From df0d98475954d655571979aa061ecb07d7e00392 Mon Sep 17 00:00:00 2001 From: "Peter Zijlstra (Intel)" Date: Wed, 1 Apr 2026 14:52:13 -0700 Subject: sched/cache: Introduce infrastructure for cache-aware load balancing Adds infrastructure to enable cache-aware load balancing, which improves cache locality by grouping tasks that share resources within the same cache domain. This reduces cache misses and improves overall data access efficiency. In this initial implementation, threads belonging to the same process are treated as entities that likely share working sets. The mechanism tracks per-process CPU occupancy across cache domains and attempts to migrate threads toward cache-hot domains where their process already has active threads, thereby enhancing locality. This provides a basic model for cache affinity. While the current code targets the last-level cache (LLC), the approach could be extended to other domain types such as clusters (L2) or node-internal groupings. At present, the mechanism selects the CPU within an LLC that has the highest recent runtime. Subsequent patches in this series will use this information in the load-balancing path to guide task placement toward preferred LLCs. In the future, more advanced policies could be integrated through NUMA balancing-for example, migrating a task to its preferred LLC when spare capacity exists, or swapping tasks across LLCs to improve cache affinity. Grouping of tasks could also be generalized from that of a process to be that of a NUMA group, or be user configurable. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/6269a53221b9439b9ca00d18a9d1946fb64d8cff.1775065312.git.tim.c.chen@linux.intel.com --- kernel/fork.c | 6 ++ kernel/sched/core.c | 6 ++ kernel/sched/fair.c | 266 +++++++++++++++++++++++++++++++++++++++++++++++++++ kernel/sched/sched.h | 14 +++ 4 files changed, 292 insertions(+) (limited to 'kernel') diff --git a/kernel/fork.c b/kernel/fork.c index 079802cb6100..61042bc3482d 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -724,6 +724,7 @@ void __mmdrop(struct mm_struct *mm) cleanup_lazy_tlbs(mm); WARN_ON_ONCE(mm == current->active_mm); + mm_destroy_sched(mm); mm_free_pgd(mm); mm_free_id(mm); destroy_context(mm); @@ -1125,6 +1126,9 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, if (mm_alloc_cid(mm, p)) goto fail_cid; + if (mm_alloc_sched(mm)) + goto fail_sched; + if (percpu_counter_init_many(mm->rss_stat, 0, GFP_KERNEL_ACCOUNT, NR_MM_COUNTERS)) goto fail_pcpu; @@ -1134,6 +1138,8 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, return mm; fail_pcpu: + mm_destroy_sched(mm); +fail_sched: mm_destroy_cid(mm); fail_cid: destroy_context(mm); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 49cd5d217161..7e0b55e7ef5c 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4434,6 +4434,7 @@ static void __sched_fork(u64 clone_flags, struct task_struct *p) init_numa_balancing(clone_flags, p); p->wake_entry.u_flags = CSD_TYPE_TTWU; p->migration_pending = NULL; + init_sched_mm(p); } DEFINE_STATIC_KEY_FALSE(sched_numa_balancing); @@ -8962,6 +8963,11 @@ void __init sched_init(void) rq->core_cookie = 0UL; #endif +#ifdef CONFIG_SCHED_CACHE + raw_spin_lock_init(&rq->cpu_epoch_lock); + rq->cpu_epoch_next = jiffies; +#endif + zalloc_cpumask_var_node(&rq->scratch_mask, GFP_KERNEL, cpu_to_node(i)); } diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 12890ef16603..c9cd064223e5 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1321,6 +1321,8 @@ void post_init_entity_util_avg(struct task_struct *p) sa->runnable_avg = sa->util_avg; } +static inline void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec); + static s64 update_se(struct rq *rq, struct sched_entity *se) { u64 now = rq_clock_task(rq); @@ -1343,6 +1345,7 @@ static s64 update_se(struct rq *rq, struct sched_entity *se) trace_sched_stat_runtime(running, delta_exec); account_group_exec_runtime(running, delta_exec); + account_mm_sched(rq, running, delta_exec); /* cgroup time is always accounted against the donor */ cgroup_account_cputime(donor, delta_exec); @@ -1364,6 +1367,267 @@ static s64 update_se(struct rq *rq, struct sched_entity *se) static void set_next_buddy(struct sched_entity *se); +#ifdef CONFIG_SCHED_CACHE + +/* + * XXX numbers come from a place the sun don't shine -- probably wants to be SD + * tunable or so. + */ +#define EPOCH_PERIOD (HZ / 100) /* 10 ms */ +#define EPOCH_LLC_AFFINITY_TIMEOUT 5 /* 50 ms */ + +static int llc_id(int cpu) +{ + if (cpu < 0) + return -1; + + return per_cpu(sd_llc_id, cpu); +} + +void mm_init_sched(struct mm_struct *mm, + struct sched_cache_time __percpu *_pcpu_sched) +{ + unsigned long epoch = 0; + int i; + + for_each_possible_cpu(i) { + struct sched_cache_time *pcpu_sched = per_cpu_ptr(_pcpu_sched, i); + struct rq *rq = cpu_rq(i); + + pcpu_sched->runtime = 0; + /* a slightly stale cpu epoch is acceptible */ + pcpu_sched->epoch = rq->cpu_epoch; + epoch = rq->cpu_epoch; + } + + raw_spin_lock_init(&mm->sc_stat.lock); + mm->sc_stat.epoch = epoch; + mm->sc_stat.cpu = -1; + + /* + * The update to mm->sc_stat should not be reordered + * before initialization to mm's other fields, in case + * the readers may get invalid mm_sched_epoch, etc. + */ + smp_store_release(&mm->sc_stat.pcpu_sched, _pcpu_sched); +} + +/* because why would C be fully specified */ +static __always_inline void __shr_u64(u64 *val, unsigned int n) +{ + if (n >= 64) { + *val = 0; + return; + } + *val >>= n; +} + +static inline void __update_mm_sched(struct rq *rq, + struct sched_cache_time *pcpu_sched) +{ + lockdep_assert_held(&rq->cpu_epoch_lock); + + unsigned long n, now = jiffies; + long delta = now - rq->cpu_epoch_next; + + if (delta > 0) { + n = (delta + EPOCH_PERIOD - 1) / EPOCH_PERIOD; + rq->cpu_epoch += n; + rq->cpu_epoch_next += n * EPOCH_PERIOD; + __shr_u64(&rq->cpu_runtime, n); + } + + n = rq->cpu_epoch - pcpu_sched->epoch; + if (n) { + pcpu_sched->epoch += n; + __shr_u64(&pcpu_sched->runtime, n); + } +} + +static unsigned long fraction_mm_sched(struct rq *rq, + struct sched_cache_time *pcpu_sched) +{ + guard(raw_spinlock_irqsave)(&rq->cpu_epoch_lock); + + __update_mm_sched(rq, pcpu_sched); + + /* + * Runtime is a geometric series (r=0.5) and as such will sum to twice + * the accumulation period, this means the multiplcation here should + * not overflow. + */ + return div64_u64(NICE_0_LOAD * pcpu_sched->runtime, rq->cpu_runtime + 1); +} + +static inline +void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) +{ + struct sched_cache_time *pcpu_sched; + struct mm_struct *mm = p->mm; + unsigned long epoch; + + if (!sched_cache_enabled()) + return; + + if (p->sched_class != &fair_sched_class) + return; + /* + * init_task, kthreads and user thread created + * by user_mode_thread() don't have mm. + */ + if (!mm || !mm->sc_stat.pcpu_sched) + return; + + pcpu_sched = per_cpu_ptr(p->mm->sc_stat.pcpu_sched, cpu_of(rq)); + + scoped_guard (raw_spinlock, &rq->cpu_epoch_lock) { + __update_mm_sched(rq, pcpu_sched); + pcpu_sched->runtime += delta_exec; + rq->cpu_runtime += delta_exec; + epoch = rq->cpu_epoch; + } + + /* + * If this process hasn't hit task_cache_work() for a while invalidate + * its preferred state. + */ + if (epoch - READ_ONCE(mm->sc_stat.epoch) > EPOCH_LLC_AFFINITY_TIMEOUT) { + if (mm->sc_stat.cpu != -1) + mm->sc_stat.cpu = -1; + } +} + +static void task_tick_cache(struct rq *rq, struct task_struct *p) +{ + struct callback_head *work = &p->cache_work; + struct mm_struct *mm = p->mm; + unsigned long epoch; + + if (!sched_cache_enabled()) + return; + + if (!mm || !mm->sc_stat.pcpu_sched) + return; + + epoch = rq->cpu_epoch; + /* avoid moving backwards */ + if (time_after_eq(mm->sc_stat.epoch, epoch)) + return; + + guard(raw_spinlock)(&mm->sc_stat.lock); + + if (work->next == work) { + task_work_add(p, work, TWA_RESUME); + WRITE_ONCE(mm->sc_stat.epoch, epoch); + } +} + +static void task_cache_work(struct callback_head *work) +{ + struct task_struct *p = current; + struct mm_struct *mm = p->mm; + unsigned long m_a_occ = 0; + unsigned long curr_m_a_occ = 0; + int cpu, m_a_cpu = -1; + cpumask_var_t cpus; + + WARN_ON_ONCE(work != &p->cache_work); + + work->next = work; + + if (p->flags & PF_EXITING) + return; + + if (!zalloc_cpumask_var(&cpus, GFP_KERNEL)) + return; + + scoped_guard (cpus_read_lock) { + guard(rcu)(); + + cpumask_copy(cpus, cpu_online_mask); + + for_each_cpu(cpu, cpus) { + /* XXX sched_cluster_active */ + struct sched_domain *sd = per_cpu(sd_llc, cpu); + unsigned long occ, m_occ = 0, a_occ = 0; + int m_cpu = -1, i; + + if (!sd) + continue; + + for_each_cpu(i, sched_domain_span(sd)) { + occ = fraction_mm_sched(cpu_rq(i), + per_cpu_ptr(mm->sc_stat.pcpu_sched, i)); + a_occ += occ; + if (occ > m_occ) { + m_occ = occ; + m_cpu = i; + } + } + + /* + * Compare the accumulated occupancy of each LLC. The + * reason for using accumulated occupancy rather than average + * per CPU occupancy is that it works better in asymmetric LLC + * scenarios. + * For example, if there are 2 threads in a 4CPU LLC and 3 + * threads in an 8CPU LLC, it might be better to choose the one + * with 3 threads. However, this would not be the case if the + * occupancy is divided by the number of CPUs in an LLC (i.e., + * if average per CPU occupancy is used). + * Besides, NUMA balancing fault statistics behave similarly: + * the total number of faults per node is compared rather than + * the average number of faults per CPU. This strategy is also + * followed here. + */ + if (a_occ > m_a_occ) { + m_a_occ = a_occ; + m_a_cpu = m_cpu; + } + + if (llc_id(cpu) == llc_id(mm->sc_stat.cpu)) + curr_m_a_occ = a_occ; + + cpumask_andnot(cpus, cpus, sched_domain_span(sd)); + } + } + + if (m_a_occ > (2 * curr_m_a_occ)) { + /* + * Avoid switching sc_stat.cpu too fast. + * The reason to choose 2X is because: + * 1. It is better to keep the preferred LLC stable, + * rather than changing it frequently and cause migrations + * 2. 2X means the new preferred LLC has at least 1 more + * busy CPU than the old one(200% vs 100%, eg) + * 3. 2X is chosen based on test results, as it delivers + * the optimal performance gain so far. + */ + mm->sc_stat.cpu = m_a_cpu; + } + + free_cpumask_var(cpus); +} + +void init_sched_mm(struct task_struct *p) +{ + struct callback_head *work = &p->cache_work; + + init_task_work(work, task_cache_work); + work->next = work; +} + +#else /* CONFIG_SCHED_CACHE */ + +static inline void account_mm_sched(struct rq *rq, struct task_struct *p, + s64 delta_exec) { } + +void init_sched_mm(struct task_struct *p) { } + +static void task_tick_cache(struct rq *rq, struct task_struct *p) { } + +#endif /* CONFIG_SCHED_CACHE */ + /* * Used by other classes to account runtime. */ @@ -13653,6 +13917,8 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) if (static_branch_unlikely(&sched_numa_balancing)) task_tick_numa(rq, curr); + task_tick_cache(rq, curr); + update_misfit_status(curr, rq); check_update_overutilized_status(task_rq(curr)); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index c95584191d58..f939d45fe043 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1178,6 +1178,12 @@ struct rq { struct scx_rq scx; struct sched_dl_entity ext_server; #endif +#ifdef CONFIG_SCHED_CACHE + raw_spinlock_t cpu_epoch_lock ____cacheline_aligned; + u64 cpu_runtime; + unsigned long cpu_epoch; + unsigned long cpu_epoch_next; +#endif struct sched_dl_entity fair_server; @@ -4041,6 +4047,14 @@ static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct *next) { } #endif /* !CONFIG_SCHED_MM_CID */ +#ifdef CONFIG_SCHED_CACHE +static inline bool sched_cache_enabled(void) +{ + return false; +} +#endif +extern void init_sched_mm(struct task_struct *p); + extern u64 avg_vruntime(struct cfs_rq *cfs_rq); extern int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se); static inline -- cgit v1.2.3 From b4606faab3188beeacc2287b8a369cca943cc8eb Mon Sep 17 00:00:00 2001 From: Chen Yu Date: Wed, 1 Apr 2026 14:52:14 -0700 Subject: sched/cache: Limit the scan number of CPUs when calculating task occupancy When NUMA balancing is enabled, the kernel currently iterates over all online CPUs to aggregate process-wide occupancy data. On large systems, this global scan introduces significant overhead. To reduce scan latency, limit the search to a subset of relevant CPUs: 1. The task's preferred NUMA node. 2. The node where the task is currently running. 3. The node that contains the task's current preferred LLC.. While focusing solely on the preferred NUMA node is ideal, a process-wide scan must remain flexible because the "preferred node" is a per-task attribute. Different threads within the same process may have different preferred nodes, causing the process-wide preference to migrate. Maintaining a mask that covers both the preferred and active running nodes ensures accuracy while significantly reducing the number of CPUs inspected. Future work may integrate numa_group to further refine task aggregation. Suggested-by: Madadi Vineeth Reddy Signed-off-by: Chen Yu Co-developed-by: Tim Chen Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/57ed5fcec9b242803fe4ea2ce6e7f3de6a6efc6b.1775065312.git.tim.c.chen@linux.intel.com --- kernel/sched/fair.c | 47 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index c9cd064223e5..a55ada22e40c 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1522,6 +1522,51 @@ static void task_tick_cache(struct rq *rq, struct task_struct *p) } } +static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p) +{ +#ifdef CONFIG_NUMA_BALANCING + int cpu, curr_cpu, nid, pref_nid; + + if (!static_branch_likely(&sched_numa_balancing)) + goto out; + + cpu = p->mm->sc_stat.cpu; + if (cpu != -1) + nid = cpu_to_node(cpu); + curr_cpu = task_cpu(p); + + /* + * Scanning in the preferred NUMA node is ideal. However, the NUMA + * preferred node is per-task rather than per-process. It is possible + * for different threads of the process to have distinct preferred + * nodes; consequently, the process-wide preferred LLC may bounce + * between different nodes. As a workaround, maintain the scan + * CPU mask to also cover the process's current preferred LLC and the + * current running node to mitigate the bouncing risk. + * TBD: numa_group should be considered during task aggregation. + */ + pref_nid = p->numa_preferred_nid; + /* honor the task's preferred node */ + if (pref_nid == NUMA_NO_NODE) + goto out; + + cpumask_or(cpus, cpus, cpumask_of_node(pref_nid)); + + /* honor the task's preferred LLC CPU */ + if (cpu != -1 && !cpumask_test_cpu(cpu, cpus) && nid != NUMA_NO_NODE) + cpumask_or(cpus, cpus, cpumask_of_node(nid)); + + /* make sure the task's current running node is included */ + if (!cpumask_test_cpu(curr_cpu, cpus)) + cpumask_or(cpus, cpus, cpumask_of_node(cpu_to_node(curr_cpu))); + + return; + +out: +#endif + cpumask_copy(cpus, cpu_online_mask); +} + static void task_cache_work(struct callback_head *work) { struct task_struct *p = current; @@ -1544,7 +1589,7 @@ static void task_cache_work(struct callback_head *work) scoped_guard (cpus_read_lock) { guard(rcu)(); - cpumask_copy(cpus, cpu_online_mask); + get_scan_cpumasks(cpus, p); for_each_cpu(cpu, cpus) { /* XXX sched_cluster_active */ -- cgit v1.2.3 From f025ef275388742643a2c33f00a0d9c0af3112ee Mon Sep 17 00:00:00 2001 From: Chen Yu Date: Wed, 1 Apr 2026 14:52:15 -0700 Subject: sched/cache: Record per LLC utilization to guide cache aware scheduling decisions When a system becomes busy and a process's preferred LLC is saturated with too many threads, tasks within that LLC migrate frequently. These in LLC migrations introduce latency and degrade performance. To avoid this, task aggregation should be suppressed when the preferred LLC is overloaded, which requires a metric to indicate LLC utilization. Record per LLC utilization/cpu capacity during periodic load balancing. These statistics will be used in later patches to decide whether tasks should be aggregated into their preferred LLC. Signed-off-by: Chen Yu Co-developed-by: Tim Chen Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/a48151b3d57f2a42a5971aaead1b7f81e69229f4.1775065312.git.tim.c.chen@linux.intel.com --- kernel/sched/fair.c | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index a55ada22e40c..6647d465b59e 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -9992,6 +9992,28 @@ static inline int task_is_ineligible_on_dst_cpu(struct task_struct *p, int dest_ return 0; } +#ifdef CONFIG_SCHED_CACHE +static __maybe_unused bool get_llc_stats(int cpu, unsigned long *util, + unsigned long *cap) +{ + struct sched_domain_shared *sd_share; + + sd_share = rcu_dereference_all(per_cpu(sd_llc_shared, cpu)); + if (!sd_share) + return false; + + *util = READ_ONCE(sd_share->util_avg); + *cap = READ_ONCE(sd_share->capacity); + + return true; +} +#else +static inline bool get_llc_stats(int cpu, unsigned long *util, + unsigned long *cap) +{ + return false; +} +#endif /* * can_migrate_task - may task p from runqueue rq be migrated to this_cpu? */ @@ -10948,6 +10970,53 @@ sched_reduced_capacity(struct rq *rq, struct sched_domain *sd) return check_cpu_capacity(rq, sd); } +#ifdef CONFIG_SCHED_CACHE +/* + * Record the statistics for this scheduler group for later + * use. These values guide load balancing on aggregating tasks + * to a LLC. + */ +static void record_sg_llc_stats(struct lb_env *env, + struct sg_lb_stats *sgs, + struct sched_group *group) +{ + struct sched_domain_shared *sd_share; + int cpu; + + if (!sched_cache_enabled() || env->idle == CPU_NEWLY_IDLE) + return; + + /* Only care about sched domain spanning multiple LLCs */ + if (env->sd->child != rcu_dereference_all(per_cpu(sd_llc, env->dst_cpu))) + return; + + /* + * At this point we know this group spans a LLC domain. + * Record the statistic of this group in its corresponding + * shared LLC domain. + * Note: sd_share cannot be obtained via sd->child->shared, + * because the latter refers to the domain that covers the + * local group. Instead, sd_share should be located using + * the first CPU of the LLC group. + */ + cpu = cpumask_first(sched_group_span(group)); + sd_share = rcu_dereference_all(per_cpu(sd_llc_shared, cpu)); + if (!sd_share) + return; + + if (READ_ONCE(sd_share->util_avg) != sgs->group_util) + WRITE_ONCE(sd_share->util_avg, sgs->group_util); + + if (unlikely(READ_ONCE(sd_share->capacity) != sgs->group_capacity)) + WRITE_ONCE(sd_share->capacity, sgs->group_capacity); +} +#else +static inline void record_sg_llc_stats(struct lb_env *env, struct sg_lb_stats *sgs, + struct sched_group *group) +{ +} +#endif + /** * update_sg_lb_stats - Update sched_group's statistics for load balancing. * @env: The load balancing environment. @@ -11035,6 +11104,7 @@ static inline void update_sg_lb_stats(struct lb_env *env, sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs); + record_sg_llc_stats(env, sgs, group); /* Computing avg_load makes sense only when group is overloaded */ if (sgs->group_type == group_overloaded) sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) / -- cgit v1.2.3 From 23b2b5ccc45ce2a38b9336a916088fffdc4cdfb1 Mon Sep 17 00:00:00 2001 From: Chen Yu Date: Wed, 1 Apr 2026 14:52:16 -0700 Subject: sched/cache: Introduce helper functions to enforce LLC migration policy Cache-aware scheduling aggregates threads onto their preferred LLC, mainly through load balancing. When the preferred LLC becomes saturated, more threads are still placed there, increasing latency. A mechanism is needed to limit aggregation so that the preferred LLC does not become overloaded. Introduce helper functions can_migrate_llc() and can_migrate_llc_task() to enforce the LLC migration policy: 1. Aggregate a task to its preferred LLC if both source and destination LLCs are not too busy, or if doing so will not leave the preferred LLC much more imbalanced than the non-preferred one (>20% utilization difference, a little higher than the default imbalance_pct(17%) of the LLC domain as hysteresis). Later this threshold will be turned into tunable debugfs. 2. Allow moving a task from overloaded preferred LLC to a non preferred LLC if this will not cause the non preferred LLC to become too imbalanced to cause a later migration back. 3. If both LLCs are too busy, let the generic load balance to spread the tasks. Further (hysteresis)action could be taken in the future to prevent tasks from being migrated into and out of the preferred LLC frequently (back and forth): the threshold for migrating a task out of its preferred LLC should be higher than that for migrating it into the LLC. Signed-off-by: Chen Yu Co-developed-by: Tim Chen Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/d19b52589cdceaee5e625980959f4d1982d6d7c9.1775065312.git.tim.c.chen@linux.intel.com --- kernel/sched/fair.c | 167 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 6647d465b59e..7860c5bc12d7 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -9993,6 +9993,38 @@ static inline int task_is_ineligible_on_dst_cpu(struct task_struct *p, int dest_ } #ifdef CONFIG_SCHED_CACHE +/* + * The margin used when comparing LLC utilization with CPU capacity. + * It determines the LLC load level where active LLC aggregation is + * done. + * Derived from fits_capacity(). + * + * (default: ~50%, tunable via debugfs) + */ +static bool fits_llc_capacity(unsigned long util, unsigned long max) +{ + u32 aggr_pct = 50; + + /* + * For single core systems, raise the aggregation + * threshold to accommodate more tasks. + */ + if (cpu_smt_num_threads == 1) + aggr_pct = (aggr_pct * 3 / 2); + + return util * 100 < max * aggr_pct; +} + +/* + * The margin used when comparing utilization. + * is 'util1' noticeably greater than 'util2' + * Derived from capacity_greater(). + * Bias is in perentage. + */ +/* Allows dst util to be bigger than src util by up to bias percent */ +#define util_greater(util1, util2) \ + ((util1) * 100 > (util2) * 120) + static __maybe_unused bool get_llc_stats(int cpu, unsigned long *util, unsigned long *cap) { @@ -10007,6 +10039,141 @@ static __maybe_unused bool get_llc_stats(int cpu, unsigned long *util, return true; } + +/* + * Decision matrix according to the LLC utilization. To + * decide whether we can do task aggregation across LLC. + * + * By default, 50% is the threshold for treating the LLC + * as busy. The reason for choosing 50% is to avoid saturation + * of SMT-2, and it is also a safe cutoff for other SMT-n + * platforms. SMT-1 has higher threshold because it is + * supposed to accommodate more tasks, see fits_llc_capacity(). + * + * 20% is the utilization imbalance percentage to decide + * if the preferred LLC is busier than the non-preferred LLC. + * 20 is a little higher than the LLC domain's imbalance_pct + * 17. The hysteresis is used to avoid task bouncing between the + * preferred LLC and the non-preferred LLC, and it will + * be turned into tunable debugfs. + * + * 1. moving towards the preferred LLC, dst is the preferred + * LLC, src is not. + * + * src \ dst 30% 40% 50% 60% + * 30% Y Y Y N + * 40% Y Y Y Y + * 50% Y Y G G + * 60% Y Y G G + * + * 2. moving out of the preferred LLC, src is the preferred + * LLC, dst is not: + * + * src \ dst 30% 40% 50% 60% + * 30% N N N N + * 40% N N N N + * 50% N N G G + * 60% Y N G G + * + * src : src_util + * dst : dst_util + * Y : Yes, migrate + * N : No, do not migrate + * G : let the Generic load balance to even the load. + * + * The intention is that if both LLCs are quite busy, cache aware + * load balance should not be performed, and generic load balance + * should take effect. However, if one is busy and the other is not, + * the preferred LLC capacity(50%) and imbalance criteria(20%) should + * be considered to determine whether LLC aggregation should be + * performed to bias the load towards the preferred LLC. + */ + +/* migration decision, 3 states are orthogonal. */ +enum llc_mig { + mig_forbid = 0, /* N: Don't migrate task, respect LLC preference */ + mig_llc, /* Y: Do LLC preference based migration */ + mig_unrestricted /* G: Don't restrict generic load balance migration */ +}; + +/* + * Check if task can be moved from the source LLC to the + * destination LLC without breaking cache aware preferrence. + * src_cpu and dst_cpu are arbitrary CPUs within the source + * and destination LLCs, respectively. + */ +static enum llc_mig can_migrate_llc(int src_cpu, int dst_cpu, + unsigned long tsk_util, + bool to_pref) +{ + unsigned long src_util, dst_util, src_cap, dst_cap; + + if (!get_llc_stats(src_cpu, &src_util, &src_cap) || + !get_llc_stats(dst_cpu, &dst_util, &dst_cap)) + return mig_unrestricted; + + src_util = src_util < tsk_util ? 0 : src_util - tsk_util; + dst_util = dst_util + tsk_util; + + if (!fits_llc_capacity(dst_util, dst_cap) && + !fits_llc_capacity(src_util, src_cap)) + return mig_unrestricted; + + if (to_pref) { + /* + * Don't migrate if we will get preferred LLC too + * heavily loaded and if the dest is much busier + * than the src, in which case migration will + * increase the imbalance too much. + */ + if (!fits_llc_capacity(dst_util, dst_cap) && + util_greater(dst_util, src_util)) + return mig_forbid; + } else { + /* + * Don't migrate if we will leave preferred LLC + * too idle, or if this migration leads to the + * non-preferred LLC falls within sysctl_aggr_imb percent + * of preferred LLC, leading to migration again + * back to preferred LLC. + */ + if (fits_llc_capacity(src_util, src_cap) || + !util_greater(src_util, dst_util)) + return mig_forbid; + } + return mig_llc; +} + +/* + * Check if task p can migrate from source LLC to + * destination LLC in terms of cache aware load balance. + */ +static __maybe_unused enum llc_mig can_migrate_llc_task(int src_cpu, int dst_cpu, + struct task_struct *p) +{ + struct mm_struct *mm; + bool to_pref; + int cpu; + + mm = p->mm; + if (!mm) + return mig_unrestricted; + + cpu = mm->sc_stat.cpu; + if (cpu < 0 || cpus_share_cache(src_cpu, dst_cpu)) + return mig_unrestricted; + + if (cpus_share_cache(dst_cpu, cpu)) + to_pref = true; + else if (cpus_share_cache(src_cpu, cpu)) + to_pref = false; + else + return mig_unrestricted; + + return can_migrate_llc(src_cpu, dst_cpu, + task_util(p), to_pref); +} + #else static inline bool get_llc_stats(int cpu, unsigned long *util, unsigned long *cap) -- cgit v1.2.3 From b5ea300a17e37eada7a98561fbd34a3054578713 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Wed, 1 Apr 2026 14:52:17 -0700 Subject: sched/cache: Make LLC id continuous Introduce an index mapping between CPUs and their LLCs. This provides a roughly continuous per LLC index needed for cache-aware load balancing in later patches. The existing per_cpu llc_id usually points to the first CPU of the LLC domain, which is sparse and unsuitable as an array index. Using llc_id directly would waste memory. With the new mapping, CPUs in the same LLC share an approximate continuous id: per_cpu(llc_id, CPU=0...15) = 0 per_cpu(llc_id, CPU=16...31) = 1 per_cpu(llc_id, CPU=32...47) = 2 ... Note that the LLC IDs are allocated via bitmask, so the IDs may be reused during CPU offline->online transitions. Suggested-by: Peter Zijlstra (Intel) Originally-by: K Prateek Nayak Co-developed-by: Chen Yu Signed-off-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/047ef46339e4db497b54a89940a7ebedf27fcf28.1775065312.git.tim.c.chen@linux.intel.com --- kernel/sched/core.c | 2 ++ kernel/sched/sched.h | 3 ++ kernel/sched/topology.c | 90 +++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 93 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 7e0b55e7ef5c..d11e27be7697 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -8630,6 +8630,8 @@ int sched_cpu_deactivate(unsigned int cpu) */ synchronize_rcu(); + sched_domains_free_llc_id(cpu); + sched_set_rq_offline(rq, cpu); scx_rq_deactivate(rq); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index f939d45fe043..3cb3ab02b1eb 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -4053,6 +4053,9 @@ static inline bool sched_cache_enabled(void) return false; } #endif + +void sched_domains_free_llc_id(int cpu); + extern void init_sched_mm(struct task_struct *p); extern u64 avg_vruntime(struct cfs_rq *cfs_rq); diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 5847b83d9d55..1200670969bb 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -19,8 +19,10 @@ void sched_domains_mutex_unlock(void) } /* Protected by sched_domains_mutex: */ +static cpumask_var_t sched_domains_llc_id_allocmask; static cpumask_var_t sched_domains_tmpmask; static cpumask_var_t sched_domains_tmpmask2; +int max_lid; static int __init sched_debug_setup(char *str) { @@ -663,7 +665,7 @@ static void destroy_sched_domains(struct sched_domain *sd) */ DEFINE_PER_CPU(struct sched_domain __rcu *, sd_llc); DEFINE_PER_CPU(int, sd_llc_size); -DEFINE_PER_CPU(int, sd_llc_id); +DEFINE_PER_CPU(int, sd_llc_id) = -1; DEFINE_PER_CPU(int, sd_share_id); DEFINE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared); DEFINE_PER_CPU(struct sched_domain __rcu *, sd_numa); @@ -692,7 +694,6 @@ static void update_top_cache_domain(int cpu) rcu_assign_pointer(per_cpu(sd_llc, cpu), sd); per_cpu(sd_llc_size, cpu) = size; - per_cpu(sd_llc_id, cpu) = id; rcu_assign_pointer(per_cpu(sd_llc_shared, cpu), sds); sd = lowest_flag_domain(cpu, SD_CLUSTER); @@ -1790,6 +1791,11 @@ const struct cpumask *tl_mc_mask(struct sched_domain_topology_level *tl, int cpu { return cpu_coregroup_mask(cpu); } + +#define llc_mask(cpu) cpu_coregroup_mask(cpu) + +#else +#define llc_mask(cpu) cpumask_of(cpu) #endif const struct cpumask *tl_pkg_mask(struct sched_domain_topology_level *tl, int cpu) @@ -2650,6 +2656,61 @@ static void adjust_numa_imbalance(struct sched_domain *sd_llc) } } +static int __sched_domains_alloc_llc_id(void) +{ + int lid, max; + + lockdep_assert_held(&sched_domains_mutex); + + lid = cpumask_first_zero(sched_domains_llc_id_allocmask); + /* + * llc_id space should never grow larger than the + * possible number of CPUs in the system. + */ + if (lid >= nr_cpu_ids) + return -1; + + __cpumask_set_cpu(lid, sched_domains_llc_id_allocmask); + max = cpumask_last(sched_domains_llc_id_allocmask); + if (max > max_lid) + max_lid = max; + + return lid; +} + +static void __sched_domains_free_llc_id(int cpu) +{ + int i, lid, max; + + lockdep_assert_held(&sched_domains_mutex); + + lid = per_cpu(sd_llc_id, cpu); + if (lid == -1 || lid >= nr_cpu_ids) + return; + + per_cpu(sd_llc_id, cpu) = -1; + + for_each_cpu(i, llc_mask(cpu)) { + /* An online CPU owns the llc_id. */ + if (per_cpu(sd_llc_id, i) == lid) + return; + } + + __cpumask_clear_cpu(lid, sched_domains_llc_id_allocmask); + + max = cpumask_last(sched_domains_llc_id_allocmask); + /* shrink max lid to save memory */ + if (max < max_lid) + max_lid = max; +} + +void sched_domains_free_llc_id(int cpu) +{ + sched_domains_mutex_lock(); + __sched_domains_free_llc_id(cpu); + sched_domains_mutex_unlock(); +} + /* * Build sched domains for a given set of CPUs and attach the sched domains * to the individual CPUs @@ -2675,6 +2736,7 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att /* Set up domains for CPUs specified by the cpu_map: */ for_each_cpu(i, cpu_map) { struct sched_domain_topology_level *tl; + int lid; sd = NULL; for_each_sd_topology(tl) { @@ -2688,6 +2750,29 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att if (cpumask_equal(cpu_map, sched_domain_span(sd))) break; } + + lid = per_cpu(sd_llc_id, i); + if (lid == -1) { + /* try to reuse the llc_id of its siblings */ + for (int j = cpumask_first(llc_mask(i)); + j < nr_cpu_ids; + j = cpumask_next(j, llc_mask(i))) { + if (i == j) + continue; + + lid = per_cpu(sd_llc_id, j); + + if (lid != -1) { + per_cpu(sd_llc_id, i) = lid; + + break; + } + } + + /* a new LLC is detected */ + if (lid == -1) + per_cpu(sd_llc_id, i) = __sched_domains_alloc_llc_id(); + } } if (WARN_ON(!topology_span_sane(cpu_map))) @@ -2831,6 +2916,7 @@ int __init sched_init_domains(const struct cpumask *cpu_map) { int err; + zalloc_cpumask_var(&sched_domains_llc_id_allocmask, GFP_KERNEL); zalloc_cpumask_var(&sched_domains_tmpmask, GFP_KERNEL); zalloc_cpumask_var(&sched_domains_tmpmask2, GFP_KERNEL); zalloc_cpumask_var(&fallback_doms, GFP_KERNEL); -- cgit v1.2.3 From 47d8696b95f7397fe7cad2d194d550ffe82efc15 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Wed, 1 Apr 2026 14:52:18 -0700 Subject: sched/cache: Assign preferred LLC ID to processes With cache-aware scheduling enabled, each task is assigned a preferred LLC ID. This allows quick identification of the LLC domain where the task prefers to run, similar to numa_preferred_nid in NUMA balancing. Co-developed-by: Chen Yu Signed-off-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/f2ceecba5858680349ad4ce9303a2121f0bb7272.1775065312.git.tim.c.chen@linux.intel.com --- kernel/sched/fair.c | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 7860c5bc12d7..6e78ecfb560e 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1459,11 +1459,43 @@ static unsigned long fraction_mm_sched(struct rq *rq, return div64_u64(NICE_0_LOAD * pcpu_sched->runtime, rq->cpu_runtime + 1); } +static int get_pref_llc(struct task_struct *p, struct mm_struct *mm) +{ + int mm_sched_llc = -1; + + if (!mm) + return -1; + + if (mm->sc_stat.cpu != -1) { + mm_sched_llc = llc_id(mm->sc_stat.cpu); + +#ifdef CONFIG_NUMA_BALANCING + /* + * Don't assign preferred LLC if it + * conflicts with NUMA balancing. + * This can happen when sched_setnuma() gets + * called, however it is not much of an issue + * because we expect account_mm_sched() to get + * called fairly regularly -- at a higher rate + * than sched_setnuma() at least -- and thus the + * conflict only exists for a short period of time. + */ + if (static_branch_likely(&sched_numa_balancing) && + p->numa_preferred_nid >= 0 && + cpu_to_node(mm->sc_stat.cpu) != p->numa_preferred_nid) + mm_sched_llc = -1; +#endif + } + + return mm_sched_llc; +} + static inline void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) { struct sched_cache_time *pcpu_sched; struct mm_struct *mm = p->mm; + int mm_sched_llc = -1; unsigned long epoch; if (!sched_cache_enabled()) @@ -1495,6 +1527,11 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) if (mm->sc_stat.cpu != -1) mm->sc_stat.cpu = -1; } + + mm_sched_llc = get_pref_llc(p, mm); + + if (READ_ONCE(p->preferred_llc) != mm_sched_llc) + WRITE_ONCE(p->preferred_llc, mm_sched_llc); } static void task_tick_cache(struct rq *rq, struct task_struct *p) @@ -1671,6 +1708,12 @@ void init_sched_mm(struct task_struct *p) { } static void task_tick_cache(struct rq *rq, struct task_struct *p) { } +static inline int get_pref_llc(struct task_struct *p, + struct mm_struct *mm) +{ + return -1; +} + #endif /* CONFIG_SCHED_CACHE */ /* -- cgit v1.2.3 From 46afe3af7ead57190b6d362e214814ec804e3b7b Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Wed, 1 Apr 2026 14:52:19 -0700 Subject: sched/cache: Track LLC-preferred tasks per runqueue For each runqueue, track the number of tasks with an LLC preference and how many of them are running on their preferred LLC. This mirrors nr_numa_running and nr_preferred_running for NUMA balancing, and will be used by cache-aware load balancing in later patches. Co-developed-by: Chen Yu Signed-off-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/459a37102f3d74a4e09ea58401d2094ac731d044.1775065312.git.tim.c.chen@linux.intel.com --- kernel/sched/core.c | 5 +++++ kernel/sched/fair.c | 47 ++++++++++++++++++++++++++++++++++++++++++++--- kernel/sched/sched.h | 8 ++++++++ 3 files changed, 57 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index d11e27be7697..eb542c97266a 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -539,6 +539,11 @@ void __trace_set_current_state(int state_value) } EXPORT_SYMBOL(__trace_set_current_state); +int task_llc(const struct task_struct *p) +{ + return per_cpu(sd_llc_id, task_cpu(p)); +} + /* * Serialization rules: * diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 6e78ecfb560e..e66da7a6be3e 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1384,6 +1384,30 @@ static int llc_id(int cpu) return per_cpu(sd_llc_id, cpu); } +static void account_llc_enqueue(struct rq *rq, struct task_struct *p) +{ + int pref_llc; + + pref_llc = p->preferred_llc; + if (pref_llc < 0) + return; + + rq->nr_llc_running++; + rq->nr_pref_llc_running += (pref_llc == task_llc(p)); +} + +static void account_llc_dequeue(struct rq *rq, struct task_struct *p) +{ + int pref_llc; + + pref_llc = p->preferred_llc; + if (pref_llc < 0) + return; + + rq->nr_llc_running--; + rq->nr_pref_llc_running -= (pref_llc == task_llc(p)); +} + void mm_init_sched(struct mm_struct *mm, struct sched_cache_time __percpu *_pcpu_sched) { @@ -1490,6 +1514,8 @@ static int get_pref_llc(struct task_struct *p, struct mm_struct *mm) return mm_sched_llc; } +static unsigned int task_running_on_cpu(int cpu, struct task_struct *p); + static inline void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) { @@ -1530,8 +1556,13 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) mm_sched_llc = get_pref_llc(p, mm); - if (READ_ONCE(p->preferred_llc) != mm_sched_llc) + /* task not on rq accounted later in account_entity_enqueue() */ + if (task_running_on_cpu(rq->cpu, p) && + READ_ONCE(p->preferred_llc) != mm_sched_llc) { + account_llc_dequeue(rq, p); WRITE_ONCE(p->preferred_llc, mm_sched_llc); + account_llc_enqueue(rq, p); + } } static void task_tick_cache(struct rq *rq, struct task_struct *p) @@ -1714,6 +1745,10 @@ static inline int get_pref_llc(struct task_struct *p, return -1; } +static void account_llc_enqueue(struct rq *rq, struct task_struct *p) {} + +static void account_llc_dequeue(struct rq *rq, struct task_struct *p) {} + #endif /* CONFIG_SCHED_CACHE */ /* @@ -4200,9 +4235,11 @@ account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) { update_load_add(&cfs_rq->load, se->load.weight); if (entity_is_task(se)) { + struct task_struct *p = task_of(se); struct rq *rq = rq_of(cfs_rq); - account_numa_enqueue(rq, task_of(se)); + account_numa_enqueue(rq, p); + account_llc_enqueue(rq, p); list_add(&se->group_node, &rq->cfs_tasks); } cfs_rq->nr_queued++; @@ -4213,7 +4250,11 @@ account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) { update_load_sub(&cfs_rq->load, se->load.weight); if (entity_is_task(se)) { - account_numa_dequeue(rq_of(cfs_rq), task_of(se)); + struct task_struct *p = task_of(se); + struct rq *rq = rq_of(cfs_rq); + + account_numa_dequeue(rq, p); + account_llc_dequeue(rq, p); list_del_init(&se->group_node); } cfs_rq->nr_queued--; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 3cb3ab02b1eb..3c9e92b79041 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1196,6 +1196,12 @@ struct rq { #ifdef CONFIG_NUMA_BALANCING unsigned int numa_migrate_on; #endif + +#ifdef CONFIG_SCHED_CACHE + unsigned int nr_pref_llc_running; + unsigned int nr_llc_running; +#endif + /* * This is part of a global counter where only the total sum * over all CPUs matters. A task can increase this counter on @@ -2077,6 +2083,8 @@ init_numa_balancing(u64 clone_flags, struct task_struct *p) #endif /* !CONFIG_NUMA_BALANCING */ +int task_llc(const struct task_struct *p); + static inline void queue_balance_callback(struct rq *rq, struct balance_callback *head, -- cgit v1.2.3 From a8d0ca0b7f2f7b53565d1e30e509d3d74d1f5460 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Wed, 1 Apr 2026 14:52:20 -0700 Subject: sched/cache: Introduce per CPU's tasks LLC preference counter The lowest level of sched domain for each CPU is assigned an array where each element tracks the number of tasks preferring a given LLC, indexed from 0 to max_lid. Since each CPU has its dedicated sd, this implies that each CPU will have a dedicated task LLC preference counter. For example, sd->llc_counts[3] = 2 signifies that there are 2 tasks on this runqueue which prefer to run within LLC3. The load balancer can use this information to identify busy runqueues and migrate tasks to their preferred LLC domains. This array will be reallocated at runtime during sched domain rebuild. Introduce the buffer allocation mechanism, and the statistics will be calculated in the subsequent patch. Note: the LLC preference statistics of each CPU are reset on sched domain rebuild and may under count temporarily, until the CPU becomes idle and the count is cleared. This is a trade off to avoid complex data synchronization across sched domain builds. Suggested-by: Peter Zijlstra (Intel) Suggested-by: K Prateek Nayak Co-developed-by: Chen Yu Signed-off-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/42e79eceb8cd6be8a032401d481d101913bc5703.1775065312.git.tim.c.chen@linux.intel.com --- kernel/sched/topology.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 1200670969bb..8954bf7900ff 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -634,6 +634,11 @@ static void destroy_sched_domain(struct sched_domain *sd) if (sd->shared && atomic_dec_and_test(&sd->shared->ref)) kfree(sd->shared); + +#ifdef CONFIG_SCHED_CACHE + /* only the bottom sd has llc_counts array */ + kfree(sd->llc_counts); +#endif kfree(sd); } @@ -763,10 +768,18 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) if (sd && sd_degenerate(sd)) { tmp = sd; sd = sd->parent; - destroy_sched_domain(tmp); + if (sd) { struct sched_group *sg = sd->groups; +#ifdef CONFIG_SCHED_CACHE + /* move buffer to parent as child is being destroyed */ + sd->llc_counts = tmp->llc_counts; + sd->llc_max = tmp->llc_max; + /* make sure destroy_sched_domain() does not free it */ + tmp->llc_counts = NULL; + tmp->llc_max = 0; +#endif /* * sched groups hold the flags of the child sched * domain for convenience. Clear such flags since @@ -778,6 +791,8 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) sd->child = NULL; } + + destroy_sched_domain(tmp); } sched_domain_debug(sd, cpu); @@ -805,6 +820,49 @@ enum s_alloc { sa_none, }; +#ifdef CONFIG_SCHED_CACHE +static bool alloc_sd_llc(const struct cpumask *cpu_map, + struct s_data *d) +{ + struct sched_domain *sd; + unsigned int *p; + int i; + + for_each_cpu(i, cpu_map) { + sd = *per_cpu_ptr(d->sd, i); + if (!sd) + goto err; + + p = kcalloc_node(max_lid + 1, sizeof(unsigned int), + GFP_KERNEL, cpu_to_node(i)); + if (!p) + goto err; + + sd->llc_max = max_lid + 1; + sd->llc_counts = p; + } + + return true; +err: + for_each_cpu(i, cpu_map) { + sd = *per_cpu_ptr(d->sd, i); + if (sd) { + kfree(sd->llc_counts); + sd->llc_counts = NULL; + sd->llc_max = 0; + } + } + + return false; +} +#else +static bool alloc_sd_llc(const struct cpumask *cpu_map, + struct s_data *d) +{ + return false; +} +#endif + /* * Return the canonical balance CPU for this group, this is the first CPU * of this group that's also in the balance mask. @@ -2828,6 +2886,8 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att init_sched_groups_capacity(i, sd); } + alloc_sd_llc(cpu_map, &d); + /* Attach the domains */ rcu_read_lock(); for_each_cpu(i, cpu_map) { -- cgit v1.2.3 From 82c960aee304bf286552046b66d5b0b3933b2418 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Wed, 1 Apr 2026 14:52:21 -0700 Subject: sched/cache: Calculate the percpu sd task LLC preference Calculate the number of tasks' LLC preferences for each runqueue. This statistic is computed during task enqueue and dequeue operations, and is used by the cache-aware load balancing. Co-developed-by: Chen Yu Signed-off-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/d15a64436d3acd19c5c53344c5e9d3d0b79b3233.1775065312.git.tim.c.chen@linux.intel.com --- kernel/sched/fair.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index e66da7a6be3e..7d52cf0b85bd 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1386,6 +1386,7 @@ static int llc_id(int cpu) static void account_llc_enqueue(struct rq *rq, struct task_struct *p) { + struct sched_domain *sd; int pref_llc; pref_llc = p->preferred_llc; @@ -1394,10 +1395,15 @@ static void account_llc_enqueue(struct rq *rq, struct task_struct *p) rq->nr_llc_running++; rq->nr_pref_llc_running += (pref_llc == task_llc(p)); + + sd = rcu_dereference_all(rq->sd); + if (sd && (unsigned int)pref_llc < sd->llc_max) + sd->llc_counts[pref_llc]++; } static void account_llc_dequeue(struct rq *rq, struct task_struct *p) { + struct sched_domain *sd; int pref_llc; pref_llc = p->preferred_llc; @@ -1406,6 +1412,24 @@ static void account_llc_dequeue(struct rq *rq, struct task_struct *p) rq->nr_llc_running--; rq->nr_pref_llc_running -= (pref_llc == task_llc(p)); + + sd = rcu_dereference_all(rq->sd); + if (sd && (unsigned int)pref_llc < sd->llc_max) { + /* + * There is a race condition between dequeue + * and CPU hotplug. After a task has been enqueued + * on CPUx, a CPU hotplug event occurs, and all online + * CPUs (including CPUx) rebuild their sched_domains + * and reset statistics to zero(including sd->llc_counts). + * This can cause temporary undercount and we have to + * check for such underflow in sd->llc_counts. + * + * This undercount is temporary and accurate accounting + * will resume once the rq has a chance to be idle. + */ + if (sd->llc_counts[pref_llc]) + sd->llc_counts[pref_llc]--; + } } void mm_init_sched(struct mm_struct *mm, -- cgit v1.2.3 From 15ad45fb80ca7fe67faf6b51dffce125a801cc5a Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Wed, 1 Apr 2026 14:52:22 -0700 Subject: sched/cache: Count tasks prefering destination LLC in a sched group During LLC load balancing, tabulate the number of tasks on each runqueue that prefer the LLC contains the env->dst_cpu in a sched group. For example, consider a system with 4 LLC sched groups (LLC0 to LLC3) balancing towards LLC3. LLC0 has 3 tasks preferring LLC3, LLC1 has 2, and LLC2 has 1. LLC0, having the most tasks preferring LLC3, is selected as the busiest source to pick tasks from. Within a source LLC, the total number of tasks preferring a destination LLC is computed by summing counts across all CPUs in that LLC. For instance, if LLC0 has CPU0 with 2 tasks and CPU1 with 1 task preferring LLC3, the total for LLC0 is 3. These statistics allow the load balancer to choose tasks from source sched groups that best match their preferred LLCs. Co-developed-by: Chen Yu Signed-off-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/3d8502a33a753c4384b368f97f64ee70b1cea0db.1775065312.git.tim.c.chen@linux.intel.com --- kernel/sched/fair.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 7d52cf0b85bd..cea625c79035 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -10834,6 +10834,9 @@ struct sg_lb_stats { unsigned int nr_numa_running; unsigned int nr_preferred_running; #endif +#ifdef CONFIG_SCHED_CACHE + unsigned int nr_pref_dst_llc; +#endif }; /* @@ -11328,6 +11331,20 @@ static inline void update_sg_lb_stats(struct lb_env *env, if (cpu_overutilized(i)) sgs->group_overutilized = 1; +#ifdef CONFIG_SCHED_CACHE + if (sched_cache_enabled()) { + struct sched_domain *sd_tmp; + int dst_llc; + + dst_llc = llc_id(env->dst_cpu); + if (llc_id(i) != dst_llc) { + sd_tmp = rcu_dereference_all(rq->sd); + if (sd_tmp && (unsigned int)dst_llc < sd_tmp->llc_max) + sgs->nr_pref_dst_llc += sd_tmp->llc_counts[dst_llc]; + } + } +#endif + /* * No need to call idle_cpu() if nr_running is not 0 */ -- cgit v1.2.3 From 9a5e22fbb0c88bff33458ede98b0fa922fab3831 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Wed, 1 Apr 2026 14:52:23 -0700 Subject: sched/cache: Check local_group only once in update_sg_lb_stats() There is no need to check the local group twice for both group_asym_packing and group_smt_balance. Adjust the code to facilitate future checks for group types (cache-aware load balancing) as well. No functional changes are expected. Suggested-by: Peter Zijlstra (Intel) Co-developed-by: Chen Yu Signed-off-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/99a57865c8ae1847087a5c00e92d24351cf3e5a8.1775065312.git.tim.c.chen@linux.intel.com --- kernel/sched/fair.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index cea625c79035..d3812d18b6d6 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -11385,14 +11385,16 @@ static inline void update_sg_lb_stats(struct lb_env *env, sgs->group_weight = group->group_weight; - /* Check if dst CPU is idle and preferred to this group */ - if (!local_group && env->idle && sgs->sum_h_nr_running && - sched_group_asym(env, sgs, group)) - sgs->group_asym_packing = 1; - - /* Check for loaded SMT group to be balanced to dst CPU */ - if (!local_group && smt_balance(env, sgs, group)) - sgs->group_smt_balance = 1; + if (!local_group) { + /* Check if dst CPU is idle and preferred to this group */ + if (env->idle && sgs->sum_h_nr_running && + sched_group_asym(env, sgs, group)) + sgs->group_asym_packing = 1; + + /* Check for loaded SMT group to be balanced to dst CPU */ + if (smt_balance(env, sgs, group)) + sgs->group_smt_balance = 1; + } sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs); -- cgit v1.2.3 From f38cc2f0d8a354551d219e7fd95fce3e96868105 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Wed, 1 Apr 2026 14:52:24 -0700 Subject: sched/cache: Prioritize tasks preferring destination LLC during balancing During LLC load balancing, first check for tasks that prefer the destination LLC and balance them to it before others. Mark source sched groups containing tasks preferring non local LLCs with the group_llc_balance flag. This ensures the load balancer later pulls or pushes these tasks toward their preferred LLCs. The priority of group_llc_balance is lower than that of group_overloaded and higher than that of all other group types. This is because group_llc_balance may exacerbate load imbalance, and if the LLC balancing attempt fails, the nr_balance_failed mechanism will trigger other group types to rebalance the load. The load balancer selects the busiest sched_group and migrates tasks to less busy groups to distribute load across CPUs. With cache-aware scheduling enabled, the busiest sched_group is the one with most tasks preferring the destination LLC. If the group has the llc_balance flag set, cache aware load balancing is triggered. Introduce the helper function update_llc_busiest() to identify the sched_group with the most tasks preferring the destination LLC. Suggested-by: K Prateek Nayak Suggested-by: Madadi Vineeth Reddy Co-developed-by: Chen Yu Signed-off-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/baa458f45eab3f602af090c6d6af63dc864f5ec6.1775065312.git.tim.c.chen@linux.intel.com --- kernel/sched/fair.c | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index d3812d18b6d6..ba4ee9aeea66 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -9925,6 +9925,16 @@ enum group_type { * from balancing the load across the system. */ group_imbalanced, + /* + * There are tasks running on non-preferred LLC, possible to move + * them to their preferred LLC without creating too much imbalance. + * The priority of group_llc_balance is lower than that of + * group_overloaded and higher than that of all other group types. + * This is because group_llc_balance may exacerbate load imbalance. + * If the LLC balancing attempt fails, the nr_balance_failed + * mechanism will trigger other group types to rebalance the load. + */ + group_llc_balance, /* * The CPU is overloaded and can't provide expected CPU cycles to all * tasks. @@ -10828,6 +10838,7 @@ struct sg_lb_stats { enum group_type group_type; unsigned int group_asym_packing; /* Tasks should be moved to preferred CPU */ unsigned int group_smt_balance; /* Task on busy SMT be moved */ + unsigned int group_llc_balance; /* Tasks should be moved to preferred LLC */ unsigned long group_misfit_task_load; /* A CPU has a task too big for its capacity */ unsigned int group_overutilized; /* At least one CPU is overutilized in the group */ #ifdef CONFIG_NUMA_BALANCING @@ -11094,6 +11105,9 @@ group_type group_classify(unsigned int imbalance_pct, if (group_is_overloaded(imbalance_pct, sgs)) return group_overloaded; + if (sgs->group_llc_balance) + return group_llc_balance; + if (sg_imbalanced(group)) return group_imbalanced; @@ -11288,11 +11302,63 @@ static void record_sg_llc_stats(struct lb_env *env, if (unlikely(READ_ONCE(sd_share->capacity) != sgs->group_capacity)) WRITE_ONCE(sd_share->capacity, sgs->group_capacity); } + +/* + * Do LLC balance on sched group that contains LLC, and have tasks preferring + * to run on LLC in idle dst_cpu. + */ +static inline bool llc_balance(struct lb_env *env, struct sg_lb_stats *sgs, + struct sched_group *group) +{ + if (!sched_cache_enabled()) + return false; + + if (env->sd->flags & SD_SHARE_LLC) + return false; + + /* + * Skip cache aware tagging if nr_balanced_failed is sufficiently high. + * Threshold of cache_nice_tries is set to 1 higher than nr_balance_failed + * to avoid excessive task migration at the same time. + */ + if (env->sd->nr_balance_failed >= env->sd->cache_nice_tries + 1) + return false; + + if (sgs->nr_pref_dst_llc && + can_migrate_llc(cpumask_first(sched_group_span(group)), + env->dst_cpu, 0, true) == mig_llc) + return true; + + return false; +} + +static bool update_llc_busiest(struct lb_env *env, + struct sg_lb_stats *busiest, + struct sg_lb_stats *sgs) +{ + /* + * There are more tasks that want to run on dst_cpu's LLC. + */ + return sgs->nr_pref_dst_llc > busiest->nr_pref_dst_llc; +} #else static inline void record_sg_llc_stats(struct lb_env *env, struct sg_lb_stats *sgs, struct sched_group *group) { } + +static inline bool llc_balance(struct lb_env *env, struct sg_lb_stats *sgs, + struct sched_group *group) +{ + return false; +} + +static bool update_llc_busiest(struct lb_env *env, + struct sg_lb_stats *busiest, + struct sg_lb_stats *sgs) +{ + return false; +} #endif /** @@ -11394,6 +11460,10 @@ static inline void update_sg_lb_stats(struct lb_env *env, /* Check for loaded SMT group to be balanced to dst CPU */ if (smt_balance(env, sgs, group)) sgs->group_smt_balance = 1; + + /* Check for tasks in this group can be moved to their preferred LLC */ + if (llc_balance(env, sgs, group)) + sgs->group_llc_balance = 1; } sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs); @@ -11457,6 +11527,10 @@ static bool update_sd_pick_busiest(struct lb_env *env, /* Select the overloaded group with highest avg_load. */ return sgs->avg_load > busiest->avg_load; + case group_llc_balance: + /* Select the group with most tasks preferring dst LLC */ + return update_llc_busiest(env, busiest, sgs); + case group_imbalanced: /* * Select the 1st imbalanced group as we don't have any way to @@ -11719,6 +11793,7 @@ static bool update_pick_idlest(struct sched_group *idlest, return false; break; + case group_llc_balance: case group_imbalanced: case group_asym_packing: case group_smt_balance: @@ -11851,6 +11926,7 @@ sched_balance_find_dst_group(struct sched_domain *sd, struct task_struct *p, int return NULL; break; + case group_llc_balance: case group_imbalanced: case group_asym_packing: case group_smt_balance: @@ -12349,7 +12425,8 @@ static struct sched_group *sched_balance_find_src_group(struct lb_env *env) * group's child domain. */ if (sds.prefer_sibling && local->group_type == group_has_spare && - sibling_imbalance(env, &sds, busiest, local) > 1) + (busiest->group_type == group_llc_balance || + sibling_imbalance(env, &sds, busiest, local) > 1)) goto force_balance; if (busiest->group_type != group_overloaded) { -- cgit v1.2.3 From e4c9a4cb244a273c58e8fd86d7c04e2502822e64 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Wed, 1 Apr 2026 14:52:25 -0700 Subject: sched/cache: Add migrate_llc_task migration type for cache-aware balancing Introduce a new migration type, migrate_llc_task, to support cache-aware load balancing. After identifying the busiest sched_group (having the most tasks preferring the destination LLC), mark migrations with this type. During load balancing, each runqueue in the busiest sched_group is examined, and the runqueue with the highest number of tasks preferring the destination CPU is selected as the busiest runqueue. Co-developed-by: Chen Yu Signed-off-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/b9df27c19cc5121ddb2a7d1be7f9d52fec1563dc.1775065312.git.tim.c.chen@linux.intel.com --- kernel/sched/fair.c | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index ba4ee9aeea66..68032efd143b 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -9946,7 +9946,8 @@ enum migration_type { migrate_load = 0, migrate_util, migrate_task, - migrate_misfit + migrate_misfit, + migrate_llc_task }; #define LBF_ALL_PINNED 0x01 @@ -10560,6 +10561,10 @@ static int detach_tasks(struct lb_env *env) env->imbalance = 0; break; + + case migrate_llc_task: + env->imbalance--; + break; } detach_task(p, env); @@ -12179,6 +12184,15 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s return; } +#ifdef CONFIG_SCHED_CACHE + if (busiest->group_type == group_llc_balance) { + /* Move a task that prefer local LLC */ + env->migration_type = migrate_llc_task; + env->imbalance = 1; + return; + } +#endif + if (busiest->group_type == group_imbalanced) { /* * In the group_imb case we cannot rely on group-wide averages @@ -12485,7 +12499,10 @@ static struct rq *sched_balance_find_src_rq(struct lb_env *env, { struct rq *busiest = NULL, *rq; unsigned long busiest_util = 0, busiest_load = 0, busiest_capacity = 1; + unsigned int __maybe_unused busiest_pref_llc = 0; + struct sched_domain __maybe_unused *sd_tmp; unsigned int busiest_nr = 0; + int __maybe_unused dst_llc; int i; for_each_cpu_and(i, sched_group_span(group), env->cpus) { @@ -12613,6 +12630,23 @@ static struct rq *sched_balance_find_src_rq(struct lb_env *env, break; + case migrate_llc_task: +#ifdef CONFIG_SCHED_CACHE + sd_tmp = rcu_dereference_all(rq->sd); + dst_llc = llc_id(env->dst_cpu); + + if (sd_tmp && (unsigned)dst_llc < sd_tmp->llc_max) { + unsigned int this_pref_llc = + sd_tmp->llc_counts[dst_llc]; + + if (busiest_pref_llc < this_pref_llc) { + busiest_pref_llc = this_pref_llc; + busiest = rq; + } + } +#endif + break; + } } @@ -12776,6 +12810,8 @@ static void update_lb_imbalance_stat(struct lb_env *env, struct sched_domain *sd case migrate_misfit: __schedstat_add(sd->lb_imbalance_misfit[idle], env->imbalance); break; + case migrate_llc_task: + break; } } -- cgit v1.2.3 From 714059f79ff0ba976cb75360064583c78bbc6f8e Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Wed, 1 Apr 2026 14:52:26 -0700 Subject: sched/cache: Handle moving single tasks to/from their preferred LLC Cache aware scheduling mainly does two things: 1. Prevent task from migrating out of its preferred LLC if not nessasary. 2. Migrating task to their preferred LLC if nessasary. For 1: In the generic load balance, if the busiest runqueue has only one task, active balancing may be invoked to move it away. However, this migration might break LLC locality. Prevent regular load balance from migrating a task that prefers the current LLC. The load level and imbalance do not warrant breaking LLC preference per the can_migrate_llc() policy. Here, the benefit of LLC locality outweighs the power efficiency gained from migrating the only runnable task away. Before migration, check whether the task is running on its preferred LLC: Do not move a lone task to another LLC if it would move the task away from its preferred LLC or cause excessive imbalance between LLCs. For 2: On the other hand, if the migration type is migrate_llc_task, it means that there are tasks on the env->src_cpu that want to be migrated to their preferred LLC, launch the active load balance anyway. Co-developed-by: Chen Yu Signed-off-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/9b816d8c27fabf2a9c0e1f61a6b90afe8ec4ad52.1775065312.git.tim.c.chen@linux.intel.com --- kernel/sched/fair.c | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 68032efd143b..bfb6c0c52221 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -10293,12 +10293,60 @@ static __maybe_unused enum llc_mig can_migrate_llc_task(int src_cpu, int dst_cpu task_util(p), to_pref); } +/* + * Check if active load balance breaks LLC locality in + * terms of cache aware load balance. The load level and + * imbalance do not warrant breaking LLC preference per + * the can_migrate_llc() policy. Here, the benefit of + * LLC locality outweighs the power efficiency gained from + * migrating the only runnable task away. + */ +static inline bool +alb_break_llc(struct lb_env *env) +{ + if (!sched_cache_enabled()) + return false; + + if (cpus_share_cache(env->src_cpu, env->dst_cpu)) + return false; + /* + * All tasks prefer to stay on their current CPU. + * Do not pull a task from its preferred CPU if: + * 1. It is the only task running there(not too imbalance); OR + * 2. Migrating it away from its preferred LLC would violate + * the cache-aware scheduling policy. + */ + if (env->src_rq->nr_pref_llc_running && + env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable) { + unsigned long util = 0; + struct task_struct *cur; + + if (env->src_rq->nr_running <= 1) + return true; + + cur = rcu_dereference_all(env->src_rq->curr); + if (cur) + util = task_util(cur); + + if (can_migrate_llc(env->src_cpu, env->dst_cpu, + util, false) == mig_forbid) + return true; + } + + return false; +} #else static inline bool get_llc_stats(int cpu, unsigned long *util, unsigned long *cap) { return false; } + +static inline bool +alb_break_llc(struct lb_env *env) +{ + return false; +} #endif /* * can_migrate_task - may task p from runqueue rq be migrated to this_cpu? @@ -12698,6 +12746,9 @@ static int need_active_balance(struct lb_env *env) { struct sched_domain *sd = env->sd; + if (alb_break_llc(env)) + return 0; + if (asym_active_balance(env)) return 1; @@ -12717,7 +12768,8 @@ static int need_active_balance(struct lb_env *env) return 1; } - if (env->migration_type == migrate_misfit) + if (env->migration_type == migrate_misfit || + env->migration_type == migrate_llc_task) return 1; return 0; -- cgit v1.2.3 From 5b1d5e6db20a6c64ffb95d04578db8c4b0228eea Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Wed, 1 Apr 2026 14:52:27 -0700 Subject: sched/cache: Respect LLC preference in task migration and detach During load balancing, make can_migrate_task() consider a task's LLC preference. Prevent a task from being moved out of its preferred LLC. During the regular load balancing, if the task cannot be migrated due to LLC locality, the nr_balance_failed also should not be increased. Suggested-by: Peter Zijlstra (Intel) Suggested-by: K Prateek Nayak Co-developed-by: Chen Yu Signed-off-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/53da65f3d59de31e1a1dc59a4093d8dd9d4dc206.1775065312.git.tim.c.chen@linux.intel.com --- kernel/sched/fair.c | 83 ++++++++++++++++++++++++++++++++++++++++++++++++---- kernel/sched/sched.h | 13 ++++++++ 2 files changed, 91 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index bfb6c0c52221..5f22e5a097cf 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -9955,6 +9955,7 @@ enum migration_type { #define LBF_DST_PINNED 0x04 #define LBF_SOME_PINNED 0x08 #define LBF_ACTIVE_LB 0x10 +#define LBF_LLC_PINNED 0x20 struct lb_env { struct sched_domain *sd; @@ -10267,8 +10268,8 @@ static enum llc_mig can_migrate_llc(int src_cpu, int dst_cpu, * Check if task p can migrate from source LLC to * destination LLC in terms of cache aware load balance. */ -static __maybe_unused enum llc_mig can_migrate_llc_task(int src_cpu, int dst_cpu, - struct task_struct *p) +static enum llc_mig can_migrate_llc_task(int src_cpu, int dst_cpu, + struct task_struct *p) { struct mm_struct *mm; bool to_pref; @@ -10335,6 +10336,46 @@ alb_break_llc(struct lb_env *env) return false; } + +/* + * Check if migrating task p from env->src_cpu to + * env->dst_cpu breaks LLC localiy. + */ +static bool migrate_degrades_llc(struct task_struct *p, struct lb_env *env) +{ + if (!sched_cache_enabled()) + return false; + + if (task_has_sched_core(p)) + return false; + /* + * Skip over tasks that would degrade LLC locality; + * only when nr_balanced_failed is sufficiently high do we + * ignore this constraint. + * + * Threshold of cache_nice_tries is set to 1 higher + * than nr_balance_failed to avoid excessive task + * migration at the same time. + */ + if (env->sd->nr_balance_failed >= env->sd->cache_nice_tries + 1) + return false; + + /* + * We know the env->src_cpu has some tasks prefer to + * run on env->dst_cpu, skip the tasks do not prefer + * env->dst_cpu, and find the one that prefers. + */ + if (env->migration_type == migrate_llc_task && + READ_ONCE(p->preferred_llc) != llc_id(env->dst_cpu)) + return true; + + if (can_migrate_llc_task(env->src_cpu, + env->dst_cpu, p) != mig_forbid) + return false; + + return true; +} + #else static inline bool get_llc_stats(int cpu, unsigned long *util, unsigned long *cap) @@ -10347,6 +10388,12 @@ alb_break_llc(struct lb_env *env) { return false; } + +static inline bool +migrate_degrades_llc(struct task_struct *p, struct lb_env *env) +{ + return false; +} #endif /* * can_migrate_task - may task p from runqueue rq be migrated to this_cpu? @@ -10444,10 +10491,29 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) return 1; degrades = migrate_degrades_locality(p, env); - if (!degrades) + if (!degrades) { + /* + * If the NUMA locality is not broken, + * further check if migration would hurt + * LLC locality. + */ + if (migrate_degrades_llc(p, env)) { + /* + * If regular load balancing fails to pull a task + * due to LLC locality, this is expected behavior + * and we set LBF_LLC_PINNED so we don't increase + * nr_balance_failed unecessarily. + */ + if (env->migration_type != migrate_llc_task) + env->flags |= LBF_LLC_PINNED; + + return 0; + } + hot = task_hot(p, env); - else + } else { hot = degrades > 0; + } if (!hot || env->sd->nr_balance_failed > env->sd->cache_nice_tries) { if (hot) @@ -13067,9 +13133,16 @@ more_balance: * * Similarly for migration_misfit which is not related to * load/util migration, don't pollute nr_balance_failed. + * + * The same for cache aware scheduling's allowance for + * load imbalance. If regular load balance does not + * migrate task due to LLC locality, it is a expected + * behavior and don't pollute nr_balance_failed. + * See can_migrate_task(). */ if (idle != CPU_NEWLY_IDLE && - env.migration_type != migrate_misfit) + env.migration_type != migrate_misfit && + !(env.flags & LBF_LLC_PINNED)) sd->nr_balance_failed++; if (need_active_balance(&env)) { diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 3c9e92b79041..a56619b3761f 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1547,6 +1547,14 @@ extern void sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags); extern void sched_core_get(void); extern void sched_core_put(void); +static inline bool task_has_sched_core(struct task_struct *p) +{ + if (sched_core_disabled()) + return false; + + return !!p->core_cookie; +} + #else /* !CONFIG_SCHED_CORE: */ static inline bool sched_core_enabled(struct rq *rq) @@ -1587,6 +1595,11 @@ static inline bool sched_group_cookie_match(struct rq *rq, return true; } +static inline bool task_has_sched_core(struct task_struct *p) +{ + return false; +} + #endif /* !CONFIG_SCHED_CORE */ #ifdef CONFIG_RT_GROUP_SCHED -- cgit v1.2.3 From d59f4fd1d303987f434bcf0b8191e89ca1d6a67c Mon Sep 17 00:00:00 2001 From: Chen Yu Date: Wed, 1 Apr 2026 14:52:30 -0700 Subject: sched/cache: Enable cache aware scheduling for multi LLCs NUMA node Introduce sched_cache_present to enable cache aware scheduling for multi LLCs NUMA node Cache-aware load balancing should only be enabled if there are more than 1 LLCs within 1 NUMA node. sched_cache_present is introduced to indicate whether this platform supports this topology. Test results: The first test platform is a 2 socket Intel Sapphire Rapids with 30 cores per socket. The DRAM interleaving is enabled in the BIOS so it essential has one NUMA node with two last level caches. There are 60 CPUs associated with each last level cache. The second test platform is a AMD Genoa. There are 4 Nodes and 32 CPUs per node. Each node has 2 CCXs and each CCX has 16 CPUs. hackbench/schbench/netperf/stream/stress-ng/chacha20 were launched on these two platforms. [TL;DR] Sappire Rapids: hackbench shows significant improvement when the number of different active threads is below the capacity of a LLC. schbench shows limitted wakeup latency improvement. ChaCha20-xiangshan(risc-v simulator) shows good throughput improvement. No obvious difference was observed in netperf/stream/stress-ng in Hmean. Genoa: Significant improvement is observed in hackbench when the active number of threads is lower than the number of CPUs within 1 LLC. On v2, Aaron reported improvement of hackbench/redis when system is underloaded. ChaCha20-xiangshan shows huge throughput improvement. Phoronix has tested v1 and shows good improvements in 30+ cases[3]. No obvious difference was observed in netperf/stream/stress-ng in Hmean. Detail: Due to length constraints, data without much difference with baseline is not presented. Sapphire Rapids: [hackbench pipe] ================ case load baseline(std%) compare%( std%) threads-pipe-10 1-groups 1.00 ( 1.22) +26.09 ( 1.10) threads-pipe-10 2-groups 1.00 ( 4.90) +22.88 ( 0.18) threads-pipe-10 4-groups 1.00 ( 2.07) +9.00 ( 3.49) threads-pipe-10 8-groups 1.00 ( 8.13) +3.45 ( 3.62) threads-pipe-16 1-groups 1.00 ( 2.11) +26.30 ( 0.08) threads-pipe-16 2-groups 1.00 ( 15.13) -1.77 ( 11.89) threads-pipe-16 4-groups 1.00 ( 4.37) +0.58 ( 7.99) threads-pipe-16 8-groups 1.00 ( 2.88) +2.71 ( 3.50) threads-pipe-2 1-groups 1.00 ( 9.40) +22.07 ( 0.71) threads-pipe-2 2-groups 1.00 ( 9.99) +18.01 ( 0.95) threads-pipe-2 4-groups 1.00 ( 3.98) +24.66 ( 0.96) threads-pipe-2 8-groups 1.00 ( 7.00) +21.83 ( 0.23) threads-pipe-20 1-groups 1.00 ( 1.03) +28.84 ( 0.21) threads-pipe-20 2-groups 1.00 ( 4.42) +31.90 ( 3.15) threads-pipe-20 4-groups 1.00 ( 9.97) +4.56 ( 1.69) threads-pipe-20 8-groups 1.00 ( 1.87) +1.25 ( 0.74) threads-pipe-4 1-groups 1.00 ( 4.48) +25.67 ( 0.78) threads-pipe-4 2-groups 1.00 ( 9.14) +4.91 ( 2.08) threads-pipe-4 4-groups 1.00 ( 7.68) +19.36 ( 1.53) threads-pipe-4 8-groups 1.00 ( 10.79) +7.20 ( 12.20) threads-pipe-8 1-groups 1.00 ( 4.69) +21.93 ( 0.03) threads-pipe-8 2-groups 1.00 ( 1.16) +25.29 ( 0.65) threads-pipe-8 4-groups 1.00 ( 2.23) -1.27 ( 3.62) threads-pipe-8 8-groups 1.00 ( 4.65) -3.08 ( 2.75) Note: The default number of fd in hackbench is changed from 20 to various values to ensure that threads fit within a single LLC, especially on AMD systems. Take "threads-pipe-8, 2-groups" for example, the number of fd is 8, and 2 groups are created. [schbench] The 99th percentile wakeup latency shows some improvements when the system is underload, while it does not bring much difference with the increasing of system utilization. 99th Wakeup Latencies Base (mean std) Compare (mean std) Change ========================================================================= thread=2 9.00(0.00) 9.00(1.73) 0.00% thread=4 7.33(0.58) 6.33(0.58) +13.64% thread=8 9.00(0.00) 7.67(1.15) +14.78% thread=16 8.67(0.58) 8.67(1.53) 0.00% thread=32 9.00(0.00) 7.00(0.00) +22.22% thread=64 9.33(0.58) 9.67(0.58) -3.64% thread=128 12.00(0.00) 12.00(0.00) 0.00% [chacha20 on simulated risc-v] baseline: Host time spent: 67861ms cache aware scheduling enabled: Host time spent: 54441ms Time reduced by 24% Genoa: [hackbench pipe] The default number of fd is 20, which exceed the number of CPUs in a LLC. So the fd is adjusted to 2, 4, 6, 8, 20 respectively. Exclude the result with large run-to-run variance, 10% ~ 50% improvement is observed when the system is underloaded: [hackbench pipe] ================ case load baseline(std%) compare%( std%) threads-pipe-2 1-groups 1.00 ( 2.89) +47.33 ( 1.20) threads-pipe-2 2-groups 1.00 ( 3.88) +39.82 ( 0.61) threads-pipe-2 4-groups 1.00 ( 8.76) +5.57 ( 13.10) threads-pipe-20 1-groups 1.00 ( 4.61) +11.72 ( 1.06) threads-pipe-20 2-groups 1.00 ( 6.18) +14.55 ( 1.47) threads-pipe-20 4-groups 1.00 ( 2.99) +10.16 ( 4.49) threads-pipe-4 1-groups 1.00 ( 4.23) +43.70 ( 2.14) threads-pipe-4 2-groups 1.00 ( 3.68) +8.45 ( 4.04) threads-pipe-4 4-groups 1.00 ( 17.72) +2.42 ( 1.14) threads-pipe-6 1-groups 1.00 ( 3.10) +7.74 ( 3.83) threads-pipe-6 2-groups 1.00 ( 3.42) +14.26 ( 4.53) threads-pipe-6 4-groups 1.00 ( 10.34) +10.94 ( 7.12) threads-pipe-8 1-groups 1.00 ( 4.21) +9.06 ( 4.43) threads-pipe-8 2-groups 1.00 ( 1.88) +3.74 ( 0.58) threads-pipe-8 4-groups 1.00 ( 2.78) +23.96 ( 1.18) [chacha20 on simulated risc-v] Host time spent: 54762ms Host time spent: 28295ms Time reduced by 48% Suggested-by: Libo Chen Suggested-by: Adam Li Signed-off-by: Chen Yu Co-developed-by: Tim Chen Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/71972e12ab4f08aff422b31e34df09bdbd94de84.1775065312.git.tim.c.chen@linux.intel.com --- kernel/sched/sched.h | 4 +++- kernel/sched/topology.c | 19 +++++++++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index a56619b3761f..71f6077da466 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -4069,9 +4069,11 @@ static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct #endif /* !CONFIG_SCHED_MM_CID */ #ifdef CONFIG_SCHED_CACHE +DECLARE_STATIC_KEY_FALSE(sched_cache_present); + static inline bool sched_cache_enabled(void) { - return false; + return static_branch_unlikely(&sched_cache_present); } #endif diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 8954bf7900ff..6a36f8f6b7b1 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -821,6 +821,7 @@ enum s_alloc { }; #ifdef CONFIG_SCHED_CACHE +DEFINE_STATIC_KEY_FALSE(sched_cache_present); static bool alloc_sd_llc(const struct cpumask *cpu_map, struct s_data *d) { @@ -2777,6 +2778,7 @@ static int build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *attr) { enum s_alloc alloc_state = sa_none; + bool has_multi_llcs = false; struct sched_domain *sd; struct s_data d; struct rq *rq = NULL; @@ -2870,8 +2872,11 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att * In presence of higher domains, adjust the * NUMA imbalance stats for the hierarchy. */ - if (IS_ENABLED(CONFIG_NUMA) && sd->parent) - adjust_numa_imbalance(sd); + if (sd->parent) { + if (IS_ENABLED(CONFIG_NUMA)) + adjust_numa_imbalance(sd); + has_multi_llcs = true; + } } } @@ -2912,6 +2917,16 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att ret = 0; error: +#ifdef CONFIG_SCHED_CACHE + /* + * TBD: check before writing to it. sched domain rebuild + * is not in the critical path, leave as-is for now. + */ + if (!ret && has_multi_llcs) + static_branch_enable_cpuslocked(&sched_cache_present); + else + static_branch_disable_cpuslocked(&sched_cache_present); +#endif __free_domain_allocs(&d, alloc_state, cpu_map); return ret; -- cgit v1.2.3 From 067a3135814334a8ea7241faef364cc48c6340bc Mon Sep 17 00:00:00 2001 From: Chen Yu Date: Wed, 1 Apr 2026 14:52:31 -0700 Subject: sched/cache: Allow the user space to turn on and off cache aware scheduling Provide a debugfs directory llc_balancing, and a knob named "enabled" under it to allow the user to turn off and on the cache aware scheduling at runtime. Suggested-by: Peter Zijlstra (Intel) Signed-off-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/0aa56f7fc48db2f8f700cd1aa34dedd0ec88351b.1775065312.git.tim.c.chen@linux.intel.com --- kernel/sched/debug.c | 48 +++++++++++++++++++++++++++++++++++- kernel/sched/sched.h | 7 +++++- kernel/sched/topology.c | 65 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 74c1617cf652..2eae67cd2ba2 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -210,6 +210,46 @@ static const struct file_operations sched_scaling_fops = { .release = single_release, }; +#ifdef CONFIG_SCHED_CACHE +static ssize_t +sched_cache_enable_write(struct file *filp, const char __user *ubuf, + size_t cnt, loff_t *ppos) +{ + bool val; + int ret; + + ret = kstrtobool_from_user(ubuf, cnt, &val); + if (ret) + return ret; + + sysctl_sched_cache_user = val; + + sched_cache_active_set_unlocked(); + + return cnt; +} + +static int sched_cache_enable_show(struct seq_file *m, void *v) +{ + seq_printf(m, "%d\n", sysctl_sched_cache_user); + return 0; +} + +static int sched_cache_enable_open(struct inode *inode, + struct file *filp) +{ + return single_open(filp, sched_cache_enable_show, NULL); +} + +static const struct file_operations sched_cache_enable_fops = { + .open = sched_cache_enable_open, + .write = sched_cache_enable_write, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; +#endif + #ifdef CONFIG_PREEMPT_DYNAMIC static ssize_t sched_dynamic_write(struct file *filp, const char __user *ubuf, @@ -593,7 +633,7 @@ static void debugfs_ext_server_init(void) static __init int sched_init_debug(void) { - struct dentry __maybe_unused *numa; + struct dentry __maybe_unused *numa, *llc; debugfs_sched = debugfs_create_dir("sched", NULL); @@ -626,6 +666,12 @@ static __init int sched_init_debug(void) debugfs_create_u32("hot_threshold_ms", 0644, numa, &sysctl_numa_balancing_hot_threshold); #endif /* CONFIG_NUMA_BALANCING */ +#ifdef CONFIG_SCHED_CACHE + llc = debugfs_create_dir("llc_balancing", debugfs_sched); + debugfs_create_file("enabled", 0644, llc, NULL, + &sched_cache_enable_fops); +#endif + debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); debugfs_fair_server_init(); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 71f6077da466..f499d5dd1130 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -4070,11 +4070,16 @@ static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct #ifdef CONFIG_SCHED_CACHE DECLARE_STATIC_KEY_FALSE(sched_cache_present); +DECLARE_STATIC_KEY_FALSE(sched_cache_active); +extern int sysctl_sched_cache_user; static inline bool sched_cache_enabled(void) { - return static_branch_unlikely(&sched_cache_present); + return static_branch_unlikely(&sched_cache_active); } + +extern void sched_cache_active_set_unlocked(void); + #endif void sched_domains_free_llc_id(int cpu); diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 6a36f8f6b7b1..9fc99346ef4f 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -821,7 +821,16 @@ enum s_alloc { }; #ifdef CONFIG_SCHED_CACHE +/* hardware support for cache aware scheduling */ DEFINE_STATIC_KEY_FALSE(sched_cache_present); +/* + * Indicator of whether cache aware scheduling + * is active, used by the scheduler. + */ +DEFINE_STATIC_KEY_FALSE(sched_cache_active); +/* user wants cache aware scheduling [0 or 1] */ +int sysctl_sched_cache_user = 1; + static bool alloc_sd_llc(const struct cpumask *cpu_map, struct s_data *d) { @@ -856,6 +865,60 @@ err: return false; } + +static void _sched_cache_active_set(bool enable, bool locked) +{ + if (enable) { + if (locked) + static_branch_enable_cpuslocked(&sched_cache_active); + else + static_branch_enable(&sched_cache_active); + } else { + if (locked) + static_branch_disable_cpuslocked(&sched_cache_active); + else + static_branch_disable(&sched_cache_active); + } +} + +/* + * Enable/disable cache aware scheduling according to + * user input and the presence of hardware support. + */ +static void sched_cache_active_set(bool locked) +{ + /* hardware does not support */ + if (!static_branch_likely(&sched_cache_present)) { + _sched_cache_active_set(false, locked); + return; + } + + /* + * user wants it or not ? + * TBD: read before writing the static key. + * It is not in the critical path, leave as-is + * for now. + */ + if (sysctl_sched_cache_user) { + _sched_cache_active_set(true, locked); + if (sched_debug()) + pr_info("%s: enabling cache aware scheduling\n", __func__); + } else { + _sched_cache_active_set(false, locked); + if (sched_debug()) + pr_info("%s: disabling cache aware scheduling\n", __func__); + } +} + +static void sched_cache_active_set_locked(void) +{ + return sched_cache_active_set(true); +} + +void sched_cache_active_set_unlocked(void) +{ + return sched_cache_active_set(false); +} #else static bool alloc_sd_llc(const struct cpumask *cpu_map, struct s_data *d) @@ -2926,6 +2989,8 @@ error: static_branch_enable_cpuslocked(&sched_cache_present); else static_branch_disable_cpuslocked(&sched_cache_present); + + sched_cache_active_set_locked(); #endif __free_domain_allocs(&d, alloc_state, cpu_map); -- cgit v1.2.3 From 5425abc222bcdb6913aafb2a1449fe9b8cdb308e Mon Sep 17 00:00:00 2001 From: Zhao Mengmeng Date: Fri, 17 Apr 2026 17:18:03 +0800 Subject: sched_ext: Print sub-scheduler disabled log and reason Take scx_qmap for example, when sub scheduler is attached, there is 'BPF sub-scheduler "qmap" enabled' message, but when detached, the log is missing. Add a new function to do the log thing, it can be used by both root scheduler and sub scheduler. Signed-off-by: Zhao Mengmeng Reviewed-by: Cheng-Yang Chou Signed-off-by: Tejun Heo --- kernel/sched/ext.c | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 012ca8bd70fb..229a82900e8f 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -5526,6 +5526,26 @@ static void scx_disable_dump(struct scx_sched *sch) sch->dump_disabled = true; } +static void scx_log_sched_disable(struct scx_sched *sch) +{ + struct scx_exit_info *ei = sch->exit_info; + const char *type = scx_parent(sch) ? "sub-scheduler" : "scheduler"; + + if (ei->kind >= SCX_EXIT_ERROR) { + pr_err("sched_ext: BPF %s \"%s\" disabled (%s)\n", type, + sch->ops.name, ei->reason); + + if (ei->msg[0] != '\0') + pr_err("sched_ext: %s: %s\n", sch->ops.name, ei->msg); +#ifdef CONFIG_STACKTRACE + stack_trace_print(ei->bt, ei->bt_len, 2); +#endif + } else { + pr_info("sched_ext: BPF %s \"%s\" disabled (%s)\n", type, + sch->ops.name, ei->reason); + } +} + #ifdef CONFIG_EXT_SUB_SCHED static DECLARE_WAIT_QUEUE_HEAD(scx_unlink_waitq); @@ -5696,6 +5716,8 @@ static void scx_sub_disable(struct scx_sched *sch) &sub_detach_args); } + scx_log_sched_disable(sch); + if (sch->ops.exit) SCX_CALL_OP(sch, exit, NULL, sch->exit_info); kobject_del(&sch->kobj); @@ -5707,7 +5729,6 @@ static void scx_sub_disable(struct scx_sched *sch) { } static void scx_root_disable(struct scx_sched *sch) { - struct scx_exit_info *ei = sch->exit_info; struct scx_task_iter sti; struct task_struct *p; int cpu; @@ -5797,22 +5818,10 @@ static void scx_root_disable(struct scx_sched *sch) scx_idle_disable(); synchronize_rcu(); - if (ei->kind >= SCX_EXIT_ERROR) { - pr_err("sched_ext: BPF scheduler \"%s\" disabled (%s)\n", - sch->ops.name, ei->reason); - - if (ei->msg[0] != '\0') - pr_err("sched_ext: %s: %s\n", sch->ops.name, ei->msg); -#ifdef CONFIG_STACKTRACE - stack_trace_print(ei->bt, ei->bt_len, 2); -#endif - } else { - pr_info("sched_ext: BPF scheduler \"%s\" disabled (%s)\n", - sch->ops.name, ei->reason); - } + scx_log_sched_disable(sch); if (sch->ops.exit) - SCX_CALL_OP(sch, exit, NULL, ei); + SCX_CALL_OP(sch, exit, NULL, sch->exit_info); scx_unlink_sched(sch); -- cgit v1.2.3 From 31f61ac33032ee87ea404d6d996ba2c386502a36 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Tue, 14 Apr 2026 12:10:14 -0700 Subject: bpf: Refactor dynptr mutability tracking Redefine dynptr mutability and fix inconsistency in the verifier and kfunc signatures. Dynptr mutability is at two levels. The first is the bpf_dynptr structure and the second is the memory the dynptr points to. The verifer currently tracks the mutability of the bpf_dynptr struct through helper and kfunc prototypes, where "const struct bpf_dynptr *" means the structure itself is immutable. The second level is tracked in upper bit of bpf_dynptr->size in runtime and is not changed in this patch. There are two type of inconsistency in the verfier regarding the mutability of the bpf_dynptr struct. First, there are many existing kfuncs whose prototypes are wrong. For example, bpf_dynptr_adjust() mutates a dynptr's start and offset but marks the argument as a const pointer. At the same time many other kfuncs that does not mutate the dynptr but mark themselves as mutable. Second, the verifier currently does not honor the const qualifier in kfunc prototypes as it determines whether tagging the arg_type with MEM_RDONLY or not based on the register state. Since all the verifier care is to prevent CONST_PTR_TO_DYNPTR from being destroyed in callback and global subprogram, redefine the mutability at the bpf_dynptr level to just bpf_dynptr_kern->data. Then, explicitly prohibit passing CONST_PTR_TO_DYNPTR to an argument tagged with MEM_UNINIT or OBJ_RELEASE. The mutability of a dynptr's view is not really interesting so drop MEM_RDONLY annotation for dynptr from the helpers and kfuncs. Plus, if the mutability of the entire bpf_dynptr were to be done correctly, it would kill the bpf_dynptr_adjust() usage in callback and global subporgram. Implementation wise - First, make sure all kfunc arg are correctly tagged: Tag the dynptr argument of bpf_dynptr_file_discard() with OBJ_RELEASE. - Then, in process_dynptr_func(), make sure CONST_PTR_TO_DYNPTR cannot be passed to argument tagged with MEM_UNINIT or OBJ_RELEASE. For MEM_UNINIT, it is already checked by is_dynptr_reg_valid_uninit(). For OBJ_RELEASE, check against OBJ_RELEASE instead of MEM_RDONLY and drop a now identical check in unmark_stack_slots_dynptr(). - Remove the mutual exclusive check between MEM_UNINIT and MEM_RDONLY, but don't add a MEM_UNINIT and OBJ_RELEASE version as it is obviously wrong. Note that while this patch stops following the C semantic for the mutability of bpf_dynptr, the prototype of kfuncs are still fixed to maintain the correct C semantics in the implementation. Adding or removing the const qualifier does not break backward compatibility. In addition, fix kfuncs dropping the const qualifier when casting the opaque bpf_dynptr to bpf_dynptr_kern. In test_kfunc_dynptr_param.c, initialize dynptr to 0 to avoid -Wuninitialized-const-pointer warning. Signed-off-by: Amery Hung Acked-by: Kumar Kartikeya Dwivedi Link: https://lore.kernel.org/bpf/20260414191014.1218567-1-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/btf.c | 2 +- kernel/bpf/helpers.c | 36 ++++++++++++------------- kernel/bpf/verifier.c | 70 +++++++++++++----------------------------------- kernel/trace/bpf_trace.c | 22 +++++++-------- 4 files changed, 49 insertions(+), 81 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index a62d78581207..3c2aaa3c5004 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7973,7 +7973,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) bpf_log(log, "arg#%d has invalid combination of tags\n", i); return -EINVAL; } - sub->args[i].arg_type = ARG_PTR_TO_DYNPTR | MEM_RDONLY; + sub->args[i].arg_type = ARG_PTR_TO_DYNPTR; continue; } if (tags & ARG_TAG_TRUSTED) { diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 2bb60200c266..baa12b24bb64 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -1944,7 +1944,7 @@ static const struct bpf_func_proto bpf_dynptr_read_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, .arg2_type = ARG_CONST_SIZE_OR_ZERO, - .arg3_type = ARG_PTR_TO_DYNPTR | MEM_RDONLY, + .arg3_type = ARG_PTR_TO_DYNPTR, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -2001,7 +2001,7 @@ static const struct bpf_func_proto bpf_dynptr_write_proto = { .func = bpf_dynptr_write, .gpl_only = false, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_DYNPTR | MEM_RDONLY, + .arg1_type = ARG_PTR_TO_DYNPTR, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg4_type = ARG_CONST_SIZE_OR_ZERO, @@ -2044,7 +2044,7 @@ static const struct bpf_func_proto bpf_dynptr_data_proto = { .func = bpf_dynptr_data, .gpl_only = false, .ret_type = RET_PTR_TO_DYNPTR_MEM_OR_NULL, - .arg1_type = ARG_PTR_TO_DYNPTR | MEM_RDONLY, + .arg1_type = ARG_PTR_TO_DYNPTR, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_CONST_ALLOC_SIZE_OR_ZERO, }; @@ -3072,7 +3072,7 @@ __bpf_kfunc void *bpf_dynptr_slice_rdwr(const struct bpf_dynptr *p, u64 offset, return bpf_dynptr_slice(p, offset, buffer__nullable, buffer__szk); } -__bpf_kfunc int bpf_dynptr_adjust(const struct bpf_dynptr *p, u64 start, u64 end) +__bpf_kfunc int bpf_dynptr_adjust(struct bpf_dynptr *p, u64 start, u64 end) { struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p; u64 size; @@ -3093,14 +3093,14 @@ __bpf_kfunc int bpf_dynptr_adjust(const struct bpf_dynptr *p, u64 start, u64 end __bpf_kfunc bool bpf_dynptr_is_null(const struct bpf_dynptr *p) { - struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p; + const struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p; return !ptr->data; } __bpf_kfunc bool bpf_dynptr_is_rdonly(const struct bpf_dynptr *p) { - struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p; + const struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p; if (!ptr->data) return false; @@ -3110,7 +3110,7 @@ __bpf_kfunc bool bpf_dynptr_is_rdonly(const struct bpf_dynptr *p) __bpf_kfunc u64 bpf_dynptr_size(const struct bpf_dynptr *p) { - struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p; + const struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p; if (!ptr->data) return -EINVAL; @@ -3122,7 +3122,7 @@ __bpf_kfunc int bpf_dynptr_clone(const struct bpf_dynptr *p, struct bpf_dynptr *clone__uninit) { struct bpf_dynptr_kern *clone = (struct bpf_dynptr_kern *)clone__uninit; - struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p; + const struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p; if (!ptr->data) { bpf_dynptr_set_null(clone); @@ -3145,11 +3145,11 @@ __bpf_kfunc int bpf_dynptr_clone(const struct bpf_dynptr *p, * Copies data from source dynptr to destination dynptr. * Returns 0 on success; negative error, otherwise. */ -__bpf_kfunc int bpf_dynptr_copy(struct bpf_dynptr *dst_ptr, u64 dst_off, - struct bpf_dynptr *src_ptr, u64 src_off, u64 size) +__bpf_kfunc int bpf_dynptr_copy(const struct bpf_dynptr *dst_ptr, u64 dst_off, + const struct bpf_dynptr *src_ptr, u64 src_off, u64 size) { - struct bpf_dynptr_kern *dst = (struct bpf_dynptr_kern *)dst_ptr; - struct bpf_dynptr_kern *src = (struct bpf_dynptr_kern *)src_ptr; + const struct bpf_dynptr_kern *dst = (struct bpf_dynptr_kern *)dst_ptr; + const struct bpf_dynptr_kern *src = (struct bpf_dynptr_kern *)src_ptr; void *src_slice, *dst_slice; char buf[256]; u64 off; @@ -3200,9 +3200,9 @@ __bpf_kfunc int bpf_dynptr_copy(struct bpf_dynptr *dst_ptr, u64 dst_off, * at @offset with the constant byte @val. * Returns 0 on success; negative error, otherwise. */ -__bpf_kfunc int bpf_dynptr_memset(struct bpf_dynptr *p, u64 offset, u64 size, u8 val) +__bpf_kfunc int bpf_dynptr_memset(const struct bpf_dynptr *p, u64 offset, u64 size, u8 val) { - struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p; + const struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p; u64 chunk_sz, write_off; char buf[256]; void* slice; @@ -4214,13 +4214,13 @@ __bpf_kfunc void bpf_key_put(struct bpf_key *bkey) * * Return: 0 on success, a negative value on error. */ -__bpf_kfunc int bpf_verify_pkcs7_signature(struct bpf_dynptr *data_p, - struct bpf_dynptr *sig_p, +__bpf_kfunc int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_p, + const struct bpf_dynptr *sig_p, struct bpf_key *trusted_keyring) { #ifdef CONFIG_SYSTEM_DATA_VERIFICATION - struct bpf_dynptr_kern *data_ptr = (struct bpf_dynptr_kern *)data_p; - struct bpf_dynptr_kern *sig_ptr = (struct bpf_dynptr_kern *)sig_p; + const struct bpf_dynptr_kern *data_ptr = (struct bpf_dynptr_kern *)data_p; + const struct bpf_dynptr_kern *sig_ptr = (struct bpf_dynptr_kern *)sig_p; const void *data, *sig; u32 data_len, sig_len; int ret; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 69d75515ed3f..185210b73385 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -717,15 +717,6 @@ static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_re struct bpf_func_state *state = bpf_func(env, reg); int spi, ref_obj_id, i; - /* - * This can only be set for PTR_TO_STACK, as CONST_PTR_TO_DYNPTR cannot - * be released by any dynptr helper. Hence, unmark_stack_slots_dynptr - * is safe to do directly. - */ - if (reg->type == CONST_PTR_TO_DYNPTR) { - verifier_bug(env, "CONST_PTR_TO_DYNPTR cannot be released"); - return -EFAULT; - } spi = dynptr_get_spi(env, reg); if (spi < 0) return spi; @@ -7434,23 +7425,12 @@ static int process_kptr_func(struct bpf_verifier_env *env, int regno, * bytes as STACK_DYNPTR in case of PTR_TO_STACK. In case of * CONST_PTR_TO_DYNPTR, we are guaranteed to get the beginning of the object. * - * Mutability of bpf_dynptr is at two levels, one is at the level of struct - * bpf_dynptr itself, i.e. whether the helper is receiving a pointer to struct - * bpf_dynptr or pointer to const struct bpf_dynptr. In the former case, it can - * mutate the view of the dynptr and also possibly destroy it. In the latter - * case, it cannot mutate the bpf_dynptr itself but it can still mutate the - * memory that dynptr points to. - * - * The verifier will keep track both levels of mutation (bpf_dynptr's in - * reg->type and the memory's in reg->dynptr.type), but there is no support for - * readonly dynptr view yet, hence only the first case is tracked and checked. - * - * This is consistent with how C applies the const modifier to a struct object, - * where the pointer itself inside bpf_dynptr becomes const but not what it - * points to. - * - * Helpers which do not mutate the bpf_dynptr set MEM_RDONLY in their argument - * type, and declare it as 'const struct bpf_dynptr *' in their prototype. + * Mutability of bpf_dynptr is at two levels: the dynptr and the memory the + * dynptr points to. At the first level, the verifier will make sure a + * CONST_PTR_TO_DYNPTR cannot be reinitialized or destroyed. The mutability of + * a dynptr's view (i.e., start and offset) is not tracked as there is not such + * use case. The second level is tracked using the upper bit of bpf_dynptr->size + * and checked dynamically during runtime. */ static int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn_idx, enum bpf_arg_type arg_type, int clone_ref_obj_id) @@ -7465,14 +7445,6 @@ static int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn return -EINVAL; } - /* MEM_UNINIT and MEM_RDONLY are exclusive, when applied to an - * ARG_PTR_TO_DYNPTR (or ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_*): - */ - if ((arg_type & (MEM_UNINIT | MEM_RDONLY)) == (MEM_UNINIT | MEM_RDONLY)) { - verifier_bug(env, "misconfigured dynptr helper type flags"); - return -EFAULT; - } - /* MEM_UNINIT - Points to memory that is an appropriate candidate for * constructing a mutable bpf_dynptr object. * @@ -7480,13 +7452,12 @@ static int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn * pointing to a region of at least 16 bytes which doesn't * contain an existing bpf_dynptr. * - * MEM_RDONLY - Points to a initialized bpf_dynptr that will not be - * mutated or destroyed. However, the memory it points to - * may be mutated. + * OBJ_RELEASE - Points to a initialized bpf_dynptr that will be + * destroyed. * - * None - Points to a initialized dynptr that can be mutated and - * destroyed, including mutation of the memory it points - * to. + * None - Points to a initialized dynptr that cannot be + * reinitialized or destroyed. However, the view of the + * dynptr and the memory it points to may be mutated. */ if (arg_type & MEM_UNINIT) { int i; @@ -7505,10 +7476,10 @@ static int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn } err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, clone_ref_obj_id); - } else /* MEM_RDONLY and None case from above */ { + } else /* OBJ_RELEASE and None case from above */ { /* For the reg->type == PTR_TO_STACK case, bpf_dynptr is never const */ - if (reg->type == CONST_PTR_TO_DYNPTR && !(arg_type & MEM_RDONLY)) { - verbose(env, "cannot pass pointer to const bpf_dynptr, the helper mutates it\n"); + if (reg->type == CONST_PTR_TO_DYNPTR && (arg_type & OBJ_RELEASE)) { + verbose(env, "CONST_PTR_TO_DYNPTR cannot be released\n"); return -EINVAL; } @@ -7519,8 +7490,8 @@ static int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn return -EINVAL; } - /* Fold modifiers (in this case, MEM_RDONLY) when checking expected type */ - if (!is_dynptr_type_expected(env, reg, arg_type & ~MEM_RDONLY)) { + /* Fold modifiers (in this case, OBJ_RELEASE) when checking expected type */ + if (!is_dynptr_type_expected(env, reg, arg_type & ~OBJ_RELEASE)) { verbose(env, "Expected a dynptr of type %s as arg #%d\n", dynptr_type_str(arg_to_dynptr_type(arg_type)), regno - 1); @@ -9366,7 +9337,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, bpf_log(log, "R%d is not a pointer to arena or scalar.\n", regno); return -EINVAL; } - } else if (arg->arg_type == (ARG_PTR_TO_DYNPTR | MEM_RDONLY)) { + } else if (arg->arg_type == ARG_PTR_TO_DYNPTR) { ret = check_func_arg_reg_off(env, reg, regno, ARG_PTR_TO_DYNPTR); if (ret) return ret; @@ -12273,9 +12244,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR; int clone_ref_obj_id = 0; - if (reg->type == CONST_PTR_TO_DYNPTR) - dynptr_arg_type |= MEM_RDONLY; - if (is_kfunc_arg_uninit(btf, &args[i])) dynptr_arg_type |= MEM_UNINIT; @@ -12288,7 +12256,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_file]) { dynptr_arg_type |= DYNPTR_TYPE_FILE; } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_file_discard]) { - dynptr_arg_type |= DYNPTR_TYPE_FILE; + dynptr_arg_type |= DYNPTR_TYPE_FILE | OBJ_RELEASE; meta->release_regno = regno; } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_clone] && (dynptr_arg_type & MEM_UNINIT)) { @@ -18745,7 +18713,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) } else if (arg->arg_type == ARG_ANYTHING) { reg->type = SCALAR_VALUE; mark_reg_unknown(env, regs, i); - } else if (arg->arg_type == (ARG_PTR_TO_DYNPTR | MEM_RDONLY)) { + } else if (arg->arg_type == ARG_PTR_TO_DYNPTR) { /* assume unspecial LOCAL dynptr type */ __mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_LOCAL, true, ++env->id_gen); } else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) { diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index af7079aa0f36..e916f0ccbed9 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -3397,12 +3397,12 @@ typedef int (*copy_fn_t)(void *dst, const void *src, u32 size, struct task_struc * direct calls into all the specific callback implementations * (copy_user_data_sleepable, copy_user_data_nofault, and so on) */ -static __always_inline int __bpf_dynptr_copy_str(struct bpf_dynptr *dptr, u64 doff, u64 size, +static __always_inline int __bpf_dynptr_copy_str(const struct bpf_dynptr *dptr, u64 doff, u64 size, const void *unsafe_src, copy_fn_t str_copy_fn, struct task_struct *tsk) { - struct bpf_dynptr_kern *dst; + const struct bpf_dynptr_kern *dst; u64 chunk_sz, off; void *dst_slice; int cnt, err; @@ -3438,7 +3438,7 @@ static __always_inline int __bpf_dynptr_copy(const struct bpf_dynptr *dptr, u64 u64 size, const void *unsafe_src, copy_fn_t copy_fn, struct task_struct *tsk) { - struct bpf_dynptr_kern *dst; + const struct bpf_dynptr_kern *dst; void *dst_slice; char buf[256]; u64 off, chunk_sz; @@ -3539,49 +3539,49 @@ __bpf_kfunc int bpf_send_signal_task(struct task_struct *task, int sig, enum pid return bpf_send_signal_common(sig, type, task, value); } -__bpf_kfunc int bpf_probe_read_user_dynptr(struct bpf_dynptr *dptr, u64 off, +__bpf_kfunc int bpf_probe_read_user_dynptr(const struct bpf_dynptr *dptr, u64 off, u64 size, const void __user *unsafe_ptr__ign) { return __bpf_dynptr_copy(dptr, off, size, (const void __force *)unsafe_ptr__ign, copy_user_data_nofault, NULL); } -__bpf_kfunc int bpf_probe_read_kernel_dynptr(struct bpf_dynptr *dptr, u64 off, +__bpf_kfunc int bpf_probe_read_kernel_dynptr(const struct bpf_dynptr *dptr, u64 off, u64 size, const void *unsafe_ptr__ign) { return __bpf_dynptr_copy(dptr, off, size, unsafe_ptr__ign, copy_kernel_data_nofault, NULL); } -__bpf_kfunc int bpf_probe_read_user_str_dynptr(struct bpf_dynptr *dptr, u64 off, +__bpf_kfunc int bpf_probe_read_user_str_dynptr(const struct bpf_dynptr *dptr, u64 off, u64 size, const void __user *unsafe_ptr__ign) { return __bpf_dynptr_copy_str(dptr, off, size, (const void __force *)unsafe_ptr__ign, copy_user_str_nofault, NULL); } -__bpf_kfunc int bpf_probe_read_kernel_str_dynptr(struct bpf_dynptr *dptr, u64 off, +__bpf_kfunc int bpf_probe_read_kernel_str_dynptr(const struct bpf_dynptr *dptr, u64 off, u64 size, const void *unsafe_ptr__ign) { return __bpf_dynptr_copy_str(dptr, off, size, unsafe_ptr__ign, copy_kernel_str_nofault, NULL); } -__bpf_kfunc int bpf_copy_from_user_dynptr(struct bpf_dynptr *dptr, u64 off, +__bpf_kfunc int bpf_copy_from_user_dynptr(const struct bpf_dynptr *dptr, u64 off, u64 size, const void __user *unsafe_ptr__ign) { return __bpf_dynptr_copy(dptr, off, size, (const void __force *)unsafe_ptr__ign, copy_user_data_sleepable, NULL); } -__bpf_kfunc int bpf_copy_from_user_str_dynptr(struct bpf_dynptr *dptr, u64 off, +__bpf_kfunc int bpf_copy_from_user_str_dynptr(const struct bpf_dynptr *dptr, u64 off, u64 size, const void __user *unsafe_ptr__ign) { return __bpf_dynptr_copy_str(dptr, off, size, (const void __force *)unsafe_ptr__ign, copy_user_str_sleepable, NULL); } -__bpf_kfunc int bpf_copy_from_user_task_dynptr(struct bpf_dynptr *dptr, u64 off, +__bpf_kfunc int bpf_copy_from_user_task_dynptr(const struct bpf_dynptr *dptr, u64 off, u64 size, const void __user *unsafe_ptr__ign, struct task_struct *tsk) { @@ -3589,7 +3589,7 @@ __bpf_kfunc int bpf_copy_from_user_task_dynptr(struct bpf_dynptr *dptr, u64 off, copy_user_data_sleepable, tsk); } -__bpf_kfunc int bpf_copy_from_user_task_str_dynptr(struct bpf_dynptr *dptr, u64 off, +__bpf_kfunc int bpf_copy_from_user_task_str_dynptr(const struct bpf_dynptr *dptr, u64 off, u64 size, const void __user *unsafe_ptr__ign, struct task_struct *tsk) { -- cgit v1.2.3 From 41e3312861eafba171d9620150aaf2e99165d044 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sun, 19 Apr 2026 08:36:45 -1000 Subject: sched_ext: add p->scx.tid and SCX_OPS_TID_TO_TASK lookup BPF schedulers that can't hold task_struct pointers (arena-backed ones in particular) key tasks by pid. During exit, pid is released before the task finishes passing through scheduler callbacks, so a dying task becomes invisible to the BPF side mid-schedule. scx_qmap hits this: an exiting task's dispatch callback can't recover its queue entry, stalling dispatch until SCX_EXIT_ERROR_STALL. Add a unique non-zero u64 p->scx.tid assigned at fork that survives the full task lifetime including exit. scx_bpf_tid_to_task() looks up the task; unlike bpf_task_from_pid(), it handles exiting tasks. The lookup costs an rhashtable insert/remove under scx_tasks_lock, so root schedulers opt in via SCX_OPS_TID_TO_TASK. Sub-schedulers that set the flag to declare a dependency are rejected at attach if root didn't opt in. scx_qmap converted: keys tasks by tid and enables SCX_OPS_ENQ_EXITING. Pre-patch it stalls within seconds under a non-leader-exec workload; with the patch it runs cleanly. v3: Warn on rhashtable_lookup_insert_fast() failure via new scx_tid_hash_insert() helper (Cheng-Yang Chou). v2: Guard scx_root deref in scx_bpf_tid_to_task() error path. The kfunc is registered via scx_kfunc_set_any and reachable from tracing and syscall programs when no scheduler is attached (Cheng-Yang Chou). Signed-off-by: Tejun Heo Reviewed-by: Cheng-Yang Chou Reviewed-by: Andrea Righi --- kernel/sched/ext.c | 154 +++++++++++++++++++++++++++++++++++++++++--- kernel/sched/ext_internal.h | 20 +++++- 2 files changed, 163 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 4b0527840f2f..b34f1e5df1c5 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -38,6 +38,15 @@ static const struct rhashtable_params scx_sched_hash_params = { static struct rhashtable scx_sched_hash; #endif +/* see SCX_OPS_TID_TO_TASK */ +static const struct rhashtable_params scx_tid_hash_params = { + .key_len = sizeof_field(struct sched_ext_entity, tid), + .key_offset = offsetof(struct sched_ext_entity, tid), + .head_offset = offsetof(struct sched_ext_entity, tid_hash_node), + .insecure_elasticity = true, /* inserted/removed under scx_tasks_lock */ +}; +static struct rhashtable scx_tid_hash; + /* * During exit, a task may schedule after losing its PIDs. When disabling the * BPF scheduler, we need to be able to iterate tasks in every state to @@ -58,10 +67,25 @@ static cpumask_var_t scx_bypass_lb_resched_cpumask; static bool scx_init_task_enabled; static bool scx_switching_all; DEFINE_STATIC_KEY_FALSE(__scx_switched_all); +static DEFINE_STATIC_KEY_FALSE(__scx_tid_to_task_enabled); + +/* + * True once SCX_OPS_TID_TO_TASK has been negotiated with the root scheduler + * and the tid->task table is live. Wraps the static key so callers don't + * take the address, and hints "likely enabled" for the common case where + * the feature is in use. + */ +static inline bool scx_tid_to_task_enabled(void) +{ + return static_branch_likely(&__scx_tid_to_task_enabled); +} static atomic_long_t scx_nr_rejected = ATOMIC_LONG_INIT(0); static atomic_long_t scx_hotplug_seq = ATOMIC_LONG_INIT(0); +/* Global cursor for the per-CPU tid allocator. Starts at 1; tid 0 is reserved. */ +static atomic64_t scx_tid_cursor = ATOMIC64_INIT(1); + #ifdef CONFIG_EXT_SUB_SCHED /* * The sub sched being enabled. Used by scx_disable_and_exit_task() to exit @@ -110,6 +134,17 @@ struct scx_kick_syncs { static DEFINE_PER_CPU(struct scx_kick_syncs __rcu *, scx_kick_syncs); +/* + * Per-CPU buffered allocator state for p->scx.tid. Each CPU pulls a chunk of + * SCX_TID_CHUNK ids from scx_tid_cursor and hands them out locally without + * further synchronization. See scx_alloc_tid(). + */ +struct scx_tid_alloc { + u64 next; + u64 end; +}; +static DEFINE_PER_CPU(struct scx_tid_alloc, scx_tid_alloc); + /* * Direct dispatch marker. * @@ -3665,6 +3700,33 @@ void init_scx_entity(struct sched_ext_entity *scx) scx->slice = SCX_SLICE_DFL; } +/* See scx_tid_alloc / scx_tid_cursor. */ +static u64 scx_alloc_tid(void) +{ + struct scx_tid_alloc *ta; + + guard(preempt)(); + ta = this_cpu_ptr(&scx_tid_alloc); + + if (unlikely(ta->next >= ta->end)) { + ta->next = atomic64_fetch_add(SCX_TID_CHUNK, &scx_tid_cursor); + ta->end = ta->next + SCX_TID_CHUNK; + } + return ta->next++; +} + +static void scx_tid_hash_insert(struct task_struct *p) +{ + int ret; + + lockdep_assert_held(&scx_tasks_lock); + + ret = rhashtable_lookup_insert_fast(&scx_tid_hash, + &p->scx.tid_hash_node, + scx_tid_hash_params); + WARN_ON_ONCE(ret); +} + void scx_pre_fork(struct task_struct *p) { /* @@ -3682,6 +3744,8 @@ int scx_fork(struct task_struct *p, struct kernel_clone_args *kargs) percpu_rwsem_assert_held(&scx_fork_rwsem); + p->scx.tid = scx_alloc_tid(); + if (scx_init_task_enabled) { #ifdef CONFIG_EXT_SUB_SCHED struct scx_sched *sch = kargs->cset->dfl_cgrp->scx_sched; @@ -3717,9 +3781,11 @@ void scx_post_fork(struct task_struct *p) } } - raw_spin_lock_irq(&scx_tasks_lock); - list_add_tail(&p->scx.tasks_node, &scx_tasks); - raw_spin_unlock_irq(&scx_tasks_lock); + scoped_guard(raw_spinlock_irq, &scx_tasks_lock) { + list_add_tail(&p->scx.tasks_node, &scx_tasks); + if (scx_tid_to_task_enabled()) + scx_tid_hash_insert(p); + } percpu_up_read(&scx_fork_rwsem); } @@ -3770,17 +3836,19 @@ static bool task_dead_and_done(struct task_struct *p) void sched_ext_dead(struct task_struct *p) { - unsigned long flags; - /* * By the time control reaches here, @p has %TASK_DEAD set, switched out * for the last time and then dropped the rq lock - task_dead_and_done() * should be returning %true nullifying the straggling sched_class ops. * Remove from scx_tasks and exit @p. */ - raw_spin_lock_irqsave(&scx_tasks_lock, flags); - list_del_init(&p->scx.tasks_node); - raw_spin_unlock_irqrestore(&scx_tasks_lock, flags); + scoped_guard(raw_spinlock_irqsave, &scx_tasks_lock) { + list_del_init(&p->scx.tasks_node); + if (scx_tid_to_task_enabled()) + rhashtable_remove_fast(&scx_tid_hash, + &p->scx.tid_hash_node, + scx_tid_hash_params); + } /* * @p is off scx_tasks and wholly ours. scx_root_enable()'s READY -> @@ -5815,9 +5883,13 @@ static void scx_root_disable(struct scx_sched *sch) /* no task is on scx, turn off all the switches and flush in-progress calls */ static_branch_disable(&__scx_enabled); + if (sch->ops.flags & SCX_OPS_TID_TO_TASK) + static_branch_disable(&__scx_tid_to_task_enabled); bitmap_zero(sch->has_op, SCX_OPI_END); scx_idle_disable(); synchronize_rcu(); + if (sch->ops.flags & SCX_OPS_TID_TO_TASK) + rhashtable_free_and_destroy(&scx_tid_hash, NULL, NULL); scx_log_sched_disable(sch); @@ -6561,6 +6633,17 @@ static int validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) return -EINVAL; } + /* + * SCX_OPS_TID_TO_TASK is enabled by the root scheduler. A sub-sched + * may set it to declare a dependency; reject if the root hasn't + * enabled it. + */ + if ((ops->flags & SCX_OPS_TID_TO_TASK) && scx_parent(sch) && + !(scx_root->ops.flags & SCX_OPS_TID_TO_TASK)) { + scx_error(sch, "SCX_OPS_TID_TO_TASK requires root scheduler to enable it"); + return -EINVAL; + } + /* * SCX_OPS_BUILTIN_IDLE_PER_NODE requires built-in CPU idle * selection policy to be enabled. @@ -6611,13 +6694,19 @@ static void scx_root_enable_workfn(struct kthread_work *work) if (ret) goto err_unlock; + if (ops->flags & SCX_OPS_TID_TO_TASK) { + ret = rhashtable_init(&scx_tid_hash, &scx_tid_hash_params); + if (ret) + goto err_free_ksyncs; + } + #if defined(CONFIG_EXT_GROUP_SCHED) || defined(CONFIG_EXT_SUB_SCHED) cgroup_get(cgrp); #endif sch = scx_alloc_and_add_sched(ops, cgrp, NULL); if (IS_ERR(sch)) { ret = PTR_ERR(sch); - goto err_free_ksyncs; + goto err_free_tid_hash; } /* @@ -6706,6 +6795,10 @@ static void scx_root_enable_workfn(struct kthread_work *work) WARN_ON_ONCE(scx_init_task_enabled); scx_init_task_enabled = true; + /* flip under fork_rwsem; the iter below covers existing tasks */ + if (ops->flags & SCX_OPS_TID_TO_TASK) + static_branch_enable(&__scx_tid_to_task_enabled); + /* * Enable ops for every task. Fork is excluded by scx_fork_rwsem * preventing new tasks from being added. No need to exclude tasks @@ -6749,6 +6842,17 @@ static void scx_root_enable_workfn(struct kthread_work *work) scx_set_task_sched(p, sch); scx_set_task_state(p, SCX_TASK_READY); + /* + * Insert into the tid hash under scx_tasks_lock so we can't + * race sched_ext_dead() and leave a stale entry for an already + * exited task. + */ + if (scx_tid_to_task_enabled()) { + guard(raw_spinlock_irq)(&scx_tasks_lock); + if (!list_empty(&p->scx.tasks_node)) + scx_tid_hash_insert(p); + } + put_task_struct(p); } scx_task_iter_stop(&sti); @@ -6808,6 +6912,9 @@ static void scx_root_enable_workfn(struct kthread_work *work) cmd->ret = 0; return; +err_free_tid_hash: + if (ops->flags & SCX_OPS_TID_TO_TASK) + rhashtable_free_and_destroy(&scx_tid_hash, NULL, NULL); err_free_ksyncs: free_kick_syncs(); err_unlock: @@ -9296,6 +9403,34 @@ __bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu, const struct bpf_prog_ return rcu_dereference(cpu_rq(cpu)->curr); } +/** + * scx_bpf_tid_to_task - Look up a task by its scx tid + * @tid: task ID previously read from p->scx.tid + * + * Returns the task with the given tid, or NULL if no such task exists. The + * returned pointer is valid until the end of the current RCU read section + * (KF_RCU_PROTECTED). Requires SCX_OPS_TID_TO_TASK to be set on the root + * scheduler; otherwise an error is raised and NULL returned. + */ +__bpf_kfunc struct task_struct *scx_bpf_tid_to_task(u64 tid) +{ + struct sched_ext_entity *scx; + + if (!scx_tid_to_task_enabled()) { + struct scx_sched *sch = rcu_dereference(scx_root); + + if (sch) + scx_error(sch, "scx_bpf_tid_to_task() called without SCX_OPS_TID_TO_TASK"); + return NULL; + } + + scx = rhashtable_lookup(&scx_tid_hash, &tid, scx_tid_hash_params); + if (!scx) + return NULL; + + return container_of(scx, struct task_struct, scx); +} + /** * scx_bpf_now - Returns a high-performance monotonically non-decreasing * clock for the current CPU. The clock returned is in nanoseconds. @@ -9479,6 +9614,7 @@ BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_cpu_rq, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_locked_rq, KF_IMPLICIT_ARGS | KF_RET_NULL) BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) +BTF_ID_FLAGS(func, scx_bpf_tid_to_task, KF_RET_NULL | KF_RCU_PROTECTED) BTF_ID_FLAGS(func, scx_bpf_now) BTF_ID_FLAGS(func, scx_bpf_events) #ifdef CONFIG_CGROUP_SCHED diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext_internal.h index 62ce4eaf6a3f..4a7ffc7f55d2 100644 --- a/kernel/sched/ext_internal.h +++ b/kernel/sched/ext_internal.h @@ -13,6 +13,9 @@ enum scx_consts { SCX_DSP_MAX_LOOPS = 32, SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, + /* per-CPU chunk size for p->scx.tid allocation, see scx_alloc_tid() */ + SCX_TID_CHUNK = 1024, + SCX_EXIT_BT_LEN = 64, SCX_EXIT_MSG_LEN = 1024, SCX_EXIT_DUMP_DFL_LEN = 32768, @@ -138,7 +141,8 @@ enum scx_ops_flags { * To mask this problem, by default, unhashed tasks are automatically * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't * depend on pid lookups and wants to handle these tasks directly, the - * following flag can be used. + * following flag can be used. With %SCX_OPS_TID_TO_TASK, + * scx_bpf_tid_to_task() can find exiting tasks reliably. */ SCX_OPS_ENQ_EXITING = 1LLU << 2, @@ -189,6 +193,17 @@ enum scx_ops_flags { */ SCX_OPS_ALWAYS_ENQ_IMMED = 1LLU << 7, + /* + * Maintain a mapping from p->scx.tid to task_struct so the BPF + * scheduler can recover task pointers from stored tids via + * scx_bpf_tid_to_task(). + * + * Only the root scheduler turns this on. A sub-sched may set the flag + * to declare a dependency on the lookup; if the root scheduler hasn't + * enabled it, attaching the sub-sched is rejected. + */ + SCX_OPS_TID_TO_TASK = 1LLU << 8, + SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | SCX_OPS_ENQ_LAST | SCX_OPS_ENQ_EXITING | @@ -196,7 +211,8 @@ enum scx_ops_flags { SCX_OPS_ALLOW_QUEUED_WAKEUP | SCX_OPS_SWITCH_PARTIAL | SCX_OPS_BUILTIN_IDLE_PER_NODE | - SCX_OPS_ALWAYS_ENQ_IMMED, + SCX_OPS_ALWAYS_ENQ_IMMED | + SCX_OPS_TID_TO_TASK, /* high 8 bits are internal, don't include in SCX_OPS_ALL_FLAGS */ __SCX_OPS_INTERNAL_MASK = 0xffLLU << 56, -- cgit v1.2.3 From f7a6b9eaff3e6693ba3b19c5812e28538049bbf2 Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Fri, 17 Apr 2026 15:30:18 +0100 Subject: bpf: Extend BTF UAPI vlen, kinds to use unused bits BTF maximum vlen is encoded using 16 bits with a maximum vlen of 65535. This has sufficed for structs, function parameters and enumerated type values. However, with upcoming BTF location information - in particular information about inline sites - this limit is surpassed. Use bits 16-23 - currently unused in BTF info - to extend to 24 bits, giving a max vlen of (2^24 - 1), or 16 million. Also extend BTF kind encoding from 5 to 7 bits, giving a maximum available number of kinds of 128. Since with the BTF location work we use another 3 kinds, we are fast approaching the current limit of 32. Convert BTF_MAX_* values to enums to allow them to be encoded in kernel BTF; this will allow us to detect if the running kernel supports a 24-bit vlen or not. Add one for max _possible_ (not used) kind. Fix up a few places in the kernel where a 16-bit vlen is assumed; remove BTF_INFO_MASK as now all bits are used. The vlen expansion was suggested by Andrii in [1]; the kind expansion is tackled here too as it may be needed also to support new kinds in BTF. [1] https://lore.kernel.org/bpf/CAEf4BzZx=X6vGqcA8SPU6D+v6k+TR=ZewebXMuXtpmML058piw@mail.gmail.com/ Suggested-by: Andrii Nakryiko Signed-off-by: Alan Maguire Acked-by: Mykyta Yatsenko Link: https://lore.kernel.org/r/20260417143023.1551481-2-alan.maguire@oracle.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/btf.c | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 3c2aaa3c5004..77af44d8a3ad 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -182,7 +182,6 @@ #define BITS_ROUNDUP_BYTES(bits) \ (BITS_ROUNDDOWN_BYTES(bits) + !!BITS_PER_BYTE_MASKED(bits)) -#define BTF_INFO_MASK 0x9f00ffff #define BTF_INT_MASK 0x0fffffff #define BTF_TYPE_ID_VALID(type_id) ((type_id) <= BTF_MAX_TYPE) #define BTF_STR_OFFSET_VALID(name_off) ((name_off) <= BTF_MAX_NAME_OFFSET) @@ -289,7 +288,7 @@ enum verifier_phase { struct resolve_vertex { const struct btf_type *t; u32 type_id; - u16 next_member; + u32 next_member; }; enum visit_state { @@ -2031,7 +2030,7 @@ static int env_stack_push(struct btf_verifier_env *env, } static void env_stack_set_next_member(struct btf_verifier_env *env, - u16 next_member) + u32 next_member) { env->stack[env->top_stack - 1].next_member = next_member; } @@ -3293,7 +3292,7 @@ static s32 btf_struct_check_meta(struct btf_verifier_env *env, struct btf *btf = env->btf; u32 struct_size = t->size; u32 offset; - u16 i; + u32 i; meta_needed = btf_type_vlen(t) * sizeof(*member); if (meta_left < meta_needed) { @@ -3369,7 +3368,7 @@ static int btf_struct_resolve(struct btf_verifier_env *env, { const struct btf_member *member; int err; - u16 i; + u32 i; /* Before continue resolving the next_member, * ensure the last member is indeed resolved to a @@ -4447,7 +4446,7 @@ static s32 btf_enum_check_meta(struct btf_verifier_env *env, const struct btf_enum *enums = btf_type_enum(t); struct btf *btf = env->btf; const char *fmt_str; - u16 i, nr_enums; + u32 i, nr_enums; u32 meta_needed; nr_enums = btf_type_vlen(t); @@ -4555,7 +4554,7 @@ static s32 btf_enum64_check_meta(struct btf_verifier_env *env, const struct btf_enum64 *enums = btf_type_enum64(t); struct btf *btf = env->btf; const char *fmt_str; - u16 i, nr_enums; + u32 i, nr_enums; u32 meta_needed; nr_enums = btf_type_vlen(t); @@ -4683,7 +4682,7 @@ static void btf_func_proto_log(struct btf_verifier_env *env, const struct btf_type *t) { const struct btf_param *args = (const struct btf_param *)(t + 1); - u16 nr_args = btf_type_vlen(t), i; + u32 nr_args = btf_type_vlen(t), i; btf_verifier_log(env, "return=%u args=(", t->type); if (!nr_args) { @@ -4929,7 +4928,7 @@ static int btf_datasec_resolve(struct btf_verifier_env *env, { const struct btf_var_secinfo *vsi; struct btf *btf = env->btf; - u16 i; + u32 i; env->resolve_mode = RESOLVE_TBD; for_each_vsi_from(i, v->next_member, v->t, vsi) { @@ -5183,7 +5182,7 @@ static int btf_func_proto_check(struct btf_verifier_env *env, const struct btf_type *ret_type; const struct btf_param *args; const struct btf *btf; - u16 nr_args, i; + u32 nr_args, i; int err; btf = env->btf; @@ -5278,7 +5277,7 @@ static int btf_func_check(struct btf_verifier_env *env, const struct btf_type *proto_type; const struct btf_param *args; const struct btf *btf; - u16 nr_args, i; + u32 nr_args, i; btf = env->btf; proto_type = btf_type_by_id(btf, t->type); @@ -5336,12 +5335,6 @@ static s32 btf_check_meta(struct btf_verifier_env *env, } meta_left -= sizeof(*t); - if (t->info & ~BTF_INFO_MASK) { - btf_verifier_log(env, "[%u] Invalid btf_info:%x", - env->log_type_id, t->info); - return -EINVAL; - } - if (BTF_INFO_KIND(t->info) > BTF_KIND_MAX || BTF_INFO_KIND(t->info) == BTF_KIND_UNKN) { btf_verifier_log(env, "[%u] Invalid kind:%u", -- cgit v1.2.3 From 439ebd5b5708f236f7a4a9784194f7ecb77cd814 Mon Sep 17 00:00:00 2001 From: Mykyta Yatsenko Date: Wed, 22 Apr 2026 12:41:06 -0700 Subject: bpf: Add sleepable support for raw tracepoint programs Rework __bpf_trace_run() to support sleepable BPF programs by using explicit RCU flavor selection, following the uprobe_prog_run() pattern. For sleepable programs, use rcu_read_lock_tasks_trace() for lifetime protection with migrate_disable(). For non-sleepable programs, use the regular rcu_read_lock_dont_migrate(). Remove the preempt_disable_notrace/preempt_enable_notrace pair from the faultable tracepoint BPF probe wrapper in bpf_probe.h, since migration protection and RCU locking are now handled per-program inside __bpf_trace_run(). Adapt bpf_prog_test_run_raw_tp() for sleepable programs: reject BPF_F_TEST_RUN_ON_CPU since sleepable programs cannot run in hardirq or preempt-disabled context, and call __bpf_prog_test_run_raw_tp() directly instead of via smp_call_function_single(). Rework __bpf_prog_test_run_raw_tp() to select RCU flavor per-program and add per-program recursion context guard for private stack safety. Signed-off-by: Mykyta Yatsenko Acked-by: Kumar Kartikeya Dwivedi Link: https://lore.kernel.org/bpf/20260422-sleepable_tracepoints-v13-1-99005dff21ef@meta.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/trace/bpf_trace.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index e916f0ccbed9..7276c72c1d31 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -2072,11 +2072,19 @@ void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp) static __always_inline void __bpf_trace_run(struct bpf_raw_tp_link *link, u64 *args) { + struct srcu_ctr __percpu *scp = NULL; struct bpf_prog *prog = link->link.prog; + bool sleepable = prog->sleepable; struct bpf_run_ctx *old_run_ctx; struct bpf_trace_run_ctx run_ctx; - rcu_read_lock_dont_migrate(); + if (sleepable) { + scp = rcu_read_lock_tasks_trace(); + migrate_disable(); + } else { + rcu_read_lock_dont_migrate(); + } + if (unlikely(!bpf_prog_get_recursion_context(prog))) { bpf_prog_inc_misses_counter(prog); goto out; @@ -2085,12 +2093,18 @@ void __bpf_trace_run(struct bpf_raw_tp_link *link, u64 *args) run_ctx.bpf_cookie = link->cookie; old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx); - (void) bpf_prog_run(prog, args); + (void)bpf_prog_run(prog, args); bpf_reset_run_ctx(old_run_ctx); out: bpf_prog_put_recursion_context(prog); - rcu_read_unlock_migrate(); + + if (sleepable) { + migrate_enable(); + rcu_read_unlock_tasks_trace(scp); + } else { + rcu_read_unlock_migrate(); + } } #define UNPACK(...) __VA_ARGS__ -- cgit v1.2.3 From 57918341dd19e5ca8a77622ffae3db19e5ba4cc7 Mon Sep 17 00:00:00 2001 From: Mykyta Yatsenko Date: Wed, 22 Apr 2026 12:41:08 -0700 Subject: bpf: Add sleepable support for classic tracepoint programs Add trace_call_bpf_faultable(), a variant of trace_call_bpf() for faultable tracepoints that supports sleepable BPF programs. It uses rcu_tasks_trace for lifetime protection and bpf_prog_run_array_sleepable() for per-program RCU flavor selection, following the uprobe_prog_run() pattern. Restructure perf_syscall_enter() and perf_syscall_exit() to run BPF programs before perf event processing. Previously, BPF ran after the per-cpu perf trace buffer was allocated under preempt_disable, requiring cleanup via perf_swevent_put_recursion_context() on filter. Now BPF runs in faultable context before preempt_disable, reading syscall arguments from local variables instead of the per-cpu trace record, removing the dependency on buffer allocation. This allows sleepable BPF programs to execute and avoids unnecessary buffer allocation when BPF filters the event. The perf event submission path (buffer allocation, fill, submit) remains under preempt_disable as before. Since BPF no longer runs within the buffer allocation context, the fake_regs output parameter to perf_trace_buf_alloc() is no longer needed and is replaced with NULL. Add an attach-time check in __perf_event_set_bpf_prog() to reject sleepable BPF_PROG_TYPE_TRACEPOINT programs on non-syscall tracepoints, since only syscall tracepoints run in faultable context. This prepares the classic tracepoint runtime and attach paths for sleepable programs. The verifier changes to allow loading sleepable BPF_PROG_TYPE_TRACEPOINT programs are in a subsequent patch. To: Peter Zijlstra To: Steven Rostedt Signed-off-by: Mykyta Yatsenko Acked-by: Kumar Kartikeya Dwivedi # for BPF bits Acked-by: Steven Rostedt Link: https://lore.kernel.org/bpf/20260422-sleepable_tracepoints-v13-3-99005dff21ef@meta.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/events/core.c | 9 ++++ kernel/trace/bpf_trace.c | 28 +++++++++++ kernel/trace/trace_syscalls.c | 110 ++++++++++++++++++++++-------------------- 3 files changed, 95 insertions(+), 52 deletions(-) (limited to 'kernel') diff --git a/kernel/events/core.c b/kernel/events/core.c index 6d1f8bad7e1c..0f9cacfa7cb8 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -11643,6 +11643,15 @@ static int __perf_event_set_bpf_prog(struct perf_event *event, /* only uprobe programs are allowed to be sleepable */ return -EINVAL; + if (prog->type == BPF_PROG_TYPE_TRACEPOINT && prog->sleepable) { + /* + * Sleepable tracepoint programs can only attach to faultable + * tracepoints. Currently only syscall tracepoints are faultable. + */ + if (!is_syscall_tp) + return -EINVAL; + } + /* Kprobe override only works for kprobes, not uprobes. */ if (prog->kprobe_override && !is_kprobe) return -EINVAL; diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 7276c72c1d31..a822c589c9bd 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -152,6 +152,34 @@ unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx) return ret; } +/** + * trace_call_bpf_faultable - invoke BPF program in faultable context + * @call: tracepoint event + * @ctx: opaque context pointer + * + * Variant of trace_call_bpf() for faultable tracepoints (syscall + * tracepoints). Supports sleepable BPF programs by using rcu_tasks_trace + * for lifetime protection and bpf_prog_run_array_sleepable() for per-program + * RCU flavor selection, following the uprobe pattern. + * + * Per-program recursion protection is provided by + * bpf_prog_run_array_sleepable(). Global bpf_prog_active is not + * needed because syscall tracepoints cannot self-recurse. + * + * Must be called from a faultable/preemptible context. + */ +unsigned int trace_call_bpf_faultable(struct trace_event_call *call, void *ctx) +{ + struct bpf_prog_array *prog_array; + + might_fault(); + guard(rcu_tasks_trace)(); + + prog_array = rcu_dereference_check(call->prog_array, + rcu_read_lock_trace_held()); + return bpf_prog_run_array_sleepable(prog_array, ctx, bpf_prog_run); +} + #ifdef CONFIG_BPF_KPROBE_OVERRIDE BPF_CALL_2(bpf_override_return, struct pt_regs *, regs, unsigned long, rc) { diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c index 8ad72e17d8eb..e98ee7e1e66f 100644 --- a/kernel/trace/trace_syscalls.c +++ b/kernel/trace/trace_syscalls.c @@ -1371,33 +1371,33 @@ static DECLARE_BITMAP(enabled_perf_exit_syscalls, NR_syscalls); static int sys_perf_refcount_enter; static int sys_perf_refcount_exit; -static int perf_call_bpf_enter(struct trace_event_call *call, struct pt_regs *regs, +static int perf_call_bpf_enter(struct trace_event_call *call, struct syscall_metadata *sys_data, - struct syscall_trace_enter *rec) + int syscall_nr, unsigned long *args) { struct syscall_tp_t { struct trace_entry ent; int syscall_nr; unsigned long args[SYSCALL_DEFINE_MAXARGS]; } __aligned(8) param; + struct pt_regs regs = {}; int i; BUILD_BUG_ON(sizeof(param.ent) < sizeof(void *)); - /* bpf prog requires 'regs' to be the first member in the ctx (a.k.a. ¶m) */ - perf_fetch_caller_regs(regs); - *(struct pt_regs **)¶m = regs; - param.syscall_nr = rec->nr; + /* bpf prog requires 'regs' to be the first member in the ctx */ + perf_fetch_caller_regs(®s); + *(struct pt_regs **)¶m = ®s; + param.syscall_nr = syscall_nr; for (i = 0; i < sys_data->nb_args; i++) - param.args[i] = rec->args[i]; - return trace_call_bpf(call, ¶m); + param.args[i] = args[i]; + return trace_call_bpf_faultable(call, ¶m); } static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) { struct syscall_metadata *sys_data; struct syscall_trace_enter *rec; - struct pt_regs *fake_regs; struct hlist_head *head; unsigned long args[6]; bool valid_prog_array; @@ -1410,12 +1410,7 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) int size = 0; int uargs = 0; - /* - * Syscall probe called with preemption enabled, but the ring - * buffer and per-cpu data require preemption to be disabled. - */ might_fault(); - guard(preempt_notrace)(); syscall_nr = trace_get_syscall_nr(current, regs); if (syscall_nr < 0 || syscall_nr >= NR_syscalls) @@ -1429,6 +1424,26 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) syscall_get_arguments(current, regs, args); + /* + * Run BPF program in faultable context before per-cpu buffer + * allocation, allowing sleepable BPF programs to execute. + */ + valid_prog_array = bpf_prog_array_valid(sys_data->enter_event); + if (valid_prog_array && + !perf_call_bpf_enter(sys_data->enter_event, sys_data, + syscall_nr, args)) + return; + + /* + * Per-cpu ring buffer and perf event list operations require + * preemption to be disabled. + */ + guard(preempt_notrace)(); + + head = this_cpu_ptr(sys_data->enter_event->perf_events); + if (hlist_empty(head)) + return; + /* Check if this syscall event faults in user space memory */ mayfault = sys_data->user_mask != 0; @@ -1438,17 +1453,12 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) return; } - head = this_cpu_ptr(sys_data->enter_event->perf_events); - valid_prog_array = bpf_prog_array_valid(sys_data->enter_event); - if (!valid_prog_array && hlist_empty(head)) - return; - /* get the size after alignment with the u32 buffer size field */ size += sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec); size = ALIGN(size + sizeof(u32), sizeof(u64)); size -= sizeof(u32); - rec = perf_trace_buf_alloc(size, &fake_regs, &rctx); + rec = perf_trace_buf_alloc(size, NULL, &rctx); if (!rec) return; @@ -1458,13 +1468,6 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) if (mayfault) syscall_put_data(sys_data, rec, user_ptr, size, user_sizes, uargs); - if ((valid_prog_array && - !perf_call_bpf_enter(sys_data->enter_event, fake_regs, sys_data, rec)) || - hlist_empty(head)) { - perf_swevent_put_recursion_context(rctx); - return; - } - perf_trace_buf_submit(rec, size, rctx, sys_data->enter_event->event.type, 1, regs, head, NULL); @@ -1514,40 +1517,35 @@ static void perf_sysenter_disable(struct trace_event_call *call) syscall_fault_buffer_disable(); } -static int perf_call_bpf_exit(struct trace_event_call *call, struct pt_regs *regs, - struct syscall_trace_exit *rec) +static int perf_call_bpf_exit(struct trace_event_call *call, + int syscall_nr, long ret_val) { struct syscall_tp_t { struct trace_entry ent; int syscall_nr; unsigned long ret; } __aligned(8) param; - - /* bpf prog requires 'regs' to be the first member in the ctx (a.k.a. ¶m) */ - perf_fetch_caller_regs(regs); - *(struct pt_regs **)¶m = regs; - param.syscall_nr = rec->nr; - param.ret = rec->ret; - return trace_call_bpf(call, ¶m); + struct pt_regs regs = {}; + + /* bpf prog requires 'regs' to be the first member in the ctx */ + perf_fetch_caller_regs(®s); + *(struct pt_regs **)¶m = ®s; + param.syscall_nr = syscall_nr; + param.ret = ret_val; + return trace_call_bpf_faultable(call, ¶m); } static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret) { struct syscall_metadata *sys_data; struct syscall_trace_exit *rec; - struct pt_regs *fake_regs; struct hlist_head *head; bool valid_prog_array; int syscall_nr; int rctx; int size; - /* - * Syscall probe called with preemption enabled, but the ring - * buffer and per-cpu data require preemption to be disabled. - */ might_fault(); - guard(preempt_notrace)(); syscall_nr = trace_get_syscall_nr(current, regs); if (syscall_nr < 0 || syscall_nr >= NR_syscalls) @@ -1559,29 +1557,37 @@ static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret) if (!sys_data) return; - head = this_cpu_ptr(sys_data->exit_event->perf_events); + /* + * Run BPF program in faultable context before per-cpu buffer + * allocation, allowing sleepable BPF programs to execute. + */ valid_prog_array = bpf_prog_array_valid(sys_data->exit_event); - if (!valid_prog_array && hlist_empty(head)) + if (valid_prog_array && + !perf_call_bpf_exit(sys_data->exit_event, syscall_nr, + syscall_get_return_value(current, regs))) + return; + + /* + * Per-cpu ring buffer and perf event list operations require + * preemption to be disabled. + */ + guard(preempt_notrace)(); + + head = this_cpu_ptr(sys_data->exit_event->perf_events); + if (hlist_empty(head)) return; /* We can probably do that at build time */ size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64)); size -= sizeof(u32); - rec = perf_trace_buf_alloc(size, &fake_regs, &rctx); + rec = perf_trace_buf_alloc(size, NULL, &rctx); if (!rec) return; rec->nr = syscall_nr; rec->ret = syscall_get_return_value(current, regs); - if ((valid_prog_array && - !perf_call_bpf_exit(sys_data->exit_event, fake_regs, rec)) || - hlist_empty(head)) { - perf_swevent_put_recursion_context(rctx); - return; - } - perf_trace_buf_submit(rec, size, rctx, sys_data->exit_event->event.type, 1, regs, head, NULL); } -- cgit v1.2.3 From 8cfb77d3092052b52582e804e644202e2b10167a Mon Sep 17 00:00:00 2001 From: Mykyta Yatsenko Date: Wed, 22 Apr 2026 12:41:09 -0700 Subject: bpf: Verifier support for sleepable tracepoint programs Allow BPF_PROG_TYPE_RAW_TRACEPOINT, BPF_PROG_TYPE_TRACEPOINT, and BPF_TRACE_RAW_TP (tp_btf) programs to be sleepable by adding them to can_be_sleepable(). For BTF-based raw tracepoints (tp_btf), add a load-time check in bpf_check_attach_target() that rejects sleepable programs attaching to non-faultable tracepoints with a descriptive error message. For raw tracepoints (raw_tp), add an attach-time check in bpf_raw_tp_link_attach() that rejects sleepable programs on non-faultable tracepoints. The attach-time check is needed because the tracepoint name is not known at load time for raw_tp. The attach-time check for classic tracepoints (tp) in __perf_event_set_bpf_prog() was added in the previous patch. Replace the verbose error message that enumerates allowed program types with a generic "Program of this type cannot be sleepable" message, since the list of sleepable-capable types keeps growing. Signed-off-by: Mykyta Yatsenko Acked-by: Kumar Kartikeya Dwivedi Link: https://lore.kernel.org/bpf/20260422-sleepable_tracepoints-v13-4-99005dff21ef@meta.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/syscall.c | 5 +++++ kernel/bpf/verifier.c | 13 +++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index a3c0214ca934..3b1f0ba02f61 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -4281,6 +4281,11 @@ static int bpf_raw_tp_link_attach(struct bpf_prog *prog, if (!btp) return -ENOENT; + if (prog->sleepable && !tracepoint_is_faultable(btp->tp)) { + bpf_put_raw_tracepoint(btp); + return -EINVAL; + } + link = kzalloc_obj(*link, GFP_USER); if (!link) { err = -ENOMEM; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 185210b73385..5b4806fdb648 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -19267,6 +19267,12 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, btp = bpf_get_raw_tracepoint(tname); if (!btp) return -EINVAL; + if (prog->sleepable && !tracepoint_is_faultable(btp->tp)) { + bpf_log(log, "Sleepable program cannot attach to non-faultable tracepoint %s\n", + tname); + bpf_put_raw_tracepoint(btp); + return -EINVAL; + } fname = kallsyms_lookup((unsigned long)btp->bpf_func, NULL, NULL, NULL, trace_symbol); bpf_put_raw_tracepoint(btp); @@ -19483,6 +19489,7 @@ static bool can_be_sleepable(struct bpf_prog *prog) case BPF_MODIFY_RETURN: case BPF_TRACE_ITER: case BPF_TRACE_FSESSION: + case BPF_TRACE_RAW_TP: return true; default: return false; @@ -19490,7 +19497,9 @@ static bool can_be_sleepable(struct bpf_prog *prog) } return prog->type == BPF_PROG_TYPE_LSM || prog->type == BPF_PROG_TYPE_KPROBE /* only for uprobes */ || - prog->type == BPF_PROG_TYPE_STRUCT_OPS; + prog->type == BPF_PROG_TYPE_STRUCT_OPS || + prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT || + prog->type == BPF_PROG_TYPE_TRACEPOINT; } static int check_attach_btf_id(struct bpf_verifier_env *env) @@ -19512,7 +19521,7 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) } if (prog->sleepable && !can_be_sleepable(prog)) { - verbose(env, "Only fentry/fexit/fsession/fmod_ret, lsm, iter, uprobe, and struct_ops programs can be sleepable\n"); + verbose(env, "Program of this type cannot be sleepable\n"); return -EINVAL; } -- cgit v1.2.3 From a7088176d8299ff74276a89dbdef3c5ce8748eeb Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Wed, 22 Apr 2026 20:34:30 -0700 Subject: bpf: Remove unused parameter from check_map_kptr_access() The parameter 'regno' in check_map_kptr_access() is unused. Remove it. Acked-by: Puranjay Mohan Acked-by: Kumar Kartikeya Dwivedi Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260423033430.2537615-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5b4806fdb648..6118743d87e6 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -4710,7 +4710,7 @@ static int mark_uptr_ld_reg(struct bpf_verifier_env *env, u32 regno, return 0; } -static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno, +static int check_map_kptr_access(struct bpf_verifier_env *env, int value_regno, int insn_idx, struct btf_field *kptr_field) { @@ -6357,7 +6357,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn kptr_field = btf_record_find(reg->map_ptr->record, off + reg->var_off.value, BPF_KPTR | BPF_UPTR); if (kptr_field) { - err = check_map_kptr_access(env, regno, value_regno, insn_idx, kptr_field); + err = check_map_kptr_access(env, value_regno, insn_idx, kptr_field); } else if (t == BPF_READ && value_regno >= 0) { struct bpf_map *map = reg->map_ptr; -- cgit v1.2.3 From 54c27ea6dadbe932a955bf40b7837947c5c202e1 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Wed, 22 Apr 2026 20:34:35 -0700 Subject: bpf: Fix tail_call_reachable leak In check_max_stack_depth_subprog(), the local variable tail_call_reachable is set when entering a callee that has a tail call, but never reset when popping back to the parent. This causes the flag to leak across sibling subprogs in the DFS traversal. This results in unnecessary JIT overhead: the JIT emits tail call counter preservation code for subprogs that can never be reached via a tail call path. Fix this by resetting tail_call_reachable to the parent's actual per-subprog flag when popping a frame. If the parent was already marked tail_call_reachable by a previous sibling's traversal, the local variable stays true. Otherwise it resets to false, so subsequent siblings start with a clean state. Acked-by: Kumar Kartikeya Dwivedi Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260423033435.2538013-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 6118743d87e6..26b6cdfd8613 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5490,6 +5490,9 @@ continue_func: frame = dinfo[idx].frame; i = dinfo[idx].ret_insn; + /* reset tail_call_reachable to the parent's actual state */ + tail_call_reachable = subprog[idx].tail_call_reachable; + goto continue_func; } -- cgit v1.2.3 From 8a16c5b2b22ff22c1a2e3ff92cc991990efceb38 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Wed, 22 Apr 2026 20:34:41 -0700 Subject: bpf: Remove WARN_ON_ONCE in check_kfunc_mem_size_reg() The warning is too late if it does happen. Remove it. Acked-by: Kumar Kartikeya Dwivedi Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260423033441.2538149-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 26b6cdfd8613..d123449f5552 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -7134,8 +7134,6 @@ static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg struct bpf_call_arg_meta meta; int err; - WARN_ON_ONCE(regno < BPF_REG_2 || regno > BPF_REG_5); - memset(&meta, 0, sizeof(meta)); if (may_be_null) { -- cgit v1.2.3 From 6a581b856c9e633c615483ad53910cba8cacbf28 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Wed, 22 Apr 2026 20:34:46 -0700 Subject: bpf: Refactor to avoid redundant calculation of bpf_reg_state In many cases, once a bpf_reg_state is defined, it can pass to callee's. Otherwise, callee will need to get bpf_reg_state again based on regno. More importantly, this is needed for later stack arguments for kfuncs since the register state for stack arguments does not have a corresponding regno. So it makes sense to pass reg state for callee's. The following is the only change to avoid compilation warning: static int sanitize_check_bounds(struct bpf_verifier_env *env, const struct bpf_insn *insn, - const struct bpf_reg_state *dst_reg) + struct bpf_reg_state *dst_reg) Acked-by: Puranjay Mohan Acked-by: Kumar Kartikeya Dwivedi Reviewed-by: Amery Hung Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260423033446.2538321-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 223 ++++++++++++++++++++++---------------------------- 1 file changed, 98 insertions(+), 125 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d123449f5552..debae6133e4c 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3908,7 +3908,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, return 0; } -/* Write the stack: 'stack[ptr_regno + off] = value_regno'. 'ptr_regno' is +/* Write the stack: 'stack[ptr_reg + off] = value_regno'. 'ptr_reg' is * known to contain a variable offset. * This function checks whether the write is permitted and conservatively * tracks the effects of the write, considering that each stack slot in the @@ -3929,13 +3929,13 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, static int check_stack_write_var_off(struct bpf_verifier_env *env, /* func where register points to */ struct bpf_func_state *state, - int ptr_regno, int off, int size, + struct bpf_reg_state *ptr_reg, int off, int size, int value_regno, int insn_idx) { struct bpf_func_state *cur; /* state of the current function */ int min_off, max_off; int i, err; - struct bpf_reg_state *ptr_reg = NULL, *value_reg = NULL; + struct bpf_reg_state *value_reg = NULL; struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; bool writing_zero = false; /* set if the fact that we're writing a zero is used to let any @@ -3944,7 +3944,6 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, bool zero_used = false; cur = env->cur_state->frame[env->cur_state->curframe]; - ptr_reg = &cur->regs[ptr_regno]; min_off = ptr_reg->smin_value + off; max_off = ptr_reg->smax_value + off + size; if (value_regno >= 0) @@ -4241,7 +4240,7 @@ enum bpf_access_src { ACCESS_HELPER = 2, /* the access is performed by a helper */ }; -static int check_stack_range_initialized(struct bpf_verifier_env *env, +static int check_stack_range_initialized(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int regno, int off, int access_size, bool zero_size_allowed, enum bpf_access_type type, @@ -4252,31 +4251,29 @@ static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) return cur_regs(env) + regno; } -/* Read the stack at 'ptr_regno + off' and put the result into the register +/* Read the stack at 'reg + off' and put the result into the register * 'dst_regno'. - * 'off' includes the pointer register's fixed offset(i.e. 'ptr_regno.off'), + * 'off' includes the pointer register's fixed offset(i.e. 'reg->off'), * but not its variable offset. * 'size' is assumed to be <= reg size and the access is assumed to be aligned. * * As opposed to check_stack_read_fixed_off, this function doesn't deal with * filling registers (i.e. reads of spilled register cannot be detected when * the offset is not fixed). We conservatively mark 'dst_regno' as containing - * SCALAR_VALUE. That's why we assert that the 'ptr_regno' has a variable + * SCALAR_VALUE. That's why we assert that the 'reg' has a variable * offset; for a fixed offset check_stack_read_fixed_off should be used * instead. */ -static int check_stack_read_var_off(struct bpf_verifier_env *env, +static int check_stack_read_var_off(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int ptr_regno, int off, int size, int dst_regno) { - /* The state of the source register. */ - struct bpf_reg_state *reg = reg_state(env, ptr_regno); struct bpf_func_state *ptr_state = bpf_func(env, reg); int err; int min_off, max_off; /* Note that we pass a NULL meta, so raw access will not be permitted. */ - err = check_stack_range_initialized(env, ptr_regno, off, size, + err = check_stack_range_initialized(env, reg, ptr_regno, off, size, false, BPF_READ, NULL); if (err) return err; @@ -4298,10 +4295,9 @@ static int check_stack_read_var_off(struct bpf_verifier_env *env, * can be -1, meaning that the read value is not going to a register. */ static int check_stack_read(struct bpf_verifier_env *env, - int ptr_regno, int off, int size, + struct bpf_reg_state *reg, int ptr_regno, int off, int size, int dst_regno) { - struct bpf_reg_state *reg = reg_state(env, ptr_regno); struct bpf_func_state *state = bpf_func(env, reg); int err; /* Some accesses are only permitted with a static offset. */ @@ -4337,7 +4333,7 @@ static int check_stack_read(struct bpf_verifier_env *env, * than fixed offset ones. Note that dst_regno >= 0 on this * branch. */ - err = check_stack_read_var_off(env, ptr_regno, off, size, + err = check_stack_read_var_off(env, reg, ptr_regno, off, size, dst_regno); } return err; @@ -4347,17 +4343,16 @@ static int check_stack_read(struct bpf_verifier_env *env, /* check_stack_write dispatches to check_stack_write_fixed_off or * check_stack_write_var_off. * - * 'ptr_regno' is the register used as a pointer into the stack. + * 'reg' is the register used as a pointer into the stack. * 'value_regno' is the register whose value we're writing to the stack. It can * be -1, meaning that we're not writing from a register. * * The caller must ensure that the offset falls within the maximum stack size. */ static int check_stack_write(struct bpf_verifier_env *env, - int ptr_regno, int off, int size, + struct bpf_reg_state *reg, int off, int size, int value_regno, int insn_idx) { - struct bpf_reg_state *reg = reg_state(env, ptr_regno); struct bpf_func_state *state = bpf_func(env, reg); int err; @@ -4370,16 +4365,15 @@ static int check_stack_write(struct bpf_verifier_env *env, * than fixed offset ones. */ err = check_stack_write_var_off(env, state, - ptr_regno, off, size, + reg, off, size, value_regno, insn_idx); } return err; } -static int check_map_access_type(struct bpf_verifier_env *env, u32 regno, +static int check_map_access_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int off, int size, enum bpf_access_type type) { - struct bpf_reg_state *reg = reg_state(env, regno); struct bpf_map *map = reg->map_ptr; u32 cap = bpf_map_flags_to_cap(map); @@ -4399,17 +4393,15 @@ static int check_map_access_type(struct bpf_verifier_env *env, u32 regno, } /* check read/write into memory region (e.g., map value, ringbuf sample, etc) */ -static int __check_mem_access(struct bpf_verifier_env *env, int regno, +static int __check_mem_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int regno, int off, int size, u32 mem_size, bool zero_size_allowed) { bool size_ok = size > 0 || (size == 0 && zero_size_allowed); - struct bpf_reg_state *reg; if (off >= 0 && size_ok && (u64)off + size <= mem_size) return 0; - reg = &cur_regs(env)[regno]; switch (reg->type) { case PTR_TO_MAP_KEY: verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n", @@ -4439,13 +4431,10 @@ static int __check_mem_access(struct bpf_verifier_env *env, int regno, } /* check read/write into a memory region with possible variable offset */ -static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno, +static int check_mem_region_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, u32 regno, int off, int size, u32 mem_size, bool zero_size_allowed) { - struct bpf_verifier_state *vstate = env->cur_state; - struct bpf_func_state *state = vstate->frame[vstate->curframe]; - struct bpf_reg_state *reg = &state->regs[regno]; int err; /* We may have adjusted the register pointing to memory region, so we @@ -4466,7 +4455,7 @@ static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno, regno); return -EACCES; } - err = __check_mem_access(env, regno, reg->smin_value + off, size, + err = __check_mem_access(env, reg, regno, reg->smin_value + off, size, mem_size, zero_size_allowed); if (err) { verbose(env, "R%d min value is outside of the allowed memory range\n", @@ -4483,7 +4472,7 @@ static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno, regno); return -EACCES; } - err = __check_mem_access(env, regno, reg->umax_value + off, size, + err = __check_mem_access(env, reg, regno, reg->umax_value + off, size, mem_size, zero_size_allowed); if (err) { verbose(env, "R%d max value is outside of the allowed memory range\n", @@ -4787,19 +4776,16 @@ static u32 map_mem_size(const struct bpf_map *map) } /* check read/write into a map element with possible variable offset */ -static int check_map_access(struct bpf_verifier_env *env, u32 regno, +static int check_map_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, u32 regno, int off, int size, bool zero_size_allowed, enum bpf_access_src src) { - struct bpf_verifier_state *vstate = env->cur_state; - struct bpf_func_state *state = vstate->frame[vstate->curframe]; - struct bpf_reg_state *reg = &state->regs[regno]; struct bpf_map *map = reg->map_ptr; u32 mem_size = map_mem_size(map); struct btf_record *rec; int err, i; - err = check_mem_region_access(env, regno, off, size, mem_size, zero_size_allowed); + err = check_mem_region_access(env, reg, regno, off, size, mem_size, zero_size_allowed); if (err) return err; @@ -4895,10 +4881,9 @@ static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, } } -static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, +static int check_packet_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, u32 regno, int off, int size, bool zero_size_allowed) { - struct bpf_reg_state *reg = reg_state(env, regno); int err; if (reg->range < 0) { @@ -4906,7 +4891,7 @@ static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, return -EINVAL; } - err = check_mem_region_access(env, regno, off, size, reg->range, zero_size_allowed); + err = check_mem_region_access(env, reg, regno, off, size, reg->range, zero_size_allowed); if (err) return err; @@ -4961,7 +4946,7 @@ static int __check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int of return -EACCES; } -static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, +static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, struct bpf_reg_state *reg, u32 regno, int off, int access_size, enum bpf_access_type t, struct bpf_insn_access_aux *info) { @@ -4971,12 +4956,10 @@ static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, u32 regn */ bool var_off_ok = is_var_ctx_off_allowed(env->prog); bool fixed_off_ok = !env->ops->convert_ctx_access; - struct bpf_reg_state *regs = cur_regs(env); - struct bpf_reg_state *reg = regs + regno; int err; if (var_off_ok) - err = check_mem_region_access(env, regno, off, access_size, U16_MAX, false); + err = check_mem_region_access(env, reg, regno, off, access_size, U16_MAX, false); else err = __check_ptr_off_reg(env, reg, regno, fixed_off_ok); if (err) @@ -5002,10 +4985,9 @@ static int check_flow_keys_access(struct bpf_verifier_env *env, int off, } static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, - u32 regno, int off, int size, + struct bpf_reg_state *reg, u32 regno, int off, int size, enum bpf_access_type t) { - struct bpf_reg_state *reg = reg_state(env, regno); struct bpf_insn_access_aux info = {}; bool valid; @@ -5971,12 +5953,11 @@ static bool type_is_trusted_or_null(struct bpf_verifier_env *env, } static int check_ptr_to_btf_access(struct bpf_verifier_env *env, - struct bpf_reg_state *regs, + struct bpf_reg_state *regs, struct bpf_reg_state *reg, int regno, int off, int size, enum bpf_access_type atype, int value_regno) { - struct bpf_reg_state *reg = regs + regno; const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id); const char *tname = btf_name_by_offset(reg->btf, t->name_off); const char *field_name = NULL; @@ -6128,12 +6109,11 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, } static int check_ptr_to_map_access(struct bpf_verifier_env *env, - struct bpf_reg_state *regs, + struct bpf_reg_state *regs, struct bpf_reg_state *reg, int regno, int off, int size, enum bpf_access_type atype, int value_regno) { - struct bpf_reg_state *reg = regs + regno; struct bpf_map *map = reg->map_ptr; struct bpf_reg_state map_reg; enum bpf_type_flag flag = 0; @@ -6222,11 +6202,10 @@ static int check_stack_slot_within_bounds(struct bpf_verifier_env *env, * 'off' includes `regno->offset`, but not its dynamic part (if any). */ static int check_stack_access_within_bounds( - struct bpf_verifier_env *env, + struct bpf_verifier_env *env, struct bpf_reg_state *reg, int regno, int off, int access_size, enum bpf_access_type type) { - struct bpf_reg_state *reg = reg_state(env, regno); struct bpf_func_state *state = bpf_func(env, reg); s64 min_off, max_off; int err; @@ -6314,12 +6293,11 @@ static void add_scalar_to_reg(struct bpf_reg_state *dst_reg, s64 val) * if t==write && value_regno==-1, some unknown value is stored into memory * if t==read && value_regno==-1, don't care what we read from memory */ -static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, +static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct bpf_reg_state *reg, u32 regno, int off, int bpf_size, enum bpf_access_type t, int value_regno, bool strict_alignment_once, bool is_ldsx) { struct bpf_reg_state *regs = cur_regs(env); - struct bpf_reg_state *reg = regs + regno; int size, err = 0; size = bpf_size_to_bytes(bpf_size); @@ -6336,7 +6314,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn return -EACCES; } - err = check_mem_region_access(env, regno, off, size, + err = check_mem_region_access(env, reg, regno, off, size, reg->map_ptr->key_size, false); if (err) return err; @@ -6350,10 +6328,10 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn verbose(env, "R%d leaks addr into map\n", value_regno); return -EACCES; } - err = check_map_access_type(env, regno, off, size, t); + err = check_map_access_type(env, reg, off, size, t); if (err) return err; - err = check_map_access(env, regno, off, size, false, ACCESS_DIRECT); + err = check_map_access(env, reg, regno, off, size, false, ACCESS_DIRECT); if (err) return err; if (tnum_is_const(reg->var_off)) @@ -6422,7 +6400,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn * instructions, hence no need to check bounds in that case. */ if (!rdonly_untrusted) - err = check_mem_region_access(env, regno, off, size, + err = check_mem_region_access(env, reg, regno, off, size, reg->mem_size, false); if (!err && value_regno >= 0 && (t == BPF_READ || rdonly_mem)) mark_reg_unknown(env, regs, value_regno); @@ -6440,7 +6418,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn return -EACCES; } - err = check_ctx_access(env, insn_idx, regno, off, size, t, &info); + err = check_ctx_access(env, insn_idx, reg, regno, off, size, t, &info); if (!err && t == BPF_READ && value_regno >= 0) { /* ctx access returns either a scalar, or a * PTR_TO_PACKET[_META,_END]. In the latter @@ -6477,15 +6455,15 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn } else if (reg->type == PTR_TO_STACK) { /* Basic bounds checks. */ - err = check_stack_access_within_bounds(env, regno, off, size, t); + err = check_stack_access_within_bounds(env, reg, regno, off, size, t); if (err) return err; if (t == BPF_READ) - err = check_stack_read(env, regno, off, size, + err = check_stack_read(env, reg, regno, off, size, value_regno); else - err = check_stack_write(env, regno, off, size, + err = check_stack_write(env, reg, off, size, value_regno, insn_idx); } else if (reg_is_pkt_pointer(reg)) { if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) { @@ -6498,7 +6476,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn value_regno); return -EACCES; } - err = check_packet_access(env, regno, off, size, false); + err = check_packet_access(env, reg, regno, off, size, false); if (!err && t == BPF_READ && value_regno >= 0) mark_reg_unknown(env, regs, value_regno); } else if (reg->type == PTR_TO_FLOW_KEYS) { @@ -6518,7 +6496,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn regno, reg_type_str(env, reg->type)); return -EACCES; } - err = check_sock_access(env, insn_idx, regno, off, size, t); + err = check_sock_access(env, insn_idx, reg, regno, off, size, t); if (!err && value_regno >= 0) mark_reg_unknown(env, regs, value_regno); } else if (reg->type == PTR_TO_TP_BUFFER) { @@ -6527,10 +6505,10 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn mark_reg_unknown(env, regs, value_regno); } else if (base_type(reg->type) == PTR_TO_BTF_ID && !type_may_be_null(reg->type)) { - err = check_ptr_to_btf_access(env, regs, regno, off, size, t, + err = check_ptr_to_btf_access(env, regs, reg, regno, off, size, t, value_regno); } else if (reg->type == CONST_PTR_TO_MAP) { - err = check_ptr_to_map_access(env, regs, regno, off, size, t, + err = check_ptr_to_map_access(env, regs, reg, regno, off, size, t, value_regno); } else if (base_type(reg->type) == PTR_TO_BUF && !type_may_be_null(reg->type)) { @@ -6599,7 +6577,7 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, /* Check if (src_reg + off) is readable. The state of dst_reg will be * updated by this call. */ - err = check_mem_access(env, env->insn_idx, insn->src_reg, insn->off, + err = check_mem_access(env, env->insn_idx, regs + insn->src_reg, insn->src_reg, insn->off, BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, strict_alignment_once, is_ldsx); err = err ?: save_aux_ptr_type(env, src_reg_type, @@ -6629,7 +6607,7 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, dst_reg_type = regs[insn->dst_reg].type; /* Check if (dst_reg + off) is writeable. */ - err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off, + err = check_mem_access(env, env->insn_idx, regs + insn->dst_reg, insn->dst_reg, insn->off, BPF_SIZE(insn->code), BPF_WRITE, insn->src_reg, strict_alignment_once, false); err = err ?: save_aux_ptr_type(env, dst_reg_type, false); @@ -6640,6 +6618,7 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, static int check_atomic_rmw(struct bpf_verifier_env *env, struct bpf_insn *insn) { + struct bpf_reg_state *dst_reg; int load_reg; int err; @@ -6701,13 +6680,15 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, load_reg = -1; } + dst_reg = cur_regs(env) + insn->dst_reg; + /* Check whether we can read the memory, with second call for fetch * case to simulate the register fill. */ - err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off, + err = check_mem_access(env, env->insn_idx, dst_reg, insn->dst_reg, insn->off, BPF_SIZE(insn->code), BPF_READ, -1, true, false); if (!err && load_reg >= 0) - err = check_mem_access(env, env->insn_idx, insn->dst_reg, + err = check_mem_access(env, env->insn_idx, dst_reg, insn->dst_reg, insn->off, BPF_SIZE(insn->code), BPF_READ, load_reg, true, false); if (err) @@ -6719,7 +6700,7 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, return err; } /* Check whether we can write into the same memory. */ - err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off, + err = check_mem_access(env, env->insn_idx, dst_reg, insn->dst_reg, insn->off, BPF_SIZE(insn->code), BPF_WRITE, -1, true, false); if (err) return err; @@ -6808,11 +6789,10 @@ static int check_atomic(struct bpf_verifier_env *env, struct bpf_insn *insn) * read offsets are marked as read. */ static int check_stack_range_initialized( - struct bpf_verifier_env *env, int regno, int off, + struct bpf_verifier_env *env, struct bpf_reg_state *reg, int regno, int off, int access_size, bool zero_size_allowed, enum bpf_access_type type, struct bpf_call_arg_meta *meta) { - struct bpf_reg_state *reg = reg_state(env, regno); struct bpf_func_state *state = bpf_func(env, reg); int err, min_off, max_off, i, j, slot, spi; /* Some accesses can write anything into the stack, others are @@ -6834,7 +6814,7 @@ static int check_stack_range_initialized( return -EACCES; } - err = check_stack_access_within_bounds(env, regno, off, access_size, type); + err = check_stack_access_within_bounds(env, reg, regno, off, access_size, type); if (err) return err; @@ -6965,7 +6945,7 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, switch (base_type(reg->type)) { case PTR_TO_PACKET: case PTR_TO_PACKET_META: - return check_packet_access(env, regno, 0, access_size, + return check_packet_access(env, reg, regno, 0, access_size, zero_size_allowed); case PTR_TO_MAP_KEY: if (access_type == BPF_WRITE) { @@ -6973,12 +6953,12 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, reg_type_str(env, reg->type)); return -EACCES; } - return check_mem_region_access(env, regno, 0, access_size, + return check_mem_region_access(env, reg, regno, 0, access_size, reg->map_ptr->key_size, false); case PTR_TO_MAP_VALUE: - if (check_map_access_type(env, regno, 0, access_size, access_type)) + if (check_map_access_type(env, reg, 0, access_size, access_type)) return -EACCES; - return check_map_access(env, regno, 0, access_size, + return check_map_access(env, reg, regno, 0, access_size, zero_size_allowed, ACCESS_HELPER); case PTR_TO_MEM: if (type_is_rdonly_mem(reg->type)) { @@ -6988,7 +6968,7 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, return -EACCES; } } - return check_mem_region_access(env, regno, 0, + return check_mem_region_access(env, reg, regno, 0, access_size, reg->mem_size, zero_size_allowed); case PTR_TO_BUF: @@ -7008,16 +6988,16 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, max_access); case PTR_TO_STACK: return check_stack_range_initialized( - env, + env, reg, regno, 0, access_size, zero_size_allowed, access_type, meta); case PTR_TO_BTF_ID: - return check_ptr_to_btf_access(env, regs, regno, 0, + return check_ptr_to_btf_access(env, regs, reg, regno, 0, access_size, BPF_READ, -1); case PTR_TO_CTX: /* Only permit reading or writing syscall context using helper calls. */ if (is_var_ctx_off_allowed(env->prog)) { - int err = check_mem_region_access(env, regno, 0, access_size, U16_MAX, + int err = check_mem_region_access(env, reg, regno, 0, access_size, U16_MAX, zero_size_allowed); if (err) return err; @@ -7178,11 +7158,10 @@ enum { * env->cur_state->active_locks remembers which map value element or allocated * object got locked and clears it after bpf_spin_unlock. */ -static int process_spin_lock(struct bpf_verifier_env *env, int regno, int flags) +static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int regno, int flags) { bool is_lock = flags & PROCESS_SPIN_LOCK, is_res_lock = flags & PROCESS_RES_LOCK; const char *lock_str = is_res_lock ? "bpf_res_spin" : "bpf_spin"; - struct bpf_reg_state *reg = reg_state(env, regno); struct bpf_verifier_state *cur = env->cur_state; bool is_const = tnum_is_const(reg->var_off); bool is_irq = flags & PROCESS_LOCK_IRQ; @@ -7295,11 +7274,10 @@ static int process_spin_lock(struct bpf_verifier_env *env, int regno, int flags) } /* Check if @regno is a pointer to a specific field in a map value */ -static int check_map_field_pointer(struct bpf_verifier_env *env, u32 regno, +static int check_map_field_pointer(struct bpf_verifier_env *env, struct bpf_reg_state *reg, u32 regno, enum btf_field_type field_type, struct bpf_map_desc *map_desc) { - struct bpf_reg_state *reg = reg_state(env, regno); bool is_const = tnum_is_const(reg->var_off); struct bpf_map *map = reg->map_ptr; u64 val = reg->var_off.value; @@ -7349,26 +7327,26 @@ static int check_map_field_pointer(struct bpf_verifier_env *env, u32 regno, return 0; } -static int process_timer_func(struct bpf_verifier_env *env, int regno, +static int process_timer_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int regno, struct bpf_map_desc *map) { if (IS_ENABLED(CONFIG_PREEMPT_RT)) { verbose(env, "bpf_timer cannot be used for PREEMPT_RT.\n"); return -EOPNOTSUPP; } - return check_map_field_pointer(env, regno, BPF_TIMER, map); + return check_map_field_pointer(env, reg, regno, BPF_TIMER, map); } -static int process_timer_helper(struct bpf_verifier_env *env, int regno, +static int process_timer_helper(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int regno, struct bpf_call_arg_meta *meta) { - return process_timer_func(env, regno, &meta->map); + return process_timer_func(env, reg, regno, &meta->map); } -static int process_timer_kfunc(struct bpf_verifier_env *env, int regno, +static int process_timer_kfunc(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int regno, struct bpf_kfunc_call_arg_meta *meta) { - return process_timer_func(env, regno, &meta->map); + return process_timer_func(env, reg, regno, &meta->map); } static int process_kptr_func(struct bpf_verifier_env *env, int regno, @@ -7433,10 +7411,9 @@ static int process_kptr_func(struct bpf_verifier_env *env, int regno, * use case. The second level is tracked using the upper bit of bpf_dynptr->size * and checked dynamically during runtime. */ -static int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn_idx, +static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int regno, int insn_idx, enum bpf_arg_type arg_type, int clone_ref_obj_id) { - struct bpf_reg_state *reg = reg_state(env, regno); int err; if (reg->type != PTR_TO_STACK && reg->type != CONST_PTR_TO_DYNPTR) { @@ -7470,7 +7447,7 @@ static int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn /* we write BPF_DW bits (8 bytes) at a time */ for (i = 0; i < BPF_DYNPTR_SIZE; i += 8) { - err = check_mem_access(env, insn_idx, regno, + err = check_mem_access(env, insn_idx, reg, regno, i, BPF_DW, BPF_WRITE, -1, false, false); if (err) return err; @@ -7540,10 +7517,9 @@ static bool is_kfunc_arg_iter(struct bpf_kfunc_call_arg_meta *meta, int arg_idx, return btf_param_match_suffix(meta->btf, arg, "__iter"); } -static int process_iter_arg(struct bpf_verifier_env *env, int regno, int insn_idx, +static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int regno, int insn_idx, struct bpf_kfunc_call_arg_meta *meta) { - struct bpf_reg_state *reg = reg_state(env, regno); const struct btf_type *t; int spi, err, i, nr_slots, btf_id; @@ -7575,7 +7551,7 @@ static int process_iter_arg(struct bpf_verifier_env *env, int regno, int insn_id } for (i = 0; i < nr_slots * 8; i += BPF_REG_SIZE) { - err = check_mem_access(env, insn_idx, regno, + err = check_mem_access(env, insn_idx, reg, regno, i, BPF_DW, BPF_WRITE, -1, false, false); if (err) return err; @@ -8014,12 +7990,11 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { [ARG_PTR_TO_DYNPTR] = &dynptr_types, }; -static int check_reg_type(struct bpf_verifier_env *env, u32 regno, +static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg, u32 regno, enum bpf_arg_type arg_type, const u32 *arg_btf_id, struct bpf_call_arg_meta *meta) { - struct bpf_reg_state *reg = reg_state(env, regno); enum bpf_reg_type expected, type = reg->type; const struct bpf_reg_types *compatible; int i, j, err; @@ -8362,7 +8337,7 @@ static int check_reg_const_str(struct bpf_verifier_env *env, return -EACCES; } - err = check_map_access(env, regno, 0, + err = check_map_access(env, reg, regno, 0, map->value_size - reg->var_off.value, false, ACCESS_HELPER); if (err) @@ -8498,7 +8473,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK) arg_btf_id = fn->arg_btf_id[arg]; - err = check_reg_type(env, regno, arg_type, arg_btf_id, meta); + err = check_reg_type(env, reg, regno, arg_type, arg_btf_id, meta); if (err) return err; @@ -8636,11 +8611,11 @@ skip_type_check: return -EACCES; } if (meta->func_id == BPF_FUNC_spin_lock) { - err = process_spin_lock(env, regno, PROCESS_SPIN_LOCK); + err = process_spin_lock(env, reg, regno, PROCESS_SPIN_LOCK); if (err) return err; } else if (meta->func_id == BPF_FUNC_spin_unlock) { - err = process_spin_lock(env, regno, 0); + err = process_spin_lock(env, reg, regno, 0); if (err) return err; } else { @@ -8649,7 +8624,7 @@ skip_type_check: } break; case ARG_PTR_TO_TIMER: - err = process_timer_helper(env, regno, meta); + err = process_timer_helper(env, reg, regno, meta); if (err) return err; break; @@ -8684,7 +8659,7 @@ skip_type_check: true, meta); break; case ARG_PTR_TO_DYNPTR: - err = process_dynptr_func(env, regno, insn_idx, arg_type, 0); + err = process_dynptr_func(env, reg, regno, insn_idx, arg_type, 0); if (err) return err; break; @@ -9343,7 +9318,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, if (ret) return ret; - ret = process_dynptr_func(env, regno, -1, arg->arg_type, 0); + ret = process_dynptr_func(env, reg, regno, -1, arg->arg_type, 0); if (ret) return ret; } else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) { @@ -9354,7 +9329,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, continue; memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */ - err = check_reg_type(env, regno, arg->arg_type, &arg->btf_id, &meta); + err = check_reg_type(env, reg, regno, arg->arg_type, &arg->btf_id, &meta); err = err ?: check_func_arg_reg_off(env, reg, regno, arg->arg_type); if (err) return err; @@ -10312,18 +10287,18 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (err) return err; + regs = cur_regs(env); + /* Mark slots with STACK_MISC in case of raw mode, stack offset * is inferred from register state. */ for (i = 0; i < meta.access_size; i++) { - err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B, + err = check_mem_access(env, insn_idx, regs + meta.regno, meta.regno, i, BPF_B, BPF_WRITE, -1, false, false); if (err) return err; } - regs = cur_regs(env); - if (meta.release_regno) { err = -EINVAL; if (arg_type_is_dynptr(fn->arg_type[meta.release_regno - BPF_REG_1])) { @@ -11327,11 +11302,10 @@ get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta, const struct btf_type *t, const struct btf_type *ref_t, const char *ref_tname, const struct btf_param *args, - int argno, int nargs) + int argno, int nargs, struct bpf_reg_state *reg) { u32 regno = argno + 1; struct bpf_reg_state *regs = cur_regs(env); - struct bpf_reg_state *reg = ®s[regno]; bool arg_mem_size = false; if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || @@ -11498,10 +11472,9 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, return 0; } -static int process_irq_flag(struct bpf_verifier_env *env, int regno, +static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int regno, struct bpf_kfunc_call_arg_meta *meta) { - struct bpf_reg_state *reg = reg_state(env, regno); int err, kfunc_class = IRQ_NATIVE_KFUNC; bool irq_save; @@ -11526,7 +11499,7 @@ static int process_irq_flag(struct bpf_verifier_env *env, int regno, return -EINVAL; } - err = check_mem_access(env, env->insn_idx, regno, 0, BPF_DW, BPF_WRITE, -1, false, false); + err = check_mem_access(env, env->insn_idx, reg, regno, 0, BPF_DW, BPF_WRITE, -1, false, false); if (err) return err; @@ -12114,7 +12087,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); ref_tname = btf_name_by_offset(btf, ref_t->name_off); - kf_arg_type = get_kfunc_ptr_arg_type(env, meta, t, ref_t, ref_tname, args, i, nargs); + kf_arg_type = get_kfunc_ptr_arg_type(env, meta, t, ref_t, ref_tname, args, i, nargs, reg); if (kf_arg_type < 0) return kf_arg_type; @@ -12276,7 +12249,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ } } - ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type, clone_ref_obj_id); + ret = process_dynptr_func(env, reg, regno, insn_idx, dynptr_arg_type, clone_ref_obj_id); if (ret < 0) return ret; @@ -12301,7 +12274,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ return -EINVAL; } } - ret = process_iter_arg(env, regno, insn_idx, meta); + ret = process_iter_arg(env, reg, regno, insn_idx, meta); if (ret < 0) return ret; break; @@ -12478,7 +12451,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ verbose(env, "arg#%d doesn't point to a map value\n", i); return -EINVAL; } - ret = check_map_field_pointer(env, regno, BPF_WORKQUEUE, &meta->map); + ret = check_map_field_pointer(env, reg, regno, BPF_WORKQUEUE, &meta->map); if (ret < 0) return ret; break; @@ -12487,7 +12460,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ verbose(env, "arg#%d doesn't point to a map value\n", i); return -EINVAL; } - ret = process_timer_kfunc(env, regno, meta); + ret = process_timer_kfunc(env, reg, regno, meta); if (ret < 0) return ret; break; @@ -12496,7 +12469,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ verbose(env, "arg#%d doesn't point to a map value\n", i); return -EINVAL; } - ret = check_map_field_pointer(env, regno, BPF_TASK_WORK, &meta->map); + ret = check_map_field_pointer(env, reg, regno, BPF_TASK_WORK, &meta->map); if (ret < 0) return ret; break; @@ -12505,7 +12478,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ verbose(env, "arg#%d doesn't point to an irq flag on stack\n", i); return -EINVAL; } - ret = process_irq_flag(env, regno, meta); + ret = process_irq_flag(env, reg, regno, meta); if (ret < 0) return ret; break; @@ -12526,7 +12499,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave] || meta->func_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]) flags |= PROCESS_LOCK_IRQ; - ret = process_spin_lock(env, regno, flags); + ret = process_spin_lock(env, reg, regno, flags); if (ret < 0) return ret; break; @@ -13660,7 +13633,7 @@ static int check_stack_access_for_ptr_arithmetic( static int sanitize_check_bounds(struct bpf_verifier_env *env, const struct bpf_insn *insn, - const struct bpf_reg_state *dst_reg) + struct bpf_reg_state *dst_reg) { u32 dst = insn->dst_reg; @@ -13677,7 +13650,7 @@ static int sanitize_check_bounds(struct bpf_verifier_env *env, return -EACCES; break; case PTR_TO_MAP_VALUE: - if (check_map_access(env, dst, 0, 1, false, ACCESS_HELPER)) { + if (check_map_access(env, dst_reg, dst, 0, 1, false, ACCESS_HELPER)) { verbose(env, "R%d pointer arithmetic of map value goes out of range, " "prohibited for !root\n", dst); return -EACCES; @@ -17563,7 +17536,7 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) dst_reg_type = cur_regs(env)[insn->dst_reg].type; - err = check_mem_access(env, env->insn_idx, insn->dst_reg, + err = check_mem_access(env, env->insn_idx, cur_regs(env) + insn->dst_reg, insn->dst_reg, insn->off, BPF_SIZE(insn->code), BPF_WRITE, -1, false, false); if (err) -- cgit v1.2.3 From 024c0ab5db9459d114304d070936c440a61d3fd4 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Wed, 22 Apr 2026 20:34:51 -0700 Subject: bpf: Refactor to handle memory and size together Similar to the previous patch, try to pass bpf_reg_state from caller to callee. Both mem_reg and size_reg are passed to helper functions. This is important for stack arguments as they may be beyond registers 1-5. Acked-by: Puranjay Mohan Acked-by: Kumar Kartikeya Dwivedi Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260423033451.2539065-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 57 +++++++++++++++++++++++++-------------------------- 1 file changed, 28 insertions(+), 29 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index debae6133e4c..1b9a4918fa57 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6934,12 +6934,12 @@ mark: return 0; } -static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, +static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int regno, int access_size, enum bpf_access_type access_type, bool zero_size_allowed, struct bpf_call_arg_meta *meta) { - struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; + struct bpf_reg_state *regs = cur_regs(env); u32 *max_access; switch (base_type(reg->type)) { @@ -7022,12 +7022,12 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, /* verify arguments to helpers or kfuncs consisting of a pointer and an access * size. * - * @regno is the register containing the access size. regno-1 is the register - * containing the pointer. + * @mem_reg contains the pointer, @size_reg contains the access size. */ static int check_mem_size_reg(struct bpf_verifier_env *env, - struct bpf_reg_state *reg, u32 regno, - enum bpf_access_type access_type, + struct bpf_reg_state *mem_reg, + struct bpf_reg_state *size_reg, u32 mem_regno, + u32 size_regno, enum bpf_access_type access_type, bool zero_size_allowed, struct bpf_call_arg_meta *meta) { @@ -7041,37 +7041,37 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, * out. Only upper bounds can be learned because retval is an * int type and negative retvals are allowed. */ - meta->msize_max_value = reg->umax_value; + meta->msize_max_value = size_reg->umax_value; /* The register is SCALAR_VALUE; the access check happens using * its boundaries. For unprivileged variable accesses, disable * raw mode so that the program is required to initialize all * the memory that the helper could just partially fill up. */ - if (!tnum_is_const(reg->var_off)) + if (!tnum_is_const(size_reg->var_off)) meta = NULL; - if (reg->smin_value < 0) { + if (size_reg->smin_value < 0) { verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n", - regno); + size_regno); return -EACCES; } - if (reg->umin_value == 0 && !zero_size_allowed) { + if (size_reg->umin_value == 0 && !zero_size_allowed) { verbose(env, "R%d invalid zero-sized read: u64=[%lld,%lld]\n", - regno, reg->umin_value, reg->umax_value); + size_regno, size_reg->umin_value, size_reg->umax_value); return -EACCES; } - if (reg->umax_value >= BPF_MAX_VAR_SIZ) { + if (size_reg->umax_value >= BPF_MAX_VAR_SIZ) { verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n", - regno); + size_regno); return -EACCES; } - err = check_helper_mem_access(env, regno - 1, reg->umax_value, + err = check_helper_mem_access(env, mem_reg, mem_regno, size_reg->umax_value, access_type, zero_size_allowed, meta); if (!err) - err = mark_chain_precision(env, regno); + err = mark_chain_precision(env, size_regno); return err; } @@ -7096,8 +7096,8 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg int size = base_type(reg->type) == PTR_TO_STACK ? -(int)mem_size : mem_size; - err = check_helper_mem_access(env, regno, size, BPF_READ, true, NULL); - err = err ?: check_helper_mem_access(env, regno, size, BPF_WRITE, true, NULL); + err = check_helper_mem_access(env, reg, regno, size, BPF_READ, true, NULL); + err = err ?: check_helper_mem_access(env, reg, regno, size, BPF_WRITE, true, NULL); if (may_be_null) *reg = saved_reg; @@ -7105,10 +7105,9 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg return err; } -static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - u32 regno) +static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *mem_reg, + struct bpf_reg_state *size_reg, u32 mem_regno, u32 size_regno) { - struct bpf_reg_state *mem_reg = &cur_regs(env)[regno - 1]; bool may_be_null = type_may_be_null(mem_reg->type); struct bpf_reg_state saved_reg; struct bpf_call_arg_meta meta; @@ -7121,8 +7120,8 @@ static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg mark_ptr_not_null_reg(mem_reg); } - err = check_mem_size_reg(env, reg, regno, BPF_READ, true, &meta); - err = err ?: check_mem_size_reg(env, reg, regno, BPF_WRITE, true, &meta); + err = check_mem_size_reg(env, mem_reg, size_reg, mem_regno, size_regno, BPF_READ, true, &meta); + err = err ?: check_mem_size_reg(env, mem_reg, size_reg, mem_regno, size_regno, BPF_WRITE, true, &meta); if (may_be_null) *mem_reg = saved_reg; @@ -8566,7 +8565,7 @@ skip_type_check: return -EFAULT; } key_size = meta->map.ptr->key_size; - err = check_helper_mem_access(env, regno, key_size, BPF_READ, false, NULL); + err = check_helper_mem_access(env, reg, regno, key_size, BPF_READ, false, NULL); if (err) return err; if (can_elide_value_nullness(meta->map.ptr->map_type)) { @@ -8593,7 +8592,7 @@ skip_type_check: return -EFAULT; } meta->raw_mode = arg_type & MEM_UNINIT; - err = check_helper_mem_access(env, regno, meta->map.ptr->value_size, + err = check_helper_mem_access(env, reg, regno, meta->map.ptr->value_size, arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, false, meta); break; @@ -8637,7 +8636,7 @@ skip_type_check: */ meta->raw_mode = arg_type & MEM_UNINIT; if (arg_type & MEM_FIXED_SIZE) { - err = check_helper_mem_access(env, regno, fn->arg_size[arg], + err = check_helper_mem_access(env, reg, regno, fn->arg_size[arg], arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, false, meta); if (err) @@ -8647,13 +8646,13 @@ skip_type_check: } break; case ARG_CONST_SIZE: - err = check_mem_size_reg(env, reg, regno, + err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, regno - 1, regno, fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, false, meta); break; case ARG_CONST_SIZE_OR_ZERO: - err = check_mem_size_reg(env, reg, regno, + err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, regno - 1, regno, fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, true, meta); @@ -12384,7 +12383,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ const struct btf_param *size_arg = &args[i + 1]; if (!bpf_register_is_null(buff_reg) || !is_kfunc_arg_nullable(meta->btf, buff_arg)) { - ret = check_kfunc_mem_size_reg(env, size_reg, regno + 1); + ret = check_kfunc_mem_size_reg(env, buff_reg, size_reg, regno, regno + 1); if (ret < 0) { verbose(env, "arg#%d arg#%d memory, len pair leads to invalid memory access\n", i, i + 1); return ret; -- cgit v1.2.3 From 053a48cb2a5459659a5034ad20e7d9f28bc56f16 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Wed, 22 Apr 2026 20:34:56 -0700 Subject: bpf: Rename existing argno to arg To support stack arguments, in later patches, argno will represent both registers and stack arguments. To avoid confusion, rename existing argno to arg. Acked-by: Kumar Kartikeya Dwivedi Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260423033456.2539340-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 54 +++++++++++++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 1b9a4918fa57..81d77dfaaaf6 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11301,9 +11301,9 @@ get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta, const struct btf_type *t, const struct btf_type *ref_t, const char *ref_tname, const struct btf_param *args, - int argno, int nargs, struct bpf_reg_state *reg) + int arg, int nargs, struct bpf_reg_state *reg) { - u32 regno = argno + 1; + u32 regno = arg + 1; struct bpf_reg_state *regs = cur_regs(env); bool arg_mem_size = false; @@ -11312,9 +11312,9 @@ get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, meta->func_id == special_kfunc_list[KF_bpf_session_cookie]) return KF_ARG_PTR_TO_CTX; - if (argno + 1 < nargs && - (is_kfunc_arg_mem_size(meta->btf, &args[argno + 1], ®s[regno + 1]) || - is_kfunc_arg_const_mem_size(meta->btf, &args[argno + 1], ®s[regno + 1]))) + if (arg + 1 < nargs && + (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1], ®s[regno + 1]) || + is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1], ®s[regno + 1]))) arg_mem_size = true; /* In this function, we verify the kfunc's BTF as per the argument type, @@ -11322,68 +11322,68 @@ get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, * type to our caller. When a set of conditions hold in the BTF type of * arguments, we resolve it to a known kfunc_ptr_arg_type. */ - if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), argno)) + if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), arg)) return KF_ARG_PTR_TO_CTX; - if (is_kfunc_arg_nullable(meta->btf, &args[argno]) && bpf_register_is_null(reg) && + if (is_kfunc_arg_nullable(meta->btf, &args[arg]) && bpf_register_is_null(reg) && !arg_mem_size) return KF_ARG_PTR_TO_NULL; - if (is_kfunc_arg_alloc_obj(meta->btf, &args[argno])) + if (is_kfunc_arg_alloc_obj(meta->btf, &args[arg])) return KF_ARG_PTR_TO_ALLOC_BTF_ID; - if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[argno])) + if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[arg])) return KF_ARG_PTR_TO_REFCOUNTED_KPTR; - if (is_kfunc_arg_dynptr(meta->btf, &args[argno])) + if (is_kfunc_arg_dynptr(meta->btf, &args[arg])) return KF_ARG_PTR_TO_DYNPTR; - if (is_kfunc_arg_iter(meta, argno, &args[argno])) + if (is_kfunc_arg_iter(meta, arg, &args[arg])) return KF_ARG_PTR_TO_ITER; - if (is_kfunc_arg_list_head(meta->btf, &args[argno])) + if (is_kfunc_arg_list_head(meta->btf, &args[arg])) return KF_ARG_PTR_TO_LIST_HEAD; - if (is_kfunc_arg_list_node(meta->btf, &args[argno])) + if (is_kfunc_arg_list_node(meta->btf, &args[arg])) return KF_ARG_PTR_TO_LIST_NODE; - if (is_kfunc_arg_rbtree_root(meta->btf, &args[argno])) + if (is_kfunc_arg_rbtree_root(meta->btf, &args[arg])) return KF_ARG_PTR_TO_RB_ROOT; - if (is_kfunc_arg_rbtree_node(meta->btf, &args[argno])) + if (is_kfunc_arg_rbtree_node(meta->btf, &args[arg])) return KF_ARG_PTR_TO_RB_NODE; - if (is_kfunc_arg_const_str(meta->btf, &args[argno])) + if (is_kfunc_arg_const_str(meta->btf, &args[arg])) return KF_ARG_PTR_TO_CONST_STR; - if (is_kfunc_arg_map(meta->btf, &args[argno])) + if (is_kfunc_arg_map(meta->btf, &args[arg])) return KF_ARG_PTR_TO_MAP; - if (is_kfunc_arg_wq(meta->btf, &args[argno])) + if (is_kfunc_arg_wq(meta->btf, &args[arg])) return KF_ARG_PTR_TO_WORKQUEUE; - if (is_kfunc_arg_timer(meta->btf, &args[argno])) + if (is_kfunc_arg_timer(meta->btf, &args[arg])) return KF_ARG_PTR_TO_TIMER; - if (is_kfunc_arg_task_work(meta->btf, &args[argno])) + if (is_kfunc_arg_task_work(meta->btf, &args[arg])) return KF_ARG_PTR_TO_TASK_WORK; - if (is_kfunc_arg_irq_flag(meta->btf, &args[argno])) + if (is_kfunc_arg_irq_flag(meta->btf, &args[arg])) return KF_ARG_PTR_TO_IRQ_FLAG; - if (is_kfunc_arg_res_spin_lock(meta->btf, &args[argno])) + if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg])) return KF_ARG_PTR_TO_RES_SPIN_LOCK; if ((base_type(reg->type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(reg->type)])) { if (!btf_type_is_struct(ref_t)) { verbose(env, "kernel function %s args#%d pointer type %s %s is not supported\n", - meta->func_name, argno, btf_type_str(ref_t), ref_tname); + meta->func_name, arg, btf_type_str(ref_t), ref_tname); return -EINVAL; } return KF_ARG_PTR_TO_BTF_ID; } - if (is_kfunc_arg_callback(env, meta->btf, &args[argno])) + if (is_kfunc_arg_callback(env, meta->btf, &args[arg])) return KF_ARG_PTR_TO_CALLBACK; /* This is the catch all argument type of register types supported by @@ -11394,7 +11394,7 @@ get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, if (!btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0) && (arg_mem_size ? !btf_type_is_void(ref_t) : 1)) { verbose(env, "arg#%d pointer type %s %s must point to %sscalar, or struct with scalar\n", - argno, btf_type_str(ref_t), ref_tname, arg_mem_size ? "void, " : ""); + arg, btf_type_str(ref_t), ref_tname, arg_mem_size ? "void, " : ""); return -EINVAL; } return arg_mem_size ? KF_ARG_PTR_TO_MEM_SIZE : KF_ARG_PTR_TO_MEM; @@ -11405,7 +11405,7 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, const struct btf_type *ref_t, const char *ref_tname, u32 ref_id, struct bpf_kfunc_call_arg_meta *meta, - int argno) + int arg) { const struct btf_type *reg_ref_t; bool strict_type_match = false; @@ -11464,7 +11464,7 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, taking_projection = btf_is_projection_of(ref_tname, reg_ref_tname); if (!taking_projection && !struct_same) { verbose(env, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\n", - meta->func_name, argno, btf_type_str(ref_t), ref_tname, argno + 1, + meta->func_name, arg, btf_type_str(ref_t), ref_tname, arg + 1, btf_type_str(reg_ref_t), reg_ref_tname); return -EINVAL; } -- cgit v1.2.3 From 9b9f0b42703ceb88332bcb19453c4288c2683e34 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Wed, 22 Apr 2026 20:35:01 -0700 Subject: bpf: Prepare verifier logs for upcoming kfunc stack arguments This change prepares verifier log reporting for upcoming kfunc stack argument support. Currently verifier log code mostly assumes that an argument can be described directly by a register number. That works for arguments passed in `R1` to `R5`, but it does not work once kfunc arguments can also be passed on the stack. Introduce an opaque `argno_t` type that encodes both register-based and arg-based references. Four helpers form the interface: - argno_from_reg(regno): create from a register number - argno_from_arg(arg): create from a 1-based arg number - reg_from_argno(a): extract register number, or -1 - arg_from_argno(a): extract arg number, or -1 reg_arg_name() converts an argno_t to a human-readable string for verifier logs: "R%d" for register arguments, or "*(R11-off)" for stack arguments beyond R5. Update selftests accordingly. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260423033501.2539667-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 694 +++++++++++++++++++++++++++++--------------------- 1 file changed, 397 insertions(+), 297 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 81d77dfaaaf6..ff6ff1c27517 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -261,6 +261,36 @@ struct bpf_kfunc_meta { struct btf *btf_vmlinux; +typedef struct argno { + int argno; +} argno_t; + +static argno_t argno_from_reg(u32 regno) +{ + return (argno_t){ .argno = regno }; +} + +static argno_t argno_from_arg(u32 arg) +{ + return (argno_t){ .argno = -arg }; +} + +static int reg_from_argno(argno_t a) +{ + if (a.argno >= 0) + return a.argno; + if (a.argno >= -MAX_BPF_FUNC_REG_ARGS) + return -a.argno; + return -1; +} + +static int arg_from_argno(argno_t a) +{ + if (a.argno < 0) + return -a.argno; + return -1; +} + static const char *btf_type_name(const struct btf *btf, u32 id) { return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off); @@ -1742,6 +1772,22 @@ static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, return &elem->st; } +static const char *reg_arg_name(struct bpf_verifier_env *env, argno_t argno) +{ + char *buf = env->tmp_arg_name; + int len = sizeof(env->tmp_arg_name); + int arg, regno = reg_from_argno(argno); + + if (regno >= 0) { + snprintf(buf, len, "R%d", regno); + } else { + arg = arg_from_argno(argno); + snprintf(buf, len, "*(R11-%u)", (arg - MAX_BPF_FUNC_REG_ARGS) * BPF_REG_SIZE); + } + + return buf; +} + static const int caller_saved[CALLER_SAVED_REGS] = { BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5 }; @@ -4241,7 +4287,7 @@ enum bpf_access_src { }; static int check_stack_range_initialized(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - int regno, int off, int access_size, + argno_t argno, int off, int access_size, bool zero_size_allowed, enum bpf_access_type type, struct bpf_call_arg_meta *meta); @@ -4265,7 +4311,7 @@ static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) * instead. */ static int check_stack_read_var_off(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - int ptr_regno, int off, int size, int dst_regno) + argno_t ptr_argno, int off, int size, int dst_regno) { struct bpf_func_state *ptr_state = bpf_func(env, reg); int err; @@ -4273,7 +4319,7 @@ static int check_stack_read_var_off(struct bpf_verifier_env *env, struct bpf_reg /* Note that we pass a NULL meta, so raw access will not be permitted. */ - err = check_stack_range_initialized(env, reg, ptr_regno, off, size, + err = check_stack_range_initialized(env, reg, ptr_argno, off, size, false, BPF_READ, NULL); if (err) return err; @@ -4295,7 +4341,7 @@ static int check_stack_read_var_off(struct bpf_verifier_env *env, struct bpf_reg * can be -1, meaning that the read value is not going to a register. */ static int check_stack_read(struct bpf_verifier_env *env, - struct bpf_reg_state *reg, int ptr_regno, int off, int size, + struct bpf_reg_state *reg, argno_t ptr_argno, int off, int size, int dst_regno) { struct bpf_func_state *state = bpf_func(env, reg); @@ -4333,7 +4379,7 @@ static int check_stack_read(struct bpf_verifier_env *env, * than fixed offset ones. Note that dst_regno >= 0 on this * branch. */ - err = check_stack_read_var_off(env, reg, ptr_regno, off, size, + err = check_stack_read_var_off(env, reg, ptr_argno, off, size, dst_regno); } return err; @@ -4393,7 +4439,7 @@ static int check_map_access_type(struct bpf_verifier_env *env, struct bpf_reg_st } /* check read/write into memory region (e.g., map value, ringbuf sample, etc) */ -static int __check_mem_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int regno, +static int __check_mem_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int off, int size, u32 mem_size, bool zero_size_allowed) { @@ -4414,8 +4460,8 @@ static int __check_mem_access(struct bpf_verifier_env *env, struct bpf_reg_state case PTR_TO_PACKET: case PTR_TO_PACKET_META: case PTR_TO_PACKET_END: - verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n", - off, size, regno, reg->id, off, mem_size); + verbose(env, "invalid access to packet, off=%d size=%d, %s(id=%d,off=%d,r=%d)\n", + off, size, reg_arg_name(env, argno), reg->id, off, mem_size); break; case PTR_TO_CTX: verbose(env, "invalid access to context, ctx_size=%d off=%d size=%d\n", @@ -4431,7 +4477,7 @@ static int __check_mem_access(struct bpf_verifier_env *env, struct bpf_reg_state } /* check read/write into a memory region with possible variable offset */ -static int check_mem_region_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, u32 regno, +static int check_mem_region_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int off, int size, u32 mem_size, bool zero_size_allowed) { @@ -4451,15 +4497,15 @@ static int check_mem_region_access(struct bpf_verifier_env *env, struct bpf_reg_ (reg->smin_value == S64_MIN || (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) || reg->smin_value + off < 0)) { - verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", - regno); + verbose(env, "%s min value is negative, either use unsigned index or do a if (index >=0) check.\n", + reg_arg_name(env, argno)); return -EACCES; } - err = __check_mem_access(env, reg, regno, reg->smin_value + off, size, + err = __check_mem_access(env, reg, argno, reg->smin_value + off, size, mem_size, zero_size_allowed); if (err) { - verbose(env, "R%d min value is outside of the allowed memory range\n", - regno); + verbose(env, "%s min value is outside of the allowed memory range\n", + reg_arg_name(env, argno)); return err; } @@ -4468,15 +4514,15 @@ static int check_mem_region_access(struct bpf_verifier_env *env, struct bpf_reg_ * If reg->umax_value + off could overflow, treat that as unbounded too. */ if (reg->umax_value >= BPF_MAX_VAR_OFF) { - verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n", - regno); + verbose(env, "%s unbounded memory access, make sure to bounds check any such access\n", + reg_arg_name(env, argno)); return -EACCES; } - err = __check_mem_access(env, reg, regno, reg->umax_value + off, size, + err = __check_mem_access(env, reg, argno, reg->umax_value + off, size, mem_size, zero_size_allowed); if (err) { - verbose(env, "R%d max value is outside of the allowed memory range\n", - regno); + verbose(env, "%s max value is outside of the allowed memory range\n", + reg_arg_name(env, argno)); return err; } @@ -4484,7 +4530,7 @@ static int check_mem_region_access(struct bpf_verifier_env *env, struct bpf_reg_ } static int __check_ptr_off_reg(struct bpf_verifier_env *env, - const struct bpf_reg_state *reg, int regno, + const struct bpf_reg_state *reg, argno_t argno, bool fixed_off_ok) { /* Access to this pointer-typed register or passing it to a helper @@ -4501,14 +4547,14 @@ static int __check_ptr_off_reg(struct bpf_verifier_env *env, } if (reg->smin_value < 0) { - verbose(env, "negative offset %s ptr R%d off=%lld disallowed\n", - reg_type_str(env, reg->type), regno, reg->var_off.value); + verbose(env, "negative offset %s ptr %s off=%lld disallowed\n", + reg_type_str(env, reg->type), reg_arg_name(env, argno), reg->var_off.value); return -EACCES; } if (!fixed_off_ok && reg->var_off.value != 0) { - verbose(env, "dereference of modified %s ptr R%d off=%lld disallowed\n", - reg_type_str(env, reg->type), regno, reg->var_off.value); + verbose(env, "dereference of modified %s ptr %s off=%lld disallowed\n", + reg_type_str(env, reg->type), reg_arg_name(env, argno), reg->var_off.value); return -EACCES; } @@ -4518,7 +4564,7 @@ static int __check_ptr_off_reg(struct bpf_verifier_env *env, static int check_ptr_off_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, int regno) { - return __check_ptr_off_reg(env, reg, regno, false); + return __check_ptr_off_reg(env, reg, argno_from_reg(regno), false); } static int map_kptr_match_type(struct bpf_verifier_env *env, @@ -4556,7 +4602,7 @@ static int map_kptr_match_type(struct bpf_verifier_env *env, * Since ref_ptr cannot be accessed directly by BPF insns, check for * reg->ref_obj_id is not needed here. */ - if (__check_ptr_off_reg(env, reg, regno, true)) + if (__check_ptr_off_reg(env, reg, argno_from_reg(regno), true)) return -EACCES; /* A full type match is needed, as BTF can be vmlinux, module or prog BTF, and @@ -4776,7 +4822,7 @@ static u32 map_mem_size(const struct bpf_map *map) } /* check read/write into a map element with possible variable offset */ -static int check_map_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, u32 regno, +static int check_map_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int off, int size, bool zero_size_allowed, enum bpf_access_src src) { @@ -4785,7 +4831,7 @@ static int check_map_access(struct bpf_verifier_env *env, struct bpf_reg_state * struct btf_record *rec; int err, i; - err = check_mem_region_access(env, reg, regno, off, size, mem_size, zero_size_allowed); + err = check_mem_region_access(env, reg, argno, off, size, mem_size, zero_size_allowed); if (err) return err; @@ -4881,17 +4927,17 @@ static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, } } -static int check_packet_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, u32 regno, int off, +static int check_packet_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int off, int size, bool zero_size_allowed) { int err; if (reg->range < 0) { - verbose(env, "R%d offset is outside of the packet\n", regno); + verbose(env, "%s offset is outside of the packet\n", reg_arg_name(env, argno)); return -EINVAL; } - err = check_mem_region_access(env, reg, regno, off, size, reg->range, zero_size_allowed); + err = check_mem_region_access(env, reg, argno, off, size, reg->range, zero_size_allowed); if (err) return err; @@ -4946,7 +4992,7 @@ static int __check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int of return -EACCES; } -static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, struct bpf_reg_state *reg, u32 regno, +static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, struct bpf_reg_state *reg, argno_t argno, int off, int access_size, enum bpf_access_type t, struct bpf_insn_access_aux *info) { @@ -4959,9 +5005,9 @@ static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, struct b int err; if (var_off_ok) - err = check_mem_region_access(env, reg, regno, off, access_size, U16_MAX, false); + err = check_mem_region_access(env, reg, argno, off, access_size, U16_MAX, false); else - err = __check_ptr_off_reg(env, reg, regno, fixed_off_ok); + err = __check_ptr_off_reg(env, reg, argno, fixed_off_ok); if (err) return err; off += reg->umax_value; @@ -4985,15 +5031,15 @@ static int check_flow_keys_access(struct bpf_verifier_env *env, int off, } static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, - struct bpf_reg_state *reg, u32 regno, int off, int size, + struct bpf_reg_state *reg, argno_t argno, int off, int size, enum bpf_access_type t) { struct bpf_insn_access_aux info = {}; bool valid; if (reg->smin_value < 0) { - verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", - regno); + verbose(env, "%s min value is negative, either use unsigned index or do a if (index >=0) check.\n", + reg_arg_name(env, argno)); return -EACCES; } @@ -5021,8 +5067,8 @@ static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, return 0; } - verbose(env, "R%d invalid %s access off=%d size=%d\n", - regno, reg_type_str(env, reg->type), off, size); + verbose(env, "%s invalid %s access off=%d size=%d\n", + reg_arg_name(env, argno), reg_type_str(env, reg->type), off, size); return -EACCES; } @@ -5535,12 +5581,12 @@ static int check_max_stack_depth(struct bpf_verifier_env *env) static int __check_buffer_access(struct bpf_verifier_env *env, const char *buf_info, const struct bpf_reg_state *reg, - int regno, int off, int size) + argno_t argno, int off, int size) { if (off < 0) { verbose(env, - "R%d invalid %s buffer access: off=%d, size=%d\n", - regno, buf_info, off, size); + "%s invalid %s buffer access: off=%d, size=%d\n", + reg_arg_name(env, argno), buf_info, off, size); return -EACCES; } if (!tnum_is_const(reg->var_off)) { @@ -5548,8 +5594,8 @@ static int __check_buffer_access(struct bpf_verifier_env *env, tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); verbose(env, - "R%d invalid variable buffer offset: off=%d, var_off=%s\n", - regno, off, tn_buf); + "%s invalid variable buffer offset: off=%d, var_off=%s\n", + reg_arg_name(env, argno), off, tn_buf); return -EACCES; } @@ -5558,11 +5604,11 @@ static int __check_buffer_access(struct bpf_verifier_env *env, static int check_tp_buffer_access(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, - int regno, int off, int size) + argno_t argno, int off, int size) { int err; - err = __check_buffer_access(env, "tracepoint", reg, regno, off, size); + err = __check_buffer_access(env, "tracepoint", reg, argno, off, size); if (err) return err; @@ -5574,14 +5620,14 @@ static int check_tp_buffer_access(struct bpf_verifier_env *env, static int check_buffer_access(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, - int regno, int off, int size, + argno_t argno, int off, int size, bool zero_size_allowed, u32 *max_access) { const char *buf_info = type_is_rdonly_mem(reg->type) ? "rdonly" : "rdwr"; int err; - err = __check_buffer_access(env, buf_info, reg, regno, off, size); + err = __check_buffer_access(env, buf_info, reg, argno, off, size); if (err) return err; @@ -5954,7 +6000,7 @@ static bool type_is_trusted_or_null(struct bpf_verifier_env *env, static int check_ptr_to_btf_access(struct bpf_verifier_env *env, struct bpf_reg_state *regs, struct bpf_reg_state *reg, - int regno, int off, int size, + argno_t argno, int off, int size, enum bpf_access_type atype, int value_regno) { @@ -5983,8 +6029,8 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); verbose(env, - "R%d is ptr_%s invalid variable offset: off=%d, var_off=%s\n", - regno, tname, off, tn_buf); + "%s is ptr_%s invalid variable offset: off=%d, var_off=%s\n", + reg_arg_name(env, argno), tname, off, tn_buf); return -EACCES; } @@ -5992,22 +6038,22 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, if (off < 0) { verbose(env, - "R%d is ptr_%s invalid negative access: off=%d\n", - regno, tname, off); + "%s is ptr_%s invalid negative access: off=%d\n", + reg_arg_name(env, argno), tname, off); return -EACCES; } if (reg->type & MEM_USER) { verbose(env, - "R%d is ptr_%s access user memory: off=%d\n", - regno, tname, off); + "%s is ptr_%s access user memory: off=%d\n", + reg_arg_name(env, argno), tname, off); return -EACCES; } if (reg->type & MEM_PERCPU) { verbose(env, - "R%d is ptr_%s access percpu memory: off=%d\n", - regno, tname, off); + "%s is ptr_%s access percpu memory: off=%d\n", + reg_arg_name(env, argno), tname, off); return -EACCES; } @@ -6110,7 +6156,7 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, static int check_ptr_to_map_access(struct bpf_verifier_env *env, struct bpf_reg_state *regs, struct bpf_reg_state *reg, - int regno, int off, int size, + argno_t argno, int off, int size, enum bpf_access_type atype, int value_regno) { @@ -6144,8 +6190,8 @@ static int check_ptr_to_map_access(struct bpf_verifier_env *env, } if (off < 0) { - verbose(env, "R%d is %s invalid negative access: off=%d\n", - regno, tname, off); + verbose(env, "%s is %s invalid negative access: off=%d\n", + reg_arg_name(env, argno), tname, off); return -EACCES; } @@ -6203,7 +6249,7 @@ static int check_stack_slot_within_bounds(struct bpf_verifier_env *env, */ static int check_stack_access_within_bounds( struct bpf_verifier_env *env, struct bpf_reg_state *reg, - int regno, int off, int access_size, + argno_t argno, int off, int access_size, enum bpf_access_type type) { struct bpf_func_state *state = bpf_func(env, reg); @@ -6222,8 +6268,8 @@ static int check_stack_access_within_bounds( } else { if (reg->smax_value >= BPF_MAX_VAR_OFF || reg->smin_value <= -BPF_MAX_VAR_OFF) { - verbose(env, "invalid unbounded variable-offset%s stack R%d\n", - err_extra, regno); + verbose(env, "invalid unbounded variable-offset%s stack %s\n", + err_extra, reg_arg_name(env, argno)); return -EACCES; } min_off = reg->smin_value + off; @@ -6241,14 +6287,14 @@ static int check_stack_access_within_bounds( if (err) { if (tnum_is_const(reg->var_off)) { - verbose(env, "invalid%s stack R%d off=%lld size=%d\n", - err_extra, regno, min_off, access_size); + verbose(env, "invalid%s stack %s off=%lld size=%d\n", + err_extra, reg_arg_name(env, argno), min_off, access_size); } else { char tn_buf[48]; tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); - verbose(env, "invalid variable-offset%s stack R%d var_off=%s off=%d size=%d\n", - err_extra, regno, tn_buf, off, access_size); + verbose(env, "invalid variable-offset%s stack %s var_off=%s off=%d size=%d\n", + err_extra, reg_arg_name(env, argno), tn_buf, off, access_size); } return err; } @@ -6293,7 +6339,7 @@ static void add_scalar_to_reg(struct bpf_reg_state *dst_reg, s64 val) * if t==write && value_regno==-1, some unknown value is stored into memory * if t==read && value_regno==-1, don't care what we read from memory */ -static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct bpf_reg_state *reg, u32 regno, +static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct bpf_reg_state *reg, argno_t argno, int off, int bpf_size, enum bpf_access_type t, int value_regno, bool strict_alignment_once, bool is_ldsx) { @@ -6310,11 +6356,12 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b if (reg->type == PTR_TO_MAP_KEY) { if (t == BPF_WRITE) { - verbose(env, "write to change key R%d not allowed\n", regno); + verbose(env, "write to change key %s not allowed\n", + reg_arg_name(env, argno)); return -EACCES; } - err = check_mem_region_access(env, reg, regno, off, size, + err = check_mem_region_access(env, reg, argno, off, size, reg->map_ptr->key_size, false); if (err) return err; @@ -6331,7 +6378,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b err = check_map_access_type(env, reg, off, size, t); if (err) return err; - err = check_map_access(env, reg, regno, off, size, false, ACCESS_DIRECT); + err = check_map_access(env, reg, argno, off, size, false, ACCESS_DIRECT); if (err) return err; if (tnum_is_const(reg->var_off)) @@ -6378,14 +6425,14 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b bool rdonly_untrusted = rdonly_mem && (reg->type & PTR_UNTRUSTED); if (type_may_be_null(reg->type)) { - verbose(env, "R%d invalid mem access '%s'\n", regno, + verbose(env, "%s invalid mem access '%s'\n", reg_arg_name(env, argno), reg_type_str(env, reg->type)); return -EACCES; } if (t == BPF_WRITE && rdonly_mem) { - verbose(env, "R%d cannot write into %s\n", - regno, reg_type_str(env, reg->type)); + verbose(env, "%s cannot write into %s\n", + reg_arg_name(env, argno), reg_type_str(env, reg->type)); return -EACCES; } @@ -6400,7 +6447,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b * instructions, hence no need to check bounds in that case. */ if (!rdonly_untrusted) - err = check_mem_region_access(env, reg, regno, off, size, + err = check_mem_region_access(env, reg, argno, off, size, reg->mem_size, false); if (!err && value_regno >= 0 && (t == BPF_READ || rdonly_mem)) mark_reg_unknown(env, regs, value_regno); @@ -6418,7 +6465,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b return -EACCES; } - err = check_ctx_access(env, insn_idx, reg, regno, off, size, t, &info); + err = check_ctx_access(env, insn_idx, reg, argno, off, size, t, &info); if (!err && t == BPF_READ && value_regno >= 0) { /* ctx access returns either a scalar, or a * PTR_TO_PACKET[_META,_END]. In the latter @@ -6455,12 +6502,12 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b } else if (reg->type == PTR_TO_STACK) { /* Basic bounds checks. */ - err = check_stack_access_within_bounds(env, reg, regno, off, size, t); + err = check_stack_access_within_bounds(env, reg, argno, off, size, t); if (err) return err; if (t == BPF_READ) - err = check_stack_read(env, reg, regno, off, size, + err = check_stack_read(env, reg, argno, off, size, value_regno); else err = check_stack_write(env, reg, off, size, @@ -6476,7 +6523,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b value_regno); return -EACCES; } - err = check_packet_access(env, reg, regno, off, size, false); + err = check_packet_access(env, reg, argno, off, size, false); if (!err && t == BPF_READ && value_regno >= 0) mark_reg_unknown(env, regs, value_regno); } else if (reg->type == PTR_TO_FLOW_KEYS) { @@ -6492,23 +6539,23 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b mark_reg_unknown(env, regs, value_regno); } else if (type_is_sk_pointer(reg->type)) { if (t == BPF_WRITE) { - verbose(env, "R%d cannot write into %s\n", - regno, reg_type_str(env, reg->type)); + verbose(env, "%s cannot write into %s\n", + reg_arg_name(env, argno), reg_type_str(env, reg->type)); return -EACCES; } - err = check_sock_access(env, insn_idx, reg, regno, off, size, t); + err = check_sock_access(env, insn_idx, reg, argno, off, size, t); if (!err && value_regno >= 0) mark_reg_unknown(env, regs, value_regno); } else if (reg->type == PTR_TO_TP_BUFFER) { - err = check_tp_buffer_access(env, reg, regno, off, size); + err = check_tp_buffer_access(env, reg, argno, off, size); if (!err && t == BPF_READ && value_regno >= 0) mark_reg_unknown(env, regs, value_regno); } else if (base_type(reg->type) == PTR_TO_BTF_ID && !type_may_be_null(reg->type)) { - err = check_ptr_to_btf_access(env, regs, reg, regno, off, size, t, + err = check_ptr_to_btf_access(env, regs, reg, argno, off, size, t, value_regno); } else if (reg->type == CONST_PTR_TO_MAP) { - err = check_ptr_to_map_access(env, regs, reg, regno, off, size, t, + err = check_ptr_to_map_access(env, regs, reg, argno, off, size, t, value_regno); } else if (base_type(reg->type) == PTR_TO_BUF && !type_may_be_null(reg->type)) { @@ -6517,8 +6564,8 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b if (rdonly_mem) { if (t == BPF_WRITE) { - verbose(env, "R%d cannot write into %s\n", - regno, reg_type_str(env, reg->type)); + verbose(env, "%s cannot write into %s\n", + reg_arg_name(env, argno), reg_type_str(env, reg->type)); return -EACCES; } max_access = &env->prog->aux->max_rdonly_access; @@ -6526,7 +6573,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b max_access = &env->prog->aux->max_rdwr_access; } - err = check_buffer_access(env, reg, regno, off, size, false, + err = check_buffer_access(env, reg, argno, off, size, false, max_access); if (!err && value_regno >= 0 && (rdonly_mem || t == BPF_READ)) @@ -6535,7 +6582,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b if (t == BPF_READ && value_regno >= 0) mark_reg_unknown(env, regs, value_regno); } else { - verbose(env, "R%d invalid mem access '%s'\n", regno, + verbose(env, "%s invalid mem access '%s'\n", reg_arg_name(env, argno), reg_type_str(env, reg->type)); return -EACCES; } @@ -6577,7 +6624,7 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, /* Check if (src_reg + off) is readable. The state of dst_reg will be * updated by this call. */ - err = check_mem_access(env, env->insn_idx, regs + insn->src_reg, insn->src_reg, insn->off, + err = check_mem_access(env, env->insn_idx, regs + insn->src_reg, argno_from_reg(insn->src_reg), insn->off, BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, strict_alignment_once, is_ldsx); err = err ?: save_aux_ptr_type(env, src_reg_type, @@ -6607,7 +6654,7 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, dst_reg_type = regs[insn->dst_reg].type; /* Check if (dst_reg + off) is writeable. */ - err = check_mem_access(env, env->insn_idx, regs + insn->dst_reg, insn->dst_reg, insn->off, + err = check_mem_access(env, env->insn_idx, regs + insn->dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_WRITE, insn->src_reg, strict_alignment_once, false); err = err ?: save_aux_ptr_type(env, dst_reg_type, false); @@ -6685,10 +6732,10 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, /* Check whether we can read the memory, with second call for fetch * case to simulate the register fill. */ - err = check_mem_access(env, env->insn_idx, dst_reg, insn->dst_reg, insn->off, + err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_READ, -1, true, false); if (!err && load_reg >= 0) - err = check_mem_access(env, env->insn_idx, dst_reg, insn->dst_reg, + err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_READ, load_reg, true, false); if (err) @@ -6700,7 +6747,7 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, return err; } /* Check whether we can write into the same memory. */ - err = check_mem_access(env, env->insn_idx, dst_reg, insn->dst_reg, insn->off, + err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_WRITE, -1, true, false); if (err) return err; @@ -6789,7 +6836,7 @@ static int check_atomic(struct bpf_verifier_env *env, struct bpf_insn *insn) * read offsets are marked as read. */ static int check_stack_range_initialized( - struct bpf_verifier_env *env, struct bpf_reg_state *reg, int regno, int off, + struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int off, int access_size, bool zero_size_allowed, enum bpf_access_type type, struct bpf_call_arg_meta *meta) { @@ -6814,7 +6861,7 @@ static int check_stack_range_initialized( return -EACCES; } - err = check_stack_access_within_bounds(env, reg, regno, off, access_size, type); + err = check_stack_access_within_bounds(env, reg, argno, off, access_size, type); if (err) return err; @@ -6831,8 +6878,8 @@ static int check_stack_range_initialized( char tn_buf[48]; tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); - verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n", - regno, tn_buf); + verbose(env, "%s variable offset stack access prohibited for !root, var_off=%s\n", + reg_arg_name(env, argno), tn_buf); return -EACCES; } /* Only initialized buffer on stack is allowed to be accessed @@ -6875,7 +6922,7 @@ static int check_stack_range_initialized( } } meta->access_size = access_size; - meta->regno = regno; + meta->regno = reg_from_argno(argno); return 0; } @@ -6915,17 +6962,17 @@ static int check_stack_range_initialized( if (*stype == STACK_POISON) { if (allow_poison) goto mark; - verbose(env, "reading from stack R%d off %d+%d size %d, slot poisoned by dead code elimination\n", - regno, min_off, i - min_off, access_size); + verbose(env, "reading from stack %s off %d+%d size %d, slot poisoned by dead code elimination\n", + reg_arg_name(env, argno), min_off, i - min_off, access_size); } else if (tnum_is_const(reg->var_off)) { - verbose(env, "invalid read from stack R%d off %d+%d size %d\n", - regno, min_off, i - min_off, access_size); + verbose(env, "invalid read from stack %s off %d+%d size %d\n", + reg_arg_name(env, argno), min_off, i - min_off, access_size); } else { char tn_buf[48]; tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); - verbose(env, "invalid read from stack R%d var_off %s+%d size %d\n", - regno, tn_buf, i - min_off, access_size); + verbose(env, "invalid read from stack %s var_off %s+%d size %d\n", + reg_arg_name(env, argno), tn_buf, i - min_off, access_size); } return -EACCES; mark: @@ -6934,7 +6981,7 @@ mark: return 0; } -static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int regno, +static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int access_size, enum bpf_access_type access_type, bool zero_size_allowed, struct bpf_call_arg_meta *meta) @@ -6945,37 +6992,37 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ switch (base_type(reg->type)) { case PTR_TO_PACKET: case PTR_TO_PACKET_META: - return check_packet_access(env, reg, regno, 0, access_size, + return check_packet_access(env, reg, argno, 0, access_size, zero_size_allowed); case PTR_TO_MAP_KEY: if (access_type == BPF_WRITE) { - verbose(env, "R%d cannot write into %s\n", regno, - reg_type_str(env, reg->type)); + verbose(env, "%s cannot write into %s\n", + reg_arg_name(env, argno), reg_type_str(env, reg->type)); return -EACCES; } - return check_mem_region_access(env, reg, regno, 0, access_size, + return check_mem_region_access(env, reg, argno, 0, access_size, reg->map_ptr->key_size, false); case PTR_TO_MAP_VALUE: if (check_map_access_type(env, reg, 0, access_size, access_type)) return -EACCES; - return check_map_access(env, reg, regno, 0, access_size, + return check_map_access(env, reg, argno, 0, access_size, zero_size_allowed, ACCESS_HELPER); case PTR_TO_MEM: if (type_is_rdonly_mem(reg->type)) { if (access_type == BPF_WRITE) { - verbose(env, "R%d cannot write into %s\n", regno, - reg_type_str(env, reg->type)); + verbose(env, "%s cannot write into %s\n", + reg_arg_name(env, argno), reg_type_str(env, reg->type)); return -EACCES; } } - return check_mem_region_access(env, reg, regno, 0, + return check_mem_region_access(env, reg, argno, 0, access_size, reg->mem_size, zero_size_allowed); case PTR_TO_BUF: if (type_is_rdonly_mem(reg->type)) { if (access_type == BPF_WRITE) { - verbose(env, "R%d cannot write into %s\n", regno, - reg_type_str(env, reg->type)); + verbose(env, "%s cannot write into %s\n", + reg_arg_name(env, argno), reg_type_str(env, reg->type)); return -EACCES; } @@ -6983,21 +7030,21 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ } else { max_access = &env->prog->aux->max_rdwr_access; } - return check_buffer_access(env, reg, regno, 0, + return check_buffer_access(env, reg, argno, 0, access_size, zero_size_allowed, max_access); case PTR_TO_STACK: return check_stack_range_initialized( env, reg, - regno, 0, access_size, + argno, 0, access_size, zero_size_allowed, access_type, meta); case PTR_TO_BTF_ID: - return check_ptr_to_btf_access(env, regs, reg, regno, 0, + return check_ptr_to_btf_access(env, regs, reg, argno, 0, access_size, BPF_READ, -1); case PTR_TO_CTX: /* Only permit reading or writing syscall context using helper calls. */ if (is_var_ctx_off_allowed(env->prog)) { - int err = check_mem_region_access(env, reg, regno, 0, access_size, U16_MAX, + int err = check_mem_region_access(env, reg, argno, 0, access_size, U16_MAX, zero_size_allowed); if (err) return err; @@ -7012,7 +7059,7 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ bpf_register_is_null(reg)) return 0; - verbose(env, "R%d type=%s ", regno, + verbose(env, "%s type=%s ", reg_arg_name(env, argno), reg_type_str(env, reg->type)); verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK)); return -EACCES; @@ -7026,8 +7073,8 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ */ static int check_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *mem_reg, - struct bpf_reg_state *size_reg, u32 mem_regno, - u32 size_regno, enum bpf_access_type access_type, + struct bpf_reg_state *size_reg, argno_t mem_argno, + argno_t size_argno, enum bpf_access_type access_type, bool zero_size_allowed, struct bpf_call_arg_meta *meta) { @@ -7052,31 +7099,31 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, meta = NULL; if (size_reg->smin_value < 0) { - verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n", - size_regno); + verbose(env, "%s min value is negative, either use unsigned or 'var &= const'\n", + reg_arg_name(env, size_argno)); return -EACCES; } if (size_reg->umin_value == 0 && !zero_size_allowed) { - verbose(env, "R%d invalid zero-sized read: u64=[%lld,%lld]\n", - size_regno, size_reg->umin_value, size_reg->umax_value); + verbose(env, "%s invalid zero-sized read: u64=[%lld,%lld]\n", + reg_arg_name(env, size_argno), size_reg->umin_value, size_reg->umax_value); return -EACCES; } if (size_reg->umax_value >= BPF_MAX_VAR_SIZ) { - verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n", - size_regno); + verbose(env, "%s unbounded memory access, use 'var &= const' or 'if (var < const)'\n", + reg_arg_name(env, size_argno)); return -EACCES; } - err = check_helper_mem_access(env, mem_reg, mem_regno, size_reg->umax_value, + err = check_helper_mem_access(env, mem_reg, mem_argno, size_reg->umax_value, access_type, zero_size_allowed, meta); if (!err) - err = mark_chain_precision(env, size_regno); + err = mark_chain_precision(env, reg_from_argno(size_argno)); return err; } static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - u32 regno, u32 mem_size) + argno_t argno, u32 mem_size) { bool may_be_null = type_may_be_null(reg->type); struct bpf_reg_state saved_reg; @@ -7096,8 +7143,8 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg int size = base_type(reg->type) == PTR_TO_STACK ? -(int)mem_size : mem_size; - err = check_helper_mem_access(env, reg, regno, size, BPF_READ, true, NULL); - err = err ?: check_helper_mem_access(env, reg, regno, size, BPF_WRITE, true, NULL); + err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, NULL); + err = err ?: check_helper_mem_access(env, reg, argno, size, BPF_WRITE, true, NULL); if (may_be_null) *reg = saved_reg; @@ -7106,7 +7153,7 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg } static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *mem_reg, - struct bpf_reg_state *size_reg, u32 mem_regno, u32 size_regno) + struct bpf_reg_state *size_reg, argno_t mem_argno, argno_t size_argno) { bool may_be_null = type_may_be_null(mem_reg->type); struct bpf_reg_state saved_reg; @@ -7120,8 +7167,8 @@ static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg mark_ptr_not_null_reg(mem_reg); } - err = check_mem_size_reg(env, mem_reg, size_reg, mem_regno, size_regno, BPF_READ, true, &meta); - err = err ?: check_mem_size_reg(env, mem_reg, size_reg, mem_regno, size_regno, BPF_WRITE, true, &meta); + err = check_mem_size_reg(env, mem_reg, size_reg, mem_argno, size_argno, BPF_READ, true, &meta); + err = err ?: check_mem_size_reg(env, mem_reg, size_reg, mem_argno, size_argno, BPF_WRITE, true, &meta); if (may_be_null) *mem_reg = saved_reg; @@ -7157,7 +7204,7 @@ enum { * env->cur_state->active_locks remembers which map value element or allocated * object got locked and clears it after bpf_spin_unlock. */ -static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int regno, int flags) +static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int flags) { bool is_lock = flags & PROCESS_SPIN_LOCK, is_res_lock = flags & PROCESS_RES_LOCK; const char *lock_str = is_res_lock ? "bpf_res_spin" : "bpf_spin"; @@ -7173,8 +7220,8 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state if (!is_const) { verbose(env, - "R%d doesn't have constant offset. %s_lock has to be at the constant offset\n", - regno, lock_str); + "%s doesn't have constant offset. %s_lock has to be at the constant offset\n", + reg_arg_name(env, argno), lock_str); return -EINVAL; } if (reg->type == PTR_TO_MAP_VALUE) { @@ -7273,7 +7320,7 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state } /* Check if @regno is a pointer to a specific field in a map value */ -static int check_map_field_pointer(struct bpf_verifier_env *env, struct bpf_reg_state *reg, u32 regno, +static int check_map_field_pointer(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, enum btf_field_type field_type, struct bpf_map_desc *map_desc) { @@ -7285,8 +7332,8 @@ static int check_map_field_pointer(struct bpf_verifier_env *env, struct bpf_reg_ if (!is_const) { verbose(env, - "R%d doesn't have constant offset. %s has to be at the constant offset\n", - regno, struct_name); + "%s doesn't have constant offset. %s has to be at the constant offset\n", + reg_arg_name(env, argno), struct_name); return -EINVAL; } if (!map->btf) { @@ -7326,26 +7373,26 @@ static int check_map_field_pointer(struct bpf_verifier_env *env, struct bpf_reg_ return 0; } -static int process_timer_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int regno, +static int process_timer_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, struct bpf_map_desc *map) { if (IS_ENABLED(CONFIG_PREEMPT_RT)) { verbose(env, "bpf_timer cannot be used for PREEMPT_RT.\n"); return -EOPNOTSUPP; } - return check_map_field_pointer(env, reg, regno, BPF_TIMER, map); + return check_map_field_pointer(env, reg, argno, BPF_TIMER, map); } -static int process_timer_helper(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int regno, +static int process_timer_helper(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, struct bpf_call_arg_meta *meta) { - return process_timer_func(env, reg, regno, &meta->map); + return process_timer_func(env, reg, argno, &meta->map); } -static int process_timer_kfunc(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int regno, +static int process_timer_kfunc(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, struct bpf_kfunc_call_arg_meta *meta) { - return process_timer_func(env, reg, regno, &meta->map); + return process_timer_func(env, reg, argno, &meta->map); } static int process_kptr_func(struct bpf_verifier_env *env, int regno, @@ -7410,15 +7457,15 @@ static int process_kptr_func(struct bpf_verifier_env *env, int regno, * use case. The second level is tracked using the upper bit of bpf_dynptr->size * and checked dynamically during runtime. */ -static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int regno, int insn_idx, +static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int insn_idx, enum bpf_arg_type arg_type, int clone_ref_obj_id) { int err; if (reg->type != PTR_TO_STACK && reg->type != CONST_PTR_TO_DYNPTR) { verbose(env, - "arg#%d expected pointer to stack or const struct bpf_dynptr\n", - regno - 1); + "%s expected pointer to stack or const struct bpf_dynptr\n", + reg_arg_name(env, argno)); return -EINVAL; } @@ -7446,7 +7493,7 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat /* we write BPF_DW bits (8 bytes) at a time */ for (i = 0; i < BPF_DYNPTR_SIZE; i += 8) { - err = check_mem_access(env, insn_idx, reg, regno, + err = check_mem_access(env, insn_idx, reg, argno, i, BPF_DW, BPF_WRITE, -1, false, false); if (err) return err; @@ -7461,17 +7508,17 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat } if (!is_dynptr_reg_valid_init(env, reg)) { - verbose(env, - "Expected an initialized dynptr as arg #%d\n", - regno - 1); + verbose(env, "Expected an initialized dynptr as %s\n", + reg_arg_name(env, argno)); return -EINVAL; } /* Fold modifiers (in this case, OBJ_RELEASE) when checking expected type */ if (!is_dynptr_type_expected(env, reg, arg_type & ~OBJ_RELEASE)) { verbose(env, - "Expected a dynptr of type %s as arg #%d\n", - dynptr_type_str(arg_to_dynptr_type(arg_type)), regno - 1); + "Expected a dynptr of type %s as %s\n", + dynptr_type_str(arg_to_dynptr_type(arg_type)), + reg_arg_name(env, argno)); return -EINVAL; } @@ -7516,14 +7563,16 @@ static bool is_kfunc_arg_iter(struct bpf_kfunc_call_arg_meta *meta, int arg_idx, return btf_param_match_suffix(meta->btf, arg, "__iter"); } -static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int regno, int insn_idx, +static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int insn_idx, struct bpf_kfunc_call_arg_meta *meta) { const struct btf_type *t; + u32 arg_idx = arg_from_argno(argno) - 1; int spi, err, i, nr_slots, btf_id; if (reg->type != PTR_TO_STACK) { - verbose(env, "arg#%d expected pointer to an iterator on stack\n", regno - 1); + verbose(env, "%s expected pointer to an iterator on stack\n", + reg_arg_name(env, argno)); return -EINVAL; } @@ -7533,9 +7582,10 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * * to any kfunc, if arg has "__iter" suffix, we need to be a bit more * conservative here. */ - btf_id = btf_check_iter_arg(meta->btf, meta->func_proto, regno - 1); + btf_id = btf_check_iter_arg(meta->btf, meta->func_proto, arg_idx); if (btf_id < 0) { - verbose(env, "expected valid iter pointer as arg #%d\n", regno - 1); + verbose(env, "expected valid iter pointer as %s\n", + reg_arg_name(env, argno)); return -EINVAL; } t = btf_type_by_id(meta->btf, btf_id); @@ -7544,13 +7594,13 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * if (is_iter_new_kfunc(meta)) { /* bpf_iter__new() expects pointer to uninit iter state */ if (!is_iter_reg_valid_uninit(env, reg, nr_slots)) { - verbose(env, "expected uninitialized iter_%s as arg #%d\n", - iter_type_str(meta->btf, btf_id), regno - 1); + verbose(env, "expected uninitialized iter_%s as %s\n", + iter_type_str(meta->btf, btf_id), reg_arg_name(env, argno)); return -EINVAL; } for (i = 0; i < nr_slots * 8; i += BPF_REG_SIZE) { - err = check_mem_access(env, insn_idx, reg, regno, + err = check_mem_access(env, insn_idx, reg, argno, i, BPF_DW, BPF_WRITE, -1, false, false); if (err) return err; @@ -7568,8 +7618,8 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * case 0: break; case -EINVAL: - verbose(env, "expected an initialized iter_%s as arg #%d\n", - iter_type_str(meta->btf, btf_id), regno - 1); + verbose(env, "expected an initialized iter_%s as %s\n", + iter_type_str(meta->btf, btf_id), reg_arg_name(env, argno)); return err; case -EPROTO: verbose(env, "expected an RCU CS when using %s\n", meta->func_name); @@ -7989,7 +8039,7 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { [ARG_PTR_TO_DYNPTR] = &dynptr_types, }; -static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg, u32 regno, +static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, enum bpf_arg_type arg_type, const u32 *arg_btf_id, struct bpf_call_arg_meta *meta) @@ -8024,7 +8074,7 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re type &= ~DYNPTR_TYPE_FLAG_MASK; /* Local kptr types are allowed as the source argument of bpf_kptr_xchg */ - if (meta->func_id == BPF_FUNC_kptr_xchg && type_is_alloc(type) && regno == BPF_REG_2) { + if (meta->func_id == BPF_FUNC_kptr_xchg && type_is_alloc(type) && reg_from_argno(argno) == BPF_REG_2) { type &= ~MEM_ALLOC; type &= ~MEM_PERCPU; } @@ -8038,7 +8088,7 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re goto found; } - verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type)); + verbose(env, "%s type=%s expected=", reg_arg_name(env, argno), reg_type_str(env, reg->type)); for (j = 0; j + 1 < i; j++) verbose(env, "%s, ", reg_type_str(env, compatible->types[j])); verbose(env, "%s\n", reg_type_str(env, compatible->types[j])); @@ -8051,9 +8101,9 @@ found: if (compatible == &mem_types) { if (!(arg_type & MEM_RDONLY)) { verbose(env, - "%s() may write into memory pointed by R%d type=%s\n", + "%s() may write into memory pointed by %s type=%s\n", func_id_name(meta->func_id), - regno, reg_type_str(env, reg->type)); + reg_arg_name(env, argno), reg_type_str(env, reg->type)); return -EACCES; } return 0; @@ -8076,7 +8126,8 @@ found: if (type_may_be_null(reg->type) && (!type_may_be_null(arg_type) || arg_type_is_release(arg_type))) { - verbose(env, "Possibly NULL pointer passed to helper arg%d\n", regno); + verbose(env, "Possibly NULL pointer passed to helper %s\n", + reg_arg_name(env, argno)); return -EACCES; } @@ -8089,25 +8140,26 @@ found: } if (meta->func_id == BPF_FUNC_kptr_xchg) { - if (map_kptr_match_type(env, meta->kptr_field, reg, regno)) + if (map_kptr_match_type(env, meta->kptr_field, reg, reg_from_argno(argno))) return -EACCES; } else { if (arg_btf_id == BPF_PTR_POISON) { verbose(env, "verifier internal error:"); - verbose(env, "R%d has non-overwritten BPF_PTR_POISON type\n", - regno); + verbose(env, "%s has non-overwritten BPF_PTR_POISON type\n", + reg_arg_name(env, argno)); return -EACCES; } - err = __check_ptr_off_reg(env, reg, regno, true); + err = __check_ptr_off_reg(env, reg, argno, true); if (err) return err; if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->var_off.value, btf_vmlinux, *arg_btf_id, strict_type_match)) { - verbose(env, "R%d is of type %s but %s is expected\n", - regno, btf_type_name(reg->btf, reg->btf_id), + verbose(env, "%s is of type %s but %s is expected\n", + reg_arg_name(env, argno), + btf_type_name(reg->btf, reg->btf_id), btf_type_name(btf_vmlinux, *arg_btf_id)); return -EACCES; } @@ -8124,8 +8176,11 @@ found: return -EFAULT; } /* Check if local kptr in src arg matches kptr in dst arg */ - if (meta->func_id == BPF_FUNC_kptr_xchg && regno == BPF_REG_2) { - if (map_kptr_match_type(env, meta->kptr_field, reg, regno)) + if (meta->func_id == BPF_FUNC_kptr_xchg) { + int regno = reg_from_argno(argno); + + if (regno == BPF_REG_2 && + map_kptr_match_type(env, meta->kptr_field, reg, regno)) return -EACCES; } break; @@ -8159,7 +8214,7 @@ reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields) } static int check_func_arg_reg_off(struct bpf_verifier_env *env, - const struct bpf_reg_state *reg, int regno, + const struct bpf_reg_state *reg, argno_t argno, enum bpf_arg_type arg_type) { u32 type = reg->type; @@ -8185,8 +8240,8 @@ static int check_func_arg_reg_off(struct bpf_verifier_env *env, * to give the user a better error message. */ if (!tnum_is_const(reg->var_off) || reg->var_off.value != 0) { - verbose(env, "R%d must have zero offset when passed to release func or trusted arg to kfunc\n", - regno); + verbose(env, "%s must have zero offset when passed to release func or trusted arg to kfunc\n", + reg_arg_name(env, argno)); return -EINVAL; } } @@ -8222,7 +8277,7 @@ static int check_func_arg_reg_off(struct bpf_verifier_env *env, * cases. var_off always must be 0 for PTR_TO_BTF_ID, hence we * still need to do checks instead of returning. */ - return __check_ptr_off_reg(env, reg, regno, true); + return __check_ptr_off_reg(env, reg, argno, true); case PTR_TO_CTX: /* * Allow fixed and variable offsets for syscall context, but @@ -8234,7 +8289,7 @@ static int check_func_arg_reg_off(struct bpf_verifier_env *env, return 0; fallthrough; default: - return __check_ptr_off_reg(env, reg, regno, false); + return __check_ptr_off_reg(env, reg, argno, false); } } @@ -8304,8 +8359,8 @@ static enum bpf_dynptr_type dynptr_get_type(struct bpf_verifier_env *env, return state->stack[spi].spilled_ptr.dynptr.type; } -static int check_reg_const_str(struct bpf_verifier_env *env, - struct bpf_reg_state *reg, u32 regno) +static int check_arg_const_str(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, argno_t argno) { struct bpf_map *map = reg->map_ptr; int err; @@ -8317,17 +8372,18 @@ static int check_reg_const_str(struct bpf_verifier_env *env, return -EINVAL; if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY) { - verbose(env, "R%d points to insn_array map which cannot be used as const string\n", regno); + verbose(env, "%s points to insn_array map which cannot be used as const string\n", + reg_arg_name(env, argno)); return -EACCES; } if (!bpf_map_is_rdonly(map)) { - verbose(env, "R%d does not point to a readonly map'\n", regno); + verbose(env, "%s does not point to a readonly map'\n", reg_arg_name(env, argno)); return -EACCES; } if (!tnum_is_const(reg->var_off)) { - verbose(env, "R%d is not a constant address'\n", regno); + verbose(env, "%s is not a constant address'\n", reg_arg_name(env, argno)); return -EACCES; } @@ -8336,7 +8392,7 @@ static int check_reg_const_str(struct bpf_verifier_env *env, return -EACCES; } - err = check_map_access(env, reg, regno, 0, + err = check_map_access(env, reg, argno, 0, map->value_size - reg->var_off.value, false, ACCESS_HELPER); if (err) @@ -8472,11 +8528,11 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK) arg_btf_id = fn->arg_btf_id[arg]; - err = check_reg_type(env, reg, regno, arg_type, arg_btf_id, meta); + err = check_reg_type(env, reg, argno_from_reg(regno), arg_type, arg_btf_id, meta); if (err) return err; - err = check_func_arg_reg_off(env, reg, regno, arg_type); + err = check_func_arg_reg_off(env, reg, argno_from_reg(regno), arg_type); if (err) return err; @@ -8565,7 +8621,7 @@ skip_type_check: return -EFAULT; } key_size = meta->map.ptr->key_size; - err = check_helper_mem_access(env, reg, regno, key_size, BPF_READ, false, NULL); + err = check_helper_mem_access(env, reg, argno_from_reg(regno), key_size, BPF_READ, false, NULL); if (err) return err; if (can_elide_value_nullness(meta->map.ptr->map_type)) { @@ -8592,7 +8648,7 @@ skip_type_check: return -EFAULT; } meta->raw_mode = arg_type & MEM_UNINIT; - err = check_helper_mem_access(env, reg, regno, meta->map.ptr->value_size, + err = check_helper_mem_access(env, reg, argno_from_reg(regno), meta->map.ptr->value_size, arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, false, meta); break; @@ -8610,11 +8666,11 @@ skip_type_check: return -EACCES; } if (meta->func_id == BPF_FUNC_spin_lock) { - err = process_spin_lock(env, reg, regno, PROCESS_SPIN_LOCK); + err = process_spin_lock(env, reg, argno_from_reg(regno), PROCESS_SPIN_LOCK); if (err) return err; } else if (meta->func_id == BPF_FUNC_spin_unlock) { - err = process_spin_lock(env, reg, regno, 0); + err = process_spin_lock(env, reg, argno_from_reg(regno), 0); if (err) return err; } else { @@ -8623,7 +8679,7 @@ skip_type_check: } break; case ARG_PTR_TO_TIMER: - err = process_timer_helper(env, reg, regno, meta); + err = process_timer_helper(env, reg, argno_from_reg(regno), meta); if (err) return err; break; @@ -8636,7 +8692,7 @@ skip_type_check: */ meta->raw_mode = arg_type & MEM_UNINIT; if (arg_type & MEM_FIXED_SIZE) { - err = check_helper_mem_access(env, reg, regno, fn->arg_size[arg], + err = check_helper_mem_access(env, reg, argno_from_reg(regno), fn->arg_size[arg], arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, false, meta); if (err) @@ -8646,19 +8702,21 @@ skip_type_check: } break; case ARG_CONST_SIZE: - err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, regno - 1, regno, + err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, argno_from_reg(regno - 1), + argno_from_reg(regno), fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, false, meta); break; case ARG_CONST_SIZE_OR_ZERO: - err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, regno - 1, regno, + err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, argno_from_reg(regno - 1), + argno_from_reg(regno), fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, true, meta); break; case ARG_PTR_TO_DYNPTR: - err = process_dynptr_func(env, reg, regno, insn_idx, arg_type, 0); + err = process_dynptr_func(env, reg, argno_from_reg(regno), insn_idx, arg_type, 0); if (err) return err; break; @@ -8675,7 +8733,7 @@ skip_type_check: break; case ARG_PTR_TO_CONST_STR: { - err = check_reg_const_str(env, reg, regno); + err = check_arg_const_str(env, reg, argno_from_reg(regno)); if (err) return err; break; @@ -9264,13 +9322,14 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, * verifier sees. */ for (i = 0; i < sub->arg_cnt; i++) { + argno_t argno = argno_from_arg(i + 1); u32 regno = i + 1; struct bpf_reg_state *reg = ®s[regno]; struct bpf_subprog_arg_info *arg = &sub->args[i]; if (arg->arg_type == ARG_ANYTHING) { if (reg->type != SCALAR_VALUE) { - bpf_log(log, "R%d is not a scalar\n", regno); + bpf_log(log, "%s is not a scalar\n", reg_arg_name(env, argno)); return -EINVAL; } } else if (arg->arg_type & PTR_UNTRUSTED) { @@ -9280,24 +9339,26 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, * invalid memory access. */ } else if (arg->arg_type == ARG_PTR_TO_CTX) { - ret = check_func_arg_reg_off(env, reg, regno, ARG_PTR_TO_CTX); + ret = check_func_arg_reg_off(env, reg, argno, ARG_PTR_TO_CTX); if (ret < 0) return ret; /* If function expects ctx type in BTF check that caller * is passing PTR_TO_CTX. */ if (reg->type != PTR_TO_CTX) { - bpf_log(log, "arg#%d expects pointer to ctx\n", i); + bpf_log(log, "%s expects pointer to ctx\n", + reg_arg_name(env, argno)); return -EINVAL; } } else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) { - ret = check_func_arg_reg_off(env, reg, regno, ARG_DONTCARE); + ret = check_func_arg_reg_off(env, reg, argno, ARG_DONTCARE); if (ret < 0) return ret; - if (check_mem_reg(env, reg, regno, arg->mem_size)) + if (check_mem_reg(env, reg, argno, arg->mem_size)) return -EINVAL; if (!(arg->arg_type & PTR_MAYBE_NULL) && (reg->type & PTR_MAYBE_NULL)) { - bpf_log(log, "arg#%d is expected to be non-NULL\n", i); + bpf_log(log, "%s is expected to be non-NULL\n", + reg_arg_name(env, argno)); return -EINVAL; } } else if (base_type(arg->arg_type) == ARG_PTR_TO_ARENA) { @@ -9309,15 +9370,16 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, * run-time debug nightmare. */ if (reg->type != PTR_TO_ARENA && reg->type != SCALAR_VALUE) { - bpf_log(log, "R%d is not a pointer to arena or scalar.\n", regno); + bpf_log(log, "%s is not a pointer to arena or scalar.\n", + reg_arg_name(env, argno)); return -EINVAL; } } else if (arg->arg_type == ARG_PTR_TO_DYNPTR) { - ret = check_func_arg_reg_off(env, reg, regno, ARG_PTR_TO_DYNPTR); + ret = check_func_arg_reg_off(env, reg, argno, ARG_PTR_TO_DYNPTR); if (ret) return ret; - ret = process_dynptr_func(env, reg, regno, -1, arg->arg_type, 0); + ret = process_dynptr_func(env, reg, argno, -1, arg->arg_type, 0); if (ret) return ret; } else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) { @@ -9328,12 +9390,13 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, continue; memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */ - err = check_reg_type(env, reg, regno, arg->arg_type, &arg->btf_id, &meta); - err = err ?: check_func_arg_reg_off(env, reg, regno, arg->arg_type); + err = check_reg_type(env, reg, argno, arg->arg_type, &arg->btf_id, &meta); + err = err ?: check_func_arg_reg_off(env, reg, argno, arg->arg_type); if (err) return err; } else { - verifier_bug(env, "unrecognized arg#%d type %d", i, arg->arg_type); + verifier_bug(env, "unrecognized %s type %d", + reg_arg_name(env, argno), arg->arg_type); return -EFAULT; } } @@ -10292,7 +10355,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn * is inferred from register state. */ for (i = 0; i < meta.access_size; i++) { - err = check_mem_access(env, insn_idx, regs + meta.regno, meta.regno, i, BPF_B, + err = check_mem_access(env, insn_idx, regs + meta.regno, argno_from_reg(meta.regno), i, BPF_B, BPF_WRITE, -1, false, false); if (err) return err; @@ -11301,7 +11364,7 @@ get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta, const struct btf_type *t, const struct btf_type *ref_t, const char *ref_tname, const struct btf_param *args, - int arg, int nargs, struct bpf_reg_state *reg) + int arg, int nargs, argno_t argno, struct bpf_reg_state *reg) { u32 regno = arg + 1; struct bpf_reg_state *regs = cur_regs(env); @@ -11376,8 +11439,9 @@ get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, if ((base_type(reg->type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(reg->type)])) { if (!btf_type_is_struct(ref_t)) { - verbose(env, "kernel function %s args#%d pointer type %s %s is not supported\n", - meta->func_name, arg, btf_type_str(ref_t), ref_tname); + verbose(env, "kernel function %s %s pointer type %s %s is not supported\n", + meta->func_name, reg_arg_name(env, argno), + btf_type_str(ref_t), ref_tname); return -EINVAL; } return KF_ARG_PTR_TO_BTF_ID; @@ -11393,8 +11457,9 @@ get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, */ if (!btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0) && (arg_mem_size ? !btf_type_is_void(ref_t) : 1)) { - verbose(env, "arg#%d pointer type %s %s must point to %sscalar, or struct with scalar\n", - arg, btf_type_str(ref_t), ref_tname, arg_mem_size ? "void, " : ""); + verbose(env, "%s pointer type %s %s must point to %sscalar, or struct with scalar\n", + reg_arg_name(env, argno), + btf_type_str(ref_t), ref_tname, arg_mem_size ? "void, " : ""); return -EINVAL; } return arg_mem_size ? KF_ARG_PTR_TO_MEM_SIZE : KF_ARG_PTR_TO_MEM; @@ -11405,7 +11470,7 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, const struct btf_type *ref_t, const char *ref_tname, u32 ref_id, struct bpf_kfunc_call_arg_meta *meta, - int arg) + int arg, argno_t argno) { const struct btf_type *reg_ref_t; bool strict_type_match = false; @@ -11463,15 +11528,16 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, */ taking_projection = btf_is_projection_of(ref_tname, reg_ref_tname); if (!taking_projection && !struct_same) { - verbose(env, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\n", - meta->func_name, arg, btf_type_str(ref_t), ref_tname, arg + 1, + verbose(env, "kernel function %s %s expected pointer to %s %s but %s has a pointer to %s %s\n", + meta->func_name, reg_arg_name(env, argno), + btf_type_str(ref_t), ref_tname, reg_arg_name(env, argno), btf_type_str(reg_ref_t), reg_ref_tname); return -EINVAL; } return 0; } -static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int regno, +static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, struct bpf_kfunc_call_arg_meta *meta) { int err, kfunc_class = IRQ_NATIVE_KFUNC; @@ -11494,11 +11560,13 @@ static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state * if (irq_save) { if (!is_irq_flag_reg_valid_uninit(env, reg)) { - verbose(env, "expected uninitialized irq flag as arg#%d\n", regno - 1); + verbose(env, "expected uninitialized irq flag as %s\n", + reg_arg_name(env, argno)); return -EINVAL; } - err = check_mem_access(env, env->insn_idx, reg, regno, 0, BPF_DW, BPF_WRITE, -1, false, false); + err = check_mem_access(env, env->insn_idx, reg, argno, 0, BPF_DW, + BPF_WRITE, -1, false, false); if (err) return err; @@ -11508,7 +11576,8 @@ static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state * } else { err = is_irq_flag_reg_valid_init(env, reg); if (err) { - verbose(env, "expected an initialized irq flag as arg#%d\n", regno - 1); + verbose(env, "expected an initialized irq flag as %s\n", + reg_arg_name(env, argno)); return err; } @@ -11799,7 +11868,7 @@ static bool check_kfunc_is_graph_node_api(struct bpf_verifier_env *env, static int __process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env, - struct bpf_reg_state *reg, u32 regno, + struct bpf_reg_state *reg, argno_t argno, struct bpf_kfunc_call_arg_meta *meta, enum btf_field_type head_field_type, struct btf_field **head_field) @@ -11820,8 +11889,8 @@ __process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env, head_type_name = btf_field_type_name(head_field_type); if (!tnum_is_const(reg->var_off)) { verbose(env, - "R%d doesn't have constant offset. %s has to be at the constant offset\n", - regno, head_type_name); + "%s doesn't have constant offset. %s has to be at the constant offset\n", + reg_arg_name(env, argno), head_type_name); return -EINVAL; } @@ -11849,24 +11918,24 @@ __process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env, } static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env, - struct bpf_reg_state *reg, u32 regno, + struct bpf_reg_state *reg, argno_t argno, struct bpf_kfunc_call_arg_meta *meta) { - return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_LIST_HEAD, + return __process_kf_arg_ptr_to_graph_root(env, reg, argno, meta, BPF_LIST_HEAD, &meta->arg_list_head.field); } static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env, - struct bpf_reg_state *reg, u32 regno, + struct bpf_reg_state *reg, argno_t argno, struct bpf_kfunc_call_arg_meta *meta) { - return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_RB_ROOT, + return __process_kf_arg_ptr_to_graph_root(env, reg, argno, meta, BPF_RB_ROOT, &meta->arg_rbtree_root.field); } static int __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env, - struct bpf_reg_state *reg, u32 regno, + struct bpf_reg_state *reg, argno_t argno, struct bpf_kfunc_call_arg_meta *meta, enum btf_field_type head_field_type, enum btf_field_type node_field_type, @@ -11888,8 +11957,8 @@ __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env, node_type_name = btf_field_type_name(node_field_type); if (!tnum_is_const(reg->var_off)) { verbose(env, - "R%d doesn't have constant offset. %s has to be at the constant offset\n", - regno, node_type_name); + "%s doesn't have constant offset. %s has to be at the constant offset\n", + reg_arg_name(env, argno), node_type_name); return -EINVAL; } @@ -11930,19 +11999,19 @@ __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env, } static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, - struct bpf_reg_state *reg, u32 regno, + struct bpf_reg_state *reg, argno_t argno, struct bpf_kfunc_call_arg_meta *meta) { - return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta, + return __process_kf_arg_ptr_to_graph_node(env, reg, argno, meta, BPF_LIST_HEAD, BPF_LIST_NODE, &meta->arg_list_head.field); } static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env, - struct bpf_reg_state *reg, u32 regno, + struct bpf_reg_state *reg, argno_t argno, struct bpf_kfunc_call_arg_meta *meta) { - return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta, + return __process_kf_arg_ptr_to_graph_node(env, reg, argno, meta, BPF_RB_ROOT, BPF_RB_NODE, &meta->arg_rbtree_root.field); } @@ -11994,6 +12063,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[i + 1]; const struct btf_type *t, *ref_t, *resolve_ret; enum bpf_arg_type arg_type = ARG_DONTCARE; + argno_t argno = argno_from_arg(i + 1); u32 regno = i + 1, ref_id, type_size; bool is_ret_buf_sz = false; int kf_arg_type; @@ -12016,7 +12086,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ if (btf_type_is_scalar(t)) { if (reg->type != SCALAR_VALUE) { - verbose(env, "R%d is not a scalar\n", regno); + verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); return -EINVAL; } @@ -12026,7 +12096,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ return -EFAULT; } if (!tnum_is_const(reg->var_off)) { - verbose(env, "R%d must be a known constant\n", regno); + verbose(env, "%s must be a known constant\n", + reg_arg_name(env, argno)); return -EINVAL; } ret = mark_chain_precision(env, regno); @@ -12048,7 +12119,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ } if (!tnum_is_const(reg->var_off)) { - verbose(env, "R%d is not a const\n", regno); + verbose(env, "%s is not a const\n", + reg_arg_name(env, argno)); return -EINVAL; } @@ -12061,20 +12133,22 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ } if (!btf_type_is_ptr(t)) { - verbose(env, "Unrecognized arg#%d type %s\n", i, btf_type_str(t)); + verbose(env, "Unrecognized %s type %s\n", + reg_arg_name(env, argno), btf_type_str(t)); return -EINVAL; } if ((bpf_register_is_null(reg) || type_may_be_null(reg->type)) && !is_kfunc_arg_nullable(meta->btf, &args[i])) { - verbose(env, "Possibly NULL pointer passed to trusted arg%d\n", i); + verbose(env, "Possibly NULL pointer passed to trusted %s\n", + reg_arg_name(env, argno)); return -EACCES; } if (reg->ref_obj_id) { if (is_kfunc_release(meta) && meta->ref_obj_id) { - verifier_bug(env, "more than one arg with ref_obj_id R%d %u %u", - regno, reg->ref_obj_id, + verifier_bug(env, "more than one arg with ref_obj_id %s %u %u", + reg_arg_name(env, argno), reg->ref_obj_id, meta->ref_obj_id); return -EFAULT; } @@ -12086,7 +12160,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); ref_tname = btf_name_by_offset(btf, ref_t->name_off); - kf_arg_type = get_kfunc_ptr_arg_type(env, meta, t, ref_t, ref_tname, args, i, nargs, reg); + kf_arg_type = get_kfunc_ptr_arg_type(env, meta, t, ref_t, ref_tname, args, i, nargs, argno, reg); if (kf_arg_type < 0) return kf_arg_type; @@ -12095,7 +12169,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ continue; case KF_ARG_PTR_TO_MAP: if (!reg->map_ptr) { - verbose(env, "pointer in R%d isn't map pointer\n", regno); + verbose(env, "pointer in %s isn't map pointer\n", + reg_arg_name(env, argno)); return -EINVAL; } if (meta->map.ptr && (reg->map_ptr->record->wq_off >= 0 || @@ -12133,11 +12208,13 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ case KF_ARG_PTR_TO_BTF_ID: if (!is_trusted_reg(reg)) { if (!is_kfunc_rcu(meta)) { - verbose(env, "R%d must be referenced or trusted\n", regno); + verbose(env, "%s must be referenced or trusted\n", + reg_arg_name(env, argno)); return -EINVAL; } if (!is_rcu_reg(reg)) { - verbose(env, "R%d must be a rcu pointer\n", regno); + verbose(env, "%s must be a rcu pointer\n", + reg_arg_name(env, argno)); return -EINVAL; } } @@ -12169,15 +12246,15 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ if (is_kfunc_release(meta) && reg->ref_obj_id) arg_type |= OBJ_RELEASE; - ret = check_func_arg_reg_off(env, reg, regno, arg_type); + ret = check_func_arg_reg_off(env, reg, argno, arg_type); if (ret < 0) return ret; switch (kf_arg_type) { case KF_ARG_PTR_TO_CTX: if (reg->type != PTR_TO_CTX) { - verbose(env, "arg#%d expected pointer to ctx, but got %s\n", - i, reg_type_str(env, reg->type)); + verbose(env, "%s expected pointer to ctx, but got %s\n", + reg_arg_name(env, argno), reg_type_str(env, reg->type)); return -EINVAL; } @@ -12191,16 +12268,19 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ case KF_ARG_PTR_TO_ALLOC_BTF_ID: if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC)) { if (!is_bpf_obj_drop_kfunc(meta->func_id)) { - verbose(env, "arg#%d expected for bpf_obj_drop()\n", i); + verbose(env, "%s expected for bpf_obj_drop()\n", + reg_arg_name(env, argno)); return -EINVAL; } } else if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU)) { if (!is_bpf_percpu_obj_drop_kfunc(meta->func_id)) { - verbose(env, "arg#%d expected for bpf_percpu_obj_drop()\n", i); + verbose(env, "%s expected for bpf_percpu_obj_drop()\n", + reg_arg_name(env, argno)); return -EINVAL; } } else { - verbose(env, "arg#%d expected pointer to allocated object\n", i); + verbose(env, "%s expected pointer to allocated object\n", + reg_arg_name(env, argno)); return -EINVAL; } if (!reg->ref_obj_id) { @@ -12248,7 +12328,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ } } - ret = process_dynptr_func(env, reg, regno, insn_idx, dynptr_arg_type, clone_ref_obj_id); + ret = process_dynptr_func(env, reg, argno, insn_idx, + dynptr_arg_type, clone_ref_obj_id); if (ret < 0) return ret; @@ -12273,55 +12354,59 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ return -EINVAL; } } - ret = process_iter_arg(env, reg, regno, insn_idx, meta); + ret = process_iter_arg(env, reg, argno, insn_idx, meta); if (ret < 0) return ret; break; case KF_ARG_PTR_TO_LIST_HEAD: if (reg->type != PTR_TO_MAP_VALUE && reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { - verbose(env, "arg#%d expected pointer to map value or allocated object\n", i); + verbose(env, "%s expected pointer to map value or allocated object\n", + reg_arg_name(env, argno)); return -EINVAL; } if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg->ref_obj_id) { verbose(env, "allocated object must be referenced\n"); return -EINVAL; } - ret = process_kf_arg_ptr_to_list_head(env, reg, regno, meta); + ret = process_kf_arg_ptr_to_list_head(env, reg, argno, meta); if (ret < 0) return ret; break; case KF_ARG_PTR_TO_RB_ROOT: if (reg->type != PTR_TO_MAP_VALUE && reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { - verbose(env, "arg#%d expected pointer to map value or allocated object\n", i); + verbose(env, "%s expected pointer to map value or allocated object\n", + reg_arg_name(env, argno)); return -EINVAL; } if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg->ref_obj_id) { verbose(env, "allocated object must be referenced\n"); return -EINVAL; } - ret = process_kf_arg_ptr_to_rbtree_root(env, reg, regno, meta); + ret = process_kf_arg_ptr_to_rbtree_root(env, reg, argno, meta); if (ret < 0) return ret; break; case KF_ARG_PTR_TO_LIST_NODE: if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { - verbose(env, "arg#%d expected pointer to allocated object\n", i); + verbose(env, "%s expected pointer to allocated object\n", + reg_arg_name(env, argno)); return -EINVAL; } if (!reg->ref_obj_id) { verbose(env, "allocated object must be referenced\n"); return -EINVAL; } - ret = process_kf_arg_ptr_to_list_node(env, reg, regno, meta); + ret = process_kf_arg_ptr_to_list_node(env, reg, argno, meta); if (ret < 0) return ret; break; case KF_ARG_PTR_TO_RB_NODE: if (is_bpf_rbtree_add_kfunc(meta->func_id)) { if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { - verbose(env, "arg#%d expected pointer to allocated object\n", i); + verbose(env, "%s expected pointer to allocated object\n", + reg_arg_name(env, argno)); return -EINVAL; } if (!reg->ref_obj_id) { @@ -12339,7 +12424,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ } } - ret = process_kf_arg_ptr_to_rbtree_node(env, reg, regno, meta); + ret = process_kf_arg_ptr_to_rbtree_node(env, reg, argno, meta); if (ret < 0) return ret; break; @@ -12354,24 +12439,26 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ if ((base_type(reg->type) != PTR_TO_BTF_ID || (bpf_type_has_unsafe_modifiers(reg->type) && !is_rcu_reg(reg))) && !reg2btf_ids[base_type(reg->type)]) { - verbose(env, "arg#%d is %s ", i, reg_type_str(env, reg->type)); + verbose(env, "%s is %s ", reg_arg_name(env, argno), + reg_type_str(env, reg->type)); verbose(env, "expected %s or socket\n", reg_type_str(env, base_type(reg->type) | (type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS))); return -EINVAL; } - ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i); + ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i, argno); if (ret < 0) return ret; break; case KF_ARG_PTR_TO_MEM: resolve_ret = btf_resolve_size(btf, ref_t, &type_size); if (IS_ERR(resolve_ret)) { - verbose(env, "arg#%d reference type('%s %s') size cannot be determined: %ld\n", - i, btf_type_str(ref_t), ref_tname, PTR_ERR(resolve_ret)); + verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n", + reg_arg_name(env, argno), btf_type_str(ref_t), + ref_tname, PTR_ERR(resolve_ret)); return -EINVAL; } - ret = check_mem_reg(env, reg, regno, type_size); + ret = check_mem_reg(env, reg, argno, type_size); if (ret < 0) return ret; break; @@ -12381,11 +12468,15 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ const struct btf_param *buff_arg = &args[i]; struct bpf_reg_state *size_reg = ®s[regno + 1]; const struct btf_param *size_arg = &args[i + 1]; + argno_t next_argno = argno_from_arg(i + 2); if (!bpf_register_is_null(buff_reg) || !is_kfunc_arg_nullable(meta->btf, buff_arg)) { - ret = check_kfunc_mem_size_reg(env, buff_reg, size_reg, regno, regno + 1); + ret = check_kfunc_mem_size_reg(env, buff_reg, size_reg, + argno, next_argno); if (ret < 0) { - verbose(env, "arg#%d arg#%d memory, len pair leads to invalid memory access\n", i, i + 1); + verbose(env, "%s and ", reg_arg_name(env, argno)); + verbose(env, "%s memory, len pair leads to invalid memory access\n", + reg_arg_name(env, next_argno)); return ret; } } @@ -12396,7 +12487,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ return -EFAULT; } if (!tnum_is_const(size_reg->var_off)) { - verbose(env, "R%d must be a known constant\n", regno + 1); + verbose(env, "%s must be a known constant\n", + reg_arg_name(env, next_argno)); return -EINVAL; } meta->arg_constant.found = true; @@ -12409,14 +12501,15 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ } case KF_ARG_PTR_TO_CALLBACK: if (reg->type != PTR_TO_FUNC) { - verbose(env, "arg%d expected pointer to func\n", i); + verbose(env, "%s expected pointer to func\n", reg_arg_name(env, argno)); return -EINVAL; } meta->subprogno = reg->subprogno; break; case KF_ARG_PTR_TO_REFCOUNTED_KPTR: if (!type_is_ptr_alloc_obj(reg->type)) { - verbose(env, "arg#%d is neither owning or non-owning ref\n", i); + verbose(env, "%s is neither owning or non-owning ref\n", + reg_arg_name(env, argno)); return -EINVAL; } if (!type_is_non_owning_ref(reg->type)) @@ -12429,7 +12522,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ } if (rec->refcount_off < 0) { - verbose(env, "arg#%d doesn't point to a type with bpf_refcount field\n", i); + verbose(env, "%s doesn't point to a type with bpf_refcount field\n", + reg_arg_name(env, argno)); return -EINVAL; } @@ -12438,46 +12532,51 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ break; case KF_ARG_PTR_TO_CONST_STR: if (reg->type != PTR_TO_MAP_VALUE) { - verbose(env, "arg#%d doesn't point to a const string\n", i); + verbose(env, "%s doesn't point to a const string\n", + reg_arg_name(env, argno)); return -EINVAL; } - ret = check_reg_const_str(env, reg, regno); + ret = check_arg_const_str(env, reg, argno); if (ret) return ret; break; case KF_ARG_PTR_TO_WORKQUEUE: if (reg->type != PTR_TO_MAP_VALUE) { - verbose(env, "arg#%d doesn't point to a map value\n", i); + verbose(env, "%s doesn't point to a map value\n", + reg_arg_name(env, argno)); return -EINVAL; } - ret = check_map_field_pointer(env, reg, regno, BPF_WORKQUEUE, &meta->map); + ret = check_map_field_pointer(env, reg, argno, BPF_WORKQUEUE, &meta->map); if (ret < 0) return ret; break; case KF_ARG_PTR_TO_TIMER: if (reg->type != PTR_TO_MAP_VALUE) { - verbose(env, "arg#%d doesn't point to a map value\n", i); + verbose(env, "%s doesn't point to a map value\n", + reg_arg_name(env, argno)); return -EINVAL; } - ret = process_timer_kfunc(env, reg, regno, meta); + ret = process_timer_kfunc(env, reg, argno, meta); if (ret < 0) return ret; break; case KF_ARG_PTR_TO_TASK_WORK: if (reg->type != PTR_TO_MAP_VALUE) { - verbose(env, "arg#%d doesn't point to a map value\n", i); + verbose(env, "%s doesn't point to a map value\n", + reg_arg_name(env, argno)); return -EINVAL; } - ret = check_map_field_pointer(env, reg, regno, BPF_TASK_WORK, &meta->map); + ret = check_map_field_pointer(env, reg, argno, BPF_TASK_WORK, &meta->map); if (ret < 0) return ret; break; case KF_ARG_PTR_TO_IRQ_FLAG: if (reg->type != PTR_TO_STACK) { - verbose(env, "arg#%d doesn't point to an irq flag on stack\n", i); + verbose(env, "%s doesn't point to an irq flag on stack\n", + reg_arg_name(env, argno)); return -EINVAL; } - ret = process_irq_flag(env, reg, regno, meta); + ret = process_irq_flag(env, reg, argno, meta); if (ret < 0) return ret; break; @@ -12486,7 +12585,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ int flags = PROCESS_RES_LOCK; if (reg->type != PTR_TO_MAP_VALUE && reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { - verbose(env, "arg#%d doesn't point to map value or allocated object\n", i); + verbose(env, "%s doesn't point to map value or allocated object\n", + reg_arg_name(env, argno)); return -EINVAL; } @@ -12498,7 +12598,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave] || meta->func_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]) flags |= PROCESS_LOCK_IRQ; - ret = process_spin_lock(env, reg, regno, flags); + ret = process_spin_lock(env, reg, argno, flags); if (ret < 0) return ret; break; @@ -13649,7 +13749,7 @@ static int sanitize_check_bounds(struct bpf_verifier_env *env, return -EACCES; break; case PTR_TO_MAP_VALUE: - if (check_map_access(env, dst_reg, dst, 0, 1, false, ACCESS_HELPER)) { + if (check_map_access(env, dst_reg, argno_from_reg(dst), 0, 1, false, ACCESS_HELPER)) { verbose(env, "R%d pointer arithmetic of map value goes out of range, " "prohibited for !root\n", dst); return -EACCES; @@ -16831,7 +16931,7 @@ static int check_return_code(struct bpf_verifier_env *env, int regno, const char prog->aux->attach_func_proto->type, NULL); if (ret_type && ret_type == reg_type && reg->ref_obj_id) - return __check_ptr_off_reg(env, reg, regno, false); + return __check_ptr_off_reg(env, reg, argno_from_reg(regno), false); } /* eBPF calling convention is such that R0 is used @@ -17535,7 +17635,7 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) dst_reg_type = cur_regs(env)[insn->dst_reg].type; - err = check_mem_access(env, env->insn_idx, cur_regs(env) + insn->dst_reg, insn->dst_reg, + err = check_mem_access(env, env->insn_idx, cur_regs(env) + insn->dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_WRITE, -1, false, false); if (err) @@ -18714,7 +18814,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) mark_reg_unknown(env, regs, i); } else { verifier_bug(env, "unhandled arg#%d type %d", - i - BPF_REG_1, arg->arg_type); + i - BPF_REG_1 + 1, arg->arg_type); ret = -EFAULT; goto out; } -- cgit v1.2.3 From 246ad6e5ee259669692bdb7fb353e8c5d5bba628 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Wed, 22 Apr 2026 20:35:06 -0700 Subject: bpf: Introduce bpf register BPF_REG_PARAMS Introduce BPF_REG_PARAMS as a dedicated BPF register for stack argument accesses. It occupies the BPF register number 11 (R11), which is used as the base pointer for the stack argument area, keeping it separate from the R10-based (BPF_REG_FP) program stack. The kernel-internal hidden register BPF_REG_AX previously occupied slot 11 (MAX_BPF_REG). With BPF_REG_PARAMS taking that slot, BPF_REG_AX moves to slot 12 and MAX_BPF_EXT_REG increases accordingly. Acked-by: Puranjay Mohan Acked-by: Kumar Kartikeya Dwivedi Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260423033506.2542005-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 8b018ff48875..ae10b9ca018d 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -1299,8 +1299,8 @@ static int bpf_jit_blind_insn(const struct bpf_insn *from, u32 imm_rnd = get_random_u32(); s16 off; - BUILD_BUG_ON(BPF_REG_AX + 1 != MAX_BPF_JIT_REG); - BUILD_BUG_ON(MAX_BPF_REG + 1 != MAX_BPF_JIT_REG); + BUILD_BUG_ON(BPF_REG_PARAMS + 2 != MAX_BPF_JIT_REG); + BUILD_BUG_ON(BPF_REG_AX + 1 != MAX_BPF_JIT_REG); /* Constraints on AX register: * -- cgit v1.2.3 From 5bbb3453e8f35e1eacde30459e1fa7625c8d6d21 Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Wed, 22 Apr 2026 10:17:09 +0800 Subject: cgroup/rdma: refactor resource parsing with match_table_t/match_token() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the hand-rolled strsep/strcmp/match_string parsing in rdmacg_resource_set_max() with a match_table_t and match_token() pattern, following the convention used by user_proactive_reclaim() and ioc_cost_model_write(). The old strncmp(value, RDMACG_MAX_STR, strlen(value)) also had two bugs that are fixed by this refactor: - It matched "ma" as "max" because strncmp only compared the shorter strlen(value) bytes. - It silently accepted "hca_handle=" (empty value) as "max" because strncmp with n=0 always returns 0. The match_token() approach also robustly handles extra whitespace in the input by splitting on " \t\n" and skipping empty tokens. Suggested-by: "Michal Koutný" Signed-off-by: Tao Cui Signed-off-by: Tejun Heo --- kernel/cgroup/rdma.c | 116 +++++++++++++++++++++++++-------------------------- 1 file changed, 57 insertions(+), 59 deletions(-) (limited to 'kernel') diff --git a/kernel/cgroup/rdma.c b/kernel/cgroup/rdma.c index 4fdab4cf49e0..3df7c38ce481 100644 --- a/kernel/cgroup/rdma.c +++ b/kernel/cgroup/rdma.c @@ -9,6 +9,7 @@ */ #include +#include #include #include #include @@ -17,6 +18,22 @@ #define RDMACG_MAX_STR "max" +enum rdmacg_limit_tokens { + RDMACG_HCA_HANDLE_VAL, + RDMACG_HCA_HANDLE_MAX, + RDMACG_HCA_OBJECT_VAL, + RDMACG_HCA_OBJECT_MAX, + NR_RDMACG_LIMIT_TOKENS, +}; + +static const match_table_t rdmacg_limit_tokens = { + { RDMACG_HCA_HANDLE_VAL, "hca_handle=%d" }, + { RDMACG_HCA_HANDLE_MAX, "hca_handle=max" }, + { RDMACG_HCA_OBJECT_VAL, "hca_object=%d" }, + { RDMACG_HCA_OBJECT_MAX, "hca_object=max" }, + { NR_RDMACG_LIMIT_TOKENS, NULL }, +}; + /* * Protects list of resource pools maintained on per cgroup basis * and rdma device list. @@ -355,62 +372,6 @@ void rdmacg_unregister_device(struct rdmacg_device *device) } EXPORT_SYMBOL(rdmacg_unregister_device); -static int parse_resource(char *c, int *intval) -{ - substring_t argstr; - char *name, *value = c; - size_t len; - int ret, i; - - name = strsep(&value, "="); - if (!name || !value) - return -EINVAL; - - i = match_string(rdmacg_resource_names, RDMACG_RESOURCE_MAX, name); - if (i < 0) - return i; - - len = strlen(value); - - argstr.from = value; - argstr.to = value + len; - - ret = match_int(&argstr, intval); - if (ret >= 0) { - if (*intval < 0) - return -EINVAL; - return i; - } - if (strncmp(value, RDMACG_MAX_STR, len) == 0) { - *intval = S32_MAX; - return i; - } - return -EINVAL; -} - -static int rdmacg_parse_limits(char *options, - int *new_limits, unsigned long *enables) -{ - char *c; - int err = -EINVAL; - - /* parse resource options */ - while ((c = strsep(&options, " ")) != NULL) { - int index, intval; - - index = parse_resource(c, &intval); - if (index < 0) - goto err; - - new_limits[index] = intval; - *enables |= BIT(index); - } - return 0; - -err: - return err; -} - static struct rdmacg_device *rdmacg_get_device_locked(const char *name) { struct rdmacg_device *device; @@ -432,6 +393,7 @@ static ssize_t rdmacg_resource_set_max(struct kernfs_open_file *of, struct rdmacg_resource_pool *rpool; struct rdmacg_device *device; char *options = strstrip(buf); + char *p; int *new_limits; unsigned long enables = 0; int i = 0, ret = 0; @@ -449,9 +411,45 @@ static ssize_t rdmacg_resource_set_max(struct kernfs_open_file *of, goto err; } - ret = rdmacg_parse_limits(options, new_limits, &enables); - if (ret) - goto parse_err; + /* parse resource limit tokens */ + while ((p = strsep(&options, " \t\n"))) { + substring_t args[MAX_OPT_ARGS]; + int tok, intval; + + if (!*p) + continue; + + tok = match_token(p, rdmacg_limit_tokens, args); + switch (tok) { + case RDMACG_HCA_HANDLE_VAL: + if (match_int(&args[0], &intval) || intval < 0) { + ret = -EINVAL; + goto parse_err; + } + new_limits[RDMACG_RESOURCE_HCA_HANDLE] = intval; + enables |= BIT(RDMACG_RESOURCE_HCA_HANDLE); + break; + case RDMACG_HCA_HANDLE_MAX: + new_limits[RDMACG_RESOURCE_HCA_HANDLE] = S32_MAX; + enables |= BIT(RDMACG_RESOURCE_HCA_HANDLE); + break; + case RDMACG_HCA_OBJECT_VAL: + if (match_int(&args[0], &intval) || intval < 0) { + ret = -EINVAL; + goto parse_err; + } + new_limits[RDMACG_RESOURCE_HCA_OBJECT] = intval; + enables |= BIT(RDMACG_RESOURCE_HCA_OBJECT); + break; + case RDMACG_HCA_OBJECT_MAX: + new_limits[RDMACG_RESOURCE_HCA_OBJECT] = S32_MAX; + enables |= BIT(RDMACG_RESOURCE_HCA_OBJECT); + break; + default: + ret = -EINVAL; + goto parse_err; + } + } /* acquire lock to synchronize with hot plug devices */ mutex_lock(&rdmacg_mutex); -- cgit v1.2.3 From 256f0071f9b61ae5028f749449fd3fdad015889d Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 24 Apr 2026 15:52:42 -0700 Subject: bpf: representation and basic operations on circular numbers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit adds basic definitions for cnum32/cnum64. This is a unified numeric range representation for signed and unsigned domains. Inspired by an old post from Shung-Hsi Yu [1] and paper [2]. Operations correctness is verified using cbmc model checker, tests source code can be found in a separate repo [3]. The cnum64_cnum32_intersect() function is notable, because it handled several cases verifier.c:deduce_bounds_64_from_32() does not. Given: - a is a 64-bit range - b is a 32-bit range - t is a refined 64-bit range, such that ∀ v ∈ a, (u32)v ∈ b: v ∈ t. cnum64_cnum32_intersect() makes the following deductions: (A): 'b' is a sub-range of the first or the last 32-bit sub-range of 'a': 64-bit number axis ---> N*2^32 (N+1)*2^32 (N+2)*2^32 (N+3)*2^32 ||------|---|=====|-------||----------|=====|-------||----------|=====|----|--|| | |< b >| |< b >| |< b >| | | | | | |<--+--------------------------- a ---------------------------+--->| | | |<-------------------------- t -------------------------->| (B) 'b' does not intersect with the first of the last 32-bit sub-range of 'a': N*2^32 (N+1)*2^32 (N+2)*2^32 (N+3)*2^32 ||--|=====|----|----------||--|=====|---------------||--|=====|------------|--|| |< b >| | |< b >| |< b >| | | | | | |<-------------+--------- a -------------------|----------->| | | |<-------- t ------------------>| (C) 'b' crosses 0/U32_MAX boundary: N*2^32 (N+1)*2^32 (N+2)*2^32 (N+3)*2^32 ||===|---------|------|===||===|----------------|===||===|---------|------|===|| |b >| | |< b||b >| |< b||b >| | |< b| | | | | |<-----+----------------- a --------------+-------->| | | |<---------------- t ------------->| Current implementation of deduce_bounds_64_from_32() only handles case (A). [1] https://lore.kernel.org/all/ZTZxoDJJbX9mrQ9w@u94a/ [2] https://jorgenavas.github.io/papers/ACM-TOPLAS-wrapped.pdf [3] https://github.com/eddyz87/cnum-verif/tree/master Signed-off-by: Eduard Zingerman Link: https://lore.kernel.org/r/20260424-cnums-everywhere-rfc-v1-v3-1-ca434b39a486@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/Makefile | 2 +- kernel/bpf/cnum.c | 120 ++++++++++++++++++++++++++ kernel/bpf/cnum_defs.h | 230 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 351 insertions(+), 1 deletion(-) create mode 100644 kernel/bpf/cnum.c create mode 100644 kernel/bpf/cnum_defs.h (limited to 'kernel') diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index 399007b67a92..4dc41bf5780c 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -6,7 +6,7 @@ cflags-nogcse-$(CONFIG_X86)$(CONFIG_CC_IS_GCC) := -fno-gcse endif CFLAGS_core.o += -Wno-override-init $(cflags-nogcse-yy) -obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o log.o token.o liveness.o const_fold.o +obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o cnum.o log.o token.o liveness.o const_fold.o obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_iter.o obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o diff --git a/kernel/bpf/cnum.c b/kernel/bpf/cnum.c new file mode 100644 index 000000000000..86142cb2aee5 --- /dev/null +++ b/kernel/bpf/cnum.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include + +#define T 32 +#include "cnum_defs.h" +#undef T + +#define T 64 +#include "cnum_defs.h" +#undef T + +struct cnum32 cnum32_from_cnum64(struct cnum64 cnum) +{ + if (cnum64_is_empty(cnum)) + return CNUM32_EMPTY; + + if (cnum.size >= U32_MAX) + return (struct cnum32){ .base = 0, .size = U32_MAX }; + else + return (struct cnum32){ .base = (u32)cnum.base, .size = cnum.size }; +} + +/* + * Suppose 'a' and 'b' are laid out as follows: + * + * 64-bit number axis ---> + * + * N*2^32 (N+1)*2^32 (N+2)*2^32 (N+3)*2^32 + * ||------|---|=====|-------||----------|=====|-------||----------|=====|----|--|| + * | |< b >| |< b >| |< b >| | + * | | | | + * |<--+--------------------------- a ---------------------------+--->| + * | | + * |<-------------------------- t -------------------------->| + * + * In such a case it is possible to infer a more tight representation t + * such that ∀ v ∈ a, (u32)v ∈ b: v ∈ t. + */ +struct cnum64 cnum64_cnum32_intersect(struct cnum64 a, struct cnum32 b) +{ + /* + * To simplify reasoning, rotate the circles so that [virtual] a1 starts + * at u32 boundary, b1 represents b in this new frame of reference. + */ + struct cnum32 b1 = { b.base - (u32)a.base, b.size }; + struct cnum64 t = a; + u64 d, b1_max; + + if (cnum64_is_empty(a) || cnum32_is_empty(b)) + return CNUM64_EMPTY; + + if (cnum32_urange_overflow(b1)) { + b1_max = (u32)b1.base + (u32)b1.size; /* overflow here is fine and necessary */ + if ((u32)a.size > b1_max && (u32)a.size < b1.base) { + /* + * N*2^32 (N+1)*2^32 + * ||=====|------------|=====||=====|---------|---|=====|| + * |b1 ->| |<- b1||b1 ->| | |<- b1| + * |<----------------- a1 ------------------>| + * |<-------------- t ------------>|<-- d -->| (after adjustment) + * ^ + * b1_max + */ + d = (u32)a.size - b1_max; + t.size -= d; + } else { + /* + * No adjustments possible in the following cases: + * + * ||=====|------------|=====||===|=|-------------|=|===|| + * |b1 ->| |<- b1||b1 +>| |<+ b1| + * |<----------------- a1 ------>| | + * |<----------------- (or) a1 ------------------->| + */ + } + } else { + if (t.size < b1.base) + /* + * N*2^32 (N+1)*2^32 + * ||----------|--|=======|--||------> + * |<-- a1 -->| |<- b ->| + */ + return CNUM64_EMPTY; + /* + * N*2^32 (N+1)*2^32 + * ||-------------|========|-||-----| -------|========|-|| + * | |<- b1 ->| | |<- b1 ->| + * |<------------+ a1 ------------>| + * |<------ t ------>| (after adjustment) + */ + t.base += b1.base; + t.size -= b1.base; + b1_max = b1.base + b1.size; + d = 0; + if ((u32)a.size < b1.base) + /* + * N*2^32 (N+1)*2^32 + * ||-------------|========|-||------|-------|========|-|| + * | |<- b1 ->| | |<- b1 ->| + * |<------------+-- a1 --+-------->| + * |<- t ->|<-- d -->| (after adjustment) + */ + d = (u32)a.size + (BIT_ULL(32) - b1_max); + else if ((u32)a.size >= b1_max) + /* + * N*2^32 (N+1)*2^32 + * ||--|========|------------||--|========|-------|-----|| + * | |<- b1 ->| |<- b1 ->| | + * |<-+------------------ a1 ------------+------>| + * |<-------------- t --------------->|<- d ->| (after adjustment) + */ + d = (u32)a.size - b1_max; + if (t.size < d) + return CNUM64_EMPTY; + t.size -= d; + } + return t; +} diff --git a/kernel/bpf/cnum_defs.h b/kernel/bpf/cnum_defs.h new file mode 100644 index 000000000000..3ebd8f723dbb --- /dev/null +++ b/kernel/bpf/cnum_defs.h @@ -0,0 +1,230 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#ifndef T +#error "Define T (bit width: 32, 64) before including cnum_defs.h" +#endif + +#include +#include +#include +#include + +#define cnum_t __PASTE(cnum, T) +#define ut __PASTE(u, T) +#define st __PASTE(s, T) +#define UT_MAX __PASTE(__PASTE(U, T), _MAX) +#define ST_MAX __PASTE(__PASTE(S, T), _MAX) +#define ST_MIN __PASTE(__PASTE(S, T), _MIN) +#define EMPTY __PASTE(__PASTE(CNUM, T), _EMPTY) +#define FN(name) __PASTE(__PASTE(cnum, T), __PASTE(_, name)) + +struct cnum_t FN(from_urange)(ut min, ut max) +{ + return (struct cnum_t){ .base = min, .size = (ut)max - min }; +} + +struct cnum_t FN(from_srange)(st min, st max) +{ + ut size = (ut)max - (ut)min; + ut base = size == UT_MAX ? 0 : (ut)min; + + return (struct cnum_t){ .base = base, .size = size }; +} + +/* True if this cnum represents two unsigned ranges. */ +static inline bool FN(urange_overflow)(struct cnum_t cnum) +{ + /* Same as cnum.base + cnum.size > UT_MAX but avoids overflow */ + return cnum.size > UT_MAX - (ut)cnum.base; +} + +/* + * cnum{T}_umin / cnum{T}_umax query an unsigned range represented by this cnum. + * If cnum represents a range crossing the UT_MAX/0 boundary, the unbound range + * [0..UT_MAX] is returned. + */ +ut FN(umin)(struct cnum_t cnum) +{ + return FN(urange_overflow)(cnum) ? 0 : cnum.base; +} + +ut FN(umax)(struct cnum_t cnum) +{ + return FN(urange_overflow)(cnum) ? UT_MAX : cnum.base + cnum.size; +} + +/* True if this cnum represents two signed ranges. */ +static inline bool FN(srange_overflow)(struct cnum_t cnum) +{ + return FN(contains)(cnum, (ut)ST_MAX) && FN(contains)(cnum, (ut)ST_MIN); +} + +/* + * cnum{T}_smin / cnum{T}_smax query a signed range represented by this cnum. + * If cnum represents a range crossing the ST_MAX/ST_MIN boundary, the unbound range + * [ST_MIN..ST_MAX] is returned. + */ +st FN(smin)(struct cnum_t cnum) +{ + return FN(srange_overflow)(cnum) + ? ST_MIN + : min((st)cnum.base, (st)(cnum.base + cnum.size)); +} + +st FN(smax)(struct cnum_t cnum) +{ + return FN(srange_overflow)(cnum) + ? ST_MAX + : max((st)cnum.base, (st)(cnum.base + cnum.size)); +} + +/* + * Returns a possibly empty intersection of cnums 'a' and 'b'. + * If 'a' and 'b' intersect in two sub-arcs, the function over-approximates + * and returns either 'a' or 'b', whichever is smaller. + */ +struct cnum_t FN(intersect)(struct cnum_t a, struct cnum_t b) +{ + struct cnum_t b1; + ut dbase; + + if (FN(is_empty)(a) || FN(is_empty)(b)) + return EMPTY; + + if (a.base > b.base) + swap(a, b); + + /* + * Rotate frame of reference such that a.base is 0. + * 'b1' is 'b' in this frame of reference. + */ + dbase = b.base - a.base; + b1 = (struct cnum_t){ dbase, b.size }; + if (FN(urange_overflow)(b1)) { + if (b1.base <= a.size) { + /* + * Rotated frame (a.base at origin): + * + * 0 UT_MAX + * |--------------------------------------------| + * [=== a ==========================] | + * [= b1 tail =] [========= b1 main ==========>] + * ^-- b1.base <= a.size + * + * 'a' and 'b' intersect in two disjoint arcs, + * can't represent as single cnum, over-approximate + * the result. + */ + return a.size <= b.size ? a : b; + } else { + /* + * Rotated frame (a.base at origin): + * + * 0 UT_MAX + * |--------------------------------------------| + * [=== a =============] | | + * [= b1 tail =] [======= b1 main ====>] + * ^-- b1.base > a.size + * + * Only 'b' tail intersects 'a'. + */ + return (struct cnum_t) { + .base = a.base, + .size = min(a.size, (ut)(b1.base + b1.size)), + }; + } + } else if (a.size >= b1.base) { + /* + * Rotated frame (a.base at origin): + * + * 0 UT_MAX + * |--------------------------------------------------| + * [=== a ==================================] | + * [== b1 =====================] + * + * 0 UT_MAX + * |--------------------------------------------------| + * [=== a ==================================] | + * [== b1 ====] + * ^-- b1.base <= a.size + * |<-- a.size - dbase -->| + * + * 'a' and 'b' intersect as one cnum. + */ + return (struct cnum_t) { + .base = b.base, + .size = min((ut)(a.size - dbase), b.size), + }; + } else { + return EMPTY; + } +} + +void FN(intersect_with)(struct cnum_t *dst, struct cnum_t src) +{ + *dst = FN(intersect)(*dst, src); +} + +void FN(intersect_with_urange)(struct cnum_t *dst, ut min, ut max) +{ + FN(intersect_with)(dst, FN(from_urange)(min, max)); +} + +void FN(intersect_with_srange)(struct cnum_t *dst, st min, st max) +{ + FN(intersect_with)(dst, FN(from_srange)(min, max)); +} + +static inline struct cnum_t FN(normalize)(struct cnum_t cnum) +{ + if (cnum.size == UT_MAX && cnum.base != 0 && cnum.base != (ut)ST_MAX) + cnum.base = 0; + return cnum; +} + +struct cnum_t FN(add)(struct cnum_t a, struct cnum_t b) +{ + if (FN(is_empty)(a) || FN(is_empty)(b)) + return EMPTY; + if (a.size > UT_MAX - b.size) + return (struct cnum_t){ 0, (ut)UT_MAX }; + else + return FN(normalize)((struct cnum_t){ a.base + b.base, a.size + b.size }); +} + +struct cnum_t FN(negate)(struct cnum_t a) +{ + if (FN(is_empty)(a)) + return EMPTY; + return FN(normalize)((struct cnum_t){ -((ut)a.base + a.size), a.size }); +} + +bool FN(is_empty)(struct cnum_t cnum) +{ + return cnum.base == EMPTY.base && cnum.size == EMPTY.size; +} + +bool FN(contains)(struct cnum_t cnum, ut v) +{ + if (FN(is_empty)(cnum)) + return false; + if (FN(urange_overflow)(cnum)) + return v >= cnum.base || v <= (ut)cnum.base + cnum.size; + else + return v >= cnum.base && v <= (ut)cnum.base + cnum.size; +} + +bool FN(is_const)(struct cnum_t cnum) +{ + return cnum.size == 0; +} + +#undef EMPTY +#undef cnum_t +#undef ut +#undef st +#undef UT_MAX +#undef ST_MAX +#undef ST_MIN +#undef FN -- cgit v1.2.3 From b93f7180f0bc37336cb26b43aa4796973d84852e Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 24 Apr 2026 15:52:43 -0700 Subject: bpf: use accessor functions for bpf_reg_state min/max fields Replace direct access to bpf_reg_state->{smin,smax,umin,umax, s32_min,s32_max,u32_min,u32_max}_value with getter/setter inline functions, preparing for future switch to cnum-based internal representation. Signed-off-by: Eduard Zingerman Link: https://lore.kernel.org/r/20260424-cnums-everywhere-rfc-v1-v3-2-ca434b39a486@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/log.c | 24 +- kernel/bpf/states.c | 16 +- kernel/bpf/verifier.c | 1233 +++++++++++++++++++++++-------------------------- 3 files changed, 610 insertions(+), 663 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index 011e4ec25acd..64566b86dd27 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c @@ -571,20 +571,20 @@ static void print_scalar_ranges(struct bpf_verifier_env *env, u64 val; bool omit; } minmaxs[] = { - {"smin", reg->smin_value, reg->smin_value == S64_MIN}, - {"smax", reg->smax_value, reg->smax_value == S64_MAX}, - {"umin", reg->umin_value, reg->umin_value == 0}, - {"umax", reg->umax_value, reg->umax_value == U64_MAX}, + {"smin", reg_smin(reg), reg_smin(reg) == S64_MIN}, + {"smax", reg_smax(reg), reg_smax(reg) == S64_MAX}, + {"umin", reg_umin(reg), reg_umin(reg) == 0}, + {"umax", reg_umax(reg), reg_umax(reg) == U64_MAX}, {"smin32", - is_snum_decimal((s64)reg->s32_min_value) - ? (s64)reg->s32_min_value - : (u32)reg->s32_min_value, reg->s32_min_value == S32_MIN}, + is_snum_decimal((s64)reg_s32_min(reg)) + ? (s64)reg_s32_min(reg) + : (u32)reg_s32_min(reg), reg_s32_min(reg) == S32_MIN}, {"smax32", - is_snum_decimal((s64)reg->s32_max_value) - ? (s64)reg->s32_max_value - : (u32)reg->s32_max_value, reg->s32_max_value == S32_MAX}, - {"umin32", reg->u32_min_value, reg->u32_min_value == 0}, - {"umax32", reg->u32_max_value, reg->u32_max_value == U32_MAX}, + is_snum_decimal((s64)reg_s32_max(reg)) + ? (s64)reg_s32_max(reg) + : (u32)reg_s32_max(reg), reg_s32_max(reg) == S32_MAX}, + {"umin32", reg_u32_min(reg), reg_u32_min(reg) == 0}, + {"umax32", reg_u32_max(reg), reg_u32_max(reg) == U32_MAX}, }, *m1, *m2, *mend = &minmaxs[ARRAY_SIZE(minmaxs)]; bool neg1, neg2; diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 8478d2c6ed5b..a78ae891b743 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -301,14 +301,14 @@ int bpf_update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_s static bool range_within(const struct bpf_reg_state *old, const struct bpf_reg_state *cur) { - return old->umin_value <= cur->umin_value && - old->umax_value >= cur->umax_value && - old->smin_value <= cur->smin_value && - old->smax_value >= cur->smax_value && - old->u32_min_value <= cur->u32_min_value && - old->u32_max_value >= cur->u32_max_value && - old->s32_min_value <= cur->s32_min_value && - old->s32_max_value >= cur->s32_max_value; + return reg_umin(old) <= reg_umin(cur) && + reg_umax(old) >= reg_umax(cur) && + reg_smin(old) <= reg_smin(cur) && + reg_smax(old) >= reg_smax(cur) && + reg_u32_min(old) <= reg_u32_min(cur) && + reg_u32_max(old) >= reg_u32_max(cur) && + reg_s32_min(old) <= reg_s32_min(cur) && + reg_s32_max(old) >= reg_s32_max(cur); } /* If in the old state two registers had the same id, then they need to have diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ff6ff1c27517..b91d2789e7b9 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -320,12 +320,12 @@ static void verbose_invalid_scalar(struct bpf_verifier_env *env, bool unknown = true; verbose(env, "%s the register %s has", ctx, reg_name); - if (reg->smin_value > S64_MIN) { - verbose(env, " smin=%lld", reg->smin_value); + if (reg_smin(reg) > S64_MIN) { + verbose(env, " smin=%lld", reg_smin(reg)); unknown = false; } - if (reg->smax_value < S64_MAX) { - verbose(env, " smax=%lld", reg->smax_value); + if (reg_smax(reg) < S64_MAX) { + verbose(env, " smax=%lld", reg_smax(reg)); unknown = false; } if (unknown) @@ -1796,15 +1796,10 @@ static const int caller_saved[CALLER_SAVED_REGS] = { static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm) { reg->var_off = tnum_const(imm); - reg->smin_value = (s64)imm; - reg->smax_value = (s64)imm; - reg->umin_value = imm; - reg->umax_value = imm; - - reg->s32_min_value = (s32)imm; - reg->s32_max_value = (s32)imm; - reg->u32_min_value = (u32)imm; - reg->u32_max_value = (u32)imm; + reg_set_srange64(reg, (s64)imm, (s64)imm); + reg_set_urange64(reg, imm, imm); + reg_set_srange32(reg, (s32)imm, (s32)imm); + reg_set_urange32(reg, (u32)imm, (u32)imm); } /* Mark the unknown part of a register (variable offset or scalar value) as @@ -1823,10 +1818,8 @@ static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm) static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm) { reg->var_off = tnum_const_subreg(reg->var_off, imm); - reg->s32_min_value = (s32)imm; - reg->s32_max_value = (s32)imm; - reg->u32_min_value = (u32)imm; - reg->u32_max_value = (u32)imm; + reg_set_srange32(reg, (s32)imm, (s32)imm); + reg_set_urange32(reg, (u32)imm, (u32)imm); } /* Mark the 'variable offset' part of a register as zero. This should be @@ -1937,34 +1930,25 @@ static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg, tnum_equals_const(reg->var_off, 0); } +static void __mark_reg32_unbounded(struct bpf_reg_state *reg) +{ + reg_set_srange32(reg, S32_MIN, S32_MAX); + reg_set_urange32(reg, 0, U32_MAX); +} + /* Reset the min/max bounds of a register */ static void __mark_reg_unbounded(struct bpf_reg_state *reg) { - reg->smin_value = S64_MIN; - reg->smax_value = S64_MAX; - reg->umin_value = 0; - reg->umax_value = U64_MAX; + reg_set_srange64(reg, S64_MIN, S64_MAX); + reg_set_urange64(reg, 0, U64_MAX); - reg->s32_min_value = S32_MIN; - reg->s32_max_value = S32_MAX; - reg->u32_min_value = 0; - reg->u32_max_value = U32_MAX; + __mark_reg32_unbounded(reg); } static void __mark_reg64_unbounded(struct bpf_reg_state *reg) { - reg->smin_value = S64_MIN; - reg->smax_value = S64_MAX; - reg->umin_value = 0; - reg->umax_value = U64_MAX; -} - -static void __mark_reg32_unbounded(struct bpf_reg_state *reg) -{ - reg->s32_min_value = S32_MIN; - reg->s32_max_value = S32_MAX; - reg->u32_min_value = 0; - reg->u32_max_value = U32_MAX; + reg_set_srange64(reg, S64_MIN, S64_MAX); + reg_set_urange64(reg, 0, U64_MAX); } static void reset_reg64_and_tnum(struct bpf_reg_state *reg) @@ -1983,15 +1967,14 @@ static void __update_reg32_bounds(struct bpf_reg_state *reg) { struct tnum var32_off = tnum_subreg(reg->var_off); - /* min signed is max(sign bit) | min(other bits) */ - reg->s32_min_value = max_t(s32, reg->s32_min_value, - var32_off.value | (var32_off.mask & S32_MIN)); - /* max signed is min(sign bit) | max(other bits) */ - reg->s32_max_value = min_t(s32, reg->s32_max_value, - var32_off.value | (var32_off.mask & S32_MAX)); - reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)var32_off.value); - reg->u32_max_value = min(reg->u32_max_value, - (u32)(var32_off.value | var32_off.mask)); + reg_set_srange32(reg, + /* min signed is max(sign bit) | min(other bits) */ + max_t(s32, reg_s32_min(reg), var32_off.value | (var32_off.mask & S32_MIN)), + /* max signed is min(sign bit) | max(other bits) */ + min_t(s32, reg_s32_max(reg), var32_off.value | (var32_off.mask & S32_MAX))); + reg_set_urange32(reg, + max_t(u32, reg_u32_min(reg), (u32)var32_off.value), + min(reg_u32_max(reg), (u32)(var32_off.value | var32_off.mask))); } static void __update_reg64_bounds(struct bpf_reg_state *reg) @@ -2000,25 +1983,27 @@ static void __update_reg64_bounds(struct bpf_reg_state *reg) bool umin_in_tnum; /* min signed is max(sign bit) | min(other bits) */ - reg->smin_value = max_t(s64, reg->smin_value, - reg->var_off.value | (reg->var_off.mask & S64_MIN)); /* max signed is min(sign bit) | max(other bits) */ - reg->smax_value = min_t(s64, reg->smax_value, - reg->var_off.value | (reg->var_off.mask & S64_MAX)); - reg->umin_value = max(reg->umin_value, reg->var_off.value); - reg->umax_value = min(reg->umax_value, - reg->var_off.value | reg->var_off.mask); + reg_set_srange64(reg, + max_t(s64, reg_smin(reg), + reg->var_off.value | (reg->var_off.mask & S64_MIN)), + min_t(s64, reg_smax(reg), + reg->var_off.value | (reg->var_off.mask & S64_MAX))); + reg_set_urange64(reg, + max(reg_umin(reg), reg->var_off.value), + min(reg_umax(reg), + reg->var_off.value | reg->var_off.mask)); /* Check if u64 and tnum overlap in a single value */ - tnum_next = tnum_step(reg->var_off, reg->umin_value); - umin_in_tnum = (reg->umin_value & ~reg->var_off.mask) == reg->var_off.value; + tnum_next = tnum_step(reg->var_off, reg_umin(reg)); + umin_in_tnum = (reg_umin(reg) & ~reg->var_off.mask) == reg->var_off.value; tmax = reg->var_off.value | reg->var_off.mask; - if (umin_in_tnum && tnum_next > reg->umax_value) { + if (umin_in_tnum && tnum_next > reg_umax(reg)) { /* The u64 range and the tnum only overlap in umin. * u64: ---[xxxxxx]----- * tnum: --xx----------x- */ - ___mark_reg_known(reg, reg->umin_value); + ___mark_reg_known(reg, reg_umin(reg)); } else if (!umin_in_tnum && tnum_next == tmax) { /* The u64 range and the tnum only overlap in the maximum value * represented by the tnum, called tmax. @@ -2026,8 +2011,8 @@ static void __update_reg64_bounds(struct bpf_reg_state *reg) * tnum: xx-----x-------- */ ___mark_reg_known(reg, tmax); - } else if (!umin_in_tnum && tnum_next <= reg->umax_value && - tnum_step(reg->var_off, tnum_next) > reg->umax_value) { + } else if (!umin_in_tnum && tnum_next <= reg_umax(reg) && + tnum_step(reg->var_off, tnum_next) > reg_umax(reg)) { /* The u64 range and the tnum only overlap in between umin * (excluded) and umax. * u64: ---[xxxxxx]----- @@ -2067,28 +2052,32 @@ static void deduce_bounds_32_from_64(struct bpf_reg_state *reg) * * So we use all these insights to derive bounds for subregisters here. */ - if ((reg->umin_value >> 32) == (reg->umax_value >> 32)) { + if ((reg_umin(reg) >> 32) == (reg_umax(reg) >> 32)) { /* u64 to u32 casting preserves validity of low 32 bits as * a range, if upper 32 bits are the same */ - reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)reg->umin_value); - reg->u32_max_value = min_t(u32, reg->u32_max_value, (u32)reg->umax_value); + reg_set_urange32(reg, + max_t(u32, reg_u32_min(reg), (u32)reg_umin(reg)), + min_t(u32, reg_u32_max(reg), (u32)reg_umax(reg))); - if ((s32)reg->umin_value <= (s32)reg->umax_value) { - reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->umin_value); - reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->umax_value); + if ((s32)reg_umin(reg) <= (s32)reg_umax(reg)) { + reg_set_srange32(reg, + max_t(s32, reg_s32_min(reg), (s32)reg_umin(reg)), + min_t(s32, reg_s32_max(reg), (s32)reg_umax(reg))); } } - if ((reg->smin_value >> 32) == (reg->smax_value >> 32)) { + if ((reg_smin(reg) >> 32) == (reg_smax(reg) >> 32)) { /* low 32 bits should form a proper u32 range */ - if ((u32)reg->smin_value <= (u32)reg->smax_value) { - reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)reg->smin_value); - reg->u32_max_value = min_t(u32, reg->u32_max_value, (u32)reg->smax_value); + if ((u32)reg_smin(reg) <= (u32)reg_smax(reg)) { + reg_set_urange32(reg, + max_t(u32, reg_u32_min(reg), (u32)reg_smin(reg)), + min_t(u32, reg_u32_max(reg), (u32)reg_smax(reg))); } /* low 32 bits should form a proper s32 range */ - if ((s32)reg->smin_value <= (s32)reg->smax_value) { - reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->smin_value); - reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->smax_value); + if ((s32)reg_smin(reg) <= (s32)reg_smax(reg)) { + reg_set_srange32(reg, + max_t(s32, reg_s32_min(reg), (s32)reg_smin(reg)), + min_t(s32, reg_s32_max(reg), (s32)reg_smax(reg))); } } /* Special case where upper bits form a small sequence of two @@ -2104,15 +2093,17 @@ static void deduce_bounds_32_from_64(struct bpf_reg_state *reg) * [0xfffffff0fffffff0; 0xfffffff100000010], forms a valid s32 range * [-16, 16] ([0xfffffff0; 0x00000010]) in its 32 bit subregister. */ - if ((u32)(reg->umin_value >> 32) + 1 == (u32)(reg->umax_value >> 32) && - (s32)reg->umin_value < 0 && (s32)reg->umax_value >= 0) { - reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->umin_value); - reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->umax_value); + if ((u32)(reg_umin(reg) >> 32) + 1 == (u32)(reg_umax(reg) >> 32) && + (s32)reg_umin(reg) < 0 && (s32)reg_umax(reg) >= 0) { + reg_set_srange32(reg, + max_t(s32, reg_s32_min(reg), (s32)reg_umin(reg)), + min_t(s32, reg_s32_max(reg), (s32)reg_umax(reg))); } - if ((u32)(reg->smin_value >> 32) + 1 == (u32)(reg->smax_value >> 32) && - (s32)reg->smin_value < 0 && (s32)reg->smax_value >= 0) { - reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->smin_value); - reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->smax_value); + if ((u32)(reg_smin(reg) >> 32) + 1 == (u32)(reg_smax(reg) >> 32) && + (s32)reg_smin(reg) < 0 && (s32)reg_smax(reg) >= 0) { + reg_set_srange32(reg, + max_t(s32, reg_s32_min(reg), (s32)reg_smin(reg)), + min_t(s32, reg_s32_max(reg), (s32)reg_smax(reg))); } } @@ -2121,19 +2112,21 @@ static void deduce_bounds_32_from_32(struct bpf_reg_state *reg) /* if u32 range forms a valid s32 range (due to matching sign bit), * try to learn from that */ - if ((s32)reg->u32_min_value <= (s32)reg->u32_max_value) { - reg->s32_min_value = max_t(s32, reg->s32_min_value, reg->u32_min_value); - reg->s32_max_value = min_t(s32, reg->s32_max_value, reg->u32_max_value); + if ((s32)reg_u32_min(reg) <= (s32)reg_u32_max(reg)) { + reg_set_srange32(reg, + max_t(s32, reg_s32_min(reg), reg_u32_min(reg)), + min_t(s32, reg_s32_max(reg), reg_u32_max(reg))); } /* If we cannot cross the sign boundary, then signed and unsigned bounds * are the same, so combine. This works even in the negative case, e.g. * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. */ - if ((u32)reg->s32_min_value <= (u32)reg->s32_max_value) { - reg->u32_min_value = max_t(u32, reg->s32_min_value, reg->u32_min_value); - reg->u32_max_value = min_t(u32, reg->s32_max_value, reg->u32_max_value); + if ((u32)reg_s32_min(reg) <= (u32)reg_s32_max(reg)) { + reg_set_urange32(reg, + max_t(u32, reg_s32_min(reg), reg_u32_min(reg)), + min_t(u32, reg_s32_max(reg), reg_u32_max(reg))); } else { - if (reg->u32_max_value < (u32)reg->s32_min_value) { + if (reg_u32_max(reg) < (u32)reg_s32_min(reg)) { /* See __reg64_deduce_bounds() for detailed explanation. * Refine ranges in the following situation: * @@ -2143,9 +2136,11 @@ static void deduce_bounds_32_from_32(struct bpf_reg_state *reg) * |xxxxx s32 range xxxxxxxxx] [xxxxxxx| * 0 S32_MAX S32_MIN -1 */ - reg->s32_min_value = (s32)reg->u32_min_value; - reg->u32_max_value = min_t(u32, reg->u32_max_value, reg->s32_max_value); - } else if ((u32)reg->s32_max_value < reg->u32_min_value) { + reg_set_srange32(reg, (s32)reg_u32_min(reg), reg_s32_max(reg)); + reg_set_urange32(reg, + reg_u32_min(reg), + min_t(u32, reg_u32_max(reg), reg_s32_max(reg))); + } else if ((u32)reg_s32_max(reg) < reg_u32_min(reg)) { /* * 0 U32_MAX * | [xxxxxxxxxxxxxx u32 range xxxxxxxxxxxxxx] | @@ -2153,8 +2148,10 @@ static void deduce_bounds_32_from_32(struct bpf_reg_state *reg) * |xxxxxxxxx] [xxxxxxxxxxxx s32 range | * 0 S32_MAX S32_MIN -1 */ - reg->s32_max_value = (s32)reg->u32_max_value; - reg->u32_min_value = max_t(u32, reg->u32_min_value, reg->s32_min_value); + reg_set_srange32(reg, reg_s32_min(reg), (s32)reg_u32_max(reg)); + reg_set_urange32(reg, + max_t(u32, reg_u32_min(reg), reg_s32_min(reg)), + reg_u32_max(reg)); } } } @@ -2228,17 +2225,19 @@ static void deduce_bounds_64_from_64(struct bpf_reg_state *reg) * casting umin/umax as smin/smax and checking if they form valid * range, and vice versa. Those are equivalent checks. */ - if ((s64)reg->umin_value <= (s64)reg->umax_value) { - reg->smin_value = max_t(s64, reg->smin_value, reg->umin_value); - reg->smax_value = min_t(s64, reg->smax_value, reg->umax_value); + if ((s64)reg_umin(reg) <= (s64)reg_umax(reg)) { + reg_set_srange64(reg, + max_t(s64, reg_smin(reg), reg_umin(reg)), + min_t(s64, reg_smax(reg), reg_umax(reg))); } /* If we cannot cross the sign boundary, then signed and unsigned bounds * are the same, so combine. This works even in the negative case, e.g. * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. */ - if ((u64)reg->smin_value <= (u64)reg->smax_value) { - reg->umin_value = max_t(u64, reg->smin_value, reg->umin_value); - reg->umax_value = min_t(u64, reg->smax_value, reg->umax_value); + if ((u64)reg_smin(reg) <= (u64)reg_smax(reg)) { + reg_set_urange64(reg, + max_t(u64, reg_smin(reg), reg_umin(reg)), + min_t(u64, reg_smax(reg), reg_umax(reg))); } else { /* If the s64 range crosses the sign boundary, then it's split * between the beginning and end of the U64 domain. In that @@ -2275,10 +2274,10 @@ static void deduce_bounds_64_from_64(struct bpf_reg_state *reg) * The first condition below corresponds to the first diagram * above. */ - if (reg->umax_value < (u64)reg->smin_value) { - reg->smin_value = (s64)reg->umin_value; - reg->umax_value = min_t(u64, reg->umax_value, reg->smax_value); - } else if ((u64)reg->smax_value < reg->umin_value) { + if (reg_umax(reg) < (u64)reg_smin(reg)) { + reg_set_srange64(reg, (s64)reg_umin(reg), reg_smax(reg)); + reg_set_urange64(reg, reg_umin(reg), min_t(u64, reg_umax(reg), reg_smax(reg))); + } else if ((u64)reg_smax(reg) < reg_umin(reg)) { /* This second condition considers the case where the u64 range * overlaps with the negative portion of the s64 range: * @@ -2288,8 +2287,8 @@ static void deduce_bounds_64_from_64(struct bpf_reg_state *reg) * |xxxxxxxxx] [xxxxxxxxxxxx s64 range | * 0 S64_MAX S64_MIN -1 */ - reg->smax_value = (s64)reg->umax_value; - reg->umin_value = max_t(u64, reg->umin_value, reg->smin_value); + reg_set_srange64(reg, reg_smin(reg), (s64)reg_umax(reg)); + reg_set_urange64(reg, max_t(u64, reg_umin(reg), reg_smin(reg)), reg_umax(reg)); } } } @@ -2312,15 +2311,17 @@ static void deduce_bounds_64_from_32(struct bpf_reg_state *reg) __s64 new_smin, new_smax; /* u32 -> u64 tightening, it's always well-formed */ - new_umin = (reg->umin_value & ~0xffffffffULL) | reg->u32_min_value; - new_umax = (reg->umax_value & ~0xffffffffULL) | reg->u32_max_value; - reg->umin_value = max_t(u64, reg->umin_value, new_umin); - reg->umax_value = min_t(u64, reg->umax_value, new_umax); + new_umin = (reg_umin(reg) & ~0xffffffffULL) | reg_u32_min(reg); + new_umax = (reg_umax(reg) & ~0xffffffffULL) | reg_u32_max(reg); + reg_set_urange64(reg, + max_t(u64, reg_umin(reg), new_umin), + min_t(u64, reg_umax(reg), new_umax)); /* u32 -> s64 tightening, u32 range embedded into s64 preserves range validity */ - new_smin = (reg->smin_value & ~0xffffffffULL) | reg->u32_min_value; - new_smax = (reg->smax_value & ~0xffffffffULL) | reg->u32_max_value; - reg->smin_value = max_t(s64, reg->smin_value, new_smin); - reg->smax_value = min_t(s64, reg->smax_value, new_smax); + new_smin = (reg_smin(reg) & ~0xffffffffULL) | reg_u32_min(reg); + new_smax = (reg_smax(reg) & ~0xffffffffULL) | reg_u32_max(reg); + reg_set_srange64(reg, + max_t(s64, reg_smin(reg), new_smin), + min_t(s64, reg_smax(reg), new_smax)); /* Here we would like to handle a special case after sign extending load, * when upper bits for a 64-bit range are all 1s or all 0s. @@ -2351,13 +2352,11 @@ static void deduce_bounds_64_from_32(struct bpf_reg_state *reg) * - 0x0000_0000_7fff_ffff == (s64)S32_MAX * These relations are used in the conditions below. */ - if (reg->s32_min_value >= 0 && reg->smin_value >= S32_MIN && reg->smax_value <= S32_MAX) { - reg->smin_value = reg->s32_min_value; - reg->smax_value = reg->s32_max_value; - reg->umin_value = reg->s32_min_value; - reg->umax_value = reg->s32_max_value; + if (reg_s32_min(reg) >= 0 && reg_smin(reg) >= S32_MIN && reg_smax(reg) <= S32_MAX) { + reg_set_srange64(reg, reg_s32_min(reg), reg_s32_max(reg)); + reg_set_urange64(reg, reg_s32_min(reg), reg_s32_max(reg)); reg->var_off = tnum_intersect(reg->var_off, - tnum_range(reg->smin_value, reg->smax_value)); + tnum_range(reg_smin(reg), reg_smax(reg))); } } @@ -2373,11 +2372,11 @@ static void __reg_deduce_bounds(struct bpf_reg_state *reg) static void __reg_bound_offset(struct bpf_reg_state *reg) { struct tnum var64_off = tnum_intersect(reg->var_off, - tnum_range(reg->umin_value, - reg->umax_value)); + tnum_range(reg_umin(reg), + reg_umax(reg))); struct tnum var32_off = tnum_intersect(tnum_subreg(var64_off), - tnum_range(reg->u32_min_value, - reg->u32_max_value)); + tnum_range(reg_u32_min(reg), + reg_u32_max(reg))); reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off); } @@ -2405,9 +2404,9 @@ static void reg_bounds_sync(struct bpf_reg_state *reg) static bool range_bounds_violation(struct bpf_reg_state *reg) { - return (reg->umin_value > reg->umax_value || reg->smin_value > reg->smax_value || - reg->u32_min_value > reg->u32_max_value || - reg->s32_min_value > reg->s32_max_value); + return (reg_umin(reg) > reg_umax(reg) || reg_smin(reg) > reg_smax(reg) || + reg_u32_min(reg) > reg_u32_max(reg) || + reg_s32_min(reg) > reg_s32_max(reg)); } static bool const_tnum_range_mismatch(struct bpf_reg_state *reg) @@ -2418,8 +2417,8 @@ static bool const_tnum_range_mismatch(struct bpf_reg_state *reg) if (!tnum_is_const(reg->var_off)) return false; - return reg->umin_value != uval || reg->umax_value != uval || - reg->smin_value != sval || reg->smax_value != sval; + return reg_umin(reg) != uval || reg_umax(reg) != uval || + reg_smin(reg) != sval || reg_smax(reg) != sval; } static bool const_tnum_range_mismatch_32(struct bpf_reg_state *reg) @@ -2430,8 +2429,8 @@ static bool const_tnum_range_mismatch_32(struct bpf_reg_state *reg) if (!tnum_subreg_is_const(reg->var_off)) return false; - return reg->u32_min_value != uval32 || reg->u32_max_value != uval32 || - reg->s32_min_value != sval32 || reg->s32_max_value != sval32; + return reg_u32_min(reg) != uval32 || reg_u32_max(reg) != uval32 || + reg_s32_min(reg) != sval32 || reg_s32_max(reg) != sval32; } static int reg_bounds_sanity_check(struct bpf_verifier_env *env, @@ -2458,10 +2457,10 @@ static int reg_bounds_sanity_check(struct bpf_verifier_env *env, out: verifier_bug(env, "REG INVARIANTS VIOLATION (%s): %s u64=[%#llx, %#llx] " "s64=[%#llx, %#llx] u32=[%#x, %#x] s32=[%#x, %#x] var_off=(%#llx, %#llx)", - ctx, msg, reg->umin_value, reg->umax_value, - reg->smin_value, reg->smax_value, - reg->u32_min_value, reg->u32_max_value, - reg->s32_min_value, reg->s32_max_value, + ctx, msg, reg_umin(reg), reg_umax(reg), + reg_smin(reg), reg_smax(reg), + reg_u32_min(reg), reg_u32_max(reg), + reg_s32_min(reg), reg_s32_max(reg), reg->var_off.value, reg->var_off.mask); if (env->test_reg_invariants) return -EFAULT; @@ -2476,21 +2475,17 @@ static bool __reg32_bound_s64(s32 a) static void __reg_assign_32_into_64(struct bpf_reg_state *reg) { - reg->umin_value = reg->u32_min_value; - reg->umax_value = reg->u32_max_value; + reg_set_urange64(reg, reg_u32_min(reg), reg_u32_max(reg)); /* Attempt to pull 32-bit signed bounds into 64-bit bounds but must * be positive otherwise set to worse case bounds and refine later * from tnum. */ - if (__reg32_bound_s64(reg->s32_min_value) && - __reg32_bound_s64(reg->s32_max_value)) { - reg->smin_value = reg->s32_min_value; - reg->smax_value = reg->s32_max_value; - } else { - reg->smin_value = 0; - reg->smax_value = U32_MAX; - } + if (__reg32_bound_s64(reg_s32_min(reg)) && + __reg32_bound_s64(reg_s32_max(reg))) + reg_set_srange64(reg, reg_s32_min(reg), reg_s32_max(reg)); + else + reg_set_srange64(reg, 0, U32_MAX); } /* Mark a register as having a completely unknown (scalar) value. */ @@ -2534,11 +2529,12 @@ static int __mark_reg_s32_range(struct bpf_verifier_env *env, { struct bpf_reg_state *reg = regs + regno; - reg->s32_min_value = max_t(s32, reg->s32_min_value, s32_min); - reg->s32_max_value = min_t(s32, reg->s32_max_value, s32_max); - - reg->smin_value = max_t(s64, reg->smin_value, s32_min); - reg->smax_value = min_t(s64, reg->smax_value, s32_max); + reg_set_srange32(reg, + max_t(s32, reg_s32_min(reg), s32_min), + min_t(s32, reg_s32_max(reg), s32_max)); + reg_set_srange64(reg, + max_t(s64, reg_smin(reg), s32_min), + min_t(s64, reg_smax(reg), s32_max)); reg_bounds_sync(reg); @@ -3801,7 +3797,7 @@ static bool is_bpf_st_mem(struct bpf_insn *insn) static int get_reg_width(struct bpf_reg_state *reg) { - return fls64(reg->umax_value); + return fls64(reg_umax(reg)); } /* See comment for mark_fastcall_pattern_for_call() */ @@ -3990,8 +3986,8 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, bool zero_used = false; cur = env->cur_state->frame[env->cur_state->curframe]; - min_off = ptr_reg->smin_value + off; - max_off = ptr_reg->smax_value + off + size; + min_off = reg_smin(ptr_reg) + off; + max_off = reg_smax(ptr_reg) + off + size; if (value_regno >= 0) value_reg = &cur->regs[value_regno]; if ((value_reg && bpf_register_is_null(value_reg)) || @@ -4324,8 +4320,8 @@ static int check_stack_read_var_off(struct bpf_verifier_env *env, struct bpf_reg if (err) return err; - min_off = reg->smin_value + off; - max_off = reg->smax_value + off; + min_off = reg_smin(reg) + off; + max_off = reg_smax(reg) + off; mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno); check_fastcall_stack_contract(env, ptr_state, env->insn_idx, min_off); return 0; @@ -4425,13 +4421,13 @@ static int check_map_access_type(struct bpf_verifier_env *env, struct bpf_reg_st if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) { verbose(env, "write into map forbidden, value_size=%d off=%lld size=%d\n", - map->value_size, reg->smin_value + off, size); + map->value_size, reg_smin(reg) + off, size); return -EACCES; } if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) { verbose(env, "read from map forbidden, value_size=%d off=%lld size=%d\n", - map->value_size, reg->smin_value + off, size); + map->value_size, reg_smin(reg) + off, size); return -EACCES; } @@ -4493,15 +4489,15 @@ static int check_mem_region_access(struct bpf_verifier_env *env, struct bpf_reg_ * index'es we need to make sure that whatever we use * will have a set floor within our range. */ - if (reg->smin_value < 0 && - (reg->smin_value == S64_MIN || - (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) || - reg->smin_value + off < 0)) { + if (reg_smin(reg) < 0 && + (reg_smin(reg) == S64_MIN || + (off + reg_smin(reg) != (s64)(s32)(off + reg_smin(reg))) || + reg_smin(reg) + off < 0)) { verbose(env, "%s min value is negative, either use unsigned index or do a if (index >=0) check.\n", reg_arg_name(env, argno)); return -EACCES; } - err = __check_mem_access(env, reg, argno, reg->smin_value + off, size, + err = __check_mem_access(env, reg, argno, reg_smin(reg) + off, size, mem_size, zero_size_allowed); if (err) { verbose(env, "%s min value is outside of the allowed memory range\n", @@ -4511,14 +4507,14 @@ static int check_mem_region_access(struct bpf_verifier_env *env, struct bpf_reg_ /* If we haven't set a max value then we need to bail since we can't be * sure we won't do bad things. - * If reg->umax_value + off could overflow, treat that as unbounded too. + * If reg_umax(reg) + off could overflow, treat that as unbounded too. */ - if (reg->umax_value >= BPF_MAX_VAR_OFF) { + if (reg_umax(reg) >= BPF_MAX_VAR_OFF) { verbose(env, "%s unbounded memory access, make sure to bounds check any such access\n", reg_arg_name(env, argno)); return -EACCES; } - err = __check_mem_access(env, reg, argno, reg->umax_value + off, size, + err = __check_mem_access(env, reg, argno, reg_umax(reg) + off, size, mem_size, zero_size_allowed); if (err) { verbose(env, "%s max value is outside of the allowed memory range\n", @@ -4546,7 +4542,7 @@ static int __check_ptr_off_reg(struct bpf_verifier_env *env, return -EACCES; } - if (reg->smin_value < 0) { + if (reg_smin(reg) < 0) { verbose(env, "negative offset %s ptr %s off=%lld disallowed\n", reg_type_str(env, reg->type), reg_arg_name(env, argno), reg->var_off.value); return -EACCES; @@ -4846,8 +4842,8 @@ static int check_map_access(struct bpf_verifier_env *env, struct bpf_reg_state * * this program. To check that [x1, x2) overlaps with [y1, y2), * it is sufficient to check x1 < y2 && y1 < x2. */ - if (reg->smin_value + off < p + field->size && - p < reg->umax_value + off + size) { + if (reg_smin(reg) + off < p + field->size && + p < reg_umax(reg) + off + size) { switch (field->type) { case BPF_KPTR_UNREF: case BPF_KPTR_REF: @@ -4942,14 +4938,14 @@ static int check_packet_access(struct bpf_verifier_env *env, struct bpf_reg_stat return err; /* __check_mem_access has made sure "off + size - 1" is within u16. - * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff, + * reg_umax(reg) can't be bigger than MAX_PACKET_OFF which is 0xffff, * otherwise find_good_pkt_pointers would have refused to set range info * that __check_mem_access would have rejected this pkt access. - * Therefore, "off + reg->umax_value + size - 1" won't overflow u32. + * Therefore, "off + reg_umax(reg) + size - 1" won't overflow u32. */ env->prog->aux->max_pkt_offset = max_t(u32, env->prog->aux->max_pkt_offset, - off + reg->umax_value + size - 1); + off + reg_umax(reg) + size - 1); return 0; } @@ -5010,7 +5006,7 @@ static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, struct b err = __check_ptr_off_reg(env, reg, argno, fixed_off_ok); if (err) return err; - off += reg->umax_value; + off += reg_umax(reg); err = __check_ctx_access(env, insn_idx, off, access_size, t, info); if (err) @@ -5037,7 +5033,7 @@ static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn_access_aux info = {}; bool valid; - if (reg->smin_value < 0) { + if (reg_smin(reg) < 0) { verbose(env, "%s min value is negative, either use unsigned index or do a if (index >=0) check.\n", reg_arg_name(env, argno)); return -EACCES; @@ -5655,15 +5651,12 @@ static void coerce_reg_to_size(struct bpf_reg_state *reg, int size) /* fix arithmetic bounds */ mask = ((u64)1 << (size * 8)) - 1; - if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) { - reg->umin_value &= mask; - reg->umax_value &= mask; + if ((reg_umin(reg) & ~mask) == (reg_umax(reg) & ~mask)) { + reg_set_urange64(reg, reg_umin(reg) & mask, reg_umax(reg) & mask); } else { - reg->umin_value = 0; - reg->umax_value = mask; + reg_set_urange64(reg, 0, mask); } - reg->smin_value = reg->umin_value; - reg->smax_value = reg->umax_value; + reg_set_srange64(reg, reg_umin(reg), reg_umax(reg)); /* If size is smaller than 32bit register the 32bit register * values are also truncated so we push 64-bit bounds into @@ -5678,19 +5671,18 @@ static void coerce_reg_to_size(struct bpf_reg_state *reg, int size) static void set_sext64_default_val(struct bpf_reg_state *reg, int size) { if (size == 1) { - reg->smin_value = reg->s32_min_value = S8_MIN; - reg->smax_value = reg->s32_max_value = S8_MAX; + reg_set_srange64(reg, S8_MIN, S8_MAX); + reg_set_srange32(reg, S8_MIN, S8_MAX); } else if (size == 2) { - reg->smin_value = reg->s32_min_value = S16_MIN; - reg->smax_value = reg->s32_max_value = S16_MAX; + reg_set_srange64(reg, S16_MIN, S16_MAX); + reg_set_srange32(reg, S16_MIN, S16_MAX); } else { /* size == 4 */ - reg->smin_value = reg->s32_min_value = S32_MIN; - reg->smax_value = reg->s32_max_value = S32_MAX; + reg_set_srange64(reg, S32_MIN, S32_MAX); + reg_set_srange32(reg, S32_MIN, S32_MAX); } - reg->umin_value = reg->u32_min_value = 0; - reg->umax_value = U64_MAX; - reg->u32_max_value = U32_MAX; + reg_set_urange64(reg, 0, U64_MAX); + reg_set_urange32(reg, 0, U32_MAX); reg->var_off = tnum_unknown; } @@ -5711,29 +5703,29 @@ static void coerce_reg_to_size_sx(struct bpf_reg_state *reg, int size) reg->var_off = tnum_const((s32)u64_cval); u64_cval = reg->var_off.value; - reg->smax_value = reg->smin_value = u64_cval; - reg->umax_value = reg->umin_value = u64_cval; - reg->s32_max_value = reg->s32_min_value = u64_cval; - reg->u32_max_value = reg->u32_min_value = u64_cval; + reg_set_srange64(reg, u64_cval, u64_cval); + reg_set_urange64(reg, u64_cval, u64_cval); + reg_set_srange32(reg, u64_cval, u64_cval); + reg_set_urange32(reg, u64_cval, u64_cval); return; } - top_smax_value = ((u64)reg->smax_value >> num_bits) << num_bits; - top_smin_value = ((u64)reg->smin_value >> num_bits) << num_bits; + top_smax_value = ((u64)reg_smax(reg) >> num_bits) << num_bits; + top_smin_value = ((u64)reg_smin(reg) >> num_bits) << num_bits; if (top_smax_value != top_smin_value) goto out; /* find the s64_min and s64_min after sign extension */ if (size == 1) { - init_s64_max = (s8)reg->smax_value; - init_s64_min = (s8)reg->smin_value; + init_s64_max = (s8)reg_smax(reg); + init_s64_min = (s8)reg_smin(reg); } else if (size == 2) { - init_s64_max = (s16)reg->smax_value; - init_s64_min = (s16)reg->smin_value; + init_s64_max = (s16)reg_smax(reg); + init_s64_min = (s16)reg_smin(reg); } else { - init_s64_max = (s32)reg->smax_value; - init_s64_min = (s32)reg->smin_value; + init_s64_max = (s32)reg_smax(reg); + init_s64_min = (s32)reg_smin(reg); } s64_max = max(init_s64_max, init_s64_min); @@ -5741,10 +5733,10 @@ static void coerce_reg_to_size_sx(struct bpf_reg_state *reg, int size) /* both of s64_max/s64_min positive or negative */ if ((s64_max >= 0) == (s64_min >= 0)) { - reg->s32_min_value = reg->smin_value = s64_min; - reg->s32_max_value = reg->smax_value = s64_max; - reg->u32_min_value = reg->umin_value = s64_min; - reg->u32_max_value = reg->umax_value = s64_max; + reg_set_srange64(reg, s64_min, s64_max); + reg_set_urange64(reg, s64_min, s64_max); + reg_set_srange32(reg, s64_min, s64_max); + reg_set_urange32(reg, s64_min, s64_max); reg->var_off = tnum_range(s64_min, s64_max); return; } @@ -5755,16 +5747,12 @@ out: static void set_sext32_default_val(struct bpf_reg_state *reg, int size) { - if (size == 1) { - reg->s32_min_value = S8_MIN; - reg->s32_max_value = S8_MAX; - } else { + if (size == 1) + reg_set_srange32(reg, S8_MIN, S8_MAX); + else /* size == 2 */ - reg->s32_min_value = S16_MIN; - reg->s32_max_value = S16_MAX; - } - reg->u32_min_value = 0; - reg->u32_max_value = U32_MAX; + reg_set_srange32(reg, S16_MIN, S16_MAX); + reg_set_urange32(reg, 0, U32_MAX); reg->var_off = tnum_subreg(tnum_unknown); } @@ -5782,34 +5770,32 @@ static void coerce_subreg_to_size_sx(struct bpf_reg_state *reg, int size) reg->var_off = tnum_const((s16)u32_val); u32_val = reg->var_off.value; - reg->s32_min_value = reg->s32_max_value = u32_val; - reg->u32_min_value = reg->u32_max_value = u32_val; + reg_set_srange32(reg, u32_val, u32_val); + reg_set_urange32(reg, u32_val, u32_val); return; } - top_smax_value = ((u32)reg->s32_max_value >> num_bits) << num_bits; - top_smin_value = ((u32)reg->s32_min_value >> num_bits) << num_bits; + top_smax_value = ((u32)reg_s32_max(reg) >> num_bits) << num_bits; + top_smin_value = ((u32)reg_s32_min(reg) >> num_bits) << num_bits; if (top_smax_value != top_smin_value) goto out; /* find the s32_min and s32_min after sign extension */ if (size == 1) { - init_s32_max = (s8)reg->s32_max_value; - init_s32_min = (s8)reg->s32_min_value; + init_s32_max = (s8)reg_s32_max(reg); + init_s32_min = (s8)reg_s32_min(reg); } else { /* size == 2 */ - init_s32_max = (s16)reg->s32_max_value; - init_s32_min = (s16)reg->s32_min_value; + init_s32_max = (s16)reg_s32_max(reg); + init_s32_min = (s16)reg_s32_min(reg); } s32_max = max(init_s32_max, init_s32_min); s32_min = min(init_s32_max, init_s32_min); if ((s32_min >= 0) == (s32_max >= 0)) { - reg->s32_min_value = s32_min; - reg->s32_max_value = s32_max; - reg->u32_min_value = (u32)s32_min; - reg->u32_max_value = (u32)s32_max; + reg_set_srange32(reg, s32_min, s32_max); + reg_set_urange32(reg, (u32)s32_min, (u32)s32_max); reg->var_off = tnum_subreg(tnum_range(s32_min, s32_max)); return; } @@ -6266,14 +6252,14 @@ static int check_stack_access_within_bounds( min_off = (s64)reg->var_off.value + off; max_off = min_off + access_size; } else { - if (reg->smax_value >= BPF_MAX_VAR_OFF || - reg->smin_value <= -BPF_MAX_VAR_OFF) { + if (reg_smax(reg) >= BPF_MAX_VAR_OFF || + reg_smin(reg) <= -BPF_MAX_VAR_OFF) { verbose(env, "invalid unbounded variable-offset%s stack %s\n", err_extra, reg_arg_name(env, argno)); return -EACCES; } - min_off = reg->smin_value + off; - max_off = reg->smax_value + off + access_size; + min_off = reg_smin(reg) + off; + max_off = reg_smax(reg) + off + access_size; } err = check_stack_slot_within_bounds(env, min_off, state, type); @@ -6891,8 +6877,8 @@ static int check_stack_range_initialized( if (meta && meta->raw_mode) meta = NULL; - min_off = reg->smin_value + off; - max_off = reg->smax_value + off; + min_off = reg_smin(reg) + off; + max_off = reg_smax(reg) + off; } if (meta && meta->raw_mode) { @@ -7048,8 +7034,8 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ zero_size_allowed); if (err) return err; - if (env->prog->aux->max_ctx_offset < reg->umax_value + access_size) - env->prog->aux->max_ctx_offset = reg->umax_value + access_size; + if (env->prog->aux->max_ctx_offset < reg_umax(reg) + access_size) + env->prog->aux->max_ctx_offset = reg_umax(reg) + access_size; return 0; } fallthrough; @@ -7088,7 +7074,7 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, * out. Only upper bounds can be learned because retval is an * int type and negative retvals are allowed. */ - meta->msize_max_value = size_reg->umax_value; + meta->msize_max_value = reg_umax(size_reg); /* The register is SCALAR_VALUE; the access check happens using * its boundaries. For unprivileged variable accesses, disable @@ -7098,24 +7084,24 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, if (!tnum_is_const(size_reg->var_off)) meta = NULL; - if (size_reg->smin_value < 0) { + if (reg_smin(size_reg) < 0) { verbose(env, "%s min value is negative, either use unsigned or 'var &= const'\n", reg_arg_name(env, size_argno)); return -EACCES; } - if (size_reg->umin_value == 0 && !zero_size_allowed) { + if (reg_umin(size_reg) == 0 && !zero_size_allowed) { verbose(env, "%s invalid zero-sized read: u64=[%lld,%lld]\n", - reg_arg_name(env, size_argno), size_reg->umin_value, size_reg->umax_value); + reg_arg_name(env, size_argno), reg_umin(size_reg), reg_umax(size_reg)); return -EACCES; } - if (size_reg->umax_value >= BPF_MAX_VAR_SIZ) { + if (reg_umax(size_reg) >= BPF_MAX_VAR_SIZ) { verbose(env, "%s unbounded memory access, use 'var &= const' or 'if (var < const)'\n", reg_arg_name(env, size_argno)); return -EACCES; } - err = check_helper_mem_access(env, mem_reg, mem_argno, size_reg->umax_value, + err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg), access_type, zero_size_allowed, meta); if (!err) err = mark_chain_precision(env, reg_from_argno(size_argno)); @@ -9848,9 +9834,9 @@ static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env) static bool retval_range_within(struct bpf_retval_range range, const struct bpf_reg_state *reg) { if (range.return_32bit) - return range.minval <= reg->s32_min_value && reg->s32_max_value <= range.maxval; + return range.minval <= reg_s32_min(reg) && reg_s32_max(reg) <= range.maxval; else - return range.minval <= reg->smin_value && reg->smax_value <= range.maxval; + return range.minval <= reg_smin(reg) && reg_smax(reg) <= range.maxval; } static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) @@ -9959,21 +9945,15 @@ static int do_refine_retval_range(struct bpf_verifier_env *env, case BPF_FUNC_probe_read_str: case BPF_FUNC_probe_read_kernel_str: case BPF_FUNC_probe_read_user_str: - ret_reg->smax_value = meta->msize_max_value; - ret_reg->s32_max_value = meta->msize_max_value; - ret_reg->smin_value = -MAX_ERRNO; - ret_reg->s32_min_value = -MAX_ERRNO; + reg_set_srange64(ret_reg, -MAX_ERRNO, meta->msize_max_value); + reg_set_srange32(ret_reg, -MAX_ERRNO, meta->msize_max_value); reg_bounds_sync(ret_reg); break; case BPF_FUNC_get_smp_processor_id: - ret_reg->umax_value = nr_cpu_ids - 1; - ret_reg->u32_max_value = nr_cpu_ids - 1; - ret_reg->smax_value = nr_cpu_ids - 1; - ret_reg->s32_max_value = nr_cpu_ids - 1; - ret_reg->umin_value = 0; - ret_reg->u32_min_value = 0; - ret_reg->smin_value = 0; - ret_reg->s32_min_value = 0; + reg_set_urange64(ret_reg, 0, nr_cpu_ids - 1); + reg_set_urange32(ret_reg, 0, nr_cpu_ids - 1); + reg_set_srange64(ret_reg, 0, nr_cpu_ids - 1); + reg_set_srange32(ret_reg, 0, nr_cpu_ids - 1); reg_bounds_sync(ret_reg); break; } @@ -10438,7 +10418,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn err = mark_chain_precision(env, BPF_REG_1); if (err) return err; - if (cur_func(env)->callback_depth < regs[BPF_REG_1].umax_value) { + if (cur_func(env)->callback_depth < reg_umax(®s[BPF_REG_1])) { err = push_callback_call(env, insn, insn_idx, meta.subprogno, set_loop_callback_state); } else { @@ -13403,7 +13383,7 @@ static bool check_reg_sane_offset_scalar(struct bpf_verifier_env *env, { bool known = tnum_is_const(reg->var_off); s64 val = reg->var_off.value; - s64 smin = reg->smin_value; + s64 smin = reg_smin(reg); if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) { verbose(env, "math between %s pointer and %lld is not allowed\n", @@ -13432,7 +13412,7 @@ static bool check_reg_sane_offset_ptr(struct bpf_verifier_env *env, { bool known = tnum_is_const(reg->var_off); s64 val = reg->var_off.value; - s64 smin = reg->smin_value; + s64 smin = reg_smin(reg); if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) { verbose(env, "%s pointer offset %lld is not allowed\n", @@ -13474,7 +13454,7 @@ static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, break; case PTR_TO_MAP_VALUE: max = ptr_reg->map_ptr->value_size; - ptr_limit = mask_to_left ? ptr_reg->smin_value : ptr_reg->umax_value; + ptr_limit = mask_to_left ? reg_smin(ptr_reg) : reg_umax(ptr_reg); break; default: return REASON_TYPE; @@ -13563,7 +13543,7 @@ static int sanitize_ptr_alu(struct bpf_verifier_env *env, struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux; struct bpf_verifier_state *vstate = env->cur_state; bool off_is_imm = tnum_is_const(off_reg->var_off); - bool off_is_neg = off_reg->smin_value < 0; + bool off_is_neg = reg_smin(off_reg) < 0; bool ptr_is_dst_reg = ptr_reg == dst_reg; u8 opcode = BPF_OP(insn->code); u32 alu_state, alu_limit; @@ -13582,7 +13562,7 @@ static int sanitize_ptr_alu(struct bpf_verifier_env *env, if (!commit_window) { if (!tnum_is_const(off_reg->var_off) && - (off_reg->smin_value < 0) != (off_reg->smax_value < 0)) + (reg_smin(off_reg) < 0) != (reg_smax(off_reg) < 0)) return REASON_BOUNDS; info->mask_to_left = (opcode == BPF_ADD && off_is_neg) || @@ -13776,10 +13756,10 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = state->regs, *dst_reg; bool known = tnum_is_const(off_reg->var_off); - s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value, - smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value; - u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value, - umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value; + s64 smin_val = reg_smin(off_reg), smax_val = reg_smax(off_reg), + smin_ptr = reg_smin(ptr_reg), smax_ptr = reg_smax(ptr_reg); + u64 umin_val = reg_umin(off_reg), umax_val = reg_umax(off_reg), + umin_ptr = reg_umin(ptr_reg), umax_ptr = reg_umax(ptr_reg); struct bpf_sanitize_info info = {}; u8 opcode = BPF_OP(insn->code); u32 dst = insn->dst_reg; @@ -13881,15 +13861,22 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, * added into the variable offset, and we copy the fixed offset * from ptr_reg. */ - if (check_add_overflow(smin_ptr, smin_val, &dst_reg->smin_value) || - check_add_overflow(smax_ptr, smax_val, &dst_reg->smax_value)) { - dst_reg->smin_value = S64_MIN; - dst_reg->smax_value = S64_MAX; + { + s64 smin_res, smax_res; + u64 umin_res, umax_res; + + if (check_add_overflow(smin_ptr, smin_val, &smin_res) || + check_add_overflow(smax_ptr, smax_val, &smax_res)) { + reg_set_srange64(dst_reg, S64_MIN, S64_MAX); + } else { + reg_set_srange64(dst_reg, smin_res, smax_res); + } + if (check_add_overflow(umin_ptr, umin_val, &umin_res) || + check_add_overflow(umax_ptr, umax_val, &umax_res)) { + reg_set_urange64(dst_reg, 0, U64_MAX); + } else { + reg_set_urange64(dst_reg, umin_res, umax_res); } - if (check_add_overflow(umin_ptr, umin_val, &dst_reg->umin_value) || - check_add_overflow(umax_ptr, umax_val, &dst_reg->umax_value)) { - dst_reg->umin_value = 0; - dst_reg->umax_value = U64_MAX; } dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); dst_reg->raw = ptr_reg->raw; @@ -13925,20 +13912,23 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, /* A new variable offset is created. If the subtrahend is known * nonnegative, then any reg->range we had before is still good. */ - if (check_sub_overflow(smin_ptr, smax_val, &dst_reg->smin_value) || - check_sub_overflow(smax_ptr, smin_val, &dst_reg->smax_value)) { + { + s64 smin_res, smax_res; + + if (check_sub_overflow(smin_ptr, smax_val, &smin_res) || + check_sub_overflow(smax_ptr, smin_val, &smax_res)) { /* Overflow possible, we know nothing */ - dst_reg->smin_value = S64_MIN; - dst_reg->smax_value = S64_MAX; + reg_set_srange64(dst_reg, S64_MIN, S64_MAX); + } else { + reg_set_srange64(dst_reg, smin_res, smax_res); + } } if (umin_ptr < umax_val) { /* Overflow possible, we know nothing */ - dst_reg->umin_value = 0; - dst_reg->umax_value = U64_MAX; + reg_set_urange64(dst_reg, 0, U64_MAX); } else { /* Cannot overflow (as long as bounds are consistent) */ - dst_reg->umin_value = umin_ptr - umax_val; - dst_reg->umax_value = umax_ptr - umin_val; + reg_set_urange64(dst_reg, umin_ptr - umax_val, umax_ptr - umin_val); } dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off); dst_reg->raw = ptr_reg->raw; @@ -13996,18 +13986,18 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) { - s32 *dst_smin = &dst_reg->s32_min_value; - s32 *dst_smax = &dst_reg->s32_max_value; - u32 *dst_umin = &dst_reg->u32_min_value; - u32 *dst_umax = &dst_reg->u32_max_value; - u32 umin_val = src_reg->u32_min_value; - u32 umax_val = src_reg->u32_max_value; + s32 smin = reg_s32_min(dst_reg); + s32 smax = reg_s32_max(dst_reg); + u32 umin = reg_u32_min(dst_reg); + u32 umax = reg_u32_max(dst_reg); + u32 umin_val = reg_u32_min(src_reg); + u32 umax_val = reg_u32_max(src_reg); bool min_overflow, max_overflow; - if (check_add_overflow(*dst_smin, src_reg->s32_min_value, dst_smin) || - check_add_overflow(*dst_smax, src_reg->s32_max_value, dst_smax)) { - *dst_smin = S32_MIN; - *dst_smax = S32_MAX; + if (check_add_overflow(smin, reg_s32_min(src_reg), &smin) || + check_add_overflow(smax, reg_s32_max(src_reg), &smax)) { + smin = S32_MIN; + smax = S32_MAX; } /* If either all additions overflow or no additions overflow, then @@ -14015,30 +14005,33 @@ static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, * dst_umax + src_umax. Otherwise (some additions overflow), set * the output bounds to unbounded. */ - min_overflow = check_add_overflow(*dst_umin, umin_val, dst_umin); - max_overflow = check_add_overflow(*dst_umax, umax_val, dst_umax); + min_overflow = check_add_overflow(umin, umin_val, &umin); + max_overflow = check_add_overflow(umax, umax_val, &umax); if (!min_overflow && max_overflow) { - *dst_umin = 0; - *dst_umax = U32_MAX; + umin = 0; + umax = U32_MAX; } + + reg_set_srange32(dst_reg, smin, smax); + reg_set_urange32(dst_reg, umin, umax); } static void scalar_min_max_add(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) { - s64 *dst_smin = &dst_reg->smin_value; - s64 *dst_smax = &dst_reg->smax_value; - u64 *dst_umin = &dst_reg->umin_value; - u64 *dst_umax = &dst_reg->umax_value; - u64 umin_val = src_reg->umin_value; - u64 umax_val = src_reg->umax_value; + s64 smin = reg_smin(dst_reg); + s64 smax = reg_smax(dst_reg); + u64 umin = reg_umin(dst_reg); + u64 umax = reg_umax(dst_reg); + u64 umin_val = reg_umin(src_reg); + u64 umax_val = reg_umax(src_reg); bool min_overflow, max_overflow; - if (check_add_overflow(*dst_smin, src_reg->smin_value, dst_smin) || - check_add_overflow(*dst_smax, src_reg->smax_value, dst_smax)) { - *dst_smin = S64_MIN; - *dst_smax = S64_MAX; + if (check_add_overflow(smin, reg_smin(src_reg), &smin) || + check_add_overflow(smax, reg_smax(src_reg), &smax)) { + smin = S64_MIN; + smax = S64_MAX; } /* If either all additions overflow or no additions overflow, then @@ -14046,31 +14039,34 @@ static void scalar_min_max_add(struct bpf_reg_state *dst_reg, * dst_umax + src_umax. Otherwise (some additions overflow), set * the output bounds to unbounded. */ - min_overflow = check_add_overflow(*dst_umin, umin_val, dst_umin); - max_overflow = check_add_overflow(*dst_umax, umax_val, dst_umax); + min_overflow = check_add_overflow(umin, umin_val, &umin); + max_overflow = check_add_overflow(umax, umax_val, &umax); if (!min_overflow && max_overflow) { - *dst_umin = 0; - *dst_umax = U64_MAX; + umin = 0; + umax = U64_MAX; } + + reg_set_srange64(dst_reg, smin, smax); + reg_set_urange64(dst_reg, umin, umax); } static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) { - s32 *dst_smin = &dst_reg->s32_min_value; - s32 *dst_smax = &dst_reg->s32_max_value; - u32 *dst_umin = &dst_reg->u32_min_value; - u32 *dst_umax = &dst_reg->u32_max_value; - u32 umin_val = src_reg->u32_min_value; - u32 umax_val = src_reg->u32_max_value; + s32 smin = reg_s32_min(dst_reg); + s32 smax = reg_s32_max(dst_reg); + u32 umin = reg_u32_min(dst_reg); + u32 umax = reg_u32_max(dst_reg); + u32 umin_val = reg_u32_min(src_reg); + u32 umax_val = reg_u32_max(src_reg); bool min_underflow, max_underflow; - if (check_sub_overflow(*dst_smin, src_reg->s32_max_value, dst_smin) || - check_sub_overflow(*dst_smax, src_reg->s32_min_value, dst_smax)) { + if (check_sub_overflow(smin, reg_s32_max(src_reg), &smin) || + check_sub_overflow(smax, reg_s32_min(src_reg), &smax)) { /* Overflow possible, we know nothing */ - *dst_smin = S32_MIN; - *dst_smax = S32_MAX; + smin = S32_MIN; + smax = S32_MAX; } /* If either all subtractions underflow or no subtractions @@ -14078,31 +14074,34 @@ static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg, * dst_umax = dst_umax - src_umin. Otherwise (some subtractions * underflow), set the output bounds to unbounded. */ - min_underflow = check_sub_overflow(*dst_umin, umax_val, dst_umin); - max_underflow = check_sub_overflow(*dst_umax, umin_val, dst_umax); + min_underflow = check_sub_overflow(umin, umax_val, &umin); + max_underflow = check_sub_overflow(umax, umin_val, &umax); if (min_underflow && !max_underflow) { - *dst_umin = 0; - *dst_umax = U32_MAX; + umin = 0; + umax = U32_MAX; } + + reg_set_srange32(dst_reg, smin, smax); + reg_set_urange32(dst_reg, umin, umax); } static void scalar_min_max_sub(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) { - s64 *dst_smin = &dst_reg->smin_value; - s64 *dst_smax = &dst_reg->smax_value; - u64 *dst_umin = &dst_reg->umin_value; - u64 *dst_umax = &dst_reg->umax_value; - u64 umin_val = src_reg->umin_value; - u64 umax_val = src_reg->umax_value; + s64 smin = reg_smin(dst_reg); + s64 smax = reg_smax(dst_reg); + u64 umin = reg_umin(dst_reg); + u64 umax = reg_umax(dst_reg); + u64 umin_val = reg_umin(src_reg); + u64 umax_val = reg_umax(src_reg); bool min_underflow, max_underflow; - if (check_sub_overflow(*dst_smin, src_reg->smax_value, dst_smin) || - check_sub_overflow(*dst_smax, src_reg->smin_value, dst_smax)) { + if (check_sub_overflow(smin, reg_smax(src_reg), &smin) || + check_sub_overflow(smax, reg_smin(src_reg), &smax)) { /* Overflow possible, we know nothing */ - *dst_smin = S64_MIN; - *dst_smax = S64_MAX; + smin = S64_MIN; + smax = S64_MAX; } /* If either all subtractions underflow or no subtractions @@ -14110,113 +14109,116 @@ static void scalar_min_max_sub(struct bpf_reg_state *dst_reg, * dst_umax = dst_umax - src_umin. Otherwise (some subtractions * underflow), set the output bounds to unbounded. */ - min_underflow = check_sub_overflow(*dst_umin, umax_val, dst_umin); - max_underflow = check_sub_overflow(*dst_umax, umin_val, dst_umax); + min_underflow = check_sub_overflow(umin, umax_val, &umin); + max_underflow = check_sub_overflow(umax, umin_val, &umax); if (min_underflow && !max_underflow) { - *dst_umin = 0; - *dst_umax = U64_MAX; + umin = 0; + umax = U64_MAX; } + + reg_set_srange64(dst_reg, smin, smax); + reg_set_urange64(dst_reg, umin, umax); } static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) { - s32 *dst_smin = &dst_reg->s32_min_value; - s32 *dst_smax = &dst_reg->s32_max_value; - u32 *dst_umin = &dst_reg->u32_min_value; - u32 *dst_umax = &dst_reg->u32_max_value; + s32 smin = reg_s32_min(dst_reg); + s32 smax = reg_s32_max(dst_reg); + u32 umin = reg_u32_min(dst_reg); + u32 umax = reg_u32_max(dst_reg); s32 tmp_prod[4]; - if (check_mul_overflow(*dst_umax, src_reg->u32_max_value, dst_umax) || - check_mul_overflow(*dst_umin, src_reg->u32_min_value, dst_umin)) { + if (check_mul_overflow(umax, reg_u32_max(src_reg), &umax) || + check_mul_overflow(umin, reg_u32_min(src_reg), &umin)) { /* Overflow possible, we know nothing */ - *dst_umin = 0; - *dst_umax = U32_MAX; + umin = 0; + umax = U32_MAX; } - if (check_mul_overflow(*dst_smin, src_reg->s32_min_value, &tmp_prod[0]) || - check_mul_overflow(*dst_smin, src_reg->s32_max_value, &tmp_prod[1]) || - check_mul_overflow(*dst_smax, src_reg->s32_min_value, &tmp_prod[2]) || - check_mul_overflow(*dst_smax, src_reg->s32_max_value, &tmp_prod[3])) { + if (check_mul_overflow(smin, reg_s32_min(src_reg), &tmp_prod[0]) || + check_mul_overflow(smin, reg_s32_max(src_reg), &tmp_prod[1]) || + check_mul_overflow(smax, reg_s32_min(src_reg), &tmp_prod[2]) || + check_mul_overflow(smax, reg_s32_max(src_reg), &tmp_prod[3])) { /* Overflow possible, we know nothing */ - *dst_smin = S32_MIN; - *dst_smax = S32_MAX; + smin = S32_MIN; + smax = S32_MAX; } else { - *dst_smin = min_array(tmp_prod, 4); - *dst_smax = max_array(tmp_prod, 4); + smin = min_array(tmp_prod, 4); + smax = max_array(tmp_prod, 4); } + + reg_set_srange32(dst_reg, smin, smax); + reg_set_urange32(dst_reg, umin, umax); } static void scalar_min_max_mul(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) { - s64 *dst_smin = &dst_reg->smin_value; - s64 *dst_smax = &dst_reg->smax_value; - u64 *dst_umin = &dst_reg->umin_value; - u64 *dst_umax = &dst_reg->umax_value; + s64 smin = reg_smin(dst_reg); + s64 smax = reg_smax(dst_reg); + u64 umin = reg_umin(dst_reg); + u64 umax = reg_umax(dst_reg); s64 tmp_prod[4]; - if (check_mul_overflow(*dst_umax, src_reg->umax_value, dst_umax) || - check_mul_overflow(*dst_umin, src_reg->umin_value, dst_umin)) { + if (check_mul_overflow(umax, reg_umax(src_reg), &umax) || + check_mul_overflow(umin, reg_umin(src_reg), &umin)) { /* Overflow possible, we know nothing */ - *dst_umin = 0; - *dst_umax = U64_MAX; + umin = 0; + umax = U64_MAX; } - if (check_mul_overflow(*dst_smin, src_reg->smin_value, &tmp_prod[0]) || - check_mul_overflow(*dst_smin, src_reg->smax_value, &tmp_prod[1]) || - check_mul_overflow(*dst_smax, src_reg->smin_value, &tmp_prod[2]) || - check_mul_overflow(*dst_smax, src_reg->smax_value, &tmp_prod[3])) { + if (check_mul_overflow(smin, reg_smin(src_reg), &tmp_prod[0]) || + check_mul_overflow(smin, reg_smax(src_reg), &tmp_prod[1]) || + check_mul_overflow(smax, reg_smin(src_reg), &tmp_prod[2]) || + check_mul_overflow(smax, reg_smax(src_reg), &tmp_prod[3])) { /* Overflow possible, we know nothing */ - *dst_smin = S64_MIN; - *dst_smax = S64_MAX; + smin = S64_MIN; + smax = S64_MAX; } else { - *dst_smin = min_array(tmp_prod, 4); - *dst_smax = max_array(tmp_prod, 4); + smin = min_array(tmp_prod, 4); + smax = max_array(tmp_prod, 4); } + + reg_set_srange64(dst_reg, smin, smax); + reg_set_urange64(dst_reg, umin, umax); } static void scalar32_min_max_udiv(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) { - u32 *dst_umin = &dst_reg->u32_min_value; - u32 *dst_umax = &dst_reg->u32_max_value; - u32 src_val = src_reg->u32_min_value; /* non-zero, const divisor */ + u32 src_val = reg_u32_min(src_reg); /* non-zero, const divisor */ - *dst_umin = *dst_umin / src_val; - *dst_umax = *dst_umax / src_val; + reg_set_urange32(dst_reg, reg_u32_min(dst_reg) / src_val, + reg_u32_max(dst_reg) / src_val); /* Reset other ranges/tnum to unbounded/unknown. */ - dst_reg->s32_min_value = S32_MIN; - dst_reg->s32_max_value = S32_MAX; + reg_set_srange32(dst_reg, S32_MIN, S32_MAX); reset_reg64_and_tnum(dst_reg); } static void scalar_min_max_udiv(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) { - u64 *dst_umin = &dst_reg->umin_value; - u64 *dst_umax = &dst_reg->umax_value; - u64 src_val = src_reg->umin_value; /* non-zero, const divisor */ + u64 src_val = reg_umin(src_reg); /* non-zero, const divisor */ - *dst_umin = div64_u64(*dst_umin, src_val); - *dst_umax = div64_u64(*dst_umax, src_val); + reg_set_urange64(dst_reg, div64_u64(reg_umin(dst_reg), src_val), + div64_u64(reg_umax(dst_reg), src_val)); /* Reset other ranges/tnum to unbounded/unknown. */ - dst_reg->smin_value = S64_MIN; - dst_reg->smax_value = S64_MAX; + reg_set_srange64(dst_reg, S64_MIN, S64_MAX); reset_reg32_and_tnum(dst_reg); } static void scalar32_min_max_sdiv(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) { - s32 *dst_smin = &dst_reg->s32_min_value; - s32 *dst_smax = &dst_reg->s32_max_value; - s32 src_val = src_reg->s32_min_value; /* non-zero, const divisor */ + s32 smin = reg_s32_min(dst_reg); + s32 smax = reg_s32_max(dst_reg); + s32 src_val = reg_s32_min(src_reg); /* non-zero, const divisor */ s32 res1, res2; /* BPF div specification: S32_MIN / -1 = S32_MIN */ - if (*dst_smin == S32_MIN && src_val == -1) { + if (smin == S32_MIN && src_val == -1) { /* * If the dividend range contains more than just S32_MIN, * we cannot precisely track the result, so it becomes unbounded. @@ -14225,35 +14227,35 @@ static void scalar32_min_max_sdiv(struct bpf_reg_state *dst_reg, * = {S32_MIN} U [S32_MAX-9, S32_MAX] = [S32_MIN, S32_MAX] * Otherwise (if dividend is exactly S32_MIN), result remains S32_MIN. */ - if (*dst_smax != S32_MIN) { - *dst_smin = S32_MIN; - *dst_smax = S32_MAX; + if (smax != S32_MIN) { + smin = S32_MIN; + smax = S32_MAX; } goto reset; } - res1 = *dst_smin / src_val; - res2 = *dst_smax / src_val; - *dst_smin = min(res1, res2); - *dst_smax = max(res1, res2); + res1 = smin / src_val; + res2 = smax / src_val; + smin = min(res1, res2); + smax = max(res1, res2); reset: + reg_set_srange32(dst_reg, smin, smax); /* Reset other ranges/tnum to unbounded/unknown. */ - dst_reg->u32_min_value = 0; - dst_reg->u32_max_value = U32_MAX; + reg_set_urange32(dst_reg, 0, U32_MAX); reset_reg64_and_tnum(dst_reg); } static void scalar_min_max_sdiv(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) { - s64 *dst_smin = &dst_reg->smin_value; - s64 *dst_smax = &dst_reg->smax_value; - s64 src_val = src_reg->smin_value; /* non-zero, const divisor */ + s64 smin = reg_smin(dst_reg); + s64 smax = reg_smax(dst_reg); + s64 src_val = reg_smin(src_reg); /* non-zero, const divisor */ s64 res1, res2; /* BPF div specification: S64_MIN / -1 = S64_MIN */ - if (*dst_smin == S64_MIN && src_val == -1) { + if (smin == S64_MIN && src_val == -1) { /* * If the dividend range contains more than just S64_MIN, * we cannot precisely track the result, so it becomes unbounded. @@ -14262,79 +14264,69 @@ static void scalar_min_max_sdiv(struct bpf_reg_state *dst_reg, * = {S64_MIN} U [S64_MAX-9, S64_MAX] = [S64_MIN, S64_MAX] * Otherwise (if dividend is exactly S64_MIN), result remains S64_MIN. */ - if (*dst_smax != S64_MIN) { - *dst_smin = S64_MIN; - *dst_smax = S64_MAX; + if (smax != S64_MIN) { + smin = S64_MIN; + smax = S64_MAX; } goto reset; } - res1 = div64_s64(*dst_smin, src_val); - res2 = div64_s64(*dst_smax, src_val); - *dst_smin = min(res1, res2); - *dst_smax = max(res1, res2); + res1 = div64_s64(smin, src_val); + res2 = div64_s64(smax, src_val); + smin = min(res1, res2); + smax = max(res1, res2); reset: + reg_set_srange64(dst_reg, smin, smax); /* Reset other ranges/tnum to unbounded/unknown. */ - dst_reg->umin_value = 0; - dst_reg->umax_value = U64_MAX; + reg_set_urange64(dst_reg, 0, U64_MAX); reset_reg32_and_tnum(dst_reg); } static void scalar32_min_max_umod(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) { - u32 *dst_umin = &dst_reg->u32_min_value; - u32 *dst_umax = &dst_reg->u32_max_value; - u32 src_val = src_reg->u32_min_value; /* non-zero, const divisor */ + u32 src_val = reg_u32_min(src_reg); /* non-zero, const divisor */ u32 res_max = src_val - 1; /* * If dst_umax <= res_max, the result remains unchanged. * e.g., [2, 5] % 10 = [2, 5]. */ - if (*dst_umax <= res_max) + if (reg_u32_max(dst_reg) <= res_max) return; - *dst_umin = 0; - *dst_umax = min(*dst_umax, res_max); + reg_set_urange32(dst_reg, 0, min(reg_u32_max(dst_reg), res_max)); /* Reset other ranges/tnum to unbounded/unknown. */ - dst_reg->s32_min_value = S32_MIN; - dst_reg->s32_max_value = S32_MAX; + reg_set_srange32(dst_reg, S32_MIN, S32_MAX); reset_reg64_and_tnum(dst_reg); } static void scalar_min_max_umod(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) { - u64 *dst_umin = &dst_reg->umin_value; - u64 *dst_umax = &dst_reg->umax_value; - u64 src_val = src_reg->umin_value; /* non-zero, const divisor */ + u64 src_val = reg_umin(src_reg); /* non-zero, const divisor */ u64 res_max = src_val - 1; /* * If dst_umax <= res_max, the result remains unchanged. * e.g., [2, 5] % 10 = [2, 5]. */ - if (*dst_umax <= res_max) + if (reg_umax(dst_reg) <= res_max) return; - *dst_umin = 0; - *dst_umax = min(*dst_umax, res_max); + reg_set_urange64(dst_reg, 0, min(reg_umax(dst_reg), res_max)); /* Reset other ranges/tnum to unbounded/unknown. */ - dst_reg->smin_value = S64_MIN; - dst_reg->smax_value = S64_MAX; + reg_set_srange64(dst_reg, S64_MIN, S64_MAX); reset_reg32_and_tnum(dst_reg); } static void scalar32_min_max_smod(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) { - s32 *dst_smin = &dst_reg->s32_min_value; - s32 *dst_smax = &dst_reg->s32_max_value; - s32 src_val = src_reg->s32_min_value; /* non-zero, const divisor */ + s32 src_val = reg_s32_min(src_reg); /* non-zero, const divisor */ /* * Safe absolute value calculation: @@ -14354,33 +14346,27 @@ static void scalar32_min_max_smod(struct bpf_reg_state *dst_reg, * If the dividend is already within the result range, * the result remains unchanged. e.g., [-2, 5] % 10 = [-2, 5]. */ - if (*dst_smin >= -res_max_abs && *dst_smax <= res_max_abs) + if (reg_s32_min(dst_reg) >= -res_max_abs && reg_s32_max(dst_reg) <= res_max_abs) return; /* General case: result has the same sign as the dividend. */ - if (*dst_smin >= 0) { - *dst_smin = 0; - *dst_smax = min(*dst_smax, res_max_abs); - } else if (*dst_smax <= 0) { - *dst_smax = 0; - *dst_smin = max(*dst_smin, -res_max_abs); + if (reg_s32_min(dst_reg) >= 0) { + reg_set_srange32(dst_reg, 0, min(reg_s32_max(dst_reg), res_max_abs)); + } else if (reg_s32_max(dst_reg) <= 0) { + reg_set_srange32(dst_reg, max(reg_s32_min(dst_reg), -res_max_abs), 0); } else { - *dst_smin = -res_max_abs; - *dst_smax = res_max_abs; + reg_set_srange32(dst_reg, -res_max_abs, res_max_abs); } /* Reset other ranges/tnum to unbounded/unknown. */ - dst_reg->u32_min_value = 0; - dst_reg->u32_max_value = U32_MAX; + reg_set_urange32(dst_reg, 0, U32_MAX); reset_reg64_and_tnum(dst_reg); } static void scalar_min_max_smod(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) { - s64 *dst_smin = &dst_reg->smin_value; - s64 *dst_smax = &dst_reg->smax_value; - s64 src_val = src_reg->smin_value; /* non-zero, const divisor */ + s64 src_val = reg_smin(src_reg); /* non-zero, const divisor */ /* * Safe absolute value calculation: @@ -14400,24 +14386,20 @@ static void scalar_min_max_smod(struct bpf_reg_state *dst_reg, * If the dividend is already within the result range, * the result remains unchanged. e.g., [-2, 5] % 10 = [-2, 5]. */ - if (*dst_smin >= -res_max_abs && *dst_smax <= res_max_abs) + if (reg_smin(dst_reg) >= -res_max_abs && reg_smax(dst_reg) <= res_max_abs) return; /* General case: result has the same sign as the dividend. */ - if (*dst_smin >= 0) { - *dst_smin = 0; - *dst_smax = min(*dst_smax, res_max_abs); - } else if (*dst_smax <= 0) { - *dst_smax = 0; - *dst_smin = max(*dst_smin, -res_max_abs); + if (reg_smin(dst_reg) >= 0) { + reg_set_srange64(dst_reg, 0, min(reg_smax(dst_reg), res_max_abs)); + } else if (reg_smax(dst_reg) <= 0) { + reg_set_srange64(dst_reg, max(reg_smin(dst_reg), -res_max_abs), 0); } else { - *dst_smin = -res_max_abs; - *dst_smax = res_max_abs; + reg_set_srange64(dst_reg, -res_max_abs, res_max_abs); } /* Reset other ranges/tnum to unbounded/unknown. */ - dst_reg->umin_value = 0; - dst_reg->umax_value = U64_MAX; + reg_set_urange64(dst_reg, 0, U64_MAX); reset_reg32_and_tnum(dst_reg); } @@ -14427,7 +14409,7 @@ static void scalar32_min_max_and(struct bpf_reg_state *dst_reg, bool src_known = tnum_subreg_is_const(src_reg->var_off); bool dst_known = tnum_subreg_is_const(dst_reg->var_off); struct tnum var32_off = tnum_subreg(dst_reg->var_off); - u32 umax_val = src_reg->u32_max_value; + u32 umax_val = reg_u32_max(src_reg); if (src_known && dst_known) { __mark_reg32_known(dst_reg, var32_off.value); @@ -14437,19 +14419,15 @@ static void scalar32_min_max_and(struct bpf_reg_state *dst_reg, /* We get our minimum from the var_off, since that's inherently * bitwise. Our maximum is the minimum of the operands' maxima. */ - dst_reg->u32_min_value = var32_off.value; - dst_reg->u32_max_value = min(dst_reg->u32_max_value, umax_val); + reg_set_urange32(dst_reg, var32_off.value, min(reg_u32_max(dst_reg), umax_val)); /* Safe to set s32 bounds by casting u32 result into s32 when u32 * doesn't cross sign boundary. Otherwise set s32 bounds to unbounded. */ - if ((s32)dst_reg->u32_min_value <= (s32)dst_reg->u32_max_value) { - dst_reg->s32_min_value = dst_reg->u32_min_value; - dst_reg->s32_max_value = dst_reg->u32_max_value; - } else { - dst_reg->s32_min_value = S32_MIN; - dst_reg->s32_max_value = S32_MAX; - } + if ((s32)reg_u32_min(dst_reg) <= (s32)reg_u32_max(dst_reg)) + reg_set_srange32(dst_reg, reg_u32_min(dst_reg), reg_u32_max(dst_reg)); + else + reg_set_srange32(dst_reg, S32_MIN, S32_MAX); } static void scalar_min_max_and(struct bpf_reg_state *dst_reg, @@ -14457,7 +14435,7 @@ static void scalar_min_max_and(struct bpf_reg_state *dst_reg, { bool src_known = tnum_is_const(src_reg->var_off); bool dst_known = tnum_is_const(dst_reg->var_off); - u64 umax_val = src_reg->umax_value; + u64 umax_val = reg_umax(src_reg); if (src_known && dst_known) { __mark_reg_known(dst_reg, dst_reg->var_off.value); @@ -14467,19 +14445,15 @@ static void scalar_min_max_and(struct bpf_reg_state *dst_reg, /* We get our minimum from the var_off, since that's inherently * bitwise. Our maximum is the minimum of the operands' maxima. */ - dst_reg->umin_value = dst_reg->var_off.value; - dst_reg->umax_value = min(dst_reg->umax_value, umax_val); + reg_set_urange64(dst_reg, dst_reg->var_off.value, min(reg_umax(dst_reg), umax_val)); /* Safe to set s64 bounds by casting u64 result into s64 when u64 * doesn't cross sign boundary. Otherwise set s64 bounds to unbounded. */ - if ((s64)dst_reg->umin_value <= (s64)dst_reg->umax_value) { - dst_reg->smin_value = dst_reg->umin_value; - dst_reg->smax_value = dst_reg->umax_value; - } else { - dst_reg->smin_value = S64_MIN; - dst_reg->smax_value = S64_MAX; - } + if ((s64)reg_umin(dst_reg) <= (s64)reg_umax(dst_reg)) + reg_set_srange64(dst_reg, reg_umin(dst_reg), reg_umax(dst_reg)); + else + reg_set_srange64(dst_reg, S64_MIN, S64_MAX); /* We may learn something more from the var_off */ __update_reg_bounds(dst_reg); } @@ -14490,7 +14464,7 @@ static void scalar32_min_max_or(struct bpf_reg_state *dst_reg, bool src_known = tnum_subreg_is_const(src_reg->var_off); bool dst_known = tnum_subreg_is_const(dst_reg->var_off); struct tnum var32_off = tnum_subreg(dst_reg->var_off); - u32 umin_val = src_reg->u32_min_value; + u32 umin_val = reg_u32_min(src_reg); if (src_known && dst_known) { __mark_reg32_known(dst_reg, var32_off.value); @@ -14500,19 +14474,16 @@ static void scalar32_min_max_or(struct bpf_reg_state *dst_reg, /* We get our maximum from the var_off, and our minimum is the * maximum of the operands' minima */ - dst_reg->u32_min_value = max(dst_reg->u32_min_value, umin_val); - dst_reg->u32_max_value = var32_off.value | var32_off.mask; + reg_set_urange32(dst_reg, max(reg_u32_min(dst_reg), umin_val), + var32_off.value | var32_off.mask); /* Safe to set s32 bounds by casting u32 result into s32 when u32 * doesn't cross sign boundary. Otherwise set s32 bounds to unbounded. */ - if ((s32)dst_reg->u32_min_value <= (s32)dst_reg->u32_max_value) { - dst_reg->s32_min_value = dst_reg->u32_min_value; - dst_reg->s32_max_value = dst_reg->u32_max_value; - } else { - dst_reg->s32_min_value = S32_MIN; - dst_reg->s32_max_value = S32_MAX; - } + if ((s32)reg_u32_min(dst_reg) <= (s32)reg_u32_max(dst_reg)) + reg_set_srange32(dst_reg, reg_u32_min(dst_reg), reg_u32_max(dst_reg)); + else + reg_set_srange32(dst_reg, S32_MIN, S32_MAX); } static void scalar_min_max_or(struct bpf_reg_state *dst_reg, @@ -14520,7 +14491,7 @@ static void scalar_min_max_or(struct bpf_reg_state *dst_reg, { bool src_known = tnum_is_const(src_reg->var_off); bool dst_known = tnum_is_const(dst_reg->var_off); - u64 umin_val = src_reg->umin_value; + u64 umin_val = reg_umin(src_reg); if (src_known && dst_known) { __mark_reg_known(dst_reg, dst_reg->var_off.value); @@ -14530,19 +14501,16 @@ static void scalar_min_max_or(struct bpf_reg_state *dst_reg, /* We get our maximum from the var_off, and our minimum is the * maximum of the operands' minima */ - dst_reg->umin_value = max(dst_reg->umin_value, umin_val); - dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; + reg_set_urange64(dst_reg, max(reg_umin(dst_reg), umin_val), + dst_reg->var_off.value | dst_reg->var_off.mask); /* Safe to set s64 bounds by casting u64 result into s64 when u64 * doesn't cross sign boundary. Otherwise set s64 bounds to unbounded. */ - if ((s64)dst_reg->umin_value <= (s64)dst_reg->umax_value) { - dst_reg->smin_value = dst_reg->umin_value; - dst_reg->smax_value = dst_reg->umax_value; - } else { - dst_reg->smin_value = S64_MIN; - dst_reg->smax_value = S64_MAX; - } + if ((s64)reg_umin(dst_reg) <= (s64)reg_umax(dst_reg)) + reg_set_srange64(dst_reg, reg_umin(dst_reg), reg_umax(dst_reg)); + else + reg_set_srange64(dst_reg, S64_MIN, S64_MAX); /* We may learn something more from the var_off */ __update_reg_bounds(dst_reg); } @@ -14560,19 +14528,15 @@ static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg, } /* We get both minimum and maximum from the var32_off. */ - dst_reg->u32_min_value = var32_off.value; - dst_reg->u32_max_value = var32_off.value | var32_off.mask; + reg_set_urange32(dst_reg, var32_off.value, var32_off.value | var32_off.mask); /* Safe to set s32 bounds by casting u32 result into s32 when u32 * doesn't cross sign boundary. Otherwise set s32 bounds to unbounded. */ - if ((s32)dst_reg->u32_min_value <= (s32)dst_reg->u32_max_value) { - dst_reg->s32_min_value = dst_reg->u32_min_value; - dst_reg->s32_max_value = dst_reg->u32_max_value; - } else { - dst_reg->s32_min_value = S32_MIN; - dst_reg->s32_max_value = S32_MAX; - } + if ((s32)reg_u32_min(dst_reg) <= (s32)reg_u32_max(dst_reg)) + reg_set_srange32(dst_reg, reg_u32_min(dst_reg), reg_u32_max(dst_reg)); + else + reg_set_srange32(dst_reg, S32_MIN, S32_MAX); } static void scalar_min_max_xor(struct bpf_reg_state *dst_reg, @@ -14588,19 +14552,16 @@ static void scalar_min_max_xor(struct bpf_reg_state *dst_reg, } /* We get both minimum and maximum from the var_off. */ - dst_reg->umin_value = dst_reg->var_off.value; - dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; + reg_set_urange64(dst_reg, dst_reg->var_off.value, + dst_reg->var_off.value | dst_reg->var_off.mask); /* Safe to set s64 bounds by casting u64 result into s64 when u64 * doesn't cross sign boundary. Otherwise set s64 bounds to unbounded. */ - if ((s64)dst_reg->umin_value <= (s64)dst_reg->umax_value) { - dst_reg->smin_value = dst_reg->umin_value; - dst_reg->smax_value = dst_reg->umax_value; - } else { - dst_reg->smin_value = S64_MIN; - dst_reg->smax_value = S64_MAX; - } + if ((s64)reg_umin(dst_reg) <= (s64)reg_umax(dst_reg)) + reg_set_srange64(dst_reg, reg_umin(dst_reg), reg_umax(dst_reg)); + else + reg_set_srange64(dst_reg, S64_MIN, S64_MAX); __update_reg_bounds(dst_reg); } @@ -14611,23 +14572,20 @@ static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, /* We lose all sign bit information (except what we can pick * up from var_off) */ - dst_reg->s32_min_value = S32_MIN; - dst_reg->s32_max_value = S32_MAX; + reg_set_srange32(dst_reg, S32_MIN, S32_MAX); /* If we might shift our top bit out, then we know nothing */ - if (umax_val > 31 || dst_reg->u32_max_value > 1ULL << (31 - umax_val)) { - dst_reg->u32_min_value = 0; - dst_reg->u32_max_value = U32_MAX; - } else { - dst_reg->u32_min_value <<= umin_val; - dst_reg->u32_max_value <<= umax_val; - } + if (umax_val > 31 || reg_u32_max(dst_reg) > 1ULL << (31 - umax_val)) + reg_set_urange32(dst_reg, 0, U32_MAX); + else + reg_set_urange32(dst_reg, reg_u32_min(dst_reg) << umin_val, + reg_u32_max(dst_reg) << umax_val); } static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) { - u32 umax_val = src_reg->u32_max_value; - u32 umin_val = src_reg->u32_min_value; + u32 umax_val = reg_u32_max(src_reg); + u32 umin_val = reg_u32_min(src_reg); /* u32 alu operation will zext upper bits */ struct tnum subreg = tnum_subreg(dst_reg->var_off); @@ -14649,29 +14607,25 @@ static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg, * because s32 bounds don't flip sign when shifting to the left by * 32bits. */ - if (umin_val == 32 && umax_val == 32) { - dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32; - dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32; - } else { - dst_reg->smax_value = S64_MAX; - dst_reg->smin_value = S64_MIN; - } + if (umin_val == 32 && umax_val == 32) + reg_set_srange64(dst_reg, (s64)reg_s32_min(dst_reg) << 32, + (s64)reg_s32_max(dst_reg) << 32); + else + reg_set_srange64(dst_reg, S64_MIN, S64_MAX); /* If we might shift our top bit out, then we know nothing */ - if (dst_reg->umax_value > 1ULL << (63 - umax_val)) { - dst_reg->umin_value = 0; - dst_reg->umax_value = U64_MAX; - } else { - dst_reg->umin_value <<= umin_val; - dst_reg->umax_value <<= umax_val; - } + if (reg_umax(dst_reg) > 1ULL << (63 - umax_val)) + reg_set_urange64(dst_reg, 0, U64_MAX); + else + reg_set_urange64(dst_reg, reg_umin(dst_reg) << umin_val, + reg_umax(dst_reg) << umax_val); } static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) { - u64 umax_val = src_reg->umax_value; - u64 umin_val = src_reg->umin_value; + u64 umax_val = reg_umax(src_reg); + u64 umin_val = reg_umin(src_reg); /* scalar64 calc uses 32bit unshifted bounds so must be called first */ __scalar64_min_max_lsh(dst_reg, umin_val, umax_val); @@ -14686,8 +14640,8 @@ static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) { struct tnum subreg = tnum_subreg(dst_reg->var_off); - u32 umax_val = src_reg->u32_max_value; - u32 umin_val = src_reg->u32_min_value; + u32 umax_val = reg_u32_max(src_reg); + u32 umin_val = reg_u32_min(src_reg); /* BPF_RSH is an unsigned shift. If the value in dst_reg might * be negative, then either: @@ -14703,12 +14657,11 @@ static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg, * and rely on inferring new ones from the unsigned bounds and * var_off of the result. */ - dst_reg->s32_min_value = S32_MIN; - dst_reg->s32_max_value = S32_MAX; + reg_set_srange32(dst_reg, S32_MIN, S32_MAX); dst_reg->var_off = tnum_rshift(subreg, umin_val); - dst_reg->u32_min_value >>= umax_val; - dst_reg->u32_max_value >>= umin_val; + reg_set_urange32(dst_reg, reg_u32_min(dst_reg) >> umax_val, + reg_u32_max(dst_reg) >> umin_val); __mark_reg64_unbounded(dst_reg); __update_reg32_bounds(dst_reg); @@ -14717,8 +14670,8 @@ static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg, static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) { - u64 umax_val = src_reg->umax_value; - u64 umin_val = src_reg->umin_value; + u64 umax_val = reg_umax(src_reg); + u64 umin_val = reg_umin(src_reg); /* BPF_RSH is an unsigned shift. If the value in dst_reg might * be negative, then either: @@ -14734,11 +14687,10 @@ static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg, * and rely on inferring new ones from the unsigned bounds and * var_off of the result. */ - dst_reg->smin_value = S64_MIN; - dst_reg->smax_value = S64_MAX; + reg_set_srange64(dst_reg, S64_MIN, S64_MAX); dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val); - dst_reg->umin_value >>= umax_val; - dst_reg->umax_value >>= umin_val; + reg_set_urange64(dst_reg, reg_umin(dst_reg) >> umax_val, + reg_umax(dst_reg) >> umin_val); /* Its not easy to operate on alu32 bounds here because it depends * on bits being shifted in. Take easy way out and mark unbounded @@ -14751,21 +14703,21 @@ static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg, static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) { - u64 umin_val = src_reg->u32_min_value; + u64 umin_val = reg_u32_min(src_reg); /* Upon reaching here, src_known is true and * umax_val is equal to umin_val. */ - dst_reg->s32_min_value = (u32)(((s32)dst_reg->s32_min_value) >> umin_val); - dst_reg->s32_max_value = (u32)(((s32)dst_reg->s32_max_value) >> umin_val); + reg_set_srange32(dst_reg, + (u32)(((s32)reg_s32_min(dst_reg)) >> umin_val), + (u32)(((s32)reg_s32_max(dst_reg)) >> umin_val)); dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32); /* blow away the dst_reg umin_value/umax_value and rely on * dst_reg var_off to refine the result. */ - dst_reg->u32_min_value = 0; - dst_reg->u32_max_value = U32_MAX; + reg_set_urange32(dst_reg, 0, U32_MAX); __mark_reg64_unbounded(dst_reg); __update_reg32_bounds(dst_reg); @@ -14774,21 +14726,20 @@ static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg, static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) { - u64 umin_val = src_reg->umin_value; + u64 umin_val = reg_umin(src_reg); /* Upon reaching here, src_known is true and umax_val is equal * to umin_val. */ - dst_reg->smin_value >>= umin_val; - dst_reg->smax_value >>= umin_val; + reg_set_srange64(dst_reg, reg_smin(dst_reg) >> umin_val, + reg_smax(dst_reg) >> umin_val); dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64); /* blow away the dst_reg umin_value/umax_value and rely on * dst_reg var_off to refine the result. */ - dst_reg->umin_value = 0; - dst_reg->umax_value = U64_MAX; + reg_set_urange64(dst_reg, 0, U64_MAX); /* Its not easy to operate on alu32 bounds here because it depends * on bits being shifted in from upper 32-bits. Take easy way out @@ -14855,13 +14806,13 @@ static bool is_safe_to_compute_dst_reg_range(struct bpf_insn *insn, if (insn_bitness == 32) { if (tnum_subreg_is_const(src_reg->var_off) - && src_reg->s32_min_value == src_reg->s32_max_value - && src_reg->u32_min_value == src_reg->u32_max_value) + && reg_s32_min(src_reg) == reg_s32_max(src_reg) + && reg_u32_min(src_reg) == reg_u32_max(src_reg)) src_is_const = true; } else { if (tnum_is_const(src_reg->var_off) - && src_reg->smin_value == src_reg->smax_value - && src_reg->umin_value == src_reg->umax_value) + && reg_smin(src_reg) == reg_smax(src_reg) + && reg_umin(src_reg) == reg_umax(src_reg)) src_is_const = true; } @@ -14891,7 +14842,7 @@ static bool is_safe_to_compute_dst_reg_range(struct bpf_insn *insn, case BPF_LSH: case BPF_RSH: case BPF_ARSH: - return (src_is_const && src_reg->umax_value < insn_bitness); + return (src_is_const && reg_umax(src_reg) < insn_bitness); default: return false; } @@ -14904,9 +14855,9 @@ static int maybe_fork_scalars(struct bpf_verifier_env *env, struct bpf_insn *ins struct bpf_reg_state *regs; bool alu32; - if (dst_reg->smin_value == -1 && dst_reg->smax_value == 0) + if (reg_smin(dst_reg) == -1 && reg_smax(dst_reg) == 0) alu32 = false; - else if (dst_reg->s32_min_value == -1 && dst_reg->s32_max_value == 0) + else if (reg_s32_min(dst_reg) == -1 && reg_s32_max(dst_reg) == 0) alu32 = true; else return 0; @@ -14990,7 +14941,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, break; case BPF_DIV: /* BPF div specification: x / 0 = 0 */ - if ((alu32 && src_reg.u32_min_value == 0) || (!alu32 && src_reg.umin_value == 0)) { + if ((alu32 && reg_u32_min(&src_reg) == 0) || (!alu32 && reg_umin(&src_reg) == 0)) { ___mark_reg_known(dst_reg, 0); break; } @@ -15007,7 +14958,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, break; case BPF_MOD: /* BPF mod specification: x % 0 = x */ - if ((alu32 && src_reg.u32_min_value == 0) || (!alu32 && src_reg.umin_value == 0)) + if ((alu32 && reg_u32_min(&src_reg) == 0) || (!alu32 && reg_umin(&src_reg) == 0)) break; if (alu32) if (off == 1) @@ -15195,7 +15146,7 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, * umax_value before the ALU operation. After adjust_scalar_min_max_vals(), * alu32 ops will have zero-extended the result, making umax_value <= U32_MAX. */ - u64 dst_umax = dst_reg->umax_value; + u64 dst_umax = reg_umax(dst_reg); err = adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); if (err) @@ -15337,7 +15288,7 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) } else if (src_reg->type == SCALAR_VALUE) { bool no_sext; - no_sext = src_reg->umax_value < (1ULL << (insn->off - 1)); + no_sext = reg_umax(src_reg) < (1ULL << (insn->off - 1)); if (no_sext) assign_scalar_id_before_mov(env, src_reg); copy_register_state(dst_reg, src_reg); @@ -15372,7 +15323,7 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) dst_reg->subreg_def = env->insn_idx + 1; } else { /* case: W1 = (s8, s16)W2 */ - bool no_sext = src_reg->umax_value < (1ULL << (insn->off - 1)); + bool no_sext = reg_umax(src_reg) < (1ULL << (insn->off - 1)); if (no_sext) assign_scalar_id_before_mov(env, src_reg); @@ -15454,17 +15405,17 @@ static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, struct bpf_reg_state *reg; int new_range; - if (dst_reg->umax_value == 0 && range_right_open) + if (reg_umax(dst_reg) == 0 && range_right_open) /* This doesn't give us any range */ return; - if (dst_reg->umax_value > MAX_PACKET_OFF) + if (reg_umax(dst_reg) > MAX_PACKET_OFF) /* Risk of overflow. For instance, ptr + (1<<63) may be less * than pkt_end, but that's because it's also less than pkt. */ return; - new_range = dst_reg->umax_value; + new_range = reg_umax(dst_reg); if (range_right_open) new_range++; @@ -15513,7 +15464,7 @@ static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, /* If our ids match, then we must have the same max_value. And we * don't care about the other reg's fixed offset, since if it's too big * the range won't allow anything. - * dst_reg->umax_value is known < MAX_PACKET_OFF, therefore it fits in a u16. + * reg_umax(dst_reg) is known < MAX_PACKET_OFF, therefore it fits in a u16. */ bpf_for_each_reg_in_vstate(vstate, state, reg, ({ if (reg->type == type && reg->id == dst_reg->id) @@ -15569,14 +15520,14 @@ static int is_scalar_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_s { struct tnum t1 = is_jmp32 ? tnum_subreg(reg1->var_off) : reg1->var_off; struct tnum t2 = is_jmp32 ? tnum_subreg(reg2->var_off) : reg2->var_off; - u64 umin1 = is_jmp32 ? (u64)reg1->u32_min_value : reg1->umin_value; - u64 umax1 = is_jmp32 ? (u64)reg1->u32_max_value : reg1->umax_value; - s64 smin1 = is_jmp32 ? (s64)reg1->s32_min_value : reg1->smin_value; - s64 smax1 = is_jmp32 ? (s64)reg1->s32_max_value : reg1->smax_value; - u64 umin2 = is_jmp32 ? (u64)reg2->u32_min_value : reg2->umin_value; - u64 umax2 = is_jmp32 ? (u64)reg2->u32_max_value : reg2->umax_value; - s64 smin2 = is_jmp32 ? (s64)reg2->s32_min_value : reg2->smin_value; - s64 smax2 = is_jmp32 ? (s64)reg2->s32_max_value : reg2->smax_value; + u64 umin1 = is_jmp32 ? (u64)reg_u32_min(reg1) : reg_umin(reg1); + u64 umax1 = is_jmp32 ? (u64)reg_u32_max(reg1) : reg_umax(reg1); + s64 smin1 = is_jmp32 ? (s64)reg_s32_min(reg1) : reg_smin(reg1); + s64 smax1 = is_jmp32 ? (s64)reg_s32_max(reg1) : reg_smax(reg1); + u64 umin2 = is_jmp32 ? (u64)reg_u32_min(reg2) : reg_umin(reg2); + u64 umax2 = is_jmp32 ? (u64)reg_u32_max(reg2) : reg_umax(reg2); + s64 smin2 = is_jmp32 ? (s64)reg_s32_min(reg2) : reg_smin(reg2); + s64 smax2 = is_jmp32 ? (s64)reg_s32_max(reg2) : reg_smax(reg2); if (reg1 == reg2) { switch (opcode) { @@ -15621,11 +15572,11 @@ static int is_scalar_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_s * utilize 32-bit subrange knowledge to eliminate * branches that can't be taken a priori */ - if (reg1->u32_min_value > reg2->u32_max_value || - reg1->u32_max_value < reg2->u32_min_value) + if (reg_u32_min(reg1) > reg_u32_max(reg2) || + reg_u32_max(reg1) < reg_u32_min(reg2)) return 0; - if (reg1->s32_min_value > reg2->s32_max_value || - reg1->s32_max_value < reg2->s32_min_value) + if (reg_s32_min(reg1) > reg_s32_max(reg2) || + reg_s32_max(reg1) < reg_s32_min(reg2)) return 0; } break; @@ -15647,11 +15598,11 @@ static int is_scalar_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_s * utilize 32-bit subrange knowledge to eliminate * branches that can't be taken a priori */ - if (reg1->u32_min_value > reg2->u32_max_value || - reg1->u32_max_value < reg2->u32_min_value) + if (reg_u32_min(reg1) > reg_u32_max(reg2) || + reg_u32_max(reg1) < reg_u32_min(reg2)) return 1; - if (reg1->s32_min_value > reg2->s32_max_value || - reg1->s32_max_value < reg2->s32_min_value) + if (reg_s32_min(reg1) > reg_s32_max(reg2) || + reg_s32_max(reg1) < reg_s32_min(reg2)) return 1; } break; @@ -15878,27 +15829,23 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state switch (opcode) { case BPF_JEQ: if (is_jmp32) { - reg1->u32_min_value = max(reg1->u32_min_value, reg2->u32_min_value); - reg1->u32_max_value = min(reg1->u32_max_value, reg2->u32_max_value); - reg1->s32_min_value = max(reg1->s32_min_value, reg2->s32_min_value); - reg1->s32_max_value = min(reg1->s32_max_value, reg2->s32_max_value); - reg2->u32_min_value = reg1->u32_min_value; - reg2->u32_max_value = reg1->u32_max_value; - reg2->s32_min_value = reg1->s32_min_value; - reg2->s32_max_value = reg1->s32_max_value; + reg_set_urange32(reg1, max(reg_u32_min(reg1), reg_u32_min(reg2)), + min(reg_u32_max(reg1), reg_u32_max(reg2))); + reg_set_srange32(reg1, max(reg_s32_min(reg1), reg_s32_min(reg2)), + min(reg_s32_max(reg1), reg_s32_max(reg2))); + reg_set_urange32(reg2, reg_u32_min(reg1), reg_u32_max(reg1)); + reg_set_srange32(reg2, reg_s32_min(reg1), reg_s32_max(reg1)); t = tnum_intersect(tnum_subreg(reg1->var_off), tnum_subreg(reg2->var_off)); reg1->var_off = tnum_with_subreg(reg1->var_off, t); reg2->var_off = tnum_with_subreg(reg2->var_off, t); } else { - reg1->umin_value = max(reg1->umin_value, reg2->umin_value); - reg1->umax_value = min(reg1->umax_value, reg2->umax_value); - reg1->smin_value = max(reg1->smin_value, reg2->smin_value); - reg1->smax_value = min(reg1->smax_value, reg2->smax_value); - reg2->umin_value = reg1->umin_value; - reg2->umax_value = reg1->umax_value; - reg2->smin_value = reg1->smin_value; - reg2->smax_value = reg1->smax_value; + reg_set_urange64(reg1, max(reg_umin(reg1), reg_umin(reg2)), + min(reg_umax(reg1), reg_umax(reg2))); + reg_set_srange64(reg1, max(reg_smin(reg1), reg_smin(reg2)), + min(reg_smax(reg1), reg_smax(reg2))); + reg_set_urange64(reg2, reg_umin(reg1), reg_umax(reg1)); + reg_set_srange64(reg2, reg_smin(reg1), reg_smax(reg1)); reg1->var_off = tnum_intersect(reg1->var_off, reg2->var_off); reg2->var_off = reg1->var_off; @@ -15915,8 +15862,8 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state */ val = reg_const_value(reg2, is_jmp32); if (is_jmp32) { - /* u32_min_value is not equal to 0xffffffff at this point, - * because otherwise u32_max_value is 0xffffffff as well, + /* u32_min is not equal to 0xffffffff at this point, + * because otherwise u32_max is 0xffffffff as well, * in such a case both reg1 and reg2 would be constants, * jump would be predicted and regs_refine_cond_op() * wouldn't be called. @@ -15924,23 +15871,23 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state * Same reasoning works for all {u,s}{min,max}{32,64} cases * below. */ - if (reg1->u32_min_value == (u32)val) - reg1->u32_min_value++; - if (reg1->u32_max_value == (u32)val) - reg1->u32_max_value--; - if (reg1->s32_min_value == (s32)val) - reg1->s32_min_value++; - if (reg1->s32_max_value == (s32)val) - reg1->s32_max_value--; + if (reg_u32_min(reg1) == (u32)val) + reg_set_urange32(reg1, reg_u32_min(reg1) + 1, reg_u32_max(reg1)); + if (reg_u32_max(reg1) == (u32)val) + reg_set_urange32(reg1, reg_u32_min(reg1), reg_u32_max(reg1) - 1); + if (reg_s32_min(reg1) == (s32)val) + reg_set_srange32(reg1, reg_s32_min(reg1) + 1, reg_s32_max(reg1)); + if (reg_s32_max(reg1) == (s32)val) + reg_set_srange32(reg1, reg_s32_min(reg1), reg_s32_max(reg1) - 1); } else { - if (reg1->umin_value == (u64)val) - reg1->umin_value++; - if (reg1->umax_value == (u64)val) - reg1->umax_value--; - if (reg1->smin_value == (s64)val) - reg1->smin_value++; - if (reg1->smax_value == (s64)val) - reg1->smax_value--; + if (reg_umin(reg1) == (u64)val) + reg_set_urange64(reg1, reg_umin(reg1) + 1, reg_umax(reg1)); + if (reg_umax(reg1) == (u64)val) + reg_set_urange64(reg1, reg_umin(reg1), reg_umax(reg1) - 1); + if (reg_smin(reg1) == (s64)val) + reg_set_srange64(reg1, reg_smin(reg1) + 1, reg_smax(reg1)); + if (reg_smax(reg1) == (s64)val) + reg_set_srange64(reg1, reg_smin(reg1), reg_smax(reg1) - 1); } break; case BPF_JSET: @@ -15987,38 +15934,38 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state break; case BPF_JLE: if (is_jmp32) { - reg1->u32_max_value = min(reg1->u32_max_value, reg2->u32_max_value); - reg2->u32_min_value = max(reg1->u32_min_value, reg2->u32_min_value); + reg_set_urange32(reg1, reg_u32_min(reg1), min(reg_u32_max(reg1), reg_u32_max(reg2))); + reg_set_urange32(reg2, max(reg_u32_min(reg1), reg_u32_min(reg2)), reg_u32_max(reg2)); } else { - reg1->umax_value = min(reg1->umax_value, reg2->umax_value); - reg2->umin_value = max(reg1->umin_value, reg2->umin_value); + reg_set_urange64(reg1, reg_umin(reg1), min(reg_umax(reg1), reg_umax(reg2))); + reg_set_urange64(reg2, max(reg_umin(reg1), reg_umin(reg2)), reg_umax(reg2)); } break; case BPF_JLT: if (is_jmp32) { - reg1->u32_max_value = min(reg1->u32_max_value, reg2->u32_max_value - 1); - reg2->u32_min_value = max(reg1->u32_min_value + 1, reg2->u32_min_value); + reg_set_urange32(reg1, reg_u32_min(reg1), min(reg_u32_max(reg1), reg_u32_max(reg2) - 1)); + reg_set_urange32(reg2, max(reg_u32_min(reg1) + 1, reg_u32_min(reg2)), reg_u32_max(reg2)); } else { - reg1->umax_value = min(reg1->umax_value, reg2->umax_value - 1); - reg2->umin_value = max(reg1->umin_value + 1, reg2->umin_value); + reg_set_urange64(reg1, reg_umin(reg1), min(reg_umax(reg1), reg_umax(reg2) - 1)); + reg_set_urange64(reg2, max(reg_umin(reg1) + 1, reg_umin(reg2)), reg_umax(reg2)); } break; case BPF_JSLE: if (is_jmp32) { - reg1->s32_max_value = min(reg1->s32_max_value, reg2->s32_max_value); - reg2->s32_min_value = max(reg1->s32_min_value, reg2->s32_min_value); + reg_set_srange32(reg1, reg_s32_min(reg1), min(reg_s32_max(reg1), reg_s32_max(reg2))); + reg_set_srange32(reg2, max(reg_s32_min(reg1), reg_s32_min(reg2)), reg_s32_max(reg2)); } else { - reg1->smax_value = min(reg1->smax_value, reg2->smax_value); - reg2->smin_value = max(reg1->smin_value, reg2->smin_value); + reg_set_srange64(reg1, reg_smin(reg1), min(reg_smax(reg1), reg_smax(reg2))); + reg_set_srange64(reg2, max(reg_smin(reg1), reg_smin(reg2)), reg_smax(reg2)); } break; case BPF_JSLT: if (is_jmp32) { - reg1->s32_max_value = min(reg1->s32_max_value, reg2->s32_max_value - 1); - reg2->s32_min_value = max(reg1->s32_min_value + 1, reg2->s32_min_value); + reg_set_srange32(reg1, reg_s32_min(reg1), min(reg_s32_max(reg1), reg_s32_max(reg2) - 1)); + reg_set_srange32(reg2, max(reg_s32_min(reg1) + 1, reg_s32_min(reg2)), reg_s32_max(reg2)); } else { - reg1->smax_value = min(reg1->smax_value, reg2->smax_value - 1); - reg2->smin_value = max(reg1->smin_value + 1, reg2->smin_value); + reg_set_srange64(reg1, reg_smin(reg1), min(reg_smax(reg1), reg_smax(reg2) - 1)); + reg_set_srange64(reg2, max(reg_smin(reg1) + 1, reg_smin(reg2)), reg_smax(reg2)); } break; default: @@ -17519,16 +17466,16 @@ static int indirect_jump_min_max_index(struct bpf_verifier_env *env, u32 *pmin_index, u32 *pmax_index) { struct bpf_reg_state *reg = reg_state(env, regno); - u64 min_index = reg->umin_value; - u64 max_index = reg->umax_value; + u64 min_index = reg_umin(reg); + u64 max_index = reg_umax(reg); const u32 size = 8; if (min_index > (u64) U32_MAX * size) { - verbose(env, "the sum of R%u umin_value %llu is too big\n", regno, reg->umin_value); + verbose(env, "the sum of R%u umin_value %llu is too big\n", regno, reg_umin(reg)); return -ERANGE; } if (max_index > (u64) U32_MAX * size) { - verbose(env, "the sum of R%u umax_value %llu is too big\n", regno, reg->umax_value); + verbose(env, "the sum of R%u umax_value %llu is too big\n", regno, reg_umax(reg)); return -ERANGE; } -- cgit v1.2.3 From bbc631085503a7fde9617be18b0657cc9a83910a Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Fri, 24 Apr 2026 15:52:44 -0700 Subject: bpf: replace min/max fields with struct cnum{32,64} Replace eight independent s64, u64, s32, u32 min/max fields in bpf_reg_state with two circular number fields: - cnum64 for a unified signed/unsigned 64-bit range tracking; - cnum32 for a unified signed/unsigned 32-bit range tracking. Each cnum represents a range as a single arc on the circular number line (base + size), from which signed and unsigned bounds are derived on demand via accessor functions introduced in the preceding commit. Notable changes: - Signed<->unsigned deductions in __reg_deduce_bounds() are removed. - 64<->32 bit deductions are replaced with: - reg->r32 = cnum32_intersect(reg->r32, cnum32_from_cnum64(reg->r64)); this is functionally equivalent to the old code. - reg->r64 = cnum64_cnum32_intersect(reg->r64, reg->r32); this handles a few additional cases, see commit message for "bpf: representation and basic operations on circular numbers". - regs_refine_cond_op() now computes results in terms of operations on sets, e.g. for JNE: /* Complement of the range [val, val] as cnum64. */ lo = (struct cnum64){ val + 1, U64_MAX - 1 }; reg1->r64 = cnum64_intersect(reg1->r64, lo); - For add, sub operations on scalars replace explicit bounds computations with cnum{32,64}_{add,negate}. - For add, sub operations on pointers deduplicate with arithmetic operations on scalars and use cnum{32,64}_{add,negate}. - For and, or, xor operations on scalars remove explicit signed bounds computations. - range_bounds_violation() reduces to checking cnum_is_empty(). - const_tnum_range_mismatch() reduces to checking cnum_is_const(). Selftest adjustments: a few existing tests are updated because a single cnum arc cannot always represent what the old system expressed as the intersection of independent signed and unsigned ranges. For example, if the old system tracked u64=[0, U64_MAX-U32_MAX+2] and s64=[S64_MIN+2, 2] independently, their intersection is a tight two-point set. A single cnum must pick the shorter arc, losing the other constraint. These cases are documented with comments in the adjusted tests. reg_bounds.c is updated with logic similar to cnum64_cnum32_intersect(). Instead of using cnums it inspects intersection between 'b' and first / last / next-after-first / previous-before-last sub-ranges of 'a'. reg_bounds.c is also updated to skip test cases that rely in signed and unsigned ranges intersecting in two intervals, as such cases are not representable by a single cnum. The following "crafted" test cases are affected: - reg_bounds_crafted/(s64)[0xffffffffffff8000; 0x7fff] (u32) [0; 0x1f] - reg_bounds_crafted/(s64)[0; 0x1f] (u32) [0xffffffffffffff80; 0x7f] - reg_bounds_crafted/(s64)[0xffffffffffffff80; 0x7f] (u32) [0; 0x1f] - reg_bounds_crafted/(u64)[0; 1] (s32) [1; 2147483648] - reg_bounds_crafted/(u64)[1; 2147483648] (s32) [0; 1] - reg_bounds_crafted/(u64)[0; 0xffffffff00000000] (s64) 0 - reg_bounds_crafted/(u64)0 (s64) [0; 0xffffffff00000000] - reg_bounds_crafted/(u64)[0; 0xffffffff00000000] (s32) 0 - reg_bounds_crafted/(u64)0 (s32) [0; 0xffffffff00000000] - reg_bounds_crafted/(s64)[S64_MIN; 0] (u64) S64_MIN - reg_bounds_crafted/(s64)S64_MIN (u64) [S64_MIN; 0] - reg_bounds_crafted/(s32)[S32_MIN; 0] (u32) S32_MIN - reg_bounds_crafted/(s32)S32_MIN (u32) [S32_MIN; 0] - reg_bounds_crafted/(s64)[0; 0x1f] (u32) [0xffffffff80000000; 0x7fffffff] - reg_bounds_crafted/(s64)[0xffffffff80000000; 0x7fffffff] (u32) [0; 0x1f] - reg_bounds_crafted/(s64)[0; 0x1f] (u32) [0xffffffffffff8000; 0x7fff] As well as some reg_bounds_roand_{consts,ranges}_A_B, where A and B differ in sign domain. Signed-off-by: Eduard Zingerman Link: https://lore.kernel.org/r/20260424-cnums-everywhere-rfc-v1-v3-3-ca434b39a486@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 843 +++++++------------------------------------------- 1 file changed, 118 insertions(+), 725 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index b91d2789e7b9..03f9e16c2abe 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -1796,10 +1797,8 @@ static const int caller_saved[CALLER_SAVED_REGS] = { static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm) { reg->var_off = tnum_const(imm); - reg_set_srange64(reg, (s64)imm, (s64)imm); - reg_set_urange64(reg, imm, imm); - reg_set_srange32(reg, (s32)imm, (s32)imm); - reg_set_urange32(reg, (u32)imm, (u32)imm); + reg->r64 = cnum64_from_urange(imm, imm); + reg->r32 = cnum32_from_urange((u32)imm, (u32)imm); } /* Mark the unknown part of a register (variable offset or scalar value) as @@ -1818,8 +1817,7 @@ static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm) static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm) { reg->var_off = tnum_const_subreg(reg->var_off, imm); - reg_set_srange32(reg, (s32)imm, (s32)imm); - reg_set_urange32(reg, (u32)imm, (u32)imm); + reg->r32 = cnum32_from_urange((u32)imm, (u32)imm); } /* Mark the 'variable offset' part of a register as zero. This should be @@ -1932,23 +1930,19 @@ static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg, static void __mark_reg32_unbounded(struct bpf_reg_state *reg) { - reg_set_srange32(reg, S32_MIN, S32_MAX); - reg_set_urange32(reg, 0, U32_MAX); + reg->r32 = CNUM32_UNBOUNDED; } -/* Reset the min/max bounds of a register */ -static void __mark_reg_unbounded(struct bpf_reg_state *reg) +static void __mark_reg64_unbounded(struct bpf_reg_state *reg) { - reg_set_srange64(reg, S64_MIN, S64_MAX); - reg_set_urange64(reg, 0, U64_MAX); - - __mark_reg32_unbounded(reg); + reg->r64 = CNUM64_UNBOUNDED; } -static void __mark_reg64_unbounded(struct bpf_reg_state *reg) +/* Reset the min/max bounds of a register */ +static void __mark_reg_unbounded(struct bpf_reg_state *reg) { - reg_set_srange64(reg, S64_MIN, S64_MAX); - reg_set_urange64(reg, 0, U64_MAX); + __mark_reg64_unbounded(reg); + __mark_reg32_unbounded(reg); } static void reset_reg64_and_tnum(struct bpf_reg_state *reg) @@ -1963,18 +1957,32 @@ static void reset_reg32_and_tnum(struct bpf_reg_state *reg) reg->var_off = tnum_unknown; } -static void __update_reg32_bounds(struct bpf_reg_state *reg) +static struct cnum32 cnum32_from_tnum(struct tnum tnum) { - struct tnum var32_off = tnum_subreg(reg->var_off); + tnum = tnum_subreg(tnum); + if ((tnum.mask & S32_MIN) || (tnum.value & S32_MIN)) + /* min signed is max(sign bit) | min(other bits) */ + /* max signed is min(sign bit) | max(other bits) */ + return cnum32_from_srange(tnum.value | (tnum.mask & S32_MIN), + tnum.value | (tnum.mask & S32_MAX)); + else + return cnum32_from_urange(tnum.value, (tnum.value | tnum.mask)); +} - reg_set_srange32(reg, - /* min signed is max(sign bit) | min(other bits) */ - max_t(s32, reg_s32_min(reg), var32_off.value | (var32_off.mask & S32_MIN)), - /* max signed is min(sign bit) | max(other bits) */ - min_t(s32, reg_s32_max(reg), var32_off.value | (var32_off.mask & S32_MAX))); - reg_set_urange32(reg, - max_t(u32, reg_u32_min(reg), (u32)var32_off.value), - min(reg_u32_max(reg), (u32)(var32_off.value | var32_off.mask))); +static struct cnum64 cnum64_from_tnum(struct tnum tnum) +{ + if ((tnum.mask & S64_MIN) || (tnum.value & S64_MIN)) + /* min signed is max(sign bit) | min(other bits) */ + /* max signed is min(sign bit) | max(other bits) */ + return cnum64_from_srange(tnum.value | (tnum.mask & S64_MIN), + tnum.value | (tnum.mask & S64_MAX)); + else + return cnum64_from_urange(tnum.value, (tnum.value | tnum.mask)); +} + +static void __update_reg32_bounds(struct bpf_reg_state *reg) +{ + cnum32_intersect_with(®->r32, cnum32_from_tnum(reg->var_off)); } static void __update_reg64_bounds(struct bpf_reg_state *reg) @@ -1982,17 +1990,7 @@ static void __update_reg64_bounds(struct bpf_reg_state *reg) u64 tnum_next, tmax; bool umin_in_tnum; - /* min signed is max(sign bit) | min(other bits) */ - /* max signed is min(sign bit) | max(other bits) */ - reg_set_srange64(reg, - max_t(s64, reg_smin(reg), - reg->var_off.value | (reg->var_off.mask & S64_MIN)), - min_t(s64, reg_smax(reg), - reg->var_off.value | (reg->var_off.mask & S64_MAX))); - reg_set_urange64(reg, - max(reg_umin(reg), reg->var_off.value), - min(reg_umax(reg), - reg->var_off.value | reg->var_off.mask)); + cnum64_intersect_with(®->r64, cnum64_from_tnum(reg->var_off)); /* Check if u64 and tnum overlap in a single value */ tnum_next = tnum_step(reg->var_off, reg_umin(reg)); @@ -2028,343 +2026,19 @@ static void __update_reg_bounds(struct bpf_reg_state *reg) __update_reg64_bounds(reg); } -/* Uses signed min/max values to inform unsigned, and vice-versa */ static void deduce_bounds_32_from_64(struct bpf_reg_state *reg) { - /* If upper 32 bits of u64/s64 range don't change, we can use lower 32 - * bits to improve our u32/s32 boundaries. - * - * E.g., the case where we have upper 32 bits as zero ([10, 20] in - * u64) is pretty trivial, it's obvious that in u32 we'll also have - * [10, 20] range. But this property holds for any 64-bit range as - * long as upper 32 bits in that entire range of values stay the same. - * - * E.g., u64 range [0x10000000A, 0x10000000F] ([4294967306, 4294967311] - * in decimal) has the same upper 32 bits throughout all the values in - * that range. As such, lower 32 bits form a valid [0xA, 0xF] ([10, 15]) - * range. - * - * Note also, that [0xA, 0xF] is a valid range both in u32 and in s32, - * following the rules outlined below about u64/s64 correspondence - * (which equally applies to u32 vs s32 correspondence). In general it - * depends on actual hexadecimal values of 32-bit range. They can form - * only valid u32, or only valid s32 ranges in some cases. - * - * So we use all these insights to derive bounds for subregisters here. - */ - if ((reg_umin(reg) >> 32) == (reg_umax(reg) >> 32)) { - /* u64 to u32 casting preserves validity of low 32 bits as - * a range, if upper 32 bits are the same - */ - reg_set_urange32(reg, - max_t(u32, reg_u32_min(reg), (u32)reg_umin(reg)), - min_t(u32, reg_u32_max(reg), (u32)reg_umax(reg))); - - if ((s32)reg_umin(reg) <= (s32)reg_umax(reg)) { - reg_set_srange32(reg, - max_t(s32, reg_s32_min(reg), (s32)reg_umin(reg)), - min_t(s32, reg_s32_max(reg), (s32)reg_umax(reg))); - } - } - if ((reg_smin(reg) >> 32) == (reg_smax(reg) >> 32)) { - /* low 32 bits should form a proper u32 range */ - if ((u32)reg_smin(reg) <= (u32)reg_smax(reg)) { - reg_set_urange32(reg, - max_t(u32, reg_u32_min(reg), (u32)reg_smin(reg)), - min_t(u32, reg_u32_max(reg), (u32)reg_smax(reg))); - } - /* low 32 bits should form a proper s32 range */ - if ((s32)reg_smin(reg) <= (s32)reg_smax(reg)) { - reg_set_srange32(reg, - max_t(s32, reg_s32_min(reg), (s32)reg_smin(reg)), - min_t(s32, reg_s32_max(reg), (s32)reg_smax(reg))); - } - } - /* Special case where upper bits form a small sequence of two - * sequential numbers (in 32-bit unsigned space, so 0xffffffff to - * 0x00000000 is also valid), while lower bits form a proper s32 range - * going from negative numbers to positive numbers. E.g., let's say we - * have s64 range [-1, 1] ([0xffffffffffffffff, 0x0000000000000001]). - * Possible s64 values are {-1, 0, 1} ({0xffffffffffffffff, - * 0x0000000000000000, 0x00000000000001}). Ignoring upper 32 bits, - * we still get a valid s32 range [-1, 1] ([0xffffffff, 0x00000001]). - * Note that it doesn't have to be 0xffffffff going to 0x00000000 in - * upper 32 bits. As a random example, s64 range - * [0xfffffff0fffffff0; 0xfffffff100000010], forms a valid s32 range - * [-16, 16] ([0xfffffff0; 0x00000010]) in its 32 bit subregister. - */ - if ((u32)(reg_umin(reg) >> 32) + 1 == (u32)(reg_umax(reg) >> 32) && - (s32)reg_umin(reg) < 0 && (s32)reg_umax(reg) >= 0) { - reg_set_srange32(reg, - max_t(s32, reg_s32_min(reg), (s32)reg_umin(reg)), - min_t(s32, reg_s32_max(reg), (s32)reg_umax(reg))); - } - if ((u32)(reg_smin(reg) >> 32) + 1 == (u32)(reg_smax(reg) >> 32) && - (s32)reg_smin(reg) < 0 && (s32)reg_smax(reg) >= 0) { - reg_set_srange32(reg, - max_t(s32, reg_s32_min(reg), (s32)reg_smin(reg)), - min_t(s32, reg_s32_max(reg), (s32)reg_smax(reg))); - } -} - -static void deduce_bounds_32_from_32(struct bpf_reg_state *reg) -{ - /* if u32 range forms a valid s32 range (due to matching sign bit), - * try to learn from that - */ - if ((s32)reg_u32_min(reg) <= (s32)reg_u32_max(reg)) { - reg_set_srange32(reg, - max_t(s32, reg_s32_min(reg), reg_u32_min(reg)), - min_t(s32, reg_s32_max(reg), reg_u32_max(reg))); - } - /* If we cannot cross the sign boundary, then signed and unsigned bounds - * are the same, so combine. This works even in the negative case, e.g. - * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. - */ - if ((u32)reg_s32_min(reg) <= (u32)reg_s32_max(reg)) { - reg_set_urange32(reg, - max_t(u32, reg_s32_min(reg), reg_u32_min(reg)), - min_t(u32, reg_s32_max(reg), reg_u32_max(reg))); - } else { - if (reg_u32_max(reg) < (u32)reg_s32_min(reg)) { - /* See __reg64_deduce_bounds() for detailed explanation. - * Refine ranges in the following situation: - * - * 0 U32_MAX - * | [xxxxxxxxxxxxxx u32 range xxxxxxxxxxxxxx] | - * |----------------------------|----------------------------| - * |xxxxx s32 range xxxxxxxxx] [xxxxxxx| - * 0 S32_MAX S32_MIN -1 - */ - reg_set_srange32(reg, (s32)reg_u32_min(reg), reg_s32_max(reg)); - reg_set_urange32(reg, - reg_u32_min(reg), - min_t(u32, reg_u32_max(reg), reg_s32_max(reg))); - } else if ((u32)reg_s32_max(reg) < reg_u32_min(reg)) { - /* - * 0 U32_MAX - * | [xxxxxxxxxxxxxx u32 range xxxxxxxxxxxxxx] | - * |----------------------------|----------------------------| - * |xxxxxxxxx] [xxxxxxxxxxxx s32 range | - * 0 S32_MAX S32_MIN -1 - */ - reg_set_srange32(reg, reg_s32_min(reg), (s32)reg_u32_max(reg)); - reg_set_urange32(reg, - max_t(u32, reg_u32_min(reg), reg_s32_min(reg)), - reg_u32_max(reg)); - } - } -} - -static void deduce_bounds_64_from_64(struct bpf_reg_state *reg) -{ - /* If u64 range forms a valid s64 range (due to matching sign bit), - * try to learn from that. Let's do a bit of ASCII art to see when - * this is happening. Let's take u64 range first: - * - * 0 0x7fffffffffffffff 0x8000000000000000 U64_MAX - * |-------------------------------|--------------------------------| - * - * Valid u64 range is formed when umin and umax are anywhere in the - * range [0, U64_MAX], and umin <= umax. u64 case is simple and - * straightforward. Let's see how s64 range maps onto the same range - * of values, annotated below the line for comparison: - * - * 0 0x7fffffffffffffff 0x8000000000000000 U64_MAX - * |-------------------------------|--------------------------------| - * 0 S64_MAX S64_MIN -1 - * - * So s64 values basically start in the middle and they are logically - * contiguous to the right of it, wrapping around from -1 to 0, and - * then finishing as S64_MAX (0x7fffffffffffffff) right before - * S64_MIN. We can try drawing the continuity of u64 vs s64 values - * more visually as mapped to sign-agnostic range of hex values. - * - * u64 start u64 end - * _______________________________________________________________ - * / \ - * 0 0x7fffffffffffffff 0x8000000000000000 U64_MAX - * |-------------------------------|--------------------------------| - * 0 S64_MAX S64_MIN -1 - * / \ - * >------------------------------ -------------------------------> - * s64 continues... s64 end s64 start s64 "midpoint" - * - * What this means is that, in general, we can't always derive - * something new about u64 from any random s64 range, and vice versa. - * - * But we can do that in two particular cases. One is when entire - * u64/s64 range is *entirely* contained within left half of the above - * diagram or when it is *entirely* contained in the right half. I.e.: - * - * |-------------------------------|--------------------------------| - * ^ ^ ^ ^ - * A B C D - * - * [A, B] and [C, D] are contained entirely in their respective halves - * and form valid contiguous ranges as both u64 and s64 values. [A, B] - * will be non-negative both as u64 and s64 (and in fact it will be - * identical ranges no matter the signedness). [C, D] treated as s64 - * will be a range of negative values, while in u64 it will be - * non-negative range of values larger than 0x8000000000000000. - * - * Now, any other range here can't be represented in both u64 and s64 - * simultaneously. E.g., [A, C], [A, D], [B, C], [B, D] are valid - * contiguous u64 ranges, but they are discontinuous in s64. [B, C] - * in s64 would be properly presented as [S64_MIN, C] and [B, S64_MAX], - * for example. Similarly, valid s64 range [D, A] (going from negative - * to positive values), would be two separate [D, U64_MAX] and [0, A] - * ranges as u64. Currently reg_state can't represent two segments per - * numeric domain, so in such situations we can only derive maximal - * possible range ([0, U64_MAX] for u64, and [S64_MIN, S64_MAX] for s64). - * - * So we use these facts to derive umin/umax from smin/smax and vice - * versa only if they stay within the same "half". This is equivalent - * to checking sign bit: lower half will have sign bit as zero, upper - * half have sign bit 1. Below in code we simplify this by just - * casting umin/umax as smin/smax and checking if they form valid - * range, and vice versa. Those are equivalent checks. - */ - if ((s64)reg_umin(reg) <= (s64)reg_umax(reg)) { - reg_set_srange64(reg, - max_t(s64, reg_smin(reg), reg_umin(reg)), - min_t(s64, reg_smax(reg), reg_umax(reg))); - } - /* If we cannot cross the sign boundary, then signed and unsigned bounds - * are the same, so combine. This works even in the negative case, e.g. - * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. - */ - if ((u64)reg_smin(reg) <= (u64)reg_smax(reg)) { - reg_set_urange64(reg, - max_t(u64, reg_smin(reg), reg_umin(reg)), - min_t(u64, reg_smax(reg), reg_umax(reg))); - } else { - /* If the s64 range crosses the sign boundary, then it's split - * between the beginning and end of the U64 domain. In that - * case, we can derive new bounds if the u64 range overlaps - * with only one end of the s64 range. - * - * In the following example, the u64 range overlaps only with - * positive portion of the s64 range. - * - * 0 U64_MAX - * | [xxxxxxxxxxxxxx u64 range xxxxxxxxxxxxxx] | - * |----------------------------|----------------------------| - * |xxxxx s64 range xxxxxxxxx] [xxxxxxx| - * 0 S64_MAX S64_MIN -1 - * - * We can thus derive the following new s64 and u64 ranges. - * - * 0 U64_MAX - * | [xxxxxx u64 range xxxxx] | - * |----------------------------|----------------------------| - * | [xxxxxx s64 range xxxxx] | - * 0 S64_MAX S64_MIN -1 - * - * If they overlap in two places, we can't derive anything - * because reg_state can't represent two ranges per numeric - * domain. - * - * 0 U64_MAX - * | [xxxxxxxxxxxxxxxxx u64 range xxxxxxxxxxxxxxxxx] | - * |----------------------------|----------------------------| - * |xxxxx s64 range xxxxxxxxx] [xxxxxxxxxx| - * 0 S64_MAX S64_MIN -1 - * - * The first condition below corresponds to the first diagram - * above. - */ - if (reg_umax(reg) < (u64)reg_smin(reg)) { - reg_set_srange64(reg, (s64)reg_umin(reg), reg_smax(reg)); - reg_set_urange64(reg, reg_umin(reg), min_t(u64, reg_umax(reg), reg_smax(reg))); - } else if ((u64)reg_smax(reg) < reg_umin(reg)) { - /* This second condition considers the case where the u64 range - * overlaps with the negative portion of the s64 range: - * - * 0 U64_MAX - * | [xxxxxxxxxxxxxx u64 range xxxxxxxxxxxxxx] | - * |----------------------------|----------------------------| - * |xxxxxxxxx] [xxxxxxxxxxxx s64 range | - * 0 S64_MAX S64_MIN -1 - */ - reg_set_srange64(reg, reg_smin(reg), (s64)reg_umax(reg)); - reg_set_urange64(reg, max_t(u64, reg_umin(reg), reg_smin(reg)), reg_umax(reg)); - } - } + cnum32_intersect_with(®->r32, cnum32_from_cnum64(reg->r64)); } static void deduce_bounds_64_from_32(struct bpf_reg_state *reg) { - /* Try to tighten 64-bit bounds from 32-bit knowledge, using 32-bit - * values on both sides of 64-bit range in hope to have tighter range. - * E.g., if r1 is [0x1'00000000, 0x3'80000000], and we learn from - * 32-bit signed > 0 operation that s32 bounds are now [1; 0x7fffffff]. - * With this, we can substitute 1 as low 32-bits of _low_ 64-bit bound - * (0x100000000 -> 0x100000001) and 0x7fffffff as low 32-bits of - * _high_ 64-bit bound (0x380000000 -> 0x37fffffff) and arrive at a - * better overall bounds for r1 as [0x1'000000001; 0x3'7fffffff]. - * We just need to make sure that derived bounds we are intersecting - * with are well-formed ranges in respective s64 or u64 domain, just - * like we do with similar kinds of 32-to-64 or 64-to-32 adjustments. - */ - __u64 new_umin, new_umax; - __s64 new_smin, new_smax; - - /* u32 -> u64 tightening, it's always well-formed */ - new_umin = (reg_umin(reg) & ~0xffffffffULL) | reg_u32_min(reg); - new_umax = (reg_umax(reg) & ~0xffffffffULL) | reg_u32_max(reg); - reg_set_urange64(reg, - max_t(u64, reg_umin(reg), new_umin), - min_t(u64, reg_umax(reg), new_umax)); - /* u32 -> s64 tightening, u32 range embedded into s64 preserves range validity */ - new_smin = (reg_smin(reg) & ~0xffffffffULL) | reg_u32_min(reg); - new_smax = (reg_smax(reg) & ~0xffffffffULL) | reg_u32_max(reg); - reg_set_srange64(reg, - max_t(s64, reg_smin(reg), new_smin), - min_t(s64, reg_smax(reg), new_smax)); - - /* Here we would like to handle a special case after sign extending load, - * when upper bits for a 64-bit range are all 1s or all 0s. - * - * Upper bits are all 1s when register is in a range: - * [0xffff_ffff_0000_0000, 0xffff_ffff_ffff_ffff] - * Upper bits are all 0s when register is in a range: - * [0x0000_0000_0000_0000, 0x0000_0000_ffff_ffff] - * Together this forms are continuous range: - * [0xffff_ffff_0000_0000, 0x0000_0000_ffff_ffff] - * - * Now, suppose that register range is in fact tighter: - * [0xffff_ffff_8000_0000, 0x0000_0000_ffff_ffff] (R) - * Also suppose that it's 32-bit range is positive, - * meaning that lower 32-bits of the full 64-bit register - * are in the range: - * [0x0000_0000, 0x7fff_ffff] (W) - * - * If this happens, then any value in a range: - * [0xffff_ffff_0000_0000, 0xffff_ffff_7fff_ffff] - * is smaller than a lowest bound of the range (R): - * 0xffff_ffff_8000_0000 - * which means that upper bits of the full 64-bit register - * can't be all 1s, when lower bits are in range (W). - * - * Note that: - * - 0xffff_ffff_8000_0000 == (s64)S32_MIN - * - 0x0000_0000_7fff_ffff == (s64)S32_MAX - * These relations are used in the conditions below. - */ - if (reg_s32_min(reg) >= 0 && reg_smin(reg) >= S32_MIN && reg_smax(reg) <= S32_MAX) { - reg_set_srange64(reg, reg_s32_min(reg), reg_s32_max(reg)); - reg_set_urange64(reg, reg_s32_min(reg), reg_s32_max(reg)); - reg->var_off = tnum_intersect(reg->var_off, - tnum_range(reg_smin(reg), reg_smax(reg))); - } + reg->r64 = cnum64_cnum32_intersect(reg->r64, reg->r32); } static void __reg_deduce_bounds(struct bpf_reg_state *reg) { - deduce_bounds_64_from_64(reg); deduce_bounds_32_from_64(reg); - deduce_bounds_32_from_32(reg); deduce_bounds_64_from_32(reg); } @@ -2402,35 +2076,25 @@ static void reg_bounds_sync(struct bpf_reg_state *reg) __update_reg_bounds(reg); } -static bool range_bounds_violation(struct bpf_reg_state *reg) -{ - return (reg_umin(reg) > reg_umax(reg) || reg_smin(reg) > reg_smax(reg) || - reg_u32_min(reg) > reg_u32_max(reg) || - reg_s32_min(reg) > reg_s32_max(reg)); -} - static bool const_tnum_range_mismatch(struct bpf_reg_state *reg) { - u64 uval = reg->var_off.value; - s64 sval = (s64)uval; - if (!tnum_is_const(reg->var_off)) return false; - return reg_umin(reg) != uval || reg_umax(reg) != uval || - reg_smin(reg) != sval || reg_smax(reg) != sval; + return !cnum64_is_const(reg->r64) || reg->r64.base != reg->var_off.value; } static bool const_tnum_range_mismatch_32(struct bpf_reg_state *reg) { - u32 uval32 = tnum_subreg(reg->var_off).value; - s32 sval32 = (s32)uval32; - if (!tnum_subreg_is_const(reg->var_off)) return false; - return reg_u32_min(reg) != uval32 || reg_u32_max(reg) != uval32 || - reg_s32_min(reg) != sval32 || reg_s32_max(reg) != sval32; + return !cnum32_is_const(reg->r32) || reg->r32.base != tnum_subreg(reg->var_off).value; +} + +static bool range_bounds_violation(struct bpf_reg_state *reg) +{ + return cnum32_is_empty(reg->r32) || cnum64_is_empty(reg->r64); } static int reg_bounds_sanity_check(struct bpf_verifier_env *env, @@ -2455,12 +2119,11 @@ static int reg_bounds_sanity_check(struct bpf_verifier_env *env, return 0; out: - verifier_bug(env, "REG INVARIANTS VIOLATION (%s): %s u64=[%#llx, %#llx] " - "s64=[%#llx, %#llx] u32=[%#x, %#x] s32=[%#x, %#x] var_off=(%#llx, %#llx)", - ctx, msg, reg_umin(reg), reg_umax(reg), - reg_smin(reg), reg_smax(reg), - reg_u32_min(reg), reg_u32_max(reg), - reg_s32_min(reg), reg_s32_max(reg), + verifier_bug(env, "REG INVARIANTS VIOLATION (%s): %s r64={.base=%#llx, .size=%#llx} " + "r32={.base=%#x, .size=%#x} var_off=(%#llx, %#llx)", + ctx, msg, + reg->r64.base, reg->r64.size, + reg->r32.base, reg->r32.size, reg->var_off.value, reg->var_off.mask); if (env->test_reg_invariants) return -EFAULT; @@ -2468,26 +2131,6 @@ out: return 0; } -static bool __reg32_bound_s64(s32 a) -{ - return a >= 0 && a <= S32_MAX; -} - -static void __reg_assign_32_into_64(struct bpf_reg_state *reg) -{ - reg_set_urange64(reg, reg_u32_min(reg), reg_u32_max(reg)); - - /* Attempt to pull 32-bit signed bounds into 64-bit bounds but must - * be positive otherwise set to worse case bounds and refine later - * from tnum. - */ - if (__reg32_bound_s64(reg_s32_min(reg)) && - __reg32_bound_s64(reg_s32_max(reg))) - reg_set_srange64(reg, reg_s32_min(reg), reg_s32_max(reg)); - else - reg_set_srange64(reg, 0, U32_MAX); -} - /* Mark a register as having a completely unknown (scalar) value. */ void bpf_mark_reg_unknown_imprecise(struct bpf_reg_state *reg) { @@ -5636,7 +5279,7 @@ static int check_buffer_access(struct bpf_verifier_env *env, static void zext_32_to_64(struct bpf_reg_state *reg) { reg->var_off = tnum_subreg(reg->var_off); - __reg_assign_32_into_64(reg); + reg_set_urange64(reg, reg_u32_min(reg), reg_u32_max(reg)); } /* truncate register to smaller size (in bytes) @@ -5651,12 +5294,10 @@ static void coerce_reg_to_size(struct bpf_reg_state *reg, int size) /* fix arithmetic bounds */ mask = ((u64)1 << (size * 8)) - 1; - if ((reg_umin(reg) & ~mask) == (reg_umax(reg) & ~mask)) { + if ((reg_umin(reg) & ~mask) == (reg_umax(reg) & ~mask)) reg_set_urange64(reg, reg_umin(reg) & mask, reg_umax(reg) & mask); - } else { + else reg_set_urange64(reg, 0, mask); - } - reg_set_srange64(reg, reg_umin(reg), reg_umax(reg)); /* If size is smaller than 32bit register the 32bit register * values are also truncated so we push 64-bit bounds into @@ -5681,8 +5322,6 @@ static void set_sext64_default_val(struct bpf_reg_state *reg, int size) reg_set_srange64(reg, S32_MIN, S32_MAX); reg_set_srange32(reg, S32_MIN, S32_MAX); } - reg_set_urange64(reg, 0, U64_MAX); - reg_set_urange32(reg, 0, U32_MAX); reg->var_off = tnum_unknown; } @@ -5703,10 +5342,8 @@ static void coerce_reg_to_size_sx(struct bpf_reg_state *reg, int size) reg->var_off = tnum_const((s32)u64_cval); u64_cval = reg->var_off.value; - reg_set_srange64(reg, u64_cval, u64_cval); - reg_set_urange64(reg, u64_cval, u64_cval); - reg_set_srange32(reg, u64_cval, u64_cval); - reg_set_urange32(reg, u64_cval, u64_cval); + reg->r64 = cnum64_from_urange(u64_cval, u64_cval); + reg->r32 = cnum32_from_urange((u32)u64_cval, (u32)u64_cval); return; } @@ -5734,9 +5371,7 @@ static void coerce_reg_to_size_sx(struct bpf_reg_state *reg, int size) /* both of s64_max/s64_min positive or negative */ if ((s64_max >= 0) == (s64_min >= 0)) { reg_set_srange64(reg, s64_min, s64_max); - reg_set_urange64(reg, s64_min, s64_max); reg_set_srange32(reg, s64_min, s64_max); - reg_set_urange32(reg, s64_min, s64_max); reg->var_off = tnum_range(s64_min, s64_max); return; } @@ -5752,7 +5387,6 @@ static void set_sext32_default_val(struct bpf_reg_state *reg, int size) else /* size == 2 */ reg_set_srange32(reg, S16_MIN, S16_MAX); - reg_set_urange32(reg, 0, U32_MAX); reg->var_off = tnum_subreg(tnum_unknown); } @@ -5771,7 +5405,6 @@ static void coerce_subreg_to_size_sx(struct bpf_reg_state *reg, int size) u32_val = reg->var_off.value; reg_set_srange32(reg, u32_val, u32_val); - reg_set_urange32(reg, u32_val, u32_val); return; } @@ -5795,7 +5428,6 @@ static void coerce_subreg_to_size_sx(struct bpf_reg_state *reg, int size) if ((s32_min >= 0) == (s32_max >= 0)) { reg_set_srange32(reg, s32_min, s32_max); - reg_set_urange32(reg, (u32)s32_min, (u32)s32_max); reg->var_off = tnum_subreg(tnum_range(s32_min, s32_max)); return; } @@ -9952,8 +9584,6 @@ static int do_refine_retval_range(struct bpf_verifier_env *env, case BPF_FUNC_get_smp_processor_id: reg_set_urange64(ret_reg, 0, nr_cpu_ids - 1); reg_set_urange32(ret_reg, 0, nr_cpu_ids - 1); - reg_set_srange64(ret_reg, 0, nr_cpu_ids - 1); - reg_set_srange32(ret_reg, 0, nr_cpu_ids - 1); reg_bounds_sync(ret_reg); break; } @@ -13756,10 +13386,8 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = state->regs, *dst_reg; bool known = tnum_is_const(off_reg->var_off); - s64 smin_val = reg_smin(off_reg), smax_val = reg_smax(off_reg), - smin_ptr = reg_smin(ptr_reg), smax_ptr = reg_smax(ptr_reg); - u64 umin_val = reg_umin(off_reg), umax_val = reg_umax(off_reg), - umin_ptr = reg_umin(ptr_reg), umax_ptr = reg_umax(ptr_reg); + s64 smin_val = reg_smin(off_reg), smax_val = reg_smax(off_reg); + u64 umin_val = reg_umin(off_reg), umax_val = reg_umax(off_reg); struct bpf_sanitize_info info = {}; u8 opcode = BPF_OP(insn->code); u32 dst = insn->dst_reg; @@ -13861,23 +13489,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, * added into the variable offset, and we copy the fixed offset * from ptr_reg. */ - { - s64 smin_res, smax_res; - u64 umin_res, umax_res; - - if (check_add_overflow(smin_ptr, smin_val, &smin_res) || - check_add_overflow(smax_ptr, smax_val, &smax_res)) { - reg_set_srange64(dst_reg, S64_MIN, S64_MAX); - } else { - reg_set_srange64(dst_reg, smin_res, smax_res); - } - if (check_add_overflow(umin_ptr, umin_val, &umin_res) || - check_add_overflow(umax_ptr, umax_val, &umax_res)) { - reg_set_urange64(dst_reg, 0, U64_MAX); - } else { - reg_set_urange64(dst_reg, umin_res, umax_res); - } - } + dst_reg->r64 = cnum64_add(ptr_reg->r64, off_reg->r64); dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); dst_reg->raw = ptr_reg->raw; if (reg_is_pkt_pointer(ptr_reg)) { @@ -13909,27 +13521,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, dst); return -EACCES; } - /* A new variable offset is created. If the subtrahend is known - * nonnegative, then any reg->range we had before is still good. - */ - { - s64 smin_res, smax_res; - - if (check_sub_overflow(smin_ptr, smax_val, &smin_res) || - check_sub_overflow(smax_ptr, smin_val, &smax_res)) { - /* Overflow possible, we know nothing */ - reg_set_srange64(dst_reg, S64_MIN, S64_MAX); - } else { - reg_set_srange64(dst_reg, smin_res, smax_res); - } - } - if (umin_ptr < umax_val) { - /* Overflow possible, we know nothing */ - reg_set_urange64(dst_reg, 0, U64_MAX); - } else { - /* Cannot overflow (as long as bounds are consistent) */ - reg_set_urange64(dst_reg, umin_ptr - umax_val, umax_ptr - umin_val); - } + dst_reg->r64 = cnum64_add(ptr_reg->r64, cnum64_negate(off_reg->r64)); dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off); dst_reg->raw = ptr_reg->raw; if (reg_is_pkt_pointer(ptr_reg)) { @@ -13986,139 +13578,25 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) { - s32 smin = reg_s32_min(dst_reg); - s32 smax = reg_s32_max(dst_reg); - u32 umin = reg_u32_min(dst_reg); - u32 umax = reg_u32_max(dst_reg); - u32 umin_val = reg_u32_min(src_reg); - u32 umax_val = reg_u32_max(src_reg); - bool min_overflow, max_overflow; - - if (check_add_overflow(smin, reg_s32_min(src_reg), &smin) || - check_add_overflow(smax, reg_s32_max(src_reg), &smax)) { - smin = S32_MIN; - smax = S32_MAX; - } - - /* If either all additions overflow or no additions overflow, then - * it is okay to set: dst_umin = dst_umin + src_umin, dst_umax = - * dst_umax + src_umax. Otherwise (some additions overflow), set - * the output bounds to unbounded. - */ - min_overflow = check_add_overflow(umin, umin_val, &umin); - max_overflow = check_add_overflow(umax, umax_val, &umax); - - if (!min_overflow && max_overflow) { - umin = 0; - umax = U32_MAX; - } - - reg_set_srange32(dst_reg, smin, smax); - reg_set_urange32(dst_reg, umin, umax); + dst_reg->r32 = cnum32_add(dst_reg->r32, src_reg->r32); } static void scalar_min_max_add(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) { - s64 smin = reg_smin(dst_reg); - s64 smax = reg_smax(dst_reg); - u64 umin = reg_umin(dst_reg); - u64 umax = reg_umax(dst_reg); - u64 umin_val = reg_umin(src_reg); - u64 umax_val = reg_umax(src_reg); - bool min_overflow, max_overflow; - - if (check_add_overflow(smin, reg_smin(src_reg), &smin) || - check_add_overflow(smax, reg_smax(src_reg), &smax)) { - smin = S64_MIN; - smax = S64_MAX; - } - - /* If either all additions overflow or no additions overflow, then - * it is okay to set: dst_umin = dst_umin + src_umin, dst_umax = - * dst_umax + src_umax. Otherwise (some additions overflow), set - * the output bounds to unbounded. - */ - min_overflow = check_add_overflow(umin, umin_val, &umin); - max_overflow = check_add_overflow(umax, umax_val, &umax); - - if (!min_overflow && max_overflow) { - umin = 0; - umax = U64_MAX; - } - - reg_set_srange64(dst_reg, smin, smax); - reg_set_urange64(dst_reg, umin, umax); + dst_reg->r64 = cnum64_add(dst_reg->r64, src_reg->r64); } static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) { - s32 smin = reg_s32_min(dst_reg); - s32 smax = reg_s32_max(dst_reg); - u32 umin = reg_u32_min(dst_reg); - u32 umax = reg_u32_max(dst_reg); - u32 umin_val = reg_u32_min(src_reg); - u32 umax_val = reg_u32_max(src_reg); - bool min_underflow, max_underflow; - - if (check_sub_overflow(smin, reg_s32_max(src_reg), &smin) || - check_sub_overflow(smax, reg_s32_min(src_reg), &smax)) { - /* Overflow possible, we know nothing */ - smin = S32_MIN; - smax = S32_MAX; - } - - /* If either all subtractions underflow or no subtractions - * underflow, it is okay to set: dst_umin = dst_umin - src_umax, - * dst_umax = dst_umax - src_umin. Otherwise (some subtractions - * underflow), set the output bounds to unbounded. - */ - min_underflow = check_sub_overflow(umin, umax_val, &umin); - max_underflow = check_sub_overflow(umax, umin_val, &umax); - - if (min_underflow && !max_underflow) { - umin = 0; - umax = U32_MAX; - } - - reg_set_srange32(dst_reg, smin, smax); - reg_set_urange32(dst_reg, umin, umax); + dst_reg->r32 = cnum32_add(dst_reg->r32, cnum32_negate(src_reg->r32)); } static void scalar_min_max_sub(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) { - s64 smin = reg_smin(dst_reg); - s64 smax = reg_smax(dst_reg); - u64 umin = reg_umin(dst_reg); - u64 umax = reg_umax(dst_reg); - u64 umin_val = reg_umin(src_reg); - u64 umax_val = reg_umax(src_reg); - bool min_underflow, max_underflow; - - if (check_sub_overflow(smin, reg_smax(src_reg), &smin) || - check_sub_overflow(smax, reg_smin(src_reg), &smax)) { - /* Overflow possible, we know nothing */ - smin = S64_MIN; - smax = S64_MAX; - } - - /* If either all subtractions underflow or no subtractions - * underflow, it is okay to set: dst_umin = dst_umin - src_umax, - * dst_umax = dst_umax - src_umin. Otherwise (some subtractions - * underflow), set the output bounds to unbounded. - */ - min_underflow = check_sub_overflow(umin, umax_val, &umin); - max_underflow = check_sub_overflow(umax, umin_val, &umax); - - if (min_underflow && !max_underflow) { - umin = 0; - umax = U64_MAX; - } - - reg_set_srange64(dst_reg, smin, smax); - reg_set_urange64(dst_reg, umin, umax); + dst_reg->r64 = cnum64_add(dst_reg->r64, cnum64_negate(src_reg->r64)); } static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg, @@ -14148,8 +13626,8 @@ static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg, smax = max_array(tmp_prod, 4); } - reg_set_srange32(dst_reg, smin, smax); - reg_set_urange32(dst_reg, umin, umax); + dst_reg->r32 = cnum32_intersect(cnum32_from_urange(umin, umax), + cnum32_from_srange(smin, smax)); } static void scalar_min_max_mul(struct bpf_reg_state *dst_reg, @@ -14179,8 +13657,8 @@ static void scalar_min_max_mul(struct bpf_reg_state *dst_reg, smax = max_array(tmp_prod, 4); } - reg_set_srange64(dst_reg, smin, smax); - reg_set_urange64(dst_reg, umin, umax); + dst_reg->r64 = cnum64_intersect(cnum64_from_urange(umin, umax), + cnum64_from_srange(smin, smax)); } static void scalar32_min_max_udiv(struct bpf_reg_state *dst_reg, @@ -14192,7 +13670,6 @@ static void scalar32_min_max_udiv(struct bpf_reg_state *dst_reg, reg_u32_max(dst_reg) / src_val); /* Reset other ranges/tnum to unbounded/unknown. */ - reg_set_srange32(dst_reg, S32_MIN, S32_MAX); reset_reg64_and_tnum(dst_reg); } @@ -14205,7 +13682,6 @@ static void scalar_min_max_udiv(struct bpf_reg_state *dst_reg, div64_u64(reg_umax(dst_reg), src_val)); /* Reset other ranges/tnum to unbounded/unknown. */ - reg_set_srange64(dst_reg, S64_MIN, S64_MAX); reset_reg32_and_tnum(dst_reg); } @@ -14242,7 +13718,6 @@ static void scalar32_min_max_sdiv(struct bpf_reg_state *dst_reg, reset: reg_set_srange32(dst_reg, smin, smax); /* Reset other ranges/tnum to unbounded/unknown. */ - reg_set_urange32(dst_reg, 0, U32_MAX); reset_reg64_and_tnum(dst_reg); } @@ -14279,7 +13754,6 @@ static void scalar_min_max_sdiv(struct bpf_reg_state *dst_reg, reset: reg_set_srange64(dst_reg, smin, smax); /* Reset other ranges/tnum to unbounded/unknown. */ - reg_set_urange64(dst_reg, 0, U64_MAX); reset_reg32_and_tnum(dst_reg); } @@ -14299,7 +13773,6 @@ static void scalar32_min_max_umod(struct bpf_reg_state *dst_reg, reg_set_urange32(dst_reg, 0, min(reg_u32_max(dst_reg), res_max)); /* Reset other ranges/tnum to unbounded/unknown. */ - reg_set_srange32(dst_reg, S32_MIN, S32_MAX); reset_reg64_and_tnum(dst_reg); } @@ -14319,7 +13792,6 @@ static void scalar_min_max_umod(struct bpf_reg_state *dst_reg, reg_set_urange64(dst_reg, 0, min(reg_umax(dst_reg), res_max)); /* Reset other ranges/tnum to unbounded/unknown. */ - reg_set_srange64(dst_reg, S64_MIN, S64_MAX); reset_reg32_and_tnum(dst_reg); } @@ -14359,7 +13831,6 @@ static void scalar32_min_max_smod(struct bpf_reg_state *dst_reg, } /* Reset other ranges/tnum to unbounded/unknown. */ - reg_set_urange32(dst_reg, 0, U32_MAX); reset_reg64_and_tnum(dst_reg); } @@ -14399,7 +13870,6 @@ static void scalar_min_max_smod(struct bpf_reg_state *dst_reg, } /* Reset other ranges/tnum to unbounded/unknown. */ - reg_set_urange64(dst_reg, 0, U64_MAX); reset_reg32_and_tnum(dst_reg); } @@ -14419,15 +13889,9 @@ static void scalar32_min_max_and(struct bpf_reg_state *dst_reg, /* We get our minimum from the var_off, since that's inherently * bitwise. Our maximum is the minimum of the operands' maxima. */ - reg_set_urange32(dst_reg, var32_off.value, min(reg_u32_max(dst_reg), umax_val)); - - /* Safe to set s32 bounds by casting u32 result into s32 when u32 - * doesn't cross sign boundary. Otherwise set s32 bounds to unbounded. - */ - if ((s32)reg_u32_min(dst_reg) <= (s32)reg_u32_max(dst_reg)) - reg_set_srange32(dst_reg, reg_u32_min(dst_reg), reg_u32_max(dst_reg)); - else - reg_set_srange32(dst_reg, S32_MIN, S32_MAX); + reg_set_urange32(dst_reg, + var32_off.value, + min(reg_u32_max(dst_reg), umax_val)); } static void scalar_min_max_and(struct bpf_reg_state *dst_reg, @@ -14445,15 +13909,10 @@ static void scalar_min_max_and(struct bpf_reg_state *dst_reg, /* We get our minimum from the var_off, since that's inherently * bitwise. Our maximum is the minimum of the operands' maxima. */ - reg_set_urange64(dst_reg, dst_reg->var_off.value, min(reg_umax(dst_reg), umax_val)); + reg_set_urange64(dst_reg, + dst_reg->var_off.value, + min(reg_umax(dst_reg), umax_val)); - /* Safe to set s64 bounds by casting u64 result into s64 when u64 - * doesn't cross sign boundary. Otherwise set s64 bounds to unbounded. - */ - if ((s64)reg_umin(dst_reg) <= (s64)reg_umax(dst_reg)) - reg_set_srange64(dst_reg, reg_umin(dst_reg), reg_umax(dst_reg)); - else - reg_set_srange64(dst_reg, S64_MIN, S64_MAX); /* We may learn something more from the var_off */ __update_reg_bounds(dst_reg); } @@ -14474,16 +13933,9 @@ static void scalar32_min_max_or(struct bpf_reg_state *dst_reg, /* We get our maximum from the var_off, and our minimum is the * maximum of the operands' minima */ - reg_set_urange32(dst_reg, max(reg_u32_min(dst_reg), umin_val), + reg_set_urange32(dst_reg, + max(reg_u32_min(dst_reg), umin_val), var32_off.value | var32_off.mask); - - /* Safe to set s32 bounds by casting u32 result into s32 when u32 - * doesn't cross sign boundary. Otherwise set s32 bounds to unbounded. - */ - if ((s32)reg_u32_min(dst_reg) <= (s32)reg_u32_max(dst_reg)) - reg_set_srange32(dst_reg, reg_u32_min(dst_reg), reg_u32_max(dst_reg)); - else - reg_set_srange32(dst_reg, S32_MIN, S32_MAX); } static void scalar_min_max_or(struct bpf_reg_state *dst_reg, @@ -14501,16 +13953,10 @@ static void scalar_min_max_or(struct bpf_reg_state *dst_reg, /* We get our maximum from the var_off, and our minimum is the * maximum of the operands' minima */ - reg_set_urange64(dst_reg, max(reg_umin(dst_reg), umin_val), + reg_set_urange64(dst_reg, + max(reg_umin(dst_reg), umin_val), dst_reg->var_off.value | dst_reg->var_off.mask); - /* Safe to set s64 bounds by casting u64 result into s64 when u64 - * doesn't cross sign boundary. Otherwise set s64 bounds to unbounded. - */ - if ((s64)reg_umin(dst_reg) <= (s64)reg_umax(dst_reg)) - reg_set_srange64(dst_reg, reg_umin(dst_reg), reg_umax(dst_reg)); - else - reg_set_srange64(dst_reg, S64_MIN, S64_MAX); /* We may learn something more from the var_off */ __update_reg_bounds(dst_reg); } @@ -14529,14 +13975,6 @@ static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg, /* We get both minimum and maximum from the var32_off. */ reg_set_urange32(dst_reg, var32_off.value, var32_off.value | var32_off.mask); - - /* Safe to set s32 bounds by casting u32 result into s32 when u32 - * doesn't cross sign boundary. Otherwise set s32 bounds to unbounded. - */ - if ((s32)reg_u32_min(dst_reg) <= (s32)reg_u32_max(dst_reg)) - reg_set_srange32(dst_reg, reg_u32_min(dst_reg), reg_u32_max(dst_reg)); - else - reg_set_srange32(dst_reg, S32_MIN, S32_MAX); } static void scalar_min_max_xor(struct bpf_reg_state *dst_reg, @@ -14552,31 +13990,21 @@ static void scalar_min_max_xor(struct bpf_reg_state *dst_reg, } /* We get both minimum and maximum from the var_off. */ - reg_set_urange64(dst_reg, dst_reg->var_off.value, + reg_set_urange64(dst_reg, + dst_reg->var_off.value, dst_reg->var_off.value | dst_reg->var_off.mask); - - /* Safe to set s64 bounds by casting u64 result into s64 when u64 - * doesn't cross sign boundary. Otherwise set s64 bounds to unbounded. - */ - if ((s64)reg_umin(dst_reg) <= (s64)reg_umax(dst_reg)) - reg_set_srange64(dst_reg, reg_umin(dst_reg), reg_umax(dst_reg)); - else - reg_set_srange64(dst_reg, S64_MIN, S64_MAX); - - __update_reg_bounds(dst_reg); } static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, u64 umin_val, u64 umax_val) { - /* We lose all sign bit information (except what we can pick - * up from var_off) - */ - reg_set_srange32(dst_reg, S32_MIN, S32_MAX); /* If we might shift our top bit out, then we know nothing */ if (umax_val > 31 || reg_u32_max(dst_reg) > 1ULL << (31 - umax_val)) reg_set_urange32(dst_reg, 0, U32_MAX); else + /* We lose all sign bit information (except what we can pick + * up from var_off) + */ reg_set_urange32(dst_reg, reg_u32_min(dst_reg) << umin_val, reg_u32_max(dst_reg) << umax_val); } @@ -14602,23 +14030,27 @@ static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg, u64 umin_val, u64 umax_val) { + struct cnum64 u, s; + /* Special case <<32 because it is a common compiler pattern to sign * extend subreg by doing <<32 s>>32. smin/smax assignments are correct * because s32 bounds don't flip sign when shifting to the left by * 32bits. */ if (umin_val == 32 && umax_val == 32) - reg_set_srange64(dst_reg, (s64)reg_s32_min(dst_reg) << 32, - (s64)reg_s32_max(dst_reg) << 32); + s = cnum64_from_srange((s64)reg_s32_min(dst_reg) << 32, + (s64)reg_s32_max(dst_reg) << 32); else - reg_set_srange64(dst_reg, S64_MIN, S64_MAX); + s = CNUM64_UNBOUNDED; /* If we might shift our top bit out, then we know nothing */ if (reg_umax(dst_reg) > 1ULL << (63 - umax_val)) - reg_set_urange64(dst_reg, 0, U64_MAX); + u = CNUM64_UNBOUNDED; else - reg_set_urange64(dst_reg, reg_umin(dst_reg) << umin_val, - reg_umax(dst_reg) << umax_val); + u = cnum64_from_urange(reg_umin(dst_reg) << umin_val, + reg_umax(dst_reg) << umax_val); + + dst_reg->r64 = cnum64_intersect(u, s); } static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg, @@ -14657,7 +14089,6 @@ static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg, * and rely on inferring new ones from the unsigned bounds and * var_off of the result. */ - reg_set_srange32(dst_reg, S32_MIN, S32_MAX); dst_reg->var_off = tnum_rshift(subreg, umin_val); reg_set_urange32(dst_reg, reg_u32_min(dst_reg) >> umax_val, @@ -14687,7 +14118,6 @@ static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg, * and rely on inferring new ones from the unsigned bounds and * var_off of the result. */ - reg_set_srange64(dst_reg, S64_MIN, S64_MAX); dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val); reg_set_urange64(dst_reg, reg_umin(dst_reg) >> umax_val, reg_umax(dst_reg) >> umin_val); @@ -14707,6 +14137,8 @@ static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg, /* Upon reaching here, src_known is true and * umax_val is equal to umin_val. + * Blow away the dst_reg umin_value/umax_value and rely on + * dst_reg var_off to refine the result. */ reg_set_srange32(dst_reg, (u32)(((s32)reg_s32_min(dst_reg)) >> umin_val), @@ -14714,11 +14146,6 @@ static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg, dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32); - /* blow away the dst_reg umin_value/umax_value and rely on - * dst_reg var_off to refine the result. - */ - reg_set_urange32(dst_reg, 0, U32_MAX); - __mark_reg64_unbounded(dst_reg); __update_reg32_bounds(dst_reg); } @@ -14736,11 +14163,6 @@ static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg, dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64); - /* blow away the dst_reg umin_value/umax_value and rely on - * dst_reg var_off to refine the result. - */ - reg_set_urange64(dst_reg, 0, U64_MAX); - /* Its not easy to operate on alu32 bounds here because it depends * on bits being shifted in from upper 32-bits. Take easy way out * and mark unbounded so we can recalculate later from tnum. @@ -15829,23 +15251,15 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state switch (opcode) { case BPF_JEQ: if (is_jmp32) { - reg_set_urange32(reg1, max(reg_u32_min(reg1), reg_u32_min(reg2)), - min(reg_u32_max(reg1), reg_u32_max(reg2))); - reg_set_srange32(reg1, max(reg_s32_min(reg1), reg_s32_min(reg2)), - min(reg_s32_max(reg1), reg_s32_max(reg2))); - reg_set_urange32(reg2, reg_u32_min(reg1), reg_u32_max(reg1)); - reg_set_srange32(reg2, reg_s32_min(reg1), reg_s32_max(reg1)); + reg1->r32 = cnum32_intersect(reg1->r32, reg2->r32); + reg2->r32 = reg1->r32; t = tnum_intersect(tnum_subreg(reg1->var_off), tnum_subreg(reg2->var_off)); reg1->var_off = tnum_with_subreg(reg1->var_off, t); reg2->var_off = tnum_with_subreg(reg2->var_off, t); } else { - reg_set_urange64(reg1, max(reg_umin(reg1), reg_umin(reg2)), - min(reg_umax(reg1), reg_umax(reg2))); - reg_set_srange64(reg1, max(reg_smin(reg1), reg_smin(reg2)), - min(reg_smax(reg1), reg_smax(reg2))); - reg_set_urange64(reg2, reg_umin(reg1), reg_umax(reg1)); - reg_set_srange64(reg2, reg_smin(reg1), reg_smax(reg1)); + reg1->r64 = cnum64_intersect(reg1->r64, reg2->r64); + reg2->r64 = reg1->r64; reg1->var_off = tnum_intersect(reg1->var_off, reg2->var_off); reg2->var_off = reg1->var_off; @@ -15862,32 +15276,11 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state */ val = reg_const_value(reg2, is_jmp32); if (is_jmp32) { - /* u32_min is not equal to 0xffffffff at this point, - * because otherwise u32_max is 0xffffffff as well, - * in such a case both reg1 and reg2 would be constants, - * jump would be predicted and regs_refine_cond_op() - * wouldn't be called. - * - * Same reasoning works for all {u,s}{min,max}{32,64} cases - * below. - */ - if (reg_u32_min(reg1) == (u32)val) - reg_set_urange32(reg1, reg_u32_min(reg1) + 1, reg_u32_max(reg1)); - if (reg_u32_max(reg1) == (u32)val) - reg_set_urange32(reg1, reg_u32_min(reg1), reg_u32_max(reg1) - 1); - if (reg_s32_min(reg1) == (s32)val) - reg_set_srange32(reg1, reg_s32_min(reg1) + 1, reg_s32_max(reg1)); - if (reg_s32_max(reg1) == (s32)val) - reg_set_srange32(reg1, reg_s32_min(reg1), reg_s32_max(reg1) - 1); + /* Complement of the range [val, val] as cnum32. */ + cnum32_intersect_with(®1->r32, (struct cnum32){ val + 1, U32_MAX - 1 }); } else { - if (reg_umin(reg1) == (u64)val) - reg_set_urange64(reg1, reg_umin(reg1) + 1, reg_umax(reg1)); - if (reg_umax(reg1) == (u64)val) - reg_set_urange64(reg1, reg_umin(reg1), reg_umax(reg1) - 1); - if (reg_smin(reg1) == (s64)val) - reg_set_srange64(reg1, reg_smin(reg1) + 1, reg_smax(reg1)); - if (reg_smax(reg1) == (s64)val) - reg_set_srange64(reg1, reg_smin(reg1), reg_smax(reg1) - 1); + /* Complement of the range [val, val] as cnum64. */ + cnum64_intersect_with(®1->r64, (struct cnum64){ val + 1, U64_MAX - 1 }); } break; case BPF_JSET: @@ -15934,38 +15327,38 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state break; case BPF_JLE: if (is_jmp32) { - reg_set_urange32(reg1, reg_u32_min(reg1), min(reg_u32_max(reg1), reg_u32_max(reg2))); - reg_set_urange32(reg2, max(reg_u32_min(reg1), reg_u32_min(reg2)), reg_u32_max(reg2)); + cnum32_intersect_with_urange(®1->r32, 0, reg_u32_max(reg2)); + cnum32_intersect_with_urange(®2->r32, reg_u32_min(reg1), U32_MAX); } else { - reg_set_urange64(reg1, reg_umin(reg1), min(reg_umax(reg1), reg_umax(reg2))); - reg_set_urange64(reg2, max(reg_umin(reg1), reg_umin(reg2)), reg_umax(reg2)); + cnum64_intersect_with_urange(®1->r64, 0, reg_umax(reg2)); + cnum64_intersect_with_urange(®2->r64, reg_umin(reg1), U64_MAX); } break; case BPF_JLT: if (is_jmp32) { - reg_set_urange32(reg1, reg_u32_min(reg1), min(reg_u32_max(reg1), reg_u32_max(reg2) - 1)); - reg_set_urange32(reg2, max(reg_u32_min(reg1) + 1, reg_u32_min(reg2)), reg_u32_max(reg2)); + cnum32_intersect_with_urange(®1->r32, 0, reg_u32_max(reg2) - 1); + cnum32_intersect_with_urange(®2->r32, reg_u32_min(reg1) + 1, U32_MAX); } else { - reg_set_urange64(reg1, reg_umin(reg1), min(reg_umax(reg1), reg_umax(reg2) - 1)); - reg_set_urange64(reg2, max(reg_umin(reg1) + 1, reg_umin(reg2)), reg_umax(reg2)); + cnum64_intersect_with_urange(®1->r64, 0, reg_umax(reg2) - 1); + cnum64_intersect_with_urange(®2->r64, reg_umin(reg1) + 1, U64_MAX); } break; case BPF_JSLE: if (is_jmp32) { - reg_set_srange32(reg1, reg_s32_min(reg1), min(reg_s32_max(reg1), reg_s32_max(reg2))); - reg_set_srange32(reg2, max(reg_s32_min(reg1), reg_s32_min(reg2)), reg_s32_max(reg2)); + cnum32_intersect_with_srange(®1->r32, S32_MIN, reg_s32_max(reg2)); + cnum32_intersect_with_srange(®2->r32, reg_s32_min(reg1), S32_MAX); } else { - reg_set_srange64(reg1, reg_smin(reg1), min(reg_smax(reg1), reg_smax(reg2))); - reg_set_srange64(reg2, max(reg_smin(reg1), reg_smin(reg2)), reg_smax(reg2)); + cnum64_intersect_with_srange(®1->r64, S64_MIN, reg_smax(reg2)); + cnum64_intersect_with_srange(®2->r64, reg_smin(reg1), S64_MAX); } break; case BPF_JSLT: if (is_jmp32) { - reg_set_srange32(reg1, reg_s32_min(reg1), min(reg_s32_max(reg1), reg_s32_max(reg2) - 1)); - reg_set_srange32(reg2, max(reg_s32_min(reg1) + 1, reg_s32_min(reg2)), reg_s32_max(reg2)); + cnum32_intersect_with_srange(®1->r32, S32_MIN, reg_s32_max(reg2) - 1); + cnum32_intersect_with_srange(®2->r32, reg_s32_min(reg1) + 1, S32_MAX); } else { - reg_set_srange64(reg1, reg_smin(reg1), min(reg_smax(reg1), reg_smax(reg2) - 1)); - reg_set_srange64(reg2, max(reg_smin(reg1) + 1, reg_smin(reg2)), reg_smax(reg2)); + cnum64_intersect_with_srange(®1->r64, S64_MIN, reg_smax(reg2) - 1); + cnum64_intersect_with_srange(®2->r64, reg_smin(reg1) + 1, S64_MAX); } break; default: -- cgit v1.2.3 From d99167df047a12cac636188f994a9e8f1f9779ab Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Mon, 6 Apr 2026 16:22:57 -0700 Subject: driver core: Replace dev->dma_skip_sync with dev_dma_skip_sync() In C, bitfields are not necessarily safe to modify from multiple threads without locking. Switch "dma_skip_sync" over to the "flags" field so modifications are safe. Cc: Alexander Lobakin Cc: Eric Dumazet Cc: Christoph Hellwig Reviewed-by: Rafael J. Wysocki (Intel) Reviewed-by: Danilo Krummrich Acked-by: Greg Kroah-Hartman Acked-by: Marek Szyprowski Signed-off-by: Douglas Anderson Link: https://patch.msgid.link/20260406162231.v5.4.Icf072aa4184dd86a88fa8ca195b09d1651984000@changeid Signed-off-by: Danilo Krummrich --- kernel/dma/mapping.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index 23ed8eb9233e..a81a316971ff 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -476,7 +476,7 @@ bool dma_need_unmap(struct device *dev) { if (!dma_map_direct(dev, get_dma_ops(dev))) return true; - if (!dev->dma_skip_sync) + if (!dev_dma_skip_sync(dev)) return true; return IS_ENABLED(CONFIG_DMA_API_DEBUG); } @@ -492,16 +492,16 @@ static void dma_setup_need_sync(struct device *dev) * mapping, if any. During the device initialization, it's * enough to check only for the DMA coherence. */ - dev->dma_skip_sync = dev_is_dma_coherent(dev); + dev_assign_dma_skip_sync(dev, dev_is_dma_coherent(dev)); else if (!ops->sync_single_for_device && !ops->sync_single_for_cpu && !ops->sync_sg_for_device && !ops->sync_sg_for_cpu) /* * Synchronization is not possible when none of DMA sync ops * is set. */ - dev->dma_skip_sync = true; + dev_set_dma_skip_sync(dev); else - dev->dma_skip_sync = false; + dev_clear_dma_skip_sync(dev); } #else /* !CONFIG_DMA_NEED_SYNC */ static inline void dma_setup_need_sync(struct device *dev) { } -- cgit v1.2.3 From 9ad67f494ba6bdb4e7bb612640158f645932dca3 Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Mon, 6 Apr 2026 16:22:58 -0700 Subject: driver core: Replace dev->dma_ops_bypass with dev_dma_ops_bypass() In C, bitfields are not necessarily safe to modify from multiple threads without locking. Switch "dma_ops_bypass" over to the "flags" field so modifications are safe. Cc: Christoph Hellwig Cc: Alexey Kardashevskiy Reviewed-by: Rafael J. Wysocki (Intel) Reviewed-by: Danilo Krummrich Acked-by: Greg Kroah-Hartman Acked-by: Marek Szyprowski Signed-off-by: Douglas Anderson Link: https://patch.msgid.link/20260406162231.v5.5.If62b84471ef2c85e7ad250f0468867d6dba965ab@changeid Signed-off-by: Danilo Krummrich --- kernel/dma/mapping.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index a81a316971ff..fe3ab94f3d83 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -126,11 +126,9 @@ static bool dma_go_direct(struct device *dev, dma_addr_t mask, if (likely(!ops)) return true; -#ifdef CONFIG_DMA_OPS_BYPASS - if (dev->dma_ops_bypass) + if (IS_ENABLED(CONFIG_DMA_OPS_BYPASS) && dev_dma_ops_bypass(dev)) return min_not_zero(mask, dev->bus_dma_limit) >= dma_direct_get_required_mask(dev); -#endif return false; } -- cgit v1.2.3 From a7cc262a11354ab104b8e55c21200d099d141bc7 Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Mon, 6 Apr 2026 16:23:02 -0700 Subject: driver core: Replace dev->offline + ->offline_disabled with accessors In C, bitfields are not necessarily safe to modify from multiple threads without locking. Switch "offline" and "offline_disabled" over to the "flags" field so modifications are safe. Cc: Rafael J. Wysocki Acked-by: Mark Brown Reviewed-by: Rafael J. Wysocki (Intel) Reviewed-by: Danilo Krummrich Acked-by: Greg Kroah-Hartman Acked-by: Marek Szyprowski Signed-off-by: Douglas Anderson Link: https://patch.msgid.link/20260406162231.v5.9.I897d478b4a9361d79cd5073207c1062fd4d0d0e4@changeid Signed-off-by: Danilo Krummrich --- kernel/cpu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/cpu.c b/kernel/cpu.c index bc4f7a9ba64e..f975bb34915b 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -2639,7 +2639,7 @@ static void cpuhp_offline_cpu_device(unsigned int cpu) { struct device *dev = get_cpu_device(cpu); - dev->offline = true; + dev_set_offline(dev); /* Tell user space about the state change */ kobject_uevent(&dev->kobj, KOBJ_OFFLINE); } @@ -2648,7 +2648,7 @@ static void cpuhp_online_cpu_device(unsigned int cpu) { struct device *dev = get_cpu_device(cpu); - dev->offline = false; + dev_clear_offline(dev); /* Tell user space about the state change */ kobject_uevent(&dev->kobj, KOBJ_ONLINE); } -- cgit v1.2.3 From cd5b460ed1eca9e48f3eb07db1ee0a522c0eaa23 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Sat, 25 Apr 2026 15:48:23 -0700 Subject: bpf: range_within() must check cnum ranges instead of min/max pairs states.c:range_within() must be updated to properly check if cnum-based range in an old state is a superset of a range in the cur state. Currently it makes the decision using min/max accessors: reg_umin(old) <= reg_umin(cur) <= reg_umax(old) This is wrong for cnums that cross both UT_MAX/0 and ST_MAX/ST_MIN boundaries. Consider cnum32{base=0x7FFFFFF0, size=0x80000020}, which represents values [0x7FFFFFF0, ..., U32_MAX, 0, ..., 0x10]. Its projections are u32_min/max=0/U32_MAX, s32_min/max=S32_MIN/MAX. A register with range [0x100, 0x200] (which lies entirely in the gap of the wrapping range) would pass the min/max check despite having no overlap with the actual cnum arc. This commit replaces min/max comparison with cnum{32,64}_is_subset() operation. The operation implementation is verified using cbmc model checker in [1]. [1] https://github.com/eddyz87/cnum-verif/ Fixes: bbc631085503 ("bpf: replace min/max fields with struct cnum{32,64}") Signed-off-by: Eduard Zingerman Link: https://lore.kernel.org/r/20260425-cnum-range-within-v1-1-2fdca70cb09d@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/cnum_defs.h | 14 ++++++++++++++ kernel/bpf/states.c | 11 +++-------- 2 files changed, 17 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/cnum_defs.h b/kernel/bpf/cnum_defs.h index 3ebd8f723dbb..1f232138b6e9 100644 --- a/kernel/bpf/cnum_defs.h +++ b/kernel/bpf/cnum_defs.h @@ -220,6 +220,20 @@ bool FN(is_const)(struct cnum_t cnum) return cnum.size == 0; } +bool FN(is_subset)(struct cnum_t bigger, struct cnum_t smaller) +{ + if (FN(is_empty(smaller))) + return true; + if (FN(is_empty(bigger))) + return false; + /* rotate both arcs such that 'bigger' starts at origin, hence does not overflow */ + smaller.base -= bigger.base; + bigger.base = 0; + if (FN(urange_overflow)(smaller) && bigger.size < UT_MAX) + return false; + return smaller.base + smaller.size <= bigger.size; +} + #undef EMPTY #undef cnum_t #undef ut diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index a78ae891b743..bd9c22945050 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -2,6 +2,7 @@ /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ #include #include +#include #include #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) @@ -301,14 +302,8 @@ int bpf_update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_s static bool range_within(const struct bpf_reg_state *old, const struct bpf_reg_state *cur) { - return reg_umin(old) <= reg_umin(cur) && - reg_umax(old) >= reg_umax(cur) && - reg_smin(old) <= reg_smin(cur) && - reg_smax(old) >= reg_smax(cur) && - reg_u32_min(old) <= reg_u32_min(cur) && - reg_u32_max(old) >= reg_u32_max(cur) && - reg_s32_min(old) <= reg_s32_min(cur) && - reg_s32_max(old) >= reg_s32_max(cur); + return cnum64_is_subset(old->r64, cur->r64) && + cnum32_is_subset(old->r32, cur->r32); } /* If in the old state two registers had the same id, then they need to have -- cgit v1.2.3 From 79b8ebcbe483fee401e1b91dd32470348d9aa5b8 Mon Sep 17 00:00:00 2001 From: Alan Maguire Date: Mon, 27 Apr 2026 12:22:05 +0100 Subject: bpf: Export cnum_umin/umax() helpers for netronome driver ERROR: modpost: "cnum64_umin" [drivers/net/ethernet/netronome/nfp/nfp.ko] undefined! ERROR: modpost: "cnum64_umax" [drivers/net/ethernet/netronome/nfp/nfp.ko] undefined! Export symbols for these references. Reported-by: Kaitao Cheng Fixes: bbc631085503 ("bpf: replace min/max fields with struct cnum{32,64}") Signed-off-by: Alan Maguire Acked-by: Eduard Zingerman Link: https://lore.kernel.org/r/20260427112205.1346733-1-alan.maguire@oracle.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/cnum_defs.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/cnum_defs.h b/kernel/bpf/cnum_defs.h index 1f232138b6e9..a90e317e3578 100644 --- a/kernel/bpf/cnum_defs.h +++ b/kernel/bpf/cnum_defs.h @@ -6,6 +6,7 @@ #endif #include +#include #include #include #include @@ -48,11 +49,13 @@ ut FN(umin)(struct cnum_t cnum) { return FN(urange_overflow)(cnum) ? 0 : cnum.base; } +EXPORT_SYMBOL_GPL(FN(umin)); ut FN(umax)(struct cnum_t cnum) { return FN(urange_overflow)(cnum) ? UT_MAX : cnum.base + cnum.size; } +EXPORT_SYMBOL_GPL(FN(umax)); /* True if this cnum represents two signed ranges. */ static inline bool FN(srange_overflow)(struct cnum_t cnum) -- cgit v1.2.3 From cfeddb4244268c246d67cbe50269a9475cb112fc Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Fri, 24 Apr 2026 17:39:05 +0200 Subject: bpf: Remove obsolete WARN_ON call The WARN_ON call in bpf_trampoline_update could never hit, because we direct the code path with (total == 0) to out label, which effectively skips the WARN_ON call. The WARN_ON made sense back then when it checked tr->selector, but now with total being set just inside the function it's useless. Signed-off-by: Jiri Olsa Acked-by: Song Liu Link: https://lore.kernel.org/r/20260424153905.354922-2-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/bpf/trampoline.c | 1 - 1 file changed, 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index f02254a21585..a4298a25d4ba 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -685,7 +685,6 @@ again: if (err) goto out_free; - WARN_ON(tr->cur_image && total == 0); if (tr->cur_image) /* progs already running at this address */ err = modify_fentry(tr, orig_flags, tr->cur_image->image, -- cgit v1.2.3 From 4939721aad2ea1ae5050ce1b0c68cb338ade5db0 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 27 Apr 2026 00:26:55 -1000 Subject: sched_ext: Collect ext_*.c include headers in build_policy.c Move from ext.c and "ext_idle.h" from ext.c (plus its self-include in ext_idle.c) into build_policy.c. Subsequent patches add their headers the same way for consistency. No functional change. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/build_policy.c | 2 ++ kernel/sched/ext.c | 2 -- kernel/sched/ext_idle.c | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c index 755883faf751..ffb386889218 100644 --- a/kernel/sched/build_policy.c +++ b/kernel/sched/build_policy.c @@ -58,7 +58,9 @@ #include "deadline.c" #ifdef CONFIG_SCHED_CLASS_EXT +# include # include "ext_internal.h" +# include "ext_idle.h" # include "ext.c" # include "ext_idle.c" #endif diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index f7b1b16e81a5..11893f00be06 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -6,8 +6,6 @@ * Copyright (c) 2022 Tejun Heo * Copyright (c) 2022 David Vernet */ -#include -#include "ext_idle.h" static DEFINE_RAW_SPINLOCK(scx_sched_lock); diff --git a/kernel/sched/ext_idle.c b/kernel/sched/ext_idle.c index 7468560a6d80..f0f4d9500997 100644 --- a/kernel/sched/ext_idle.c +++ b/kernel/sched/ext_idle.c @@ -9,7 +9,6 @@ * Copyright (c) 2022 David Vernet * Copyright (c) 2024 Andrea Righi */ -#include "ext_idle.h" /* Enable/disable built-in idle CPU selection policy */ static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); -- cgit v1.2.3 From 3f63ddda0b969410585757309cc95add4f256f45 Mon Sep 17 00:00:00 2001 From: Changwoo Min Date: Wed, 29 Apr 2026 17:23:16 +0900 Subject: sched_ext: Extract scx_dump_cpu() from scx_dump_state() Factor out the per-CPU state dump logic from the for_each_possible_cpu loop in scx_dump_state() into a new scx_dump_cpu() helper to improve readability. No functional change. Signed-off-by: Changwoo Min Signed-off-by: Tejun Heo --- kernel/sched/ext.c | 171 ++++++++++++++++++++++++++++------------------------- 1 file changed, 89 insertions(+), 82 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 11893f00be06..fff3c8ab0175 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -6254,6 +6254,94 @@ static void scx_dump_task(struct scx_sched *sch, struct seq_buf *s, struct scx_d } } +static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, + struct scx_dump_ctx *dctx, int cpu, + bool dump_all_tasks) +{ + struct rq *rq = cpu_rq(cpu); + struct rq_flags rf; + struct task_struct *p; + struct seq_buf ns; + size_t avail, used; + char *buf; + bool idle; + + rq_lock_irqsave(rq, &rf); + + idle = list_empty(&rq->scx.runnable_list) && + rq->curr->sched_class == &idle_sched_class; + + if (idle && !SCX_HAS_OP(sch, dump_cpu)) + goto next; + + /* + * We don't yet know whether ops.dump_cpu() will produce output + * and we may want to skip the default CPU dump if it doesn't. + * Use a nested seq_buf to generate the standard dump so that we + * can decide whether to commit later. + */ + avail = seq_buf_get_buf(s, &buf); + seq_buf_init(&ns, buf, avail); + + dump_newline(&ns); + dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu ksync=%lu", + cpu, rq->scx.nr_running, rq->scx.flags, + rq->scx.cpu_released, rq->scx.ops_qseq, + rq->scx.kick_sync); + dump_line(&ns, " curr=%s[%d] class=%ps", + rq->curr->comm, rq->curr->pid, + rq->curr->sched_class); + if (!cpumask_empty(rq->scx.cpus_to_kick)) + dump_line(&ns, " cpus_to_kick : %*pb", + cpumask_pr_args(rq->scx.cpus_to_kick)); + if (!cpumask_empty(rq->scx.cpus_to_kick_if_idle)) + dump_line(&ns, " idle_to_kick : %*pb", + cpumask_pr_args(rq->scx.cpus_to_kick_if_idle)); + if (!cpumask_empty(rq->scx.cpus_to_preempt)) + dump_line(&ns, " cpus_to_preempt: %*pb", + cpumask_pr_args(rq->scx.cpus_to_preempt)); + if (!cpumask_empty(rq->scx.cpus_to_wait)) + dump_line(&ns, " cpus_to_wait : %*pb", + cpumask_pr_args(rq->scx.cpus_to_wait)); + if (!cpumask_empty(rq->scx.cpus_to_sync)) + dump_line(&ns, " cpus_to_sync : %*pb", + cpumask_pr_args(rq->scx.cpus_to_sync)); + + used = seq_buf_used(&ns); + if (SCX_HAS_OP(sch, dump_cpu)) { + ops_dump_init(&ns, " "); + SCX_CALL_OP(sch, dump_cpu, rq, dctx, cpu, idle); + ops_dump_exit(); + } + + /* + * If idle && nothing generated by ops.dump_cpu(), there's + * nothing interesting. Skip. + */ + if (idle && used == seq_buf_used(&ns)) + goto next; + + /* + * $s may already have overflowed when $ns was created. If so, + * calling commit on it will trigger BUG. + */ + if (avail) { + seq_buf_commit(s, seq_buf_used(&ns)); + if (seq_buf_has_overflowed(&ns)) + seq_buf_set_overflow(s); + } + + if (rq->curr->sched_class == &ext_sched_class && + (dump_all_tasks || scx_task_on_sched(sch, rq->curr))) + scx_dump_task(sch, s, dctx, rq, rq->curr, '*'); + + list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) + if (dump_all_tasks || scx_task_on_sched(sch, p)) + scx_dump_task(sch, s, dctx, rq, p, ' '); +next: + rq_unlock_irqrestore(rq, &rf); +} + /* * Dump scheduler state. If @dump_all_tasks is true, dump all tasks regardless * of which scheduler they belong to. If false, only dump tasks owned by @sch. @@ -6274,7 +6362,6 @@ static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei, }; struct seq_buf s; struct scx_event_stats events; - char *buf; int cpu; guard(raw_spinlock_irqsave)(&scx_dump_lock); @@ -6314,87 +6401,7 @@ static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei, dump_line(&s, "----------"); for_each_possible_cpu(cpu) { - struct rq *rq = cpu_rq(cpu); - struct rq_flags rf; - struct task_struct *p; - struct seq_buf ns; - size_t avail, used; - bool idle; - - rq_lock_irqsave(rq, &rf); - - idle = list_empty(&rq->scx.runnable_list) && - rq->curr->sched_class == &idle_sched_class; - - if (idle && !SCX_HAS_OP(sch, dump_cpu)) - goto next; - - /* - * We don't yet know whether ops.dump_cpu() will produce output - * and we may want to skip the default CPU dump if it doesn't. - * Use a nested seq_buf to generate the standard dump so that we - * can decide whether to commit later. - */ - avail = seq_buf_get_buf(&s, &buf); - seq_buf_init(&ns, buf, avail); - - dump_newline(&ns); - dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu ksync=%lu", - cpu, rq->scx.nr_running, rq->scx.flags, - rq->scx.cpu_released, rq->scx.ops_qseq, - rq->scx.kick_sync); - dump_line(&ns, " curr=%s[%d] class=%ps", - rq->curr->comm, rq->curr->pid, - rq->curr->sched_class); - if (!cpumask_empty(rq->scx.cpus_to_kick)) - dump_line(&ns, " cpus_to_kick : %*pb", - cpumask_pr_args(rq->scx.cpus_to_kick)); - if (!cpumask_empty(rq->scx.cpus_to_kick_if_idle)) - dump_line(&ns, " idle_to_kick : %*pb", - cpumask_pr_args(rq->scx.cpus_to_kick_if_idle)); - if (!cpumask_empty(rq->scx.cpus_to_preempt)) - dump_line(&ns, " cpus_to_preempt: %*pb", - cpumask_pr_args(rq->scx.cpus_to_preempt)); - if (!cpumask_empty(rq->scx.cpus_to_wait)) - dump_line(&ns, " cpus_to_wait : %*pb", - cpumask_pr_args(rq->scx.cpus_to_wait)); - if (!cpumask_empty(rq->scx.cpus_to_sync)) - dump_line(&ns, " cpus_to_sync : %*pb", - cpumask_pr_args(rq->scx.cpus_to_sync)); - - used = seq_buf_used(&ns); - if (SCX_HAS_OP(sch, dump_cpu)) { - ops_dump_init(&ns, " "); - SCX_CALL_OP(sch, dump_cpu, rq, &dctx, cpu, idle); - ops_dump_exit(); - } - - /* - * If idle && nothing generated by ops.dump_cpu(), there's - * nothing interesting. Skip. - */ - if (idle && used == seq_buf_used(&ns)) - goto next; - - /* - * $s may already have overflowed when $ns was created. If so, - * calling commit on it will trigger BUG. - */ - if (avail) { - seq_buf_commit(&s, seq_buf_used(&ns)); - if (seq_buf_has_overflowed(&ns)) - seq_buf_set_overflow(&s); - } - - if (rq->curr->sched_class == &ext_sched_class && - (dump_all_tasks || scx_task_on_sched(sch, rq->curr))) - scx_dump_task(sch, &s, &dctx, rq, rq->curr, '*'); - - list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) - if (dump_all_tasks || scx_task_on_sched(sch, p)) - scx_dump_task(sch, &s, &dctx, rq, p, ' '); - next: - rq_unlock_irqrestore(rq, &rf); + scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks); } dump_newline(&s); -- cgit v1.2.3 From 78683079ca6d91392e84ed313e15bb330d7da851 Mon Sep 17 00:00:00 2001 From: Changwoo Min Date: Wed, 29 Apr 2026 17:23:17 +0900 Subject: sched_ext: Dump the exit CPU first When sched_ext is disabled by an error, the CPU that triggered the exit is the most relevant piece of information for diagnosing the problem. However, if there are many CPUs, the dump can get truncated and that CPU's information may not appear in the output. Add an exit_cpu field to scx_exit_info and thread it through scx_vexit() / __scx_exit(). For the watchdog stall path, populate it from cpu_of(rq) in check_rq_for_timeouts(). For all other exit paths, define a scx_exit() macro that wraps __scx_exit() with raw_smp_processor_id(), so the CPU that initiated the exit is captured automatically, with no call-site changes needed. In scx_dump_state(), report the exit CPU in the dump header ("on cpu N") and dump that CPU first, skipping it in the per-CPU loop, so the most relevant CPU is never truncated out of the dump. The SysRq-D path initializes exit_cpu to -1 so debug dumps not tied to an exit don't arbitrarily promote CPU 0. Signed-off-by: Changwoo Min Signed-off-by: Tejun Heo --- kernel/sched/ext.c | 52 +++++++++++++++++++++++++++++++++------------ kernel/sched/ext_internal.h | 6 ++++++ 2 files changed, 44 insertions(+), 14 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index fff3c8ab0175..920ebd441e52 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -232,24 +232,29 @@ static bool task_dead_and_done(struct task_struct *p); static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind); static bool scx_vexit(struct scx_sched *sch, enum scx_exit_kind kind, - s64 exit_code, const char *fmt, va_list args); + s64 exit_code, s32 exit_cpu, const char *fmt, + va_list args); -static __printf(4, 5) bool scx_exit(struct scx_sched *sch, - enum scx_exit_kind kind, s64 exit_code, - const char *fmt, ...) +static __printf(5, 6) bool __scx_exit(struct scx_sched *sch, + enum scx_exit_kind kind, s64 exit_code, + s32 exit_cpu, const char *fmt, ...) { va_list args; bool ret; va_start(args, fmt); - ret = scx_vexit(sch, kind, exit_code, fmt, args); + ret = scx_vexit(sch, kind, exit_code, exit_cpu, fmt, args); va_end(args); return ret; } +#define scx_exit(sch, kind, exit_code, fmt, args...) \ + __scx_exit(sch, kind, exit_code, raw_smp_processor_id(), fmt, ##args) + #define scx_error(sch, fmt, args...) scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args) -#define scx_verror(sch, fmt, args) scx_vexit((sch), SCX_EXIT_ERROR, 0, fmt, args) +#define scx_verror(sch, fmt, args) \ + scx_vexit((sch), SCX_EXIT_ERROR, 0, raw_smp_processor_id(), fmt, args) #define SCX_HAS_OP(sch, op) test_bit(SCX_OP_IDX(op), (sch)->has_op) @@ -3387,9 +3392,10 @@ static bool check_rq_for_timeouts(struct rq *rq) last_runnable + READ_ONCE(sch->watchdog_timeout)))) { u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); - scx_exit(sch, SCX_EXIT_ERROR_STALL, 0, - "%s[%d] failed to run for %u.%03us", - p->comm, p->pid, dur_ms / 1000, dur_ms % 1000); + __scx_exit(sch, SCX_EXIT_ERROR_STALL, 0, cpu_of(rq), + "%s[%d] failed to run for %u.%03us", + p->comm, p->pid, dur_ms / 1000, + dur_ms % 1000); timed_out = true; break; } @@ -5526,6 +5532,7 @@ static struct scx_exit_info *alloc_exit_info(size_t exit_dump_len) if (!ei) return NULL; + ei->exit_cpu = -1; ei->bt = kzalloc_objs(ei->bt[0], SCX_EXIT_BT_LEN); ei->msg = kzalloc(SCX_EXIT_MSG_LEN, GFP_KERNEL); ei->dump = kvzalloc(exit_dump_len, GFP_KERNEL); @@ -6382,8 +6389,13 @@ static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei, if (ei->kind == SCX_EXIT_NONE) { dump_line(&s, "Debug dump triggered by %s", ei->reason); } else { - dump_line(&s, "%s[%d] triggered exit kind %d:", - current->comm, current->pid, ei->kind); + if (ei->exit_cpu >= 0) + dump_line(&s, "%s[%d] triggered exit kind %d on cpu %d:", + current->comm, current->pid, ei->kind, + ei->exit_cpu); + else + dump_line(&s, "%s[%d] triggered exit kind %d:", + current->comm, current->pid, ei->kind); dump_line(&s, " %s (%s)", ei->reason, ei->msg); dump_newline(&s); dump_line(&s, "Backtrace:"); @@ -6400,8 +6412,15 @@ static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei, dump_line(&s, "CPU states"); dump_line(&s, "----------"); + /* + * Dump the exit CPU first so it isn't lost to dump truncation, then + * walk the rest in order, skipping the one already dumped. + */ + if (ei->exit_cpu >= 0) + scx_dump_cpu(sch, &s, &dctx, ei->exit_cpu, dump_all_tasks); for_each_possible_cpu(cpu) { - scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks); + if (cpu != ei->exit_cpu) + scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks); } dump_newline(&s); @@ -6440,7 +6459,7 @@ static void scx_disable_irq_workfn(struct irq_work *irq_work) } static bool scx_vexit(struct scx_sched *sch, - enum scx_exit_kind kind, s64 exit_code, + enum scx_exit_kind kind, s64 exit_code, s32 exit_cpu, const char *fmt, va_list args) { struct scx_exit_info *ei = sch->exit_info; @@ -6463,6 +6482,7 @@ static bool scx_vexit(struct scx_sched *sch, */ ei->kind = kind; ei->reason = scx_exit_reason(ei->kind); + ei->exit_cpu = exit_cpu; irq_work_queue(&sch->disable_irq_work); return true; @@ -7728,7 +7748,11 @@ static const struct sysrq_key_op sysrq_sched_ext_reset_op = { static void sysrq_handle_sched_ext_dump(u8 key) { - struct scx_exit_info ei = { .kind = SCX_EXIT_NONE, .reason = "SysRq-D" }; + struct scx_exit_info ei = { + .kind = SCX_EXIT_NONE, + .exit_cpu = -1, + .reason = "SysRq-D", + }; struct scx_sched *sch; list_for_each_entry_rcu(sch, &scx_sched_all, all) diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext_internal.h index a54903bb74b3..54c6ed43b6c7 100644 --- a/kernel/sched/ext_internal.h +++ b/kernel/sched/ext_internal.h @@ -97,6 +97,12 @@ struct scx_exit_info { /* %SCX_EXIT_* - broad category of the exit reason */ enum scx_exit_kind kind; + /* + * CPU that initiated the exit, valid once @kind has been set. + * Negative if the exit path didn't identify a CPU. + */ + s32 exit_cpu; + /* exit code if gracefully exiting */ s64 exit_code; -- cgit v1.2.3 From 0e781a7853785989553f8a4e658145623ef695ac Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 29 Apr 2026 08:09:10 -1000 Subject: sched_ext: Add ext_types.h for early subsystem-wide defs Introduce kernel/sched/ext_types.h as the early-def header for the sched_ext compilation unit. Included from kernel/sched/build_policy.c before ext_internal.h so every later header and source in the unit sees its content without re-inclusion. Later patches add their types here (struct scx_cid_topo, scx_cmask, scx_cid_shard, etc.) so the subsystem has one place to stash types shared across the TU. Move enum scx_consts (SCX_DSP_DFL_MAX_BATCH, SCX_WATCHDOG_MAX_TIMEOUT, SCX_SUB_MAX_DEPTH, etc.) here as the initial content. Ops-facing content stays in ext_internal.h. Signed-off-by: Tejun Heo Reviewed-by: Changwoo Min Reviewed-by: Andrea Righi --- kernel/sched/build_policy.c | 1 + kernel/sched/ext_internal.h | 32 -------------------------------- kernel/sched/ext_types.h | 43 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 32 deletions(-) create mode 100644 kernel/sched/ext_types.h (limited to 'kernel') diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c index ffb386889218..1d92f7d7a19f 100644 --- a/kernel/sched/build_policy.c +++ b/kernel/sched/build_policy.c @@ -59,6 +59,7 @@ #ifdef CONFIG_SCHED_CLASS_EXT # include +# include "ext_types.h" # include "ext_internal.h" # include "ext_idle.h" # include "ext.c" diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext_internal.h index 54c6ed43b6c7..26668bbce2b3 100644 --- a/kernel/sched/ext_internal.h +++ b/kernel/sched/ext_internal.h @@ -8,38 +8,6 @@ #define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) #define SCX_MOFF_IDX(moff) ((moff) / sizeof(void (*)(void))) -enum scx_consts { - SCX_DSP_DFL_MAX_BATCH = 32, - SCX_DSP_MAX_LOOPS = 32, - SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, - - /* per-CPU chunk size for p->scx.tid allocation, see scx_alloc_tid() */ - SCX_TID_CHUNK = 1024, - - SCX_EXIT_BT_LEN = 64, - SCX_EXIT_MSG_LEN = 1024, - SCX_EXIT_DUMP_DFL_LEN = 32768, - - SCX_CPUPERF_ONE = SCHED_CAPACITY_SCALE, - - /* - * Iterating all tasks may take a while. Periodically drop - * scx_tasks_lock to avoid causing e.g. CSD and RCU stalls. - */ - SCX_TASK_ITER_BATCH = 32, - - SCX_BYPASS_HOST_NTH = 2, - - SCX_BYPASS_LB_DFL_INTV_US = 500 * USEC_PER_MSEC, - SCX_BYPASS_LB_DONOR_PCT = 125, - SCX_BYPASS_LB_MIN_DELTA_DIV = 4, - SCX_BYPASS_LB_BATCH = 256, - - SCX_REENQ_LOCAL_MAX_REPEAT = 256, - - SCX_SUB_MAX_DEPTH = 4, -}; - enum scx_exit_kind { SCX_EXIT_NONE, SCX_EXIT_DONE, diff --git a/kernel/sched/ext_types.h b/kernel/sched/ext_types.h new file mode 100644 index 000000000000..19299ec3920e --- /dev/null +++ b/kernel/sched/ext_types.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Early sched_ext type definitions. + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2026 Tejun Heo + */ +#ifndef _KERNEL_SCHED_EXT_TYPES_H +#define _KERNEL_SCHED_EXT_TYPES_H + +enum scx_consts { + SCX_DSP_DFL_MAX_BATCH = 32, + SCX_DSP_MAX_LOOPS = 32, + SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, + + /* per-CPU chunk size for p->scx.tid allocation, see scx_alloc_tid() */ + SCX_TID_CHUNK = 1024, + + SCX_EXIT_BT_LEN = 64, + SCX_EXIT_MSG_LEN = 1024, + SCX_EXIT_DUMP_DFL_LEN = 32768, + + SCX_CPUPERF_ONE = SCHED_CAPACITY_SCALE, + + /* + * Iterating all tasks may take a while. Periodically drop + * scx_tasks_lock to avoid causing e.g. CSD and RCU stalls. + */ + SCX_TASK_ITER_BATCH = 32, + + SCX_BYPASS_HOST_NTH = 2, + + SCX_BYPASS_LB_DFL_INTV_US = 500 * USEC_PER_MSEC, + SCX_BYPASS_LB_DONOR_PCT = 125, + SCX_BYPASS_LB_MIN_DELTA_DIV = 4, + SCX_BYPASS_LB_BATCH = 256, + + SCX_REENQ_LOCAL_MAX_REPEAT = 256, + + SCX_SUB_MAX_DEPTH = 4, +}; + +#endif /* _KERNEL_SCHED_EXT_TYPES_H */ -- cgit v1.2.3 From 93292808b24355126620d814fff7cc076f262855 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 29 Apr 2026 08:09:10 -1000 Subject: sched_ext: Rename ops_cpu_valid() to scx_cpu_valid() and expose it Rename the static ext.c helper and declare it in ext_internal.h so ext_idle.c and the upcoming cid code can call it directly instead of relying on build_policy.c textual inclusion. Pure rename and visibility change. Signed-off-by: Tejun Heo Reviewed-by: Cheng-Yang Chou Reviewed-by: Changwoo Min Reviewed-by: Andrea Righi --- kernel/sched/ext.c | 22 +++++++++++----------- kernel/sched/ext_idle.c | 6 +++--- kernel/sched/ext_internal.h | 2 ++ 3 files changed, 16 insertions(+), 14 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 920ebd441e52..0d0833bd8d20 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -1067,7 +1067,7 @@ static inline bool __cpu_valid(s32 cpu) } /** - * ops_cpu_valid - Verify a cpu number, to be used on ops input args + * scx_cpu_valid - Verify a cpu number, to be used on ops input args * @sch: scx_sched to abort on error * @cpu: cpu number which came from a BPF ops * @where: extra information reported on error @@ -1076,7 +1076,7 @@ static inline bool __cpu_valid(s32 cpu) * Verify that it is in range and one of the possible cpus. If invalid, trigger * an ops error. */ -static bool ops_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where) +bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where) { if (__cpu_valid(cpu)) { return true; @@ -1691,7 +1691,7 @@ static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch, if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; - if (!ops_cpu_valid(sch, cpu, "in SCX_DSQ_LOCAL_ON dispatch verdict")) + if (!scx_cpu_valid(sch, cpu, "in SCX_DSQ_LOCAL_ON dispatch verdict")) return find_global_dsq(sch, tcpu); return &cpu_rq(cpu)->scx.local_dsq; @@ -3274,7 +3274,7 @@ static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flag this_rq()->scx.in_select_cpu = false; p->scx.selected_cpu = cpu; *ddsp_taskp = NULL; - if (ops_cpu_valid(sch, cpu, "from ops.select_cpu()")) + if (scx_cpu_valid(sch, cpu, "from ops.select_cpu()")) return cpu; else return prev_cpu; @@ -8822,7 +8822,7 @@ static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) struct rq *this_rq; unsigned long irq_flags; - if (!ops_cpu_valid(sch, cpu, NULL)) + if (!scx_cpu_valid(sch, cpu, NULL)) return; local_irq_save(irq_flags); @@ -8919,7 +8919,7 @@ __bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id, const struct bpf_prog_aux *aux } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; - if (ops_cpu_valid(sch, cpu, NULL)) { + if (scx_cpu_valid(sch, cpu, NULL)) { ret = READ_ONCE(cpu_rq(cpu)->scx.local_dsq.nr); goto out; } @@ -9308,7 +9308,7 @@ __bpf_kfunc u32 scx_bpf_cpuperf_cap(s32 cpu, const struct bpf_prog_aux *aux) guard(rcu)(); sch = scx_prog_sched(aux); - if (likely(sch) && ops_cpu_valid(sch, cpu, NULL)) + if (likely(sch) && scx_cpu_valid(sch, cpu, NULL)) return arch_scale_cpu_capacity(cpu); else return SCX_CPUPERF_ONE; @@ -9336,7 +9336,7 @@ __bpf_kfunc u32 scx_bpf_cpuperf_cur(s32 cpu, const struct bpf_prog_aux *aux) guard(rcu)(); sch = scx_prog_sched(aux); - if (likely(sch) && ops_cpu_valid(sch, cpu, NULL)) + if (likely(sch) && scx_cpu_valid(sch, cpu, NULL)) return arch_scale_freq_capacity(cpu); else return SCX_CPUPERF_ONE; @@ -9372,7 +9372,7 @@ __bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf, const struct bpf_prog_au return; } - if (ops_cpu_valid(sch, cpu, NULL)) { + if (scx_cpu_valid(sch, cpu, NULL)) { struct rq *rq = cpu_rq(cpu), *locked_rq = scx_locked_rq(); struct rq_flags rf; @@ -9485,7 +9485,7 @@ __bpf_kfunc struct rq *scx_bpf_cpu_rq(s32 cpu, const struct bpf_prog_aux *aux) if (unlikely(!sch)) return NULL; - if (!ops_cpu_valid(sch, cpu, NULL)) + if (!scx_cpu_valid(sch, cpu, NULL)) return NULL; if (!sch->warned_deprecated_rq) { @@ -9542,7 +9542,7 @@ __bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu, const struct bpf_prog_ if (unlikely(!sch)) return NULL; - if (!ops_cpu_valid(sch, cpu, NULL)) + if (!scx_cpu_valid(sch, cpu, NULL)) return NULL; return rcu_dereference(cpu_rq(cpu)->curr); diff --git a/kernel/sched/ext_idle.c b/kernel/sched/ext_idle.c index f0f4d9500997..860c4634f60e 100644 --- a/kernel/sched/ext_idle.c +++ b/kernel/sched/ext_idle.c @@ -916,7 +916,7 @@ static s32 select_cpu_from_kfunc(struct scx_sched *sch, struct task_struct *p, bool we_locked = false; s32 cpu; - if (!ops_cpu_valid(sch, prev_cpu, NULL)) + if (!scx_cpu_valid(sch, prev_cpu, NULL)) return -EINVAL; if (!check_builtin_idle_enabled(sch)) @@ -989,7 +989,7 @@ __bpf_kfunc s32 scx_bpf_cpu_node(s32 cpu, const struct bpf_prog_aux *aux) guard(rcu)(); sch = scx_prog_sched(aux); - if (unlikely(!sch) || !ops_cpu_valid(sch, cpu, NULL)) + if (unlikely(!sch) || !scx_cpu_valid(sch, cpu, NULL)) return NUMA_NO_NODE; return cpu_to_node(cpu); } @@ -1271,7 +1271,7 @@ __bpf_kfunc bool scx_bpf_test_and_clear_cpu_idle(s32 cpu, const struct bpf_prog_ if (!check_builtin_idle_enabled(sch)) return false; - if (!ops_cpu_valid(sch, cpu, NULL)) + if (!scx_cpu_valid(sch, cpu, NULL)) return false; return scx_idle_test_and_clear_cpu(cpu); diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext_internal.h index 26668bbce2b3..d8fb0b0b652c 100644 --- a/kernel/sched/ext_internal.h +++ b/kernel/sched/ext_internal.h @@ -1358,6 +1358,8 @@ DECLARE_PER_CPU(struct rq *, scx_locked_rq_state); int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id); +bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where); + /* * Return the rq currently locked from an scx callback, or NULL if no rq is * locked. -- cgit v1.2.3 From 7f19c089cc4da60c6084fd5f92f13637c20f92ad Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 29 Apr 2026 08:09:10 -1000 Subject: sched_ext: Move scx_exit(), scx_error() and friends to ext_internal.h Things shared across multiple .c files belong in a header. scx_exit() / scx_error() (and their scx_vexit() / scx_verror() siblings) are already called from ext_idle.c and the upcoming ext_cid.c, and it was only build_policy.c's textual inclusion of ext.c that made the references resolve. Move the whole family to ext_internal.h. Pure visibility change. v4: Rebased over the exit_cpu plumbing. scx_exit() and scx_verror() are now macros wrapping raw_smp_processor_id(); move both macros plus the underlying __scx_exit() / scx_vexit() declarations to the header. Signed-off-by: Tejun Heo Reviewed-by: Cheng-Yang Chou Reviewed-by: Changwoo Min Reviewed-by: Andrea Righi --- kernel/sched/ext.c | 22 ++++++---------------- kernel/sched/ext_internal.h | 12 ++++++++++++ 2 files changed, 18 insertions(+), 16 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 0d0833bd8d20..2f16dc2b814b 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -231,13 +231,10 @@ static void run_deferred(struct rq *rq); static bool task_dead_and_done(struct task_struct *p); static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind); -static bool scx_vexit(struct scx_sched *sch, enum scx_exit_kind kind, - s64 exit_code, s32 exit_cpu, const char *fmt, - va_list args); -static __printf(5, 6) bool __scx_exit(struct scx_sched *sch, - enum scx_exit_kind kind, s64 exit_code, - s32 exit_cpu, const char *fmt, ...) +__printf(5, 6) bool __scx_exit(struct scx_sched *sch, + enum scx_exit_kind kind, s64 exit_code, + s32 exit_cpu, const char *fmt, ...) { va_list args; bool ret; @@ -249,13 +246,6 @@ static __printf(5, 6) bool __scx_exit(struct scx_sched *sch, return ret; } -#define scx_exit(sch, kind, exit_code, fmt, args...) \ - __scx_exit(sch, kind, exit_code, raw_smp_processor_id(), fmt, ##args) - -#define scx_error(sch, fmt, args...) scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args) -#define scx_verror(sch, fmt, args) \ - scx_vexit((sch), SCX_EXIT_ERROR, 0, raw_smp_processor_id(), fmt, args) - #define SCX_HAS_OP(sch, op) test_bit(SCX_OP_IDX(op), (sch)->has_op) static long jiffies_delta_msecs(unsigned long at, unsigned long now) @@ -6458,9 +6448,9 @@ static void scx_disable_irq_workfn(struct irq_work *irq_work) kthread_queue_work(sch->helper, &sch->disable_work); } -static bool scx_vexit(struct scx_sched *sch, - enum scx_exit_kind kind, s64 exit_code, s32 exit_cpu, - const char *fmt, va_list args) +bool scx_vexit(struct scx_sched *sch, + enum scx_exit_kind kind, s64 exit_code, s32 exit_cpu, + const char *fmt, va_list args) { struct scx_exit_info *ei = sch->exit_info; diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext_internal.h index d8fb0b0b652c..0c55c5481fa5 100644 --- a/kernel/sched/ext_internal.h +++ b/kernel/sched/ext_internal.h @@ -1360,6 +1360,18 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id); bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where); +bool scx_vexit(struct scx_sched *sch, enum scx_exit_kind kind, s64 exit_code, + s32 exit_cpu, const char *fmt, va_list args); +__printf(5, 6) bool __scx_exit(struct scx_sched *sch, enum scx_exit_kind kind, + s64 exit_code, s32 exit_cpu, const char *fmt, ...); + +#define scx_exit(sch, kind, exit_code, fmt, args...) \ + __scx_exit(sch, kind, exit_code, raw_smp_processor_id(), fmt, ##args) +#define scx_error(sch, fmt, args...) \ + scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args) +#define scx_verror(sch, fmt, args) \ + scx_vexit((sch), SCX_EXIT_ERROR, 0, raw_smp_processor_id(), fmt, args) + /* * Return the rq currently locked from an scx callback, or NULL if no rq is * locked. -- cgit v1.2.3 From 017ec7d5fe178acbd3c9488843b800d3a32be295 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 29 Apr 2026 08:09:10 -1000 Subject: sched_ext: Shift scx_kick_cpu() validity check to scx_bpf_kick_cpu() Callers that already know the cpu is valid shouldn't have to pay for a redundant check. scx_kick_cpu() is called from the in-kernel balance loop break-out path with the current cpu (trivially valid) and from scx_bpf_kick_cpu() with a BPF-supplied cpu that does need validation. Move the check out of scx_kick_cpu() into scx_bpf_kick_cpu() so the backend is reusable by callers that have already validated. Signed-off-by: Tejun Heo Reviewed-by: Cheng-Yang Chou Reviewed-by: Changwoo Min Reviewed-by: Andrea Righi --- kernel/sched/ext.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 2f16dc2b814b..6575b52f4b9d 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -8812,9 +8812,6 @@ static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) struct rq *this_rq; unsigned long irq_flags; - if (!scx_cpu_valid(sch, cpu, NULL)) - return; - local_irq_save(irq_flags); this_rq = this_rq(); @@ -8877,7 +8874,7 @@ __bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux guard(rcu)(); sch = scx_prog_sched(aux); - if (likely(sch)) + if (likely(sch) && scx_cpu_valid(sch, cpu, NULL)) scx_kick_cpu(sch, cpu, flags); } -- cgit v1.2.3 From 1e315c44aa91b4475e445d007151c13429ce7bce Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 29 Apr 2026 08:09:10 -1000 Subject: sched_ext: Relocate cpu_acquire/cpu_release to end of struct sched_ext_ops cpu_acquire and cpu_release are deprecated and slated for removal. Move their declarations to the end of struct sched_ext_ops so an upcoming cid-form struct (sched_ext_ops_cid) can omit them entirely without disturbing the offsets of the shared fields. Switch the two SCX_HAS_OP() callers for these ops to direct field checks since the relocated ops sit outside the SCX_OPI_END range covered by the has_op bitmap. scx_kf_allow_flags[] auto-sizes to the highest used SCX_OP_IDX, so SCX_OP_IDX(cpu_release) moving to a higher index just enlarges the sparse array; the lookup logic is unchanged. Signed-off-by: Tejun Heo Reviewed-by: Cheng-Yang Chou Reviewed-by: Changwoo Min Reviewed-by: Andrea Righi --- kernel/sched/ext.c | 4 ++-- kernel/sched/ext_internal.h | 54 +++++++++++++++++++++++++++------------------ 2 files changed, 34 insertions(+), 24 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 6575b52f4b9d..6bf1418c4237 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -2823,7 +2823,7 @@ static int balance_one(struct rq *rq, struct task_struct *prev) * core. This callback complements ->cpu_release(), which is * emitted in switch_class(). */ - if (SCX_HAS_OP(sch, cpu_acquire)) + if (sch->ops.cpu_acquire) SCX_CALL_OP(sch, cpu_acquire, rq, cpu, NULL); rq->scx.cpu_released = false; } @@ -2969,7 +2969,7 @@ static void switch_class(struct rq *rq, struct task_struct *next) * next time that balance_one() is invoked. */ if (!rq->scx.cpu_released) { - if (SCX_HAS_OP(sch, cpu_release)) { + if (sch->ops.cpu_release) { struct scx_cpu_release_args args = { .reason = preempt_reason_from_class(next_class), .task = next, diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext_internal.h index 0c55c5481fa5..c6de974eaf48 100644 --- a/kernel/sched/ext_internal.h +++ b/kernel/sched/ext_internal.h @@ -529,28 +529,6 @@ struct sched_ext_ops { */ void (*update_idle)(s32 cpu, bool idle); - /** - * @cpu_acquire: A CPU is becoming available to the BPF scheduler - * @cpu: The CPU being acquired by the BPF scheduler. - * @args: Acquire arguments, see the struct definition. - * - * A CPU that was previously released from the BPF scheduler is now once - * again under its control. - */ - void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); - - /** - * @cpu_release: A CPU is taken away from the BPF scheduler - * @cpu: The CPU being released by the BPF scheduler. - * @args: Release arguments, see the struct definition. - * - * The specified CPU is no longer under the control of the BPF - * scheduler. This could be because it was preempted by a higher - * priority sched_class, though there may be other reasons as well. The - * caller should consult @args->reason to determine the cause. - */ - void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); - /** * @init_task: Initialize a task to run in a BPF scheduler * @p: task to initialize for BPF scheduling @@ -841,6 +819,38 @@ struct sched_ext_ops { /* internal use only, must be NULL */ void __rcu *priv; + + /* + * Deprecated callbacks. Kept at the end of the struct so the cid-form + * struct (sched_ext_ops_cid) can omit them without affecting the + * shared field offsets. Use SCX_ENQ_IMMED instead. Sitting past + * SCX_OPI_END means has_op doesn't cover them, so SCX_HAS_OP() cannot + * be used; callers must test sch->ops.cpu_acquire / cpu_release + * directly. + */ + + /** + * @cpu_acquire: A CPU is becoming available to the BPF scheduler + * @cpu: The CPU being acquired by the BPF scheduler. + * @args: Acquire arguments, see the struct definition. + * + * A CPU that was previously released from the BPF scheduler is now once + * again under its control. Deprecated; use SCX_ENQ_IMMED instead. + */ + void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); + + /** + * @cpu_release: A CPU is taken away from the BPF scheduler + * @cpu: The CPU being released by the BPF scheduler. + * @args: Release arguments, see the struct definition. + * + * The specified CPU is no longer under the control of the BPF + * scheduler. This could be because it was preempted by a higher + * priority sched_class, though there may be other reasons as well. The + * caller should consult @args->reason to determine the cause. + * Deprecated; use SCX_ENQ_IMMED instead. + */ + void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); }; enum scx_opi { -- cgit v1.2.3 From c97ce728bcc43adcc861375f48433ac7adfd4e2e Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 29 Apr 2026 08:09:10 -1000 Subject: sched_ext: Make scx_enable() take scx_enable_cmd Pass struct scx_enable_cmd to scx_enable() rather than unpacking @ops at every call site and re-packing into a fresh cmd inside. bpf_scx_reg() now builds the cmd on its stack and hands it in; scx_enable() just wires up the kthread work and waits. Relocate struct scx_enable_cmd above scx_alloc_and_add_sched() so upcoming patches that also want the cmd can see it. No behavior change. Signed-off-by: Tejun Heo Reviewed-by: Cheng-Yang Chou Reviewed-by: Changwoo Min Reviewed-by: Andrea Righi --- kernel/sched/ext.c | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 6bf1418c4237..cff6047632ec 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -6529,6 +6529,19 @@ static struct scx_sched_pnode *alloc_pnode(struct scx_sched *sch, int node) return pnode; } +/* + * scx_enable() is offloaded to a dedicated system-wide RT kthread to avoid + * starvation. During the READY -> ENABLED task switching loop, the calling + * thread's sched_class gets switched from fair to ext. As fair has higher + * priority than ext, the calling thread can be indefinitely starved under + * fair-class saturation, leading to a system hang. + */ +struct scx_enable_cmd { + struct kthread_work work; + struct sched_ext_ops *ops; + int ret; +}; + /* * Allocate and initialize a new scx_sched. @cgrp's reference is always * consumed whether the function succeeds or fails. @@ -6771,19 +6784,6 @@ static int validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) return 0; } -/* - * scx_enable() is offloaded to a dedicated system-wide RT kthread to avoid - * starvation. During the READY -> ENABLED task switching loop, the calling - * thread's sched_class gets switched from fair to ext. As fair has higher - * priority than ext, the calling thread can be indefinitely starved under - * fair-class saturation, leading to a system hang. - */ -struct scx_enable_cmd { - struct kthread_work work; - struct sched_ext_ops *ops; - int ret; -}; - static void scx_root_enable_workfn(struct kthread_work *work) { struct scx_enable_cmd *cmd = container_of(work, struct scx_enable_cmd, work); @@ -7368,11 +7368,10 @@ static s32 __init scx_cgroup_lifetime_notifier_init(void) core_initcall(scx_cgroup_lifetime_notifier_init); #endif /* CONFIG_EXT_SUB_SCHED */ -static s32 scx_enable(struct sched_ext_ops *ops, struct bpf_link *link) +static s32 scx_enable(struct scx_enable_cmd *cmd, struct bpf_link *link) { static struct kthread_worker *helper; static DEFINE_MUTEX(helper_mutex); - struct scx_enable_cmd cmd; if (!cpumask_equal(housekeeping_cpumask(HK_TYPE_DOMAIN), cpu_possible_mask)) { @@ -7396,16 +7395,15 @@ static s32 scx_enable(struct sched_ext_ops *ops, struct bpf_link *link) } #ifdef CONFIG_EXT_SUB_SCHED - if (ops->sub_cgroup_id > 1) - kthread_init_work(&cmd.work, scx_sub_enable_workfn); + if (cmd->ops->sub_cgroup_id > 1) + kthread_init_work(&cmd->work, scx_sub_enable_workfn); else #endif /* CONFIG_EXT_SUB_SCHED */ - kthread_init_work(&cmd.work, scx_root_enable_workfn); - cmd.ops = ops; + kthread_init_work(&cmd->work, scx_root_enable_workfn); - kthread_queue_work(READ_ONCE(helper), &cmd.work); - kthread_flush_work(&cmd.work); - return cmd.ret; + kthread_queue_work(READ_ONCE(helper), &cmd->work); + kthread_flush_work(&cmd->work); + return cmd->ret; } @@ -7577,7 +7575,9 @@ static int bpf_scx_check_member(const struct btf_type *t, static int bpf_scx_reg(void *kdata, struct bpf_link *link) { - return scx_enable(kdata, link); + struct scx_enable_cmd cmd = { .ops = kdata }; + + return scx_enable(&cmd, link); } static void bpf_scx_unreg(void *kdata, struct bpf_link *link) -- cgit v1.2.3 From e9b55af47edf6e60c9a47b5604c3e15c5162ec86 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 29 Apr 2026 08:09:10 -1000 Subject: sched_ext: Add topological CPU IDs (cids) Raw cpu numbers are clumsy for sharding and cross-sched communication, especially from BPF. The space is sparse, numerical closeness doesn't track topological closeness (x86 hyperthreading often scatters SMT siblings), and a range of cpu ids doesn't describe anything meaningful. Sub-sched support makes this acute: cpu allocation, revocation, and state constantly flow across sub-scheds. Passing whole cpumasks scales poorly (every op scans 4K bits) and cpumasks are awkward in BPF. cids assign every cpu a dense, topology-ordered id. CPUs sharing a core, LLC, or NUMA node occupy contiguous cid ranges, so a topology unit becomes a (start, length) slice. Communication passes slices; BPF can process a u64 word of cids at a time. Build the mapping once at root enable by walking online cpus node -> LLC -> core. Possible-but-not-online cpus tail the space with no-topo cids. Expose kfuncs to map cpu <-> cid in either direction and to query each cid's topology metadata. v2: Use kzalloc_objs()/kmalloc_objs() for the three allocs in scx_cid_arrays_alloc() (Cheng-Yang Chou). v3: scx_cid_init() failure path now drops cpus_read_lock(); BUILD_BUG_ON tightened to match BPF cmask helpers' NR_CPUS<=8192. (Sashiko) Signed-off-by: Tejun Heo Reviewed-by: Cheng-Yang Chou Reviewed-by: Changwoo Min Reviewed-by: Andrea Righi --- kernel/sched/build_policy.c | 2 + kernel/sched/ext.c | 18 +++ kernel/sched/ext_cid.c | 301 ++++++++++++++++++++++++++++++++++++++++++++ kernel/sched/ext_cid.h | 129 +++++++++++++++++++ kernel/sched/ext_types.h | 23 ++++ 5 files changed, 473 insertions(+) create mode 100644 kernel/sched/ext_cid.c create mode 100644 kernel/sched/ext_cid.h (limited to 'kernel') diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c index 1d92f7d7a19f..5e76c9177d54 100644 --- a/kernel/sched/build_policy.c +++ b/kernel/sched/build_policy.c @@ -61,8 +61,10 @@ # include # include "ext_types.h" # include "ext_internal.h" +# include "ext_cid.h" # include "ext_idle.h" # include "ext.c" +# include "ext_cid.c" # include "ext_idle.c" #endif diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index cff6047632ec..a83f8ce57781 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -6842,6 +6842,18 @@ static void scx_root_enable_workfn(struct kthread_work *work) */ cpus_read_lock(); + /* + * Build the cid mapping before publishing scx_root. The cid kfuncs + * dereference the cid arrays unconditionally once scx_prog_sched() + * returns non-NULL; the rcu_assign_pointer() below pairs with their + * rcu_dereference() to make the populated arrays visible. + */ + ret = scx_cid_init(sch); + if (ret) { + cpus_read_unlock(); + goto err_disable; + } + /* * Make the scheduler instance visible. Must be inside cpus_read_lock(). * See handle_hotplug(). @@ -9914,6 +9926,12 @@ static int __init scx_init(void) return ret; } + ret = scx_cid_kfunc_init(); + if (ret) { + pr_err("sched_ext: Failed to register cid kfuncs (%d)\n", ret); + return ret; + } + ret = register_bpf_struct_ops(&bpf_sched_ext_ops, sched_ext_ops); if (ret) { pr_err("sched_ext: Failed to register struct_ops (%d)\n", ret); diff --git a/kernel/sched/ext_cid.c b/kernel/sched/ext_cid.c new file mode 100644 index 000000000000..5b73900edc87 --- /dev/null +++ b/kernel/sched/ext_cid.c @@ -0,0 +1,301 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2026 Tejun Heo + */ +#include + +/* + * cid tables. + * + * Pointers are published once on first enable and never revoked. The default + * mapping is populated before ops.init() runs; scx_bpf_cid_override() commits + * before it returns. As long as the BPF scheduler only uses the tables from + * those points onward, it sees a consistent view. + */ +s16 *scx_cid_to_cpu_tbl; +s16 *scx_cpu_to_cid_tbl; +struct scx_cid_topo *scx_cid_topo; + +#define SCX_CID_TOPO_NEG (struct scx_cid_topo) { \ + .core_cid = -1, .core_idx = -1, .llc_cid = -1, .llc_idx = -1, \ + .node_cid = -1, .node_idx = -1, \ +} + +/* + * Return @cpu's LLC shared_cpu_map. If cacheinfo isn't populated (offline or + * !present), record @cpu in @fallbacks and return its node mask instead - the + * worst that can happen is that the cpu's LLC becomes coarser than reality. + */ +static const struct cpumask *cpu_llc_mask(int cpu, struct cpumask *fallbacks) +{ + struct cpu_cacheinfo *ci = get_cpu_cacheinfo(cpu); + + if (!ci || !ci->info_list || !ci->num_leaves) { + cpumask_set_cpu(cpu, fallbacks); + return cpumask_of_node(cpu_to_node(cpu)); + } + return &ci->info_list[ci->num_leaves - 1].shared_cpu_map; +} + +/* Allocate the cid tables once on first enable; never freed. */ +static s32 scx_cid_arrays_alloc(void) +{ + u32 npossible = num_possible_cpus(); + s16 *cid_to_cpu, *cpu_to_cid; + struct scx_cid_topo *cid_topo; + + if (scx_cid_to_cpu_tbl) + return 0; + + cid_to_cpu = kzalloc_objs(*scx_cid_to_cpu_tbl, npossible, GFP_KERNEL); + cpu_to_cid = kzalloc_objs(*scx_cpu_to_cid_tbl, nr_cpu_ids, GFP_KERNEL); + cid_topo = kmalloc_objs(*scx_cid_topo, npossible, GFP_KERNEL); + + if (!cid_to_cpu || !cpu_to_cid || !cid_topo) { + kfree(cid_to_cpu); + kfree(cpu_to_cid); + kfree(cid_topo); + return -ENOMEM; + } + + WRITE_ONCE(scx_cid_to_cpu_tbl, cid_to_cpu); + WRITE_ONCE(scx_cpu_to_cid_tbl, cpu_to_cid); + WRITE_ONCE(scx_cid_topo, cid_topo); + return 0; +} + +/** + * scx_cid_init - build the cid mapping + * @sch: the scx_sched being initialized; used as the scx_error() target + * + * See "Topological CPU IDs" in ext_cid.h for the model. Walk online cpus by + * intersection at each level (parent_scratch & this_level_mask), which keeps + * containment correct by construction and naturally splits a physical LLC + * straddling two NUMA nodes into two LLC units. The caller must hold + * cpus_read_lock. + */ +s32 scx_cid_init(struct scx_sched *sch) +{ + cpumask_var_t to_walk __free(free_cpumask_var) = CPUMASK_VAR_NULL; + cpumask_var_t node_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; + cpumask_var_t llc_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; + cpumask_var_t core_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; + cpumask_var_t llc_fallback __free(free_cpumask_var) = CPUMASK_VAR_NULL; + cpumask_var_t online_no_topo __free(free_cpumask_var) = CPUMASK_VAR_NULL; + u32 next_cid = 0; + s32 next_node_idx = 0, next_llc_idx = 0, next_core_idx = 0; + s32 cpu, ret; + + /* CMASK_MAX_WORDS in cid.bpf.h covers NR_CPUS up to 8192 */ + BUILD_BUG_ON(NR_CPUS > 8192); + + lockdep_assert_cpus_held(); + + ret = scx_cid_arrays_alloc(); + if (ret) + return ret; + + if (!zalloc_cpumask_var(&to_walk, GFP_KERNEL) || + !zalloc_cpumask_var(&node_scratch, GFP_KERNEL) || + !zalloc_cpumask_var(&llc_scratch, GFP_KERNEL) || + !zalloc_cpumask_var(&core_scratch, GFP_KERNEL) || + !zalloc_cpumask_var(&llc_fallback, GFP_KERNEL) || + !zalloc_cpumask_var(&online_no_topo, GFP_KERNEL)) + return -ENOMEM; + + /* -1 sentinels for sparse-possible cpu id holes (0 is a valid cid) */ + for (cpu = 0; cpu < nr_cpu_ids; cpu++) + scx_cpu_to_cid_tbl[cpu] = -1; + + cpumask_copy(to_walk, cpu_online_mask); + + while (!cpumask_empty(to_walk)) { + s32 next_cpu = cpumask_first(to_walk); + s32 nid = cpu_to_node(next_cpu); + s32 node_cid = next_cid; + s32 node_idx; + + /* + * No NUMA info: skip and let the tail loop assign a no-topo + * cid. cpumask_of_node(-1) is undefined. + */ + if (nid < 0) { + cpumask_clear_cpu(next_cpu, to_walk); + continue; + } + + node_idx = next_node_idx++; + + /* node_scratch = to_walk & this node */ + cpumask_and(node_scratch, to_walk, cpumask_of_node(nid)); + if (WARN_ON_ONCE(!cpumask_test_cpu(next_cpu, node_scratch))) + return -EINVAL; + + while (!cpumask_empty(node_scratch)) { + s32 ncpu = cpumask_first(node_scratch); + const struct cpumask *llc_mask = cpu_llc_mask(ncpu, llc_fallback); + s32 llc_cid = next_cid; + s32 llc_idx = next_llc_idx++; + + /* llc_scratch = node_scratch & this llc */ + cpumask_and(llc_scratch, node_scratch, llc_mask); + if (WARN_ON_ONCE(!cpumask_test_cpu(ncpu, llc_scratch))) + return -EINVAL; + + while (!cpumask_empty(llc_scratch)) { + s32 lcpu = cpumask_first(llc_scratch); + const struct cpumask *sib = topology_sibling_cpumask(lcpu); + s32 core_cid = next_cid; + s32 core_idx = next_core_idx++; + s32 ccpu; + + /* core_scratch = llc_scratch & this core */ + cpumask_and(core_scratch, llc_scratch, sib); + if (WARN_ON_ONCE(!cpumask_test_cpu(lcpu, core_scratch))) + return -EINVAL; + + for_each_cpu(ccpu, core_scratch) { + s32 cid = next_cid++; + + scx_cid_to_cpu_tbl[cid] = ccpu; + scx_cpu_to_cid_tbl[ccpu] = cid; + scx_cid_topo[cid] = (struct scx_cid_topo){ + .core_cid = core_cid, + .core_idx = core_idx, + .llc_cid = llc_cid, + .llc_idx = llc_idx, + .node_cid = node_cid, + .node_idx = node_idx, + }; + + cpumask_clear_cpu(ccpu, llc_scratch); + cpumask_clear_cpu(ccpu, node_scratch); + cpumask_clear_cpu(ccpu, to_walk); + } + } + } + } + + /* + * No-topo section: any possible cpu without a cid - normally just the + * not-online ones. Collect any currently-online cpus that land here in + * @online_no_topo so we can warn about them at the end. + */ + for_each_cpu(cpu, cpu_possible_mask) { + s32 cid; + + if (__scx_cpu_to_cid(cpu) != -1) + continue; + if (cpu_online(cpu)) + cpumask_set_cpu(cpu, online_no_topo); + + cid = next_cid++; + scx_cid_to_cpu_tbl[cid] = cpu; + scx_cpu_to_cid_tbl[cpu] = cid; + scx_cid_topo[cid] = SCX_CID_TOPO_NEG; + } + + if (!cpumask_empty(llc_fallback)) + pr_warn("scx_cid: cpus without cacheinfo, using node mask as llc: %*pbl\n", + cpumask_pr_args(llc_fallback)); + if (!cpumask_empty(online_no_topo)) + pr_warn("scx_cid: online cpus with no usable topology: %*pbl\n", + cpumask_pr_args(online_no_topo)); + + return 0; +} + +__bpf_kfunc_start_defs(); + +/** + * scx_bpf_cid_to_cpu - Return the raw CPU id for @cid + * @cid: cid to look up + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Return the raw CPU id for @cid. Trigger scx_error() and return -EINVAL if + * @cid is invalid. The cid<->cpu mapping is static for the lifetime of the + * loaded scheduler, so the BPF side can cache the result to avoid repeated + * kfunc invocations. + */ +__bpf_kfunc s32 scx_bpf_cid_to_cpu(s32 cid, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return -EINVAL; + return scx_cid_to_cpu(sch, cid); +} + +/** + * scx_bpf_cpu_to_cid - Return the cid for @cpu + * @cpu: cpu to look up + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Return the cid for @cpu. Trigger scx_error() and return -EINVAL if @cpu is + * invalid. The cid<->cpu mapping is static for the lifetime of the loaded + * scheduler, so the BPF side can cache the result to avoid repeated kfunc + * invocations. + */ +__bpf_kfunc s32 scx_bpf_cpu_to_cid(s32 cpu, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return -EINVAL; + return scx_cpu_to_cid(sch, cpu); +} + +/** + * scx_bpf_cid_topo - Copy out per-cid topology info + * @cid: cid to look up + * @out__uninit: where to copy the topology info; fully written by this call + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Fill @out__uninit with the topology info for @cid. Trigger scx_error() if + * @cid is out of range. If @cid is valid but in the no-topo section, all fields + * are set to -1. + */ +__bpf_kfunc void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out__uninit, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch) || !cid_valid(sch, cid)) { + *out__uninit = SCX_CID_TOPO_NEG; + return; + } + + *out__uninit = READ_ONCE(scx_cid_topo)[cid]; +} + +__bpf_kfunc_end_defs(); + +BTF_KFUNCS_START(scx_kfunc_ids_cid) +BTF_ID_FLAGS(func, scx_bpf_cid_to_cpu, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cpu_to_cid, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cid_topo, KF_IMPLICIT_ARGS) +BTF_KFUNCS_END(scx_kfunc_ids_cid) + +static const struct btf_kfunc_id_set scx_kfunc_set_cid = { + .owner = THIS_MODULE, + .set = &scx_kfunc_ids_cid, +}; + +int scx_cid_kfunc_init(void) +{ + return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_cid) ?: + register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &scx_kfunc_set_cid) ?: + register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_cid); +} diff --git a/kernel/sched/ext_cid.h b/kernel/sched/ext_cid.h new file mode 100644 index 000000000000..1dbe8262ccdd --- /dev/null +++ b/kernel/sched/ext_cid.h @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Topological CPU IDs (cids) + * -------------------------- + * + * Raw cpu numbers are clumsy for sharding work and communication across + * topology units, especially from BPF: the space can be sparse, numerical + * closeness doesn't imply topological closeness (x86 hyperthreading often puts + * SMT siblings far apart), and a range of cpu ids doesn't mean anything. + * Sub-scheds make this acute - cpu allocation, revocation and other state are + * constantly communicated across sub-scheds, and passing whole cpumasks scales + * poorly with cpu count. cpumasks are also awkward in BPF: a variable-length + * kernel type sized for the maximum NR_CPUS (4k), with verbose helper sequences + * for every op. + * + * cids give every cpu a dense, topology-ordered id. CPUs sharing a core, LLC or + * NUMA node get contiguous cid ranges, so a topology unit becomes a (start, + * length) slice of cid space. Communication can pass a slice instead of a + * cpumask, and BPF code can process, for example, a u64 word's worth of cids at + * a time. + * + * The mapping is built once at root scheduler enable time by walking the + * topology of online cpus only. Going by online cpus is out of necessity: + * depending on the arch, topology info isn't reliably available for offline + * cpus. The expected usage model is restarting the scheduler on hotplug events + * so the mapping is rebuilt against the new online set. A scheduler that wants + * to handle hotplug without a restart can provide its own cid and shard mapping + * through the override interface. + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2026 Tejun Heo + */ +#ifndef _KERNEL_SCHED_EXT_CID_H +#define _KERNEL_SCHED_EXT_CID_H + +struct scx_sched; + +/* + * Cid space (total is always num_possible_cpus()) is laid out with + * topology-annotated cids first, then no-topo cids at the tail. The + * topology-annotated block covers the cpus that were online when scx_cid_init() + * ran and remains valid even after those cpus go offline. The tail block covers + * possible-but-not-online cpus and carries all-(-1) topo info (see + * scx_cid_topo); callers detect it via the -1 sentinels. + * + * See the comment above the table definitions in ext_cid.c for the + * memory-ordering and visibility contract. + */ +extern s16 *scx_cid_to_cpu_tbl; +extern s16 *scx_cpu_to_cid_tbl; +extern struct scx_cid_topo *scx_cid_topo; + +s32 scx_cid_init(struct scx_sched *sch); +int scx_cid_kfunc_init(void); + +/** + * cid_valid - Verify a cid value, to be used on ops input args + * @sch: scx_sched to abort on error + * @cid: cid which came from a BPF ops + * + * Return true if @cid is in [0, num_possible_cpus()). On failure, trigger + * scx_error() and return false. + */ +static inline bool cid_valid(struct scx_sched *sch, s32 cid) +{ + if (likely(cid >= 0 && cid < num_possible_cpus())) + return true; + scx_error(sch, "invalid cid %d", cid); + return false; +} + +/** + * __scx_cid_to_cpu - Unchecked cid->cpu table lookup + * @cid: cid to look up. Must be in [0, num_possible_cpus()). + * + * Intended for callsites that have already validated @cid and that hold a + * non-NULL @sch from scx_prog_sched() - a live sched implies the table has + * been allocated, so no NULL check is needed here. + */ +static inline s32 __scx_cid_to_cpu(s32 cid) +{ + /* READ_ONCE pairs with WRITE_ONCE in scx_cid_arrays_alloc() */ + return READ_ONCE(scx_cid_to_cpu_tbl)[cid]; +} + +/** + * __scx_cpu_to_cid - Unchecked cpu->cid table lookup + * @cpu: cpu to look up. Must be a valid possible cpu id. + * + * Same usage constraints as __scx_cid_to_cpu(). + */ +static inline s32 __scx_cpu_to_cid(s32 cpu) +{ + return READ_ONCE(scx_cpu_to_cid_tbl)[cpu]; +} + +/** + * scx_cid_to_cpu - Translate @cid to its cpu + * @sch: scx_sched for error reporting + * @cid: cid to look up + * + * Return the cpu for @cid or a negative errno on failure. Invalid cid triggers + * scx_error() on @sch. The cid arrays are allocated on first scheduler enable + * and never freed, so the returned cpu is stable for the lifetime of the loaded + * scheduler. + */ +static inline s32 scx_cid_to_cpu(struct scx_sched *sch, s32 cid) +{ + if (!cid_valid(sch, cid)) + return -EINVAL; + return __scx_cid_to_cpu(cid); +} + +/** + * scx_cpu_to_cid - Translate @cpu to its cid + * @sch: scx_sched for error reporting + * @cpu: cpu to look up + * + * Return the cid for @cpu or a negative errno on failure. Invalid cpu triggers + * scx_error() on @sch. Same lifetime guarantee as scx_cid_to_cpu(). + */ +static inline s32 scx_cpu_to_cid(struct scx_sched *sch, s32 cpu) +{ + if (!scx_cpu_valid(sch, cpu, NULL)) + return -EINVAL; + return __scx_cpu_to_cid(cpu); +} + +#endif /* _KERNEL_SCHED_EXT_CID_H */ diff --git a/kernel/sched/ext_types.h b/kernel/sched/ext_types.h index 19299ec3920e..be4d3565ae8d 100644 --- a/kernel/sched/ext_types.h +++ b/kernel/sched/ext_types.h @@ -40,4 +40,27 @@ enum scx_consts { SCX_SUB_MAX_DEPTH = 4, }; +/* + * Per-cid topology info. For each topology level (core, LLC, node), records + * the first cid in the unit and its global index. Global indices are + * consecutive integers assigned in cid-walk order, so e.g. core_idx ranges + * over [0, nr_cores_at_init) with no gaps. No-topo cids have all fields set + * to -1. + * + * @core_cid: first cid of this cid's core (smt-sibling group) + * @core_idx: global index of that core, in [0, nr_cores_at_init) + * @llc_cid: first cid of this cid's LLC + * @llc_idx: global index of that LLC, in [0, nr_llcs_at_init) + * @node_cid: first cid of this cid's NUMA node + * @node_idx: global index of that node, in [0, nr_nodes_at_init) + */ +struct scx_cid_topo { + s32 core_cid; + s32 core_idx; + s32 llc_cid; + s32 llc_idx; + s32 node_cid; + s32 node_idx; +}; + #endif /* _KERNEL_SCHED_EXT_TYPES_H */ -- cgit v1.2.3 From df7b5ae038d6f2707ec0f81c257a55b4062f77c7 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 29 Apr 2026 08:09:10 -1000 Subject: sched_ext: Add scx_bpf_cid_override() kfunc The auto-probed cid mapping reflects the kernel's view of topology (node -> LLC -> core), but a BPF scheduler may want a different layout - to align cid slices with its own partitioning, or to work around how the kernel reports a particular machine. Add scx_bpf_cid_override(), callable from ops.init() of the root scheduler. It validates the caller-supplied cpu->cid array and replaces the in-place mapping; topo info is invalidated. A compat.bpf.h wrapper silently no-ops on kernels that lack the kfunc. A new SCX_KF_ALLOW_INIT bit in the kfunc context filter restricts the kfunc to ops.init() at verifier load time. Signed-off-by: Tejun Heo Reviewed-by: Cheng-Yang Chou Reviewed-by: Changwoo Min --- kernel/sched/ext.c | 16 +++++++---- kernel/sched/ext_cid.c | 75 +++++++++++++++++++++++++++++++++++++++++++++++++- kernel/sched/ext_cid.h | 1 + 3 files changed, 85 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index a83f8ce57781..e3a6de5efe02 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -9781,10 +9781,11 @@ static const struct btf_kfunc_id_set scx_kfunc_set_any = { */ enum scx_kf_allow_flags { SCX_KF_ALLOW_UNLOCKED = 1 << 0, - SCX_KF_ALLOW_CPU_RELEASE = 1 << 1, - SCX_KF_ALLOW_DISPATCH = 1 << 2, - SCX_KF_ALLOW_ENQUEUE = 1 << 3, - SCX_KF_ALLOW_SELECT_CPU = 1 << 4, + SCX_KF_ALLOW_INIT = 1 << 1, + SCX_KF_ALLOW_CPU_RELEASE = 1 << 2, + SCX_KF_ALLOW_DISPATCH = 1 << 3, + SCX_KF_ALLOW_ENQUEUE = 1 << 4, + SCX_KF_ALLOW_SELECT_CPU = 1 << 5, }; /* @@ -9812,7 +9813,7 @@ static const u32 scx_kf_allow_flags[] = { [SCX_OP_IDX(sub_detach)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(cpu_online)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(cpu_offline)] = SCX_KF_ALLOW_UNLOCKED, - [SCX_OP_IDX(init)] = SCX_KF_ALLOW_UNLOCKED, + [SCX_OP_IDX(init)] = SCX_KF_ALLOW_UNLOCKED | SCX_KF_ALLOW_INIT, [SCX_OP_IDX(exit)] = SCX_KF_ALLOW_UNLOCKED, }; @@ -9827,6 +9828,7 @@ static const u32 scx_kf_allow_flags[] = { int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) { bool in_unlocked = btf_id_set8_contains(&scx_kfunc_ids_unlocked, kfunc_id); + bool in_init = btf_id_set8_contains(&scx_kfunc_ids_init, kfunc_id); bool in_select_cpu = btf_id_set8_contains(&scx_kfunc_ids_select_cpu, kfunc_id); bool in_enqueue = btf_id_set8_contains(&scx_kfunc_ids_enqueue_dispatch, kfunc_id); bool in_dispatch = btf_id_set8_contains(&scx_kfunc_ids_dispatch, kfunc_id); @@ -9836,7 +9838,7 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) u32 moff, flags; /* Not an SCX kfunc - allow. */ - if (!(in_unlocked || in_select_cpu || in_enqueue || in_dispatch || + if (!(in_unlocked || in_init || in_select_cpu || in_enqueue || in_dispatch || in_cpu_release || in_idle || in_any)) return 0; @@ -9872,6 +9874,8 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) if ((flags & SCX_KF_ALLOW_UNLOCKED) && in_unlocked) return 0; + if ((flags & SCX_KF_ALLOW_INIT) && in_init) + return 0; if ((flags & SCX_KF_ALLOW_CPU_RELEASE) && in_cpu_release) return 0; if ((flags & SCX_KF_ALLOW_DISPATCH) && in_dispatch) diff --git a/kernel/sched/ext_cid.c b/kernel/sched/ext_cid.c index 5b73900edc87..607937d9e4d1 100644 --- a/kernel/sched/ext_cid.c +++ b/kernel/sched/ext_cid.c @@ -210,6 +210,68 @@ s32 scx_cid_init(struct scx_sched *sch) __bpf_kfunc_start_defs(); +/** + * scx_bpf_cid_override - Install an explicit cpu->cid mapping + * @cpu_to_cid: array of nr_cpu_ids s32 entries (cid for each cpu) + * @cpu_to_cid__sz: must be nr_cpu_ids * sizeof(s32) bytes + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * May only be called from ops.init() of the root scheduler. Replace the + * topology-probed cid mapping with the caller-provided one. Each possible cpu + * must map to a unique cid in [0, num_possible_cpus()). Topo info is cleared. + * On invalid input, trigger scx_error() to abort the scheduler. + */ +__bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid, u32 cpu_to_cid__sz, + const struct bpf_prog_aux *aux) +{ + cpumask_var_t seen __free(free_cpumask_var) = CPUMASK_VAR_NULL; + struct scx_sched *sch; + bool alloced; + s32 cpu, cid; + + /* GFP_KERNEL alloc must happen before the rcu read section */ + alloced = zalloc_cpumask_var(&seen, GFP_KERNEL); + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return; + + if (!alloced) { + scx_error(sch, "scx_bpf_cid_override: failed to allocate cpumask"); + return; + } + + if (scx_parent(sch)) { + scx_error(sch, "scx_bpf_cid_override() only allowed from root sched"); + return; + } + + if (cpu_to_cid__sz != nr_cpu_ids * sizeof(s32)) { + scx_error(sch, "scx_bpf_cid_override: expected %zu bytes, got %u", + nr_cpu_ids * sizeof(s32), cpu_to_cid__sz); + return; + } + + for_each_possible_cpu(cpu) { + s32 c = cpu_to_cid[cpu]; + + if (!cid_valid(sch, c)) + return; + if (cpumask_test_and_set_cpu(c, seen)) { + scx_error(sch, "cid %d assigned to multiple cpus", c); + return; + } + scx_cpu_to_cid_tbl[cpu] = c; + scx_cid_to_cpu_tbl[c] = cpu; + } + + /* Invalidate stale topo info - the override carries no topology. */ + for (cid = 0; cid < num_possible_cpus(); cid++) + scx_cid_topo[cid] = SCX_CID_TOPO_NEG; +} + /** * scx_bpf_cid_to_cpu - Return the raw CPU id for @cid * @cid: cid to look up @@ -282,6 +344,16 @@ __bpf_kfunc void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out__uninit, __bpf_kfunc_end_defs(); +BTF_KFUNCS_START(scx_kfunc_ids_init) +BTF_ID_FLAGS(func, scx_bpf_cid_override, KF_IMPLICIT_ARGS | KF_SLEEPABLE) +BTF_KFUNCS_END(scx_kfunc_ids_init) + +static const struct btf_kfunc_id_set scx_kfunc_set_init = { + .owner = THIS_MODULE, + .set = &scx_kfunc_ids_init, + .filter = scx_kfunc_context_filter, +}; + BTF_KFUNCS_START(scx_kfunc_ids_cid) BTF_ID_FLAGS(func, scx_bpf_cid_to_cpu, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_cpu_to_cid, KF_IMPLICIT_ARGS) @@ -295,7 +367,8 @@ static const struct btf_kfunc_id_set scx_kfunc_set_cid = { int scx_cid_kfunc_init(void) { - return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_cid) ?: + return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_init) ?: + register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_cid) ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &scx_kfunc_set_cid) ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_cid); } diff --git a/kernel/sched/ext_cid.h b/kernel/sched/ext_cid.h index 1dbe8262ccdd..52edb66b53fd 100644 --- a/kernel/sched/ext_cid.h +++ b/kernel/sched/ext_cid.h @@ -49,6 +49,7 @@ struct scx_sched; extern s16 *scx_cid_to_cpu_tbl; extern s16 *scx_cpu_to_cid_tbl; extern struct scx_cid_topo *scx_cid_topo; +extern struct btf_id_set8 scx_kfunc_ids_init; s32 scx_cid_init(struct scx_sched *sch); int scx_cid_kfunc_init(void); -- cgit v1.2.3 From a58e6b79b432f2f08e6a2bbe638a60e4424a7cba Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 29 Apr 2026 08:09:10 -1000 Subject: sched_ext: Add cmask, a base-windowed bitmap over cid space Sub-scheduler code built on cids needs bitmaps scoped to a slice of cid space (e.g. the idle cids of a shard). A cpumask sized for NR_CPUS wastes most of its bits for a small window and is awkward in BPF. scx_cmask covers [base, base + nr_bits). bits[] is aligned to the global 64-cid grid: bits[0] spans [base & ~63, (base & ~63) + 64). Any two cmasks therefore address bits[] against the same global windows, so cross-cmask word ops reduce to dest->bits[i] OP= operand->bits[i - delta] with no bit-shifting, at the cost of up to one extra storage word for head misalignment. This alignment guarantee is the reason binary ops can stay word-level; every mutating helper preserves it. Kernel side in ext_cid.[hc]; BPF side in tools/sched_ext/include/scx/ cid.bpf.h. BPF side drops the scx_ prefix (redundant in BPF code) and adds the extra helpers that basic idle-cpu selection needs. No callers yet. v2: Narrow to helpers that will be used in the planned changes; set/bit/find/zero ops will be added as usage develops. v3: cmask_copy_from_kernel: validate src->base == 0 via probe-read; bit-level nr_bits check instead of round-up word count. (Sashiko) v4: Bump CMASK_CAS_TRIES to 1<<23 so abort fires only after seconds of real spinning, not on plausible contention. Switch __builtin_ctzll() to the ctzll() wrapper for clang compat (Changwoo). Signed-off-by: Tejun Heo Reviewed-by: Cheng-Yang Chou Reviewed-by: Changwoo Min Reviewed-by: Andrea Righi --- kernel/sched/ext_cid.h | 25 +++++++++++++++++++++++++ kernel/sched/ext_types.h | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/ext_cid.h b/kernel/sched/ext_cid.h index 52edb66b53fd..c3c429d2c8e2 100644 --- a/kernel/sched/ext_cid.h +++ b/kernel/sched/ext_cid.h @@ -127,4 +127,29 @@ static inline s32 scx_cpu_to_cid(struct scx_sched *sch, s32 cpu) return __scx_cpu_to_cid(cpu); } +static inline bool __scx_cmask_contains(const struct scx_cmask *m, u32 cid) +{ + return likely(cid >= m->base && cid < m->base + m->nr_bits); +} + +/* Word in bits[] covering @cid. @cid must satisfy __scx_cmask_contains(). */ +static inline u64 *__scx_cmask_word(const struct scx_cmask *m, u32 cid) +{ + return (u64 *)&m->bits[cid / 64 - m->base / 64]; +} + +static inline void scx_cmask_init(struct scx_cmask *m, u32 base, u32 nr_bits) +{ + m->base = base; + m->nr_bits = nr_bits; + memset(m->bits, 0, SCX_CMASK_NR_WORDS(nr_bits) * sizeof(u64)); +} + +static inline void __scx_cmask_set(struct scx_cmask *m, u32 cid) +{ + if (!__scx_cmask_contains(m, cid)) + return; + *__scx_cmask_word(m, cid) |= BIT_U64(cid & 63); +} + #endif /* _KERNEL_SCHED_EXT_CID_H */ diff --git a/kernel/sched/ext_types.h b/kernel/sched/ext_types.h index be4d3565ae8d..ebb8cdf90612 100644 --- a/kernel/sched/ext_types.h +++ b/kernel/sched/ext_types.h @@ -63,4 +63,42 @@ struct scx_cid_topo { s32 node_idx; }; +/* + * cmask: variable-length, base-windowed bitmap over cid space + * ----------------------------------------------------------- + * + * A cmask covers the cid range [base, base + nr_bits). bits[] is aligned to the + * global 64-cid grid: bits[0] spans [base & ~63, (base & ~63) + 64), so the + * first (base & 63) bits of bits[0] are head padding and any tail past base + + * nr_bits is tail padding. Both must stay zero for the lifetime of the mask; + * all mutating helpers preserve that invariant. + * + * Grid alignment means two cmasks always address bits[] against the same global + * 64-cid windows, so cross-cmask word ops (AND, OR, ...) reduce to + * + * dst->bits[i] OP= src->bits[i - delta] + * + * with no bit-shifting, regardless of how the two bases relate mod 64. + */ +struct scx_cmask { + u32 base; + u32 nr_bits; + DECLARE_FLEX_ARRAY(u64, bits); +}; + +/* + * Number of u64 words of bits[] storage that covers @nr_bits regardless of base + * alignment. The +1 absorbs up to 63 bits of head padding when base is not + * 64-aligned - always allocating one extra word beats branching on base or + * splitting the compute. + */ +#define SCX_CMASK_NR_WORDS(nr_bits) (((nr_bits) + 63) / 64 + 1) + +/* + * Define an on-stack cmask for up to @cap_bits. @name is a struct scx_cmask * + * aliasing zero-initialized storage; call scx_cmask_init() to set base/nr_bits. + */ +#define SCX_CMASK_DEFINE(name, cap_bits) \ + DEFINE_RAW_FLEX(struct scx_cmask, name, bits, SCX_CMASK_NR_WORDS(cap_bits)) + #endif /* _KERNEL_SCHED_EXT_TYPES_H */ -- cgit v1.2.3 From 5ba0a42423335f76d3e0513df42416c69dc6b742 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 29 Apr 2026 08:09:11 -1000 Subject: sched_ext: Add cid-form kfunc wrappers alongside cpu-form cpumask is awkward from BPF and unusable from arena; cid/cmask work in both. Sub-sched enqueue will need cmask. Without full cid coverage a scheduler has to mix cid and cpu forms, which is a subtle-bug factory. Close the gap with a cid-native interface. Pair every cpu-form kfunc that takes a cpu id with a cid-form equivalent (kick, task placement, cpuperf query/set, per-cpu current task, nr-cpu-ids). Add two cid-natives with no cpu-form sibling: scx_bpf_this_cid() (cid of the running cpu, scx equivalent of bpf_get_smp_processor_id) and scx_bpf_nr_online_cids(). scx_bpf_cpu_rq is deprecated; no cid-form counterpart. NUMA node info is reachable via scx_bpf_cid_topo() on the BPF side. Each cid-form wrapper is a thin cid -> cpu translation that delegates to the cpu path, registered in the same context sets so usage constraints match. Signed-off-by: Tejun Heo Reviewed-by: Cheng-Yang Chou Reviewed-by: Changwoo Min Reviewed-by: Andrea Righi --- kernel/sched/ext.c | 184 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index e3a6de5efe02..12e43df08376 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -8890,6 +8890,28 @@ __bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux scx_kick_cpu(sch, cpu, flags); } +/** + * scx_bpf_kick_cid - Trigger reschedule on the CPU mapped to @cid + * @cid: cid to kick + * @flags: %SCX_KICK_* flags + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * cid-addressed equivalent of scx_bpf_kick_cpu(). + */ +__bpf_kfunc void scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + s32 cpu; + + guard(rcu)(); + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return; + cpu = scx_cid_to_cpu(sch, cid); + if (cpu >= 0) + scx_kick_cpu(sch, cpu, flags); +} + /** * scx_bpf_dsq_nr_queued - Return the number of queued tasks * @dsq_id: id of the DSQ @@ -9313,6 +9335,29 @@ __bpf_kfunc u32 scx_bpf_cpuperf_cap(s32 cpu, const struct bpf_prog_aux *aux) return SCX_CPUPERF_ONE; } +/** + * scx_bpf_cidperf_cap - Query the maximum relative capacity of the CPU at @cid + * @cid: cid of the CPU to query + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * cid-addressed equivalent of scx_bpf_cpuperf_cap(). + */ +__bpf_kfunc u32 scx_bpf_cidperf_cap(s32 cid, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + s32 cpu; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return SCX_CPUPERF_ONE; + cpu = scx_cid_to_cpu(sch, cid); + if (cpu < 0) + return SCX_CPUPERF_ONE; + return arch_scale_cpu_capacity(cpu); +} + /** * scx_bpf_cpuperf_cur - Query the current relative performance of a CPU * @cpu: CPU of interest @@ -9341,6 +9386,29 @@ __bpf_kfunc u32 scx_bpf_cpuperf_cur(s32 cpu, const struct bpf_prog_aux *aux) return SCX_CPUPERF_ONE; } +/** + * scx_bpf_cidperf_cur - Query the current performance of the CPU at @cid + * @cid: cid of the CPU to query + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * cid-addressed equivalent of scx_bpf_cpuperf_cur(). + */ +__bpf_kfunc u32 scx_bpf_cidperf_cur(s32 cid, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + s32 cpu; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return SCX_CPUPERF_ONE; + cpu = scx_cid_to_cpu(sch, cid); + if (cpu < 0) + return SCX_CPUPERF_ONE; + return arch_scale_freq_capacity(cpu); +} + /** * scx_bpf_cpuperf_set - Set the relative performance target of a CPU * @cpu: CPU of interest @@ -9401,6 +9469,31 @@ __bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf, const struct bpf_prog_au } } +/** + * scx_bpf_cidperf_set - Set the performance target of the CPU at @cid + * @cid: cid of the CPU to target + * @perf: target performance level [0, %SCX_CPUPERF_ONE] + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * cid-addressed equivalent of scx_bpf_cpuperf_set(). + */ +__bpf_kfunc void scx_bpf_cidperf_set(s32 cid, u32 perf, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + s32 cpu; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return; + cpu = scx_cid_to_cpu(sch, cid); + if (cpu < 0) + return; + scx_bpf_cpuperf_set(cpu, perf, aux); +} + /** * scx_bpf_nr_node_ids - Return the number of possible node IDs * @@ -9421,6 +9514,47 @@ __bpf_kfunc u32 scx_bpf_nr_cpu_ids(void) return nr_cpu_ids; } +/** + * scx_bpf_nr_cids - Return the size of the cid space + * + * Equals num_possible_cpus(). All valid cids are in [0, return value). + */ +__bpf_kfunc u32 scx_bpf_nr_cids(void) +{ + return num_possible_cpus(); +} + +/** + * scx_bpf_nr_online_cids - Return current count of online CPUs in cid space + * + * Return num_online_cpus(). The standard model restarts the scheduler on + * hotplug, which lets schedulers treat [0, nr_online_cids) as the online + * range. Schedulers that prefer to handle hotplug without a restart should + * install a custom mapping via scx_bpf_cid_override() and track onlining + * through the ops.cid_online / ops.cid_offline callbacks. + */ +__bpf_kfunc u32 scx_bpf_nr_online_cids(void) +{ + return num_online_cpus(); +} + +/** + * scx_bpf_this_cid - Return the cid of the CPU this program is running on + * + * cid-addressed equivalent of bpf_get_smp_processor_id() for scx programs. + * The current cpu is trivially valid, so this is just a table lookup. Return + * -EINVAL if called from a non-SCX program before any scheduler has ever + * been enabled (the cid table is still unallocated at that point). + */ +__bpf_kfunc s32 scx_bpf_this_cid(void) +{ + s16 *tbl = READ_ONCE(scx_cpu_to_cid_tbl); + + if (!tbl) + return -EINVAL; + return tbl[raw_smp_processor_id()]; +} + /** * scx_bpf_get_possible_cpumask - Get a referenced kptr to cpu_possible_mask */ @@ -9469,6 +9603,23 @@ __bpf_kfunc s32 scx_bpf_task_cpu(const struct task_struct *p) return task_cpu(p); } +/** + * scx_bpf_task_cid - cid a task is currently associated with + * @p: task of interest + * + * cid-addressed equivalent of scx_bpf_task_cpu(). task_cpu(p) is always a + * valid cpu, so this is just a table lookup. Return -EINVAL if called from + * a non-SCX program before any scheduler has ever been enabled. + */ +__bpf_kfunc s32 scx_bpf_task_cid(const struct task_struct *p) +{ + s16 *tbl = READ_ONCE(scx_cpu_to_cid_tbl); + + if (!tbl) + return -EINVAL; + return tbl[task_cpu(p)]; +} + /** * scx_bpf_cpu_rq - Fetch the rq of a CPU * @cpu: CPU of the rq @@ -9547,6 +9698,30 @@ __bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu, const struct bpf_prog_ return rcu_dereference(cpu_rq(cpu)->curr); } +/** + * scx_bpf_cid_curr - Return the curr task on the CPU at @cid + * @cid: cid of interest + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * cid-addressed equivalent of scx_bpf_cpu_curr(). Callers must hold RCU + * read lock (KF_RCU). + */ +__bpf_kfunc struct task_struct *scx_bpf_cid_curr(s32 cid, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + s32 cpu; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return NULL; + cpu = scx_cid_to_cpu(sch, cid); + if (cpu < 0) + return NULL; + return rcu_dereference(cpu_rq(cpu)->curr); +} + /** * scx_bpf_tid_to_task - Look up a task by its scx tid * @tid: task ID previously read from p->scx.tid @@ -9734,6 +9909,7 @@ BTF_KFUNCS_START(scx_kfunc_ids_any) BTF_ID_FLAGS(func, scx_bpf_task_set_slice, KF_IMPLICIT_ARGS | KF_RCU); BTF_ID_FLAGS(func, scx_bpf_task_set_dsq_vtime, KF_IMPLICIT_ARGS | KF_RCU); BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_kick_cid, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_destroy_dsq, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_dsq_peek, KF_IMPLICIT_ARGS | KF_RCU_PROTECTED | KF_RET_NULL) @@ -9748,16 +9924,24 @@ BTF_ID_FLAGS(func, scx_bpf_dump_bstr, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cidperf_cap, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cidperf_cur, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cidperf_set, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_nr_node_ids) BTF_ID_FLAGS(func, scx_bpf_nr_cpu_ids) +BTF_ID_FLAGS(func, scx_bpf_nr_cids) +BTF_ID_FLAGS(func, scx_bpf_nr_online_cids) +BTF_ID_FLAGS(func, scx_bpf_this_cid) BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE) BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE) BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE) BTF_ID_FLAGS(func, scx_bpf_task_running, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_task_cid, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_cpu_rq, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_locked_rq, KF_IMPLICIT_ARGS | KF_RET_NULL) BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) +BTF_ID_FLAGS(func, scx_bpf_cid_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) BTF_ID_FLAGS(func, scx_bpf_tid_to_task, KF_RET_NULL | KF_RCU_PROTECTED) BTF_ID_FLAGS(func, scx_bpf_now) BTF_ID_FLAGS(func, scx_bpf_events) -- cgit v1.2.3 From 7e655ed7b953d194265837978ca21bb17985d4aa Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 29 Apr 2026 08:09:11 -1000 Subject: sched_ext: Add bpf_sched_ext_ops_cid struct_ops type cpumask is awkward from BPF and unusable from arena; cid/cmask work in both. Sub-sched enqueue will need cmask. Without a full cid interface, schedulers end up mixing forms - a subtle-bug factory. Add sched_ext_ops_cid, which mirrors sched_ext_ops with cid/cmask replacing cpu/cpumask in the topology-carrying callbacks. cpu_acquire/cpu_release are deprecated and absent; a prior patch moved them past @priv so the cid-form can omit them without disturbing shared-field offsets. The two structs share byte-identical layout up to @priv, so the existing bpf_scx init/check hooks, has_op bitmap, and scx_kf_allow_flags[] are offset-indexed and apply to both. BUILD_BUG_ON in scx_init() pins the shared-field and renamed-callback offsets so any future drift trips at boot. The kernel<->BPF boundary translates between cpu and cid: - A static key, enabled on cid-form sched load, gates the translation so cpu-form schedulers pay nothing. - dispatch, update_idle, cpu_online/offline and dump_cpu translate the cpu arg at the callsite. - select_cpu also translates the returned cid back to a cpu. - set_cpumask is wrapped to synthesize a cmask in a per-cpu scratch before calling the cid-form callback. All scheds in a hierarchy share one form. The static key drives the hot-path branch. v2: Use struct_size() for the set_cmask_scratch percpu alloc. Move cid-shard fields and assertions into the later cid-shard patch. v3: Drop `static` on scx_set_cmask_scratch; add extern in ext_internal.h. Signed-off-by: Tejun Heo Reviewed-by: Cheng-Yang Chou Reviewed-by: Changwoo Min Reviewed-by: Andrea Righi --- kernel/sched/ext.c | 281 +++++++++++++++++++++++++++++++++++++++++--- kernel/sched/ext_cid.c | 37 +++++- kernel/sched/ext_cid.h | 9 ++ kernel/sched/ext_idle.c | 2 +- kernel/sched/ext_internal.h | 109 ++++++++++++++++- 5 files changed, 417 insertions(+), 21 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 12e43df08376..79565fabd9b4 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -513,6 +513,33 @@ do { \ update_locked_rq(__prev_locked_rq); \ } while (0) +/* + * Flipped on enable per sch->is_cid_type. Declared in ext_internal.h so + * subsystem inlines can read it. + */ +DEFINE_STATIC_KEY_FALSE(__scx_is_cid_type); + +/* + * scx_cpu_arg() wraps a cpu arg being handed to an SCX op. For cid-form + * schedulers it resolves to the matching cid; for cpu-form it passes @cpu + * through. scx_cpu_ret() is the inverse for a cpu/cid returned from an op + * (currently only ops.select_cpu); it validates the BPF-supplied cid and + * triggers scx_error() on @sch if invalid. + */ +static s32 scx_cpu_arg(s32 cpu) +{ + if (scx_is_cid_type()) + return __scx_cpu_to_cid(cpu); + return cpu; +} + +static s32 scx_cpu_ret(struct scx_sched *sch, s32 cpu_or_cid) +{ + if (cpu_or_cid < 0 || !scx_is_cid_type()) + return cpu_or_cid; + return scx_cid_to_cpu(sch, cpu_or_cid); +} + #define SCX_CALL_OP_RET(sch, op, locked_rq, args...) \ ({ \ struct rq *__prev_locked_rq; \ @@ -574,6 +601,41 @@ do { \ __ret; \ }) +/** + * scx_call_op_set_cpumask - invoke ops.set_cpumask / ops_cid.set_cmask for @task + * @sch: scx_sched being invoked + * @rq: rq to update as the currently-locked rq, or NULL + * @task: task whose affinity is changing + * @cpumask: new cpumask + * + * For cid-form schedulers, translate @cpumask to a cmask via the per-cpu + * scratch in ext_cid.c and dispatch through the ops_cid union view. Caller + * must hold @rq's rq lock so this_cpu_ptr is stable across the call. + */ +static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, + struct task_struct *task, + const struct cpumask *cpumask) +{ + WARN_ON_ONCE(current->scx.kf_tasks[0]); + current->scx.kf_tasks[0] = task; + if (rq) + update_locked_rq(rq); + + if (scx_is_cid_type()) { + struct scx_cmask *cmask = this_cpu_ptr(scx_set_cmask_scratch); + + lockdep_assert_irqs_disabled(); + scx_cpumask_to_cmask(cpumask, cmask); + sch->ops_cid.set_cmask(task, cmask); + } else { + sch->ops.set_cpumask(task, cpumask); + } + + if (rq) + update_locked_rq(NULL); + current->scx.kf_tasks[0] = NULL; +} + /* see SCX_CALL_OP_TASK() */ static __always_inline bool scx_kf_arg_task_ok(struct scx_sched *sch, struct task_struct *p) @@ -1679,7 +1741,7 @@ static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch, return &rq->scx.local_dsq; if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { - s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; + s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK); if (!scx_cpu_valid(sch, cpu, "in SCX_DSQ_LOCAL_ON dispatch verdict")) return find_global_dsq(sch, tcpu); @@ -2761,11 +2823,13 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, dspc->nr_tasks = 0; if (nested) { - SCX_CALL_OP(sch, dispatch, rq, cpu, prev_on_sch ? prev : NULL); + SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), + prev_on_sch ? prev : NULL); } else { /* stash @prev so that nested invocations can access it */ rq->scx.sub_dispatch_prev = prev; - SCX_CALL_OP(sch, dispatch, rq, cpu, prev_on_sch ? prev : NULL); + SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), + prev_on_sch ? prev : NULL); rq->scx.sub_dispatch_prev = NULL; } @@ -3260,7 +3324,9 @@ static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flag *ddsp_taskp = p; this_rq()->scx.in_select_cpu = true; - cpu = SCX_CALL_OP_TASK_RET(sch, select_cpu, NULL, p, prev_cpu, wake_flags); + cpu = SCX_CALL_OP_TASK_RET(sch, select_cpu, NULL, p, + scx_cpu_arg(prev_cpu), wake_flags); + cpu = scx_cpu_ret(sch, cpu); this_rq()->scx.in_select_cpu = false; p->scx.selected_cpu = cpu; *ddsp_taskp = NULL; @@ -3310,7 +3376,7 @@ static void set_cpus_allowed_scx(struct task_struct *p, * designation pointless. Cast it away when calling the operation. */ if (SCX_HAS_OP(sch, set_cpumask)) - SCX_CALL_OP_TASK(sch, set_cpumask, task_rq(p), p, (struct cpumask *)p->cpus_ptr); + scx_call_op_set_cpumask(sch, task_rq(p), p, (struct cpumask *)p->cpus_ptr); } static void handle_hotplug(struct rq *rq, bool online) @@ -3332,9 +3398,9 @@ static void handle_hotplug(struct rq *rq, bool online) scx_idle_update_selcpu_topology(&sch->ops); if (online && SCX_HAS_OP(sch, cpu_online)) - SCX_CALL_OP(sch, cpu_online, NULL, cpu); + SCX_CALL_OP(sch, cpu_online, NULL, scx_cpu_arg(cpu)); else if (!online && SCX_HAS_OP(sch, cpu_offline)) - SCX_CALL_OP(sch, cpu_offline, NULL, cpu); + SCX_CALL_OP(sch, cpu_offline, NULL, scx_cpu_arg(cpu)); else scx_exit(sch, SCX_EXIT_UNREG_KERN, SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG, @@ -3920,7 +3986,7 @@ static void switching_to_scx(struct rq *rq, struct task_struct *p) * different scheduler class. Keep the BPF scheduler up-to-date. */ if (SCX_HAS_OP(sch, set_cpumask)) - SCX_CALL_OP_TASK(sch, set_cpumask, rq, p, (struct cpumask *)p->cpus_ptr); + scx_call_op_set_cpumask(sch, rq, p, (struct cpumask *)p->cpus_ptr); } static void switched_from_scx(struct rq *rq, struct task_struct *p) @@ -5947,6 +6013,7 @@ static void scx_root_disable(struct scx_sched *sch) /* no task is on scx, turn off all the switches and flush in-progress calls */ static_branch_disable(&__scx_enabled); + static_branch_disable(&__scx_is_cid_type); if (sch->ops.flags & SCX_OPS_TID_TO_TASK) static_branch_disable(&__scx_tid_to_task_enabled); bitmap_zero(sch->has_op, SCX_OPI_END); @@ -6307,7 +6374,7 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, used = seq_buf_used(&ns); if (SCX_HAS_OP(sch, dump_cpu)) { ops_dump_init(&ns, " "); - SCX_CALL_OP(sch, dump_cpu, rq, dctx, cpu, idle); + SCX_CALL_OP(sch, dump_cpu, rq, dctx, scx_cpu_arg(cpu), idle); ops_dump_exit(); } @@ -6538,7 +6605,11 @@ static struct scx_sched_pnode *alloc_pnode(struct scx_sched *sch, int node) */ struct scx_enable_cmd { struct kthread_work work; - struct sched_ext_ops *ops; + union { + struct sched_ext_ops *ops; + struct sched_ext_ops_cid *ops_cid; + }; + bool is_cid_type; int ret; }; @@ -6546,10 +6617,11 @@ struct scx_enable_cmd { * Allocate and initialize a new scx_sched. @cgrp's reference is always * consumed whether the function succeeds or fails. */ -static struct scx_sched *scx_alloc_and_add_sched(struct sched_ext_ops *ops, +static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, struct cgroup *cgrp, struct scx_sched *parent) { + struct sched_ext_ops *ops = cmd->ops; struct scx_sched *sch; s32 level = parent ? parent->level + 1 : 0; s32 node, cpu, ret, bypass_fail_cpu = nr_cpu_ids; @@ -6641,7 +6713,18 @@ static struct scx_sched *scx_alloc_and_add_sched(struct sched_ext_ops *ops, ret = -ENOMEM; goto err_free_lb_cpumask; } - sch->ops = *ops; + /* + * Copy ops through the right union view. For cid-form the source is + * struct sched_ext_ops_cid which lacks the trailing cpu_acquire/ + * cpu_release; those stay zero from kzalloc. + */ + if (cmd->is_cid_type) { + sch->ops_cid = *cmd->ops_cid; + sch->is_cid_type = true; + } else { + sch->ops = *cmd->ops; + } + rcu_assign_pointer(ops->priv, sch); sch->kobj.kset = scx_kset; @@ -6778,7 +6861,12 @@ static int validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) return -EINVAL; } - if (ops->cpu_acquire || ops->cpu_release) + /* + * cid-form's struct is shorter and doesn't include the cpu_acquire / + * cpu_release tail; reading those fields off a cid-form @ops would + * run past the BPF allocation. Skip for cid-form. + */ + if (!sch->is_cid_type && (ops->cpu_acquire || ops->cpu_release)) pr_warn("ops->cpu_acquire/release() are deprecated, use sched_switch TP instead\n"); return 0; @@ -6814,12 +6902,15 @@ static void scx_root_enable_workfn(struct kthread_work *work) #ifdef CONFIG_EXT_SUB_SCHED cgroup_get(cgrp); #endif - sch = scx_alloc_and_add_sched(ops, cgrp, NULL); + sch = scx_alloc_and_add_sched(cmd, cgrp, NULL); if (IS_ERR(sch)) { ret = PTR_ERR(sch); goto err_free_tid_hash; } + if (sch->is_cid_type) + static_branch_enable(&__scx_is_cid_type); + /* * Transition to ENABLING and clear exit info to arm the disable path. * Failure triggers full disabling from here on. @@ -7141,7 +7232,7 @@ static void scx_sub_enable_workfn(struct kthread_work *work) raw_spin_unlock_irq(&scx_sched_lock); /* scx_alloc_and_add_sched() consumes @cgrp whether it succeeds or not */ - sch = scx_alloc_and_add_sched(ops, cgrp, parent); + sch = scx_alloc_and_add_sched(cmd, cgrp, parent); kobject_put(&parent->kobj); if (IS_ERR(sch)) { ret = PTR_ERR(sch); @@ -7592,6 +7683,13 @@ static int bpf_scx_reg(void *kdata, struct bpf_link *link) return scx_enable(&cmd, link); } +static int bpf_scx_reg_cid(void *kdata, struct bpf_link *link) +{ + struct scx_enable_cmd cmd = { .ops_cid = kdata, .is_cid_type = true }; + + return scx_enable(&cmd, link); +} + static void bpf_scx_unreg(void *kdata, struct bpf_link *link) { struct sched_ext_ops *ops = kdata; @@ -7723,6 +7821,73 @@ static struct bpf_struct_ops bpf_sched_ext_ops = { .cfi_stubs = &__bpf_ops_sched_ext_ops }; +/* + * cid-form cfi stubs. Stubs whose signatures match the cpu-form (param types + * identical, only param names differ across structs) are reused; only + * set_cmask needs a fresh stub since the second argument type differs. + */ +static void sched_ext_ops_cid__set_cmask(struct task_struct *p, + const struct scx_cmask *cmask) {} + +static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { + .select_cid = sched_ext_ops__select_cpu, + .enqueue = sched_ext_ops__enqueue, + .dequeue = sched_ext_ops__dequeue, + .dispatch = sched_ext_ops__dispatch, + .tick = sched_ext_ops__tick, + .runnable = sched_ext_ops__runnable, + .running = sched_ext_ops__running, + .stopping = sched_ext_ops__stopping, + .quiescent = sched_ext_ops__quiescent, + .yield = sched_ext_ops__yield, + .core_sched_before = sched_ext_ops__core_sched_before, + .set_weight = sched_ext_ops__set_weight, + .set_cmask = sched_ext_ops_cid__set_cmask, + .update_idle = sched_ext_ops__update_idle, + .init_task = sched_ext_ops__init_task, + .exit_task = sched_ext_ops__exit_task, + .enable = sched_ext_ops__enable, + .disable = sched_ext_ops__disable, +#ifdef CONFIG_EXT_GROUP_SCHED + .cgroup_init = sched_ext_ops__cgroup_init, + .cgroup_exit = sched_ext_ops__cgroup_exit, + .cgroup_prep_move = sched_ext_ops__cgroup_prep_move, + .cgroup_move = sched_ext_ops__cgroup_move, + .cgroup_cancel_move = sched_ext_ops__cgroup_cancel_move, + .cgroup_set_weight = sched_ext_ops__cgroup_set_weight, + .cgroup_set_bandwidth = sched_ext_ops__cgroup_set_bandwidth, + .cgroup_set_idle = sched_ext_ops__cgroup_set_idle, +#endif + .sub_attach = sched_ext_ops__sub_attach, + .sub_detach = sched_ext_ops__sub_detach, + .cid_online = sched_ext_ops__cpu_online, + .cid_offline = sched_ext_ops__cpu_offline, + .init = sched_ext_ops__init, + .exit = sched_ext_ops__exit, + .dump = sched_ext_ops__dump, + .dump_cid = sched_ext_ops__dump_cpu, + .dump_task = sched_ext_ops__dump_task, +}; + +/* + * The cid-form struct_ops shares all bpf_struct_ops hooks with the cpu form. + * init_member, check_member, reg, unreg, etc. process kdata as the byte block + * verified to match by the BUILD_BUG_ON checks in scx_init(). + */ +static struct bpf_struct_ops bpf_sched_ext_ops_cid = { + .verifier_ops = &bpf_scx_verifier_ops, + .reg = bpf_scx_reg_cid, + .unreg = bpf_scx_unreg, + .check_member = bpf_scx_check_member, + .init_member = bpf_scx_init_member, + .init = bpf_scx_init, + .update = bpf_scx_update, + .validate = bpf_scx_validate, + .name = "sched_ext_ops_cid", + .owner = THIS_MODULE, + .cfi_stubs = &__bpf_ops_sched_ext_ops_cid +}; + /******************************************************************************** * System integration and init. @@ -8938,7 +9103,7 @@ __bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id, const struct bpf_prog_aux *aux ret = READ_ONCE(this_rq()->scx.local_dsq.nr); goto out; } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { - s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; + s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK); if (scx_cpu_valid(sch, cpu, NULL)) { ret = READ_ONCE(cpu_rq(cpu)->scx.local_dsq.nr); @@ -10045,8 +10210,15 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) /* * Non-SCX struct_ops: SCX kfuncs are not permitted. - */ - if (prog->aux->st_ops != &bpf_sched_ext_ops) + * + * Both bpf_sched_ext_ops (cpu-form) and bpf_sched_ext_ops_cid + * (cid-form) are valid SCX struct_ops. Member offsets match between + * the two (verified by BUILD_BUG_ON in scx_init()), so the shared + * scx_kf_allow_flags[] table indexed by SCX_MOFF_IDX(moff) applies to + * both. + */ + if (prog->aux->st_ops != &bpf_sched_ext_ops && + prog->aux->st_ops != &bpf_sched_ext_ops_cid) return -EACCES; /* SCX struct_ops: check the per-op allow list. */ @@ -10076,6 +10248,73 @@ static int __init scx_init(void) { int ret; + /* + * sched_ext_ops_cid mirrors sched_ext_ops up to and including @priv. + * Both bpf_scx_init_member() and bpf_scx_check_member() use offsets + * from struct sched_ext_ops; sched_ext_ops_cid relies on those offsets + * matching for the shared fields. Catch any drift at boot. + */ +#define CID_OFFSET_MATCH(cpu_field, cid_field) \ + BUILD_BUG_ON(offsetof(struct sched_ext_ops, cpu_field) != \ + offsetof(struct sched_ext_ops_cid, cid_field)) + /* data fields used by bpf_scx_init_member() */ + CID_OFFSET_MATCH(dispatch_max_batch, dispatch_max_batch); + CID_OFFSET_MATCH(flags, flags); + CID_OFFSET_MATCH(name, name); + CID_OFFSET_MATCH(timeout_ms, timeout_ms); + CID_OFFSET_MATCH(exit_dump_len, exit_dump_len); + CID_OFFSET_MATCH(hotplug_seq, hotplug_seq); + CID_OFFSET_MATCH(sub_cgroup_id, sub_cgroup_id); + /* shared callbacks: the union view requires byte-for-byte offset match */ + CID_OFFSET_MATCH(enqueue, enqueue); + CID_OFFSET_MATCH(dequeue, dequeue); + CID_OFFSET_MATCH(dispatch, dispatch); + CID_OFFSET_MATCH(tick, tick); + CID_OFFSET_MATCH(runnable, runnable); + CID_OFFSET_MATCH(running, running); + CID_OFFSET_MATCH(stopping, stopping); + CID_OFFSET_MATCH(quiescent, quiescent); + CID_OFFSET_MATCH(yield, yield); + CID_OFFSET_MATCH(core_sched_before, core_sched_before); + CID_OFFSET_MATCH(set_weight, set_weight); + CID_OFFSET_MATCH(update_idle, update_idle); + CID_OFFSET_MATCH(init_task, init_task); + CID_OFFSET_MATCH(exit_task, exit_task); + CID_OFFSET_MATCH(enable, enable); + CID_OFFSET_MATCH(disable, disable); + CID_OFFSET_MATCH(dump, dump); + CID_OFFSET_MATCH(dump_task, dump_task); + CID_OFFSET_MATCH(sub_attach, sub_attach); + CID_OFFSET_MATCH(sub_detach, sub_detach); + CID_OFFSET_MATCH(init, init); + CID_OFFSET_MATCH(exit, exit); +#ifdef CONFIG_EXT_GROUP_SCHED + CID_OFFSET_MATCH(cgroup_init, cgroup_init); + CID_OFFSET_MATCH(cgroup_exit, cgroup_exit); + CID_OFFSET_MATCH(cgroup_prep_move, cgroup_prep_move); + CID_OFFSET_MATCH(cgroup_move, cgroup_move); + CID_OFFSET_MATCH(cgroup_cancel_move, cgroup_cancel_move); + CID_OFFSET_MATCH(cgroup_set_weight, cgroup_set_weight); + CID_OFFSET_MATCH(cgroup_set_bandwidth, cgroup_set_bandwidth); + CID_OFFSET_MATCH(cgroup_set_idle, cgroup_set_idle); +#endif + /* renamed callbacks must occupy the same slot as their cpu-form sibling */ + CID_OFFSET_MATCH(select_cpu, select_cid); + CID_OFFSET_MATCH(set_cpumask, set_cmask); + CID_OFFSET_MATCH(cpu_online, cid_online); + CID_OFFSET_MATCH(cpu_offline, cid_offline); + CID_OFFSET_MATCH(dump_cpu, dump_cid); + /* @priv tail must align since both share the same data block */ + CID_OFFSET_MATCH(priv, priv); + /* + * cid-form must end exactly at @priv - validate_ops() skips + * cpu_acquire/cpu_release for cid-form because reading those fields + * past the BPF allocation would be UB. + */ + BUILD_BUG_ON(sizeof(struct sched_ext_ops_cid) != + offsetofend(struct sched_ext_ops, priv)); +#undef CID_OFFSET_MATCH + /* * kfunc registration can't be done from init_sched_ext_class() as * register_btf_kfunc_id_set() needs most of the system to be up. @@ -10126,6 +10365,12 @@ static int __init scx_init(void) return ret; } + ret = register_bpf_struct_ops(&bpf_sched_ext_ops_cid, sched_ext_ops_cid); + if (ret) { + pr_err("sched_ext: Failed to register cid struct_ops (%d)\n", ret); + return ret; + } + ret = register_pm_notifier(&scx_pm_notifier); if (ret) { pr_err("sched_ext: Failed to register PM notifier (%d)\n", ret); diff --git a/kernel/sched/ext_cid.c b/kernel/sched/ext_cid.c index 607937d9e4d1..bdd8ef8eae3d 100644 --- a/kernel/sched/ext_cid.c +++ b/kernel/sched/ext_cid.c @@ -7,6 +7,14 @@ */ #include +/* + * Per-cpu scratch cmask used by scx_call_op_set_cpumask() to synthesize a + * cmask from a cpumask. Allocated alongside the cid arrays on first enable + * and never freed. Sized to the full cid space. Caller holds rq lock so + * this_cpu_ptr is safe. + */ +struct scx_cmask __percpu *scx_set_cmask_scratch; + /* * cid tables. * @@ -46,6 +54,7 @@ static s32 scx_cid_arrays_alloc(void) u32 npossible = num_possible_cpus(); s16 *cid_to_cpu, *cpu_to_cid; struct scx_cid_topo *cid_topo; + struct scx_cmask __percpu *set_cmask_scratch; if (scx_cid_to_cpu_tbl) return 0; @@ -53,17 +62,22 @@ static s32 scx_cid_arrays_alloc(void) cid_to_cpu = kzalloc_objs(*scx_cid_to_cpu_tbl, npossible, GFP_KERNEL); cpu_to_cid = kzalloc_objs(*scx_cpu_to_cid_tbl, nr_cpu_ids, GFP_KERNEL); cid_topo = kmalloc_objs(*scx_cid_topo, npossible, GFP_KERNEL); + set_cmask_scratch = __alloc_percpu(struct_size(set_cmask_scratch, bits, + SCX_CMASK_NR_WORDS(npossible)), + sizeof(u64)); - if (!cid_to_cpu || !cpu_to_cid || !cid_topo) { + if (!cid_to_cpu || !cpu_to_cid || !cid_topo || !set_cmask_scratch) { kfree(cid_to_cpu); kfree(cpu_to_cid); kfree(cid_topo); + free_percpu(set_cmask_scratch); return -ENOMEM; } WRITE_ONCE(scx_cid_to_cpu_tbl, cid_to_cpu); WRITE_ONCE(scx_cpu_to_cid_tbl, cpu_to_cid); WRITE_ONCE(scx_cid_topo, cid_topo); + WRITE_ONCE(scx_set_cmask_scratch, set_cmask_scratch); return 0; } @@ -208,6 +222,27 @@ s32 scx_cid_init(struct scx_sched *sch) return 0; } +/** + * scx_cpumask_to_cmask - Translate a kernel cpumask into a cmask + * @src: source cpumask + * @dst: cmask to write + * + * Initialize @dst to cover the full cid space [0, num_possible_cpus()) and + * set the bit for each cid whose cpu is in @src. + */ +void scx_cpumask_to_cmask(const struct cpumask *src, struct scx_cmask *dst) +{ + s32 cpu; + + scx_cmask_init(dst, 0, num_possible_cpus()); + for_each_cpu(cpu, src) { + s32 cid = __scx_cpu_to_cid(cpu); + + if (cid >= 0) + __scx_cmask_set(dst, cid); + } +} + __bpf_kfunc_start_defs(); /** diff --git a/kernel/sched/ext_cid.h b/kernel/sched/ext_cid.h index c3c429d2c8e2..f41d48afb7d1 100644 --- a/kernel/sched/ext_cid.h +++ b/kernel/sched/ext_cid.h @@ -53,6 +53,7 @@ extern struct btf_id_set8 scx_kfunc_ids_init; s32 scx_cid_init(struct scx_sched *sch); int scx_cid_kfunc_init(void); +void scx_cpumask_to_cmask(const struct cpumask *src, struct scx_cmask *dst); /** * cid_valid - Verify a cid value, to be used on ops input args @@ -127,6 +128,14 @@ static inline s32 scx_cpu_to_cid(struct scx_sched *sch, s32 cpu) return __scx_cpu_to_cid(cpu); } +/** + * scx_is_cid_type - Test whether the active scheduler hierarchy is cid-form + */ +static inline bool scx_is_cid_type(void) +{ + return static_branch_unlikely(&__scx_is_cid_type); +} + static inline bool __scx_cmask_contains(const struct scx_cmask *m, u32 cid) { return likely(cid >= m->base && cid < m->base + m->nr_bits); diff --git a/kernel/sched/ext_idle.c b/kernel/sched/ext_idle.c index 860c4634f60e..41785f65bbb2 100644 --- a/kernel/sched/ext_idle.c +++ b/kernel/sched/ext_idle.c @@ -788,7 +788,7 @@ void __scx_update_idle(struct rq *rq, bool idle, bool do_notify) */ if (SCX_HAS_OP(sch, update_idle) && do_notify && !scx_bypassing(sch, cpu_of(rq))) - SCX_CALL_OP(sch, update_idle, rq, cpu_of(rq), idle); + SCX_CALL_OP(sch, update_idle, rq, scx_cpu_arg(cpu_of(rq)), idle); } static void reset_idle_masks(struct sched_ext_ops *ops) diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext_internal.h index c6de974eaf48..b4f5dd28855e 100644 --- a/kernel/sched/ext_internal.h +++ b/kernel/sched/ext_internal.h @@ -853,6 +853,93 @@ struct sched_ext_ops { void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); }; +/** + * struct sched_ext_ops_cid - cid-form alternative to struct sched_ext_ops + * + * Mirrors struct sched_ext_ops with cpu/cpumask substituted with cid/cmask + * where applicable. Layout up to and including @priv matches sched_ext_ops + * byte-for-byte (verified by BUILD_BUG_ON checks at scx_init() time) so + * shared field offsets work for both struct types in bpf_scx_init_member() + * and bpf_scx_check_member(). The deprecated cpu_acquire/cpu_release + * callbacks at the tail of sched_ext_ops are omitted here entirely. + * + * Differences from sched_ext_ops: + * - select_cpu -> select_cid (returns cid) + * - dispatch -> dispatch (cpu arg is now cid) + * - update_idle -> update_idle (cpu arg is now cid) + * - set_cpumask -> set_cmask (cmask instead of cpumask) + * - cpu_online -> cid_online + * - cpu_offline -> cid_offline + * - dump_cpu -> dump_cid + * - cpu_acquire/cpu_release -> not present (deprecated in sched_ext_ops) + * + * BPF schedulers using this type cannot call cpu-form scx_bpf_* kfuncs; + * use the cid-form variants instead. Enforced at BPF verifier time via + * scx_kfunc_context_filter() branching on prog->aux->st_ops. + * + * See sched_ext_ops for callback documentation. + */ +struct sched_ext_ops_cid { + s32 (*select_cid)(struct task_struct *p, s32 prev_cid, u64 wake_flags); + void (*enqueue)(struct task_struct *p, u64 enq_flags); + void (*dequeue)(struct task_struct *p, u64 deq_flags); + void (*dispatch)(s32 cid, struct task_struct *prev); + void (*tick)(struct task_struct *p); + void (*runnable)(struct task_struct *p, u64 enq_flags); + void (*running)(struct task_struct *p); + void (*stopping)(struct task_struct *p, bool runnable); + void (*quiescent)(struct task_struct *p, u64 deq_flags); + bool (*yield)(struct task_struct *from, struct task_struct *to); + bool (*core_sched_before)(struct task_struct *a, + struct task_struct *b); + void (*set_weight)(struct task_struct *p, u32 weight); + void (*set_cmask)(struct task_struct *p, + const struct scx_cmask *cmask); + void (*update_idle)(s32 cid, bool idle); + s32 (*init_task)(struct task_struct *p, + struct scx_init_task_args *args); + void (*exit_task)(struct task_struct *p, + struct scx_exit_task_args *args); + void (*enable)(struct task_struct *p); + void (*disable)(struct task_struct *p); + void (*dump)(struct scx_dump_ctx *ctx); + void (*dump_cid)(struct scx_dump_ctx *ctx, s32 cid, bool idle); + void (*dump_task)(struct scx_dump_ctx *ctx, struct task_struct *p); +#ifdef CONFIG_EXT_GROUP_SCHED + s32 (*cgroup_init)(struct cgroup *cgrp, + struct scx_cgroup_init_args *args); + void (*cgroup_exit)(struct cgroup *cgrp); + s32 (*cgroup_prep_move)(struct task_struct *p, + struct cgroup *from, struct cgroup *to); + void (*cgroup_move)(struct task_struct *p, + struct cgroup *from, struct cgroup *to); + void (*cgroup_cancel_move)(struct task_struct *p, + struct cgroup *from, struct cgroup *to); + void (*cgroup_set_weight)(struct cgroup *cgrp, u32 weight); + void (*cgroup_set_bandwidth)(struct cgroup *cgrp, + u64 period_us, u64 quota_us, u64 burst_us); + void (*cgroup_set_idle)(struct cgroup *cgrp, bool idle); +#endif /* CONFIG_EXT_GROUP_SCHED */ + s32 (*sub_attach)(struct scx_sub_attach_args *args); + void (*sub_detach)(struct scx_sub_detach_args *args); + void (*cid_online)(s32 cid); + void (*cid_offline)(s32 cid); + s32 (*init)(void); + void (*exit)(struct scx_exit_info *info); + + /* Data fields - must match sched_ext_ops layout exactly */ + u32 dispatch_max_batch; + u64 flags; + u32 timeout_ms; + u32 exit_dump_len; + u64 hotplug_seq; + u64 sub_cgroup_id; + char name[SCX_OPS_NAME_LEN]; + + /* internal use only, must be NULL */ + void __rcu *priv; +}; + enum scx_opi { SCX_OPI_BEGIN = 0, SCX_OPI_NORMAL_BEGIN = 0, @@ -1009,7 +1096,18 @@ struct scx_sched_pnode { }; struct scx_sched { - struct sched_ext_ops ops; + /* + * cpu-form and cid-form ops share field offsets up to .priv (verified + * by BUILD_BUG_ON in scx_init()). The anonymous union lets the kernel + * access either view of the same storage without function-pointer + * casts: use .ops for cpu-form and shared fields, .ops_cid for the + * cid-renamed callbacks (set_cmask, select_cid, cid_online, ...). + */ + union { + struct sched_ext_ops ops; + struct sched_ext_ops_cid ops_cid; + }; + bool is_cid_type; /* true if registered via bpf_sched_ext_ops_cid */ DECLARE_BITMAP(has_op, SCX_OPI_END); /* @@ -1366,6 +1464,15 @@ enum scx_ops_state { extern struct scx_sched __rcu *scx_root; DECLARE_PER_CPU(struct rq *, scx_locked_rq_state); +extern struct scx_cmask __percpu *scx_set_cmask_scratch; + +/* + * True when the currently loaded scheduler hierarchy is cid-form. All scheds + * in a hierarchy share one form, so this single key tells callsites which + * view to use without per-sch dereferences. Use scx_is_cid_type() to test. + */ +DECLARE_STATIC_KEY_FALSE(__scx_is_cid_type); + int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id); bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where); -- cgit v1.2.3 From 67edfa9eefe693bcc48e615241a89240006551ce Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 29 Apr 2026 08:09:11 -1000 Subject: sched_ext: Forbid cpu-form kfuncs from cid-form schedulers cid and cpu are both small s32s, trivially confused when a cid-form scheduler calls a cpu-keyed kfunc. Reject cid-form programs that reference any kfunc in the new scx_kfunc_ids_cpu_only at verifier load time. The reverse direction is intentionally permissive: cpu-form schedulers can freely call cid-form kfuncs to ease a gradual cpumask -> cid migration. The check sits in scx_kfunc_context_filter() right after the SCX struct_ops gate and before the any/idle allow and per-op allow-list checks, so it catches cpu-only kfuncs regardless of which set they belong to (any, idle, or select_cpu). v2: Sync per-entry kfunc flags with their primary declarations (Zhao). pahole intersects flags across BTF_ID_FLAGS() occurrences, so omitting them drops the flags globally. Signed-off-by: Tejun Heo Reviewed-by: Cheng-Yang Chou Reviewed-by: Zhao Mengmeng Reviewed-by: Changwoo Min Reviewed-by: Andrea Righi --- kernel/sched/ext.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 79565fabd9b4..65a00c979691 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -10121,6 +10121,47 @@ static const struct btf_kfunc_id_set scx_kfunc_set_any = { .filter = scx_kfunc_context_filter, }; +/* + * cpu-form kfuncs that are forbidden from cid-form schedulers + * (bpf_sched_ext_ops_cid). Programs targeting the cid struct_ops type must + * use the cid-form alternative (cid/cmask kfuncs). + * + * Membership overlaps with scx_kfunc_ids_{any,idle,select_cpu}; the filter + * tests this set independently and rejects matches before the per-op + * allow-list check runs. + * + * pahole/resolve_btfids scans every BTF_ID_FLAGS() at build time and + * intersects flags across duplicate entries, so each entry must carry the + * same flags as the kfunc's primary declaration; otherwise the flags get + * dropped globally. + */ +BTF_KFUNCS_START(scx_kfunc_ids_cpu_only) +BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_cpu_rq, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) +BTF_ID_FLAGS(func, scx_bpf_cpu_node, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE) +BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_IMPLICIT_ARGS | KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_IMPLICIT_ARGS | KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) +BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu_node, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu_node, KF_IMPLICIT_ARGS | KF_RCU) +BTF_KFUNCS_END(scx_kfunc_ids_cpu_only) + /* * Per-op kfunc allow flags. Each bit corresponds to a context-sensitive kfunc * group; an op may permit zero or more groups, with the union expressed in @@ -10184,6 +10225,7 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) bool in_cpu_release = btf_id_set8_contains(&scx_kfunc_ids_cpu_release, kfunc_id); bool in_idle = btf_id_set8_contains(&scx_kfunc_ids_idle, kfunc_id); bool in_any = btf_id_set8_contains(&scx_kfunc_ids_any, kfunc_id); + bool in_cpu_only = btf_id_set8_contains(&scx_kfunc_ids_cpu_only, kfunc_id); u32 moff, flags; /* Not an SCX kfunc - allow. */ @@ -10221,6 +10263,15 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) prog->aux->st_ops != &bpf_sched_ext_ops_cid) return -EACCES; + /* + * cid-form schedulers must use cid/cmask kfuncs. cid and cpu are both + * small s32s and trivially confused, so cpu-only kfuncs are rejected at + * load time. The reverse (cpu-form calling cid-form kfuncs) is + * intentionally permissive to ease gradual cpumask -> cid migration. + */ + if (prog->aux->st_ops == &bpf_sched_ext_ops_cid && in_cpu_only) + return -EACCES; + /* SCX struct_ops: check the per-op allow list. */ if (in_any || in_idle) return 0; -- cgit v1.2.3 From 97f86c38abe62c911ff20bc3e00b0937842f79c0 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 29 Apr 2026 08:09:11 -1000 Subject: sched_ext: Require cid-form struct_ops for sub-sched support Sub-scheduler support is tied to the cid-form struct_ops: sub_attach / sub_detach will communicate allocation via cmask, and the hierarchy assumes all participants share a single topological cid space. A cpu-form root that accepts sub-scheds would need cpu <-> cid translation on every cross-sched interaction, defeating the purpose. Enforce this at validate_ops(): - A sub-scheduler (scx_parent(sch) non-NULL) must be cid-form. - A root that exposes sub_attach / sub_detach must be cid-form. scx_qmap, which is currently the only scheduler demoing sub-sched support, was converted to cid-form in the preceding patch, so this doesn't cause breakage. Signed-off-by: Tejun Heo Reviewed-by: Cheng-Yang Chou Reviewed-by: Changwoo Min Reviewed-by: Andrea Righi --- kernel/sched/ext.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 65a00c979691..f3a585e32db3 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -6869,6 +6869,23 @@ static int validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) if (!sch->is_cid_type && (ops->cpu_acquire || ops->cpu_release)) pr_warn("ops->cpu_acquire/release() are deprecated, use sched_switch TP instead\n"); + /* + * Sub-scheduler support is tied to the cid-form struct_ops. A sub-sched + * attaches through a cid-form-only interface (sub_attach/sub_detach), + * and a root that accepts sub-scheds must expose cid-form state to + * them. Reject cpu-form schedulers on either side. + */ + if (!sch->is_cid_type) { + if (scx_parent(sch)) { + scx_error(sch, "sub-sched requires cid-form struct_ops"); + return -EINVAL; + } + if (ops->sub_attach || ops->sub_detach) { + scx_error(sch, "sub_attach/sub_detach requires cid-form struct_ops"); + return -EINVAL; + } + } + return 0; } -- cgit v1.2.3 From 22d0213e55fbb723c2c00dd5aa855a6eaad95b23 Mon Sep 17 00:00:00 2001 From: Petr Tesarik Date: Fri, 10 Apr 2026 13:35:06 +0200 Subject: dma-direct: fix use of max_pfn Calculate the correct physical address of the last byte of memory. Since max_pfn is in fact "the PFN of the first page after the highest system RAM in physical address space", the highest address that might be used for a DMA buffer is one byte below max_pfn << PAGE_SHIFT. This fix is unlikely to make any difference in practice. It's just that the current formula is slightly confusing. Signed-off-by: Petr Tesarik Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/20260410113506.262579-1-ptesarik@suse.com --- kernel/dma/direct.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index ec887f443741..583c5922bca2 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -39,7 +39,7 @@ static inline struct page *dma_direct_to_page(struct device *dev, u64 dma_direct_get_required_mask(struct device *dev) { - phys_addr_t phys = (phys_addr_t)(max_pfn - 1) << PAGE_SHIFT; + phys_addr_t phys = ((phys_addr_t)max_pfn << PAGE_SHIFT) - 1; u64 max_dma = phys_to_dma_direct(dev, phys); return (1ULL << (fls64(max_dma) - 1)) * 2 - 1; @@ -553,7 +553,7 @@ int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma, int dma_direct_supported(struct device *dev, u64 mask) { - u64 min_mask = (max_pfn - 1) << PAGE_SHIFT; + u64 min_mask = ((u64)max_pfn << PAGE_SHIFT) - 1; /* * Because 32-bit DMA masks are so common we expect every architecture -- cgit v1.2.3 From f603e84ab7918db6470c0b06b46ece7fbdb71e9a Mon Sep 17 00:00:00 2001 From: Paul Chaignon Date: Thu, 30 Apr 2026 10:44:28 +0200 Subject: bpf: Print breakdown of insns processed by subprogs When using global functions (i.e. subprogs), the verifier performs function-by-function verification. In that case, the sum of the instructions processed in each global function and in the main program counts towards the 1 million instructions limit. Only that sum is reported in the verifier logs. While starting to use global functions in Cilium (finally!), we found it can be useful to have the breakdown per global function, to understand exactly where the budget is currently spent. This patch implements this breakdown, under BPF_LOG_STATS, as done for the stack depths. When iterating over subprogs, we need to skip the hidden subprogs at the end because they don't have a corresponding func_info_aux entry and calling bpf_subprog_is_global() would result in an OOB access. Signed-off-by: Paul Chaignon Link: https://lore.kernel.org/bpf/5590f9c67e614ec9054d0c7e74e87cc690a52c56.1777538384.git.paul.chaignon@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 03f9e16c2abe..11054ad89c14 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -18215,6 +18215,7 @@ static int do_check_subprogs(struct bpf_verifier_env *env) struct bpf_prog_aux *aux = env->prog->aux; struct bpf_func_info_aux *sub_aux; int i, ret, new_cnt; + u32 insn_processed; if (!aux->func_info) return 0; @@ -18229,6 +18230,8 @@ again: if (!bpf_subprog_is_global(env, i)) continue; + insn_processed = env->insn_processed; + sub_aux = subprog_aux(env, i); if (!sub_aux->called || sub_aux->verified) continue; @@ -18236,6 +18239,7 @@ again: env->insn_idx = env->subprog_info[i].start; WARN_ON_ONCE(env->insn_idx == 0); ret = do_check_common(env, i); + env->subprog_info[i].insn_processed = env->insn_processed - insn_processed; if (ret) { return ret; } else if (env->log.level & BPF_LOG_LEVEL) { @@ -18262,10 +18266,12 @@ again: static int do_check_main(struct bpf_verifier_env *env) { + u32 insn_processed = env->insn_processed; int ret; env->insn_idx = 0; ret = do_check_common(env, 0); + env->subprog_info[0].insn_processed = env->insn_processed - insn_processed; if (!ret) env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; return ret; @@ -18274,19 +18280,20 @@ static int do_check_main(struct bpf_verifier_env *env) static void print_verification_stats(struct bpf_verifier_env *env) { - int i; + /* Skip over hidden subprogs which are not verified. */ + int i, subprog_cnt = env->subprog_cnt - env->hidden_subprog_cnt; if (env->log.level & BPF_LOG_STATS) { verbose(env, "verification time %lld usec\n", div_u64(env->verification_time, 1000)); - verbose(env, "stack depth "); - for (i = 0; i < env->subprog_cnt; i++) { - u32 depth = env->subprog_info[i].stack_depth; - - verbose(env, "%d", depth); - if (i + 1 < env->subprog_cnt) - verbose(env, "+"); - } + verbose(env, "stack depth %d", env->subprog_info[0].stack_depth); + for (i = 1; i < subprog_cnt; i++) + verbose(env, "+%d", env->subprog_info[i].stack_depth); + verbose(env, "\n"); + verbose(env, "insns processed %d", env->subprog_info[0].insn_processed); + for (i = 1; i < subprog_cnt; i++) + if (bpf_subprog_is_global(env, i)) + verbose(env, "+%d", env->subprog_info[i].insn_processed); verbose(env, "\n"); } verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " -- cgit v1.2.3 From bd5956166d20adbde3af0f6f265dc2f0ce5f4df9 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 8 Apr 2026 13:53:46 +0200 Subject: hrtimer: Provide hrtimer_start_range_ns_user() Calvin reported an odd NMI watchdog lockup which claims that the CPU locked up in user space. He provided a reproducer, which set's up a timerfd based timer and then rearms it in a loop with an absolute expiry time of 1ns. As the expiry time is in the past, the timer ends up as the first expiring timer in the per CPU hrtimer base and the clockevent device is programmed with the minimum delta value. If the machine is fast enough, this ends up in a endless loop of programming the delta value to the minimum value defined by the clock event device, before the timer interrupt can fire, which starves the interrupt and consequently triggers the lockup detector because the hrtimer callback of the lockup mechanism is never invoked. The clockevents code already has a last resort mechanism to prevent that, but it's sensible to catch such issues before trying to reprogram the clock event device. Provide a variant of hrtimer_start_range_ns(), which sanity checks the timer after queueing it. It does not so before because the timer might be armed and therefore needs to be dequeued. also we optimize for the latest possible point to check, so that the clock event prevention is avoided as much as possible. If the timer is already expired _before_ the clock event is reprogrammed, remove the timer from the queue and signal to the caller that the operation failed by returning false. That allows the caller to take immediate action without going through the loops and hoops of the hrtimer interrupt. The queueing code can't invoke the timer callback as the caller might hold a lock which is taken in the callback. Add a tracepoint which allows to analyze the expired at start situation. Reported-by: Calvin Owens Signed-off-by: Thomas Gleixner Tested-by: Calvin Owens Reviewed-by: Frederic Weisbecker Acked-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260408114951.995031895@kernel.org --- kernel/time/hrtimer.c | 134 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 118 insertions(+), 16 deletions(-) (limited to 'kernel') diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 5bd6efe598f0..79aaac471164 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1352,6 +1352,12 @@ static inline bool hrtimer_keep_base(struct hrtimer *timer, bool is_local, bool return hrtimer_prefer_local(is_local, is_first, is_pinned); } +enum { + HRTIMER_REPROGRAM_NONE, + HRTIMER_REPROGRAM, + HRTIMER_REPROGRAM_FORCE, +}; + static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 delta_ns, const enum hrtimer_mode mode, struct hrtimer_clock_base *base) { @@ -1410,7 +1416,7 @@ static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 del /* If a deferred rearm is pending skip reprogramming the device */ if (cpu_base->deferred_rearm) { cpu_base->deferred_needs_update = true; - return false; + return HRTIMER_REPROGRAM_NONE; } if (!was_first || cpu_base != this_cpu_base) { @@ -1423,7 +1429,7 @@ static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 del * callbacks. */ if (likely(hrtimer_base_is_online(this_cpu_base))) - return first; + return first ? HRTIMER_REPROGRAM : HRTIMER_REPROGRAM_NONE; /* * Timer was enqueued remote because the current base is @@ -1432,7 +1438,7 @@ static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 del */ if (first) smp_call_function_single_async(cpu_base->cpu, &cpu_base->csd); - return false; + return HRTIMER_REPROGRAM_NONE; } /* @@ -1446,7 +1452,7 @@ static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 del */ if (timer->is_lazy) { if (cpu_base->expires_next <= hrtimer_get_expires(timer)) - return false; + return HRTIMER_REPROGRAM_NONE; } /* @@ -1455,8 +1461,24 @@ static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 del * reprogram the hardware by evaluating the new first expiring * timer. */ - hrtimer_force_reprogram(cpu_base, /* skip_equal */ true); - return false; + return HRTIMER_REPROGRAM_FORCE; +} + +static int hrtimer_start_range_ns_common(struct hrtimer *timer, ktime_t tim, + u64 delta_ns, const enum hrtimer_mode mode, + struct hrtimer_clock_base *base) +{ + /* + * Check whether the HRTIMER_MODE_SOFT bit and hrtimer.is_soft + * match on CONFIG_PREEMPT_RT = n. With PREEMPT_RT check the hard + * expiry mode because unmarked timers are moved to softirq expiry. + */ + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft); + else + WARN_ON_ONCE(!(mode & HRTIMER_MODE_HARD) ^ !timer->is_hard); + + return __hrtimer_start_range_ns(timer, tim, delta_ns, mode, base); } /** @@ -1476,24 +1498,104 @@ void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 delta_ns, debug_hrtimer_assert_init(timer); + base = lock_hrtimer_base(timer, &flags); + + switch (hrtimer_start_range_ns_common(timer, tim, delta_ns, mode, base)) { + case HRTIMER_REPROGRAM: + hrtimer_reprogram(timer, true); + break; + case HRTIMER_REPROGRAM_FORCE: + hrtimer_force_reprogram(timer->base->cpu_base, 1); + break; + case HRTIMER_REPROGRAM_NONE: + break; + } + + unlock_hrtimer_base(timer, &flags); +} +EXPORT_SYMBOL_GPL(hrtimer_start_range_ns); + +static inline bool hrtimer_check_user_timer(struct hrtimer *timer) +{ + struct hrtimer_cpu_base *cpu_base = timer->base->cpu_base; + ktime_t expires; + /* - * Check whether the HRTIMER_MODE_SOFT bit and hrtimer.is_soft - * match on CONFIG_PREEMPT_RT = n. With PREEMPT_RT check the hard - * expiry mode because unmarked timers are moved to softirq expiry. + * This uses soft expires because that's the user provided + * expiry time, while expires can be further in the past + * due to a slack value added to the user expiry time. */ - if (!IS_ENABLED(CONFIG_PREEMPT_RT)) - WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft); - else - WARN_ON_ONCE(!(mode & HRTIMER_MODE_HARD) ^ !timer->is_hard); + expires = hrtimer_get_softexpires(timer); + + /* Convert to monotonic */ + expires = ktime_sub(expires, timer->base->offset); + + /* + * Check whether this timer will end up as the first expiring timer in + * the CPU base. If not, no further checks required as it's then + * guaranteed to expire in the future. + */ + if (expires >= cpu_base->expires_next) + return true; + + /* Validate that the expiry time is in the future. */ + if (expires > ktime_get()) + return true; + + debug_hrtimer_deactivate(timer); + __remove_hrtimer(timer, timer->base, HRTIMER_STATE_INACTIVE, false); + trace_hrtimer_start_expired(timer); + return false; +} + +/** + * hrtimer_start_range_ns_user - (re)start an user controlled hrtimer + * @timer: the timer to be added + * @tim: expiry time + * @delta_ns: "slack" range for the timer + * @mode: timer mode: absolute (HRTIMER_MODE_ABS) or + * relative (HRTIMER_MODE_REL), and pinned (HRTIMER_MODE_PINNED); + * softirq based mode is considered for debug purpose only! + * + * Returns: True when the timer was queued, false if it was already expired + * + * This function cannot invoke the timer callback for expired timers as it might + * be called under a lock which the timer callback needs to acquire. So the + * caller has to handle that case. + */ +bool hrtimer_start_range_ns_user(struct hrtimer *timer, ktime_t tim, + u64 delta_ns, const enum hrtimer_mode mode) +{ + struct hrtimer_clock_base *base; + unsigned long flags; + bool ret = true; + + debug_hrtimer_assert_init(timer); base = lock_hrtimer_base(timer, &flags); - if (__hrtimer_start_range_ns(timer, tim, delta_ns, mode, base)) - hrtimer_reprogram(timer, true); + switch (hrtimer_start_range_ns_common(timer, tim, delta_ns, mode, base)) { + case HRTIMER_REPROGRAM: + ret = hrtimer_check_user_timer(timer); + if (ret) + hrtimer_reprogram(timer, true); + break; + case HRTIMER_REPROGRAM_FORCE: + ret = hrtimer_check_user_timer(timer); + /* + * The base must always be reevaluated, independent of the + * result above because the timer was the first pending timer. + */ + hrtimer_force_reprogram(timer->base->cpu_base, 1); + break; + case HRTIMER_REPROGRAM_NONE: + break; + } unlock_hrtimer_base(timer, &flags); + return ret; } -EXPORT_SYMBOL_GPL(hrtimer_start_range_ns); +EXPORT_SYMBOL_GPL(hrtimer_start_range_ns_user); /** * hrtimer_try_to_cancel - try to deactivate a timer -- cgit v1.2.3 From b40c927345a91e687eac4c3c5ca03a99643cd0c4 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 8 Apr 2026 13:53:52 +0200 Subject: hrtimer: Use hrtimer_start_expires_user() for hrtimer sleepers Most hrtimer sleepers are user controlled and user space can hand arbitrary expiry values in as long as they are valid timespecs. If the expiry value is in the past then this requires a full loop through reprogramming the clock event device, taking the hrtimer interrupt, waking the task and reprogram again. Use hrtimer_start_expires_user() which avoids the full round trip by checking the timer for expiry on enqueue. Signed-off-by: Thomas Gleixner Tested-by: Calvin Owens Reviewed-by: Frederic Weisbecker Acked-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260408114952.062400833@kernel.org --- kernel/time/hrtimer.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 79aaac471164..8cfc7aacece1 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -2315,7 +2315,11 @@ void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl, enum hrtimer_mode if (IS_ENABLED(CONFIG_PREEMPT_RT) && sl->timer.is_hard) mode |= HRTIMER_MODE_HARD; - hrtimer_start_expires(&sl->timer, mode); + /* If already expired, clear the task pointer and set current state to running */ + if (!hrtimer_start_expires_user(&sl->timer, mode)) { + sl->task = NULL; + __set_current_state(TASK_RUNNING); + } } EXPORT_SYMBOL_GPL(hrtimer_sleeper_start_expires); -- cgit v1.2.3 From 6fdb2677a594ab38eade927919bbd4d9688bfa1c Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 8 Apr 2026 13:53:56 +0200 Subject: posix-timers: Expand timer_[re]arm() callbacks with a boolean return value In order to catch expiry times which are already in the past the timer_arm() and timer_rearm() callbacks need to be able to report back to the caller whether the timer has been queued or not. Change the function signature and let all implementations return true for now. While at it simplify posix_cpu_timer_rearm(). No functional change intended. Signed-off-by: Thomas Gleixner Reviewed-by: Frederic Weisbecker Acked-by: Peter Zijlstra (Intel) Acked-by: John Stultz Link: https://patch.msgid.link/20260408114952.130222296@kernel.org --- kernel/time/alarmtimer.c | 6 ++++-- kernel/time/posix-cpu-timers.c | 18 ++++++++++-------- kernel/time/posix-timers.c | 6 ++++-- kernel/time/posix-timers.h | 4 ++-- 4 files changed, 20 insertions(+), 14 deletions(-) (limited to 'kernel') diff --git a/kernel/time/alarmtimer.c b/kernel/time/alarmtimer.c index 6e173d70d825..e10021be190f 100644 --- a/kernel/time/alarmtimer.c +++ b/kernel/time/alarmtimer.c @@ -527,12 +527,13 @@ static void alarm_handle_timer(struct alarm *alarm, ktime_t now) * alarm_timer_rearm - Posix timer callback for rearming timer * @timr: Pointer to the posixtimer data struct */ -static void alarm_timer_rearm(struct k_itimer *timr) +static bool alarm_timer_rearm(struct k_itimer *timr) { struct alarm *alarm = &timr->it.alarm.alarmtimer; timr->it_overrun += alarm_forward_now(alarm, timr->it_interval); alarm_start(alarm, alarm->node.expires); + return true; } /** @@ -588,7 +589,7 @@ static void alarm_timer_wait_running(struct k_itimer *timr) * @absolute: Expiry value is absolute time * @sigev_none: Posix timer does not deliver signals */ -static void alarm_timer_arm(struct k_itimer *timr, ktime_t expires, +static bool alarm_timer_arm(struct k_itimer *timr, ktime_t expires, bool absolute, bool sigev_none) { struct alarm *alarm = &timr->it.alarm.alarmtimer; @@ -600,6 +601,7 @@ static void alarm_timer_arm(struct k_itimer *timr, ktime_t expires, alarm->node.expires = expires; else alarm_start(&timr->it.alarm.alarmtimer, expires); + return true; } /** diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 0de2bb7cbec0..395e297093f8 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -19,7 +19,7 @@ #include "posix-timers.h" -static void posix_cpu_timer_rearm(struct k_itimer *timer); +static bool posix_cpu_timer_rearm(struct k_itimer *timer); void posix_cputimers_group_init(struct posix_cputimers *pct, u64 cpu_limit) { @@ -1011,24 +1011,27 @@ static void check_process_timers(struct task_struct *tsk, /* * This is called from the signal code (via posixtimer_rearm) * when the last timer signal was delivered and we have to reload the timer. + * + * Return true unconditionally so the core code assumes the timer to be + * armed. Otherwise it would requeue the signal. */ -static void posix_cpu_timer_rearm(struct k_itimer *timer) +static bool posix_cpu_timer_rearm(struct k_itimer *timer) { clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock); - struct task_struct *p; struct sighand_struct *sighand; + struct task_struct *p; unsigned long flags; u64 now; - rcu_read_lock(); + guard(rcu)(); p = cpu_timer_task_rcu(timer); if (!p) - goto out; + return true; /* Protect timer list r/w in arm_timer() */ sighand = lock_task_sighand(p, &flags); if (unlikely(sighand == NULL)) - goto out; + return true; /* * Fetch the current sample and update the timer's expiry time. @@ -1045,8 +1048,7 @@ static void posix_cpu_timer_rearm(struct k_itimer *timer) */ arm_timer(timer, p); unlock_task_sighand(p, &flags); -out: - rcu_read_unlock(); + return true; } /** diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c index 9331e1614124..da04ed42bf82 100644 --- a/kernel/time/posix-timers.c +++ b/kernel/time/posix-timers.c @@ -288,12 +288,13 @@ static inline int timer_overrun_to_int(struct k_itimer *timr) return (int)timr->it_overrun_last; } -static void common_hrtimer_rearm(struct k_itimer *timr) +static bool common_hrtimer_rearm(struct k_itimer *timr) { struct hrtimer *timer = &timr->it.real.timer; timr->it_overrun += hrtimer_forward_now(timer, timr->it_interval); hrtimer_restart(timer); + return true; } static bool __posixtimer_deliver_signal(struct kernel_siginfo *info, struct k_itimer *timr) @@ -795,7 +796,7 @@ SYSCALL_DEFINE1(timer_getoverrun, timer_t, timer_id) return timer_overrun_to_int(scoped_timer); } -static void common_hrtimer_arm(struct k_itimer *timr, ktime_t expires, +static bool common_hrtimer_arm(struct k_itimer *timr, ktime_t expires, bool absolute, bool sigev_none) { struct hrtimer *timer = &timr->it.real.timer; @@ -822,6 +823,7 @@ static void common_hrtimer_arm(struct k_itimer *timr, ktime_t expires, if (!sigev_none) hrtimer_start_expires(timer, HRTIMER_MODE_ABS); + return true; } static int common_hrtimer_try_to_cancel(struct k_itimer *timr) diff --git a/kernel/time/posix-timers.h b/kernel/time/posix-timers.h index 7f259e845d24..4ea9611dd716 100644 --- a/kernel/time/posix-timers.h +++ b/kernel/time/posix-timers.h @@ -27,11 +27,11 @@ struct k_clock { int (*timer_del)(struct k_itimer *timr); void (*timer_get)(struct k_itimer *timr, struct itimerspec64 *cur_setting); - void (*timer_rearm)(struct k_itimer *timr); + bool (*timer_rearm)(struct k_itimer *timr); s64 (*timer_forward)(struct k_itimer *timr, ktime_t now); ktime_t (*timer_remaining)(struct k_itimer *timr, ktime_t now); int (*timer_try_to_cancel)(struct k_itimer *timr); - void (*timer_arm)(struct k_itimer *timr, ktime_t expires, + bool (*timer_arm)(struct k_itimer *timr, ktime_t expires, bool absolute, bool sigev_none); void (*timer_wait_running)(struct k_itimer *timr); }; -- cgit v1.2.3 From cfb7fe3fdd4ca1d37da1ed15a1897d4a27c47a8a Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 8 Apr 2026 13:54:01 +0200 Subject: posix-timers: Handle the timer_[re]arm() return value The [re]arm callbacks will return true when the timer was queued and false if it was already expired at enqueue time. In both cases the call sites can trivially queue the signal right there, when the timer was already expired. That avoids a full round trip through the hrtimer interrupt. Signed-off-by: Thomas Gleixner Reviewed-by: Frederic Weisbecker Acked-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260408114952.198028466@kernel.org --- kernel/time/posix-timers.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c index da04ed42bf82..db62cfac169d 100644 --- a/kernel/time/posix-timers.c +++ b/kernel/time/posix-timers.c @@ -299,6 +299,8 @@ static bool common_hrtimer_rearm(struct k_itimer *timr) static bool __posixtimer_deliver_signal(struct kernel_siginfo *info, struct k_itimer *timr) { + bool queued; + guard(spinlock)(&timr->it_lock); /* @@ -312,12 +314,18 @@ static bool __posixtimer_deliver_signal(struct kernel_siginfo *info, struct k_it if (!timr->it_interval || WARN_ON_ONCE(timr->it_status != POSIX_TIMER_REQUEUE_PENDING)) return true; - timr->kclock->timer_rearm(timr); - timr->it_status = POSIX_TIMER_ARMED; + /* timer_rearm() updates timr::it_overrun */ + queued = timr->kclock->timer_rearm(timr); + timr->it_overrun_last = timr->it_overrun; timr->it_overrun = -1LL; ++timr->it_signal_seq; info->si_overrun = timer_overrun_to_int(timr); + + if (queued) + timr->it_status = POSIX_TIMER_ARMED; + else + posix_timer_queue_signal(timr); return true; } @@ -905,9 +913,13 @@ int common_timer_set(struct k_itimer *timr, int flags, expires = timens_ktime_to_host(timr->it_clock, expires); sigev_none = timr->it_sigev_notify == SIGEV_NONE; - kc->timer_arm(timr, expires, flags & TIMER_ABSTIME, sigev_none); - if (!sigev_none) - timr->it_status = POSIX_TIMER_ARMED; + if (kc->timer_arm(timr, expires, flags & TIMER_ABSTIME, sigev_none)) { + if (!sigev_none) + timr->it_status = POSIX_TIMER_ARMED; + } else { + /* Timer was already expired, queue the signal */ + posix_timer_queue_signal(timr); + } return 0; } -- cgit v1.2.3 From acc071343d29c2361619b05ad50ea3de9ef9a3ac Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 8 Apr 2026 13:54:06 +0200 Subject: posix-timers: Switch to hrtimer_start_expires_user() Switch the arm and rearm callbacks for hrtimer based posix timers over to hrtimer_start_expires_user() so that already expired timers are not queued. Hand the result back to the caller, which then queues the signal. Signed-off-by: Thomas Gleixner Reviewed-by: Frederic Weisbecker Acked-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260408114952.266001916@kernel.org --- kernel/time/posix-timers.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c index db62cfac169d..436ba794cc0b 100644 --- a/kernel/time/posix-timers.c +++ b/kernel/time/posix-timers.c @@ -293,8 +293,7 @@ static bool common_hrtimer_rearm(struct k_itimer *timr) struct hrtimer *timer = &timr->it.real.timer; timr->it_overrun += hrtimer_forward_now(timer, timr->it_interval); - hrtimer_restart(timer); - return true; + return hrtimer_start_expires_user(timer, HRTIMER_MODE_ABS); } static bool __posixtimer_deliver_signal(struct kernel_siginfo *info, struct k_itimer *timr) @@ -829,9 +828,11 @@ static bool common_hrtimer_arm(struct k_itimer *timr, ktime_t expires, expires = ktime_add_safe(expires, hrtimer_cb_get_time(timer)); hrtimer_set_expires(timer, expires); - if (!sigev_none) - hrtimer_start_expires(timer, HRTIMER_MODE_ABS); - return true; + /* For sigev_none pretend that the timer is queued */ + if (sigev_none) + return true; + + return hrtimer_start_expires_user(timer, HRTIMER_MODE_ABS); } static int common_hrtimer_try_to_cancel(struct k_itimer *timr) -- cgit v1.2.3 From 183d00b727139cf3b4be78d66a5602ce71a3acec Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 8 Apr 2026 13:54:11 +0200 Subject: alarmtimer: Provide alarm_start_timer() Alarm timers utilize hrtimers for normal operation and only switch to the RTC on suspend. In order to catch already expired timers early and without going through a timer interrupt cycle, provide a new start function which internally uses hrtimer_start_range_ns_user(). If hrtimer_start_range_ns_user() detects an already expired timer, it does not queue it. In that case remove the timer from the alarm base as well. Return the status queued or not back to the caller to handle the early expiry. Signed-off-by: Thomas Gleixner Reviewed-by: Frederic Weisbecker Acked-by: John Stultz Link: https://patch.msgid.link/20260408114952.332822525@kernel.org --- kernel/time/alarmtimer.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 'kernel') diff --git a/kernel/time/alarmtimer.c b/kernel/time/alarmtimer.c index e10021be190f..2348b0839114 100644 --- a/kernel/time/alarmtimer.c +++ b/kernel/time/alarmtimer.c @@ -369,6 +369,34 @@ void alarm_start_relative(struct alarm *alarm, ktime_t start) } EXPORT_SYMBOL_GPL(alarm_start_relative); +/** + * alarm_start_timer - Sets an alarm to fire + * @alarm: Pointer to alarm to set + * @expires: Expiry time + * @relative: True if @expires is relative + * + * Returns: True if the alarm was queued. False if it already expired + */ +bool alarm_start_timer(struct alarm *alarm, ktime_t expires, bool relative) +{ + struct alarm_base *base = &alarm_bases[alarm->type]; + + if (relative) + expires = ktime_add_safe(expires, base->get_ktime()); + + trace_alarmtimer_start(alarm, base->get_ktime()); + + guard(spinlock_irqsave)(&base->lock); + alarm->node.expires = expires; + alarmtimer_enqueue(base, alarm); + if (!hrtimer_start_range_ns_user(&alarm->timer, expires, 0, HRTIMER_MODE_ABS)) { + alarmtimer_dequeue(base, alarm); + return false; + } + return true; +} +EXPORT_SYMBOL_GPL(alarm_start_timer); + void alarm_restart(struct alarm *alarm) { struct alarm_base *base = &alarm_bases[alarm->type]; -- cgit v1.2.3 From f4b58f61da79032b03d25f8f1a5a697db84a46f3 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 8 Apr 2026 13:54:16 +0200 Subject: alarmtimer: Convert posix timer functions to alarm_start_timer() Use the new alarm_start_timer() for arming and rearming posix interval timers and for clock_nanosleep() so that already expired timers do not go through the full timer interrupt cycle. Signed-off-by: Thomas Gleixner Reviewed-by: Frederic Weisbecker Acked-by: John Stultz Link: https://patch.msgid.link/20260408114952.400451460@kernel.org --- kernel/time/alarmtimer.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/time/alarmtimer.c b/kernel/time/alarmtimer.c index 2348b0839114..9baa69cfa158 100644 --- a/kernel/time/alarmtimer.c +++ b/kernel/time/alarmtimer.c @@ -560,8 +560,7 @@ static bool alarm_timer_rearm(struct k_itimer *timr) struct alarm *alarm = &timr->it.alarm.alarmtimer; timr->it_overrun += alarm_forward_now(alarm, timr->it_interval); - alarm_start(alarm, alarm->node.expires); - return true; + return alarm_start_timer(alarm, alarm->node.expires, false); } /** @@ -625,11 +624,16 @@ static bool alarm_timer_arm(struct k_itimer *timr, ktime_t expires, if (!absolute) expires = ktime_add_safe(expires, base->get_ktime()); - if (sigev_none) + + /* + * sigev_none needs to update the expires value and pretend + * that the timer is queued + */ + if (sigev_none) { alarm->node.expires = expires; - else - alarm_start(&timr->it.alarm.alarmtimer, expires); - return true; + return true; + } + return alarm_start_timer(&timr->it.alarm.alarmtimer, expires, false); } /** @@ -736,7 +740,9 @@ static int alarmtimer_do_nsleep(struct alarm *alarm, ktime_t absexp, alarm->data = (void *)current; do { set_current_state(TASK_INTERRUPTIBLE); - alarm_start(alarm, absexp); + if (!alarm_start_timer(alarm, absexp, false)) + alarm->data = NULL; + if (likely(alarm->data)) schedule(); -- cgit v1.2.3 From ed78a701941999635389c41ddd638e8e7ea2470f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 8 Apr 2026 13:54:33 +0200 Subject: alarmtimer: Remove unused interfaces All alarmtimer users are converted to alarm_start_timer(). Remove the now unused interfaces. Signed-off-by: Thomas Gleixner Reviewed-by: Frederic Weisbecker Link: https://patch.msgid.link/20260408114952.670899355@kernel.org --- kernel/time/alarmtimer.c | 44 -------------------------------------------- 1 file changed, 44 deletions(-) (limited to 'kernel') diff --git a/kernel/time/alarmtimer.c b/kernel/time/alarmtimer.c index 9baa69cfa158..9275fe4cdc1b 100644 --- a/kernel/time/alarmtimer.c +++ b/kernel/time/alarmtimer.c @@ -336,39 +336,6 @@ void alarm_init(struct alarm *alarm, enum alarmtimer_type type, } EXPORT_SYMBOL_GPL(alarm_init); -/** - * alarm_start - Sets an absolute alarm to fire - * @alarm: ptr to alarm to set - * @start: time to run the alarm - */ -void alarm_start(struct alarm *alarm, ktime_t start) -{ - struct alarm_base *base = &alarm_bases[alarm->type]; - - scoped_guard(spinlock_irqsave, &base->lock) { - alarm->node.expires = start; - alarmtimer_enqueue(base, alarm); - hrtimer_start(&alarm->timer, alarm->node.expires, HRTIMER_MODE_ABS); - } - - trace_alarmtimer_start(alarm, base->get_ktime()); -} -EXPORT_SYMBOL_GPL(alarm_start); - -/** - * alarm_start_relative - Sets a relative alarm to fire - * @alarm: ptr to alarm to set - * @start: time relative to now to run the alarm - */ -void alarm_start_relative(struct alarm *alarm, ktime_t start) -{ - struct alarm_base *base = &alarm_bases[alarm->type]; - - start = ktime_add_safe(start, base->get_ktime()); - alarm_start(alarm, start); -} -EXPORT_SYMBOL_GPL(alarm_start_relative); - /** * alarm_start_timer - Sets an alarm to fire * @alarm: Pointer to alarm to set @@ -397,17 +364,6 @@ bool alarm_start_timer(struct alarm *alarm, ktime_t expires, bool relative) } EXPORT_SYMBOL_GPL(alarm_start_timer); -void alarm_restart(struct alarm *alarm) -{ - struct alarm_base *base = &alarm_bases[alarm->type]; - - guard(spinlock_irqsave)(&base->lock); - hrtimer_set_expires(&alarm->timer, alarm->node.expires); - hrtimer_restart(&alarm->timer); - alarmtimer_enqueue(base, alarm); -} -EXPORT_SYMBOL_GPL(alarm_restart); - /** * alarm_try_to_cancel - Tries to cancel an alarm timer * @alarm: ptr to alarm to be canceled -- cgit v1.2.3 From d757ac2ee7bfda90c64b60a6593a2139a06f79b2 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Thu, 30 Apr 2026 16:45:43 +0800 Subject: sched_ext: Remove redundant rcu_read_lock/unlock() in sysrq_handle_sched_ext_reset() sysrq_handle_sched_ext_reset() is called from __handle_sysrq(), which already holds rcu_read_lock() while invoking the sysrq handler. Remove the redundant rcu_read_lock/unlock() pair. Signed-off-by: Zqiang Reviewed-by: Cheng-Yang Chou Reviewed-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index f3a585e32db3..4287654c746f 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -7914,13 +7914,11 @@ static void sysrq_handle_sched_ext_reset(u8 key) { struct scx_sched *sch; - rcu_read_lock(); sch = rcu_dereference(scx_root); if (likely(sch)) scx_disable(sch, SCX_EXIT_SYSRQ); else pr_info("sched_ext: BPF schedulers not loaded\n"); - rcu_read_unlock(); } static const struct sysrq_key_op sysrq_sched_ext_reset_op = { -- cgit v1.2.3 From 4c81b2b2a293539674cc51e726bc7d3c13602a42 Mon Sep 17 00:00:00 2001 From: Cheng-Yang Chou Date: Tue, 5 May 2026 00:08:19 +0800 Subject: sched_ext: Normalize exit dump header to "on CPU N" Unify to uppercase to match the UEI output. Signed-off-by: Cheng-Yang Chou Signed-off-by: Tejun Heo --- kernel/sched/ext.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 4287654c746f..966a846c213c 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -6447,7 +6447,7 @@ static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei, dump_line(&s, "Debug dump triggered by %s", ei->reason); } else { if (ei->exit_cpu >= 0) - dump_line(&s, "%s[%d] triggered exit kind %d on cpu %d:", + dump_line(&s, "%s[%d] triggered exit kind %d on CPU %d:", current->comm, current->pid, ei->kind, ei->exit_cpu); else -- cgit v1.2.3 From d91b36de53c563f6f65aa1dea747f9bcb4c56d1d Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 4 May 2026 11:21:56 -1000 Subject: sched_ext: Add __printf format attributes to scx_vexit() and bstr formatters scx_vexit() forwards (fmt, args) to vscnprintf(); bstr_format() and __bstr_format() forward fmt to bstr_printf(); the BPF kfunc wrappers scx_bpf_exit_bstr(), scx_bpf_error_bstr() and scx_bpf_dump_bstr() in turn forward fmt to those formatters. None of them have __printf(), so clang -Wmissing-format-attribute fires on the forwarded calls and C-side callers don't get format-string checking. Annotate the six functions with __printf(N, 0) matching the fmt parameter position in each. Reported-by: kernel test robot Closes: https://lore.kernel.org/all/202605041112.Y6OG7v9r-lkp@intel.com/ Signed-off-by: Tejun Heo --- kernel/sched/ext.c | 5 +++++ kernel/sched/ext_internal.h | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 966a846c213c..7ac7d10a41be 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -9338,6 +9338,7 @@ __bpf_kfunc void scx_bpf_reenqueue_local___v2(const struct bpf_prog_aux *aux) __bpf_kfunc_end_defs(); +__printf(5, 0) static s32 __bstr_format(struct scx_sched *sch, u64 *data_buf, char *line_buf, size_t line_size, char *fmt, unsigned long long *data, u32 data__sz) @@ -9375,6 +9376,7 @@ static s32 __bstr_format(struct scx_sched *sch, u64 *data_buf, char *line_buf, return ret; } +__printf(3, 0) static s32 bstr_format(struct scx_sched *sch, struct scx_bstr_buf *buf, char *fmt, unsigned long long *data, u32 data__sz) { @@ -9395,6 +9397,7 @@ __bpf_kfunc_start_defs(); * Indicate that the BPF scheduler wants to exit gracefully, and initiate ops * disabling. */ +__printf(2, 0) __bpf_kfunc void scx_bpf_exit_bstr(s64 exit_code, char *fmt, unsigned long long *data, u32 data__sz, const struct bpf_prog_aux *aux) @@ -9420,6 +9423,7 @@ __bpf_kfunc void scx_bpf_exit_bstr(s64 exit_code, char *fmt, * Indicate that the BPF scheduler encountered a fatal error and initiate ops * disabling. */ +__printf(1, 0) __bpf_kfunc void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data__sz, const struct bpf_prog_aux *aux) { @@ -9447,6 +9451,7 @@ __bpf_kfunc void scx_bpf_error_bstr(char *fmt, unsigned long long *data, * The extra dump may be multiple lines. A single line may be split over * multiple calls. The last line is automatically terminated. */ +__printf(1, 0) __bpf_kfunc void scx_bpf_dump_bstr(char *fmt, unsigned long long *data, u32 data__sz, const struct bpf_prog_aux *aux) { diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext_internal.h index b4f5dd28855e..0ed79bd891c7 100644 --- a/kernel/sched/ext_internal.h +++ b/kernel/sched/ext_internal.h @@ -1477,8 +1477,9 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id); bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where); -bool scx_vexit(struct scx_sched *sch, enum scx_exit_kind kind, s64 exit_code, - s32 exit_cpu, const char *fmt, va_list args); +__printf(5, 0) bool scx_vexit(struct scx_sched *sch, enum scx_exit_kind kind, + s64 exit_code, s32 exit_cpu, const char *fmt, + va_list args); __printf(5, 6) bool __scx_exit(struct scx_sched *sch, enum scx_exit_kind kind, s64 exit_code, s32 exit_cpu, const char *fmt, ...); -- cgit v1.2.3 From 89976cd73739dcb73745705a63ccc67a8be26cdf Mon Sep 17 00:00:00 2001 From: Aniket Gattani Date: Sun, 3 May 2026 21:22:03 +0000 Subject: sched/membarrier: Use per-CPU mutexes for targeted commands Currently, the membarrier system call uses a single global mutex (`membarrier_ipi_mutex`) to serialize expedited commands. This causes significant contention on large systems when multiple threads invoke membarrier concurrently, even if they target different CPUs. This contention becomes critical when combined with CFS bandwidth throttling/unthrottling, during which interrupts can be disabled for relatively long periods on target CPUs. If membarrier is waiting for a response from such a CPU, it holds the global mutex, blocking all other membarrier calls on the system. This cascade effect can lead to hard lockups when thousands of threads stall waiting for the mutex. Optimize `MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ` when a specific CPU is targeted by introducing per-CPU mutexes. Broadcast commands and commands without a specific CPU target continue to use the global mutex. This prevents the cascade lockup scenario. As measured by the stress test introduced in the subsequent patch, on an AMD Turin machine with 384 CPUs (2 NUMA nodes with SMT=2), this optimization yields 200x more throughput. Signed-off-by: Aniket Gattani Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260503212205.3714217-2-aniketgattani@google.com --- kernel/sched/membarrier.c | 77 +++++++++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 33 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/membarrier.c b/kernel/sched/membarrier.c index 623445603725..3d88e900a17f 100644 --- a/kernel/sched/membarrier.c +++ b/kernel/sched/membarrier.c @@ -164,8 +164,26 @@ | MEMBARRIER_PRIVATE_EXPEDITED_RSEQ_BITMASK \ | MEMBARRIER_CMD_GET_REGISTRATIONS) +/* + * Scoped guard for memory barriers on entry and exit. + * Matches memory barriers before & after rq->curr modification in scheduler. + */ +DEFINE_LOCK_GUARD_0(mb, smp_mb(), smp_mb()) static DEFINE_MUTEX(membarrier_ipi_mutex); +static DEFINE_PER_CPU(struct mutex, membarrier_cpu_mutexes); + #define SERIALIZE_IPI() guard(mutex)(&membarrier_ipi_mutex) +#define SERIALIZE_IPI_CPU(cpu_id) guard(mutex)(&per_cpu(membarrier_cpu_mutexes, cpu_id)) + +static int __init membarrier_init(void) +{ + int i; + + for_each_possible_cpu(i) + mutex_init(&per_cpu(membarrier_cpu_mutexes, i)); + return 0; +} +core_initcall(membarrier_init); static void ipi_mb(void *info) { @@ -315,7 +333,6 @@ static int membarrier_global_expedited(void) static int membarrier_private_expedited(int flags, int cpu_id) { - cpumask_var_t tmpmask; struct mm_struct *mm = current->mm; smp_call_func_t ipi_func = ipi_mb; @@ -352,30 +369,45 @@ static int membarrier_private_expedited(int flags, int cpu_id) * On RISC-V, this barrier pairing is also needed for the * SYNC_CORE command when switching between processes, cf. * the inline comments in membarrier_arch_switch_mm(). + * + * Memory barrier on the caller thread _after_ we finished + * waiting for the last IPI. Matches memory barriers before + * rq->curr modification in scheduler. */ - smp_mb(); /* system call entry is not a mb. */ - - if (cpu_id < 0 && !zalloc_cpumask_var(&tmpmask, GFP_KERNEL)) - return -ENOMEM; - - SERIALIZE_IPI(); - cpus_read_lock(); - + guard(mb)(); if (cpu_id >= 0) { + if (cpu_id >= nr_cpu_ids || !cpu_possible(cpu_id)) + return 0; + + SERIALIZE_IPI_CPU(cpu_id); + guard(cpus_read_lock)(); struct task_struct *p; - if (cpu_id >= nr_cpu_ids || !cpu_online(cpu_id)) - goto out; + if (!cpu_online(cpu_id)) + return 0; + rcu_read_lock(); p = rcu_dereference(cpu_rq(cpu_id)->curr); if (!p || p->mm != mm) { rcu_read_unlock(); - goto out; + return 0; } rcu_read_unlock(); + /* + * smp_call_function_single() will call ipi_func() if cpu_id + * is the calling CPU. + */ + smp_call_function_single(cpu_id, ipi_func, NULL, 1); } else { + cpumask_var_t __free(free_cpumask_var) tmpmask = CPUMASK_VAR_NULL; int cpu; + if (!zalloc_cpumask_var(&tmpmask, GFP_KERNEL)) + return -ENOMEM; + + SERIALIZE_IPI(); + guard(cpus_read_lock)(); + rcu_read_lock(); for_each_online_cpu(cpu) { struct task_struct *p; @@ -385,15 +417,6 @@ static int membarrier_private_expedited(int flags, int cpu_id) __cpumask_set_cpu(cpu, tmpmask); } rcu_read_unlock(); - } - - if (cpu_id >= 0) { - /* - * smp_call_function_single() will call ipi_func() if cpu_id - * is the calling CPU. - */ - smp_call_function_single(cpu_id, ipi_func, NULL, 1); - } else { /* * For regular membarrier, we can save a few cycles by * skipping the current cpu -- we're about to do smp_mb() @@ -420,18 +443,6 @@ static int membarrier_private_expedited(int flags, int cpu_id) } } -out: - if (cpu_id < 0) - free_cpumask_var(tmpmask); - cpus_read_unlock(); - - /* - * Memory barrier on the caller thread _after_ we finished - * waiting for the last IPI. Matches memory barriers before - * rq->curr modification in scheduler. - */ - smp_mb(); /* exit from system call is not a mb */ - return 0; } -- cgit v1.2.3 From a5959728548caf0aa17879342ee6571a29d54574 Mon Sep 17 00:00:00 2001 From: Aniket Gattani Date: Sun, 3 May 2026 21:22:04 +0000 Subject: sched/membarrier: Modernize membarrier_global_expedited with cleanup guards Replace explicit lock/unlock and free calls with scoped guards and automatic cleanup constructs. Signed-off-by: Aniket Gattani Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260503212205.3714217-3-aniketgattani@google.com --- kernel/sched/membarrier.c | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/membarrier.c b/kernel/sched/membarrier.c index 3d88e900a17f..12b68a77630a 100644 --- a/kernel/sched/membarrier.c +++ b/kernel/sched/membarrier.c @@ -267,23 +267,19 @@ void membarrier_update_current_mm(struct mm_struct *next_mm) static int membarrier_global_expedited(void) { + cpumask_var_t __free(free_cpumask_var) tmpmask = CPUMASK_VAR_NULL; int cpu; - cpumask_var_t tmpmask; if (num_online_cpus() == 1) return 0; - /* - * Matches memory barriers after rq->curr modification in - * scheduler. - */ - smp_mb(); /* system call entry is not a mb. */ - if (!zalloc_cpumask_var(&tmpmask, GFP_KERNEL)) return -ENOMEM; + guard(mb)(); SERIALIZE_IPI(); - cpus_read_lock(); + guard(cpus_read_lock)(); + rcu_read_lock(); for_each_online_cpu(cpu) { struct task_struct *p; @@ -319,15 +315,6 @@ static int membarrier_global_expedited(void) smp_call_function_many(tmpmask, ipi_mb, NULL, 1); preempt_enable(); - free_cpumask_var(tmpmask); - cpus_read_unlock(); - - /* - * Memory barrier on the caller thread _after_ we finished - * waiting for the last IPI. Matches memory barriers before - * rq->curr modification in scheduler. - */ - smp_mb(); /* exit from system call is not a mb */ return 0; } -- cgit v1.2.3 From 4ac4d6549a6563878d7c19c154e017f6cb7114d3 Mon Sep 17 00:00:00 2001 From: Gabriele Monaco Date: Wed, 29 Apr 2026 11:41:37 +0200 Subject: sched: Use trace_call__() to save a static branch The wrapper functions __trace_set_current_state() and __trace_set_need_resched() allow the tracepoints to be called from code outside sched/core.c, those calls are already guarded by a tracepoint_enabled() so there is no need to repeat this check once again inside the call using trace_(). Use the new trace_call__() API to directly call the tracepoint without check. Those helper functions must be called after the appropriate check. Signed-off-by: Gabriele Monaco Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260429094227.34087-1-gmonaco@redhat.com --- kernel/sched/core.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index b8871449d3c6..b905805bbcbe 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -537,10 +537,14 @@ sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) { } /* need a wrapper since we may need to trace from modules */ EXPORT_TRACEPOINT_SYMBOL(sched_set_state_tp); -/* Call via the helper macro trace_set_current_state. */ +/* + * Call via the helper macro trace_set_current_state. + * Calls to this function MUST be guarded by a + * tracepoint_enabled(sched_set_state_tp) + */ void __trace_set_current_state(int state_value) { - trace_sched_set_state_tp(current, state_value); + trace_call__sched_set_state_tp(current, state_value); } EXPORT_SYMBOL(__trace_set_current_state); @@ -1203,9 +1207,13 @@ static void __resched_curr(struct rq *rq, int tif) } } +/* + * Calls to this function MUST be guarded by a + * tracepoint_enabled(sched_set_need_resched_tp) + */ void __trace_set_need_resched(struct task_struct *curr, int tif) { - trace_sched_set_need_resched_tp(curr, smp_processor_id(), tif); + trace_call__sched_set_need_resched_tp(curr, smp_processor_id(), tif); } EXPORT_SYMBOL_GPL(__trace_set_need_resched); -- cgit v1.2.3 From ff65875f80d1a662d2d39e3e36345a0918766b3c Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Thu, 23 Apr 2026 18:53:50 +0200 Subject: timers/migration: Abstract out hierarchy to prepare for CPU capacity awareness In order to prepare for separating out CPUs from different capacities in distinct hierarchies, create a hierarchy structure that group setup must rely upon. Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Link: https://patch.msgid.link/20260423165354.95152-3-frederic@kernel.org --- kernel/time/timer_migration.c | 100 +++++++++++++++++++++++++----------------- kernel/time/timer_migration.h | 10 +++++ 2 files changed, 69 insertions(+), 41 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timer_migration.c b/kernel/time/timer_migration.c index 1d0d3a4058d5..a8ec85f2f5c7 100644 --- a/kernel/time/timer_migration.c +++ b/kernel/time/timer_migration.c @@ -102,7 +102,7 @@ * active CPU/group information atomic_try_cmpxchg() is used instead and only * the per CPU tmigr_cpu->lock is held. * - * During the setup of groups tmigr_level_list is required. It is protected by + * During the setup of groups, hier->level_list is required. It is protected by * @tmigr_mutex. * * When @timer_base->lock as well as tmigr related locks are required, the lock @@ -416,13 +416,12 @@ */ static DEFINE_MUTEX(tmigr_mutex); -static struct list_head *tmigr_level_list __read_mostly; + +static struct tmigr_hierarchy *hierarchy; static unsigned int tmigr_hierarchy_levels __read_mostly; static unsigned int tmigr_crossnode_level __read_mostly; -static struct tmigr_group *tmigr_root; - static DEFINE_PER_CPU(struct tmigr_cpu, tmigr_cpu); /* @@ -1653,14 +1652,14 @@ static void tmigr_init_group(struct tmigr_group *group, unsigned int lvl, group->groupevt.ignore = true; } -static struct tmigr_group *tmigr_get_group(int node, unsigned int lvl) +static struct tmigr_group *tmigr_get_group(struct tmigr_hierarchy *hier, int node, unsigned int lvl) { struct tmigr_group *tmp, *group = NULL; lockdep_assert_held(&tmigr_mutex); /* Try to attach to an existing group first */ - list_for_each_entry(tmp, &tmigr_level_list[lvl], list) { + list_for_each_entry(tmp, &hier->level_list[lvl], list) { /* * If @lvl is below the cross NUMA node level, check whether * this group belongs to the same NUMA node. @@ -1694,14 +1693,14 @@ static struct tmigr_group *tmigr_get_group(int node, unsigned int lvl) tmigr_init_group(group, lvl, node); /* Setup successful. Add it to the hierarchy */ - list_add(&group->list, &tmigr_level_list[lvl]); + list_add(&group->list, &hier->level_list[lvl]); trace_tmigr_group_set(group); return group; } -static bool tmigr_init_root(struct tmigr_group *group, bool activate) +static bool tmigr_init_root(struct tmigr_hierarchy *hier, struct tmigr_group *group, bool activate) { - if (!group->parent && group != tmigr_root) { + if (!group->parent && group != hier->root) { /* * This is the new top-level, prepare its groupmask in advance * to avoid accidents where yet another new top-level is @@ -1717,11 +1716,10 @@ static bool tmigr_init_root(struct tmigr_group *group, bool activate) } -static void tmigr_connect_child_parent(struct tmigr_group *child, - struct tmigr_group *parent, - bool activate) +static void tmigr_connect_child_parent(struct tmigr_hierarchy *hier, struct tmigr_group *child, + struct tmigr_group *parent, bool activate) { - if (tmigr_init_root(parent, activate)) { + if (tmigr_init_root(hier, parent, activate)) { /* * The previous top level had prepared its groupmask already, * simply account it in advance as the first child. If some groups @@ -1757,10 +1755,10 @@ static void tmigr_connect_child_parent(struct tmigr_group *child, trace_tmigr_connect_child_parent(child); } -static int tmigr_setup_groups(unsigned int cpu, unsigned int node, - struct tmigr_group *start, bool activate) +static int tmigr_setup_groups(struct tmigr_hierarchy *hier, unsigned int cpu, + unsigned int node, struct tmigr_group *start, bool activate) { - struct tmigr_group *group, *child, **stack; + struct tmigr_group *root = hier->root, *group, *child, **stack; int i, top = 0, err = 0, start_lvl = 0; bool root_mismatch = false; @@ -1773,11 +1771,11 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node, start_lvl = start->level + 1; } - if (tmigr_root) - root_mismatch = tmigr_root->numa_node != node; + if (root) + root_mismatch = root->numa_node != node; for (i = start_lvl; i < tmigr_hierarchy_levels; i++) { - group = tmigr_get_group(node, i); + group = tmigr_get_group(hier, node, i); if (IS_ERR(group)) { err = PTR_ERR(group); i--; @@ -1799,7 +1797,7 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node, if (group->parent) break; if ((!root_mismatch || i >= tmigr_crossnode_level) && - list_is_singular(&tmigr_level_list[i])) + list_is_singular(&hier->level_list[i])) break; } @@ -1827,7 +1825,7 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node, tmc->tmgroup = group; tmc->groupmask = BIT(group->num_children++); - tmigr_init_root(group, activate); + tmigr_init_root(hier, group, activate); trace_tmigr_connect_cpu_parent(tmc); @@ -1835,7 +1833,7 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node, continue; } else { child = stack[i - 1]; - tmigr_connect_child_parent(child, group, activate); + tmigr_connect_child_parent(hier, child, group, activate); } } @@ -1894,15 +1892,14 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node, } /* Root update */ - if (list_is_singular(&tmigr_level_list[top])) { - group = list_first_entry(&tmigr_level_list[top], - typeof(*group), list); + if (list_is_singular(&hier->level_list[top])) { + group = list_first_entry(&hier->level_list[top], typeof(*group), list); WARN_ON_ONCE(group->parent); - if (tmigr_root) { + if (root) { /* Old root should be the same or below */ - WARN_ON_ONCE(tmigr_root->level > top); + WARN_ON_ONCE(root->level > top); } - tmigr_root = group; + hier->root = group; } out: kfree(stack); @@ -1910,18 +1907,47 @@ out: return err; } +static struct tmigr_hierarchy *tmigr_get_hierarchy(void) +{ + if (hierarchy) + return hierarchy; + + hierarchy = kzalloc(sizeof(*hierarchy), GFP_KERNEL); + if (!hierarchy) + return ERR_PTR(-ENOMEM); + + hierarchy->level_list = kzalloc_objs(struct list_head, tmigr_hierarchy_levels); + if (!hierarchy->level_list) { + kfree(hierarchy); + hierarchy = NULL; + return ERR_PTR(-ENOMEM); + } + + for (int i = 0; i < tmigr_hierarchy_levels; i++) + INIT_LIST_HEAD(&hierarchy->level_list[i]); + + return hierarchy; +} + static int tmigr_add_cpu(unsigned int cpu) { - struct tmigr_group *old_root = tmigr_root; + struct tmigr_hierarchy *hier; + struct tmigr_group *old_root; int node = cpu_to_node(cpu); int ret; guard(mutex)(&tmigr_mutex); - ret = tmigr_setup_groups(cpu, node, NULL, false); + hier = tmigr_get_hierarchy(); + if (IS_ERR(hier)) + return PTR_ERR(hier); + + old_root = hier->root; + + ret = tmigr_setup_groups(hier, cpu, node, NULL, false); /* Root has changed? Connect the old one to the new */ - if (ret >= 0 && old_root && old_root != tmigr_root) { + if (ret >= 0 && old_root && old_root != hier->root) { /* * The target CPU must never do the prepare work, except * on early boot when the boot CPU is the target. Otherwise @@ -1935,7 +1961,7 @@ static int tmigr_add_cpu(unsigned int cpu) * otherwise the old root may not be active as expected. */ WARN_ON_ONCE(!per_cpu_ptr(&tmigr_cpu, raw_smp_processor_id())->available); - ret = tmigr_setup_groups(-1, old_root->numa_node, old_root, true); + ret = tmigr_setup_groups(hier, -1, old_root->numa_node, old_root, true); } return ret; @@ -1970,7 +1996,7 @@ static int tmigr_cpu_prepare(unsigned int cpu) static int __init tmigr_init(void) { - unsigned int cpulvl, nodelvl, cpus_per_node, i; + unsigned int cpulvl, nodelvl, cpus_per_node; unsigned int nnodes = num_possible_nodes(); unsigned int ncpus = num_possible_cpus(); int ret = -ENOMEM; @@ -2017,14 +2043,6 @@ static int __init tmigr_init(void) */ tmigr_crossnode_level = cpulvl; - tmigr_level_list = kzalloc_objs(struct list_head, - tmigr_hierarchy_levels); - if (!tmigr_level_list) - goto err; - - for (i = 0; i < tmigr_hierarchy_levels; i++) - INIT_LIST_HEAD(&tmigr_level_list[i]); - pr_info("Timer migration: %d hierarchy levels; %d children per group;" " %d crossnode level\n", tmigr_hierarchy_levels, TMIGR_CHILDREN_PER_GROUP, diff --git a/kernel/time/timer_migration.h b/kernel/time/timer_migration.h index 70879cde6fdd..77df422e5f9a 100644 --- a/kernel/time/timer_migration.h +++ b/kernel/time/timer_migration.h @@ -5,6 +5,16 @@ /* Per group capacity. Must be a power of 2! */ #define TMIGR_CHILDREN_PER_GROUP 8 +/** + * struct tmigr_hierarchy - a hierarchy associated to a given CPU capacity. + * @level_list: Per level lists of tmigr groups + * @root: The current root of the hierarchy + */ +struct tmigr_hierarchy { + struct list_head *level_list; + struct tmigr_group *root; +}; + /** * struct tmigr_event - a timer event associated to a CPU * @nextevt: The node to enqueue an event in the parent group queue -- cgit v1.2.3 From 3ba25488380fd76230442df366c464c6e1fd6485 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Thu, 23 Apr 2026 18:53:51 +0200 Subject: timers/migration: Track CPUs in a hierarchy When a new root is created, the old root is connected to it and propagates up its own assumed to be active state, since the hotplug control CPU is itself active and part of the old root. However with per-capacity hierarchies, this assumption won't be true anymore because the hotplug control CPU calling the timer migration prepare callback may not belong to the same hierarchy as the booting CPU. To solve this, track the available CPUs per hierarchies so that the root connection can be offlined to safe CPUs. Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Link: https://patch.msgid.link/20260423165354.95152-4-frederic@kernel.org --- kernel/time/timer_migration.c | 24 ++++++++++++++++++------ kernel/time/timer_migration.h | 2 ++ 2 files changed, 20 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timer_migration.c b/kernel/time/timer_migration.c index a8ec85f2f5c7..a68b9c746e12 100644 --- a/kernel/time/timer_migration.c +++ b/kernel/time/timer_migration.c @@ -1916,17 +1916,23 @@ static struct tmigr_hierarchy *tmigr_get_hierarchy(void) if (!hierarchy) return ERR_PTR(-ENOMEM); + hierarchy->cpumask = kzalloc(cpumask_size(), GFP_KERNEL); + if (!hierarchy->cpumask) + goto err; + hierarchy->level_list = kzalloc_objs(struct list_head, tmigr_hierarchy_levels); - if (!hierarchy->level_list) { - kfree(hierarchy); - hierarchy = NULL; - return ERR_PTR(-ENOMEM); - } + if (!hierarchy->level_list) + goto err; for (int i = 0; i < tmigr_hierarchy_levels; i++) INIT_LIST_HEAD(&hierarchy->level_list[i]); return hierarchy; +err: + kfree(hierarchy->cpumask); + kfree(hierarchy); + hierarchy = NULL; + return ERR_PTR(-ENOMEM); } static int tmigr_add_cpu(unsigned int cpu) @@ -1946,8 +1952,11 @@ static int tmigr_add_cpu(unsigned int cpu) ret = tmigr_setup_groups(hier, cpu, node, NULL, false); + if (ret < 0) + return ret; + /* Root has changed? Connect the old one to the new */ - if (ret >= 0 && old_root && old_root != hier->root) { + if (old_root && old_root != hier->root) { /* * The target CPU must never do the prepare work, except * on early boot when the boot CPU is the target. Otherwise @@ -1964,6 +1973,9 @@ static int tmigr_add_cpu(unsigned int cpu) ret = tmigr_setup_groups(hier, -1, old_root->numa_node, old_root, true); } + if (ret >= 0) + cpumask_set_cpu(cpu, hier->cpumask); + return ret; } diff --git a/kernel/time/timer_migration.h b/kernel/time/timer_migration.h index 77df422e5f9a..0cfbb8d799a6 100644 --- a/kernel/time/timer_migration.h +++ b/kernel/time/timer_migration.h @@ -8,10 +8,12 @@ /** * struct tmigr_hierarchy - a hierarchy associated to a given CPU capacity. * @level_list: Per level lists of tmigr groups + * @cpumask: CPUs belonging to this hierarchy * @root: The current root of the hierarchy */ struct tmigr_hierarchy { struct list_head *level_list; + struct cpumask *cpumask; struct tmigr_group *root; }; -- cgit v1.2.3 From 098cbaad8e573cf6cac9e68e7ca2e7b7363d2434 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Thu, 23 Apr 2026 18:53:52 +0200 Subject: timers/migration: Split per-capacity hierarchies Systems with heterogeneous CPU capacities, such as big.LITTLE, have reported power issues since the introduction of the new timer migration code. Timers migrate from small capacity CPUs to big ones, degrading their target residency and thus overall power consumption. Solve this with splitting hierarchies per CPU capacity. For example in a big.LITTLE machine, split a single hierarchy in two: one for big capacity CPUs and another one for small capacity CPUs. This way global timers only migrate across CPUs of the same capacity. For simplicity purpose, split hierarchies keep the same number of possible levels as if there were a single hierarchy, even though the CPUs are distributed between multiple hierarchies. This could be a problem on NUMA systems with heterogeneous CPU capacities (provided that ever exists yet) where useless intermediate nodes may be created. Solving this properly will imply on boot to know in advance how many capacities are available and the number of CPUs for each of them. Reported-by: Sehee Jeong Suggested-by: Thomas Gleixner Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Link: https://patch.msgid.link/20260423165354.95152-5-frederic@kernel.org --- kernel/time/timer_migration.c | 123 +++++++++++++++++++++++++++++++----------- kernel/time/timer_migration.h | 7 +++ 2 files changed, 100 insertions(+), 30 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timer_migration.c b/kernel/time/timer_migration.c index a68b9c746e12..03ae8c7dc331 100644 --- a/kernel/time/timer_migration.c +++ b/kernel/time/timer_migration.c @@ -417,7 +417,7 @@ static DEFINE_MUTEX(tmigr_mutex); -static struct tmigr_hierarchy *hierarchy; +static LIST_HEAD(tmigr_hierarchy_list); static unsigned int tmigr_hierarchy_levels __read_mostly; static unsigned int tmigr_crossnode_level __read_mostly; @@ -1889,6 +1889,12 @@ static int tmigr_setup_groups(struct tmigr_hierarchy *hier, unsigned int cpu, data.childmask = start->groupmask; __walk_groups_from(tmigr_active_up, &data, start, start->parent); } + } else if (start) { + union tmigr_state state; + + /* Remote activation assumes the whole target's hierarchy is inactive */ + state.state = atomic_read(&start->migr_state); + WARN_ON_ONCE(state.active); } /* Root update */ @@ -1907,34 +1913,78 @@ out: return err; } -static struct tmigr_hierarchy *tmigr_get_hierarchy(void) +static struct tmigr_hierarchy *tmigr_get_hierarchy(unsigned int capacity) { - if (hierarchy) - return hierarchy; + struct tmigr_hierarchy *hier = NULL, *iter; + + list_for_each_entry(iter, &tmigr_hierarchy_list, node) { + if (iter->capacity == capacity) + hier = iter; + } + + if (hier) + return hier; - hierarchy = kzalloc(sizeof(*hierarchy), GFP_KERNEL); - if (!hierarchy) + hier = kzalloc(sizeof(*hier), GFP_KERNEL); + if (!hier) return ERR_PTR(-ENOMEM); - hierarchy->cpumask = kzalloc(cpumask_size(), GFP_KERNEL); - if (!hierarchy->cpumask) + hier->cpumask = kzalloc(cpumask_size(), GFP_KERNEL); + if (!hier->cpumask) goto err; - hierarchy->level_list = kzalloc_objs(struct list_head, tmigr_hierarchy_levels); - if (!hierarchy->level_list) + hier->level_list = kzalloc_objs(struct list_head, tmigr_hierarchy_levels); + if (!hier->level_list) goto err; for (int i = 0; i < tmigr_hierarchy_levels; i++) - INIT_LIST_HEAD(&hierarchy->level_list[i]); + INIT_LIST_HEAD(&hier->level_list[i]); - return hierarchy; + hier->capacity = capacity; + list_add_tail(&hier->node, &tmigr_hierarchy_list); + + return hier; err: - kfree(hierarchy->cpumask); - kfree(hierarchy); - hierarchy = NULL; + kfree(hier->cpumask); + kfree(hier); return ERR_PTR(-ENOMEM); } +static int tmigr_connect_old_root(struct tmigr_hierarchy *hier, int cpu, + struct tmigr_group *old_root, bool activate) +{ + /* + * The target CPU must never do the prepare work, except + * on early boot when the boot CPU is the target. Otherwise + * it may spuriously activate the old top level group inside + * the new one (nevertheless whether old top level group is + * active or not) and/or release an uninitialized childmask. + */ + WARN_ON_ONCE(cpu == smp_processor_id()); + if (activate) { + /* + * The current CPU is expected to be online in the hierarchy, + * otherwise the old root may not be active as expected. + */ + WARN_ON_ONCE(!__this_cpu_read(tmigr_cpu.available)); + } + + return tmigr_setup_groups(hier, -1, old_root->numa_node, old_root, activate); +} + +static long connect_old_root_work(void *arg) +{ + struct tmigr_group *old_root = arg; + struct tmigr_hierarchy *hier; + int cpu = smp_processor_id(); + + hier = tmigr_get_hierarchy(arch_scale_cpu_capacity(cpu)); + if (IS_ERR(hier)) + return PTR_ERR(hier); + + return tmigr_connect_old_root(hier, cpu, old_root, true); +} + static int tmigr_add_cpu(unsigned int cpu) { struct tmigr_hierarchy *hier; @@ -1944,7 +1994,7 @@ static int tmigr_add_cpu(unsigned int cpu) guard(mutex)(&tmigr_mutex); - hier = tmigr_get_hierarchy(); + hier = tmigr_get_hierarchy(arch_scale_cpu_capacity(cpu)); if (IS_ERR(hier)) return PTR_ERR(hier); @@ -1957,20 +2007,33 @@ static int tmigr_add_cpu(unsigned int cpu) /* Root has changed? Connect the old one to the new */ if (old_root && old_root != hier->root) { - /* - * The target CPU must never do the prepare work, except - * on early boot when the boot CPU is the target. Otherwise - * it may spuriously activate the old top level group inside - * the new one (nevertheless whether old top level group is - * active or not) and/or release an uninitialized childmask. - */ - WARN_ON_ONCE(cpu == raw_smp_processor_id()); - /* - * The (likely) current CPU is expected to be online in the hierarchy, - * otherwise the old root may not be active as expected. - */ - WARN_ON_ONCE(!per_cpu_ptr(&tmigr_cpu, raw_smp_processor_id())->available); - ret = tmigr_setup_groups(hier, -1, old_root->numa_node, old_root, true); + guard(migrate)(); + + if (cpumask_test_cpu(smp_processor_id(), hier->cpumask)) { + /* + * If the target belong to the same hierarchy, the old root is expected + * to be active. Link and propagate to the new root. + */ + ret = tmigr_connect_old_root(hier, cpu, old_root, true); + } else { + int target = cpumask_first_and(hier->cpumask, tmigr_available_cpumask); + + if (target < nr_cpu_ids) { + /* + * If the target doesn't belong to the same hierarchy as the current + * CPU, activate from a relevant one to make sure the old root is + * active. + */ + ret = work_on_cpu(target, connect_old_root_work, old_root); + } else { + /* + * No other available CPUs in the remote hierarchy. Link the + * old root remotely but don't propagate activation since the + * old root is not expected to be active. + */ + ret = tmigr_connect_old_root(hier, cpu, old_root, false); + } + } } if (ret >= 0) diff --git a/kernel/time/timer_migration.h b/kernel/time/timer_migration.h index 0cfbb8d799a6..291bfb6adfc3 100644 --- a/kernel/time/timer_migration.h +++ b/kernel/time/timer_migration.h @@ -7,14 +7,21 @@ /** * struct tmigr_hierarchy - a hierarchy associated to a given CPU capacity. + * Homogeneous systems have only one hierarchy. + * Heterogenous have one hierarchy per CPU capacity. * @level_list: Per level lists of tmigr groups * @cpumask: CPUs belonging to this hierarchy * @root: The current root of the hierarchy + * @capacity: CPU capacity associated to this hierarchy + * @node: Node in the global hierarchy list */ struct tmigr_hierarchy { struct list_head *level_list; struct cpumask *cpumask; struct tmigr_group *root; + unsigned long capacity; + struct list_head node; + }; /** -- cgit v1.2.3 From 5a7dfbcbbdb683e6f704966e73c02f4ba8eb6014 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Thu, 23 Apr 2026 18:53:53 +0200 Subject: timers/migration: Handle capacity in connect tracepoints This let tracers know to which hierarchy a CPU belongs to. Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Link: https://patch.msgid.link/20260423165354.95152-6-frederic@kernel.org --- kernel/time/timer_migration.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timer_migration.c b/kernel/time/timer_migration.c index 03ae8c7dc331..25e3c563eb74 100644 --- a/kernel/time/timer_migration.c +++ b/kernel/time/timer_migration.c @@ -1752,7 +1752,7 @@ static void tmigr_connect_child_parent(struct tmigr_hierarchy *hier, struct tmig */ smp_store_release(&child->parent, parent); - trace_tmigr_connect_child_parent(child); + trace_tmigr_connect_child_parent(hier, child); } static int tmigr_setup_groups(struct tmigr_hierarchy *hier, unsigned int cpu, @@ -1827,7 +1827,7 @@ static int tmigr_setup_groups(struct tmigr_hierarchy *hier, unsigned int cpu, tmigr_init_root(hier, group, activate); - trace_tmigr_connect_cpu_parent(tmc); + trace_tmigr_connect_cpu_parent(hier, tmc); /* There are no children that need to be connected */ continue; -- cgit v1.2.3 From ed3b3c4976686b63b28e44f9805a88abc20ff18a Mon Sep 17 00:00:00 2001 From: Zhan Xusheng Date: Wed, 29 Apr 2026 16:06:35 +0800 Subject: alarmtimer: Remove stale return description from alarm_handle_timer() alarm_handle_timer() was converted from returning enum alarmtimer_restart to void, but the kernel-doc "Return:" line was not removed. Remove the stale description. Fixes: 2634303f8773 ("alarmtimers: Remove return value from alarm functions") Signed-off-by: Zhan Xusheng Signed-off-by: Thomas Gleixner Link: https://patch.msgid.link/20260429080635.166790-1-zhanxusheng@xiaomi.com --- kernel/time/alarmtimer.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'kernel') diff --git a/kernel/time/alarmtimer.c b/kernel/time/alarmtimer.c index 9275fe4cdc1b..ea5be5870e51 100644 --- a/kernel/time/alarmtimer.c +++ b/kernel/time/alarmtimer.c @@ -496,8 +496,6 @@ static enum alarmtimer_type clock2alarm(clockid_t clockid) * @now: time at the timer expiration * * Posix timer callback for expired alarm timers. - * - * Return: whether the timer is to be restarted */ static void alarm_handle_timer(struct alarm *alarm, ktime_t now) { -- cgit v1.2.3 From 33d4bfc49613301c8e451a597e377aaa331944bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Mon, 4 May 2026 08:54:27 +0200 Subject: clocksource: Clean up clocksource_update_freq() functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the unused functions __clocksource_update_freq_hz() and __clocksource_update_freq_khz(). Then make __clocksource_update_freq_scale() static as it is not used from external callers anymore. Also clean up the comment accordingly. Signed-off-by: Thomas Weißschuh Signed-off-by: Thomas Gleixner Acked-by: John Stultz Link: https://patch.msgid.link/20260504-clocksource-update_freq-v2-1-3e696fb01776@linutronix.de --- kernel/time/clocksource.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index baee13a1f87f..f8d15ed3ec98 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -1222,14 +1222,8 @@ static void clocksource_enqueue(struct clocksource *cs) * @cs: clocksource to be registered * @scale: Scale factor multiplied against freq to get clocksource hz * @freq: clocksource frequency (cycles per second) divided by scale - * - * This should only be called from the clocksource->enable() method. - * - * This *SHOULD NOT* be called directly! Please use the - * __clocksource_update_freq_hz() or __clocksource_update_freq_khz() helper - * functions. */ -void __clocksource_update_freq_scale(struct clocksource *cs, u32 scale, u32 freq) +static void __clocksource_update_freq_scale(struct clocksource *cs, u32 scale, u32 freq) { u64 sec; @@ -1287,7 +1281,6 @@ void __clocksource_update_freq_scale(struct clocksource *cs, u32 scale, u32 freq pr_info("%s: mask: 0x%llx max_cycles: 0x%llx, max_idle_ns: %lld ns\n", cs->name, cs->mask, cs->max_cycles, cs->max_idle_ns); } -EXPORT_SYMBOL_GPL(__clocksource_update_freq_scale); /** * __clocksource_register_scale - Used to install new clocksources -- cgit v1.2.3 From 3af1f49f415dcac8c0df8bfc593df0371c219876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Mon, 4 May 2026 08:56:10 +0200 Subject: hrtimer: Return ktime_t from hrtimer_get_next_event()/hrtimer_next_event_without() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These functions really work in terms of ktime_t and not u64. Change their return types and adapt the callers. Signed-off-by: Thomas Weißschuh Signed-off-by: Thomas Gleixner Link: https://patch.msgid.link/20260504-hrtimer-next_event-v2-1-7a5d0550b42f@linutronix.de --- kernel/time/hrtimer.c | 8 ++++---- kernel/time/tick-sched.c | 3 +-- kernel/time/timer.c | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 8cfc7aacece1..8a19a61f6fee 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1783,10 +1783,10 @@ EXPORT_SYMBOL_GPL(__hrtimer_get_remaining); * * Returns the next expiry time or KTIME_MAX if no timer is pending. */ -u64 hrtimer_get_next_event(void) +ktime_t hrtimer_get_next_event(void) { struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); - u64 expires = KTIME_MAX; + ktime_t expires = KTIME_MAX; guard(raw_spinlock_irqsave)(&cpu_base->lock); if (!hrtimer_hres_active(cpu_base)) @@ -1802,10 +1802,10 @@ u64 hrtimer_get_next_event(void) * Returns the next expiry time over all timers except for the @exclude one or * KTIME_MAX if none of them is pending. */ -u64 hrtimer_next_event_without(const struct hrtimer *exclude) +ktime_t hrtimer_next_event_without(const struct hrtimer *exclude) { struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); - u64 expires = KTIME_MAX; + ktime_t expires = KTIME_MAX; unsigned int active; guard(raw_spinlock_irqsave)(&cpu_base->lock); diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index cbbb87a0c6e7..3026a301dff7 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -1407,8 +1407,7 @@ ktime_t tick_nohz_get_sleep_length(ktime_t *delta_next) * If the next highres timer to expire is earlier than 'next_event', the * idle governor needs to know that. */ - next_event = min_t(u64, next_event, - hrtimer_next_event_without(&ts->sched_timer)); + next_event = min(next_event, hrtimer_next_event_without(&ts->sched_timer)); return ktime_sub(next_event, now); } diff --git a/kernel/time/timer.c b/kernel/time/timer.c index 04d928c21aba..655a8c6cd84d 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -1932,7 +1932,7 @@ static void timer_recalc_next_expiry(struct timer_base *base) */ static u64 cmp_next_hrtimer_event(u64 basem, u64 expires) { - u64 nextevt = hrtimer_get_next_event(); + u64 nextevt = ktime_to_ns(hrtimer_get_next_event()); /* * If high resolution timers are enabled -- cgit v1.2.3 From 7fcb7afb9b5eecb85904b16747b6eb04b0880c3b Mon Sep 17 00:00:00 2001 From: Marco Crivellari Date: Tue, 5 May 2026 14:41:02 +0200 Subject: umh: replace use of system_unbound_wq with system_dfl_wq This patch continues the effort to refactor workqueue APIs, which has begun with the changes introducing new workqueues and a new alloc_workqueue flag: commit 128ea9f6ccfb ("workqueue: Add system_percpu_wq and system_dfl_wq") commit 930c2ea566af ("workqueue: Add new WQ_PERCPU flag") The point of the refactoring is to eventually alter the default behavior of workqueues to become unbound by default so that their workload placement is optimized by the scheduler. Before that to happen, workqueue users must be converted to the better named new workqueues with no intended behaviour changes: system_wq -> system_percpu_wq system_unbound_wq -> system_dfl_wq This way the old obsolete workqueues (system_wq, system_unbound_wq) can be removed in the future. Cc: Luis Chamberlain Link: https://lore.kernel.org/all/20250221112003.1dSuoGyc@linutronix.de/ Suggested-by: Tejun Heo Signed-off-by: Marco Crivellari Signed-off-by: Tejun Heo Reviewed-by: Frederic Weisbecker --- kernel/umh.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/umh.c b/kernel/umh.c index cffda97d961c..48117c569e1a 100644 --- a/kernel/umh.c +++ b/kernel/umh.c @@ -430,7 +430,7 @@ int call_usermodehelper_exec(struct subprocess_info *sub_info, int wait) sub_info->complete = (wait == UMH_NO_WAIT) ? NULL : &done; sub_info->wait = wait; - queue_work(system_unbound_wq, &sub_info->work); + queue_work(system_dfl_wq, &sub_info->work); if (wait == UMH_NO_WAIT) /* task has freed sub_info */ goto unlock; -- cgit v1.2.3 From ca1d48a86fab82da94cf0ddf586b484dcd04df6e Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 8 May 2026 05:50:08 -1000 Subject: sched_ext: Use offsetofend on both sides of the ops_cid layout assert sizeof() includes trailing struct pad, offsetofend() doesn't. On 32-bit PPC, sched_ext_ops_cid tail-pads 4 bytes past @priv and the assert trips. Use offsetofend() on both sides. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202605081637.DbH4SZ1E-lkp@intel.com/ Fixes: 7e655ed7b953 ("sched_ext: Add bpf_sched_ext_ops_cid struct_ops type") Signed-off-by: Tejun Heo --- kernel/sched/ext.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 7ac7d10a41be..f86ee15be7cb 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -10380,9 +10380,11 @@ static int __init scx_init(void) /* * cid-form must end exactly at @priv - validate_ops() skips * cpu_acquire/cpu_release for cid-form because reading those fields - * past the BPF allocation would be UB. + * past the BPF allocation would be UB. offsetofend() on both sides + * instead of sizeof() on sched_ext_ops_cid to sidestep trailing + * struct padding (e.g. 32-bit PPC tail-pads ops_cid past @priv). */ - BUILD_BUG_ON(sizeof(struct sched_ext_ops_cid) != + BUILD_BUG_ON(offsetofend(struct sched_ext_ops_cid, priv) != offsetofend(struct sched_ext_ops, priv)); #undef CID_OFFSET_MATCH -- cgit v1.2.3 From 5da80b8fb38006c7e79b06cba711c28219f17ab2 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 1 May 2026 09:35:07 +0300 Subject: dma-debug: Remove unused DMA attribute parameter debug_dma_alloc_pages() always receives a DMA attribute value of 0, because dma_alloc_pages() never receives any attributes from its callers. As preparation for upcoming patches, remove this unused attribute from the debug routine. Signed-off-by: Leon Romanovsky Reviewed-by: Samiullah Khawaja Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/20260501-dma-attrs-debug-v2-3-8dbac75cd501@nvidia.com --- kernel/dma/debug.c | 5 ++--- kernel/dma/debug.h | 6 ++---- kernel/dma/mapping.c | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c index 1a725edbbbf6..3b53495337f5 100644 --- a/kernel/dma/debug.c +++ b/kernel/dma/debug.c @@ -1567,8 +1567,7 @@ void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, void debug_dma_alloc_pages(struct device *dev, struct page *page, size_t size, int direction, - dma_addr_t dma_addr, - unsigned long attrs) + dma_addr_t dma_addr) { struct dma_debug_entry *entry; @@ -1586,7 +1585,7 @@ void debug_dma_alloc_pages(struct device *dev, struct page *page, entry->dev_addr = dma_addr; entry->direction = direction; - add_dma_entry(entry, attrs); + add_dma_entry(entry, 0); } void debug_dma_free_pages(struct device *dev, struct page *page, diff --git a/kernel/dma/debug.h b/kernel/dma/debug.h index da7be0bddcf6..24b8610850fb 100644 --- a/kernel/dma/debug.h +++ b/kernel/dma/debug.h @@ -47,8 +47,7 @@ extern void debug_dma_sync_sg_for_device(struct device *dev, int nelems, int direction); extern void debug_dma_alloc_pages(struct device *dev, struct page *page, size_t size, int direction, - dma_addr_t dma_addr, - unsigned long attrs); + dma_addr_t dma_addr); extern void debug_dma_free_pages(struct device *dev, struct page *page, size_t size, int direction, dma_addr_t dma_addr); @@ -113,8 +112,7 @@ static inline void debug_dma_sync_sg_for_device(struct device *dev, static inline void debug_dma_alloc_pages(struct device *dev, struct page *page, size_t size, int direction, - dma_addr_t dma_addr, - unsigned long attrs) + dma_addr_t dma_addr) { } diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index 23ed8eb9233e..6cbefbd4158c 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -733,7 +733,7 @@ struct page *dma_alloc_pages(struct device *dev, size_t size, if (page) { trace_dma_alloc_pages(dev, page_to_virt(page), *dma_handle, size, dir, gfp, 0); - debug_dma_alloc_pages(dev, page, size, dir, *dma_handle, 0); + debug_dma_alloc_pages(dev, page, size, dir, *dma_handle); } else { trace_dma_alloc_pages(dev, NULL, 0, size, dir, gfp, 0); } -- cgit v1.2.3 From 04d492ab964a6256a93ed11cdf99b126281e793a Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 1 May 2026 09:35:08 +0300 Subject: dma-debug: Record DMA attributes in debug entry To enable reliable comparison of DMA attributes between map and unmap operations, store the attribute value in dma_debug_entry. Signed-off-by: Leon Romanovsky Reviewed-by: Samiullah Khawaja Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/20260501-dma-attrs-debug-v2-4-8dbac75cd501@nvidia.com --- kernel/dma/debug.c | 48 +++++++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 19 deletions(-) (limited to 'kernel') diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c index 3b53495337f5..f07e6a1e9fba 100644 --- a/kernel/dma/debug.c +++ b/kernel/dma/debug.c @@ -63,7 +63,7 @@ enum map_err_types { * @sg_mapped_ents: 'mapped_ents' from dma_map_sg * @paddr: physical start address of the mapping * @map_err_type: track whether dma_mapping_error() was checked - * @is_cache_clean: driver promises not to write to buffer while mapped + * @attrs: dma attributes * @stack_len: number of backtrace entries in @stack_entries * @stack_entries: stack of backtrace history */ @@ -78,7 +78,7 @@ struct dma_debug_entry { int sg_mapped_ents; phys_addr_t paddr; enum map_err_types map_err_type; - bool is_cache_clean; + unsigned long attrs; #ifdef CONFIG_STACKTRACE unsigned int stack_len; unsigned long stack_entries[DMA_DEBUG_STACKTRACE_ENTRIES]; @@ -478,6 +478,9 @@ static int active_cacheline_insert(struct dma_debug_entry *entry, bool *overlap_cache_clean) { phys_addr_t cln = to_cacheline_number(entry); + bool is_cache_clean = entry->attrs & + (DMA_ATTR_DEBUGGING_IGNORE_CACHELINES | + DMA_ATTR_REQUIRE_COHERENT); unsigned long flags; int rc; @@ -495,12 +498,15 @@ static int active_cacheline_insert(struct dma_debug_entry *entry, if (rc == -EEXIST) { struct dma_debug_entry *existing; - active_cacheline_inc_overlap(cln, entry->is_cache_clean); + active_cacheline_inc_overlap(cln, is_cache_clean); existing = radix_tree_lookup(&dma_active_cacheline, cln); /* A lookup failure here after we got -EEXIST is unexpected. */ WARN_ON(!existing); if (existing) - *overlap_cache_clean = existing->is_cache_clean; + *overlap_cache_clean = + existing->attrs & + (DMA_ATTR_DEBUGGING_IGNORE_CACHELINES | + DMA_ATTR_REQUIRE_COHERENT); } spin_unlock_irqrestore(&radix_lock, flags); @@ -544,12 +550,13 @@ void debug_dma_dump_mappings(struct device *dev) if (!dev || dev == entry->dev) { cln = to_cacheline_number(entry); dev_info(entry->dev, - "%s idx %d P=%pa D=%llx L=%llx cln=%pa %s %s\n", + "%s idx %d P=%pa D=%llx L=%llx cln=%pa %s %s attrs=0x%lx\n", type2name[entry->type], idx, &entry->paddr, entry->dev_addr, entry->size, &cln, dir2name[entry->direction], - maperr2str[entry->map_err_type]); + maperr2str[entry->map_err_type], + entry->attrs); } } spin_unlock_irqrestore(&bucket->lock, flags); @@ -575,14 +582,15 @@ static int dump_show(struct seq_file *seq, void *v) list_for_each_entry(entry, &bucket->list, list) { cln = to_cacheline_number(entry); seq_printf(seq, - "%s %s %s idx %d P=%pa D=%llx L=%llx cln=%pa %s %s\n", + "%s %s %s idx %d P=%pa D=%llx L=%llx cln=%pa %s %s attrs=0x%lx\n", dev_driver_string(entry->dev), dev_name(entry->dev), type2name[entry->type], idx, &entry->paddr, entry->dev_addr, entry->size, &cln, dir2name[entry->direction], - maperr2str[entry->map_err_type]); + maperr2str[entry->map_err_type], + entry->attrs); } spin_unlock_irqrestore(&bucket->lock, flags); } @@ -594,16 +602,14 @@ DEFINE_SHOW_ATTRIBUTE(dump); * Wrapper function for adding an entry to the hash. * This function takes care of locking itself. */ -static void add_dma_entry(struct dma_debug_entry *entry, unsigned long attrs) +static void add_dma_entry(struct dma_debug_entry *entry) { + unsigned long attrs = entry->attrs; bool overlap_cache_clean; struct hash_bucket *bucket; unsigned long flags; int rc; - entry->is_cache_clean = attrs & (DMA_ATTR_DEBUGGING_IGNORE_CACHELINES | - DMA_ATTR_REQUIRE_COHERENT); - bucket = get_hash_bucket(entry, &flags); hash_bucket_add(bucket, entry); put_hash_bucket(bucket, flags); @@ -612,9 +618,10 @@ static void add_dma_entry(struct dma_debug_entry *entry, unsigned long attrs) if (rc == -ENOMEM) { pr_err_once("cacheline tracking ENOMEM, dma-debug disabled\n"); global_disable = true; - } else if (rc == -EEXIST && - !(attrs & DMA_ATTR_SKIP_CPU_SYNC) && - !(entry->is_cache_clean && overlap_cache_clean) && + } else if (rc == -EEXIST && !(attrs & DMA_ATTR_SKIP_CPU_SYNC) && + !(attrs & (DMA_ATTR_DEBUGGING_IGNORE_CACHELINES | + DMA_ATTR_REQUIRE_COHERENT) && + overlap_cache_clean) && dma_get_cache_alignment() >= L1_CACHE_BYTES && !(IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) && is_swiotlb_active(entry->dev))) { @@ -1250,6 +1257,7 @@ void debug_dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, entry->size = size; entry->direction = direction; entry->map_err_type = MAP_ERR_NOT_CHECKED; + entry->attrs = attrs; if (!(attrs & DMA_ATTR_MMIO)) { check_for_stack(dev, phys); @@ -1258,7 +1266,7 @@ void debug_dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, check_for_illegal_area(dev, phys_to_virt(phys), size); } - add_dma_entry(entry, attrs); + add_dma_entry(entry); } void debug_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) @@ -1345,10 +1353,11 @@ void debug_dma_map_sg(struct device *dev, struct scatterlist *sg, entry->direction = direction; entry->sg_call_ents = nents; entry->sg_mapped_ents = mapped_ents; + entry->attrs = attrs; check_sg_segment(dev, s); - add_dma_entry(entry, attrs); + add_dma_entry(entry); } } @@ -1440,8 +1449,9 @@ void debug_dma_alloc_coherent(struct device *dev, size_t size, entry->size = size; entry->dev_addr = dma_addr; entry->direction = DMA_BIDIRECTIONAL; + entry->attrs = attrs; - add_dma_entry(entry, attrs); + add_dma_entry(entry); } void debug_dma_free_coherent(struct device *dev, size_t size, @@ -1585,7 +1595,7 @@ void debug_dma_alloc_pages(struct device *dev, struct page *page, entry->dev_addr = dma_addr; entry->direction = direction; - add_dma_entry(entry, 0); + add_dma_entry(entry); } void debug_dma_free_pages(struct device *dev, struct page *page, -- cgit v1.2.3 From c8411b1d1e524cbe4a12aacad7bf2163fb2be062 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 1 May 2026 09:35:09 +0300 Subject: dma-debug: Feed DMA attribute for unmapping flows too There are multiple unmapping flows which didn't provide DMA attributes, which limited DMA debug code to compare the mapping and unmapping attributes. Let's fix it. Signed-off-by: Leon Romanovsky Reviewed-by: Samiullah Khawaja Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/20260501-dma-attrs-debug-v2-5-8dbac75cd501@nvidia.com --- kernel/dma/debug.c | 13 ++++++++----- kernel/dma/debug.h | 19 +++++++++++-------- kernel/dma/mapping.c | 8 ++++---- 3 files changed, 23 insertions(+), 17 deletions(-) (limited to 'kernel') diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c index f07e6a1e9fba..3dfed51c3d9a 100644 --- a/kernel/dma/debug.c +++ b/kernel/dma/debug.c @@ -1307,8 +1307,8 @@ void debug_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) } EXPORT_SYMBOL(debug_dma_mapping_error); -void debug_dma_unmap_phys(struct device *dev, dma_addr_t dma_addr, - size_t size, int direction) +void debug_dma_unmap_phys(struct device *dev, dma_addr_t dma_addr, size_t size, + int direction, unsigned long attrs) { struct dma_debug_entry ref = { .type = dma_debug_phy, @@ -1316,6 +1316,7 @@ void debug_dma_unmap_phys(struct device *dev, dma_addr_t dma_addr, .dev_addr = dma_addr, .size = size, .direction = direction, + .attrs = attrs, }; if (unlikely(dma_debug_disabled())) @@ -1381,7 +1382,7 @@ static int get_nr_mapped_entries(struct device *dev, } void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, - int nelems, int dir) + int nelems, int dir, unsigned long attrs) { struct scatterlist *s; int mapped_ents = 0, i; @@ -1399,6 +1400,7 @@ void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, .size = sg_dma_len(s), .direction = dir, .sg_call_ents = nelems, + .attrs = attrs, }; if (mapped_ents && i >= mapped_ents) @@ -1454,8 +1456,8 @@ void debug_dma_alloc_coherent(struct device *dev, size_t size, add_dma_entry(entry); } -void debug_dma_free_coherent(struct device *dev, size_t size, - void *virt, dma_addr_t dma_addr) +void debug_dma_free_coherent(struct device *dev, size_t size, void *virt, + dma_addr_t dma_addr, unsigned long attrs) { struct dma_debug_entry ref = { .type = dma_debug_coherent, @@ -1463,6 +1465,7 @@ void debug_dma_free_coherent(struct device *dev, size_t size, .dev_addr = dma_addr, .size = size, .direction = DMA_BIDIRECTIONAL, + .attrs = attrs, }; /* handle vmalloc and linear addresses */ diff --git a/kernel/dma/debug.h b/kernel/dma/debug.h index 24b8610850fb..13e384633c32 100644 --- a/kernel/dma/debug.h +++ b/kernel/dma/debug.h @@ -14,21 +14,22 @@ extern void debug_dma_map_phys(struct device *dev, phys_addr_t phys, unsigned long attrs); extern void debug_dma_unmap_phys(struct device *dev, dma_addr_t addr, - size_t size, int direction); + size_t size, int direction, + unsigned long attrs); extern void debug_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, int mapped_ents, int direction, unsigned long attrs); extern void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, - int nelems, int dir); + int nelems, int dir, unsigned long attrs); extern void debug_dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t dma_addr, void *virt, unsigned long attrs); -extern void debug_dma_free_coherent(struct device *dev, size_t size, - void *virt, dma_addr_t addr); +extern void debug_dma_free_coherent(struct device *dev, size_t size, void *virt, + dma_addr_t addr, unsigned long attrs); extern void debug_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, size_t size, @@ -59,7 +60,8 @@ static inline void debug_dma_map_phys(struct device *dev, phys_addr_t phys, } static inline void debug_dma_unmap_phys(struct device *dev, dma_addr_t addr, - size_t size, int direction) + size_t size, int direction, + unsigned long attrs) { } @@ -70,8 +72,8 @@ static inline void debug_dma_map_sg(struct device *dev, struct scatterlist *sg, } static inline void debug_dma_unmap_sg(struct device *dev, - struct scatterlist *sglist, - int nelems, int dir) + struct scatterlist *sglist, int nelems, + int dir, unsigned long attrs) { } @@ -82,7 +84,8 @@ static inline void debug_dma_alloc_coherent(struct device *dev, size_t size, } static inline void debug_dma_free_coherent(struct device *dev, size_t size, - void *virt, dma_addr_t addr) + void *virt, dma_addr_t addr, + unsigned long attrs) { } diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index 6cbefbd4158c..f010b3cc0ece 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -225,7 +225,7 @@ void dma_unmap_phys(struct device *dev, dma_addr_t addr, size_t size, else if (ops->unmap_phys) ops->unmap_phys(dev, addr, size, dir, attrs); trace_dma_unmap_phys(dev, addr, size, dir, attrs); - debug_dma_unmap_phys(dev, addr, size, dir); + debug_dma_unmap_phys(dev, addr, size, dir, attrs); } EXPORT_SYMBOL_GPL(dma_unmap_phys); @@ -351,7 +351,7 @@ void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg, BUG_ON(!valid_dma_direction(dir)); trace_dma_unmap_sg(dev, sg, nents, dir, attrs); - debug_dma_unmap_sg(dev, sg, nents, dir); + debug_dma_unmap_sg(dev, sg, nents, dir, attrs); if (dma_map_direct(dev, ops) || arch_dma_unmap_sg_direct(dev, sg, nents)) dma_direct_unmap_sg(dev, sg, nents, dir, attrs); @@ -693,7 +693,7 @@ void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, if (!cpu_addr) return; - debug_dma_free_coherent(dev, size, cpu_addr, dma_handle); + debug_dma_free_coherent(dev, size, cpu_addr, dma_handle, attrs); if (dma_alloc_direct(dev, ops) || arch_dma_free_direct(dev, dma_handle)) dma_direct_free(dev, size, cpu_addr, dma_handle, attrs); else if (use_dma_iommu(dev)) @@ -840,7 +840,7 @@ void dma_free_noncontiguous(struct device *dev, size_t size, struct sg_table *sgt, enum dma_data_direction dir) { trace_dma_free_sgt(dev, sgt, size, dir); - debug_dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir); + debug_dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir, 0); if (use_dma_iommu(dev)) iommu_dma_free_noncontiguous(dev, size, sgt, dir); -- cgit v1.2.3 From a34bd60bdaebb1db4631cad45ff87dcabf4fc962 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 1 May 2026 09:35:10 +0300 Subject: dma-debug: Ensure mappings are created and released with matching attributes The DMA API expects that callers use the same attributes when mapping and unmapping. Add tracking to verify this and catch mismatches. Signed-off-by: Leon Romanovsky Reviewed-by: Samiullah Khawaja Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/20260501-dma-attrs-debug-v2-6-8dbac75cd501@nvidia.com --- kernel/dma/debug.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'kernel') diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c index 3dfed51c3d9a..c38efc1ac8d6 100644 --- a/kernel/dma/debug.c +++ b/kernel/dma/debug.c @@ -1074,6 +1074,29 @@ static void check_unmap(struct dma_debug_entry *ref) type2name[entry->type]); } + /* + * This may be no bug in reality - but DMA API still expects + * that entry is unmapped with same attributes as it was mapped. + * + * DMA_ATTR_UNMAP_VALID lists the attributes that must be identical + * between map and unmap. Any attribute outside this set (e.g. + * DMA_ATTR_NO_WARN, DMA_ATTR_SKIP_CPU_SYNC) is allowed to differ. + */ +#define DMA_ATTR_UNMAP_VALID \ + (DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_FORCE_CONTIGUOUS | \ + DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT | DMA_ATTR_PRIVILEGED | \ + DMA_ATTR_CC_SHARED) + if ((ref->attrs & DMA_ATTR_UNMAP_VALID) != + (entry->attrs & DMA_ATTR_UNMAP_VALID)) { + err_printk(ref->dev, entry, + "device driver frees " + "DMA memory with different attributes " + "[device address=0x%016llx] [size=%llu bytes] " + "[mapped with 0x%lx] [unmapped with 0x%lx]\n", + ref->dev_addr, ref->size, entry->attrs, ref->attrs); + } +#undef DMA_ATTR_UNMAP_VALID + hash_bucket_del(entry); put_hash_bucket(bucket, flags); -- cgit v1.2.3 From 03d958da4f355c380819a3baf339123c25db8e54 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 8 May 2026 13:48:29 -1000 Subject: sched_ext: Fix ops_cid layout assert ca1d48a86fab ("sched_ext: Use offsetofend on both sides of the ops_cid layout assert") replaced sizeof() with offsetofend() to dodge 32-bit PPC trailing padding, but the resulting check is tautological: with CID_OFFSET_MATCH(priv, priv) already enforcing offsetof(priv) equality and @priv being the same type in both structs, the two offsetofends are equal by construction. The original protection - catching a stray field added past @priv in sched_ext_ops_cid - is gone. Anchor on a zero-size __end[] marker appended after @priv. Its offset sits flush after @priv regardless of trailing struct padding; if a field is inserted past @priv, __end shifts and the assert fires. Closes: https://lore.kernel.org/all/20260508215211.0C03AC2BCB0@smtp.kernel.org/ Signed-off-by: Tejun Heo Reviewed-by: Emil Tsalapatis --- kernel/sched/ext.c | 6 ++---- kernel/sched/ext_internal.h | 3 +++ 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index f86ee15be7cb..b685f45b4fd0 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -10380,11 +10380,9 @@ static int __init scx_init(void) /* * cid-form must end exactly at @priv - validate_ops() skips * cpu_acquire/cpu_release for cid-form because reading those fields - * past the BPF allocation would be UB. offsetofend() on both sides - * instead of sizeof() on sched_ext_ops_cid to sidestep trailing - * struct padding (e.g. 32-bit PPC tail-pads ops_cid past @priv). + * past the BPF allocation would be UB. */ - BUILD_BUG_ON(offsetofend(struct sched_ext_ops_cid, priv) != + BUILD_BUG_ON(offsetof(struct sched_ext_ops_cid, __end) != offsetofend(struct sched_ext_ops, priv)); #undef CID_OFFSET_MATCH diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext_internal.h index 0ed79bd891c7..cd33984cffcf 100644 --- a/kernel/sched/ext_internal.h +++ b/kernel/sched/ext_internal.h @@ -938,6 +938,9 @@ struct sched_ext_ops_cid { /* internal use only, must be NULL */ void __rcu *priv; + + /* layout end anchor for the BUILD_BUG_ON in scx_init(); keep last */ + char __end[0]; }; enum scx_opi { -- cgit v1.2.3 From f25ad1e3cbaa4c87bb2b11496786f79db54c294f Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sun, 10 May 2026 12:21:58 -1000 Subject: sched_ext: Add scx_task_iter_relock() and use it in scx_root_enable_workfn() scx_root_enable_workfn()'s post-init block re-acquires scx_tasks_lock briefly via a scoped_guard() for the tid hash insertion. c941d7391f25 ("sched_ext: Close root-enable vs sched_ext_dead() race with SCX_TASK_INIT_BEGIN") on for-7.1-fixes adds a post-init DEAD recheck that holds the task's rq lock across the state-machine updates in the same region. A naive merge would acquire scx_tasks_lock while the rq lock is held, inverting the iter's outer/inner order (scx_tasks_lock then rq lock). Add scx_task_iter_relock(iter, p), the counterpart to scx_task_iter_unlock(), that re-acquires scx_tasks_lock and, if @p is non-NULL, @p's rq lock. The locks are tracked in @iter so subsequent iteration releases them. Use it in scx_root_enable_workfn()'s post-init block and drop the now-redundant scoped_guard on the hash insertion. The post-init region now runs with both scx_tasks_lock and the task's rq lock held across the init failure check, the state-machine updates and the hash insert. v2: Move scx_task_iter_relock() earlier to ease the for-7.1-fixes merge. Signed-off-by: Tejun Heo --- kernel/sched/ext.c | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index b685f45b4fd0..fbdc1819d4cb 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -902,6 +902,24 @@ static void __scx_task_iter_maybe_relock(struct scx_task_iter *iter) } } +/** + * scx_task_iter_relock - Re-acquire scx_tasks_lock and, optionally, @p's rq + * @iter: iterator to relock + * @p: task whose rq to lock, or %NULL for scx_tasks_lock only + * + * Counterpart to scx_task_iter_unlock(). Locking @p's rq is optional. Once + * re-acquired, both locks are managed by the iterator from here on. + */ +static void scx_task_iter_relock(struct scx_task_iter *iter, + struct task_struct *p) +{ + __scx_task_iter_maybe_relock(iter); + if (p) { + iter->rq = task_rq_lock(p, &iter->rf); + iter->locked_task = p; + } +} + /** * scx_task_iter_stop - Stop a task iteration and unlock scx_tasks_lock * @iter: iterator to exit @@ -7064,6 +7082,9 @@ static void scx_root_enable_workfn(struct kthread_work *work) scx_task_iter_unlock(&sti); ret = scx_init_task(sch, p, false); + + scx_task_iter_relock(&sti, p); + if (ret) { put_task_struct(p); scx_task_iter_stop(&sti); @@ -7076,15 +7097,12 @@ static void scx_root_enable_workfn(struct kthread_work *work) scx_set_task_state(p, SCX_TASK_READY); /* - * Insert into the tid hash under scx_tasks_lock so we can't - * race sched_ext_dead() and leave a stale entry for an already - * exited task. + * Insert into the tid hash. scx_tasks_lock is held by the iter; + * list_empty() guards against sched_ext_dead() having taken @p + * off the list while init ran unlocked. */ - if (scx_tid_to_task_enabled()) { - guard(raw_spinlock_irq)(&scx_tasks_lock); - if (!list_empty(&p->scx.tasks_node)) - scx_tid_hash_insert(p); - } + if (scx_tid_to_task_enabled() && !list_empty(&p->scx.tasks_node)) + scx_tid_hash_insert(p); put_task_struct(p); } -- cgit v1.2.3 From be2689c9ed47bde6f15a24c9845f6fc1ec52502f Mon Sep 17 00:00:00 2001 From: Chen Wandun Date: Mon, 11 May 2026 16:18:38 +0800 Subject: cgroup/cpuset: Skip hardwall ancestor scan in cpuset v2 in cpuset_current_node_allowed() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cgroup v2 doesn't have the concept of memory hardwall, only top_cpuset has CS_MEM_EXCLUSIVE/CS_MEM_HARDWALL flags, nearest_hardwall_ancestor always returns top_cpuset with all nodes set, so no need to acquire callback_lock and scan up cpuset. Suggested-by: Michal Koutný Signed-off-by: Chen Wandun Signed-off-by: Tejun Heo --- kernel/cgroup/cpuset.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'kernel') diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 3fbf6e7f68c3..74d5c494d6ae 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -4231,6 +4231,9 @@ bool cpuset_current_node_allowed(int node, gfp_t gfp_mask) if (gfp_mask & __GFP_HARDWALL) /* If hardwall request, stop here */ return false; + if (cpuset_v2()) + return true; + /* Not hardwall and node outside mems_allowed: scan up cpusets */ spin_lock_irqsave(&callback_lock, flags); -- cgit v1.2.3 From c2c7983c93f5d86962318be7e7298f1bc3feb1a6 Mon Sep 17 00:00:00 2001 From: Pengpeng Hou Date: Fri, 3 Apr 2026 16:55:56 +0800 Subject: genirq/proc: Size interrupt directory names for 10-digit interrupt numbers /proc/irq// directory names are built in `char name[10]` buffers with `sprintf(name, "%u", irq)`. Ten-digit IRQ numbers already need 11 bytes including the trailing NUL, and current sparse-IRQ configurations allow interrupt numbers in that range. Size the temporary name buffer for the current decimal form and switch to bounded formatting when creating or removing the proc entry. Signed-off-by: Pengpeng Hou Signed-off-by: Thomas Gleixner Link: https://patch.msgid.link/20260404101001.1-genirq-proc-pengpeng@iscas.ac.cn --- kernel/irq/proc.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index b0999a4f1f68..dfa0b0723642 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -326,7 +327,7 @@ void register_handler_proc(unsigned int irq, struct irqaction *action) #undef MAX_NAMELEN -#define MAX_NAMELEN 10 +#define MAX_NAMELEN 11 void register_irq_proc(unsigned int irq, struct irq_desc *desc) { @@ -348,7 +349,7 @@ void register_irq_proc(unsigned int irq, struct irq_desc *desc) return; /* create /proc/irq/1234 */ - sprintf(name, "%u", irq); + snprintf(name, MAX_NAMELEN, "%u", irq); desc->dir = proc_mkdir(name, root_irq_dir); if (!desc->dir) return; @@ -401,7 +402,7 @@ void unregister_irq_proc(unsigned int irq, struct irq_desc *desc) #endif remove_proc_entry("spurious", desc->dir); - sprintf(name, "%u", irq); + snprintf(name, MAX_NAMELEN, "%u", irq); remove_proc_entry(name, root_irq_dir); } -- cgit v1.2.3 From 4314a44564eb1565349fed7a4192344c5f46fc85 Mon Sep 17 00:00:00 2001 From: Yazhou Tang Date: Wed, 6 May 2026 17:47:12 +0800 Subject: bpf: Fix out-of-bounds read in bpf_patch_call_args() The interpreters_args array only accommodates stack depths up to MAX_BPF_STACK (512 bytes). However, do_misc_fixups() may allow a larger stack depth if JIT is requested. If JIT compilation later fails and falls back to the interpreter, the verifier invokes bpf_patch_call_args() with this oversized stack depth. This causes a load-time out-of-bounds (OOB) read when calculating the interpreter function pointer index. Fix this by changing bpf_patch_call_args() to return an int and explicitly rejecting the JIT fallback (returning -EINVAL) if the stack depth exceeds MAX_BPF_STACK. Fixes: 1ea47e01ad6e ("bpf: add support for bpf_call to interpreter") Co-developed-by: Tianci Cao Signed-off-by: Tianci Cao Co-developed-by: Shenghao Yuan Signed-off-by: Shenghao Yuan Signed-off-by: Yazhou Tang Acked-by: Xu Kuohai Link: https://lore.kernel.org/r/20260506094714.419842-2-tangyazhou@zju.edu.cn Signed-off-by: Alexei Starovoitov --- kernel/bpf/core.c | 6 +++++- kernel/bpf/fixups.c | 7 ++++++- 2 files changed, 11 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 8b018ff48875..63044ebe5721 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -2394,13 +2394,17 @@ EVAL4(PROG_NAME_LIST, 416, 448, 480, 512) #undef PROG_NAME_LIST #ifdef CONFIG_BPF_SYSCALL -void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth) +int bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth) { stack_depth = max_t(u32, stack_depth, 1); + /* Prevent out-of-bounds read to interpreters_args */ + if (stack_depth > MAX_BPF_STACK) + return -EINVAL; insn->off = (s16) insn->imm; insn->imm = interpreters_args[(round_up(stack_depth, 32) / 32) - 1] - __bpf_call_base_args; insn->code = BPF_JMP | BPF_CALL_ARGS; + return 0; } #endif #endif diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index fba9e8c00878..df8f48091321 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -1416,7 +1416,12 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env) depth = get_callee_stack_depth(env, insn, i); if (depth < 0) return depth; - bpf_patch_call_args(insn, depth); + err = bpf_patch_call_args(insn, depth); + if (err) { + verbose(env, "stack depth %d exceeds interpreter stack depth limit\n", + depth); + return err; + } } err = 0; #endif -- cgit v1.2.3 From 58a8f3e2501dc14b8e00e883d6aaf0600a239da7 Mon Sep 17 00:00:00 2001 From: Yazhou Tang Date: Wed, 6 May 2026 17:47:13 +0800 Subject: bpf: Fix s16 truncation for large bpf-to-bpf call offsets Currently, the BPF instruction set allows bpf-to-bpf calls (or internal calls, pseudo calls) to use a 32-bit imm field to represent the relative jump offset. However, when JIT is disabled or falls back to the interpreter, the verifier invokes bpf_patch_call_args() to rewrite the call instruction. In this function, the 32-bit imm is downcast to s16 and stored in the off field. void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth) { stack_depth = max_t(u32, stack_depth, 1); insn->off = (s16) insn->imm; insn->imm = interpreters_args[(round_up(stack_depth, 32) / 32) - 1] - __bpf_call_base_args; insn->code = BPF_JMP | BPF_CALL_ARGS; } If the original imm exceeds the s16 range (i.e., a jump offset greater than 32767 instructions), this downcast silently truncates the offset, resulting in an incorrect call target. Fix this by: 1. In bpf_patch_call_args(), keeping the imm field unchanged and using the off field to store the index of the interpreter function. 2. In ___bpf_prog_run() for the JMP_CALL_ARGS case, retrieving the interpreter function pointer from the interpreters_args array using the off field as the index, and passing the original imm to calculate the last argument of the interpreter function. After these changes, the truncation issue is resolved, and __bpf_call_base_args is also no longer needed and can be removed, which makes the code cleaner. Performance: In ___bpf_prog_run() for the JMP_CALL_ARGS case, changing the retrieval of the interpreter function pointer from pointer addition to direct array indexing improves performance. The possible reason is that the latter has better instruction-level parallelism. See the v5 discussion [1] for more details. [1] https://lore.kernel.org/bpf/f120c3c4-6999-414a-b514-518bb64b4758@zju.edu.cn/ To avoid requiring bpftool changes, keep the new imm/off encoding internal and restore the legacy xlated dump layout in bpf_insn_prepare_dump(). For bpf-to-bpf call offsets that do not fit in s16, export off as 0 instead of a truncated and misleading value. Fixes: 1ea47e01ad6e ("bpf: add support for bpf_call to interpreter") Fixes: 7105e828c087 ("bpf: allow for correlation of maps and helpers in dump") Suggested-by: Xu Kuohai Suggested-by: Puranjay Mohan Co-developed-by: Tianci Cao Signed-off-by: Tianci Cao Co-developed-by: Shenghao Yuan Signed-off-by: Shenghao Yuan Signed-off-by: Yazhou Tang Link: https://lore.kernel.org/r/20260506094714.419842-3-tangyazhou@zju.edu.cn Signed-off-by: Alexei Starovoitov --- kernel/bpf/core.c | 21 ++++++++++++++------- kernel/bpf/fixups.c | 6 +++--- kernel/bpf/syscall.c | 26 ++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 63044ebe5721..6aa2a8b24030 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -1771,6 +1771,9 @@ static u32 abs_s32(s32 x) return x >= 0 ? (u32)x : -(u32)x; } +static u64 (*interpreters_args[])(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5, + const struct bpf_insn *insn); + /** * ___bpf_prog_run - run eBPF program on a given context * @regs: is the array of MAX_BPF_EXT_REG eBPF pseudo-registers @@ -2077,10 +2080,9 @@ select_insn: CONT; JMP_CALL_ARGS: - BPF_R0 = (__bpf_call_base_args + insn->imm)(BPF_R1, BPF_R2, - BPF_R3, BPF_R4, - BPF_R5, - insn + insn->off + 1); + BPF_R0 = interpreters_args[insn->off](BPF_R1, BPF_R2, BPF_R3, + BPF_R4, BPF_R5, + insn + insn->imm + 1); CONT; JMP_TAIL_CALL: { @@ -2400,12 +2402,17 @@ int bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth) /* Prevent out-of-bounds read to interpreters_args */ if (stack_depth > MAX_BPF_STACK) return -EINVAL; - insn->off = (s16) insn->imm; - insn->imm = interpreters_args[(round_up(stack_depth, 32) / 32) - 1] - - __bpf_call_base_args; + insn->off = (round_up(stack_depth, 32) / 32) - 1; insn->code = BPF_JMP | BPF_CALL_ARGS; return 0; } + +s32 bpf_call_args_imm(s16 idx) +{ + if (WARN_ON_ONCE(idx < 0 || idx >= ARRAY_SIZE(interpreters_args))) + return 0; + return BPF_CALL_IMM(interpreters_args[idx]); +} #endif #endif diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index df8f48091321..3692adf62558 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -1250,9 +1250,9 @@ static int jit_subprogs(struct bpf_verifier_env *env) } if (!bpf_pseudo_call(insn)) continue; - insn->off = env->insn_aux_data[i].call_imm; - subprog = bpf_find_subprog(env, i + insn->off + 1); - insn->imm = subprog; + insn->imm = env->insn_aux_data[i].call_imm; + subprog = bpf_find_subprog(env, i + insn->imm + 1); + insn->off = subprog; } prog->jited = 1; diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index a3c0214ca934..630d530782fe 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -4919,6 +4919,29 @@ out: return map; } +static void prepare_dump_pseudo_call(struct bpf_insn *insn) +{ + s32 call_off = insn->imm; + + /* + * BPF_CALL_ARGS only exists for interpreter fallback. + * 1. For interpreter (BPF_CALL_ARGS): insn->off is the index of + * interpreters_args array, so here using bpf_call_args_imm() + * to get the real address offset. + * 2. For JIT (BPF_CALL): insn->off is the subprog id. + */ + if (insn->code == (BPF_JMP | BPF_CALL_ARGS)) + insn->imm = bpf_call_args_imm(insn->off); + else + insn->imm = insn->off; + + /* Avoid dumping a truncated and misleading pc-relative offset. */ + if (call_off > S16_MAX || call_off < S16_MIN) + insn->off = 0; + else + insn->off = call_off; +} + static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog, const struct cred *f_cred) { @@ -4944,6 +4967,9 @@ static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog, } if (code == (BPF_JMP | BPF_CALL) || code == (BPF_JMP | BPF_CALL_ARGS)) { + /* Restore the legacy xlated dump layout. */ + if (insns[i].src_reg == BPF_PSEUDO_CALL) + prepare_dump_pseudo_call(&insns[i]); if (code == (BPF_JMP | BPF_CALL_ARGS)) insns[i].code = BPF_JMP | BPF_CALL; if (!bpf_dump_raw_ok(f_cred)) -- cgit v1.2.3 From dfca46365afc030fb09bb40226514c500202dcdc Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Mon, 11 May 2026 05:14:18 -0700 Subject: workqueue: drop apply_wqattrs_lock()/unlock() wrappers The apply_wqattrs_lock()/unlock() helpers were introduced by commit a0111cf6710b ("workqueue: separate out and refactor the locking of applying attrs") to encapsulate the get_online_cpus() (later cpus_read_lock()) + mutex_lock(&wq_pool_mutex) acquire pair that was duplicated across the apply-attrs paths. Since commit 19af45757383 ("workqueue: Remove cpus_read_lock() from apply_wqattrs_lock()") removed the cpus_read_lock() (pwq creation and installation now operate on wq_online_cpumask, so CPU hotplug no longer needs to be excluded), the wrappers have been one-line forwarders to mutex_lock(&wq_pool_mutex)/mutex_unlock(&wq_pool_mutex). They no longer encode any non-trivial locking rule and obscure the fact that callers just take the existing wq_pool_mutex. This align with the "unnecessary" helpers that got discussed in [1] Inline the eight call sites and remove the wrappers. No functional change. Link: https://lore.kernel.org/all/afs_44-6ToJJVZTn@gmail.com/ [1] Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 38 ++++++++++++++------------------------ 1 file changed, 14 insertions(+), 24 deletions(-) (limited to 'kernel') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 3d2e3b2ec528..9adee917e2bb 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -5300,16 +5300,6 @@ static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq, return pwq; } -static void apply_wqattrs_lock(void) -{ - mutex_lock(&wq_pool_mutex); -} - -static void apply_wqattrs_unlock(void) -{ - mutex_unlock(&wq_pool_mutex); -} - /** * wq_calc_pod_cpumask - calculate a wq_attrs' cpumask for a pod * @attrs: the wq_attrs of the default pwq of the target workqueue @@ -5863,7 +5853,7 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt, * wq_pool_mutex protects the workqueues list, allocations of PWQs, * and the global freeze state. */ - apply_wqattrs_lock(); + mutex_lock(&wq_pool_mutex); if (alloc_and_link_pwqs(wq) < 0) goto err_unlock_free_node_nr_active; @@ -5877,7 +5867,7 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt, if (wq_online && init_rescuer(wq) < 0) goto err_unlock_destroy; - apply_wqattrs_unlock(); + mutex_unlock(&wq_pool_mutex); if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq)) goto err_destroy; @@ -5885,7 +5875,7 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt, return wq; err_unlock_free_node_nr_active: - apply_wqattrs_unlock(); + mutex_unlock(&wq_pool_mutex); /* * Failed alloc_and_link_pwqs() may leave pending pwq->release_work, * flushing the pwq_release_worker ensures that the pwq_release_workfn() @@ -5900,7 +5890,7 @@ err_free_wq: kfree(wq); return NULL; err_unlock_destroy: - apply_wqattrs_unlock(); + mutex_unlock(&wq_pool_mutex); err_destroy: destroy_workqueue(wq); return NULL; @@ -7301,7 +7291,7 @@ static ssize_t wq_nice_store(struct device *dev, struct device_attribute *attr, struct workqueue_attrs *attrs; int ret = -ENOMEM; - apply_wqattrs_lock(); + mutex_lock(&wq_pool_mutex); attrs = wq_sysfs_prep_attrs(wq); if (!attrs) @@ -7314,7 +7304,7 @@ static ssize_t wq_nice_store(struct device *dev, struct device_attribute *attr, ret = -EINVAL; out_unlock: - apply_wqattrs_unlock(); + mutex_unlock(&wq_pool_mutex); free_workqueue_attrs(attrs); return ret ?: count; } @@ -7340,7 +7330,7 @@ static ssize_t wq_cpumask_store(struct device *dev, struct workqueue_attrs *attrs; int ret = -ENOMEM; - apply_wqattrs_lock(); + mutex_lock(&wq_pool_mutex); attrs = wq_sysfs_prep_attrs(wq); if (!attrs) @@ -7351,7 +7341,7 @@ static ssize_t wq_cpumask_store(struct device *dev, ret = apply_workqueue_attrs_locked(wq, attrs); out_unlock: - apply_wqattrs_unlock(); + mutex_unlock(&wq_pool_mutex); free_workqueue_attrs(attrs); return ret ?: count; } @@ -7387,13 +7377,13 @@ static ssize_t wq_affn_scope_store(struct device *dev, if (affn < 0) return affn; - apply_wqattrs_lock(); + mutex_lock(&wq_pool_mutex); attrs = wq_sysfs_prep_attrs(wq); if (attrs) { attrs->affn_scope = affn; ret = apply_workqueue_attrs_locked(wq, attrs); } - apply_wqattrs_unlock(); + mutex_unlock(&wq_pool_mutex); free_workqueue_attrs(attrs); return ret ?: count; } @@ -7418,13 +7408,13 @@ static ssize_t wq_affinity_strict_store(struct device *dev, if (sscanf(buf, "%d", &v) != 1) return -EINVAL; - apply_wqattrs_lock(); + mutex_lock(&wq_pool_mutex); attrs = wq_sysfs_prep_attrs(wq); if (attrs) { attrs->affn_strict = (bool)v; ret = apply_workqueue_attrs_locked(wq, attrs); } - apply_wqattrs_unlock(); + mutex_unlock(&wq_pool_mutex); free_workqueue_attrs(attrs); return ret ?: count; } @@ -7465,12 +7455,12 @@ static int workqueue_set_unbound_cpumask(cpumask_var_t cpumask) cpumask_and(cpumask, cpumask, cpu_possible_mask); if (!cpumask_empty(cpumask)) { ret = 0; - apply_wqattrs_lock(); + mutex_lock(&wq_pool_mutex); if (!cpumask_equal(cpumask, wq_unbound_cpumask)) ret = workqueue_apply_unbound_cpumask(cpumask); if (!ret) cpumask_copy(wq_requested_unbound_cpumask, cpumask); - apply_wqattrs_unlock(); + mutex_unlock(&wq_pool_mutex); } return ret; -- cgit v1.2.3 From fbe3fb103596becc7825327fea281a6bbbb454e7 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Mon, 11 May 2026 21:19:40 +0200 Subject: sched_ext: Replace tryget_task_struct() with get_task_struct() The tryget_task_struct() calls in scx_sub_disable(), scx_root_enable_workfn() and scx_sub_enable_workfn() can never fail at the points they're invoked: - scx_root_enable_workfn() iterates over scx_tasks under scx_tasks_lock and rq lock. sched_ext_dead() removes tasks from scx_tasks under the same scx_tasks_lock before put_task_struct_rcu_user() runs in finish_task_switch(). So any task observed in scx_tasks must have usage > 0; put_task_struct_rcu_user() hasn't been called and the delayed_put_task_struct() callback that decrements usage cannot have been queued. - scx_sub_disable() and scx_sub_enable_workfn() iterate via css_task_iter, which takes a reference on each task in css_task_iter_next() and holds it until the next iter_next() call, so usage > 0 is guaranteed by the iter itself. The actual filter for dead tasks is the SCX_TASK_DEAD check inside scx_task_iter_next_locked(), not tryget; tryget only fails on zero usage, a state that can't be reached for tasks visible to these iters. Commit b7d4b28db7da ("sched_ext: Use SCX_TASK_READY test instead of tryget_task_struct() during class switch") removed an analogous tryget in the class-switch loop. Convert the remaining tryget calls to plain get_task_struct() and update the comment in scx_root_enable_workfn() that suggested tasks could be observed with zero @usage waiting for an RCU grace period. Link: https://lore.kernel.org/all/agCLBxHEUqWIepx8@google.com Suggested-by: Alice Ryhl Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext.c | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index b8dd3358959d..64dfaf4dc5b2 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -5933,14 +5933,11 @@ static void scx_sub_disable(struct scx_sched *sch) WARN_ON_ONCE(!scx_task_on_sched(sch, p)); /* - * If $p is about to be freed, nothing prevents $sch from - * unloading before $p reaches sched_ext_free(). Disable and - * exit $p right away. + * @p is pinned by the iter: css_task_iter_next() takes a + * reference and holds it until the next iter_next() call, so + * @p->usage is guaranteed > 0. */ - if (!tryget_task_struct(p)) { - scx_disable_and_exit_task(sch, p); - continue; - } + get_task_struct(p); scx_task_iter_unlock(&sti); @@ -7181,12 +7178,13 @@ static void scx_root_enable_workfn(struct kthread_work *work) scx_task_iter_start(&sti, NULL); while ((p = scx_task_iter_next_locked(&sti))) { /* - * @p may already be dead, have lost all its usages counts and - * be waiting for RCU grace period before being freed. @p can't - * be initialized for SCX in such cases and should be ignored. + * @p is in scx_tasks under scx_tasks_lock, and SCX_TASK_DEAD + * tasks are filtered by scx_task_iter_next_locked(). + * sched_ext_dead() removes @p from scx_tasks under the same + * lock before put_task_struct_rcu_user() runs, so @p->usage + * is guaranteed > 0 here. */ - if (!tryget_task_struct(p)) - continue; + get_task_struct(p); /* * Set %INIT_BEGIN under the iter's rq lock so that a concurrent @@ -7487,9 +7485,8 @@ static void scx_sub_enable_workfn(struct kthread_work *work) if (p->scx.flags & SCX_TASK_SUB_INIT) continue; - /* see scx_root_enable() */ - if (!tryget_task_struct(p)) - continue; + /* @p is pinned by the iter; see scx_sub_disable() */ + get_task_struct(p); if (!assert_task_ready_or_enabled(p)) { ret = -EINVAL; -- cgit v1.2.3 From dc4edae7f41dceb236553b61cda0383895293c90 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Mon, 27 Apr 2026 10:26:03 +0200 Subject: fs: move SB_I_USERNS_VISIBLE to FS_USERNS_MOUNT_RESTRICTED Whether a filesystem's mounts need to undergo a visibility check in user namespaces is a static property of the filesystem type, not a runtime property of each superblock instance. Both proc and sysfs always set SB_I_USERNS_VISIBLE on their superblocks unconditionally (sysfs does so on first creation, and subsequent mounts reuse the same superblock). Move this flag from sb->s_iflags (SB_I_USERNS_VISIBLE) to file_system_type->fs_flags (FS_USERNS_MOUNT_RESTRICTED) so the intent is expressed at the filesystem type level where it belongs. All check sites are updated to test sb->s_type->fs_flags instead of sb->s_iflags. The SB_I_NOEXEC and SB_I_NODEV flags remain on the superblock as they are runtime properties set during fill_super. Link: https://patch.msgid.link/72887c5b6204dc3adf5a53104f0be6bd8bc4f6cd.1777278334.git.legion@kernel.org Reviewed-by: Aleksa Sarai Signed-off-by: Christian Brauner --- kernel/acct.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/acct.c b/kernel/acct.c index cbbf79d718cf..c440d43479ca 100644 --- a/kernel/acct.c +++ b/kernel/acct.c @@ -249,7 +249,7 @@ static int acct_on(const char __user *name) return -EINVAL; /* Exclude procfs and sysfs. */ - if (file_inode(file)->i_sb->s_iflags & SB_I_USERNS_VISIBLE) + if (file_inode(file)->i_sb->s_type->fs_flags & FS_USERNS_MOUNT_RESTRICTED) return -EINVAL; if (!(file->f_mode & FMODE_CAN_WRITE)) -- cgit v1.2.3 From 9740914d36e5ab4f22ab215e68ee8ab0b5771e8e Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 11 May 2026 11:06:05 -1000 Subject: sched_ext: Mark !CONFIG_EXT_SUB_SCHED dummy stubs static inline Mark !CONFIG_EXT_SUB_SCHED dummy stubs static inline to avoid -Wunused-function in configs without callers. No functional change. Signed-off-by: Tejun Heo --- kernel/sched/ext.c | 20 ++++++++++---------- kernel/sched/ext_internal.h | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 64dfaf4dc5b2..87146f6e2385 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -323,9 +323,9 @@ static void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) rcu_assign_pointer(p->scx.sched, sch); } #else /* CONFIG_EXT_SUB_SCHED */ -static struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; } -static struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; } -static void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {} +static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; } +static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; } +static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {} #endif /* CONFIG_EXT_SUB_SCHED */ /** @@ -4649,9 +4649,9 @@ static void scx_cgroup_unlock(void) #endif } #else /* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */ -static struct cgroup *root_cgroup(void) { return NULL; } -static void scx_cgroup_lock(void) {} -static void scx_cgroup_unlock(void) {} +static inline struct cgroup *root_cgroup(void) { return NULL; } +static inline void scx_cgroup_lock(void) {} +static inline void scx_cgroup_unlock(void) {} #endif /* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */ #ifdef CONFIG_EXT_SUB_SCHED @@ -4670,8 +4670,8 @@ static void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) rcu_assign_pointer(pos->scx_sched, sch); } #else /* CONFIG_EXT_SUB_SCHED */ -static struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; } -static void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {} +static inline struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; } +static inline void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {} #endif /* CONFIG_EXT_SUB_SCHED */ /* @@ -6039,8 +6039,8 @@ static void scx_sub_disable(struct scx_sched *sch) kobject_del(&sch->kobj); } #else /* CONFIG_EXT_SUB_SCHED */ -static void drain_descendants(struct scx_sched *sch) { } -static void scx_sub_disable(struct scx_sched *sch) { } +static inline void drain_descendants(struct scx_sched *sch) { } +static inline void scx_sub_disable(struct scx_sched *sch) { } #endif /* CONFIG_EXT_SUB_SCHED */ static void scx_root_disable(struct scx_sched *sch) diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext_internal.h index cd33984cffcf..7258aea94b9f 100644 --- a/kernel/sched/ext_internal.h +++ b/kernel/sched/ext_internal.h @@ -1601,7 +1601,7 @@ static inline bool scx_task_on_sched(struct scx_sched *sch, return true; } -static struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux) +static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux) { return rcu_dereference_all(scx_root); } -- cgit v1.2.3 From f28771c0691bcb7f477a0f35550b17b88c32dea8 Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Tue, 12 May 2026 23:31:50 +0800 Subject: bpf: Extend BPF syscall with common attributes support Add generic BPF syscall support for passing common attributes. The initial set of common attributes includes: 1. 'log_buf': User-provided buffer for storing logs. 2. 'log_size': Size of the log buffer. 3. 'log_level': Log verbosity level. 4. 'log_true_size': Actual log size reported by kernel. The common-attribute pointer and its size are passed as the 4th and 5th syscall arguments. A new command bit, 'BPF_COMMON_ATTRS' ('1 << 16'), indicates that common attributes are supplied. This commit adds syscall and uapi plumbing. Command-specific handling is added in follow-up patches. Signed-off-by: Leon Hwang Link: https://lore.kernel.org/r/20260512153157.28382-2-leon.hwang@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/syscall.c | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 3b1f0ba02f61..354f6f471a08 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -6211,8 +6211,10 @@ put_prog: return ret; } -static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size) +static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size, + bpfptr_t uattr_common, unsigned int size_common) { + struct bpf_common_attr attr_common; union bpf_attr attr; int err; @@ -6226,6 +6228,20 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size) if (copy_from_bpfptr(&attr, uattr, size) != 0) return -EFAULT; + memset(&attr_common, 0, sizeof(attr_common)); + if (cmd & BPF_COMMON_ATTRS) { + err = bpf_check_uarg_tail_zero(uattr_common, sizeof(attr_common), size_common); + if (err) + return err; + + cmd &= ~BPF_COMMON_ATTRS; + size_common = min_t(u32, size_common, sizeof(attr_common)); + if (copy_from_bpfptr(&attr_common, uattr_common, size_common) != 0) + return -EFAULT; + } else { + size_common = 0; + } + err = security_bpf(cmd, &attr, size, uattr.is_kernel); if (err < 0) return err; @@ -6361,9 +6377,10 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size) return err; } -SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) +SYSCALL_DEFINE5(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size, + struct bpf_common_attr __user *, uattr_common, unsigned int, size_common) { - return __sys_bpf(cmd, USER_BPFPTR(uattr), size); + return __sys_bpf(cmd, USER_BPFPTR(uattr), size, USER_BPFPTR(uattr_common), size_common); } static bool syscall_prog_is_valid_access(int off, int size, @@ -6393,7 +6410,7 @@ BPF_CALL_3(bpf_sys_bpf, int, cmd, union bpf_attr *, attr, u32, attr_size) default: return -EINVAL; } - return __sys_bpf(cmd, KERNEL_BPFPTR(attr), attr_size); + return __sys_bpf(cmd, KERNEL_BPFPTR(attr), attr_size, KERNEL_BPFPTR(NULL), 0); } -- cgit v1.2.3 From 503c039ffeca7530ce9d6446a07b4bb776180b45 Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Tue, 12 May 2026 23:31:52 +0800 Subject: bpf: Refactor reporting log_true_size for prog_load The next commit will add support for reporting logs via extended common attributes, including 'log_true_size'. To prepare for that, refactor the 'log_true_size' reporting logic by introducing a new struct bpf_log_attr to encapsulate log-related behavior: * bpf_log_attr_init(): initialize log fields, which will support extended common attributes in the next commit. * bpf_log_attr_finalize(): handle log finalization and write back 'log_true_size' to userspace. Acked-by: Andrii Nakryiko Signed-off-by: Leon Hwang Link: https://lore.kernel.org/r/20260512153157.28382-4-leon.hwang@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/log.c | 29 +++++++++++++++++++++++++++++ kernel/bpf/syscall.c | 12 +++++++++--- kernel/bpf/verifier.c | 17 ++++------------- 3 files changed, 42 insertions(+), 16 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index 64566b86dd27..1b1efe75398b 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c @@ -825,3 +825,32 @@ void print_insn_state(struct bpf_verifier_env *env, const struct bpf_verifier_st } print_verifier_state(env, vstate, frameno, false); } + +int bpf_log_attr_init(struct bpf_log_attr *log, u64 log_buf, u32 log_size, u32 log_level, + u32 offsetof_log_true_size, bpfptr_t uattr) +{ + char __user *ubuf = u64_to_user_ptr(log_buf); + + memset(log, 0, sizeof(*log)); + log->ubuf = ubuf; + log->size = log_size; + log->level = log_level; + log->offsetof_true_size = offsetof_log_true_size; + log->uattr = uattr; + return 0; +} + +int bpf_log_attr_finalize(struct bpf_log_attr *attr, struct bpf_verifier_log *log) +{ + u32 log_true_size; + int err; + + err = bpf_vlog_finalize(log, &log_true_size); + + if (attr->offsetof_true_size && + copy_to_bpfptr_offset(attr->uattr, attr->offsetof_true_size, &log_true_size, + sizeof(log_true_size))) + return -EFAULT; + + return err; +} diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 354f6f471a08..70b78ddcdedb 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2861,7 +2861,7 @@ static int bpf_prog_mark_insn_arrays_ready(struct bpf_prog *prog) /* last field in 'union bpf_attr' used by this command */ #define BPF_PROG_LOAD_LAST_FIELD keyring_id -static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) +static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_attr *attr_log) { enum bpf_prog_type type = attr->prog_type; struct bpf_prog *prog, *dst_prog = NULL; @@ -3079,7 +3079,7 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) goto free_prog_sec; /* run eBPF verifier */ - err = bpf_check(&prog, attr, uattr, uattr_size); + err = bpf_check(&prog, attr, uattr, attr_log); if (err < 0) goto free_used_maps; @@ -6215,6 +6215,8 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size, bpfptr_t uattr_common, unsigned int size_common) { struct bpf_common_attr attr_common; + u32 offsetof_log_true_size = 0; + struct bpf_log_attr attr_log; union bpf_attr attr; int err; @@ -6266,7 +6268,11 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size, err = map_freeze(&attr); break; case BPF_PROG_LOAD: - err = bpf_prog_load(&attr, uattr, size); + if (size >= offsetofend(union bpf_attr, log_true_size)) + offsetof_log_true_size = offsetof(union bpf_attr, log_true_size); + err = bpf_log_attr_init(&attr_log, attr.log_buf, attr.log_size, attr.log_level, + offsetof_log_true_size, uattr); + err = err ?: bpf_prog_load(&attr, uattr, &attr_log); break; case BPF_OBJ_PIN: err = bpf_obj_pin(&attr); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 11054ad89c14..0e654ef01ae0 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -19294,12 +19294,12 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return 0; } -int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u32 uattr_size) +int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, + struct bpf_log_attr *attr_log) { u64 start_time = ktime_get_ns(); struct bpf_verifier_env *env; int i, len, ret = -EINVAL, err; - u32 log_true_size; bool is_priv; BTF_TYPE_EMIT(enum bpf_features); @@ -19346,9 +19346,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3 /* user could have requested verbose verifier output * and supplied buffer to store the verification trace */ - ret = bpf_vlog_init(&env->log, attr->log_level, - (char __user *) (unsigned long) attr->log_buf, - attr->log_size); + ret = bpf_vlog_init(&env->log, attr_log->level, attr_log->ubuf, attr_log->size); if (ret) goto err_unlock; @@ -19510,17 +19508,10 @@ skip_full_check: env->prog->aux->verified_insns = env->insn_processed; /* preserve original error even if log finalization is successful */ - err = bpf_vlog_finalize(&env->log, &log_true_size); + err = bpf_log_attr_finalize(attr_log, &env->log); if (err) ret = err; - if (uattr_size >= offsetofend(union bpf_attr, log_true_size) && - copy_to_bpfptr_offset(uattr, offsetof(union bpf_attr, log_true_size), - &log_true_size, sizeof(log_true_size))) { - ret = -EFAULT; - goto err_release_maps; - } - if (ret) goto err_release_maps; -- cgit v1.2.3 From ac89d33fdd8183df39fe92ffa525be7af6feb9d1 Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Tue, 12 May 2026 23:31:53 +0800 Subject: bpf: Add syscall common attributes support for prog_load BPF_PROG_LOAD can now take log parameters from both union bpf_attr and struct bpf_common_attr. The merge rules are: - if both sides provide a complete log tuple (buf/size/level) and they match, use it; - if only one side provides log parameters, use that one; - if both sides provide complete tuples but they differ, return -EINVAL. Signed-off-by: Leon Hwang Link: https://lore.kernel.org/r/20260512153157.28382-5-leon.hwang@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/log.c | 34 +++++++++++++++++++++++++++------- kernel/bpf/syscall.c | 3 ++- 2 files changed, 29 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index 1b1efe75398b..fd12ad5a0338 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c @@ -13,17 +13,17 @@ #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) -static bool bpf_verifier_log_attr_valid(const struct bpf_verifier_log *log) +static bool bpf_verifier_log_attr_valid(u32 log_level, char __user *log_buf, u32 log_size) { /* ubuf and len_total should both be specified (or not) together */ - if (!!log->ubuf != !!log->len_total) + if (!!log_buf != !!log_size) return false; /* log buf without log_level is meaningless */ - if (log->ubuf && log->level == 0) + if (log_buf && log_level == 0) return false; - if (log->level & ~BPF_LOG_MASK) + if (log_level & ~BPF_LOG_MASK) return false; - if (log->len_total > UINT_MAX >> 2) + if (log_size > UINT_MAX >> 2) return false; return true; } @@ -36,7 +36,7 @@ int bpf_vlog_init(struct bpf_verifier_log *log, u32 log_level, log->len_total = log_size; /* log attributes have to be sane */ - if (!bpf_verifier_log_attr_valid(log)) + if (!bpf_verifier_log_attr_valid(log_level, log_buf, log_size)) return -EINVAL; return 0; @@ -827,16 +827,36 @@ void print_insn_state(struct bpf_verifier_env *env, const struct bpf_verifier_st } int bpf_log_attr_init(struct bpf_log_attr *log, u64 log_buf, u32 log_size, u32 log_level, - u32 offsetof_log_true_size, bpfptr_t uattr) + u32 offsetof_log_true_size, bpfptr_t uattr, struct bpf_common_attr *common, + bpfptr_t uattr_common, u32 size_common) { + char __user *ubuf_common = u64_to_user_ptr(common->log_buf); char __user *ubuf = u64_to_user_ptr(log_buf); + if (!bpf_verifier_log_attr_valid(common->log_level, ubuf_common, common->log_size) || + !bpf_verifier_log_attr_valid(log_level, ubuf, log_size)) + return -EINVAL; + + if (ubuf && ubuf_common && (ubuf != ubuf_common || log_size != common->log_size || + log_level != common->log_level)) + return -EINVAL; + memset(log, 0, sizeof(*log)); log->ubuf = ubuf; log->size = log_size; log->level = log_level; log->offsetof_true_size = offsetof_log_true_size; log->uattr = uattr; + + if (!ubuf && ubuf_common) { + log->ubuf = ubuf_common; + log->size = common->log_size; + log->level = common->log_level; + log->uattr = uattr_common; + log->offsetof_true_size = 0; + if (size_common >= offsetofend(struct bpf_common_attr, log_true_size)) + log->offsetof_true_size = offsetof(struct bpf_common_attr, log_true_size); + } return 0; } diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 70b78ddcdedb..db893cae826c 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -6271,7 +6271,8 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size, if (size >= offsetofend(union bpf_attr, log_true_size)) offsetof_log_true_size = offsetof(union bpf_attr, log_true_size); err = bpf_log_attr_init(&attr_log, attr.log_buf, attr.log_size, attr.log_level, - offsetof_log_true_size, uattr); + offsetof_log_true_size, uattr, &attr_common, uattr_common, + size_common); err = err ?: bpf_prog_load(&attr, uattr, &attr_log); break; case BPF_OBJ_PIN: -- cgit v1.2.3 From ceeb7eda94a3548958b30818495ef7eb12898727 Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Tue, 12 May 2026 23:31:54 +0800 Subject: bpf: Add syscall common attributes support for btf_load BPF_BTF_LOAD can now take log parameters from both union bpf_attr and struct bpf_common_attr, with the same merge rules as BPF_PROG_LOAD: - if both sides provide a complete log tuple (buf/size/level) and they match, use it; - if only one side provides log parameters, use that one; - if both sides provide complete tuples but they differ, return -EINVAL. Acked-by: Andrii Nakryiko Signed-off-by: Leon Hwang Link: https://lore.kernel.org/r/20260512153157.28382-6-leon.hwang@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/btf.c | 30 +++++++----------------------- kernel/bpf/syscall.c | 11 ++++++++--- 2 files changed, 15 insertions(+), 26 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 77af44d8a3ad..a6bf4781943c 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -5907,25 +5907,10 @@ static int btf_check_type_tags(struct btf_verifier_env *env, return 0; } -static int finalize_log(struct bpf_verifier_log *log, bpfptr_t uattr, u32 uattr_size) -{ - u32 log_true_size; - int err; - - err = bpf_vlog_finalize(log, &log_true_size); - - if (uattr_size >= offsetofend(union bpf_attr, btf_log_true_size) && - copy_to_bpfptr_offset(uattr, offsetof(union bpf_attr, btf_log_true_size), - &log_true_size, sizeof(log_true_size))) - err = -EFAULT; - - return err; -} - -static struct btf *btf_parse(const union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) +static struct btf *btf_parse(const union bpf_attr *attr, bpfptr_t uattr, + struct bpf_log_attr *attr_log) { bpfptr_t btf_data = make_bpfptr(attr->btf, uattr.is_kernel); - char __user *log_ubuf = u64_to_user_ptr(attr->btf_log_buf); struct btf_struct_metas *struct_meta_tab; struct btf_verifier_env *env = NULL; struct btf *btf = NULL; @@ -5942,8 +5927,7 @@ static struct btf *btf_parse(const union bpf_attr *attr, bpfptr_t uattr, u32 uat /* user could have requested verbose verifier output * and supplied buffer to store the verification trace */ - err = bpf_vlog_init(&env->log, attr->btf_log_level, - log_ubuf, attr->btf_log_size); + err = bpf_vlog_init(&env->log, attr_log->level, attr_log->ubuf, attr_log->size); if (err) goto errout_free; @@ -6008,7 +5992,7 @@ static struct btf *btf_parse(const union bpf_attr *attr, bpfptr_t uattr, u32 uat } } - err = finalize_log(&env->log, uattr, uattr_size); + err = bpf_log_attr_finalize(attr_log, &env->log); if (err) goto errout_free; @@ -6020,7 +6004,7 @@ errout_meta: btf_free_struct_meta_tab(btf); errout: /* overwrite err with -ENOSPC or -EFAULT */ - ret = finalize_log(&env->log, uattr, uattr_size); + ret = bpf_log_attr_finalize(attr_log, &env->log); if (ret) err = ret; errout_free: @@ -8189,12 +8173,12 @@ static int __btf_new_fd(struct btf *btf) return anon_inode_getfd("btf", &btf_fops, btf, O_RDONLY | O_CLOEXEC); } -int btf_new_fd(const union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) +int btf_new_fd(const union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_attr *attr_log) { struct btf *btf; int ret; - btf = btf_parse(attr, uattr, uattr_size); + btf = btf_parse(attr, uattr, attr_log); if (IS_ERR(btf)) return PTR_ERR(btf); diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index db893cae826c..2fa05ba8f161 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -5474,7 +5474,7 @@ static int bpf_obj_get_info_by_fd(const union bpf_attr *attr, #define BPF_BTF_LOAD_LAST_FIELD btf_token_fd -static int bpf_btf_load(const union bpf_attr *attr, bpfptr_t uattr, __u32 uattr_size) +static int bpf_btf_load(const union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_attr *attr_log) { struct bpf_token *token = NULL; @@ -5501,7 +5501,7 @@ static int bpf_btf_load(const union bpf_attr *attr, bpfptr_t uattr, __u32 uattr_ bpf_token_put(token); - return btf_new_fd(attr, uattr, uattr_size); + return btf_new_fd(attr, uattr, attr_log); } #define BPF_BTF_GET_FD_BY_ID_LAST_FIELD fd_by_id_token_fd @@ -6318,7 +6318,12 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size, err = bpf_raw_tracepoint_open(&attr); break; case BPF_BTF_LOAD: - err = bpf_btf_load(&attr, uattr, size); + if (size >= offsetofend(union bpf_attr, btf_log_true_size)) + offsetof_log_true_size = offsetof(union bpf_attr, btf_log_true_size); + err = bpf_log_attr_init(&attr_log, attr.btf_log_buf, attr.btf_log_size, + attr.btf_log_level, offsetof_log_true_size, uattr, + &attr_common, uattr_common, size_common); + err = err ?: bpf_btf_load(&attr, uattr, &attr_log); break; case BPF_BTF_GET_FD_BY_ID: err = bpf_btf_get_fd_by_id(&attr); -- cgit v1.2.3 From 49f9b2b2a18c5ce06b21fc2b3399352d80dee0c6 Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Tue, 12 May 2026 23:31:55 +0800 Subject: bpf: Add syscall common attributes support for map_create Many BPF_MAP_CREATE validation failures currently return -EINVAL without any explanation to userspace. Plumb common syscall log attributes into map_create(), create a verifier log from bpf_common_attr::log_buf/log_size/log_level, and report map-creation failure reasons through that buffer. This improves debuggability by allowing userspace to inspect why map creation failed and read back log_true_size from common attributes. Signed-off-by: Leon Hwang Link: https://lore.kernel.org/r/20260512153157.28382-7-leon.hwang@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/log.c | 29 +++++++++++++++++++++++ kernel/bpf/syscall.c | 66 ++++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 85 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index fd12ad5a0338..62fe6ed18374 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c @@ -860,6 +860,35 @@ int bpf_log_attr_init(struct bpf_log_attr *log, u64 log_buf, u32 log_size, u32 l return 0; } +struct bpf_verifier_log *bpf_log_attr_create_vlog(struct bpf_log_attr *attr_log, + struct bpf_common_attr *common, bpfptr_t uattr, + u32 size) +{ + struct bpf_verifier_log *log; + int err; + + memset(attr_log, 0, sizeof(*attr_log)); + attr_log->uattr = uattr; + if (size >= offsetofend(struct bpf_common_attr, log_true_size)) + attr_log->offsetof_true_size = offsetof(struct bpf_common_attr, log_true_size); + + if (!size) + return NULL; + + log = kzalloc_obj(*log, GFP_KERNEL); + if (!log) + return ERR_PTR(-ENOMEM); + + err = bpf_vlog_init(log, common->log_level, u64_to_user_ptr(common->log_buf), + common->log_size); + if (err) { + kfree(log); + return ERR_PTR(err); + } + + return log; +} + int bpf_log_attr_finalize(struct bpf_log_attr *attr, struct bpf_verifier_log *log) { u32 log_true_size; diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 2fa05ba8f161..6600e126fbfb 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1359,7 +1359,7 @@ free_map_tab: #define BPF_MAP_CREATE_LAST_FIELD excl_prog_hash_size /* called via syscall */ -static int map_create(union bpf_attr *attr, bpfptr_t uattr) +static int __map_create(union bpf_attr *attr, bpfptr_t uattr, struct bpf_verifier_log *log) { const struct bpf_map_ops *ops; struct bpf_token *token = NULL; @@ -1371,8 +1371,10 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr) int err; err = CHECK_ATTR(BPF_MAP_CREATE); - if (err) + if (err) { + bpf_log(log, "Invalid attr.\n"); return -EINVAL; + } /* check BPF_F_TOKEN_FD flag, remember if it's set, and then clear it * to avoid per-map type checks tripping on unknown flag @@ -1381,17 +1383,25 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr) attr->map_flags &= ~BPF_F_TOKEN_FD; if (attr->btf_vmlinux_value_type_id) { - if (attr->map_type != BPF_MAP_TYPE_STRUCT_OPS || - attr->btf_key_type_id || attr->btf_value_type_id) + if (attr->map_type != BPF_MAP_TYPE_STRUCT_OPS) { + bpf_log(log, "btf_vmlinux_value_type_id can only be used with struct_ops maps.\n"); return -EINVAL; + } + if (attr->btf_key_type_id || attr->btf_value_type_id) { + bpf_log(log, "btf_vmlinux_value_type_id is mutually exclusive with btf_key_type_id and btf_value_type_id.\n"); + return -EINVAL; + } } else if (attr->btf_key_type_id && !attr->btf_value_type_id) { + bpf_log(log, "Invalid btf_value_type_id.\n"); return -EINVAL; } if (attr->map_type != BPF_MAP_TYPE_BLOOM_FILTER && attr->map_type != BPF_MAP_TYPE_ARENA && - attr->map_extra != 0) + attr->map_extra != 0) { + bpf_log(log, "Invalid map_extra.\n"); return -EINVAL; + } f_flags = bpf_get_file_flag(attr->map_flags); if (f_flags < 0) @@ -1399,13 +1409,17 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr) if (numa_node != NUMA_NO_NODE && ((unsigned int)numa_node >= nr_node_ids || - !node_online(numa_node))) + !node_online(numa_node))) { + bpf_log(log, "Invalid numa_node.\n"); return -EINVAL; + } /* find map type and init map: hashtable vs rbtree vs bloom vs ... */ map_type = attr->map_type; - if (map_type >= ARRAY_SIZE(bpf_map_types)) + if (map_type >= ARRAY_SIZE(bpf_map_types)) { + bpf_log(log, "Invalid map_type.\n"); return -EINVAL; + } map_type = array_index_nospec(map_type, ARRAY_SIZE(bpf_map_types)); ops = bpf_map_types[map_type]; if (!ops) @@ -1423,8 +1437,10 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr) if (token_flag) { token = bpf_token_get_from_fd(attr->map_token_fd); - if (IS_ERR(token)) + if (IS_ERR(token)) { + bpf_log(log, "Invalid map_token_fd.\n"); return PTR_ERR(token); + } /* if current token doesn't grant map creation permissions, * then we can't use this token, so ignore it and rely on @@ -1507,8 +1523,10 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr) err = bpf_obj_name_cpy(map->name, attr->map_name, sizeof(attr->map_name)); - if (err < 0) + if (err < 0) { + bpf_log(log, "Invalid map_name.\n"); goto free_map; + } preempt_disable(); map->cookie = gen_cookie_next(&bpf_map_cookie); @@ -1531,6 +1549,7 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr) btf = btf_get_by_fd(attr->btf_fd); if (IS_ERR(btf)) { + bpf_log(log, "Invalid btf_fd.\n"); err = PTR_ERR(btf); goto free_map; } @@ -1558,6 +1577,7 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr) bpfptr_t uprog_hash = make_bpfptr(attr->excl_prog_hash, uattr.is_kernel); if (attr->excl_prog_hash_size != SHA256_DIGEST_SIZE) { + bpf_log(log, "Invalid excl_prog_hash_size.\n"); err = -EINVAL; goto free_map; } @@ -1573,6 +1593,7 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr) goto free_map; } } else if (attr->excl_prog_hash_size) { + bpf_log(log, "Invalid excl_prog_hash_size.\n"); err = -EINVAL; goto free_map; } @@ -1611,6 +1632,31 @@ put_token: return err; } +static int map_create(union bpf_attr *attr, bpfptr_t uattr, struct bpf_common_attr *attr_common, + bpfptr_t uattr_common, u32 size_common) +{ + struct bpf_verifier_log *log; + struct bpf_log_attr attr_log; + int err, ret; + + log = bpf_log_attr_create_vlog(&attr_log, attr_common, uattr_common, size_common); + if (IS_ERR(log)) + return PTR_ERR(log); + + err = __map_create(attr, uattr, log); + + /* preserve original error even if log finalization is successful */ + ret = bpf_log_attr_finalize(&attr_log, log); + if (ret) { + if (err >= 0) + close_fd(err); + err = ret; + } + + kfree(log); + return err; +} + void bpf_map_inc(struct bpf_map *map) { atomic64_inc(&map->refcnt); @@ -6250,7 +6296,7 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size, switch (cmd) { case BPF_MAP_CREATE: - err = map_create(&attr, uattr); + err = map_create(&attr, uattr, &attr_common, uattr_common, size_common); break; case BPF_MAP_LOOKUP_ELEM: err = map_lookup_elem(&attr); -- cgit v1.2.3 From 1cb229a54af1e36f19f7e8359692fa0d76fbc360 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 12 May 2026 21:50:00 -0700 Subject: bpf: Remove copy_register_state wrapper function Remove the copy_register_state() helper which was just a plain struct assignment wrapper and replace all call sites with direct struct assignment. This simplifies the code in preparation for upcoming stack argument support. No functional change. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260513045000.2382933-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 44 +++++++++++++++++++------------------------- 1 file changed, 19 insertions(+), 25 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 0e654ef01ae0..1dd9736c2a13 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3410,12 +3410,6 @@ static void assign_scalar_id_before_mov(struct bpf_verifier_env *env, src_reg->id = ++env->id_gen; } -/* Copy src state preserving dst->parent and dst->live fields */ -static void copy_register_state(struct bpf_reg_state *dst, const struct bpf_reg_state *src) -{ - *dst = *src; -} - static void save_register_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi, struct bpf_reg_state *reg, @@ -3423,7 +3417,7 @@ static void save_register_state(struct bpf_verifier_env *env, { int i; - copy_register_state(&state->stack[spi].spilled_ptr, reg); + state->stack[spi].spilled_ptr = *reg; for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--) state->stack[spi].slot_type[i - 1] = STACK_SPILL; @@ -3822,7 +3816,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, * with the destination register on fill. */ assign_scalar_id_before_mov(env, reg); - copy_register_state(&state->regs[dst_regno], reg); + state->regs[dst_regno] = *reg; state->regs[dst_regno].subreg_def = subreg_def; /* Break the relation on a narrowing fill. @@ -3877,7 +3871,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, * with the destination register on fill. */ assign_scalar_id_before_mov(env, reg); - copy_register_state(&state->regs[dst_regno], reg); + state->regs[dst_regno] = *reg; /* mark reg as written since spilled pointer state likely * has its liveness marks cleared by is_state_visited() * which resets stack/reg liveness for state transitions @@ -6031,7 +6025,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b size); return -EACCES; } - copy_register_state(®s[value_regno], reg); + regs[value_regno] = *reg; add_scalar_to_reg(®s[value_regno], off); regs[value_regno].type = PTR_TO_INSN; } else { @@ -13248,7 +13242,7 @@ do_sim: */ if (!ptr_is_dst_reg) { tmp = *dst_reg; - copy_register_state(dst_reg, ptr_reg); + *dst_reg = *ptr_reg; } err = sanitize_speculative_path(env, NULL, env->insn_idx + 1, env->insn_idx); if (err < 0) @@ -14698,7 +14692,7 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) * copy register state to dest reg */ assign_scalar_id_before_mov(env, src_reg); - copy_register_state(dst_reg, src_reg); + *dst_reg = *src_reg; dst_reg->subreg_def = DEF_NOT_SUBREG; } else { /* case: R1 = (s8, s16 s32)R2 */ @@ -14713,7 +14707,7 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) no_sext = reg_umax(src_reg) < (1ULL << (insn->off - 1)); if (no_sext) assign_scalar_id_before_mov(env, src_reg); - copy_register_state(dst_reg, src_reg); + *dst_reg = *src_reg; if (!no_sext) clear_scalar_id(dst_reg); coerce_reg_to_size_sx(dst_reg, insn->off >> 3); @@ -14735,7 +14729,7 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) if (is_src_reg_u32) assign_scalar_id_before_mov(env, src_reg); - copy_register_state(dst_reg, src_reg); + *dst_reg = *src_reg; /* Make sure ID is cleared if src_reg is not in u32 * range otherwise dst_reg min/max could be incorrectly * propagated into src_reg by sync_linked_regs() @@ -14749,7 +14743,7 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) if (no_sext) assign_scalar_id_before_mov(env, src_reg); - copy_register_state(dst_reg, src_reg); + *dst_reg = *src_reg; if (!no_sext) clear_scalar_id(dst_reg); dst_reg->subreg_def = env->insn_idx + 1; @@ -15629,7 +15623,7 @@ static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_s reg->delta == known_reg->delta) { s32 saved_subreg_def = reg->subreg_def; - copy_register_state(reg, known_reg); + *reg = *known_reg; reg->subreg_def = saved_subreg_def; } else { s32 saved_subreg_def = reg->subreg_def; @@ -15640,7 +15634,7 @@ static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_s __mark_reg_known(&fake_reg, (s64)reg->delta - (s64)known_reg->delta); /* reg = known_reg; reg += delta */ - copy_register_state(reg, known_reg); + *reg = *known_reg; /* * Must preserve off, id and subreg_def flag, * otherwise another sync_linked_regs() will be incorrect. @@ -15743,10 +15737,10 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, } is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; - copy_register_state(&env->false_reg1, dst_reg); - copy_register_state(&env->false_reg2, src_reg); - copy_register_state(&env->true_reg1, dst_reg); - copy_register_state(&env->true_reg2, src_reg); + env->false_reg1 = *dst_reg; + env->false_reg2 = *src_reg; + env->true_reg1 = *dst_reg; + env->true_reg2 = *src_reg; pred = is_branch_taken(env, dst_reg, src_reg, opcode, is_jmp32); if (pred >= 0) { /* If we get here with a dst_reg pointer type it is because @@ -15815,11 +15809,11 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, if (err) return err; - copy_register_state(dst_reg, &env->false_reg1); - copy_register_state(src_reg, &env->false_reg2); - copy_register_state(&other_branch_regs[insn->dst_reg], &env->true_reg1); + *dst_reg = env->false_reg1; + *src_reg = env->false_reg2; + other_branch_regs[insn->dst_reg] = env->true_reg1; if (BPF_SRC(insn->code) == BPF_X) - copy_register_state(&other_branch_regs[insn->src_reg], &env->true_reg2); + other_branch_regs[insn->src_reg] = env->true_reg2; if (BPF_SRC(insn->code) == BPF_X && src_reg->type == SCALAR_VALUE && src_reg->id && -- cgit v1.2.3 From 3ab5bd317ee280b198b00ea2114adaad7a458ef8 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 12 May 2026 21:50:10 -0700 Subject: bpf: Set sub->arg_cnt earlier in btf_prepare_func_args() Move the "sub->arg_cnt = nargs" assignment to immediately after nargs is computed from btf_type_vlen(), instead of at the end of btf_prepare_func_args(). btf_prepare_func_args() can return -EINVAL early in several cases, e.g. when a static function has some non-int/enum arguments. Since -EINVAL from btf_prepare_func_args() does not immediately reject verification, arg_cnt remains zero after the early return. This causes later stack argument based load/store insns to incorrectly assume the function has no arguments. Setting arg_cnt right after nargs ensures it is available regardless of which path btf_prepare_func_args() takes. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260513045010.2384635-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/btf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index a6bf4781943c..099d7ca5a980 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7864,6 +7864,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) } args = (const struct btf_param *)(t + 1); nargs = btf_type_vlen(t); + sub->arg_cnt = nargs; if (nargs > MAX_BPF_FUNC_REG_ARGS) { if (!is_global) return -EINVAL; @@ -8051,7 +8052,6 @@ skip_pointer: return -EINVAL; } - sub->arg_cnt = nargs; sub->args_cached = true; return 0; -- cgit v1.2.3 From 0f6bd5e7a804af27e7f34b8306afde7a6b269318 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 12 May 2026 21:50:15 -0700 Subject: bpf: Support stack arguments for bpf functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently BPF functions (subprogs) are limited to 5 register arguments. With [1], the compiler can emit code that passes additional arguments via a dedicated stack area through bpf register BPF_REG_PARAMS (r11), introduced in an earlier patch ([2]). The compiler uses positive r11 offsets for incoming (callee-side) args and negative r11 offsets for outgoing (caller-side) args, following the x86_64/arm64 calling convention direction. There is an 8-byte gap at offset 0 separating two regions: Incoming (callee reads): r11+8 (arg6), r11+16 (arg7), ... Outgoing (caller writes): r11-8 (arg6), r11-16 (arg7), ... The following is an example to show how stack arguments are saved and transferred between caller and callee: int foo(int a1, int a2, int a3, int a4, int a5, int a6, int a7) { ... bar(a1, a2, a3, a4, a5, a6, a7, a8); ... } Caller (foo) Callee (bar) ============ ============ Incoming (positive offsets): Incoming (positive offsets): r11+8: [incoming arg 6] r11+8: [incoming arg 6] <-+ r11+16: [incoming arg 7] r11+16: [incoming arg 7] <-|+ r11+24: [incoming arg 8] <-||+ Outgoing (negative offsets): ||| r11-8: [outgoing arg 6 to bar] -------->-------------------------+|| r11-16: [outgoing arg 7 to bar] -------->--------------------------+| r11-24: [outgoing arg 8 to bar] -------->---------------------------+ If the bpf function has more than one call: int foo(int a1, int a2, int a3, int a4, int a5, int a6, int a7) { ... bar1(a1, a2, a3, a4, a5, a6, a7, a8); ... bar2(a1, a2, a3, a4, a5, a6, a7, a8, a9); ... } Caller (foo) Callee (bar2) ============ ============== Incoming (positive offsets): Incoming (positive offsets): r11+8: [incoming arg 6] r11+8: [incoming arg 6] <+ r11+16: [incoming arg 7] r11+16: [incoming arg 7] <|+ r11+24: [incoming arg 8] <||+ Outgoing for bar2 (negative offsets): r11+32: [incoming arg 9] <|||+ r11-8: [outgoing arg 6] ---->----------->-------------------------+||| r11-16: [outgoing arg 7] ---->----------->--------------------------+|| r11-24: [outgoing arg 8] ---->----------->---------------------------+| r11-32: [outgoing arg 9] ---->----------->----------------------------+ The verifier tracks outgoing stack arguments in stack_arg_regs[] and out_stack_arg_cnt in bpf_func_state, separately from the regular r10 stack. The callee does not copy incoming args — it reads them directly from the caller's outgoing slots at positive r11 offsets. Similar to stacksafe(), introduce stack_arg_safe() to do pruning check. Outgoing stack arg slots are invalidated when the callee returns (e.g. in prepare_func_exit), not at call time. This allows the callee to read incoming args from the caller's outgoing slots during verification. The following are a few examples. Example 1: *(u64 *)(r11 - 8) = r6; *(u64 *)(r11 - 16) = r7; call bar1; // arg6 = r6, arg7 = r7 call bar2; // expected with 2 stack arguments, failed Example 2: To fix the Example 1: *(u64 *)(r11 - 8) = r6; *(u64 *)(r11 - 16) = r7; call bar1; // arg6 = r6, arg7 = r7 *(u64 *)(r11 - 8) = r8; *(u64 *)(r11 - 16) = r9; call bar2; // arg6 = r8, arg7 = r9 Example 3: The compiler can hoist the shared stack arg stores above the branch: *(u64 *)(r11 - 16) = r7; if cond goto else; *(u64 *)(r11 - 8) = r8; call bar1; // arg6 = r8, arg7 = r7 goto end; else: *(u64 *)(r11 - 8) = r9; call bar2; // arg6 = r9, arg7 = r7 end: Example 4: Within a loop: loop: *(u64 *)(r11 - 8) = r6; // arg6, before loop call bar; // reuses arg6 each iteration if ... goto loop; A separate max_out_stack_arg_cnt field in bpf_subprog_info tracks the deepest outgoing slot actually written. This intends to reject programs that write to slots beyond what any callee expects. It is necessary for JIT. Similar to typical compiler generated code, enforce the following orderings: - all stack arg reads must be ahead of any stack arg write - all stack arg reads must be before any bpf func, kfunc and helpers This is needed as JIT may emit 'mov' insns for read/write with the same register and bpf function, kfunc and helper will invalidate all arguments immediately after the call. Callback functions with stack arguments need kernel setup parameter types (including stack parameters) properly and then callback function can retrieve such information for verification purpose. Global subprogs and freplace with >5 args are not yet supported. [1] https://github.com/llvm/llvm-project/pull/189060 [2] https://lore.kernel.org/bpf/20260423033506.2542005-1-yonghong.song@linux.dev/ Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260513045015.2385013-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/btf.c | 14 +++- kernel/bpf/fixups.c | 16 +++- kernel/bpf/states.c | 32 ++++++++ kernel/bpf/verifier.c | 203 +++++++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 255 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 099d7ca5a980..4fb8641546b8 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7865,10 +7865,16 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) args = (const struct btf_param *)(t + 1); nargs = btf_type_vlen(t); sub->arg_cnt = nargs; - if (nargs > MAX_BPF_FUNC_REG_ARGS) { - if (!is_global) - return -EINVAL; - bpf_log(log, "Global function %s() with %d > %d args. Buggy compiler.\n", + if (nargs > MAX_BPF_FUNC_ARGS) { + bpf_log(log, "kernel supports at most %d parameters, function %s has %d\n", + MAX_BPF_FUNC_ARGS, tname, nargs); + return -EFAULT; + } + if (nargs > MAX_BPF_FUNC_REG_ARGS) + sub->stack_arg_cnt = nargs - MAX_BPF_FUNC_REG_ARGS; + + if (is_global && nargs > MAX_BPF_FUNC_REG_ARGS) { + bpf_log(log, "global function %s has %d > %d args, stack args not supported\n", tname, nargs, MAX_BPF_FUNC_REG_ARGS); return -EINVAL; } diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index fba9e8c00878..ba86039789fd 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -1378,9 +1378,21 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env) struct bpf_prog *prog = env->prog; struct bpf_insn *insn = prog->insnsi; bool has_kfunc_call = bpf_prog_has_kfunc_call(prog); - int i, depth; + int depth; #endif - int err = 0; + int i, err = 0; + + for (i = 0; i < env->subprog_cnt; i++) { + struct bpf_subprog_info *subprog = &env->subprog_info[i]; + u16 outgoing = subprog->stack_arg_cnt - bpf_in_stack_arg_cnt(subprog); + + if (subprog->max_out_stack_arg_cnt > outgoing) { + verbose(env, + "func#%d writes %u stack arg slots, but calls only require %u\n", + i, subprog->max_out_stack_arg_cnt, outgoing); + return -EINVAL; + } + } if (env->prog->jit_requested && !bpf_prog_is_offloaded(env->prog->aux)) { diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index bd9c22945050..3ce6d2652b27 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -833,6 +833,32 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, return true; } +/* + * Compare stack arg slots between old and current states. + * Outgoing stack args are path-local state and must agree for pruning. + */ +static bool stack_arg_safe(struct bpf_verifier_env *env, struct bpf_func_state *old, + struct bpf_func_state *cur, struct bpf_idmap *idmap, + enum exact_level exact) +{ + int i, nslots; + + nslots = max(old->out_stack_arg_cnt, cur->out_stack_arg_cnt); + for (i = 0; i < nslots; i++) { + struct bpf_reg_state *old_arg, *cur_arg; + struct bpf_reg_state not_init = { .type = NOT_INIT }; + + old_arg = i < old->out_stack_arg_cnt ? + &old->stack_arg_regs[i] : ¬_init; + cur_arg = i < cur->out_stack_arg_cnt ? + &cur->stack_arg_regs[i] : ¬_init; + if (!regsafe(env, old_arg, cur_arg, idmap, exact)) + return false; + } + + return true; +} + static bool refsafe(struct bpf_verifier_state *old, struct bpf_verifier_state *cur, struct bpf_idmap *idmap) { @@ -915,6 +941,9 @@ static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_stat if (old->callback_depth > cur->callback_depth) return false; + if (!old->no_stack_arg_load && cur->no_stack_arg_load) + return false; + for (i = 0; i < MAX_BPF_REG; i++) if (((1 << i) & live_regs) && !regsafe(env, &old->regs[i], &cur->regs[i], @@ -924,6 +953,9 @@ static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_stat if (!stacksafe(env, old, cur, &env->idmap_scratch, exact)) return false; + if (!stack_arg_safe(env, old, cur, &env->idmap_scratch, exact)) + return false; + return true; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 1dd9736c2a13..a29b3003cbec 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -1362,6 +1362,18 @@ static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_st return -ENOMEM; dst->allocated_stack = src->allocated_stack; + + /* copy stack args state */ + n = src->out_stack_arg_cnt; + if (n) { + dst->stack_arg_regs = copy_array(dst->stack_arg_regs, src->stack_arg_regs, n, + sizeof(struct bpf_reg_state), + GFP_KERNEL_ACCOUNT); + if (!dst->stack_arg_regs) + return -ENOMEM; + } + + dst->out_stack_arg_cnt = src->out_stack_arg_cnt; return 0; } @@ -1403,6 +1415,23 @@ static int grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state return 0; } +static int grow_stack_arg_slots(struct bpf_verifier_env *env, + struct bpf_func_state *state, int cnt) +{ + size_t old_n = state->out_stack_arg_cnt; + + if (old_n >= cnt) + return 0; + + state->stack_arg_regs = realloc_array(state->stack_arg_regs, old_n, cnt, + sizeof(struct bpf_reg_state)); + if (!state->stack_arg_regs) + return -ENOMEM; + + state->out_stack_arg_cnt = cnt; + return 0; +} + /* Acquire a pointer id from the env and update the state->refs to include * this new pointer reference. * On success, returns a valid pointer id to associate with the register @@ -1565,6 +1594,7 @@ static void free_func_state(struct bpf_func_state *state) { if (!state) return; + kfree(state->stack_arg_regs); kfree(state->stack); kfree(state); } @@ -4050,6 +4080,103 @@ static int check_stack_write(struct bpf_verifier_env *env, return err; } +/* + * Write a value to the outgoing stack arg area. + * off is a negative offset from r11 (e.g. -8 for arg6, -16 for arg7). + */ +static int check_stack_arg_write(struct bpf_verifier_env *env, struct bpf_func_state *state, + int off, struct bpf_reg_state *value_reg) +{ + int max_stack_arg_regs = MAX_BPF_FUNC_ARGS - MAX_BPF_FUNC_REG_ARGS; + struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno]; + int spi = -off / BPF_REG_SIZE - 1; + struct bpf_reg_state *arg; + int err; + + if (spi >= max_stack_arg_regs) { + verbose(env, "stack arg write offset %d exceeds max %d stack args\n", + off, max_stack_arg_regs); + return -EINVAL; + } + + err = grow_stack_arg_slots(env, state, spi + 1); + if (err) + return err; + + /* Track the max outgoing stack arg slot count. */ + if (spi + 1 > subprog->max_out_stack_arg_cnt) + subprog->max_out_stack_arg_cnt = spi + 1; + + if (value_reg) { + state->stack_arg_regs[spi] = *value_reg; + } else { + /* BPF_ST: store immediate, treat as scalar */ + arg = &state->stack_arg_regs[spi]; + arg->type = SCALAR_VALUE; + __mark_reg_known(arg, env->prog->insnsi[env->insn_idx].imm); + } + state->no_stack_arg_load = true; + return 0; +} + +/* + * Read a value from the incoming stack arg area. + * off is a positive offset from r11 (e.g. +8 for arg6, +16 for arg7). + */ +static int check_stack_arg_read(struct bpf_verifier_env *env, struct bpf_func_state *state, + int off, int dst_regno) +{ + struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno]; + struct bpf_verifier_state *vstate = env->cur_state; + int spi = off / BPF_REG_SIZE - 1; + struct bpf_func_state *caller, *cur; + struct bpf_reg_state *arg; + + if (state->no_stack_arg_load) { + verbose(env, "r11 load must be before any r11 store or call insn\n"); + return -EINVAL; + } + + if (spi + 1 > bpf_in_stack_arg_cnt(subprog)) { + verbose(env, "invalid read from stack arg off %d depth %d\n", + off, bpf_in_stack_arg_cnt(subprog) * BPF_REG_SIZE); + return -EACCES; + } + + caller = vstate->frame[vstate->curframe - 1]; + arg = &caller->stack_arg_regs[spi]; + cur = vstate->frame[vstate->curframe]; + cur->regs[dst_regno] = *arg; + return 0; +} + +static int check_outgoing_stack_args(struct bpf_verifier_env *env, struct bpf_func_state *caller, + int nargs) +{ + int i, spi; + + for (i = MAX_BPF_FUNC_REG_ARGS; i < nargs; i++) { + spi = i - MAX_BPF_FUNC_REG_ARGS; + if (spi >= caller->out_stack_arg_cnt || + caller->stack_arg_regs[spi].type == NOT_INIT) { + verbose(env, "callee expects %d args, stack arg%d is not initialized\n", + nargs, spi + 1); + return -EFAULT; + } + } + + return 0; +} + +static struct bpf_reg_state *get_func_arg_reg(struct bpf_func_state *caller, + struct bpf_reg_state *regs, int arg) +{ + if (arg < MAX_BPF_FUNC_REG_ARGS) + return ®s[arg + 1]; + + return &caller->stack_arg_regs[arg - MAX_BPF_FUNC_REG_ARGS]; +} + static int check_map_access_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int off, int size, enum bpf_access_type type) { @@ -6217,10 +6344,20 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, bool strict_alignment_once, bool is_ldsx, bool allow_trust_mismatch, const char *ctx) { + struct bpf_verifier_state *vstate = env->cur_state; + struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = cur_regs(env); enum bpf_reg_type src_reg_type; int err; + /* Handle stack arg read */ + if (is_stack_arg_ldx(insn)) { + err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); + if (err) + return err; + return check_stack_arg_read(env, state, insn->off, insn->dst_reg); + } + /* check src operand */ err = check_reg_arg(env, insn->src_reg, SRC_OP); if (err) @@ -6249,10 +6386,20 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, bool strict_alignment_once) { + struct bpf_verifier_state *vstate = env->cur_state; + struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = cur_regs(env); enum bpf_reg_type dst_reg_type; int err; + /* Handle stack arg write */ + if (is_stack_arg_stx(insn)) { + err = check_reg_arg(env, insn->src_reg, SRC_OP); + if (err) + return err; + return check_stack_arg_write(env, state, insn->off, regs + insn->src_reg); + } + /* check src1 operand */ err = check_reg_arg(env, insn->src_reg, SRC_OP); if (err) @@ -8860,6 +9007,15 @@ static void clear_caller_saved_regs(struct bpf_verifier_env *env, } } +static void invalidate_outgoing_stack_args(const struct bpf_verifier_env *env, + struct bpf_func_state *state) +{ + int i, nslots = state->out_stack_arg_cnt; + + for (i = 0; i < nslots; i++) + bpf_mark_reg_not_init(env, &state->stack_arg_regs[i]); +} + typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env, struct bpf_func_state *caller, struct bpf_func_state *callee, @@ -8922,6 +9078,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, struct bpf_reg_state *regs) { struct bpf_subprog_info *sub = subprog_info(env, subprog); + struct bpf_func_state *caller = cur_func(env); struct bpf_verifier_log *log = &env->log; u32 i; int ret; @@ -8930,13 +9087,16 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, if (ret) return ret; + ret = check_outgoing_stack_args(env, caller, sub->arg_cnt); + if (ret) + return ret; + /* check that BTF function arguments match actual types that the * verifier sees. */ for (i = 0; i < sub->arg_cnt; i++) { argno_t argno = argno_from_arg(i + 1); - u32 regno = i + 1; - struct bpf_reg_state *reg = ®s[regno]; + struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, i); struct bpf_subprog_arg_info *arg = &sub->args[i]; if (arg->arg_type == ARG_ANYTHING) { @@ -9124,6 +9284,8 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, int *insn_idx) { struct bpf_verifier_state *state = env->cur_state; + struct bpf_subprog_info *caller_info; + u16 callee_incoming, stack_arg_cnt; struct bpf_func_state *caller; int err, subprog, target_insn; @@ -9166,6 +9328,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, /* mark global subprog for verifying after main prog */ subprog_aux(env, subprog)->called = true; clear_caller_saved_regs(env, caller->regs); + invalidate_outgoing_stack_args(env, cur_func(env)); /* All non-void global functions return a 64-bit SCALAR_VALUE. */ if (!subprog_returns_void(env, subprog)) { @@ -9177,6 +9340,16 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return 0; } + /* + * Track caller's total stack arg count (incoming + max outgoing). + * This is needed so the JIT knows how much stack arg space to allocate. + */ + caller_info = &env->subprog_info[caller->subprogno]; + callee_incoming = bpf_in_stack_arg_cnt(&env->subprog_info[subprog]); + stack_arg_cnt = bpf_in_stack_arg_cnt(caller_info) + callee_incoming; + if (stack_arg_cnt > caller_info->stack_arg_cnt) + caller_info->stack_arg_cnt = stack_arg_cnt; + /* for regular function entry setup new frame and continue * from that frame. */ @@ -9534,6 +9707,7 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) * bpf_throw, this will be done by copy_verifier_state for extra frames. */ free_func_state(callee); state->frame[state->curframe--] = NULL; + invalidate_outgoing_stack_args(env, caller); /* for callbacks widen imprecise scalars to make programs like below verify: * @@ -10160,6 +10334,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn bpf_mark_reg_not_init(env, ®s[caller_saved[i]]); check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); } + invalidate_outgoing_stack_args(env, cur_func(env)); /* helper call returns 64-bit value. */ regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; @@ -12842,6 +13017,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, bpf_mark_reg_not_init(env, ®s[regno]); regs[regno].subreg_def = DEF_NOT_SUBREG; } + invalidate_outgoing_stack_args(env, cur_func(env)); /* Check return type */ t = btf_type_skip_modifiers(desc_btf, meta.func_proto->type, NULL); @@ -16961,6 +17137,14 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) return check_store_reg(env, insn, false); case BPF_ST: { + /* Handle stack arg write (store immediate) */ + if (is_stack_arg_st(insn)) { + struct bpf_verifier_state *vstate = env->cur_state; + struct bpf_func_state *state = vstate->frame[vstate->curframe]; + + return check_stack_arg_write(env, state, insn->off, NULL); + } + enum bpf_reg_type dst_reg_type; err = check_reg_arg(env, insn->dst_reg, SRC_OP); @@ -16995,6 +17179,8 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) } } mark_reg_scratched(env, BPF_REG_0); + if (bpf_in_stack_arg_cnt(&env->subprog_info[cur_func(env)->subprogno])) + cur_func(env)->no_stack_arg_load = true; if (insn->src_reg == BPF_PSEUDO_CALL) return check_func_call(env, insn, &env->insn_idx); if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) @@ -18110,7 +18296,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) goto out; } } - for (i = BPF_REG_1; i <= sub->arg_cnt; i++) { + for (i = BPF_REG_1; i <= min_t(u32, sub->arg_cnt, MAX_BPF_FUNC_REG_ARGS); i++) { arg = &sub->args[i - BPF_REG_1]; reg = ®s[i]; @@ -18153,6 +18339,12 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) goto out; } } + if (env->prog->type == BPF_PROG_TYPE_EXT && sub->arg_cnt > MAX_BPF_FUNC_REG_ARGS) { + verbose(env, "freplace programs with >%d args not supported yet\n", + MAX_BPF_FUNC_REG_ARGS); + ret = -EINVAL; + goto out; + } } else { /* if main BPF program has associated BTF info, validate that * it's matching expected signature, and otherwise mark BTF @@ -18160,8 +18352,11 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) */ if (env->prog->aux->func_info_aux) { ret = btf_prepare_func_args(env, 0); - if (ret || sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_PTR_TO_CTX) + if (ret || sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_PTR_TO_CTX) { env->prog->aux->func_info_aux[0].unreliable = true; + sub->arg_cnt = 1; + sub->stack_arg_cnt = 0; + } } /* 1st arg to a function */ -- cgit v1.2.3 From 3a656670fd6da624f6241038ca4cf350f24fd5e8 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 12 May 2026 21:50:20 -0700 Subject: bpf: Refactor jmp history to use dedicated spi/frame fields Move stack slot index (spi) and frame number out of the flags field in bpf_jmp_history_entry into dedicated bitfields. This simplifies the encoding and makes room for new flags. Previously, spi and frame were packed into the lower 9 bits of the 12-bit flags field (3 bits frame + 6 bits spi), with INSN_F_STACK_ACCESS at BIT(9) and INSN_F_DST/SRC_REG_STACK at BIT(10)/BIT(11). But this has no room for an INSN_F_* flag for stack arguments. To resolve this issue, bpf_jmp_history_entry field idx is narrowed to 20 bits (sufficient for insn indices up to 1M), and the freed bits hold spi (6 bits) and frame (3 bits) as dedicated struct fields. The flags enum is simplified accordingly: INSN_F_STACK_ACCESS -> BIT(0) INSN_F_DST_REG_STACK -> BIT(1) INSN_F_SRC_REG_STACK -> BIT(2) which allows more room for additional INSN_F_* flags. bpf_push_jmp_history() now takes explicit spi and frame parameters instead of encoding them into flags. The insn_stack_access_flags(), insn_stack_access_spi(), and insn_stack_access_frameno() helpers are removed. No functional change. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260513045020.2385962-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/backtrack.c | 24 +++++++++--------------- kernel/bpf/states.c | 2 +- kernel/bpf/verifier.c | 23 +++++++++++------------ 3 files changed, 21 insertions(+), 28 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c index 854731dc93fe..5e93e57fb7ae 100644 --- a/kernel/bpf/backtrack.c +++ b/kernel/bpf/backtrack.c @@ -9,7 +9,7 @@ /* for any branch, call, exit record the history of jmps in the given state */ int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur, - int insn_flags, u64 linked_regs) + int insn_flags, int spi, int frame, u64 linked_regs) { u32 cnt = cur->jmp_history_cnt; struct bpf_jmp_history_entry *p; @@ -25,6 +25,8 @@ int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state env, "insn history: insn_idx %d cur flags %x new flags %x", env->insn_idx, env->cur_hist_ent->flags, insn_flags); env->cur_hist_ent->flags |= insn_flags; + env->cur_hist_ent->spi = spi; + env->cur_hist_ent->frame = frame; verifier_bug_if(env->cur_hist_ent->linked_regs != 0, env, "insn history: insn_idx %d linked_regs: %#llx", env->insn_idx, env->cur_hist_ent->linked_regs); @@ -43,6 +45,8 @@ int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state p->idx = env->insn_idx; p->prev_idx = env->prev_insn_idx; p->flags = insn_flags; + p->spi = spi; + p->frame = frame; p->linked_regs = linked_regs; cur->jmp_history_cnt = cnt; env->cur_hist_ent = p; @@ -64,16 +68,6 @@ static bool is_atomic_fetch_insn(const struct bpf_insn *insn) (insn->imm & BPF_FETCH); } -static int insn_stack_access_spi(int insn_flags) -{ - return (insn_flags >> INSN_F_SPI_SHIFT) & INSN_F_SPI_MASK; -} - -static int insn_stack_access_frameno(int insn_flags) -{ - return insn_flags & INSN_F_FRAMENO_MASK; -} - /* Backtrack one insn at a time. If idx is not at the top of recorded * history then previous instruction came from straight line execution. * Return -ENOENT if we exhausted all instructions within given state. @@ -353,8 +347,8 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, * that [fp - off] slot contains scalar that needs to be * tracked with precision */ - spi = insn_stack_access_spi(hist->flags); - fr = insn_stack_access_frameno(hist->flags); + spi = hist->spi; + fr = hist->frame; bpf_bt_set_frame_slot(bt, fr, spi); } else if (class == BPF_STX || class == BPF_ST) { if (bt_is_reg_set(bt, dreg)) @@ -366,8 +360,8 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, /* scalars can only be spilled into stack */ if (!hist || !(hist->flags & INSN_F_STACK_ACCESS)) return 0; - spi = insn_stack_access_spi(hist->flags); - fr = insn_stack_access_frameno(hist->flags); + spi = hist->spi; + fr = hist->frame; if (!bt_is_frame_slot_set(bt, fr, spi)) return 0; bt_clear_frame_slot(bt, fr, spi); diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 3ce6d2652b27..877338136009 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -1403,7 +1403,7 @@ hit: */ err = 0; if (bpf_is_jmp_point(env, env->insn_idx)) - err = bpf_push_jmp_history(env, cur, 0, 0); + err = bpf_push_jmp_history(env, cur, 0, 0, 0, 0); err = err ? : propagate_precision(env, &sl->state, cur, NULL); if (err) return err; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a29b3003cbec..d15aef2fe4a1 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3198,11 +3198,6 @@ static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, return __check_reg_arg(env, state->regs, regno, t); } -static int insn_stack_access_flags(int frameno, int spi) -{ - return INSN_F_STACK_ACCESS | (spi << INSN_F_SPI_SHIFT) | frameno; -} - static void mark_indirect_target(struct bpf_verifier_env *env, int idx) { env->insn_aux_data[idx].indirect_target = true; @@ -3517,7 +3512,8 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err; struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; struct bpf_reg_state *reg = NULL; - int insn_flags = insn_stack_access_flags(state->frameno, spi); + int insn_flags = INSN_F_STACK_ACCESS; + int hist_spi = spi, hist_frame = state->frameno; /* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0, * so it's aligned access and [off, off + size) are within stack limits @@ -3613,7 +3609,8 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, } if (insn_flags) - return bpf_push_jmp_history(env, env->cur_state, insn_flags, 0); + return bpf_push_jmp_history(env, env->cur_state, insn_flags, + hist_spi, hist_frame, 0); return 0; } @@ -3809,7 +3806,8 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, int i, slot = -off - 1, spi = slot / BPF_REG_SIZE; struct bpf_reg_state *reg; u8 *stype, type; - int insn_flags = insn_stack_access_flags(reg_state->frameno, spi); + int insn_flags = INSN_F_STACK_ACCESS; + int hist_spi = spi, hist_frame = reg_state->frameno; stype = reg_state->stack[spi].slot_type; reg = ®_state->stack[spi].spilled_ptr; @@ -3940,7 +3938,8 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, insn_flags = 0; /* we are not restoring spilled register */ } if (insn_flags) - return bpf_push_jmp_history(env, env->cur_state, insn_flags, 0); + return bpf_push_jmp_history(env, env->cur_state, insn_flags, + hist_spi, hist_frame, 0); return 0; } @@ -15907,7 +15906,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, } if (insn_flags) { - err = bpf_push_jmp_history(env, this_branch, insn_flags, 0); + err = bpf_push_jmp_history(env, this_branch, insn_flags, 0, 0, 0); if (err) return err; } @@ -15971,7 +15970,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, if (dst_reg->type == SCALAR_VALUE && dst_reg->id) collect_linked_regs(env, this_branch, dst_reg->id, &linked_regs); if (linked_regs.cnt > 1) { - err = bpf_push_jmp_history(env, this_branch, 0, linked_regs_pack(&linked_regs)); + err = bpf_push_jmp_history(env, this_branch, 0, 0, 0, linked_regs_pack(&linked_regs)); if (err) return err; } @@ -17278,7 +17277,7 @@ static int do_check(struct bpf_verifier_env *env) } if (bpf_is_jmp_point(env, env->insn_idx)) { - err = bpf_push_jmp_history(env, state, 0, 0); + err = bpf_push_jmp_history(env, state, 0, 0, 0, 0); if (err) return err; } -- cgit v1.2.3 From 0a0fdc64b68c28dab40f9deb0cffdf544e04b0ba Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 12 May 2026 21:50:25 -0700 Subject: bpf: Add precision marking and backtracking for stack argument slots Extend the precision marking and backtracking infrastructure to support stack argument slots (r11-based accesses). Without this, precision demands for scalar values passed through stack arguments are silently dropped, which could allow the verifier to incorrectly prune states with different constant values in stack arg slots. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260513045025.2387526-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/backtrack.c | 58 +++++++++++++++++++++++++++++++++++++++++++++++++- kernel/bpf/verifier.c | 32 +++++++++++++++++++++++----- 2 files changed, 84 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c index 5e93e57fb7ae..2e4ae0ef0860 100644 --- a/kernel/bpf/backtrack.c +++ b/kernel/bpf/backtrack.c @@ -129,11 +129,21 @@ static inline u32 bt_empty(struct backtrack_state *bt) int i; for (i = 0; i <= bt->frame; i++) - mask |= bt->reg_masks[i] | bt->stack_masks[i]; + mask |= bt->reg_masks[i] | bt->stack_masks[i] | bt->stack_arg_masks[i]; return mask == 0; } +static inline void bt_clear_frame_stack_arg_slot(struct backtrack_state *bt, u32 frame, u32 slot) +{ + bt->stack_arg_masks[frame] &= ~(1 << slot); +} + +static inline bool bt_is_frame_stack_arg_slot_set(struct backtrack_state *bt, u32 frame, u32 slot) +{ + return bt->stack_arg_masks[frame] & (1 << slot); +} + static inline int bt_subprog_enter(struct backtrack_state *bt) { if (bt->frame == MAX_CALL_FRAMES - 1) { @@ -194,6 +204,11 @@ static inline u64 bt_stack_mask(struct backtrack_state *bt) return bt->stack_masks[bt->frame]; } +static inline u8 bt_stack_arg_mask(struct backtrack_state *bt) +{ + return bt->stack_arg_masks[bt->frame]; +} + static inline bool bt_is_reg_set(struct backtrack_state *bt, u32 reg) { return bt->reg_masks[bt->frame] & (1 << reg); @@ -335,6 +350,19 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, return 0; bt_clear_reg(bt, load_reg); + if (hist && hist->flags & INSN_F_STACK_ARG_ACCESS) { + spi = hist->spi; + /* + * Stack arg read: callee reads from r11+off, but + * the data lives in the caller's stack_arg_regs. + * Set the mask in the caller frame so precision + * is marked in the caller's slot at the callee + * entry checkpoint. + */ + bt_set_frame_stack_arg_slot(bt, bt->frame - 1, spi); + return 0; + } + /* scalars can only be spilled into stack w/o losing precision. * Load from any other memory can be zero extended. * The desire to keep that precision is already indicated @@ -357,6 +385,17 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, * encountered a case of pointer subtraction. */ return -ENOTSUPP; + + if (hist && hist->flags & INSN_F_STACK_ARG_ACCESS) { + spi = hist->spi; + if (!bt_is_frame_stack_arg_slot_set(bt, bt->frame, spi)) + return 0; + bt_clear_frame_stack_arg_slot(bt, bt->frame, spi); + if (class == BPF_STX) + bt_set_reg(bt, sreg); + return 0; + } + /* scalars can only be spilled into stack */ if (!hist || !(hist->flags & INSN_F_STACK_ACCESS)) return 0; @@ -425,6 +464,12 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, bpf_bt_set_frame_reg(bt, bt->frame - 1, i); } } + if (bt_stack_arg_mask(bt)) { + verifier_bug(env, + "static subprog leftover stack arg slots %x", + bt_stack_arg_mask(bt)); + return -EFAULT; + } if (bt_subprog_exit(bt)) return -EFAULT; return 0; @@ -895,6 +940,17 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env, *changed = true; } } + for (i = 0; i < func->out_stack_arg_cnt; i++) { + if (!bt_is_frame_stack_arg_slot_set(bt, fr, i)) + continue; + reg = &func->stack_arg_regs[i]; + if (reg->type != SCALAR_VALUE || reg->precise) { + bt_clear_frame_stack_arg_slot(bt, fr, i); + } else { + reg->precise = true; + *changed = true; + } + } if (env->log.level & BPF_LOG_LEVEL2) { fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_frame_reg_mask(bt, fr)); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d15aef2fe4a1..ebd13661933e 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -292,6 +292,11 @@ static int arg_from_argno(argno_t a) return -1; } +static int arg_idx_from_argno(argno_t a) +{ + return arg_from_argno(a) - 1; +} + static const char *btf_type_name(const struct btf *btf, u32 id) { return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off); @@ -4115,7 +4120,8 @@ static int check_stack_arg_write(struct bpf_verifier_env *env, struct bpf_func_s __mark_reg_known(arg, env->prog->insnsi[env->insn_idx].imm); } state->no_stack_arg_load = true; - return 0; + return bpf_push_jmp_history(env, env->cur_state, + INSN_F_STACK_ARG_ACCESS, spi, 0, 0); } /* @@ -4146,7 +4152,17 @@ static int check_stack_arg_read(struct bpf_verifier_env *env, struct bpf_func_st arg = &caller->stack_arg_regs[spi]; cur = vstate->frame[vstate->curframe]; cur->regs[dst_regno] = *arg; - return 0; + return bpf_push_jmp_history(env, env->cur_state, + INSN_F_STACK_ARG_ACCESS, spi, 0, 0); +} + +static int mark_stack_arg_precision(struct bpf_verifier_env *env, int arg_idx) +{ + struct bpf_func_state *caller = cur_func(env); + int spi = arg_idx - MAX_BPF_FUNC_REG_ARGS; + + bt_set_frame_stack_arg_slot(&env->bt, caller->frameno, spi); + return mark_chain_precision_batch(env, env->cur_state); } static int check_outgoing_stack_args(struct bpf_verifier_env *env, struct bpf_func_state *caller, @@ -6875,8 +6891,14 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, } err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg), access_type, zero_size_allowed, meta); - if (!err) - err = mark_chain_precision(env, reg_from_argno(size_argno)); + if (!err) { + int regno = reg_from_argno(size_argno); + + if (regno >= 0) + err = mark_chain_precision(env, regno); + else + err = mark_stack_arg_precision(env, arg_idx_from_argno(size_argno)); + } return err; } @@ -7325,7 +7347,7 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * struct bpf_kfunc_call_arg_meta *meta) { const struct btf_type *t; - u32 arg_idx = arg_from_argno(argno) - 1; + u32 arg_idx = arg_idx_from_argno(argno); int spi, err, i, nr_slots, btf_id; if (reg->type != PTR_TO_STACK) { -- cgit v1.2.3 From 84dd7df76efef9fecb6f3e0defe2ea3ad89cd3cb Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 12 May 2026 21:50:30 -0700 Subject: bpf: Refactor record_call_access() to extract per-arg logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract the per-argument FP-derived pointer handling from record_call_access() into a new record_arg_access() helper. The existing loop body — checking arg_is_fp, querying stack access bytes, and calling record_stack_access/record_imprecise — will be reused for stack argument slots in the next patch. Factoring it out now avoids duplicating the logic. No functional change. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260513045030.2388067-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/liveness.c | 65 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 27 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 58197d73b120..c81337dfbfc7 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -1343,6 +1343,42 @@ static int record_load_store_access(struct bpf_verifier_env *env, return 0; } +static int record_arg_access(struct bpf_verifier_env *env, + struct func_instance *instance, + struct bpf_insn *insn, + struct arg_track *at, int arg_idx, + int insn_idx) +{ + int depth = instance->depth; + int frame = at->frame; + int err = 0; + s64 bytes; + + if (!arg_is_fp(at)) + return 0; + + if (bpf_helper_call(insn)) { + bytes = bpf_helper_stack_access_bytes(env, insn, arg_idx, insn_idx); + } else if (bpf_pseudo_kfunc_call(insn)) { + bytes = bpf_kfunc_stack_access_bytes(env, insn, arg_idx, insn_idx); + } else { + for (int f = 0; f <= depth; f++) { + err = mark_stack_read(instance, f, insn_idx, SPIS_ALL); + if (err) + return err; + } + return 0; + } + if (bytes == 0) + return 0; + + if (frame >= 0 && frame <= depth) + err = record_stack_access(instance, at, bytes, frame, insn_idx); + else if (frame == ARG_IMPRECISE) + err = record_imprecise(instance, at->mask, insn_idx); + return err; +} + /* Record stack access for a given 'at' state of helper/kfunc 'insn' */ static int record_call_access(struct bpf_verifier_env *env, struct func_instance *instance, @@ -1350,9 +1386,8 @@ static int record_call_access(struct bpf_verifier_env *env, int insn_idx) { struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; - int depth = instance->depth; struct bpf_call_summary cs; - int r, err = 0, num_params = 5; + int r, err, num_params = 5; if (bpf_pseudo_call(insn)) return 0; @@ -1361,31 +1396,7 @@ static int record_call_access(struct bpf_verifier_env *env, num_params = cs.num_params; for (r = BPF_REG_1; r < BPF_REG_1 + num_params; r++) { - int frame = at[r].frame; - s64 bytes; - - if (!arg_is_fp(&at[r])) - continue; - - if (bpf_helper_call(insn)) { - bytes = bpf_helper_stack_access_bytes(env, insn, r - 1, insn_idx); - } else if (bpf_pseudo_kfunc_call(insn)) { - bytes = bpf_kfunc_stack_access_bytes(env, insn, r - 1, insn_idx); - } else { - for (int f = 0; f <= depth; f++) { - err = mark_stack_read(instance, f, insn_idx, SPIS_ALL); - if (err) - return err; - } - return 0; - } - if (bytes == 0) - continue; - - if (frame >= 0 && frame <= depth) - err = record_stack_access(instance, &at[r], bytes, frame, insn_idx); - else if (frame == ARG_IMPRECISE) - err = record_imprecise(instance, at[r].mask, insn_idx); + err = record_arg_access(env, instance, insn, &at[r], r - 1, insn_idx); if (err) return err; } -- cgit v1.2.3 From f44e815e65a30f465fe4dd793df8cb98f1f9c0b1 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 12 May 2026 21:50:35 -0700 Subject: bpf: Use arg_is_fp() in has_fp_args() Replace "frame != ARG_NONE" with arg_is_fp() in has_fp_args(). The function's purpose is to check whether any argument is derived from a frame pointer, which is exactly what arg_is_fp() tests (frame >= 0 || frame == ARG_IMPRECISE). Using the dedicated predicate is clearer and more consistent with the rest of the file. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260513045035.2388671-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/liveness.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index c81337dfbfc7..13dc5ae44d2b 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -1689,7 +1689,7 @@ err_free: static bool has_fp_args(struct arg_track *args) { for (int r = BPF_REG_1; r <= BPF_REG_5; r++) - if (args[r].frame != ARG_NONE) + if (arg_is_fp(&args[r])) return true; return false; } -- cgit v1.2.3 From 2af4e792773f9fc05e5dbd5f297707cfe15cd817 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 12 May 2026 21:50:40 -0700 Subject: bpf: Extend liveness analysis to track stack argument slots BPF_REG_PARAMS (R11) is at index MAX_BPF_REG, which is beyond the register tracking arrays in const_fold.c and liveness.c. Handle it explicitly to avoid out-of-bounds accesses. Extend the arg tracking dataflow to cover stack arg slots. Otherwise, pointers passed through stack args are invisible to liveness, causing the pointed-to stack slots to be incorrectly poisoned. Extend the at_out tracking array to MAX_AT_TRACK_REGS (registers plus stack arg slots) so that outgoing stack arg stores are tracked alongside registers. Add a separate at_stack_arg_entry array in compute_subprog_args(), passed to arg_track_xfer(), to restore FP-derived values on incoming stack arg reads. Extend record_call_access() to check stack arg slots for FP-derived pointers at kfunc call sites, reusing the record_arg_access() helper extracted in the previous patch. Pass stack arg state from caller to callee in analyze_subprog() so that callees can track pointers received through stack args, hence avoid poisoning. Skip stack arg instructions in record_load_store_access(). Stack arg STX uses dst_reg=BPF_REG_PARAMS (index 11), but at[11] is repurposed to track the value stored in stack arg slot 0. Without the skip, if a prior stack arg STX stored an FP-derived pointer (e.g., fp-64) into slot 0, a subsequent stack arg STX would read that FP-derived value as the base pointer and spuriously mark a regular stack slot (e.g., fp-72 from -64 + -8) as accessed in the liveness bitmap. Extend arg_track_log() to log state transitions for outgoing stack arg slots at indices MAX_BPF_REG through MAX_AT_TRACK_REGS-1. Without this, changes to at_out[11..17] caused by stack arg store instructions are silently omitted from BPF_LOG_LEVEL2 output. For example, when a caller passes fp-64 through a stack argument: subprog#0: 10: (bf) r6 = r10 11: (07) r6 += -64 12: (7b) *(u64 *)(r11 -8) = r6 sa0: none -> fp0-64 13: (85) call pc+5 Without the fix, the "sa0: none -> fp0-64" transition at insn 12 would not appear. Extend print_subprog_arg_access() to include stack arg slots in the per-instruction FP-derived state dump. For example: subprog#0: 12: (7b) *(u64 *)(r11 - 8) = r6 // r6=fp0-64 13: (85) call pc+5 // r6=fp0-64 sa0=fp0-64 Without the fix, the "sa0=fp0-64" annotation at insn 13 would not appear, making it harder to debug liveness analysis for programs that pass FP-derived pointers through stack arguments. Extend has_fp_args() to also check stack arg slots for FP-derived pointers, so that callees receiving pointers only through stack args are still recursively analyzed. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260513045043.2389049-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/const_fold.c | 8 ++++ kernel/bpf/liveness.c | 114 +++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 106 insertions(+), 16 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/const_fold.c b/kernel/bpf/const_fold.c index db73c4740b1e..b2a19acadb91 100644 --- a/kernel/bpf/const_fold.c +++ b/kernel/bpf/const_fold.c @@ -58,6 +58,14 @@ static void const_reg_xfer(struct bpf_verifier_env *env, struct const_arg_info * u8 opcode = BPF_OP(insn->code) | BPF_SRC(insn->code); int r; + /* Stack arg stores (r11-based) are outside the tracked register set. */ + if (is_stack_arg_st(insn) || is_stack_arg_stx(insn)) + return; + if (is_stack_arg_ldx(insn)) { + ci_out[insn->dst_reg] = unknown; + return; + } + switch (class) { case BPF_ALU: case BPF_ALU64: diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 13dc5ae44d2b..7f4a0e4c2c49 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -610,6 +610,21 @@ enum arg_track_state { /* Track callee stack slots fp-8 through fp-512 (64 slots of 8 bytes each) */ #define MAX_ARG_SPILL_SLOTS 64 +/* + * Combined register + stack arg tracking: R0-R10 at indices 0-10, + * outgoing stack arg slots at indices MAX_BPF_REG..MAX_BPF_REG+6. + */ +#define MAX_AT_TRACK_REGS (MAX_BPF_REG + MAX_STACK_ARG_SLOTS) + +static int stack_arg_off_to_slot(s16 off) +{ + int aoff = off < 0 ? -off : off; + + if (aoff / 8 > MAX_STACK_ARG_SLOTS) + return -1; + return aoff / 8 - 1; +} + static bool arg_is_visited(const struct arg_track *at) { return at->frame != ARG_UNVISITED; @@ -1032,6 +1047,21 @@ static void arg_track_log(struct bpf_verifier_env *env, struct bpf_insn *insn, i verbose(env, "\tr%d: ", i); verbose_arg_track(env, &at_in[i]); verbose(env, " -> "); verbose_arg_track(env, &at_out[i]); } + /* Log outgoing stack arg slot transitions at indices MAX_BPF_REG..MAX_AT_TRACK_REGS-1 */ + for (i = 0; i < MAX_STACK_ARG_SLOTS; i++) { + int ai = MAX_BPF_REG + i; + + if (arg_track_eq(&at_out[ai], &at_in[ai])) + continue; + if (!printed) { + verbose(env, "%3d: ", idx); + bpf_verbose_insn(env, insn); + bpf_vlog_reset(&env->log, env->log.end_pos - 1); + printed = true; + } + verbose(env, "\tsa%d: ", i); verbose_arg_track(env, &at_in[ai]); + verbose(env, " -> "); verbose_arg_track(env, &at_out[ai]); + } for (i = 0; i < MAX_ARG_SPILL_SLOTS; i++) { if (arg_track_eq(&at_stack_out[i], &at_stack_in[i])) continue; @@ -1062,6 +1092,7 @@ static bool can_be_local_fp(int depth, int regno, struct arg_track *at) static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, int insn_idx, struct arg_track *at_out, struct arg_track *at_stack_out, + const struct arg_track *at_stack_arg_entry, struct func_instance *instance, u32 *callsites) { @@ -1071,9 +1102,21 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, struct arg_track *dst = &at_out[insn->dst_reg]; struct arg_track *src = &at_out[insn->src_reg]; struct arg_track none = { .frame = ARG_NONE }; - int r; - - if (class == BPF_ALU64 && BPF_SRC(insn->code) == BPF_K) { + int r, slot; + + /* Handle stack arg stores and loads. */ + if (is_stack_arg_st(insn) || is_stack_arg_stx(insn)) { + slot = stack_arg_off_to_slot(insn->off); + if (slot >= 0) { + if (is_stack_arg_stx(insn)) + at_out[MAX_BPF_REG + slot] = at_out[insn->src_reg]; + else + at_out[MAX_BPF_REG + slot] = none; + } + } else if (is_stack_arg_ldx(insn)) { + slot = stack_arg_off_to_slot(insn->off); + at_out[insn->dst_reg] = (slot >= 0) ? at_stack_arg_entry[slot] : none; + } else if (class == BPF_ALU64 && BPF_SRC(insn->code) == BPF_K) { if (code == BPF_MOV) { *dst = none; } else if (dst->frame >= 0) { @@ -1297,6 +1340,16 @@ static int record_load_store_access(struct bpf_verifier_env *env, struct arg_track resolved, *ptr; int oi; + /* + * Stack arg insns use dst_reg/src_reg=BPF_REG_PARAMS(11). Since at[] + * is extended to MAX_AT_TRACK_REGS, at[11] holds the arg_track for + * outgoing stack arg slot 0 — not the pointer used for the memory + * access. Skip so the slot's tracked value isn't confused with the + * base register that record_stack_access() expects. + */ + if (is_stack_arg_stx(insn) || is_stack_arg_st(insn) || is_stack_arg_ldx(insn)) + return 0; + switch (class) { case BPF_LDX: ptr = &at[insn->src_reg]; @@ -1395,11 +1448,18 @@ static int record_call_access(struct bpf_verifier_env *env, if (bpf_get_call_summary(env, insn, &cs)) num_params = cs.num_params; - for (r = BPF_REG_1; r < BPF_REG_1 + num_params; r++) { + for (r = BPF_REG_1; r < BPF_REG_1 + min(num_params, MAX_BPF_FUNC_REG_ARGS); r++) { err = record_arg_access(env, instance, insn, &at[r], r - 1, insn_idx); if (err) return err; } + + for (r = 0; r < MAX_STACK_ARG_SLOTS && r < num_params - MAX_BPF_FUNC_REG_ARGS; r++) { + err = record_arg_access(env, instance, insn, &at[MAX_BPF_REG + r], + r + MAX_BPF_FUNC_REG_ARGS, insn_idx); + if (err) + return err; + } return 0; } @@ -1456,7 +1516,7 @@ static int find_callback_subprog(struct bpf_verifier_env *env, /* Per-subprog intermediate state kept alive across analysis phases */ struct subprog_at_info { - struct arg_track (*at_in)[MAX_BPF_REG]; + struct arg_track (*at_in)[MAX_AT_TRACK_REGS]; int len; }; @@ -1490,6 +1550,9 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env, for (r = 0; r < MAX_BPF_REG - 1; r++) if (arg_is_fp(&info->at_in[i][r])) has_extra = true; + for (r = 0; r < MAX_STACK_ARG_SLOTS; r++) + if (arg_is_fp(&info->at_in[i][MAX_BPF_REG + r])) + has_extra = true; } if (is_ldx_stx_call) { for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++) @@ -1514,6 +1577,12 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env, verbose(env, " r%d=", r); verbose_arg_track(env, &info->at_in[i][r]); } + for (r = 0; r < MAX_STACK_ARG_SLOTS; r++) { + if (!arg_is_fp(&info->at_in[i][MAX_BPF_REG + r])) + continue; + verbose(env, " sa%d=", r); + verbose_arg_track(env, &info->at_in[i][MAX_BPF_REG + r]); + } } if (is_ldx_stx_call) { @@ -1536,7 +1605,7 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env, * Runs forward fixed-point with arg_track_xfer(), then records * memory accesses in a single linear pass over converged state. * - * @callee_entry: pre-populated entry state for R1-R5 + * @callee_entry: pre-populated entry state for R1-R5 and stack args * NULL for main (subprog 0). * @info: stores at_in, len for debug printing. */ @@ -1554,10 +1623,11 @@ static int compute_subprog_args(struct bpf_verifier_env *env, int end = env->subprog_info[subprog + 1].start; int po_end = env->subprog_info[subprog + 1].postorder_start; int len = end - start; - struct arg_track (*at_in)[MAX_BPF_REG] = NULL; - struct arg_track at_out[MAX_BPF_REG]; + struct arg_track (*at_in)[MAX_AT_TRACK_REGS] = NULL; + struct arg_track at_out[MAX_AT_TRACK_REGS]; struct arg_track (*at_stack_in)[MAX_ARG_SPILL_SLOTS] = NULL; struct arg_track *at_stack_out = NULL; + struct arg_track at_stack_arg_entry[MAX_STACK_ARG_SLOTS]; struct arg_track unvisited = { .frame = ARG_UNVISITED }; struct arg_track none = { .frame = ARG_NONE }; bool changed; @@ -1576,13 +1646,13 @@ static int compute_subprog_args(struct bpf_verifier_env *env, goto err_free; for (i = 0; i < len; i++) { - for (r = 0; r < MAX_BPF_REG; r++) + for (r = 0; r < MAX_AT_TRACK_REGS; r++) at_in[i][r] = unvisited; for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++) at_stack_in[i][r] = unvisited; } - for (r = 0; r < MAX_BPF_REG; r++) + for (r = 0; r < MAX_AT_TRACK_REGS; r++) at_in[0][r] = none; /* Entry: R10 is always precisely the current frame's FP */ @@ -1598,6 +1668,10 @@ static int compute_subprog_args(struct bpf_verifier_env *env, for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++) at_stack_in[0][r] = none; + /* Entry: incoming stack args from caller, or ARG_NONE for main */ + for (r = 0; r < MAX_STACK_ARG_SLOTS; r++) + at_stack_arg_entry[r] = callee_entry ? callee_entry[MAX_BPF_REG + r] : none; + if (env->log.level & BPF_LOG_LEVEL2) verbose(env, "subprog#%d: analyzing (depth %d)...\n", subprog, depth); @@ -1616,7 +1690,8 @@ redo: memcpy(at_out, at_in[i], sizeof(at_out)); memcpy(at_stack_out, at_stack_in[i], MAX_ARG_SPILL_SLOTS * sizeof(*at_stack_out)); - arg_track_xfer(env, insn, idx, at_out, at_stack_out, instance, callsites); + arg_track_xfer(env, insn, idx, at_out, at_stack_out, + at_stack_arg_entry, instance, callsites); arg_track_log(env, insn, idx, at_in[i], at_stack_in[i], at_out, at_stack_out); /* Propagate to successors within this subprogram */ @@ -1630,7 +1705,7 @@ redo: continue; ti = target - start; - for (r = 0; r < MAX_BPF_REG; r++) + for (r = 0; r < MAX_AT_TRACK_REGS; r++) changed |= arg_track_join(env, idx, target, r, &at_in[ti][r], at_out[r]); @@ -1685,12 +1760,15 @@ err_free: return err; } -/* Return true if any of R1-R5 is derived from a frame pointer. */ +/* Return true if any of R1-R5 or stack args is derived from a frame pointer. */ static bool has_fp_args(struct arg_track *args) { for (int r = BPF_REG_1; r <= BPF_REG_5; r++) if (arg_is_fp(&args[r])) return true; + for (int r = 0; r < MAX_STACK_ARG_SLOTS; r++) + if (arg_is_fp(&args[MAX_BPF_REG + r])) + return true; return false; } @@ -1814,7 +1892,7 @@ static int analyze_subprog(struct bpf_verifier_env *env, /* For each reachable call site in the subprog, recurse into callees */ for (int p = po_start; p < po_end; p++) { int idx = env->cfg.insn_postorder[p]; - struct arg_track callee_args[BPF_REG_5 + 1]; + struct arg_track callee_args[MAX_AT_TRACK_REGS] = {}; struct arg_track none = { .frame = ARG_NONE }; struct bpf_insn *insn = &insns[idx]; struct func_instance *callee_instance; @@ -1829,9 +1907,11 @@ static int analyze_subprog(struct bpf_verifier_env *env, if (callee < 0) continue; - /* Build entry args: R1-R5 from at_in at call site */ + /* Build entry args: R1-R5 and stack args from at_in at call site */ for (int r = BPF_REG_1; r <= BPF_REG_5; r++) callee_args[r] = info[subprog].at_in[j][r]; + for (int r = 0; r < MAX_STACK_ARG_SLOTS; r++) + callee_args[MAX_BPF_REG + r] = info[subprog].at_in[j][MAX_BPF_REG + r]; } else if (bpf_calls_callback(env, idx)) { callee = find_callback_subprog(env, insn, idx, &caller_reg, &cb_callee_reg); if (callee == -2) { @@ -1853,6 +1933,8 @@ static int analyze_subprog(struct bpf_verifier_env *env, for (int r = BPF_REG_1; r <= BPF_REG_5; r++) callee_args[r] = none; + for (int r = 0; r < MAX_STACK_ARG_SLOTS; r++) + callee_args[MAX_BPF_REG + r] = none; callee_args[cb_callee_reg] = info[subprog].at_in[j][caller_reg]; } else { continue; @@ -2096,7 +2178,7 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, def = ALL_CALLER_SAVED_REGS; use = def & ~BIT(BPF_REG_0); if (bpf_get_call_summary(env, insn, &cs)) - use = GENMASK(cs.num_params, 1); + use = GENMASK(min_t(u8, cs.num_params, MAX_BPF_FUNC_REG_ARGS), 1); break; default: def = 0; -- cgit v1.2.3 From dc8f1cf6787c4bb1d8cabfac1e44d2d0ab435caa Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 12 May 2026 21:50:49 -0700 Subject: bpf: Reject stack arguments in non-JITed programs The interpreter does not understand the bpf register r11 (BPF_REG_PARAMS) used for stack arguments. So reject interpreter usage if stack arguments are used either in the main program or any subprogram. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260513045049.2390444-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/core.c | 2 +- kernel/bpf/fixups.c | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index ae10b9ca018d..958d86f0beac 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -2599,7 +2599,7 @@ struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct goto finalize; if (IS_ENABLED(CONFIG_BPF_JIT_ALWAYS_ON) || - bpf_prog_has_kfunc_call(fp)) + bpf_prog_has_kfunc_call(fp) || (env && env->subprog_info[0].stack_arg_cnt)) jit_needed = true; if (!bpf_prog_select_interpreter(fp)) diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index ba86039789fd..19056016eed8 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -1407,6 +1407,12 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env) verbose(env, "calling kernel functions are not allowed in non-JITed programs\n"); return -EINVAL; } + for (i = 1; i < env->subprog_cnt; i++) { + if (bpf_in_stack_arg_cnt(&env->subprog_info[i])) { + verbose(env, "stack args are not supported in non-JITed programs\n"); + return -EINVAL; + } + } if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { /* When JIT fails the progs with bpf2bpf calls and tail_calls * have to be rejected, since interpreter doesn't support them yet. -- cgit v1.2.3 From 848d624acf668ae0d71b128f163d1d18d2ac6b90 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 12 May 2026 21:50:54 -0700 Subject: bpf: Prepare architecture JIT support for stack arguments Add bpf_jit_supports_stack_args() as a weak function defaulting to false. Architectures that implement JIT support for stack arguments override it to return true. Reject BPF functions with more than 5 parameters at verification time if the architecture does not support stack arguments. Acked-by: Puranjay Mohan Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260513045054.2390945-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/btf.c | 8 +++++++- kernel/bpf/core.c | 5 +++++ 2 files changed, 12 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 4fb8641546b8..17d4ab0a8206 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7870,8 +7870,14 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) MAX_BPF_FUNC_ARGS, tname, nargs); return -EFAULT; } - if (nargs > MAX_BPF_FUNC_REG_ARGS) + if (nargs > MAX_BPF_FUNC_REG_ARGS) { + if (!bpf_jit_supports_stack_args()) { + bpf_log(log, "JIT does not support function %s() with %d args\n", + tname, nargs); + return -EFAULT; + } sub->stack_arg_cnt = nargs - MAX_BPF_FUNC_REG_ARGS; + } if (is_global && nargs > MAX_BPF_FUNC_REG_ARGS) { bpf_log(log, "global function %s has %d > %d args, stack args not supported\n", diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 958d86f0beac..e6b836f846eb 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3217,6 +3217,11 @@ bool __weak bpf_jit_supports_kfunc_call(void) return false; } +bool __weak bpf_jit_supports_stack_args(void) +{ + return false; +} + bool __weak bpf_jit_supports_far_kfunc_call(void) { return false; -- cgit v1.2.3 From 9fae4cba3bfd583198afe15ed4b4433eafafd11c Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 12 May 2026 21:50:59 -0700 Subject: bpf: Enable r11 based insns BPF_REG_PARAMS (r11) is used for stack argument accesses and the following are only insns with r11 presence: - load incoming stack arg - store register to outgoing stack arg - store immediate to outgoing stack arg The detailed insn format can be found in is_stack_arg_ldx/st/stx() helpers. After this patch, stack arg ldx/st/stx insns become valid for kernel and these insns can be properly checked by verifier. The LLVM compiler [1] implemented the above BPF_REG_PARAMS insns. [1] https://github.com/llvm/llvm-project/pull/189060 Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260513045059.2391192-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ebd13661933e..b0d3c2d179e4 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -18006,11 +18006,12 @@ static int check_and_resolve_insns(struct bpf_verifier_env *env) return err; for (i = 0; i < insn_cnt; i++, insn++) { - if (insn->dst_reg >= MAX_BPF_REG) { + if (insn->dst_reg >= MAX_BPF_REG && + !is_stack_arg_st(insn) && !is_stack_arg_stx(insn)) { verbose(env, "R%d is invalid\n", insn->dst_reg); return -EINVAL; } - if (insn->src_reg >= MAX_BPF_REG) { + if (insn->src_reg >= MAX_BPF_REG && !is_stack_arg_ldx(insn)) { verbose(env, "R%d is invalid\n", insn->src_reg); return -EINVAL; } -- cgit v1.2.3 From e0b7b91c72db6dae0392dd90db3b866218a7870b Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 12 May 2026 21:51:04 -0700 Subject: bpf: Support stack arguments for kfunc calls Extend the stack argument mechanism to kfunc calls, allowing kfuncs with more than 5 parameters to receive additional arguments via the r11-based stack arg area. For kfuncs, the caller is a BPF program and the callee is a kernel function. The BPF program writes outgoing args at negative r11 offsets, following the same convention as BPF-to-BPF calls: Outgoing: r11 - 8 (arg6), ..., r11 - N*8 (last arg) The following is an example: int foo(int a1, int a2, int a3, int a4, int a5, int a6, int a7) { ... kfunc1(a1, a2, a3, a4, a5, a6, a7, a8); ... kfunc2(a1, a2, a3, a4, a5, a6, a7, a8, a9); ... } Caller (foo), generated by llvm =============================== Incoming (positive offsets): r11+8: [incoming arg 6] r11+16: [incoming arg 7] Outgoing for kfunc1 (negative offsets): r11-8: [outgoing arg 6] r11-16: [outgoing arg 7] r11-24: [outgoing arg 8] Outgoing for kfunc2 (negative offsets): r11-8: [outgoing arg 6] r11-16: [outgoing arg 7] r11-24: [outgoing arg 8] r11-32: [outgoing arg 9] Later JIT will marshal outgoing arguments to the native calling convention for kfunc1() and kfunc2(). For kfunc calls where stack args are used as constant or size parameters, a mark_stack_arg_precision() helper is used to propagate precision and do proper backtracking. There are two places where meta->release_regno needs to keep regno for later releasing the reference. Also, 'cur_aux(env)->arg_prog = regno' is also keeping regno for later fixup. Since stack arguments don't have a valid register number (regno is negative), these three cases are rejected for now if the argument is on the stack. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260513045104.2391543-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 77 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 60 insertions(+), 17 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index b0d3c2d179e4..1a734ab91a31 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11160,14 +11160,12 @@ bool bpf_is_kfunc_pkt_changing(struct bpf_kfunc_call_arg_meta *meta) } static enum kfunc_ptr_arg_type -get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, - struct bpf_kfunc_call_arg_meta *meta, +get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, struct bpf_func_state *caller, + struct bpf_reg_state *regs, struct bpf_kfunc_call_arg_meta *meta, const struct btf_type *t, const struct btf_type *ref_t, const char *ref_tname, const struct btf_param *args, int arg, int nargs, argno_t argno, struct bpf_reg_state *reg) { - u32 regno = arg + 1; - struct bpf_reg_state *regs = cur_regs(env); bool arg_mem_size = false; if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || @@ -11176,8 +11174,8 @@ get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, return KF_ARG_PTR_TO_CTX; if (arg + 1 < nargs && - (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1], ®s[regno + 1]) || - is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1], ®s[regno + 1]))) + (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1], get_func_arg_reg(caller, regs, arg + 1)) || + is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1], get_func_arg_reg(caller, regs, arg + 1)))) arg_mem_size = true; /* In this function, we verify the kfunc's BTF as per the argument type, @@ -11842,6 +11840,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ int insn_idx) { const char *func_name = meta->func_name, *ref_tname; + struct bpf_func_state *caller = cur_func(env); + struct bpf_reg_state *regs = cur_regs(env); const struct btf *btf = meta->btf; const struct btf_param *args; struct btf_record *rec; @@ -11850,21 +11850,31 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ args = (const struct btf_param *)(meta->func_proto + 1); nargs = btf_type_vlen(meta->func_proto); - if (nargs > MAX_BPF_FUNC_REG_ARGS) { + if (nargs > MAX_BPF_FUNC_ARGS) { verbose(env, "Function %s has %d > %d args\n", func_name, nargs, - MAX_BPF_FUNC_REG_ARGS); + MAX_BPF_FUNC_ARGS); return -EINVAL; } + if (nargs > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) { + verbose(env, "JIT does not support kfunc %s() with %d args\n", + func_name, nargs); + return -ENOTSUPP; + } + + ret = check_outgoing_stack_args(env, caller, nargs); + if (ret) + return ret; /* Check that BTF function arguments match actual types that the * verifier sees. */ for (i = 0; i < nargs; i++) { - struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[i + 1]; + struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, i); const struct btf_type *t, *ref_t, *resolve_ret; enum bpf_arg_type arg_type = ARG_DONTCARE; argno_t argno = argno_from_arg(i + 1); - u32 regno = i + 1, ref_id, type_size; + int regno = reg_from_argno(argno); + u32 ref_id, type_size; bool is_ret_buf_sz = false; int kf_arg_type; @@ -11874,6 +11884,11 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ verifier_bug(env, "Only 1 prog->aux argument supported per-kfunc"); return -EFAULT; } + if (regno < 0) { + verbose(env, "%s prog->aux cannot be a stack argument\n", + reg_arg_name(env, argno)); + return -EINVAL; + } meta->arg_prog = true; cur_aux(env)->arg_prog = regno; continue; @@ -11900,7 +11915,10 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ reg_arg_name(env, argno)); return -EINVAL; } - ret = mark_chain_precision(env, regno); + if (regno >= 0) + ret = mark_chain_precision(env, regno); + else + ret = mark_stack_arg_precision(env, i); if (ret < 0) return ret; meta->arg_constant.found = true; @@ -11925,7 +11943,10 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ } meta->r0_size = reg->var_off.value; - ret = mark_chain_precision(env, regno); + if (regno >= 0) + ret = mark_chain_precision(env, regno); + else + ret = mark_stack_arg_precision(env, i); if (ret) return ret; } @@ -11953,14 +11974,21 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ return -EFAULT; } meta->ref_obj_id = reg->ref_obj_id; - if (is_kfunc_release(meta)) + if (is_kfunc_release(meta)) { + if (regno < 0) { + verbose(env, "%s release arg cannot be a stack argument\n", + reg_arg_name(env, argno)); + return -EINVAL; + } meta->release_regno = regno; + } } ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); ref_tname = btf_name_by_offset(btf, ref_t->name_off); - kf_arg_type = get_kfunc_ptr_arg_type(env, meta, t, ref_t, ref_tname, args, i, nargs, argno, reg); + kf_arg_type = get_kfunc_ptr_arg_type(env, caller, regs, meta, t, ref_t, ref_tname, + args, i, nargs, argno, reg); if (kf_arg_type < 0) return kf_arg_type; @@ -12110,6 +12138,11 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ dynptr_arg_type |= DYNPTR_TYPE_FILE; } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_file_discard]) { dynptr_arg_type |= DYNPTR_TYPE_FILE | OBJ_RELEASE; + if (regno < 0) { + verbose(env, "%s release arg cannot be a stack argument\n", + reg_arg_name(env, argno)); + return -EINVAL; + } meta->release_regno = regno; } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_clone] && (dynptr_arg_type & MEM_UNINIT)) { @@ -12264,9 +12297,9 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ break; case KF_ARG_PTR_TO_MEM_SIZE: { - struct bpf_reg_state *buff_reg = ®s[regno]; + struct bpf_reg_state *buff_reg = reg; const struct btf_param *buff_arg = &args[i]; - struct bpf_reg_state *size_reg = ®s[regno + 1]; + struct bpf_reg_state *size_reg = get_func_arg_reg(caller, regs, i + 1); const struct btf_param *size_arg = &args[i + 1]; argno_t next_argno = argno_from_arg(i + 2); @@ -13171,8 +13204,18 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, clear_all_pkt_pointers(env); nargs = btf_type_vlen(meta.func_proto); + if (nargs > MAX_BPF_FUNC_REG_ARGS) { + struct bpf_func_state *caller = cur_func(env); + struct bpf_subprog_info *caller_info = &env->subprog_info[caller->subprogno]; + u16 out_stack_arg_cnt = nargs - MAX_BPF_FUNC_REG_ARGS; + u16 stack_arg_cnt = bpf_in_stack_arg_cnt(caller_info) + out_stack_arg_cnt; + + if (stack_arg_cnt > caller_info->stack_arg_cnt) + caller_info->stack_arg_cnt = stack_arg_cnt; + } + args = (const struct btf_param *)(meta.func_proto + 1); - for (i = 0; i < nargs; i++) { + for (i = 0; i < min_t(int, nargs, MAX_BPF_FUNC_REG_ARGS); i++) { u32 regno = i + 1; t = btf_type_skip_modifiers(desc_btf, args[i].type, NULL); -- cgit v1.2.3 From 35b78733160c120767332d924a0447a87109bbde Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 12 May 2026 21:51:09 -0700 Subject: bpf: Reject stack arguments if tail call reachable Tail calls are deprecated and will be replaced by indirect calls in the future. Reject programs that combine tail calls with stack arguments rather than adding complexity for a deprecated feature. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260513045109.2392108-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 1a734ab91a31..a10cc045057d 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5267,14 +5267,23 @@ continue_func: * this info will be utilized by JIT so that we will be preserving the * tail call counter throughout bpf2bpf calls combined with tailcalls */ - if (tail_call_reachable) + if (tail_call_reachable) { for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) { if (subprog[tmp].is_exception_cb) { verbose(env, "cannot tail call within exception cb\n"); return -EINVAL; } + if (subprog[tmp].stack_arg_cnt) { + verbose(env, "tail_calls are not allowed in programs with stack args\n"); + return -EINVAL; + } subprog[tmp].tail_call_reachable = true; } + } else if (!idx && subprog[0].has_tail_call && subprog[0].stack_arg_cnt) { + verbose(env, "tail_calls are not allowed in programs with stack args\n"); + return -EINVAL; + } + if (subprog[0].tail_call_reachable) env->prog->aux->tail_call_reachable = true; -- cgit v1.2.3 From cb6af5314056cb06456cfa8774aa158d61929bcd Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 12 May 2026 21:51:14 -0700 Subject: bpf: Disable private stack for x86_64 if stack arguments used Other architectures like arm64, riscv, etc. have enough register and for them private stack can be used together with stack arguments. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260513045114.2392291-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a10cc045057d..82b9531f87f6 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5163,7 +5163,10 @@ process_func: } subprog_depth = round_up_stack_depth(env, subprog[idx].stack_depth); - if (priv_stack_supported) { + if (IS_ENABLED(CONFIG_X86_64) && subprog[idx].stack_arg_cnt) { + /* x86-64 uses R9 for both private stack frame pointer and arg6. */ + subprog[idx].priv_stack_mode = NO_PRIV_STACK; + } else if (priv_stack_supported) { /* Request private stack support only if the subprog stack * depth is no less than BPF_PRIV_STACK_MIN_SIZE. This is to * avoid jit penalty if the stack usage is small. -- cgit v1.2.3 From 324c3ca6eed6fb7ec4e50f31d537953038b13c5f Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 12 May 2026 21:51:19 -0700 Subject: bpf,x86: Implement JIT support for stack arguments Add x86_64 JIT support for BPF functions and kfuncs with more than 5 arguments. The extra arguments are passed through a stack area addressed by register r11 (BPF_REG_PARAMS) in BPF bytecode, which the JIT translates to native code. The JIT follows the x86-64 calling convention for both BPF-to-BPF and kfunc calls: - Arg 6 is passed in the R9 register - Args 7+ are passed on the stack Incoming arg 6 (BPF r11+8) is translated to a MOV from R9 rather than a memory load. Incoming args 7+ (BPF r11+16, r11+24, ...) map directly to [rbp + 16], [rbp + 24], ..., matching the x86-64 stack layout after CALL + PUSH RBP, so no offset adjustment is needed. tail_call_reachable is rejected by the verifier and priv_stack is disabled by the JIT when stack args exist, so R9 is always available. When BPF bytecode writes to the arg-6 stack slot (offset -8), the JIT emits a MOV into R9 instead of a memory store. Outgoing args 7+ are placed at [rsp] in a pre-allocated area below callee-saved registers, using: native_off = outgoing_arg_base - outgoing_rsp - bpf_off - 16 The native x86_64 stack layout with stack arguments: high address +-------------------------+ | incoming stack arg N | [rbp + 16 + (N-7)*8] (from caller) | ... | | incoming stack arg 7 | [rbp + 16] +-------------------------+ | return address | [rbp + 8] | saved rbp | [rbp] +-------------------------+ | BPF program stack | (round_up(stack_depth, 8) bytes) +-------------------------+ | callee-saved regs | (r12, rbx, r13, r14, r15 as needed) +-------------------------+ | outgoing arg M | [rsp + (M-7)*8] | ... | | outgoing arg 7 | [rsp] +-------------------------+ rsp low address Acked-by: Puranjay Mohan Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260513045122.2393118-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/core.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index e6b836f846eb..427a6d828e01 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -1582,6 +1582,16 @@ bool bpf_insn_is_indirect_target(const struct bpf_verifier_env *env, const struc insn_idx += prog->aux->subprog_start; return env->insn_aux_data[insn_idx].indirect_target; } + +u16 bpf_out_stack_arg_cnt(const struct bpf_verifier_env *env, const struct bpf_prog *prog) +{ + const struct bpf_subprog_info *sub; + + if (!env) + return 0; + sub = &env->subprog_info[prog->aux->func_idx]; + return sub->stack_arg_cnt - bpf_in_stack_arg_cnt(sub); +} #endif /* CONFIG_BPF_JIT */ /* Base function for offset calculation. Needs to go into .text section, -- cgit v1.2.3 From 5d330d652d7a455b2215c38e7b0c6149c6f8225d Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 13 May 2026 14:59:29 +0200 Subject: hrtimer: Fix the bogus return type of __hrtimer_start_range_ns() __hrtimer_start_range_ns() has a bool return type, but returns actually three different values, which are checked at the call site. Make the return type int. Fixes: bd5956166d20 ("hrtimer: Provide hrtimer_start_range_ns_user()") Reported-by: Dan Carpenter --- kernel/time/hrtimer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 8a19a61f6fee..638ce623c342 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1358,8 +1358,8 @@ enum { HRTIMER_REPROGRAM_FORCE, }; -static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 delta_ns, - const enum hrtimer_mode mode, struct hrtimer_clock_base *base) +static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 delta_ns, + const enum hrtimer_mode mode, struct hrtimer_clock_base *base) { struct hrtimer_cpu_base *this_cpu_base = this_cpu_ptr(&hrtimer_bases); bool is_pinned, first, was_first, keep_base = false; -- cgit v1.2.3 From f41f34ec64748e16e5a90ab391cec39e30942f32 Mon Sep 17 00:00:00 2001 From: Paul Chaignon Date: Wed, 13 May 2026 21:34:50 +0200 Subject: bpf: Report maximum combined stack depth We've hit the 512 bytes limit on stack depth a few times in Cilium recently. As a result, we started reporting in CI our current maximum stack depth across all configurations for each BPF program. Unfortunately, that is not trivial to compute in userspace. The verifier reports the stack depths of individual subprogs at the end of the logs. However the maximum combined stack depth also depends on the callgraph of those subprogs (the max combined stack depth is the height of the callgraph weighted by per-subprog stack depths). We can compute a callgraph in userspace from the loaded instructions, but it often doesn't match the verifier's own callgraph because of dead code elimination. Our current approach relies on dumping the BPF_LOG_LEVEL2 logs, but this feels overkill considering the verifier already has the information we need. The patch lets the verifier dump the maximum combined stack depth in the logs, on the same line as the per-subprog stack depths: stack depth 16+256 max 272 The per-subprog stack depths and the new max stack depth are not directly comparable. The former is sometimes updated during fixups, while the latter is not. As a result, even with a single subprog, we may end up with two slightly different values. The aim of the new max value is to be closest to what is actually enforced by the verifier. Signed-off-by: Paul Chaignon Acked-by: Eduard Zingerman Link: https://lore.kernel.org/r/d3d23a0410f87f116f3bbaa98a815dbae113bda2.1778700777.git.paul.chaignon@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 82b9531f87f6..76a07f09ab64 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5177,6 +5177,8 @@ process_func: } if (subprog[idx].priv_stack_mode == PRIV_STACK_ADAPTIVE) { + if (subprog_depth > env->max_stack_depth) + env->max_stack_depth = subprog_depth; if (subprog_depth > MAX_BPF_STACK) { verbose(env, "stack size of subprog %d is %d. Too large\n", idx, subprog_depth); @@ -5184,6 +5186,8 @@ process_func: } } else { depth += subprog_depth; + if (depth > env->max_stack_depth) + env->max_stack_depth = depth; if (depth > MAX_BPF_STACK) { total = 0; for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) @@ -18555,7 +18559,7 @@ static void print_verification_stats(struct bpf_verifier_env *env) verbose(env, "stack depth %d", env->subprog_info[0].stack_depth); for (i = 1; i < subprog_cnt; i++) verbose(env, "+%d", env->subprog_info[i].stack_depth); - verbose(env, "\n"); + verbose(env, " max %d\n", env->max_stack_depth); verbose(env, "insns processed %d", env->subprog_info[0].insn_processed); for (i = 1; i < subprog_cnt; i++) if (bpf_subprog_is_global(env, i)) -- cgit v1.2.3 From feb662d9168b63e1d4c02671ec96005410c6f3ce Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Mon, 11 May 2026 22:00:48 +0200 Subject: slab: support for compiler-assisted type-based slab cache partitioning Rework the general infrastructure around RANDOM_KMALLOC_CACHES into more flexible KMALLOC_PARTITION_CACHES, with the former being a partitioning mode of the latter. Introduce a new mode, KMALLOC_PARTITION_TYPED, which leverages a feature available in Clang 22 and later, called "allocation tokens" via __builtin_infer_alloc_token() [1]. Unlike KMALLOC_PARTITION_RANDOM (formerly RANDOM_KMALLOC_CACHES), this mode deterministically assigns a slab cache to an allocation of type T, regardless of allocation site. The builtin __builtin_infer_alloc_token(, ...) instructs the compiler to infer an allocation type from arguments commonly passed to memory-allocating functions and returns a type-derived token ID. The implementation passes kmalloc-args to the builtin: the compiler performs best-effort type inference, and then recognizes common patterns such as `kmalloc(sizeof(T), ...)`, `kmalloc(sizeof(T) * n, ...)`, but also `(T *)kmalloc(...)`. Where the compiler fails to infer a type the fallback token (default: 0) is chosen. Note: kmalloc_obj(..) APIs fix the pattern how size and result type are expressed, and therefore ensures there's not much drift in which patterns the compiler needs to recognize. Specifically, kmalloc_obj() and friends expand to `(TYPE *)KMALLOC(__obj_size, GFP)`, which the compiler recognizes via the cast to TYPE*. Clang's default token ID calculation is described as [1]: typehashpointersplit: This mode assigns a token ID based on the hash of the allocated type's name, where the top half ID-space is reserved for types that contain pointers and the bottom half for types that do not contain pointers. Separating pointer-containing objects from pointerless objects and data allocations can help mitigate certain classes of memory corruption exploits [2]: attackers who gains a buffer overflow on a primitive buffer cannot use it to directly corrupt pointers or other critical metadata in an object residing in a different, isolated heap region. It is important to note that heap isolation strategies offer a best-effort approach, and do not provide a 100% security guarantee, albeit achievable at relatively low performance cost. Note that this also does not prevent cross-cache attacks: while waiting for future features like SLAB_VIRTUAL [3] to provide physical page isolation, this feature should be deployed alongside SHUFFLE_PAGE_ALLOCATOR and init_on_free=1 to mitigate cross-cache attacks and page-reuse attacks as much as possible today. With all that, my kernel (x86 defconfig) shows me a histogram of slab cache object distribution per /proc/slabinfo (after boot): kmalloc-part-15 1465 ++++++++++++++ kmalloc-part-14 2988 +++++++++++++++++++++++++++++ kmalloc-part-13 1656 ++++++++++++++++ kmalloc-part-12 1045 ++++++++++ kmalloc-part-11 1697 ++++++++++++++++ kmalloc-part-10 1489 ++++++++++++++ kmalloc-part-09 965 +++++++++ kmalloc-part-08 710 +++++++ kmalloc-part-07 100 + kmalloc-part-06 217 ++ kmalloc-part-05 105 + kmalloc-part-04 4047 ++++++++++++++++++++++++++++++++++++++++ kmalloc-part-03 183 + kmalloc-part-02 283 ++ kmalloc-part-01 316 +++ kmalloc 1422 ++++++++++++++ The above /proc/slabinfo snapshot shows me there are 6673 allocated objects (slabs 00 - 07) that the compiler claims contain no pointers or it was unable to infer the type of, and 12015 objects that contain pointers (slabs 08 - 15). On a whole, this looks relatively sane. Additionally, when I compile my kernel with -Rpass=alloc-token, which provides diagnostics where (after dead-code elimination) type inference failed, I see 186 allocation sites where the compiler failed to identify a type (down from 966 when I sent the RFC [4]). Some initial review confirms these are mostly variable sized buffers, but also include structs with trailing flexible length arrays. Link: https://clang.llvm.org/docs/AllocToken.html [1] Link: https://blog.dfsec.com/ios/2025/05/30/blasting-past-ios-18/ [2] Link: https://lwn.net/Articles/944647/ [3] Link: https://lore.kernel.org/all/20250825154505.1558444-1-elver@google.com/ [4] Link: https://discourse.llvm.org/t/rfc-a-framework-for-allocator-partitioning-hints/87434 Acked-by: GONG Ruiqi Co-developed-by: Harry Yoo (Oracle) Signed-off-by: Harry Yoo (Oracle) Signed-off-by: Marco Elver Reviewed-by: Harry Yoo (Oracle) Link: https://patch.msgid.link/20260511200136.3201646-1-elver@google.com Signed-off-by: Vlastimil Babka (SUSE) --- kernel/configs/hardening.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/configs/hardening.config b/kernel/configs/hardening.config index 7c3924614e01..26831a2a5739 100644 --- a/kernel/configs/hardening.config +++ b/kernel/configs/hardening.config @@ -22,7 +22,7 @@ CONFIG_SLAB_FREELIST_RANDOM=y CONFIG_SLAB_FREELIST_HARDENED=y CONFIG_SLAB_BUCKETS=y CONFIG_SHUFFLE_PAGE_ALLOCATOR=y -CONFIG_RANDOM_KMALLOC_CACHES=y +CONFIG_KMALLOC_PARTITION_CACHES=y # Sanity check userspace page table mappings. CONFIG_PAGE_TABLE_CHECK=y -- cgit v1.2.3 From 593980175389a05793f6060aa20e626330960395 Mon Sep 17 00:00:00 2001 From: Guannan Wang Date: Thu, 14 May 2026 15:44:54 +0800 Subject: bpf: Use array_map_meta_equal for percpu array inner map replacement percpu_array_map_ops.map_meta_equal points to the generic bpf_map_meta_equal(), which does not compare max_entries. When a percpu array serves as an inner map, replacing it with one that has fewer max_entries bypasses the check. Since percpu_array_map_gen_lookup() inlines the original template's index_mask as a JIT immediate, a lookup on the replacement map can access pptrs[] out of bounds. Point percpu_array_map_ops.map_meta_equal to array_map_meta_equal(), which already enforces the max_entries equality check. Add a selftest to verify that replacing a percpu array inner map with a differently-sized one is rejected. Fixes: db69718b8efa ("bpf: inline bpf_map_lookup_elem() for PERCPU_ARRAY maps") Signed-off-by: Guannan Wang Acked-by: Mykyta Yatsenko Link: https://lore.kernel.org/r/20260514074454.77491-1-wgnbuaa@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/arraymap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index 5e25e0353509..dfb2110ab733 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -827,7 +827,7 @@ const struct bpf_map_ops array_map_ops = { }; const struct bpf_map_ops percpu_array_map_ops = { - .map_meta_equal = bpf_map_meta_equal, + .map_meta_equal = array_map_meta_equal, .map_alloc_check = array_map_alloc_check, .map_alloc = array_map_alloc, .map_free = array_map_free, -- cgit v1.2.3 From 85347718ab0dd7ede9c3e1dcff2d604c7073df05 Mon Sep 17 00:00:00 2001 From: Alessandro Carminati Date: Thu, 14 May 2026 13:06:37 +0200 Subject: bug/kunit: Core support for suppressing warning backtraces Some unit tests intentionally trigger warning backtraces by passing bad parameters to kernel API functions. Such unit tests typically check the return value from such calls, not the existence of the warning backtrace. Such intentionally generated warning backtraces are neither desirable nor useful for a number of reasons: - They can result in overlooked real problems. - A warning that suddenly starts to show up in unit tests needs to be investigated and has to be marked to be ignored, for example by adjusting filter scripts. Such filters are ad hoc because there is no real standard format for warnings. On top of that, such filter scripts would require constant maintenance. Solve the problem by providing a means to suppress warning backtraces originating from the current kthread while executing test code. Since each KUnit test runs in its own kthread, this effectively scopes suppression to the test that enabled it. Limit changes to generic code to the absolute minimum. Implementation details: Suppression is integrated into the existing KUnit hooks infrastructure in test-bug.h, reusing the kunit_running static branch for zero overhead when no tests are running. Suppression is checked at three points in the warning path: - In warn_slowpath_fmt(), the check runs before any output, fully suppressing both message and backtrace. This covers architectures without __WARN_FLAGS. - In __warn_printk(), the check suppresses the warning message text. This covers architectures that define __WARN_FLAGS but not their own __WARN_printf (arm64, loongarch, parisc, powerpc, riscv, sh), where the message is printed before the trap enters __report_bug(). - In __report_bug(), the check runs before __warn() is called, suppressing the backtrace and stack dump. To avoid double-counting on architectures where both __warn_printk() and __report_bug() run for the same warning, kunit_is_suppressed_warning() takes a bool parameter: true to increment the suppression counter (used in warn_slowpath_fmt and __report_bug), false to check only (used in __warn_printk). The suppression state is dynamically allocated via kunit_kzalloc() and tied to the KUnit test lifecycle via kunit_add_action(), ensuring automatic cleanup at test exit. Writer-side access to the global suppression list is serialized with a spinlock; readers use RCU. Two API forms are provided: - kunit_warning_suppress(test) { ... }: scoped, uses __cleanup for automatic teardown on scope exit, kunit_add_action() as safety net for abnormal exits (e.g. kthread_exit from failed assertions). Suppression handle is only accessible inside the block. - kunit_start/end_suppress_warning(test): direct functions returning an explicit handle, for retaining the handle within the test, or for cross-function usage. Link: https://lore.kernel.org/r/20260514-kunit_add_support-v11-1-b36a530a6d8f@redhat.com Signed-off-by: Guenter Roeck Signed-off-by: Alessandro Carminati Reviewed-by: Kees Cook Reviewed-by: David Gow Signed-off-by: Albert Esteve Signed-off-by: Shuah Khan --- kernel/panic.c | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'kernel') diff --git a/kernel/panic.c b/kernel/panic.c index 20feada5319d..213725b612aa 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -39,6 +39,7 @@ #include #include #include +#include #define PANIC_TIMER_STEP 100 #define PANIC_BLINK_SPD 18 @@ -1124,6 +1125,11 @@ void warn_slowpath_fmt(const char *file, int line, unsigned taint, bool rcu = warn_rcu_enter(); struct warn_args args; + if (kunit_is_suppressed_warning(true)) { + warn_rcu_exit(rcu); + return; + } + pr_warn(CUT_HERE); if (!fmt) { @@ -1146,6 +1152,11 @@ void __warn_printk(const char *fmt, ...) bool rcu = warn_rcu_enter(); va_list args; + if (kunit_is_suppressed_warning(false)) { + warn_rcu_exit(rcu); + return; + } + pr_warn(CUT_HERE); va_start(args, fmt); -- cgit v1.2.3 From c68095c4a4c919cbd7de016fdfa25d19fa918a74 Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Thu, 14 May 2026 14:50:31 +0800 Subject: cgroup/rdma: add rdma.peak for per-device peak usage tracking rdma.peak tracks the high watermark of resource usage per device, giving a better baseline on which to set rdma.max. Polling rdma.current isn't feasible since it would miss short-lived spikes. This interface is analogous to memory.peak. Signed-off-by: Tao Cui Signed-off-by: Tejun Heo --- kernel/cgroup/rdma.c | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) (limited to 'kernel') diff --git a/kernel/cgroup/rdma.c b/kernel/cgroup/rdma.c index 3df7c38ce481..4e3bf0bade18 100644 --- a/kernel/cgroup/rdma.c +++ b/kernel/cgroup/rdma.c @@ -44,6 +44,7 @@ static LIST_HEAD(rdmacg_devices); enum rdmacg_file_type { RDMACG_RESOURCE_TYPE_MAX, RDMACG_RESOURCE_TYPE_STAT, + RDMACG_RESOURCE_TYPE_PEAK, }; /* @@ -60,6 +61,7 @@ static char const *rdmacg_resource_names[] = { struct rdmacg_resource { int max; int usage; + int peak; }; /* @@ -204,6 +206,17 @@ uncharge_cg_locked(struct rdma_cgroup *cg, rpool->usage_sum--; if (rpool->usage_sum == 0 && rpool->num_max_cnt == RDMACG_RESOURCE_MAX) { + int i; + + /* + * Keep the rpool alive if any peak value is non-zero, + * so that rdma.peak persists as a historical high- + * watermark even after all resources are freed. + */ + for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { + if (rpool->resources[i].peak) + return; + } /* * No user of the rpool and all entries are set to max, so * safe to delete this rpool. @@ -310,6 +323,12 @@ int rdmacg_try_charge(struct rdma_cgroup **rdmacg, } } } + /* Update peak only after all charges succeed */ + for (p = cg; p; p = parent_rdmacg(p)) { + rpool = find_cg_rpool_locked(p, device); + if (rpool && rpool->resources[index].usage > rpool->resources[index].peak) + rpool->resources[index].peak = rpool->resources[index].usage; + } mutex_unlock(&rdmacg_mutex); *rdmacg = cg; @@ -472,6 +491,12 @@ static ssize_t rdmacg_resource_set_max(struct kernfs_open_file *of, if (rpool->usage_sum == 0 && rpool->num_max_cnt == RDMACG_RESOURCE_MAX) { + int i; + + for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { + if (rpool->resources[i].peak) + goto dev_err; + } /* * No user of the rpool and all entries are set to max, so * safe to delete this rpool. @@ -506,6 +531,8 @@ static void print_rpool_values(struct seq_file *sf, value = rpool->resources[i].max; else value = S32_MAX; + } else if (sf_type == RDMACG_RESOURCE_TYPE_PEAK) { + value = rpool ? rpool->resources[i].peak : 0; } else { if (rpool) value = rpool->resources[i].usage; @@ -556,6 +583,12 @@ static struct cftype rdmacg_files[] = { .private = RDMACG_RESOURCE_TYPE_STAT, .flags = CFTYPE_NOT_ON_ROOT, }, + { + .name = "peak", + .seq_show = rdmacg_resource_read, + .private = RDMACG_RESOURCE_TYPE_PEAK, + .flags = CFTYPE_NOT_ON_ROOT, + }, { } /* terminate */ }; @@ -575,6 +608,13 @@ rdmacg_css_alloc(struct cgroup_subsys_state *parent) static void rdmacg_css_free(struct cgroup_subsys_state *css) { struct rdma_cgroup *cg = css_rdmacg(css); + struct rdmacg_resource_pool *rpool, *tmp; + + /* Clean up rpools kept alive by non-zero peak values */ + mutex_lock(&rdmacg_mutex); + list_for_each_entry_safe(rpool, tmp, &cg->rpools, cg_node) + free_cg_rpool_locked(rpool); + mutex_unlock(&rdmacg_mutex); kfree(cg); } -- cgit v1.2.3 From 009bcbd0b201d4dc125eb960a61cb6d4d9fdfc72 Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Thu, 14 May 2026 14:50:32 +0800 Subject: cgroup/rdma: add rdma.events to track resource limit exhaustion Add per-device hierarchical event counters to track when RDMA resource limits are exceeded. The rdma.events file reports max event counts propagated upward from the cgroup whose limit was hit to all ancestors. This mirrors the design of pids.events, where events are attributed to the cgroup that imposed the limit, not necessarily the cgroup where the allocation was attempted. Userspace can monitor this file via poll/epoll for real-time notification of resource exhaustion. Signed-off-by: Tao Cui Signed-off-by: Tejun Heo --- kernel/cgroup/rdma.c | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/cgroup/rdma.c b/kernel/cgroup/rdma.c index 4e3bf0bade18..927bbf1eb949 100644 --- a/kernel/cgroup/rdma.c +++ b/kernel/cgroup/rdma.c @@ -81,6 +81,9 @@ struct rdmacg_resource_pool { u64 usage_sum; /* total number counts which are set to max */ int num_max_cnt; + + /* per-resource hierarchical max event counters */ + u64 events_max[RDMACG_RESOURCE_MAX]; }; static struct rdma_cgroup *css_rdmacg(struct cgroup_subsys_state *css) @@ -214,7 +217,8 @@ uncharge_cg_locked(struct rdma_cgroup *cg, * watermark even after all resources are freed. */ for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { - if (rpool->resources[i].peak) + if (rpool->resources[i].peak || + READ_ONCE(rpool->events_max[i])) return; } /* @@ -225,6 +229,34 @@ uncharge_cg_locked(struct rdma_cgroup *cg, } } +/** + * rdmacg_event_locked - fire hierarchical max event when resource limit is hit + * @over_cg: cgroup whose limit was exceeded + * @device: rdma device + * @index: resource type index + * + * Must be called under rdmacg_mutex. Propagates max event counts + * from @over_cg (including itself) upward to all ancestors with + * an rpool and notifies userspace. + */ +static void rdmacg_event_locked(struct rdma_cgroup *over_cg, + struct rdmacg_device *device, + enum rdmacg_resource_type index) +{ + struct rdmacg_resource_pool *rpool; + struct rdma_cgroup *p; + + lockdep_assert_held(&rdmacg_mutex); + + for (p = over_cg; parent_rdmacg(p); p = parent_rdmacg(p)) { + rpool = get_cg_rpool_locked(p, device); + if (!IS_ERR(rpool)) { + rpool->events_max[index]++; + cgroup_file_notify(&p->events_file); + } + } +} + /** * rdmacg_uncharge_hierarchy - hierarchically uncharge rdma resource count * @cg: pointer to cg to uncharge and all parents in hierarchy @@ -335,6 +367,8 @@ int rdmacg_try_charge(struct rdma_cgroup **rdmacg, return 0; err: + if (ret == -EAGAIN) + rdmacg_event_locked(p, device, index); mutex_unlock(&rdmacg_mutex); rdmacg_uncharge_hierarchy(cg, device, p, index); return ret; @@ -494,7 +528,8 @@ static ssize_t rdmacg_resource_set_max(struct kernfs_open_file *of, int i; for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { - if (rpool->resources[i].peak) + if (rpool->resources[i].peak || + READ_ONCE(rpool->events_max[i])) goto dev_err; } /* @@ -569,6 +604,33 @@ static int rdmacg_resource_read(struct seq_file *sf, void *v) return 0; } +static int rdmacg_events_show(struct seq_file *sf, void *v) +{ + struct rdma_cgroup *cg = css_rdmacg(seq_css(sf)); + struct rdmacg_resource_pool *rpool; + struct rdmacg_device *device; + int i; + + mutex_lock(&rdmacg_mutex); + + list_for_each_entry(device, &rdmacg_devices, dev_node) { + rpool = find_cg_rpool_locked(cg, device); + + seq_printf(sf, "%s ", device->name); + for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { + seq_printf(sf, "%s.max=%llu", + rdmacg_resource_names[i], + rpool ? READ_ONCE(rpool->events_max[i]) : 0ULL); + if (i < RDMACG_RESOURCE_MAX - 1) + seq_putc(sf, ' '); + } + seq_putc(sf, '\n'); + } + + mutex_unlock(&rdmacg_mutex); + return 0; +} + static struct cftype rdmacg_files[] = { { .name = "max", @@ -589,6 +651,12 @@ static struct cftype rdmacg_files[] = { .private = RDMACG_RESOURCE_TYPE_PEAK, .flags = CFTYPE_NOT_ON_ROOT, }, + { + .name = "events", + .seq_show = rdmacg_events_show, + .file_offset = offsetof(struct rdma_cgroup, events_file), + .flags = CFTYPE_NOT_ON_ROOT, + }, { } /* terminate */ }; -- cgit v1.2.3 From aefe4847f0891e2e71bedf5478d1cf350f86fc61 Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Thu, 14 May 2026 14:50:33 +0800 Subject: cgroup/rdma: add rdma.events.local for per-cgroup allocation failure attribution Add per-cgroup local event counters to track RDMA resource limit exhaustion from the perspective of individual cgroups. The rdma.events.local file reports two per-resource counters: - max: number of times this cgroup's limit was the one that blocked an allocation in the subtree - alloc_fail: number of allocation attempts originating from this cgroup that failed due to an ancestor's limit This mirrors the design of pids.events.local, where events are attributed to the cgroup that imposed the limit, not necessarily the cgroup where the allocation was attempted. Also extend rdma.events with a hierarchical alloc_fail counter that tracks allocation failures propagating upward from the requesting cgroup, complementing the existing max counter, so that rdma.events and rdma.events.local share the same output format. Signed-off-by: Tao Cui Signed-off-by: Tejun Heo --- kernel/cgroup/rdma.c | 143 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 107 insertions(+), 36 deletions(-) (limited to 'kernel') diff --git a/kernel/cgroup/rdma.c b/kernel/cgroup/rdma.c index 927bbf1eb949..7c238a9d64d4 100644 --- a/kernel/cgroup/rdma.c +++ b/kernel/cgroup/rdma.c @@ -82,8 +82,11 @@ struct rdmacg_resource_pool { /* total number counts which are set to max */ int num_max_cnt; - /* per-resource hierarchical max event counters */ + /* per-resource event counters */ u64 events_max[RDMACG_RESOURCE_MAX]; + u64 events_alloc_fail[RDMACG_RESOURCE_MAX]; + u64 events_local_max[RDMACG_RESOURCE_MAX]; + u64 events_local_alloc_fail[RDMACG_RESOURCE_MAX]; }; static struct rdma_cgroup *css_rdmacg(struct cgroup_subsys_state *css) @@ -131,6 +134,26 @@ static void free_cg_rpool_locked(struct rdmacg_resource_pool *rpool) kfree(rpool); } +static bool rpool_has_persistent_state(struct rdmacg_resource_pool *rpool) +{ + int i; + + /* + * Keep the rpool alive if any peak value is non-zero, + * so that rdma.peak persists as a historical high- + * watermark even after all resources are freed. + */ + for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { + if (rpool->resources[i].peak || + READ_ONCE(rpool->events_max[i]) || + READ_ONCE(rpool->events_local_max[i]) || + READ_ONCE(rpool->events_alloc_fail[i]) || + READ_ONCE(rpool->events_local_alloc_fail[i])) + return true; + } + return false; +} + static struct rdmacg_resource_pool * find_cg_rpool_locked(struct rdma_cgroup *cg, struct rdmacg_device *device) @@ -209,37 +232,30 @@ uncharge_cg_locked(struct rdma_cgroup *cg, rpool->usage_sum--; if (rpool->usage_sum == 0 && rpool->num_max_cnt == RDMACG_RESOURCE_MAX) { - int i; - - /* - * Keep the rpool alive if any peak value is non-zero, - * so that rdma.peak persists as a historical high- - * watermark even after all resources are freed. - */ - for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { - if (rpool->resources[i].peak || - READ_ONCE(rpool->events_max[i])) - return; + if (!rpool_has_persistent_state(rpool)) { + /* + * No user of the rpool and all entries are set to max, so + * safe to delete this rpool. + */ + free_cg_rpool_locked(rpool); } - /* - * No user of the rpool and all entries are set to max, so - * safe to delete this rpool. - */ - free_cg_rpool_locked(rpool); } } /** - * rdmacg_event_locked - fire hierarchical max event when resource limit is hit + * rdmacg_event_locked - fire event when resource allocation exceeds limit + * @cg: requesting cgroup * @over_cg: cgroup whose limit was exceeded * @device: rdma device * @index: resource type index * - * Must be called under rdmacg_mutex. Propagates max event counts - * from @over_cg (including itself) upward to all ancestors with - * an rpool and notifies userspace. + * Must be called under rdmacg_mutex. Updates event counters in the + * resource pools of @cg and @over_cg, propagates hierarchical max + * events from @over_cg (including itself) upward, and notifies + * userspace via cgroup_file_notify(). */ -static void rdmacg_event_locked(struct rdma_cgroup *over_cg, +static void rdmacg_event_locked(struct rdma_cgroup *cg, + struct rdma_cgroup *over_cg, struct rdmacg_device *device, enum rdmacg_resource_type index) { @@ -248,6 +264,21 @@ static void rdmacg_event_locked(struct rdma_cgroup *over_cg, lockdep_assert_held(&rdmacg_mutex); + /* Increment local alloc_fail in requesting cgroup */ + rpool = find_cg_rpool_locked(cg, device); + if (rpool) { + rpool->events_local_alloc_fail[index]++; + cgroup_file_notify(&cg->events_local_file); + } + + /* Increment local max in the over-limit cgroup */ + rpool = find_cg_rpool_locked(over_cg, device); + if (rpool) { + rpool->events_local_max[index]++; + cgroup_file_notify(&over_cg->events_local_file); + } + + /* Propagate hierarchical max events upward */ for (p = over_cg; parent_rdmacg(p); p = parent_rdmacg(p)) { rpool = get_cg_rpool_locked(p, device); if (!IS_ERR(rpool)) { @@ -255,6 +286,14 @@ static void rdmacg_event_locked(struct rdma_cgroup *over_cg, cgroup_file_notify(&p->events_file); } } + /* Propagate hierarchical alloc_fail from requesting cgroup upward */ + for (p = cg; parent_rdmacg(p); p = parent_rdmacg(p)) { + rpool = get_cg_rpool_locked(p, device); + if (!IS_ERR(rpool)) { + rpool->events_alloc_fail[index]++; + cgroup_file_notify(&p->events_file); + } + } } /** @@ -368,7 +407,7 @@ int rdmacg_try_charge(struct rdma_cgroup **rdmacg, err: if (ret == -EAGAIN) - rdmacg_event_locked(p, device, index); + rdmacg_event_locked(cg, p, device, index); mutex_unlock(&rdmacg_mutex); rdmacg_uncharge_hierarchy(cg, device, p, index); return ret; @@ -525,18 +564,13 @@ static ssize_t rdmacg_resource_set_max(struct kernfs_open_file *of, if (rpool->usage_sum == 0 && rpool->num_max_cnt == RDMACG_RESOURCE_MAX) { - int i; - - for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { - if (rpool->resources[i].peak || - READ_ONCE(rpool->events_max[i])) - goto dev_err; + if (!rpool_has_persistent_state(rpool)) { + /* + * No user of the rpool and all entries are set to max, so + * safe to delete this rpool. + */ + free_cg_rpool_locked(rpool); } - /* - * No user of the rpool and all entries are set to max, so - * safe to delete this rpool. - */ - free_cg_rpool_locked(rpool); } dev_err: @@ -618,9 +652,40 @@ static int rdmacg_events_show(struct seq_file *sf, void *v) seq_printf(sf, "%s ", device->name); for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { - seq_printf(sf, "%s.max=%llu", + seq_printf(sf, "%s.max=%llu %s.alloc_fail=%llu", + rdmacg_resource_names[i], + rpool ? READ_ONCE(rpool->events_max[i]) : 0ULL, + rdmacg_resource_names[i], + rpool ? READ_ONCE(rpool->events_alloc_fail[i]) : 0ULL); + if (i < RDMACG_RESOURCE_MAX - 1) + seq_putc(sf, ' '); + } + seq_putc(sf, '\n'); + } + + mutex_unlock(&rdmacg_mutex); + return 0; +} + +static int rdmacg_events_local_show(struct seq_file *sf, void *v) +{ + struct rdma_cgroup *cg = css_rdmacg(seq_css(sf)); + struct rdmacg_resource_pool *rpool; + struct rdmacg_device *device; + int i; + + mutex_lock(&rdmacg_mutex); + + list_for_each_entry(device, &rdmacg_devices, dev_node) { + rpool = find_cg_rpool_locked(cg, device); + + seq_printf(sf, "%s ", device->name); + for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { + seq_printf(sf, "%s.max=%llu %s.alloc_fail=%llu", + rdmacg_resource_names[i], + rpool ? READ_ONCE(rpool->events_local_max[i]) : 0ULL, rdmacg_resource_names[i], - rpool ? READ_ONCE(rpool->events_max[i]) : 0ULL); + rpool ? READ_ONCE(rpool->events_local_alloc_fail[i]) : 0ULL); if (i < RDMACG_RESOURCE_MAX - 1) seq_putc(sf, ' '); } @@ -657,6 +722,12 @@ static struct cftype rdmacg_files[] = { .file_offset = offsetof(struct rdma_cgroup, events_file), .flags = CFTYPE_NOT_ON_ROOT, }, + { + .name = "events.local", + .seq_show = rdmacg_events_local_show, + .file_offset = offsetof(struct rdma_cgroup, events_local_file), + .flags = CFTYPE_NOT_ON_ROOT, + }, { } /* terminate */ }; -- cgit v1.2.3 From a828abbb897657451d96ad7bf20f1893ac983bb9 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Fri, 15 May 2026 13:32:32 +0200 Subject: bpf: make bpf_session_is_return() reference optional Building without CONFIG_BPF_EVENTS produces a build-time warning: WARN: resolve_btfids: unresolved symbol bpf_session_is_return The function is actually defined in kernel/trace/bpf_trace.o, which is built conditionally based on configuration. Make the reference to this function conditional as well, as is already done in the bpf verifier for other functions. Fixes: 8fe4dc4f6456 ("bpf: change prototype of bpf_session_{cookie,is_return}") Signed-off-by: Arnd Bergmann Link: https://lore.kernel.org/r/20260515113242.2706303-1-arnd@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 69d75515ed3f..88b40c979b56 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11263,7 +11263,11 @@ BTF_ID(func, bpf_task_work_schedule_resume) BTF_ID(func, bpf_arena_alloc_pages) BTF_ID(func, bpf_arena_free_pages) BTF_ID(func, bpf_arena_reserve_pages) +#ifdef CONFIG_BPF_EVENTS BTF_ID(func, bpf_session_is_return) +#else +BTF_ID_UNUSED +#endif BTF_ID(func, bpf_stream_vprintk) BTF_ID(func, bpf_stream_print_stack) -- cgit v1.2.3 From 3360a5c16d87933fb74b530f5e016eb3dfffee5d Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 4 May 2026 14:51:17 -1000 Subject: cgroup: Inline cgroup_has_tasks() in cgroup.h cpuset reads cs->css.cgroup->nr_populated_csets directly in two places to test whether a cgroup has tasks. cgroup.c already has a matching helper, cgroup_has_tasks(). Move it to cgroup.h as static inline and use that instead. This is to prepare for relocation of cgroup->nr_populated_csets. No semantic change. Signed-off-by: Tejun Heo --- kernel/cgroup/cgroup.c | 5 ----- kernel/cgroup/cpuset-v1.c | 2 +- kernel/cgroup/cpuset.c | 2 +- 3 files changed, 2 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index bd10a7e2f9c5..7a94c2ea1036 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -376,11 +376,6 @@ static void cgroup_idr_remove(struct idr *idr, int id) spin_unlock_bh(&cgroup_idr_lock); } -static bool cgroup_has_tasks(struct cgroup *cgrp) -{ - return cgrp->nr_populated_csets; -} - static bool cgroup_is_threaded(struct cgroup *cgrp) { return cgrp->dom_cgrp != cgrp; diff --git a/kernel/cgroup/cpuset-v1.c b/kernel/cgroup/cpuset-v1.c index 7308e9b02495..3e9968dd91e9 100644 --- a/kernel/cgroup/cpuset-v1.c +++ b/kernel/cgroup/cpuset-v1.c @@ -312,7 +312,7 @@ void cpuset1_hotplug_update_tasks(struct cpuset *cs, * This is full cgroup operation which will also call back into * cpuset. Execute it asynchronously using workqueue. */ - if (is_empty && cs->css.cgroup->nr_populated_csets && + if (is_empty && cgroup_has_tasks(cs->css.cgroup) && css_tryget_online(&cs->css)) { struct cpuset_remove_tasks_struct *s; diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 74d5c494d6ae..8500e4341c60 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -432,7 +432,7 @@ static inline bool partition_is_populated(struct cpuset *cs, * nr_populated_domain_children may include populated * csets from descendants that are partitions. */ - if (cs->css.cgroup->nr_populated_csets || + if (cgroup_has_tasks(cs->css.cgroup) || cs->attach_in_progress) return true; -- cgit v1.2.3 From 44fabf05634ce9e90b3fb179ea962995b7bbaa09 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 4 May 2026 14:51:18 -1000 Subject: cgroup: Annotate unlocked nr_populated_* accesses with READ_ONCE/WRITE_ONCE cgroup_update_populated() updates nr_populated_csets, nr_populated_domain_children, and nr_populated_threaded_children under css_set_lock, but cgroup_has_tasks(), cgroup_is_populated(), and cgroup_can_be_thread_root() read them without holding it. Use READ_ONCE/WRITE_ONCE. Signed-off-by: Tejun Heo --- kernel/cgroup/cgroup.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 7a94c2ea1036..d1395784871a 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -404,7 +404,7 @@ static bool cgroup_can_be_thread_root(struct cgroup *cgrp) return false; /* can only have either domain or threaded children */ - if (cgrp->nr_populated_domain_children) + if (READ_ONCE(cgrp->nr_populated_domain_children)) return false; /* and no domain controllers can be enabled */ @@ -783,12 +783,15 @@ static void cgroup_update_populated(struct cgroup *cgrp, bool populated) bool was_populated = cgroup_is_populated(cgrp); if (!child) { - cgrp->nr_populated_csets += adj; + WRITE_ONCE(cgrp->nr_populated_csets, + cgrp->nr_populated_csets + adj); } else { if (cgroup_is_threaded(child)) - cgrp->nr_populated_threaded_children += adj; + WRITE_ONCE(cgrp->nr_populated_threaded_children, + cgrp->nr_populated_threaded_children + adj); else - cgrp->nr_populated_domain_children += adj; + WRITE_ONCE(cgrp->nr_populated_domain_children, + cgrp->nr_populated_domain_children + adj); } if (was_populated == cgroup_is_populated(cgrp)) -- cgit v1.2.3 From c4799253a3ee74ebb27be72fb991c597a5902c01 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 4 May 2026 14:51:19 -1000 Subject: cgroup: Move populated counters to cgroup_subsys_state Later patches replace the cgroup-level finish_destroy_work deferral added by 93618edf7538 ("cgroup: Defer css percpu_ref kill on rmdir until cgroup is depopulated") with a per-subsys-css deferral. That needs each subsystem css to track its own populated count. Move the populated counters from cgroup onto cgroup_subsys_state. cgroup->self is itself a cgroup_subsys_state and self.parent walks the same chain as cgroup_parent(), so cgroup_update_populated() generalizes to a single css_update_populated() taking a css. The cgroup-side bookkeeping runs only when the walk started from a self css. Keep nr_populated_{domain,threaded}_children on cgroup. Both sum to self.nr_populated_children, but staying as dedicated fields to allow readers like cgroup_can_be_thread_root() unlocked access. css_set_update_populated() also walks the per-subsys-css chain so each subsystem css's hierarchical populated count is maintained. No reader consumes those counts yet. Signed-off-by: Tejun Heo --- kernel/cgroup/cgroup.c | 95 +++++++++++++++++++++++++++++--------------------- 1 file changed, 55 insertions(+), 40 deletions(-) (limited to 'kernel') diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index d1395784871a..dd4ea9d83100 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -756,65 +756,70 @@ static bool css_set_populated(struct css_set *cset) } /** - * cgroup_update_populated - update the populated count of a cgroup - * @cgrp: the target cgroup - * @populated: inc or dec populated count - * - * One of the css_sets associated with @cgrp is either getting its first - * task or losing the last. Update @cgrp->nr_populated_* accordingly. The - * count is propagated towards root so that a given cgroup's - * nr_populated_children is zero iff none of its descendants contain any - * tasks. - * - * @cgrp's interface file "cgroup.populated" is zero if both - * @cgrp->nr_populated_csets and @cgrp->nr_populated_children are zero and - * 1 otherwise. When the sum changes from or to zero, userland is notified - * that the content of the interface file has changed. This can be used to - * detect when @cgrp and its descendants become populated or empty. + * css_update_populated - update the populated state of a css and ancestors + * @css: leaf css whose own populated count is changing + * @populated: inc or dec + * + * One of the css_sets pinned by @css is getting its first task or losing the + * last. Propagate the transition up the parent chain so that a css's + * nr_populated_children is zero iff none of its descendants contain any tasks. + * + * For a cgroup->self walk, also runs cgroup-side bookkeeping at each level: + * domain/threaded child split, deferred-destroy trigger, and notification via + * "cgroup.populated" (zero iff cgrp->self has neither populated csets nor + * populated children; userland is notified on transitions). */ -static void cgroup_update_populated(struct cgroup *cgrp, bool populated) +static void css_update_populated(struct cgroup_subsys_state *css, bool populated) { - struct cgroup *child = NULL; + struct cgroup_subsys_state *child = NULL; int adj = populated ? 1 : -1; lockdep_assert_held(&css_set_lock); do { - bool was_populated = cgroup_is_populated(cgrp); + /* non-NULL only on the cgroup->self walk */ + struct cgroup *cgrp = css_is_self(css) ? css->cgroup : NULL; + bool was_populated = css_is_populated(css); if (!child) { - WRITE_ONCE(cgrp->nr_populated_csets, - cgrp->nr_populated_csets + adj); + WRITE_ONCE(css->nr_populated_csets, + css->nr_populated_csets + adj); } else { - if (cgroup_is_threaded(child)) - WRITE_ONCE(cgrp->nr_populated_threaded_children, - cgrp->nr_populated_threaded_children + adj); - else - WRITE_ONCE(cgrp->nr_populated_domain_children, - cgrp->nr_populated_domain_children + adj); + WRITE_ONCE(css->nr_populated_children, + css->nr_populated_children + adj); + if (cgrp) { + if (cgroup_is_threaded(child->cgroup)) + WRITE_ONCE(cgrp->nr_populated_threaded_children, + cgrp->nr_populated_threaded_children + adj); + else + WRITE_ONCE(cgrp->nr_populated_domain_children, + cgrp->nr_populated_domain_children + adj); + } } - if (was_populated == cgroup_is_populated(cgrp)) + if (was_populated == css_is_populated(css)) break; /* * Subtree just emptied below an offlined cgrp. Fire deferred * destroy. The transition is one-shot. */ - if (was_populated && !css_is_online(&cgrp->self)) { + if (cgrp && was_populated && !css_is_online(css)) { cgroup_get(cgrp); WARN_ON_ONCE(!queue_work(cgroup_offline_wq, &cgrp->finish_destroy_work)); } - cgroup1_check_for_release(cgrp); - TRACE_CGROUP_PATH(notify_populated, cgrp, - cgroup_is_populated(cgrp)); - cgroup_file_notify(&cgrp->events_file); + if (cgrp) { + cgroup1_check_for_release(cgrp); + TRACE_CGROUP_PATH(notify_populated, cgrp, + cgroup_is_populated(cgrp)); + cgroup_file_notify(&cgrp->events_file); + } - child = cgrp; - cgrp = cgroup_parent(cgrp); - } while (cgrp); + child = css; + css = css->parent; + } while (css); } /** @@ -822,17 +827,27 @@ static void cgroup_update_populated(struct cgroup *cgrp, bool populated) * @cset: target css_set * @populated: whether @cset is populated or depopulated * - * @cset is either getting the first task or losing the last. Update the - * populated counters of all associated cgroups accordingly. + * @cset is either getting the first task or losing the last. Update the + * populated counters along each linked cgroup's self chain and each + * subsystem css that @cset pins. */ static void css_set_update_populated(struct css_set *cset, bool populated) { struct cgrp_cset_link *link; + struct cgroup_subsys *ss; + int ssid; lockdep_assert_held(&css_set_lock); list_for_each_entry(link, &cset->cgrp_links, cgrp_link) - cgroup_update_populated(link->cgrp, populated); + css_update_populated(&link->cgrp->self, populated); + + for_each_subsys(ss, ssid) { + struct cgroup_subsys_state *css = cset->subsys[ssid]; + + if (css) + css_update_populated(css, populated); + } } /* @@ -2190,7 +2205,7 @@ int cgroup_setup_root(struct cgroup_root *root, u32 ss_mask) hash_for_each(css_set_table, i, cset, hlist) { link_css_set(&tmp_links, cset, root_cgrp); if (css_set_populated(cset)) - cgroup_update_populated(root_cgrp, true); + css_update_populated(&root_cgrp->self, true); } spin_unlock_irq(&css_set_lock); @@ -6145,7 +6160,7 @@ static void kill_css_finish(struct cgroup_subsys_state *css) * * - cgroup_finish_destroy(): kicks the percpu_ref kill via kill_css_finish() on * each subsystem css. Fires once @cgrp's subtree is fully drained, either - * inline here or from cgroup_update_populated(). + * inline here or from css_update_populated(). * * - The percpu_ref kill chain: css_killed_ref_fn -> css_killed_work_fn -> * ->css_offline() -> release/free. -- cgit v1.2.3 From cfc1da7e1127b4c8787f4dc25d59987c10c9107f Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 4 May 2026 14:51:20 -1000 Subject: cgroup: Add per-subsys-css kill_css_finish deferral 93618edf7538 ("cgroup: Defer css percpu_ref kill on rmdir until cgroup is depopulated") deferred kill_css_finish() at the cgroup level: rmdir waits for the entire cgroup's populated count to drop to zero, then fires kill_css_finish() on every subsystem css at once. Replace that with per-subsys-css deferral. Each subsystem css now tracks its own hierarchical populated count and independently defers its kill_css_finish() until its own subtree drains. The rmdir-race fix carries through unchanged in shape. The dying css's ->css_offline() still waits until no PF_EXITING task references it, and v2's cgroup-level machinery goes away. cgroup_apply_control_disable() has the same race shape (PF_EXITING tasks pinning a css whose ->css_offline() is about to run) and stays synchronous here. This patch lays the groundwork for fixing it - per-cgroup waiting can't gate one subsys css being killed while the rest of the cgroup stays live, but per-css can. Subtree-wide invariant preserved: a dying ancestor css stays populated through nr_populated_children until every dying descendant's task drains, so the walker fires the ancestor's kill_finish_work only after all descendants have drained. Add paired smp_mb()s in kill_css_sync() and css_update_populated() to fence the StoreLoad on (CSS_DYING, populated counter), guaranteeing that either the walker queues kill_finish_work or the caller fires synchronously. cgroup_destroy_locked() was implicitly fenced by an unrelated css_set_lock pair; cgroup_apply_control_disable() in the next patch is not. Signed-off-by: Tejun Heo --- kernel/cgroup/cgroup.c | 83 ++++++++++++++++++++++++++------------------------ 1 file changed, 43 insertions(+), 40 deletions(-) (limited to 'kernel') diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index dd4ea9d83100..fa24102535d9 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -264,7 +264,6 @@ static void cgroup_finalize_control(struct cgroup *cgrp, int ret); static void css_task_iter_skip(struct css_task_iter *it, struct task_struct *task); static int cgroup_destroy_locked(struct cgroup *cgrp); -static void cgroup_finish_destroy(struct cgroup *cgrp); static void kill_css_sync(struct cgroup_subsys_state *css); static void kill_css_finish(struct cgroup_subsys_state *css); static struct cgroup_subsys_state *css_create(struct cgroup *cgrp, @@ -801,13 +800,19 @@ static void css_update_populated(struct cgroup_subsys_state *css, bool populated break; /* - * Subtree just emptied below an offlined cgrp. Fire deferred - * destroy. The transition is one-shot. + * Pair with smp_mb() in kill_css_sync(). Either we observe + * CSS_DYING and queue, or the caller observes our decrement + * and fires synchronously. */ - if (cgrp && was_populated && !css_is_online(css)) { - cgroup_get(cgrp); - WARN_ON_ONCE(!queue_work(cgroup_offline_wq, - &cgrp->finish_destroy_work)); + smp_mb(); + + /* + * Subtree just emptied below a dying css. Fire deferred kill. + * The transition is one-shot for a dying css. + */ + if (was_populated && css_is_dying(css)) { + css_get(css); + WARN_ON_ONCE(!queue_work(cgroup_offline_wq, &css->kill_finish_work)); } if (cgrp) { @@ -2064,16 +2069,6 @@ static int cgroup_reconfigure(struct fs_context *fc) return 0; } -static void cgroup_finish_destroy_work_fn(struct work_struct *work) -{ - struct cgroup *cgrp = container_of(work, struct cgroup, finish_destroy_work); - - cgroup_lock(); - cgroup_finish_destroy(cgrp); - cgroup_unlock(); - cgroup_put(cgrp); -} - static void init_cgroup_housekeeping(struct cgroup *cgrp) { struct cgroup_subsys *ss; @@ -2100,7 +2095,6 @@ static void init_cgroup_housekeeping(struct cgroup *cgrp) #endif init_waitqueue_head(&cgrp->offline_waitq); - INIT_WORK(&cgrp->finish_destroy_work, cgroup_finish_destroy_work_fn); INIT_WORK(&cgrp->release_agent_work, cgroup1_release_agent); } @@ -5695,6 +5689,22 @@ static void css_release(struct percpu_ref *ref) queue_work(cgroup_release_wq, &css->destroy_work); } +/* + * Deferred kill_css_finish() fired from css_update_populated() once a dying + * css's hierarchical populated state drops to zero. Pinned by css_get() at the + * queue site; matched by css_put() here. + */ +static void kill_css_finish_work_fn(struct work_struct *work) +{ + struct cgroup_subsys_state *css = + container_of(work, struct cgroup_subsys_state, kill_finish_work); + + cgroup_lock(); + kill_css_finish(css); + cgroup_unlock(); + css_put(css); +} + static void init_and_link_css(struct cgroup_subsys_state *css, struct cgroup_subsys *ss, struct cgroup *cgrp) { @@ -5708,6 +5718,7 @@ static void init_and_link_css(struct cgroup_subsys_state *css, css->id = -1; INIT_LIST_HEAD(&css->sibling); INIT_LIST_HEAD(&css->children); + INIT_WORK(&css->kill_finish_work, kill_css_finish_work_fn); css->serial_nr = css_serial_nr_next++; atomic_set(&css->online_cnt, 0); @@ -6083,6 +6094,13 @@ static void kill_css_sync(struct cgroup_subsys_state *css) css->flags |= CSS_DYING; + /* + * Pair with smp_mb() in css_update_populated(). Either our + * caller observes the walker's decrement and fires + * synchronously, or the walker observes CSS_DYING and queues. + */ + smp_mb(); + /* * This must happen before css is disassociated with its cgroup. * See seq_css() for details. @@ -6158,9 +6176,9 @@ static void kill_css_finish(struct cgroup_subsys_state *css) * - This function: synchronous user-visible state teardown plus kill_css_sync() * on each subsystem css. * - * - cgroup_finish_destroy(): kicks the percpu_ref kill via kill_css_finish() on - * each subsystem css. Fires once @cgrp's subtree is fully drained, either - * inline here or from css_update_populated(). + * - For each subsys css: fire kill_css_finish() synchronously if the subtree is + * already drained, otherwise rely on css_update_populated() to queue + * kill_finish_work when the last populated cset under the css empties. * * - The percpu_ref kill chain: css_killed_ref_fn -> css_killed_work_fn -> * ->css_offline() -> release/free. @@ -6238,29 +6256,14 @@ static int cgroup_destroy_locked(struct cgroup *cgrp) /* put the base reference */ percpu_ref_kill(&cgrp->self.refcnt); - if (!cgroup_is_populated(cgrp)) - cgroup_finish_destroy(cgrp); + for_each_css(css, ssid, cgrp) { + if (!css_is_populated(css)) + kill_css_finish(css); + } return 0; }; -/** - * cgroup_finish_destroy - deferred half of @cgrp destruction - * @cgrp: cgroup whose subtree just became empty - * - * See cgroup_destroy_locked() for the rationale. - */ -static void cgroup_finish_destroy(struct cgroup *cgrp) -{ - struct cgroup_subsys_state *css; - int ssid; - - lockdep_assert_held(&cgroup_mutex); - - for_each_css(css, ssid, cgrp) - kill_css_finish(css); -} - int cgroup_rmdir(struct kernfs_node *kn) { struct cgroup *cgrp; -- cgit v1.2.3 From 1dffd95575eb05bc7ec20ec096ce73be4c5d1ed5 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 4 May 2026 14:51:21 -1000 Subject: cgroup: Defer kill_css_finish() in cgroup_apply_control_disable() Same race shape as the rmdir path that 93618edf7538 ("cgroup: Defer css percpu_ref kill on rmdir until cgroup is depopulated") fixed: a task past exit_signals() whose cset subsys[ssid] still pins the disabled controller's css can be touching subsys state while ->css_offline() runs. The earlier patches in this series built up the per-subsys-css deferral machinery and routed cgroup_destroy_locked() through it. Apply the same shape to cgroup_apply_control_disable(): kill_css_sync(css); if (!css_is_populated(css)) kill_css_finish(css); When the dying css is still populated, kill_css_finish() is deferred. The walker in css_update_populated() fires kill_finish_work once the css's hierarchical populated count drops to zero. cgroup_lock_and_drain_offline()'s wait predicate switches from percpu_ref_is_dying() to css_is_dying(). CSS_DYING is set by kill_css_sync() and is a strict superset of percpu_ref_is_dying. Without this change, a +cpu re-enable after a deferred -cpu disable would skip the drain (percpu_ref isn't killed yet) and observe the still-CSS_DYING css through cgroup_css(), treating it as live. Signed-off-by: Tejun Heo --- kernel/cgroup/cgroup.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index fa24102535d9..bdc8deedb4f7 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -3237,7 +3237,7 @@ restart: struct cgroup_subsys_state *css = cgroup_css(dsct, ss); DEFINE_WAIT(wait); - if (!css || !percpu_ref_is_dying(&css->refcnt)) + if (!css || !css_is_dying(css)) continue; cgroup_get_live(dsct); @@ -3405,7 +3405,8 @@ static void cgroup_apply_control_disable(struct cgroup *cgrp) if (css->parent && !(cgroup_ss_mask(dsct) & (1 << ss->id))) { kill_css_sync(css); - kill_css_finish(css); + if (!css_is_populated(css)) + kill_css_finish(css); } else if (!css_visible(css)) { css_clear_dir(css); if (ss->css_reset) -- cgit v1.2.3 From 4286f5deee14b26a9f0447b566d4c7cb7e2e2702 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 15 May 2026 15:50:40 -0700 Subject: bpf: Validate outgoing stack args when btf_prepare_func_args fails btf_prepare_func_args() sets sub->arg_cnt before validating arg types. If validation fails (e.g. unsupported pointer type in a static subprog), check_outgoing_stack_args() is skipped because btf_check_func_arg_match() returns early. For static subprogs, check_func_call() ignores non-EFAULT errors and proceeds with the call. This causes the callee to read stack arg slots that the caller never stored or not initialized, potentially dereferencing NULL caller->stack_arg_regs or getting no-initialized value. To fix the issue, when btf_prepare_func_args() fails and the subprog expects stack args, call check_outgoing_stack_args() to verify the caller initialized the slots. Return -EFAULT on failure so the error is not ignored. Fixes: 3ab5bd317ee2 ("bpf: Set sub->arg_cnt earlier in btf_prepare_func_args()") Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260515225040.821515-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 76a07f09ab64..8dd79b735a69 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -9118,11 +9118,17 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, struct bpf_func_state *caller = cur_func(env); struct bpf_verifier_log *log = &env->log; u32 i; - int ret; + int ret, err; ret = btf_prepare_func_args(env, subprog); - if (ret) + if (ret) { + if (bpf_in_stack_arg_cnt(sub) > 0) { + err = check_outgoing_stack_args(env, caller, sub->arg_cnt); + if (err) + return err; + } return ret; + } ret = check_outgoing_stack_args(env, caller, sub->arg_cnt); if (ret) -- cgit v1.2.3 From d1dbe443a0abb4ea3ec35a16e36efe6d3bbf72f6 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 15 May 2026 15:50:56 -0700 Subject: bpf: Fix arg_track_join log to use sa prefix for stack arg slots arg_track_join() logs state transitions at CFG merge points. For stack arg slots (r >= MAX_BPF_REG), it printed "r11:", "r12:", etc., which is misleading since r11 is a special register (BPF_REG_PARAMS) not meaningful to the user. Fix it to print "sa0:", "sa1:", etc., matching the per-instruction transition log in arg_track_log() which already uses the "sa" prefix. Update the existing stack_arg_pruning_type_mismatch selftest to expect the corrected format. Fixes: 2af4e792773f ("bpf: Extend liveness analysis to track stack argument slots") Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260515225056.823086-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/liveness.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 7f4a0e4c2c49..0aadfbae0acc 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -806,7 +806,9 @@ static bool arg_track_join(struct bpf_verifier_env *env, int idx, int target, in return true; verbose(env, "arg JOIN insn %d -> %d ", idx, target); - if (r >= 0) + if (r >= MAX_BPF_REG) + verbose(env, "sa%d: ", r - MAX_BPF_REG); + else if (r >= 0) verbose(env, "r%d: ", r); else verbose(env, "fp%+d: ", r * 8); -- cgit v1.2.3 From 98540a12823a016e2e1fa0db15543b22ac1fa056 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 15 May 2026 15:51:01 -0700 Subject: bpf: Clean up redundant stack arg checks for non-JITed programs Remove a redundant stack_arg_cnt check in __bpf_prog_select_runtime() and start the stack arg loop from index 0 in bpf_fixup_call_args(). Both changes are no-ops that simplify the code: In __bpf_prog_select_runtime(), the subprog_info[0].stack_arg_cnt check is unreachable: - when there is only a main program (no bpf-to-bpf calls), subprog_info[0].stack_arg_cnt is always 0 because the main program's arg_cnt is forced to 1 - when bpf-to-bpf calls use stack args and JIT succeeds, fp->bpf_func is set and this code is skipped - when JIT fails, bpf_fixup_call_args() rejects the program before we get to __bpf_prog_select_runtime(). In bpf_fixup_call_args(), starting the loop at i=1 skipped subprog 0, which is safe since the main program always has arg_cnt=1 and thus bpf_in_stack_arg_cnt() returns 0. Starting at i=0 removes the need to reason about this invariant. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260515225101.824054-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/core.c | 2 +- kernel/bpf/fixups.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 427a6d828e01..cdbe9fdf474f 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -2609,7 +2609,7 @@ struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct goto finalize; if (IS_ENABLED(CONFIG_BPF_JIT_ALWAYS_ON) || - bpf_prog_has_kfunc_call(fp) || (env && env->subprog_info[0].stack_arg_cnt)) + bpf_prog_has_kfunc_call(fp)) jit_needed = true; if (!bpf_prog_select_interpreter(fp)) diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 19056016eed8..2cec4e8cd4a0 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -1407,7 +1407,7 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env) verbose(env, "calling kernel functions are not allowed in non-JITed programs\n"); return -EINVAL; } - for (i = 1; i < env->subprog_cnt; i++) { + for (i = 0; i < env->subprog_cnt; i++) { if (bpf_in_stack_arg_cnt(&env->subprog_info[i])) { verbose(env, "stack args are not supported in non-JITed programs\n"); return -EINVAL; -- cgit v1.2.3 From 3d562d35a044ae798cab421c65a116f8cedfa5d4 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sun, 17 May 2026 09:55:28 +0200 Subject: bpf: Check global subprog exception paths Global subprogs are verified independently and are not descended into when their callers are symbolically executed. This means a caller can hold references or locks across a global subprog call that may throw, while the verifier only checks the non-exceptional return path at the call site. Record whether a subprog might throw in the CFG summary pass, alongside the existing might_sleep and packet-data-changing summaries, and propagate that effect through reachable callees. When a global subprog is marked as possibly throwing, push the normal continuation and validate the exceptional path immediately at the call site, avoiding a synthetic exception state and associated special case in the pruning checks. Fixes: f18b03fabaa9 ("bpf: Implement BPF exceptions") Signed-off-by: Kumar Kartikeya Dwivedi Link: https://lore.kernel.org/r/20260517075530.3461166-2-memxor@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/cfg.c | 13 ++++++++++++- kernel/bpf/verifier.c | 23 +++++++++++++++++------ 2 files changed, 29 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c index 998f42a8189a..26d37066465f 100644 --- a/kernel/bpf/cfg.c +++ b/kernel/bpf/cfg.c @@ -64,11 +64,19 @@ static void mark_subprog_might_sleep(struct bpf_verifier_env *env, int off) subprog->might_sleep = true; } +static void mark_subprog_might_throw(struct bpf_verifier_env *env, int off) +{ + struct bpf_subprog_info *subprog; + + subprog = bpf_find_containing_subprog(env, off); + subprog->might_throw = true; +} + /* 't' is an index of a call-site. * 'w' is a callee entry point. * Eventually this function would be called when env->cfg.insn_state[w] == EXPLORED. * Rely on DFS traversal order and absence of recursive calls to guarantee that - * callee's change_pkt_data marks would be correct at that moment. + * callee's effect marks would be correct at that moment. */ static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w) { @@ -78,6 +86,7 @@ static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w) callee = bpf_find_containing_subprog(env, w); caller->changes_pkt_data |= callee->changes_pkt_data; caller->might_sleep |= callee->might_sleep; + caller->might_throw |= callee->might_throw; } enum { @@ -509,6 +518,8 @@ static int visit_insn(int t, struct bpf_verifier_env *env) mark_subprog_might_sleep(env, t); if (ret == 0 && bpf_is_kfunc_pkt_changing(&meta)) mark_subprog_changes_pkt_data(env, t); + if (ret == 0 && bpf_is_throw_kfunc(insn)) + mark_subprog_might_throw(env, t); } return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 88b40c979b56..7fb88e1cd7c4 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -442,7 +442,6 @@ static bool is_dynptr_ref_function(enum bpf_func_id func_id) static bool is_sync_callback_calling_kfunc(u32 btf_id); static bool is_async_callback_calling_kfunc(u32 btf_id); static bool is_callback_calling_kfunc(u32 btf_id); -static bool is_bpf_throw_kfunc(struct bpf_insn *insn); static bool is_bpf_wq_set_callback_kfunc(u32 btf_id); static bool is_task_work_add_kfunc(u32 func_id); @@ -5405,7 +5404,7 @@ continue_func: if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off) { bool err = false; - if (!is_bpf_throw_kfunc(insn + i)) + if (!bpf_is_throw_kfunc(insn + i)) continue; for (tmp = idx; tmp >= 0 && !err; tmp = dinfo[tmp].caller) { if (subprog[tmp].is_cb) { @@ -9499,6 +9498,9 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins return 0; } +static int process_bpf_exit_full(struct bpf_verifier_env *env, + bool *do_print_state, bool exception_exit); + static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, int *insn_idx) { @@ -9552,6 +9554,17 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; } + if (env->subprog_info[subprog].might_throw) { + struct bpf_verifier_state *branch; + + branch = push_stack(env, *insn_idx + 1, *insn_idx, false); + if (IS_ERR(branch)) { + verbose(env, "failed to push state for global subprog exception path\n"); + return PTR_ERR(branch); + } + return process_bpf_exit_full(env, NULL, true); + } + /* continue with next insn after call */ return 0; } @@ -11782,7 +11795,7 @@ static bool is_async_callback_calling_kfunc(u32 btf_id) is_task_work_add_kfunc(btf_id); } -static bool is_bpf_throw_kfunc(struct bpf_insn *insn) +bool bpf_is_throw_kfunc(struct bpf_insn *insn) { return bpf_pseudo_kfunc_call(insn) && insn->off == 0 && insn->imm == special_kfunc_list[KF_bpf_throw]; @@ -12972,8 +12985,6 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_ca } static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name); -static int process_bpf_exit_full(struct bpf_verifier_env *env, - bool *do_print_state, bool exception_exit); static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, int *insn_idx_p) @@ -13354,7 +13365,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie]) env->prog->call_session_cookie = true; - if (is_bpf_throw_kfunc(insn)) + if (bpf_is_throw_kfunc(insn)) return process_bpf_exit_full(env, NULL, true); return 0; -- cgit v1.2.3 From 515e3996a4c26e7f955c13b3b19522a2c8642af9 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sun, 17 May 2026 07:43:16 -1000 Subject: sched_ext: Fix deadlock between scx_root_disable() and concurrent forks scx_root_disable() enters SCX_DISABLING before it grabs scx_enable_mutex to clear __scx_switched_all and scx_switching_all. task_should_scx() short-circuits on DISABLING, so forks in that window land on fair while next_active_class() still skips fair - the new tasks stall. This can deadlock the disable path itself: scx_alloc_and_add_sched() runs under scx_enable_mutex and creates a helper kthread; if that new kthread is one of the stalled fair tasks, the mutex holder waits forever and scx_root_disable() can never make progress. Only sub-sched support exposes this, since sub-sched enables are the only path where scx_alloc_and_add_sched() can race the root's disable. Move the DISABLING check after @scx_switching_all. @scx_switching_all serves as a proxy for __scx_switched_all, so while it's set, forks keep going to scx. Once cleared, DISABLING applies normally. v2: Reword in-source comment and description. (Andrea) Fixes: 337ec00b1d9c ("sched_ext: Implement cgroup sub-sched enabling and disabling") Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext.c | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index a6d0a93d8174..547ca398f646 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -4946,10 +4946,30 @@ static const struct kset_uevent_ops scx_uevent_ops = { */ bool task_should_scx(int policy) { - if (!scx_enabled() || unlikely(scx_enable_state() == SCX_DISABLING)) + /* if disabled, nothing should be on it */ + if (!scx_enabled()) return false; + + /* scx is taking over all SCHED_OTHER and SCHED_EXT tasks */ if (READ_ONCE(scx_switching_all)) return true; + + /* + * scx is tearing down - keep new SCHED_EXT tasks out. + * + * Must come after scx_switching_all test, which serves as a proxy + * for __scx_switched_all. While __scx_switched_all is set, we must + * return true via the branch above: a fork routed to fair would + * stall because next_active_class() skips fair. + * + * This can develop into a deadlock - scx holds scx_enable_mutex across + * kthread_create() in scx_alloc_and_add_sched(); if the new kthread is + * the stalled task, the disable path can never grab the mutex to clear + * scx_switching_all. + */ + if (unlikely(scx_enable_state() == SCX_DISABLING)) + return false; + return policy == SCHED_EXT; } -- cgit v1.2.3 From 18a37465b0ab5237a1d0ebf93a2a3b6a2da540b3 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sun, 17 May 2026 08:07:02 -0700 Subject: bpf,x86: Fix exception unwinding with outgoing stack arguments When a main program with exception_boundary has outgoing stack arguments (e.g. from calling subprogs with >5 args), bpf_throw() fails to correctly restore callee-saved registers, causing a kernel crash. The x86 JIT allocates the outgoing stack arg area below the callee-saved registers via 'sub rsp, outgoing_rsp' in the prologue. When bpf_throw() unwinds, it captures the main program's sp (which includes this outgoing area) and passes it to the exception callback. The callback gets rsp and rbp, followed by pop_callee_regs, but rsp points into the outgoing arg area rather than the callee-saved registers, so the pops restore garbage values. Returning to the kernel with corrupted callee-saved registers causes a crash. Fix this by adjusting the sp (adding stack_arg_sp_adjust) passed to the exception callback, so it points to the bottom of the callee-saved registers instead of the outgoing arg area. When stack_arg_sp_adjust is 0 (the common case), this is a no-op. Fixes: 324c3ca6eed6 ("bpf,x86: Implement JIT support for stack arguments") Acked-by: Kumar Kartikeya Dwivedi Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260517150702.288031-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/fixups.c | 1 + kernel/bpf/helpers.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 2cec4e8cd4a0..52535671cb9a 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -1265,6 +1265,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) prog->aux->real_func_cnt = env->subprog_cnt; prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func; prog->aux->exception_boundary = func[0]->aux->exception_boundary; + prog->aux->stack_arg_sp_adjust = func[0]->aux->stack_arg_sp_adjust; bpf_prog_jit_attempt_done(prog); return 0; out_free: diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index baa12b24bb64..07de26e7314c 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -3301,7 +3301,7 @@ __bpf_kfunc void bpf_throw(u64 cookie) * which skips compiler generated instrumentation to do the same. */ kasan_unpoison_task_stack_below((void *)(long)ctx.sp); - ctx.aux->bpf_exception_cb(cookie, ctx.sp, ctx.bp, 0, 0); + ctx.aux->bpf_exception_cb(cookie, ctx.sp + ctx.aux->stack_arg_sp_adjust, ctx.bp, 0, 0); WARN(1, "A call to BPF exception callback should never return\n"); } -- cgit v1.2.3 From af0c3f05866237f7592219bfe05387bc3bfc99b5 Mon Sep 17 00:00:00 2001 From: Jianpeng Chang Date: Wed, 13 May 2026 15:22:09 +0800 Subject: dma-mapping: move dma_map_resource() sanity check into debug code dma_map_resource() uses pfn_valid() to ensure the range is not RAM. However, pfn_valid() only checks for availability of the memory map for a PFN but it does not ensure that the PFN is actually backed by RAM. On ARM64 with SPARSEMEM (128MB section granularity), MMIO addresses that share a section with RAM will falsely trigger the WARN_ON_ONCE and cause dma_map_resource() to return DMA_MAPPING_ERROR. This causes a WARNING on Raspberry Pi 4 during spi_bcm2835 probe because the SPI FIFO register (0xfe204004) falls in the same sparsemem section as the end of RAM (0xf8000000-0xfbffffff), both in section 31 (0xf8000000-0xffffffff). Move the sanity check from dma_map_resource() into debug_dma_map_phys() and replace the unreliable pfn_valid() with pfn_valid() && !PageReserved(), which correctly identifies actual usable RAM without false positives for MMIO regions that happen to have struct pages. Since dma_map_resource() is dma_map_phys(DMA_ATTR_MMIO), the check applies equally to both APIs. Any non-reserved page represents kernel memory to a sufficient degree that using DMA_ATTR_MMIO on it is almost certainly wrong and risks breaking coherency on non-coherent platforms. ZONE_DEVICE pages used for PCI P2P DMA (MEMORY_DEVICE_PCI_P2PDMA) have PageReserved set, so they will not trigger a false positive. The check no longer blocks the mapping and uses err_printk() to integrate with dma-debug filtering. Fixes: f7326196a781 ("dma-mapping: export new dma_*map_phys() interface") Reviewed-by: Robin Murphy Signed-off-by: Jianpeng Chang Reviewed-by: Leon Romanovsky Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/20260513072209.1486986-1-jianpeng.chang.cn@windriver.com --- kernel/dma/debug.c | 9 ++++++++- kernel/dma/mapping.c | 4 ---- 2 files changed, 8 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c index 1a725edbbbf6..3248f8b4d096 100644 --- a/kernel/dma/debug.c +++ b/kernel/dma/debug.c @@ -1251,7 +1251,14 @@ void debug_dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, entry->direction = direction; entry->map_err_type = MAP_ERR_NOT_CHECKED; - if (!(attrs & DMA_ATTR_MMIO)) { + if (attrs & DMA_ATTR_MMIO) { + unsigned long pfn = PHYS_PFN(phys); + + if (pfn_valid(pfn) && !PageReserved(pfn_to_page(pfn))) + err_printk(dev, entry, + "dma_map_resource called for RAM address %pa\n", + &phys); + } else { check_for_stack(dev, phys); if (!PhysHighMem(phys)) diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index 23ed8eb9233e..e6b07f160d20 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -365,10 +365,6 @@ EXPORT_SYMBOL(dma_unmap_sg_attrs); dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr, size_t size, enum dma_data_direction dir, unsigned long attrs) { - if (IS_ENABLED(CONFIG_DMA_API_DEBUG) && - WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr)))) - return DMA_MAPPING_ERROR; - return dma_map_phys(dev, phys_addr, size, dir, attrs | DMA_ATTR_MMIO); } EXPORT_SYMBOL(dma_map_resource); -- cgit v1.2.3 From 0d25e3865841ea5edfedb5af42bf15cef075192e Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Sat, 16 May 2026 13:25:37 +0800 Subject: cgroup/rdma: Drop unnecessary READ_ONCE() on event counters All accesses to the event counters are serialized by rdmacg_mutex, making the READ_ONCE() annotations unnecessary. Remove them. Signed-off-by: Tao Cui Signed-off-by: Tejun Heo --- kernel/cgroup/rdma.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/cgroup/rdma.c b/kernel/cgroup/rdma.c index 7c238a9d64d4..5e82a03b3270 100644 --- a/kernel/cgroup/rdma.c +++ b/kernel/cgroup/rdma.c @@ -145,10 +145,10 @@ static bool rpool_has_persistent_state(struct rdmacg_resource_pool *rpool) */ for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { if (rpool->resources[i].peak || - READ_ONCE(rpool->events_max[i]) || - READ_ONCE(rpool->events_local_max[i]) || - READ_ONCE(rpool->events_alloc_fail[i]) || - READ_ONCE(rpool->events_local_alloc_fail[i])) + rpool->events_max[i] || + rpool->events_local_max[i] || + rpool->events_alloc_fail[i] || + rpool->events_local_alloc_fail[i]) return true; } return false; @@ -654,9 +654,9 @@ static int rdmacg_events_show(struct seq_file *sf, void *v) for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { seq_printf(sf, "%s.max=%llu %s.alloc_fail=%llu", rdmacg_resource_names[i], - rpool ? READ_ONCE(rpool->events_max[i]) : 0ULL, + rpool ? rpool->events_max[i] : 0ULL, rdmacg_resource_names[i], - rpool ? READ_ONCE(rpool->events_alloc_fail[i]) : 0ULL); + rpool ? rpool->events_alloc_fail[i] : 0ULL); if (i < RDMACG_RESOURCE_MAX - 1) seq_putc(sf, ' '); } @@ -683,9 +683,9 @@ static int rdmacg_events_local_show(struct seq_file *sf, void *v) for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { seq_printf(sf, "%s.max=%llu %s.alloc_fail=%llu", rdmacg_resource_names[i], - rpool ? READ_ONCE(rpool->events_local_max[i]) : 0ULL, + rpool ? rpool->events_local_max[i] : 0ULL, rdmacg_resource_names[i], - rpool ? READ_ONCE(rpool->events_local_alloc_fail[i]) : 0ULL); + rpool ? rpool->events_local_alloc_fail[i] : 0ULL); if (i < RDMACG_RESOURCE_MAX - 1) seq_putc(sf, ' '); } -- cgit v1.2.3 From 593889c401426004bd0ea0f6d4fcece728b03420 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 11 May 2026 19:54:41 +0200 Subject: srcu: Don't queue workqueue handlers to never-online CPUs While an srcu_struct structure is in the midst of switching from CPU-0 to all-CPUs state, it can attempt to invoke callbacks for CPUs that have never been online. Worse yet, it can attempt in invoke callbacks for CPUs that never will be online, even including imaginary CPUs not in cpu_possible_mask. This can cause hangs on s390, which is not set up to deal with workqueue handlers being scheduled on such CPUs. This commit therefore causes Tree SRCU to refrain from queueing workqueue handlers on CPUs that have not yet (and might never) come online. Because callbacks are not invoked on CPUs that have not been online, it is an error to invoke call_srcu(), synchronize_srcu(), or synchronize_srcu_expedited() on a CPU that is not yet fully online. However, it turns out to be less code to redirect the callbacks from too-early invocations of call_srcu() than to warn about such invocations. This commit therefore also redirects callbacks queued on not-yet-fully-online CPUs to the boot CPU. Reported-by: Vasily Gorbik Fixes: 61bbcfb50514 ("srcu: Push srcu_node allocation to GP when non-preemptible") Signed-off-by: Paul E. McKenney Tested-by: Vasily Gorbik Tested-by: Samir Reviewed-by: Shrikanth Hegde Cc: Tejun Heo Signed-off-by: Uladzislau Rezki (Sony) Signed-off-by: Boqun Feng --- kernel/rcu/srcutree.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 0d01cd8c4b4a..7c2f7cc131f7 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -897,11 +897,9 @@ static void srcu_schedule_cbs_snp(struct srcu_struct *ssp, struct srcu_node *snp { int cpu; - for (cpu = snp->grplo; cpu <= snp->grphi; cpu++) { - if (!(mask & (1UL << (cpu - snp->grplo)))) - continue; - srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, cpu), delay); - } + for (cpu = snp->grplo; cpu <= snp->grphi; cpu++) + if ((mask & (1UL << (cpu - snp->grplo))) && rcu_cpu_beenfullyonline(cpu)) + srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, cpu), delay); } /* @@ -1322,7 +1320,9 @@ static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp, */ idx = __srcu_read_lock_nmisafe(ssp); ss_state = smp_load_acquire(&ssp->srcu_sup->srcu_size_state); - if (ss_state < SRCU_SIZE_WAIT_CALL) + // If !rcu_cpu_beenfullyonline(), interrupts are still disabled, + // so no migration is possible in either direction from this CPU. + if (ss_state < SRCU_SIZE_WAIT_CALL || !rcu_cpu_beenfullyonline(raw_smp_processor_id())) sdp = per_cpu_ptr(ssp->sda, get_boot_cpu_id()); else sdp = raw_cpu_ptr(ssp->sda); -- cgit v1.2.3 From 8817005efbdfdf5d4e4814cb5dc52b53d12917d7 Mon Sep 17 00:00:00 2001 From: Qing Ming Date: Sat, 16 May 2026 15:08:49 +0800 Subject: cgroup/rstat: validate cpu before css_rstat_cpu() access css_rstat_updated() is exposed as a BPF kfunc and accepts a caller-provided cpu argument. The function uses cpu for per-cpu rstat lookups without checking whether it refers to a valid possible CPU. A BPF iter/cgroup program with CAP_BPF and CAP_PERFMON can pass an invalid cpu value. On an unfixed UBSCAN_BOUNDS test kernel, cpu == 0x7fffffff triggers: UBSAN: array-index-out-of-bounds in kernel/cgroup/rstat.c:31:9 index 2147483647 is out of range for type 'long unsigned int [64]' Call Trace: css_rstat_updated bpf_iter_run_prog cgroup_iter_seq_show bpf_seq_read Add cpu validation to the BPF-facing css_rstat_updated() kfunc and move the common implementation to __css_rstat_updated() for in-kernel callers. Fixes: a319185be9f5 ("cgroup: bpf: enable bpf programs to integrate with rstat") Signed-off-by: Qing Ming Signed-off-by: Tejun Heo --- kernel/cgroup/rstat.c | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/cgroup/rstat.c b/kernel/cgroup/rstat.c index 150e5871e66f..ed60ba119c68 100644 --- a/kernel/cgroup/rstat.c +++ b/kernel/cgroup/rstat.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only #include "cgroup-internal.h" +#include #include #include @@ -53,7 +54,7 @@ static inline struct llist_head *ss_lhead_cpu(struct cgroup_subsys *ss, int cpu) } /** - * css_rstat_updated - keep track of updated rstat_cpu + * __css_rstat_updated - keep track of updated rstat_cpu * @css: target cgroup subsystem state * @cpu: cpu on which rstat_cpu was updated * @@ -63,20 +64,17 @@ static inline struct llist_head *ss_lhead_cpu(struct cgroup_subsys *ss, int cpu) * * NOTE: if the user needs the guarantee that the updater either add itself in * the lockless list or the concurrent flusher flushes its updated stats, a - * memory barrier is needed before the call to css_rstat_updated() i.e. a + * memory barrier is needed before the call to __css_rstat_updated() i.e. a * barrier after updating the per-cpu stats and before calling - * css_rstat_updated(). + * __css_rstat_updated(). */ -__bpf_kfunc void css_rstat_updated(struct cgroup_subsys_state *css, int cpu) +void __css_rstat_updated(struct cgroup_subsys_state *css, int cpu) { struct llist_head *lhead; struct css_rstat_cpu *rstatc; struct llist_node *self; - /* - * Since bpf programs can call this function, prevent access to - * uninitialized rstat pointers. - */ + /* Prevent access to uninitialized rstat pointers. */ if (!css_uses_rstat(css)) return; @@ -125,6 +123,18 @@ __bpf_kfunc void css_rstat_updated(struct cgroup_subsys_state *css, int cpu) llist_add(&rstatc->lnode, lhead); } +/* + * BPF-facing wrapper for __css_rstat_updated(). Validate the caller-provided + * CPU before passing it to the internal rstat updater. + */ +__bpf_kfunc void css_rstat_updated(struct cgroup_subsys_state *css, int cpu) +{ + if (unlikely(cpu < 0 || cpu >= nr_cpu_ids || !cpu_possible(cpu))) + return; + + __css_rstat_updated(css, cpu); +} + static void __css_process_update_tree(struct cgroup_subsys_state *css, int cpu) { /* put @css and all ancestors on the corresponding updated lists */ @@ -170,7 +180,7 @@ static void css_process_update_tree(struct cgroup_subsys *ss, int cpu) * flusher flush the stats updated by the updater who have * observed that they are already on the list. The * corresponding barrier pair for this one should be before - * css_rstat_updated() by the user. + * __css_rstat_updated() by the user. * * For now, there aren't any such user, so not adding the * barrier here but if such a use-case arise, please add @@ -614,7 +624,7 @@ static void cgroup_base_stat_cputime_account_end(struct cgroup *cgrp, unsigned long flags) { u64_stats_update_end_irqrestore(&rstatbc->bsync, flags); - css_rstat_updated(&cgrp->self, smp_processor_id()); + __css_rstat_updated(&cgrp->self, smp_processor_id()); put_cpu_ptr(rstatbc); } -- cgit v1.2.3 From a2b4cf39d9d333bfeb9262dbaafe3d24d405a5c0 Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Wed, 13 May 2026 13:39:12 -0700 Subject: sched/cache: Allow only 1 thread of the process to calculate the LLC occupancy Scanning online CPUs to calculate the occupancy might be time-consuming. Only allow 1 thread of the process to scan the CPUs at the same time, which is similar to what NUMA balance does in task_numa_work(). Signed-off-by: Jianyong Wu Signed-off-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/5672b52e588b855b01e5a1a17822f7c6c7237a3d.1778703694.git.tim.c.chen@linux.intel.com --- kernel/sched/fair.c | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 5f22e5a097cf..a759ea669d74 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1451,6 +1451,7 @@ void mm_init_sched(struct mm_struct *mm, raw_spin_lock_init(&mm->sc_stat.lock); mm->sc_stat.epoch = epoch; mm->sc_stat.cpu = -1; + mm->sc_stat.next_scan = jiffies; /* * The update to mm->sc_stat should not be reordered @@ -1661,6 +1662,7 @@ out: static void task_cache_work(struct callback_head *work) { + unsigned long next_scan, now = jiffies; struct task_struct *p = current; struct mm_struct *mm = p->mm; unsigned long m_a_occ = 0; @@ -1675,6 +1677,15 @@ static void task_cache_work(struct callback_head *work) if (p->flags & PF_EXITING) return; + next_scan = READ_ONCE(mm->sc_stat.next_scan); + if (time_before(now, next_scan)) + return; + + /* only 1 thread is allowed to scan */ + if (!try_cmpxchg(&mm->sc_stat.next_scan, &next_scan, + now + EPOCH_PERIOD)) + return; + if (!zalloc_cpumask_var(&cpus, GFP_KERNEL)) return; -- cgit v1.2.3 From deee5e27d5b608323c04dc99979e55f944016a13 Mon Sep 17 00:00:00 2001 From: Chen Yu Date: Wed, 13 May 2026 13:39:13 -0700 Subject: sched/cache: Disable cache aware scheduling for processes with high thread counts A performance regression was observed by Prateek when running hackbench with many threads per process (high fd count). To avoid this, processes with a large number of active threads are excluded from cache-aware scheduling. With sched_cache enabled, record the number of active threads in each process during the periodic task_cache_work(). While iterating over CPUs, if the currently running task belongs to the same process as the task that launched task_cache_work(), increment the active thread count. If the number of active threads within the process exceeds the number of Cores (divided by the SMT number) in the LLC, do not enable cache-aware scheduling. However, on systems with a smaller number of CPUs within 1 LLC, like Power10/Power11 with SMT4 and an LLC size of 4, this check effectively disables cache-aware scheduling for any process. One possible solution suggested by Peter is to use an LLC-mask instead of a single LLC value for preference. Once there are a 'few' LLCs as preference, this constraint becomes a little easier. It could be an enhancement in the future. For users who wish to perform task aggregation regardless, a debugfs knob is provided for tuning in a subsequent change. Suggested-by: K Prateek Nayak Suggested-by: Aaron Lu Signed-off-by: Chen Yu Co-developed-by: Tim Chen Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Tested-by: Tingyin Duan Link: https://patch.msgid.link/d076cd21a8e6c6341d1e2d927e118db770ebb650.1778703694.git.tim.c.chen@linux.intel.com --- kernel/sched/fair.c | 48 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index a759ea669d74..808f614fc2d2 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1384,6 +1384,12 @@ static int llc_id(int cpu) return per_cpu(sd_llc_id, cpu); } +static bool invalid_llc_nr(struct mm_struct *mm, int cpu) +{ + return !fits_capacity((mm->sc_stat.nr_running_avg * cpu_smt_num_threads), + per_cpu(sd_llc_size, cpu)); +} + static void account_llc_enqueue(struct rq *rq, struct task_struct *p) { struct sched_domain *sd; @@ -1452,7 +1458,7 @@ void mm_init_sched(struct mm_struct *mm, mm->sc_stat.epoch = epoch; mm->sc_stat.cpu = -1; mm->sc_stat.next_scan = jiffies; - + mm->sc_stat.nr_running_avg = 0; /* * The update to mm->sc_stat should not be reordered * before initialization to mm's other fields, in case @@ -1574,7 +1580,8 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) * If this process hasn't hit task_cache_work() for a while invalidate * its preferred state. */ - if (epoch - READ_ONCE(mm->sc_stat.epoch) > EPOCH_LLC_AFFINITY_TIMEOUT) { + if (epoch - READ_ONCE(mm->sc_stat.epoch) > EPOCH_LLC_AFFINITY_TIMEOUT || + invalid_llc_nr(mm, cpu_of(rq))) { if (mm->sc_stat.cpu != -1) mm->sc_stat.cpu = -1; } @@ -1660,14 +1667,32 @@ out: cpumask_copy(cpus, cpu_online_mask); } +static inline void update_avg_scale(u64 *avg, u64 sample) +{ + int factor = per_cpu(sd_llc_size, raw_smp_processor_id()); + s64 diff = sample - *avg; + u32 divisor; + + /* + * Scale the divisor based on the number of CPUs contained + * in the LLC. This scaling ensures smaller LLC domains use + * a smaller divisor to achieve more precise sensitivity to + * changes in nr_running, while larger LLC domains are capped + * at a maximum divisor of 8 which is the default smoothing + * factor of EWMA in update_avg(). + */ + divisor = clamp_t(u32, (factor >> 2), 2, 8); + *avg += div64_s64(diff, divisor); +} + static void task_cache_work(struct callback_head *work) { unsigned long next_scan, now = jiffies; - struct task_struct *p = current; + struct task_struct *p = current, *cur; + int cpu, m_a_cpu = -1, nr_running = 0; + unsigned long curr_m_a_occ = 0; struct mm_struct *mm = p->mm; unsigned long m_a_occ = 0; - unsigned long curr_m_a_occ = 0; - int cpu, m_a_cpu = -1; cpumask_var_t cpus; WARN_ON_ONCE(work != &p->cache_work); @@ -1711,6 +1736,11 @@ static void task_cache_work(struct callback_head *work) m_occ = occ; m_cpu = i; } + + cur = rcu_dereference_all(cpu_rq(i)->curr); + if (cur && !(cur->flags & (PF_EXITING | PF_KTHREAD)) && + cur->mm == mm) + nr_running++; } /* @@ -1754,6 +1784,7 @@ static void task_cache_work(struct callback_head *work) mm->sc_stat.cpu = m_a_cpu; } + update_avg_scale(&mm->sc_stat.nr_running_avg, nr_running); free_cpumask_var(cpus); } @@ -10294,6 +10325,13 @@ static enum llc_mig can_migrate_llc_task(int src_cpu, int dst_cpu, if (cpu < 0 || cpus_share_cache(src_cpu, dst_cpu)) return mig_unrestricted; + /* skip cache aware load balance for too many threads */ + if (invalid_llc_nr(mm, dst_cpu)) { + if (mm->sc_stat.cpu != -1) + mm->sc_stat.cpu = -1; + return mig_unrestricted; + } + if (cpus_share_cache(dst_cpu, cpu)) to_pref = true; else if (cpus_share_cache(src_cpu, cpu)) -- cgit v1.2.3 From 7b34bb1ca324451c84c0a69136ce92e7928cf72b Mon Sep 17 00:00:00 2001 From: Chen Yu Date: Wed, 13 May 2026 13:39:14 -0700 Subject: sched/cache: Skip cache-aware scheduling for single-threaded processes For a single thread, the current wakeup path tends to place it on the same LLC where it was previously running with cache-hot data. There is no need to enable cache-aware scheduling for single-threaded processes for the following reasons: 1. Cache-aware scheduling primarily benefits multi-threaded processes where threads share data. Single-threaded processes typically have no inter-thread data sharing and thus gain little. 2. Enabling it incurs the additional overhead of tracking the thread's residency in the LLCs. 3. Bypassing single-threaded processes avoids excessive concentration of such tasks on a single LLC. Nevertheless, this check can be omitted if users explicitly provide hints for such single-threaded workloads where different processes have shared memory, e.g., via prctl() or other interfaces to be added in the future. Signed-off-by: Chen Yu Co-developed-by: Tim Chen Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Tested-by: Tingyin Duan Link: https://patch.msgid.link/8a59a13aa58fdb48e410ecb2aabd97fe3ea5d256.1778703694.git.tim.c.chen@linux.intel.com --- kernel/sched/fair.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 808f614fc2d2..df21366ba1ca 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1384,8 +1384,12 @@ static int llc_id(int cpu) return per_cpu(sd_llc_id, cpu); } -static bool invalid_llc_nr(struct mm_struct *mm, int cpu) +static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p, + int cpu) { + if (get_nr_threads(p) <= 1) + return true; + return !fits_capacity((mm->sc_stat.nr_running_avg * cpu_smt_num_threads), per_cpu(sd_llc_size, cpu)); } @@ -1581,7 +1585,7 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) * its preferred state. */ if (epoch - READ_ONCE(mm->sc_stat.epoch) > EPOCH_LLC_AFFINITY_TIMEOUT || - invalid_llc_nr(mm, cpu_of(rq))) { + invalid_llc_nr(mm, p, cpu_of(rq))) { if (mm->sc_stat.cpu != -1) mm->sc_stat.cpu = -1; } @@ -1687,9 +1691,9 @@ static inline void update_avg_scale(u64 *avg, u64 sample) static void task_cache_work(struct callback_head *work) { + int cpu, m_a_cpu = -1, nr_running = 0, curr_cpu; unsigned long next_scan, now = jiffies; struct task_struct *p = current, *cur; - int cpu, m_a_cpu = -1, nr_running = 0; unsigned long curr_m_a_occ = 0; struct mm_struct *mm = p->mm; unsigned long m_a_occ = 0; @@ -1711,6 +1715,14 @@ static void task_cache_work(struct callback_head *work) now + EPOCH_PERIOD)) return; + curr_cpu = task_cpu(p); + if (invalid_llc_nr(mm, p, curr_cpu)) { + if (mm->sc_stat.cpu != -1) + mm->sc_stat.cpu = -1; + + return; + } + if (!zalloc_cpumask_var(&cpus, GFP_KERNEL)) return; @@ -10326,7 +10338,7 @@ static enum llc_mig can_migrate_llc_task(int src_cpu, int dst_cpu, return mig_unrestricted; /* skip cache aware load balance for too many threads */ - if (invalid_llc_nr(mm, dst_cpu)) { + if (invalid_llc_nr(mm, p, dst_cpu)) { if (mm->sc_stat.cpu != -1) mm->sc_stat.cpu = -1; return mig_unrestricted; -- cgit v1.2.3 From 7030513a08776b2ca70fccd5dfddf7bb5c5c88ba Mon Sep 17 00:00:00 2001 From: Chen Yu Date: Wed, 13 May 2026 13:39:15 -0700 Subject: sched/cache: Calculate the LLC size and store it in sched_domain Cache aware scheduling needs to know the LLC size that a process can use, so as to avoid memory-intensive tasks from being over-aggregated on a single LLC. Introduce a preparation patch to add get_effective_llc_bytes() to get the LLC size that a CPU can use. The function can be further enhanced by subtracting the LLC cache ways reserved by resctrl (CAT in Intel RDT, etc). Suggested-by: Peter Zijlstra (Intel) Signed-off-by: Chen Yu Co-developed-by: Tim Chen Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Tested-by: Tingyin Duan Link: https://patch.msgid.link/37afee09ff608034da0ce149e72d33b6f4698edf.1778703694.git.tim.c.chen@linux.intel.com --- kernel/sched/topology.c | 98 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 95 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 9fc99346ef4f..7248a7279abe 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -776,9 +776,11 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) /* move buffer to parent as child is being destroyed */ sd->llc_counts = tmp->llc_counts; sd->llc_max = tmp->llc_max; + sd->llc_bytes = tmp->llc_bytes; /* make sure destroy_sched_domain() does not free it */ tmp->llc_counts = NULL; tmp->llc_max = 0; + tmp->llc_bytes = 0; #endif /* * sched groups hold the flags of the child sched @@ -831,10 +833,42 @@ DEFINE_STATIC_KEY_FALSE(sched_cache_active); /* user wants cache aware scheduling [0 or 1] */ int sysctl_sched_cache_user = 1; +/* + * Get the effective LLC size in bytes that @cpu's bottom sched_domain + * can use. A CPU within a cpuset partition can only use a proportion + * of the physical LLC, scaled by the ratio of the partition's span + * weight to the hardware LLC sharing weight. @sd should be the + * topmost domain with SD_SHARE_LLC. + * + * Returns 0 if cacheinfo is not yet populated. This happens during + * early boot when build_sched_domains() runs before the generic + * cacheinfo framework has been initialized (cacheinfo_cpu_online() + * is a device_initcall cpuhp callback). In that case, + * cacheinfo_cpu_online() will later call sched_update_llc_bytes() + * to fill in the bottom domain's llc_bytes once the cache attributes + * are available. + */ +static unsigned long get_effective_llc_bytes(int cpu, + struct sched_domain *sd) +{ + struct cacheinfo *ci; + unsigned int hw_weight; + + ci = get_cpu_cacheinfo_llc(cpu); + if (!ci) + return 0; + + hw_weight = cpumask_weight(&ci->shared_cpu_map); + if (!hw_weight) + return 0; + + return div_u64((u64)ci->size * sd->span_weight, hw_weight); +} + static bool alloc_sd_llc(const struct cpumask *cpu_map, struct s_data *d) { - struct sched_domain *sd; + struct sched_domain *sd, *top_llc, *parent; unsigned int *p; int i; @@ -848,8 +882,24 @@ static bool alloc_sd_llc(const struct cpumask *cpu_map, if (!p) goto err; - sd->llc_max = max_lid + 1; - sd->llc_counts = p; + top_llc = sd; + /* + * Find the topmost SD_SHARE_LLC domain. + * Not yet attached to the CPU, so per_cpu(sd_llc, i) + * can not be used. + */ + while ((parent = rcu_dereference_protected(top_llc->parent, true)) && + (parent->flags & SD_SHARE_LLC)) + top_llc = parent; + + if (top_llc->flags & SD_SHARE_LLC) { + sd->llc_max = max_lid + 1; + sd->llc_counts = p; + sd->llc_bytes = get_effective_llc_bytes(i, top_llc); + } else { + /* avoid memory leak */ + kfree(p); + } } return true; @@ -860,6 +910,7 @@ err: kfree(sd->llc_counts); sd->llc_counts = NULL; sd->llc_max = 0; + sd->llc_bytes = 0; } } @@ -919,6 +970,47 @@ void sched_cache_active_set_unlocked(void) { return sched_cache_active_set(false); } + +/* + * Update the bottom sched_domain's llc_bytes for @cpu and all its + * LLC siblings. Called from cacheinfo_cpu_online() or + * cacheinfo_cpu_pre_down() with cpu hotplug lock held. + * + * Note: get_effective_llc_bytes() returns 0 on PowerPC. + * thus cache aware scheduling is disabled on PowerPC for + * now. PowerPC does not use the generic cacheinfo framework -- + * it has its own cacheinfo with a separate struct cache hierarchy + * and does not populates the per-CPU struct cpu_cacheinfo array + * that get_cpu_cacheinfo_llc() reads. + */ +void sched_update_llc_bytes(unsigned int cpu) +{ + struct sched_domain *sd, *sdp; + unsigned int i; + + sched_domains_mutex_lock(); + + sdp = rcu_dereference_sched_domain(per_cpu(sd_llc, cpu)); + if (!sdp) + goto unlock; + + /* + * ci->shared_cpu_map is built incrementally as CPUs come + * online, so the first CPU in an LLC initially sees + * hw_weight == 1 and computes an inflated llc_bytes in + * get_effective_llc_bytes(). Re-evaluating every LLC + * sibling on each online event corrects this once the full + * shared_cpu_map is known. + */ + for_each_cpu(i, sched_domain_span(sdp)) { + sd = rcu_dereference_sched_domain(cpu_rq(i)->sd); + if (sd) + sd->llc_bytes = get_effective_llc_bytes(i, sdp); + } + +unlock: + sched_domains_mutex_unlock(); +} #else static bool alloc_sd_llc(const struct cpumask *cpu_map, struct s_data *d) -- cgit v1.2.3 From 808915f982c2a52f5d148510ecfab52284de67cf Mon Sep 17 00:00:00 2001 From: Chen Yu Date: Wed, 13 May 2026 13:39:16 -0700 Subject: sched/cache: Avoid cache-aware scheduling for memory-heavy processes Prateek and Tingyin reported that memory-intensive workloads (such as stream) can saturate memory bandwidth and caches on the preferred LLC when sched_cache aggregates too many threads. To mitigate this, estimate a process's memory footprint by comparing its NUMA balancing fault statistics to the size of the LLC. If the footprint exceeds the LLC size, skip cache-aware scheduling. Note that footprint is only an approximation of the memory footprint, since the kernel lacks suitable metrics to estimate the real working set. If a user-provided hint is available in the future, it would be more accurate. A later patch will allow users to provide a hint to adjust this threshold. Suggested-by: K Prateek Nayak Suggested-by: Vern Hao Signed-off-by: Chen Yu Co-developed-by: Tim Chen Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Tested-by: Tingyin Duan Link: https://patch.msgid.link/95cf64a385bcc12f18dcebe9d59e8d3ba8bb318f.1778703694.git.tim.c.chen@linux.intel.com --- kernel/exit.c | 29 +++++++++++++++++++++++++ kernel/sched/fair.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 88 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/exit.c b/kernel/exit.c index ede3117fa7d4..77275c26a2a1 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -543,6 +543,32 @@ void mm_update_next_owner(struct mm_struct *mm) } #endif /* CONFIG_MEMCG */ +#if defined(CONFIG_SCHED_CACHE) && defined(CONFIG_NUMA_BALANCING) +/* + * Subtract the memory footprint of the current task from + * mm. + */ +static void exit_mm_sched_cache(struct mm_struct *mm) +{ + unsigned long fp, sub; + + if (!current->total_numa_faults) + return; + /* + * No lock protection due to performance considerations. + * Make sure mm->sc_stat.footprint does not become + * negative. + */ + fp = READ_ONCE(mm->sc_stat.footprint); + sub = min(fp, current->total_numa_faults); + WRITE_ONCE(mm->sc_stat.footprint, fp - sub); +} +#else +static inline void exit_mm_sched_cache(struct mm_struct *mm) +{ +} +#endif /* CONFIG_SCHED_CACHE CONFIG_NUMA_BALANCING */ + /* * Turn us into a lazy TLB process if we * aren't already.. @@ -554,6 +580,9 @@ static void exit_mm(void) exit_mm_release(current, mm); if (!mm) return; + + exit_mm_sched_cache(mm); + mmap_read_lock(mm); mmgrab_lazy_tlb(mm); BUG_ON(mm != current->active_mm); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index df21366ba1ca..a10116ffe0d1 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1384,6 +1384,32 @@ static int llc_id(int cpu) return per_cpu(sd_llc_id, cpu); } +static bool exceed_llc_capacity(struct mm_struct *mm, int cpu) +{ +#ifdef CONFIG_NUMA_BALANCING + unsigned long llc, footprint; + struct sched_domain *sd; + + guard(rcu)(); + + sd = rcu_dereference_sched_domain(cpu_rq(cpu)->sd); + if (!sd) + return true; + + if (static_branch_likely(&sched_numa_balancing)) { + /* + * TBD: RDT exclusive LLC ways reserved should be + * excluded. + */ + llc = sd->llc_bytes; + footprint = READ_ONCE(mm->sc_stat.footprint); + + return (llc < (footprint * PAGE_SIZE)); + } +#endif + return false; +} + static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p, int cpu) { @@ -1463,6 +1489,7 @@ void mm_init_sched(struct mm_struct *mm, mm->sc_stat.cpu = -1; mm->sc_stat.next_scan = jiffies; mm->sc_stat.nr_running_avg = 0; + mm->sc_stat.footprint = 0; /* * The update to mm->sc_stat should not be reordered * before initialization to mm's other fields, in case @@ -1585,7 +1612,8 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) * its preferred state. */ if (epoch - READ_ONCE(mm->sc_stat.epoch) > EPOCH_LLC_AFFINITY_TIMEOUT || - invalid_llc_nr(mm, p, cpu_of(rq))) { + invalid_llc_nr(mm, p, cpu_of(rq)) || + exceed_llc_capacity(mm, cpu_of(rq))) { if (mm->sc_stat.cpu != -1) mm->sc_stat.cpu = -1; } @@ -1716,7 +1744,8 @@ static void task_cache_work(struct callback_head *work) return; curr_cpu = task_cpu(p); - if (invalid_llc_nr(mm, p, curr_cpu)) { + if (invalid_llc_nr(mm, p, curr_cpu) || + exceed_llc_capacity(mm, curr_cpu)) { if (mm->sc_stat.cpu != -1) mm->sc_stat.cpu = -1; @@ -3515,6 +3544,7 @@ static void task_numa_placement(struct task_struct *p) unsigned long total_faults; u64 runtime, period; spinlock_t *group_lock = NULL; + long __maybe_unused new_fp; struct numa_group *ng; /* @@ -3589,6 +3619,31 @@ static void task_numa_placement(struct task_struct *p) ng->total_faults += diff; group_faults += ng->faults[mem_idx]; } +#ifdef CONFIG_SCHED_CACHE + /* + * Per task p->numa_faults[mem_idx] converges, + * so the accumulation of each task's faults + * converges too - Given the number of threads, + * it cannot overflow an unsigned long. + * Racy with concurrent updates from other threads + * sharing this mm. Acceptable since footprint is a + * heuristic and occasional lost updates are tolerable. + * + * If a task exits, its corresponding footprint must + * be subtracted from the mm->sc_stat.footprint, otherwise + * the mm->sc_stat.footprint will not converge: + * the exiting thread's footprint remains unchanged/undecayed + * in mm->sc_stat.footprint. See exit_mm(). + * + * Lost updates and unsynchronized subtraction + * in exit_mm() can cause footprint + diff to + * go negative. Clamp to zero to prevent the + * unsigned footprint from wrapping. + */ + new_fp = (long)READ_ONCE(p->mm->sc_stat.footprint) + diff; + WRITE_ONCE(p->mm->sc_stat.footprint, + max(new_fp, 0L)); +#endif } if (!ng) { @@ -10338,7 +10393,8 @@ static enum llc_mig can_migrate_llc_task(int src_cpu, int dst_cpu, return mig_unrestricted; /* skip cache aware load balance for too many threads */ - if (invalid_llc_nr(mm, p, dst_cpu)) { + if (invalid_llc_nr(mm, p, dst_cpu) || + exceed_llc_capacity(mm, dst_cpu)) { if (mm->sc_stat.cpu != -1) mm->sc_stat.cpu = -1; return mig_unrestricted; -- cgit v1.2.3 From c1e7fe5e75ed11fa85368e5a186472afd3858f3a Mon Sep 17 00:00:00 2001 From: Chen Yu Date: Wed, 13 May 2026 13:39:17 -0700 Subject: sched/cache: Add user control to adjust the aggressiveness of cache-aware scheduling Introduce a set of debugfs knobs to control how aggressively the cache aware scheduling does the task aggregation. (1) aggr_tolerance With sched_cache enabled, the scheduler uses a process's footprint as a proxy for its LLC footprint to determine if aggregating tasks on the preferred LLC could cause cache contention. If the footprint exceeds the LLC size, aggregation is skipped. Since the kernel cannot efficiently track per-task cache usage (resctrl is user-space only), userspace can provide a more accurate hint. Introduce /sys/kernel/debug/sched/llc_balancing/aggr_tolerance to let users control how strictly footprint limits aggregation. Values range from 0 to 100: - 0: Cache-aware scheduling is disabled. - 1: Strict; tasks with footprint larger than LLC size are skipped. - >=100: Aggressive; tasks are aggregated regardless of footprint. For example, with a 32MB L3 cache: - aggr_tolerance=1 -> tasks with footprint > 32MB are skipped. - aggr_tolerance=99 -> tasks with footprint > 784GB are skipped (784GB = (1 + (99 - 1) * 256) * 32MB). Similarly, /sys/kernel/debug/sched/llc_balancing/aggr_tolerance also controls how strictly the number of active threads is considered when doing cache aware load balance. The number of SMTs is also considered. High SMT counts reduce the aggregation capacity, preventing excessive task aggregation on SMT-heavy systems like Power10/Power11. Yangyu suggested introducing separate aggregation controls for the number of active threads and memory footprint checks. Since there are plans to add per-process/task group controls, fine-grained tunables are deferred to that implementation. (2) epoch_period, epoch_affinity_timeout, imb_pct, overaggr_pct are also turned into tunables. Suggested-by: K Prateek Nayak Suggested-by: Madadi Vineeth Reddy Suggested-by: Shrikanth Hegde Suggested-by: Tingyin Duan Suggested-by: Jianyong Wu Suggested-by: Yangyu Chen Signed-off-by: Chen Yu Co-developed-by: Tim Chen Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Tested-by: Tingyin Duan Link: https://patch.msgid.link/1c62cc060ba2b33d7b1f0ed98b3390128edbae93.1778703694.git.tim.c.chen@linux.intel.com --- kernel/sched/debug.c | 10 ++++++++ kernel/sched/fair.c | 68 +++++++++++++++++++++++++++++++++++++++++++++------- kernel/sched/sched.h | 5 ++++ 3 files changed, 75 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 2eae67cd2ba2..fe569539e888 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -670,6 +670,16 @@ static __init int sched_init_debug(void) llc = debugfs_create_dir("llc_balancing", debugfs_sched); debugfs_create_file("enabled", 0644, llc, NULL, &sched_cache_enable_fops); + debugfs_create_u32("aggr_tolerance", 0644, llc, + &llc_aggr_tolerance); + debugfs_create_u32("epoch_period", 0644, llc, + &llc_epoch_period); + debugfs_create_u32("epoch_affinity_timeout", 0644, llc, + &llc_epoch_affinity_timeout); + debugfs_create_u32("overaggr_pct", 0644, llc, + &llc_overaggr_pct); + debugfs_create_u32("imb_pct", 0644, llc, + &llc_imb_pct); #endif debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index a10116ffe0d1..76ac6a8100fc 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1375,6 +1375,11 @@ static void set_next_buddy(struct sched_entity *se); */ #define EPOCH_PERIOD (HZ / 100) /* 10 ms */ #define EPOCH_LLC_AFFINITY_TIMEOUT 5 /* 50 ms */ +__read_mostly unsigned int llc_aggr_tolerance = 1; +__read_mostly unsigned int llc_epoch_period = EPOCH_PERIOD; +__read_mostly unsigned int llc_epoch_affinity_timeout = EPOCH_LLC_AFFINITY_TIMEOUT; +__read_mostly unsigned int llc_imb_pct = 20; +__read_mostly unsigned int llc_overaggr_pct = 50; static int llc_id(int cpu) { @@ -1384,11 +1389,25 @@ static int llc_id(int cpu) return per_cpu(sd_llc_id, cpu); } +static inline int get_sched_cache_scale(int mul) +{ + unsigned int tol = READ_ONCE(llc_aggr_tolerance); + + if (!tol) + return 0; + + if (tol >= 100) + return INT_MAX; + + return (1 + (tol - 1) * mul); +} + static bool exceed_llc_capacity(struct mm_struct *mm, int cpu) { #ifdef CONFIG_NUMA_BALANCING unsigned long llc, footprint; struct sched_domain *sd; + int scale; guard(rcu)(); @@ -1404,7 +1423,28 @@ static bool exceed_llc_capacity(struct mm_struct *mm, int cpu) llc = sd->llc_bytes; footprint = READ_ONCE(mm->sc_stat.footprint); - return (llc < (footprint * PAGE_SIZE)); + /* + * Scale the LLC size by 256*llc_aggr_tolerance + * and compare it to the task's footprint. + * + * Suppose the L3 size is 32MB. If the + * llc_aggr_tolerance is 1: + * When the footprint is larger than 32MB, the + * process is regarded as exceeding the LLC + * capacity. If the llc_aggr_tolerance is 99: + * When the footprint is larger than 784GB, the + * process is regarded as exceeding the LLC + * capacity: + * 784GB = (1 + (99 - 1) * 256) * 32MB + * If the llc_aggr_tolerance is 100: + * ignore the footprint and do the aggregation + * anyway. + */ + scale = get_sched_cache_scale(256); + if (scale == INT_MAX) + return false; + + return ((llc * (u64)scale) < (footprint * PAGE_SIZE)); } #endif return false; @@ -1413,11 +1453,21 @@ static bool exceed_llc_capacity(struct mm_struct *mm, int cpu) static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p, int cpu) { + int scale; + if (get_nr_threads(p) <= 1) return true; + /* + * Scale the number of 'cores' in a LLC by llc_aggr_tolerance + * and compare it to the task's active threads. + */ + scale = get_sched_cache_scale(1); + if (scale == INT_MAX) + return false; + return !fits_capacity((mm->sc_stat.nr_running_avg * cpu_smt_num_threads), - per_cpu(sd_llc_size, cpu)); + (scale * per_cpu(sd_llc_size, cpu))); } static void account_llc_enqueue(struct rq *rq, struct task_struct *p) @@ -1513,13 +1563,14 @@ static inline void __update_mm_sched(struct rq *rq, { lockdep_assert_held(&rq->cpu_epoch_lock); + unsigned int period = max(READ_ONCE(llc_epoch_period), 1U); unsigned long n, now = jiffies; long delta = now - rq->cpu_epoch_next; if (delta > 0) { - n = (delta + EPOCH_PERIOD - 1) / EPOCH_PERIOD; + n = (delta + period - 1) / period; rq->cpu_epoch += n; - rq->cpu_epoch_next += n * EPOCH_PERIOD; + rq->cpu_epoch_next += n * period; __shr_u64(&rq->cpu_runtime, n); } @@ -1611,7 +1662,7 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) * If this process hasn't hit task_cache_work() for a while invalidate * its preferred state. */ - if (epoch - READ_ONCE(mm->sc_stat.epoch) > EPOCH_LLC_AFFINITY_TIMEOUT || + if ((long)(epoch - READ_ONCE(mm->sc_stat.epoch)) > llc_epoch_affinity_timeout || invalid_llc_nr(mm, p, cpu_of(rq)) || exceed_llc_capacity(mm, cpu_of(rq))) { if (mm->sc_stat.cpu != -1) @@ -1740,7 +1791,8 @@ static void task_cache_work(struct callback_head *work) /* only 1 thread is allowed to scan */ if (!try_cmpxchg(&mm->sc_stat.next_scan, &next_scan, - now + EPOCH_PERIOD)) + now + max_t(unsigned long, + READ_ONCE(llc_epoch_period), 1))) return; curr_cpu = task_cpu(p); @@ -10232,7 +10284,7 @@ static inline int task_is_ineligible_on_dst_cpu(struct task_struct *p, int dest_ */ static bool fits_llc_capacity(unsigned long util, unsigned long max) { - u32 aggr_pct = 50; + u32 aggr_pct = llc_overaggr_pct; /* * For single core systems, raise the aggregation @@ -10252,7 +10304,7 @@ static bool fits_llc_capacity(unsigned long util, unsigned long max) */ /* Allows dst util to be bigger than src util by up to bias percent */ #define util_greater(util1, util2) \ - ((util1) * 100 > (util2) * 120) + ((util1) * 100 > (util2) * (100 + llc_imb_pct)) static __maybe_unused bool get_llc_stats(int cpu, unsigned long *util, unsigned long *cap) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index f499d5dd1130..27409399137c 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -4072,6 +4072,11 @@ static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct DECLARE_STATIC_KEY_FALSE(sched_cache_present); DECLARE_STATIC_KEY_FALSE(sched_cache_active); extern int sysctl_sched_cache_user; +extern unsigned int llc_aggr_tolerance; +extern unsigned int llc_epoch_period; +extern unsigned int llc_epoch_affinity_timeout; +extern unsigned int llc_imb_pct; +extern unsigned int llc_overaggr_pct; static inline bool sched_cache_enabled(void) { -- cgit v1.2.3 From d943b86dfbf4e9b76be30cf90b1b3f82ff9abbac Mon Sep 17 00:00:00 2001 From: Chen Yu Date: Wed, 13 May 2026 13:39:18 -0700 Subject: sched/cache: Fix rcu warning when accessing sd_llc domain rcu_dereference_all() should be used to access the sd_llc domain under RCU protection. This bug was reported by sashiko. Fixes: df0d98475954 ("sched/cache: Introduce infrastructure for cache-aware load balancing") Signed-off-by: Chen Yu Co-developed-by: Tim Chen Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/2dc49455e861215d8059a1c877953f0b95990038.1778703694.git.tim.c.chen@linux.intel.com --- kernel/sched/fair.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 76ac6a8100fc..c549ad489c6d 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1814,7 +1814,7 @@ static void task_cache_work(struct callback_head *work) for_each_cpu(cpu, cpus) { /* XXX sched_cluster_active */ - struct sched_domain *sd = per_cpu(sd_llc, cpu); + struct sched_domain *sd = rcu_dereference_all(per_cpu(sd_llc, cpu)); unsigned long occ, m_occ = 0, a_occ = 0; int m_cpu = -1, i; -- cgit v1.2.3 From 9f23469401b04cfd9a5d0a8b61760a48cce35dc1 Mon Sep 17 00:00:00 2001 From: Chen Yu Date: Wed, 13 May 2026 13:39:19 -0700 Subject: sched/cache: Fix potential NULL mm pointer access A concurrent task exit might cause a NULL pointer dereference in account_mm_sched(). Use the locally cached mm pointer instead, since the active_mm reference guarantees the structure remains allocated. Meanwhile, skip the kernel thread because it has nothing to do with cache aware scheduling. This bug was reported by sashiko and Vern. Fixes: df0d98475954 ("sched/cache: Introduce infrastructure for cache-aware load balancing") Reported-by: Vern Hao Signed-off-by: Chen Yu Co-developed-by: Tim Chen Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/all/09cf7ee3-6e27-4505-9692-4b4a4707c8b2@gmail.com/ Link: https://patch.msgid.link/066d8cfa45d4822bf4367e788c50377c66bbcc82.1778703694.git.tim.c.chen@linux.intel.com --- kernel/sched/fair.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index c549ad489c6d..663968b46e13 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1649,7 +1649,7 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) if (!mm || !mm->sc_stat.pcpu_sched) return; - pcpu_sched = per_cpu_ptr(p->mm->sc_stat.pcpu_sched, cpu_of(rq)); + pcpu_sched = per_cpu_ptr(mm->sc_stat.pcpu_sched, cpu_of(rq)); scoped_guard (raw_spinlock, &rq->cpu_epoch_lock) { __update_mm_sched(rq, pcpu_sched); @@ -1689,7 +1689,8 @@ static void task_tick_cache(struct rq *rq, struct task_struct *p) if (!sched_cache_enabled()) return; - if (!mm || !mm->sc_stat.pcpu_sched) + if (!mm || p->flags & PF_KTHREAD || + !mm->sc_stat.pcpu_sched) return; epoch = rq->cpu_epoch; -- cgit v1.2.3 From 91d07324c9305c0e4afff0cc859cac96594daa88 Mon Sep 17 00:00:00 2001 From: Chen Yu Date: Wed, 13 May 2026 13:39:20 -0700 Subject: sched/cache: Annotate lockless accesses to mm->sc_stat.cpu mm->sc_stat.cpu is written by task_cache_work() and could be read locklessly by several functions on other CPUs. Use READ_ONCE and WRITE_ONCE on mm->sc_stat.cpu access and write to prevent inconsistent values from compiler optimizations when there are multiple accesses. For example in get_pref_llc(), if the writer updated the field between two compiler-generated loads, the validation (e.g., cpu != -1) and subsequent use (e.g., llc_id(cpu)) could operate on different values, allowing a negative CPU ID to be used as an index. Leave plain write in mm_init_sched(), where the mm is not yet visible to other CPUs. This bug was reported by sashiko. Fixes: 47d8696b95f7 ("sched/cache: Assign preferred LLC ID to processes") Signed-off-by: Chen Yu Co-developed-by: Tim Chen Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/63ea494f12efcf265d7134400a06cd75d7f2c310.1778703694.git.tim.c.chen@linux.intel.com --- kernel/sched/fair.c | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 663968b46e13..087445ea6bc9 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1598,13 +1598,14 @@ static unsigned long fraction_mm_sched(struct rq *rq, static int get_pref_llc(struct task_struct *p, struct mm_struct *mm) { - int mm_sched_llc = -1; + int mm_sched_llc = -1, mm_sched_cpu; if (!mm) return -1; - if (mm->sc_stat.cpu != -1) { - mm_sched_llc = llc_id(mm->sc_stat.cpu); + mm_sched_cpu = READ_ONCE(mm->sc_stat.cpu); + if (mm_sched_cpu != -1) { + mm_sched_llc = llc_id(mm_sched_cpu); #ifdef CONFIG_NUMA_BALANCING /* @@ -1619,7 +1620,7 @@ static int get_pref_llc(struct task_struct *p, struct mm_struct *mm) */ if (static_branch_likely(&sched_numa_balancing) && p->numa_preferred_nid >= 0 && - cpu_to_node(mm->sc_stat.cpu) != p->numa_preferred_nid) + cpu_to_node(mm_sched_cpu) != p->numa_preferred_nid) mm_sched_llc = -1; #endif } @@ -1665,8 +1666,8 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) if ((long)(epoch - READ_ONCE(mm->sc_stat.epoch)) > llc_epoch_affinity_timeout || invalid_llc_nr(mm, p, cpu_of(rq)) || exceed_llc_capacity(mm, cpu_of(rq))) { - if (mm->sc_stat.cpu != -1) - mm->sc_stat.cpu = -1; + if (READ_ONCE(mm->sc_stat.cpu) != -1) + WRITE_ONCE(mm->sc_stat.cpu, -1); } mm_sched_llc = get_pref_llc(p, mm); @@ -1714,7 +1715,7 @@ static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p) if (!static_branch_likely(&sched_numa_balancing)) goto out; - cpu = p->mm->sc_stat.cpu; + cpu = READ_ONCE(p->mm->sc_stat.cpu); if (cpu != -1) nid = cpu_to_node(cpu); curr_cpu = task_cpu(p); @@ -1799,8 +1800,8 @@ static void task_cache_work(struct callback_head *work) curr_cpu = task_cpu(p); if (invalid_llc_nr(mm, p, curr_cpu) || exceed_llc_capacity(mm, curr_cpu)) { - if (mm->sc_stat.cpu != -1) - mm->sc_stat.cpu = -1; + if (READ_ONCE(mm->sc_stat.cpu) != -1) + WRITE_ONCE(mm->sc_stat.cpu, -1); return; } @@ -1857,7 +1858,7 @@ static void task_cache_work(struct callback_head *work) m_a_cpu = m_cpu; } - if (llc_id(cpu) == llc_id(mm->sc_stat.cpu)) + if (llc_id(cpu) == llc_id(READ_ONCE(mm->sc_stat.cpu))) curr_m_a_occ = a_occ; cpumask_andnot(cpus, cpus, sched_domain_span(sd)); @@ -1875,7 +1876,7 @@ static void task_cache_work(struct callback_head *work) * 3. 2X is chosen based on test results, as it delivers * the optimal performance gain so far. */ - mm->sc_stat.cpu = m_a_cpu; + WRITE_ONCE(mm->sc_stat.cpu, m_a_cpu); } update_avg_scale(&mm->sc_stat.nr_running_avg, nr_running); @@ -10441,15 +10442,15 @@ static enum llc_mig can_migrate_llc_task(int src_cpu, int dst_cpu, if (!mm) return mig_unrestricted; - cpu = mm->sc_stat.cpu; + cpu = READ_ONCE(mm->sc_stat.cpu); if (cpu < 0 || cpus_share_cache(src_cpu, dst_cpu)) return mig_unrestricted; /* skip cache aware load balance for too many threads */ if (invalid_llc_nr(mm, p, dst_cpu) || exceed_llc_capacity(mm, dst_cpu)) { - if (mm->sc_stat.cpu != -1) - mm->sc_stat.cpu = -1; + if (READ_ONCE(mm->sc_stat.cpu) != -1) + WRITE_ONCE(mm->sc_stat.cpu, -1); return mig_unrestricted; } -- cgit v1.2.3 From 03755348b8e74421f92ffed9da159175a698290b Mon Sep 17 00:00:00 2001 From: Chen Yu Date: Wed, 13 May 2026 13:39:21 -0700 Subject: sched/cache: Fix unpaired account_llc_enqueue/dequeue There is a race condition that, after a task is enqueued on a runqueue, task_llc(p) may change due to CPU hotplug, because the llc_id is dynamically allocated and adjusted at runtime. Therefore, checking task_llc(p) to determine whether the task is being dequeued from its preferred LLC is unreliable and can cause inconsistent values. To fix this problem, record whether p is enqueued on its preferred LLC, in order to pair with account_llc_dequeue() to maintain a consistent nr_pref_llc_running per runqueue. This bug was reported by sashiko, and the solution was once suggested by Prateek. Fixes: 46afe3af7ead ("sched/cache: Track LLC-preferred tasks per runqueue") Suggested-by: K Prateek Nayak Signed-off-by: Chen Yu Co-developed-by: Tim Chen Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/0c8c6a1571d66792a4d2ff0103ba3cc13e059046.1778703694.git.tim.c.chen@linux.intel.com --- kernel/sched/fair.c | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 087445ea6bc9..96c61ce366c2 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1472,15 +1472,32 @@ static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p, static void account_llc_enqueue(struct rq *rq, struct task_struct *p) { + int pref_llc, pref_llc_queued; struct sched_domain *sd; - int pref_llc; pref_llc = p->preferred_llc; if (pref_llc < 0) return; + pref_llc_queued = (pref_llc == task_llc(p)); rq->nr_llc_running++; - rq->nr_pref_llc_running += (pref_llc == task_llc(p)); + rq->nr_pref_llc_running += pref_llc_queued; + + /* + * Record whether p is enqueued on its preferred + * LLC, in order to pair with account_llc_dequeue() + * to maintain a consistent nr_pref_llc_running per + * runqueue. + * This is necessary because a race condition exists: + * after a task is enqueued on a runqueue, task_llc(p) + * may change due to CPU hotplug. Therefore, checking + * task_llc(p) to determine whether the task is being + * dequeued from its preferred LLC is unreliable and + * can cause inconsistent values - checking the + * p->pref_llc_queued in account_llc_dequeue() would + * be reliable. + */ + p->pref_llc_queued = pref_llc_queued; sd = rcu_dereference_all(rq->sd); if (sd && (unsigned int)pref_llc < sd->llc_max) @@ -1497,7 +1514,15 @@ static void account_llc_dequeue(struct rq *rq, struct task_struct *p) return; rq->nr_llc_running--; - rq->nr_pref_llc_running -= (pref_llc == task_llc(p)); + if (p->pref_llc_queued) { + rq->nr_pref_llc_running--; + /* + * Update the status in case + * other logic might query + * this. + */ + p->pref_llc_queued = 0; + } sd = rcu_dereference_all(rq->sd); if (sd && (unsigned int)pref_llc < sd->llc_max) { -- cgit v1.2.3 From d6b9afab44e23d537fb85ecf50330baaf9ec82e9 Mon Sep 17 00:00:00 2001 From: Chen Yu Date: Wed, 13 May 2026 13:39:22 -0700 Subject: sched/cache: Fix checking active load balance by only considering the CFS task The currently running task cur may not be a CFS task, such as an RT or Deadline task. For non-CFS tasks, the task_util(cur) utilization average is not maintained, so this might pass a stale or meaningless value to can_migrate_llc(). Check if the task is CFS before getting its task_util(). This bug was reported by sashiko. Fixes: 714059f79ff0 ("sched/cache: Handle moving single tasks to/from their preferred LLC") Signed-off-by: Chen Yu Co-developed-by: Tim Chen Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/f9161133cf040d286dca11344a112c5ef2a5253d.1778703694.git.tim.c.chen@linux.intel.com --- kernel/sched/fair.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 96c61ce366c2..c249caea3862 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -10509,7 +10509,8 @@ alb_break_llc(struct lb_env *env) /* * All tasks prefer to stay on their current CPU. * Do not pull a task from its preferred CPU if: - * 1. It is the only task running there(not too imbalance); OR + * 1. It is the only task running and does not exceed + * imbalance allowance; OR * 2. Migrating it away from its preferred LLC would violate * the cache-aware scheduling policy. */ @@ -10522,7 +10523,7 @@ alb_break_llc(struct lb_env *env) return true; cur = rcu_dereference_all(env->src_rq->curr); - if (cur) + if (cur && cur->sched_class == &fair_sched_class) util = task_util(cur); if (can_migrate_llc(env->src_cpu, env->dst_cpu, -- cgit v1.2.3 From 9f7c745850b4b1b7e4706ae81f04c43f204a6a8d Mon Sep 17 00:00:00 2001 From: Chen Yu Date: Wed, 13 May 2026 13:39:23 -0700 Subject: sched/cache: Fix race condition during sched domain rebuild sched_cache_active_set_unlocked() checks hardware support without locks: static void sched_cache_active_set(bool locked) { /* hardware does not support */ if (!static_branch_likely(&sched_cache_present)) { _sched_cache_active_set(false, locked); return; } ... If build_sched_domains() runs concurrently during CPU hotplug, it can disable sched_cache_present under sched_domains_mutex and the CPU hotplug lock. If a debugfs write thread evaluates sched_cache_present as true right before that, and then blocks or gets preempted, it might proceed to enable sched_cache_active after the hardware support has been marked as absent. Make it safer by acquiring cpus_read_lock() and sched_domains_mutex_lock() when the user changes sched_cache_active via debugfs. This bug was reported by sashiko. Fixes: 067a31358143 ("sched/cache: Allow the user space to turn on and off cache aware scheduling") Signed-off-by: Chen Yu Co-developed-by: Tim Chen Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/9afddf439687f04bb56b46625bd9f153eb8abad5.1778703694.git.tim.c.chen@linux.intel.com --- kernel/sched/debug.c | 4 +++- kernel/sched/sched.h | 2 +- kernel/sched/topology.c | 43 ++++++++++++++++--------------------------- 3 files changed, 20 insertions(+), 29 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index fe569539e888..ed3a0d65da0c 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -224,7 +224,9 @@ sched_cache_enable_write(struct file *filp, const char __user *ubuf, sysctl_sched_cache_user = val; - sched_cache_active_set_unlocked(); + sched_cache_active_set(); + + *ppos += cnt; return cnt; } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 27409399137c..45a3b77f46aa 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -4083,7 +4083,7 @@ static inline bool sched_cache_enabled(void) return static_branch_unlikely(&sched_cache_active); } -extern void sched_cache_active_set_unlocked(void); +extern void sched_cache_active_set(void); #endif diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 7248a7279abe..c257134f613d 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -917,30 +917,20 @@ err: return false; } -static void _sched_cache_active_set(bool enable, bool locked) -{ - if (enable) { - if (locked) - static_branch_enable_cpuslocked(&sched_cache_active); - else - static_branch_enable(&sched_cache_active); - } else { - if (locked) - static_branch_disable_cpuslocked(&sched_cache_active); - else - static_branch_disable(&sched_cache_active); - } -} - /* * Enable/disable cache aware scheduling according to * user input and the presence of hardware support. */ -static void sched_cache_active_set(bool locked) +static void _sched_cache_active_set(void) { + lockdep_assert_cpus_held(); + lockdep_assert_held(&sched_domains_mutex); + /* hardware does not support */ if (!static_branch_likely(&sched_cache_present)) { - _sched_cache_active_set(false, locked); + static_branch_disable_cpuslocked(&sched_cache_active); + if (sched_debug()) + pr_info("%s: cache aware scheduling not supported on this platform\n", __func__); return; } @@ -951,24 +941,23 @@ static void sched_cache_active_set(bool locked) * for now. */ if (sysctl_sched_cache_user) { - _sched_cache_active_set(true, locked); + static_branch_enable_cpuslocked(&sched_cache_active); if (sched_debug()) pr_info("%s: enabling cache aware scheduling\n", __func__); } else { - _sched_cache_active_set(false, locked); + static_branch_disable_cpuslocked(&sched_cache_active); if (sched_debug()) pr_info("%s: disabling cache aware scheduling\n", __func__); } } -static void sched_cache_active_set_locked(void) +void sched_cache_active_set(void) { - return sched_cache_active_set(true); -} - -void sched_cache_active_set_unlocked(void) -{ - return sched_cache_active_set(false); + cpus_read_lock(); + sched_domains_mutex_lock(); + _sched_cache_active_set(); + sched_domains_mutex_unlock(); + cpus_read_unlock(); } /* @@ -3082,7 +3071,7 @@ error: else static_branch_disable_cpuslocked(&sched_cache_present); - sched_cache_active_set_locked(); + _sched_cache_active_set(); #endif __free_domain_allocs(&d, alloc_state, cpu_map); -- cgit v1.2.3 From 5beff4f087277901444787d4b7af6b3a93c34c54 Mon Sep 17 00:00:00 2001 From: Chen Yu Date: Wed, 13 May 2026 13:39:24 -0700 Subject: sched/cache: Fix cache aware scheduling enabling for multi LLCs system If there are multiple LLCs in the system, cache aware scheduling should be enabled. However, there is a corner case where, if there is a single NUMA node and a single LLC per node, cache aware scheduling will be turned on in the current implementation - because at this moment, the parent domain has not yet been degenerated, and it is possible that the current domain has the same cpu span as its parent. There is no need to turn cache aware scheduling on in this scenario. Fix it by iterating the parent domains to find a domain that is a superset of the current sd_llc, so that later, after the duplicated parent domains have been degenerated, cache aware scheduling will take effect. For example, the expected behavior would be: 2 sockets, 1 LLC per socket: MC span=0-3, PKG span=0-7, has_multi_llcs=true 1 socket, 2 LLCs per socket: MC span=0-3, PKG span=0-7, has_multi_llcs=true 2 sockets, 2 LLCs per socket: MC span=0-3, PKG span=0-7, has_multi_llcs=true 1 socket, 1 LLC per socket: MC span=0-3, PKG span=0-3, has_multi_llcs=false This bug was reported by sashiko. Fixes: d59f4fd1d303 ("sched/cache: Enable cache aware scheduling for multi LLCs NUMA node") Signed-off-by: Chen Yu Co-developed-by: Tim Chen Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/6328a8a7f40925cec2a712d81ee58128a4c4444a.1778703694.git.tim.c.chen@linux.intel.com --- kernel/sched/topology.c | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index c257134f613d..4b7c64cbe854 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -1008,6 +1008,37 @@ static bool alloc_sd_llc(const struct cpumask *cpu_map, } #endif +/* + * Return true if @sd belongs to an LLC group whose enclosing + * partition spans more than one LLC. @sd must be the topmost + * SD_SHARE_LLC domain. + * + * Any duplicated parent domains with the same span as @sd are + * skipped: before cpu_attach_domain() degeneration these still + * exist, after degeneration the loop is a no-op. This makes the + * helper usable both during sched domain build and against an + * already-attached domain tree. + * + * Note: For systems with a single LLC per node, cache-aware + * scheduling is still enabled when multiple nodes exist. + * However, NUMA balancing decisions take precedence over + * cache-aware scheduling. Conversely, if there is only one + * LLC per partition, cache-aware scheduling should be disabled. + */ +static bool sd_in_multi_llcs(struct sched_domain *sd) +{ + struct sched_domain *sdp = sd->parent; + + /* it does not make sense to aggregate to 1 CPU */ + if (sd->span_weight == 1) + return false; + + while (sdp && sdp->span_weight == sd->span_weight) + sdp = sdp->parent; + + return !!sdp; +} + /* * Return the canonical balance CPU for this group, this is the first CPU * of this group that's also in the balance mask. @@ -3017,9 +3048,11 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att * NUMA imbalance stats for the hierarchy. */ if (sd->parent) { - if (IS_ENABLED(CONFIG_NUMA)) - adjust_numa_imbalance(sd); - has_multi_llcs = true; + if (IS_ENABLED(CONFIG_NUMA)) + adjust_numa_imbalance(sd); + + if (sd_in_multi_llcs(sd)) + has_multi_llcs = true; } } } -- cgit v1.2.3 From a7660ce1590fc1316a44cc2af53a07a21dfc25da Mon Sep 17 00:00:00 2001 From: Chen Yu Date: Wed, 13 May 2026 13:39:25 -0700 Subject: sched/cache: Fix has_multi_llcs iff at least one partition has multiple LLCs sched_cache_present is a global static key, but build_sched_domains() is called per partition from the "Build new domains" loop in partition_sched_domains_locked(). Each call unconditionally sets the key based solely on the has_multi_llcs local variable for that partition. The call to the last partition set the value even when there are previous partitions with multiple LLCs. If partition A (multi-LLC) is built first, the key is enabled. Then when partition B (single-LLC) is built, the key is disabled. The multi-LLC partition A is still active but the key is now off. Fix it by doing a similar thing as sched_energy_present: check the multi-LLCs during the iteration over all the partitions rather than checking it on a single partition. This bug was reported by sashiko. Fixes: d59f4fd1d303 ("sched/cache: Enable cache aware scheduling for multi LLCs NUMA node") Signed-off-by: Chen Yu Co-developed-by: Tim Chen Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/c541af2547d54509fbfd3b3a1e8072e2e5c7ff68.1778703694.git.tim.c.chen@linux.intel.com --- kernel/sched/topology.c | 69 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 53 insertions(+), 16 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 4b7c64cbe854..e47a3f72eb72 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -951,6 +951,7 @@ static void _sched_cache_active_set(void) } } +/* used by debugfs */ void sched_cache_active_set(void) { cpus_read_lock(); @@ -1000,12 +1001,27 @@ void sched_update_llc_bytes(unsigned int cpu) unlock: sched_domains_mutex_unlock(); } + +static void sched_cache_set(bool has_multi_llcs) +{ + /* + * TBD: check before writing to it. sched domain rebuild + * is not in the critical path, leave as-is for now. + */ + if (has_multi_llcs) + static_branch_enable_cpuslocked(&sched_cache_present); + else + static_branch_disable_cpuslocked(&sched_cache_present); + + _sched_cache_active_set(); +} #else static bool alloc_sd_llc(const struct cpumask *cpu_map, struct s_data *d) { return false; } +static inline void sched_cache_set(bool has_multi_llcs) { } #endif /* @@ -2950,7 +2966,8 @@ void sched_domains_free_llc_id(int cpu) * to the individual CPUs */ static int -build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *attr) +build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *attr, + bool *multi_llcs) { enum s_alloc alloc_state = sa_none; bool has_multi_llcs = false; @@ -3094,18 +3111,7 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att ret = 0; error: -#ifdef CONFIG_SCHED_CACHE - /* - * TBD: check before writing to it. sched domain rebuild - * is not in the critical path, leave as-is for now. - */ - if (!ret && has_multi_llcs) - static_branch_enable_cpuslocked(&sched_cache_present); - else - static_branch_disable_cpuslocked(&sched_cache_present); - - _sched_cache_active_set(); -#endif + *multi_llcs = has_multi_llcs; __free_domain_allocs(&d, alloc_state, cpu_map); return ret; @@ -3168,6 +3174,7 @@ void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms) */ int __init sched_init_domains(const struct cpumask *cpu_map) { + bool multi_llcs; int err; zalloc_cpumask_var(&sched_domains_llc_id_allocmask, GFP_KERNEL); @@ -3182,7 +3189,9 @@ int __init sched_init_domains(const struct cpumask *cpu_map) if (!doms_cur) doms_cur = &fallback_doms; cpumask_and(doms_cur[0], cpu_map, housekeeping_cpumask(HK_TYPE_DOMAIN)); - err = build_sched_domains(doms_cur[0], NULL); + err = build_sched_domains(doms_cur[0], NULL, &multi_llcs); + if (!err) + sched_cache_set(multi_llcs); return err; } @@ -3255,6 +3264,7 @@ static void partition_sched_domains_locked(int ndoms_new, cpumask_var_t doms_new struct sched_domain_attr *dattr_new) { bool __maybe_unused has_eas = false; + bool has_multi_llcs = false, multi_llcs; int i, j, n; int new_topology; @@ -3304,14 +3314,41 @@ match1: for (i = 0; i < ndoms_new; i++) { for (j = 0; j < n && !new_topology; j++) { if (cpumask_equal(doms_new[i], doms_cur[j]) && - dattrs_equal(dattr_new, i, dattr_cur, j)) + dattrs_equal(dattr_new, i, dattr_cur, j)) { + /* + * Reused partition has to be taken care + * of here, because there could be a corner + * case that if the reused partition is skipped + * and only new partition is considered, an + * incorrect has_multi_llcs would be set. For + * example: + * If the only multi-LLC partition is reused + * and a new single-LLC partition is built, + * sched_cache_set(false) disables cache-aware + * scheduling globally despite the reused + * multi-LLC partition still being active. + */ + struct sched_domain *sd; + int cpu = cpumask_first(doms_cur[j]); + + guard(rcu)(); + sd = rcu_dereference(cpu_rq(cpu)->sd); + while (sd && sd->parent && (sd->parent->flags & SD_SHARE_LLC)) + sd = sd->parent; + if (sd && (sd->flags & SD_SHARE_LLC) && sd->parent && + sd_in_multi_llcs(sd)) + has_multi_llcs = true; goto match2; + } } /* No match - add a new doms_new */ - build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL); + build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL, + &multi_llcs); + has_multi_llcs |= multi_llcs; match2: ; } + sched_cache_set(has_multi_llcs); #if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) /* Build perf domains: */ -- cgit v1.2.3 From c99b8593b060931c5a0a4b701689f8d6a2c00dbf Mon Sep 17 00:00:00 2001 From: Chen Yu Date: Wed, 13 May 2026 13:39:27 -0700 Subject: sched/cache: Fix stale preferred_llc for a new task On fork without CLONE_VM, the child gets a new mm, the parent's preferred_llc value is stale for the child. Fix this by resetting the task's preferred_llc to -1. This bug was reported by sashiko. Fixes: 47d8696b95f7 ("sched/cache: Assign preferred LLC ID to processes") Signed-off-by: Chen Yu Co-developed-by: Tim Chen Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/0ec7309d0e24ede97656754d1505b7490403d966.1778703694.git.tim.c.chen@linux.intel.com --- kernel/sched/fair.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index c249caea3862..2614315a25e0 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1914,6 +1914,11 @@ void init_sched_mm(struct task_struct *p) init_task_work(work, task_cache_work); work->next = work; + /* + * Reset new task's preference to avoid + * polluting account_llc_enqueue(). + */ + p->preferred_llc = -1; } #else /* CONFIG_SCHED_CACHE */ -- cgit v1.2.3 From f05ddc6771c4a2eb1801dfdd0f7a212a78fa18a7 Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Mon, 18 May 2026 22:54:42 +0800 Subject: bpf: Check tail zero of bpf_common_attr using offsetofend Because of the 8-byte alignment, the compiler will pad struct bpf_common_attr to 24 bytes. That said, sizeof(attr_common) is 24 instead of 20. When check tail zero using sizeof(attr_common) in bpf_check_uarg_tail_zero(), there will be 4 bytes that won't be checked. To also check the padding 4 bytes, replace sizeof(attr_common) with offsetofend(struct bpf_common_attr, log_true_size). Fixes: f28771c0691b ("bpf: Extend BPF syscall with common attributes support") Signed-off-by: Leon Hwang Link: https://lore.kernel.org/r/20260518145446.6794-2-leon.hwang@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/syscall.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 6600e126fbfb..83de8fb9b9aa 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -6278,7 +6278,9 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size, memset(&attr_common, 0, sizeof(attr_common)); if (cmd & BPF_COMMON_ATTRS) { - err = bpf_check_uarg_tail_zero(uattr_common, sizeof(attr_common), size_common); + err = bpf_check_uarg_tail_zero(uattr_common, + offsetofend(struct bpf_common_attr, log_true_size), + size_common); if (err) return err; -- cgit v1.2.3 From 62c4d31d78294bd61cf3403626b789e854357177 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Mon, 18 May 2026 10:32:11 +0200 Subject: pidfd: refuse access to tasks that have started exiting harder The recent ptrace fix closed a hole where someone could rely on task->mm becoming NULL during do_exit() to bypass dumpability checks. This api here leans on on the very same check and so inherits the fix. But there is no good reason to let it succeed at all once the target has entered do_exit(). PF_EXITING is set by exit_signals() at the very top of do_exit(), before exit_mm() and exit_files() run. Once we observe it, the task is committed to dying and exit_files() will release the fdtable shortly. Fixes: 8649c322f75c ("pid: Implement pidfd_getfd syscall") Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260518-obgleich-petersilie-2d77ccccf9b9@brauner Signed-off-by: Christian Brauner (Amutable) --- kernel/pid.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/pid.c b/kernel/pid.c index fd5c2d4aa349..f55189a3d07d 100644 --- a/kernel/pid.c +++ b/kernel/pid.c @@ -885,10 +885,12 @@ static struct file *__pidfd_fget(struct task_struct *task, int fd) if (ret) return ERR_PTR(ret); - if (ptrace_may_access(task, PTRACE_MODE_ATTACH_REALCREDS)) - file = fget_task(task, fd); - else + if (!ptrace_may_access(task, PTRACE_MODE_ATTACH_REALCREDS)) file = ERR_PTR(-EPERM); + else if (task->flags & PF_EXITING) + file = ERR_PTR(-ESRCH); + else + file = fget_task(task, fd); up_read(&task->signal->exec_update_lock); -- cgit v1.2.3 From c2e390197ad1360db6686a8c89abaafaf83adf72 Mon Sep 17 00:00:00 2001 From: Yuri Andriaccio Date: Thu, 30 Apr 2026 23:38:25 +0200 Subject: sched/rt: Update default bandwidth for real-time tasks to ONE Set the default total bandwidth for SCHED_DEADLINE tasks and servers to ONE. FIFO/RR tasks are already throttled by fair-servers and ext-servers, and the sysctl_sched_rt_runtime parameter now only defines the total bw that is allowed to deadline entities. Signed-off-by: Yuri Andriaccio Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260430213835.62217-22-yurand2000@gmail.com --- kernel/sched/rt.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 4ee8faf01441..e6ea728f519e 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -19,9 +19,9 @@ int sysctl_sched_rt_period = 1000000; /* * part of the period that we allow rt tasks to run in us. - * default: 0.95s + * default: 1s */ -int sysctl_sched_rt_runtime = 950000; +int sysctl_sched_rt_runtime = 1000000; #ifdef CONFIG_SYSCTL static int sysctl_sched_rr_timeslice = (MSEC_PER_SEC * RR_TIMESLICE) / HZ; -- cgit v1.2.3 From eecd5e117cfa63a353f4c69fdcea5d9b14af698e Mon Sep 17 00:00:00 2001 From: Yuri Andriaccio Date: Thu, 30 Apr 2026 23:38:05 +0200 Subject: sched/deadline: Fix replenishment logic for non-deferred servers Enqueue and replenish non-deferred deadline servers when their runtime is exhausted and the replenishment timer could not be started because it is too close to the wake-up instant. Signed-off-by: Yuri Andriaccio Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260430213835.62217-2-yurand2000@gmail.com --- kernel/sched/deadline.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index edca7849b165..b60e2df8ff9d 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -1515,8 +1515,12 @@ throttle: if (unlikely(is_dl_boosted(dl_se) || !start_dl_timer(dl_se))) { if (dl_server(dl_se)) { - replenish_dl_new_period(dl_se, rq); - start_dl_timer(dl_se); + if (dl_se->dl_defer) { + replenish_dl_new_period(dl_se, rq); + start_dl_timer(dl_se); + } else { + enqueue_dl_entity(dl_se, ENQUEUE_REPLENISH); + } } else { enqueue_task_dl(rq, dl_task_of(dl_se), ENQUEUE_REPLENISH); } -- cgit v1.2.3 From 95f44886afec7cbce0ff2a5ed8158fbe8aa6f2ec Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Thu, 14 May 2026 16:26:29 -0400 Subject: sched/cputime: Drop now-stale mul_u64_u64_div_u64() over-approximation guard Commit 77baa5bafcbe ("sched/cputime: Fix mul_u64_u64_div_u64() precision for cputime") added a clamp in cputime_adjust(): if (unlikely(stime > rtime)) stime = rtime; The justification was that mul_u64_u64_div_u64() could over-approximate on some architectures (notably arm64 and the old 32-bit fallback), so the mathematically impossible stime > rtime was nevertheless reachable and would underflow utime = rtime - stime. That premise no longer holds. Commit b29a62d87cc0 ("mul_u64_u64_div_u64: make it precise always") replaced the fallback implementation with an exact 128-bit long division, and the x86_64 inline asm already produced exact results. The helper now returns the mathematically correct floor(a*b/d) on every architecture, so stime <= rtime is guaranteed by stime <= stime + utime and the clamp is dead code. Remove it along with its stale comment. Signed-off-by: Nicolas Pitre Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260514202629.673539-1-nico@fluxnic.net --- kernel/sched/cputime.c | 6 ------ 1 file changed, 6 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index fbf31db0d2f3..6e85023a81ff 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -587,12 +587,6 @@ void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev, } stime = mul_u64_u64_div_u64(stime, rtime, stime + utime); - /* - * Because mul_u64_u64_div_u64() can approximate on some - * achitectures; enforce the constraint that: a*b/(b+c) <= a. - */ - if (unlikely(stime > rtime)) - stime = rtime; update: /* -- cgit v1.2.3 From ea19506013ad13685573e4674fbeddb790e27906 Mon Sep 17 00:00:00 2001 From: Yiyang Chen Date: Fri, 15 May 2026 00:05:05 +0800 Subject: sched/clock: Provide !HAVE_UNSTABLE_SCHED_CLOCK stub for sched_clock_stable() When CONFIG_HAVE_UNSTABLE_SCHED_CLOCK is disabled, sched_clock() is already assumed to provide stable semantics, but the public header doesn't provide a sched_clock_stable() stub for that case. Add a header stub that always returns true and clean up the duplicate local stub in ring_buffer.c, so callers can use sched_clock_stable() unconditionally. Signed-off-by: Yiyang Chen Signed-off-by: Peter Zijlstra (Intel) Acked-by: Steven Rostedt Link: https://patch.msgid.link/56e45338858946cd9581b75c8bd45dd37dba52c5.1778773587.git.cyyzero16@gmail.com --- kernel/trace/ring_buffer.c | 7 ------- 1 file changed, 7 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 5326924615a4..02691c3c6dd6 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -3769,13 +3769,6 @@ rb_add_time_stamp(struct ring_buffer_per_cpu *cpu_buffer, return skip_time_extend(event); } -#ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK -static inline bool sched_clock_stable(void) -{ - return true; -} -#endif - static void rb_check_timestamp(struct ring_buffer_per_cpu *cpu_buffer, struct rb_event_info *info) -- cgit v1.2.3 From 6d2051403d6c93832d3058a1b275c6aef2c97f44 Mon Sep 17 00:00:00 2001 From: Vincent Guittot Date: Mon, 18 May 2026 12:23:45 +0200 Subject: sched/fair: Update util_est after updating util_avg during dequeue util_est_update() must be called after updating util_avg during the dequeue of a task and only when the task is not delayed dequeue. Move util_est_update() in update_load_avg(). Fixes: b55945c500c5 ("sched: Fix pick_next_task_fair() vs try_to_wake_up() race") Closes: https://lore.kernel.org/all/20260512124653.305275-1-qyousef@layalina.io/ Reported-by: Qais Yousef Reviewed-and-tested-by: Qais Yousef Signed-off-by: Vincent Guittot Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260518102345.268452-1-vincent.guittot@linaro.org --- kernel/sched/fair.c | 188 +++++++++++++++++++++++++--------------------------- 1 file changed, 92 insertions(+), 96 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 728965851842..09d3acd2d2bc 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -4930,13 +4930,86 @@ static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *s trace_pelt_cfs_tp(cfs_rq); } +#define UTIL_EST_MARGIN (SCHED_CAPACITY_SCALE / 100) + +static inline void util_est_update(struct sched_entity *se) +{ + unsigned int ewma, dequeued, last_ewma_diff; + + if (!sched_feat(UTIL_EST)) + return; + + /* Get current estimate of utilization */ + ewma = READ_ONCE(se->avg.util_est); + + /* + * If the PELT values haven't changed since enqueue time, + * skip the util_est update. + */ + if (ewma & UTIL_AVG_UNCHANGED) + return; + + /* Get utilization at dequeue */ + dequeued = READ_ONCE(se->avg.util_avg); + + /* + * Reset EWMA on utilization increases, the moving average is used only + * to smooth utilization decreases. + */ + if (ewma <= dequeued) { + ewma = dequeued; + goto done; + } + + /* + * Skip update of task's estimated utilization when its members are + * already ~1% close to its last activation value. + */ + last_ewma_diff = ewma - dequeued; + if (last_ewma_diff < UTIL_EST_MARGIN) + goto done; + + /* + * To avoid underestimate of task utilization, skip updates of EWMA if + * we cannot grant that thread got all CPU time it wanted. + */ + if ((dequeued + UTIL_EST_MARGIN) < READ_ONCE(se->avg.runnable_avg)) + goto done; + + /* + * Update Task's estimated utilization + * + * When *p completes an activation we can consolidate another sample + * of the task size. This is done by using this value to update the + * Exponential Weighted Moving Average (EWMA): + * + * ewma(t) = w * task_util(p) + (1-w) * ewma(t-1) + * = w * task_util(p) + ewma(t-1) - w * ewma(t-1) + * = w * (task_util(p) - ewma(t-1)) + ewma(t-1) + * = w * ( -last_ewma_diff ) + ewma(t-1) + * = w * (-last_ewma_diff + ewma(t-1) / w) + * + * Where 'w' is the weight of new samples, which is configured to be + * 0.25, thus making w=1/4 ( >>= UTIL_EST_WEIGHT_SHIFT) + */ + ewma <<= UTIL_EST_WEIGHT_SHIFT; + ewma -= last_ewma_diff; + ewma >>= UTIL_EST_WEIGHT_SHIFT; +done: + ewma |= UTIL_AVG_UNCHANGED; + WRITE_ONCE(se->avg.util_est, ewma); + + trace_sched_util_est_se_tp(se); +} + /* * Optional action to be done while updating the load average */ -#define UPDATE_TG 0x1 -#define SKIP_AGE_LOAD 0x2 -#define DO_ATTACH 0x4 -#define DO_DETACH 0x8 +#define UPDATE_TG 0x01 +#define SKIP_AGE_LOAD 0x02 +#define DO_ATTACH 0x04 +#define DO_DETACH 0x08 +#define UPDATE_UTIL_EST 0x10 /* Update task and its cfs_rq load average */ static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) @@ -4979,6 +5052,9 @@ static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *s if (flags & UPDATE_TG) update_tg_load_avg(cfs_rq); } + + if (flags & UPDATE_UTIL_EST) + util_est_update(se); } /* @@ -5037,11 +5113,6 @@ static inline unsigned long task_util(struct task_struct *p) return READ_ONCE(p->se.avg.util_avg); } -static inline unsigned long task_runnable(struct task_struct *p) -{ - return READ_ONCE(p->se.avg.runnable_avg); -} - static inline unsigned long _task_util_est(struct task_struct *p) { return READ_ONCE(p->se.avg.util_est) & ~UTIL_AVG_UNCHANGED; @@ -5084,88 +5155,6 @@ static inline void util_est_dequeue(struct cfs_rq *cfs_rq, trace_sched_util_est_cfs_tp(cfs_rq); } -#define UTIL_EST_MARGIN (SCHED_CAPACITY_SCALE / 100) - -static inline void util_est_update(struct cfs_rq *cfs_rq, - struct task_struct *p, - bool task_sleep) -{ - unsigned int ewma, dequeued, last_ewma_diff; - - if (!sched_feat(UTIL_EST)) - return; - - /* - * Skip update of task's estimated utilization when the task has not - * yet completed an activation, e.g. being migrated. - */ - if (!task_sleep) - return; - - /* Get current estimate of utilization */ - ewma = READ_ONCE(p->se.avg.util_est); - - /* - * If the PELT values haven't changed since enqueue time, - * skip the util_est update. - */ - if (ewma & UTIL_AVG_UNCHANGED) - return; - - /* Get utilization at dequeue */ - dequeued = task_util(p); - - /* - * Reset EWMA on utilization increases, the moving average is used only - * to smooth utilization decreases. - */ - if (ewma <= dequeued) { - ewma = dequeued; - goto done; - } - - /* - * Skip update of task's estimated utilization when its members are - * already ~1% close to its last activation value. - */ - last_ewma_diff = ewma - dequeued; - if (last_ewma_diff < UTIL_EST_MARGIN) - goto done; - - /* - * To avoid underestimate of task utilization, skip updates of EWMA if - * we cannot grant that thread got all CPU time it wanted. - */ - if ((dequeued + UTIL_EST_MARGIN) < task_runnable(p)) - goto done; - - - /* - * Update Task's estimated utilization - * - * When *p completes an activation we can consolidate another sample - * of the task size. This is done by using this value to update the - * Exponential Weighted Moving Average (EWMA): - * - * ewma(t) = w * task_util(p) + (1-w) * ewma(t-1) - * = w * task_util(p) + ewma(t-1) - w * ewma(t-1) - * = w * (task_util(p) - ewma(t-1)) + ewma(t-1) - * = w * ( -last_ewma_diff ) + ewma(t-1) - * = w * (-last_ewma_diff + ewma(t-1) / w) - * - * Where 'w' is the weight of new samples, which is configured to be - * 0.25, thus making w=1/4 ( >>= UTIL_EST_WEIGHT_SHIFT) - */ - ewma <<= UTIL_EST_WEIGHT_SHIFT; - ewma -= last_ewma_diff; - ewma >>= UTIL_EST_WEIGHT_SHIFT; -done: - ewma |= UTIL_AVG_UNCHANGED; - WRITE_ONCE(p->se.avg.util_est, ewma); - - trace_sched_util_est_se_tp(&p->se); -} - static inline unsigned long get_actual_cpu_capacity(int cpu) { unsigned long capacity = arch_scale_cpu_capacity(cpu); @@ -5618,7 +5607,7 @@ static bool dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) { bool sleep = flags & DEQUEUE_SLEEP; - int action = UPDATE_TG; + int action = 0; update_curr(cfs_rq); clear_buddies(cfs_rq, se); @@ -5638,15 +5627,23 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) if (sched_feat(DELAY_DEQUEUE) && delay && !entity_eligible(cfs_rq, se)) { - update_load_avg(cfs_rq, se, 0); + if (entity_is_task(se)) + action |= UPDATE_UTIL_EST; + update_load_avg(cfs_rq, se, action); update_entity_lag(cfs_rq, se); set_delayed(se); return false; } } - if (entity_is_task(se) && task_on_rq_migrating(task_of(se))) - action |= DO_DETACH; + action = UPDATE_TG; + if (entity_is_task(se)) { + if (task_on_rq_migrating(task_of(se))) + action |= DO_DETACH; + + if (sleep && !(flags & DEQUEUE_DELAYED)) + action |= UPDATE_UTIL_EST; + } /* * When dequeuing a sched_entity, we must: @@ -7409,7 +7406,6 @@ static bool dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (!p->se.sched_delayed) util_est_dequeue(&rq->cfs, p); - util_est_update(&rq->cfs, p, flags & DEQUEUE_SLEEP); if (dequeue_entities(rq, &p->se, flags) < 0) return false; -- cgit v1.2.3 From 5bc6ab2d42e545f816def21cfcdb4ba35cc74bf6 Mon Sep 17 00:00:00 2001 From: Shrikanth Hegde Date: Fri, 15 May 2026 22:54:54 +0530 Subject: sched: Simplify ifdeffery around cpu_smt_mask Now, that cpu_smt_mask is defined as cpumask_of(cpu) for CONFIG_SCHED_SMT=n, it is possible to get rid of the ifdeffery. Effectively, - This makes sched_smt_present is defined always - cpumask_weight(cpumask_of(cpu)) == 1. So sched_smt_present_inc/dec will never enable the sched_smt_present. Which is expected. - Paths that were compile-time eliminated become runtime guarded using static keys. - Defines set_idle_cores, test_idle_cores, etc which could likely benefit the CONFIG_SCHED_SMT=n systems to use the same optimizations within the LLC at wakeups. - This will expose sched_smt_present symbol for CONFIG_SCHED_SMT=n. Likely not a concern. - There is a bloat of code CONFIG_SCHED_SMT=n. (NR_CPUS=2048) add/remove: 24/18 grow/shrink: 26/28 up/down: 6396/-3188 (3208) Total: Before=30629880, After=30633088, chg +0.01% - No code bloat for CONFIG_SCHED_SMT=y, which is expected. - Add comments around stop_core_cpuslocked on why ifdefs are not removed. - This leaves the remaining uses of CONFIG_SCHED_SMT mainly for topology building bits which has a policy based decision. Signed-off-by: Shrikanth Hegde Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Phil Auld Reviewed-by: Valentin Schneider Acked-by: Tejun Heo Tested-by: K Prateek Nayak Link: https://patch.msgid.link/20260515172456.542799-3-sshegde@linux.ibm.com --- kernel/sched/core.c | 6 ------ kernel/sched/ext_idle.c | 6 ------ kernel/sched/fair.c | 35 ----------------------------------- kernel/sched/sched.h | 6 ------ kernel/sched/topology.c | 2 -- kernel/stop_machine.c | 5 +++++ kernel/workqueue.c | 4 ---- 7 files changed, 5 insertions(+), 59 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index b905805bbcbe..3ae5f19c1b7e 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -8612,18 +8612,14 @@ static void cpuset_cpu_inactive(unsigned int cpu) static inline void sched_smt_present_inc(int cpu) { -#ifdef CONFIG_SCHED_SMT if (cpumask_weight(cpu_smt_mask(cpu)) == 2) static_branch_inc_cpuslocked(&sched_smt_present); -#endif } static inline void sched_smt_present_dec(int cpu) { -#ifdef CONFIG_SCHED_SMT if (cpumask_weight(cpu_smt_mask(cpu)) == 2) static_branch_dec_cpuslocked(&sched_smt_present); -#endif } int sched_cpu_activate(unsigned int cpu) @@ -8711,9 +8707,7 @@ int sched_cpu_deactivate(unsigned int cpu) */ sched_smt_present_dec(cpu); -#ifdef CONFIG_SCHED_SMT sched_core_cpu_deactivate(cpu); -#endif if (!sched_smp_initialized) return 0; diff --git a/kernel/sched/ext_idle.c b/kernel/sched/ext_idle.c index 7468560a6d80..2bcf58e99c9b 100644 --- a/kernel/sched/ext_idle.c +++ b/kernel/sched/ext_idle.c @@ -79,7 +79,6 @@ static bool scx_idle_test_and_clear_cpu(int cpu) int node = scx_cpu_node_if_enabled(cpu); struct cpumask *idle_cpus = idle_cpumask(node)->cpu; -#ifdef CONFIG_SCHED_SMT /* * SMT mask should be cleared whether we can claim @cpu or not. The SMT * cluster is not wholly idle either way. This also prevents @@ -104,7 +103,6 @@ static bool scx_idle_test_and_clear_cpu(int cpu) else if (cpumask_test_cpu(cpu, idle_smts)) __cpumask_clear_cpu(cpu, idle_smts); } -#endif return cpumask_test_and_clear_cpu(cpu, idle_cpus); } @@ -622,7 +620,6 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, goto out_unlock; } -#ifdef CONFIG_SCHED_SMT /* * Use @prev_cpu's sibling if it's idle. */ @@ -634,7 +631,6 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, goto out_unlock; } } -#endif /* * Search for any idle CPU in the same LLC domain. @@ -714,7 +710,6 @@ static void update_builtin_idle(int cpu, bool idle) assign_cpu(cpu, idle_cpus, idle); -#ifdef CONFIG_SCHED_SMT if (sched_smt_active()) { const struct cpumask *smt = cpu_smt_mask(cpu); struct cpumask *idle_smts = idle_cpumask(node)->smt; @@ -731,7 +726,6 @@ static void update_builtin_idle(int cpu, bool idle) cpumask_andnot(idle_smts, idle_smts, smt); } } -#endif } /* diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 09d3acd2d2bc..233bd2ebbb73 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1555,7 +1555,6 @@ update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) static inline bool is_core_idle(int cpu) { -#ifdef CONFIG_SCHED_SMT int sibling; for_each_cpu(sibling, cpu_smt_mask(cpu)) { @@ -1565,7 +1564,6 @@ static inline bool is_core_idle(int cpu) if (!idle_cpu(sibling)) return false; } -#endif return true; } @@ -2248,7 +2246,6 @@ numa_type numa_classify(unsigned int imbalance_pct, return node_fully_busy; } -#ifdef CONFIG_SCHED_SMT /* Forward declarations of select_idle_sibling helpers */ static inline bool test_idle_cores(int cpu); static inline int numa_idle_core(int idle_core, int cpu) @@ -2266,12 +2263,6 @@ static inline int numa_idle_core(int idle_core, int cpu) return idle_core; } -#else /* !CONFIG_SCHED_SMT: */ -static inline int numa_idle_core(int idle_core, int cpu) -{ - return idle_core; -} -#endif /* !CONFIG_SCHED_SMT */ /* * Gather all necessary information to make NUMA balancing placement @@ -7778,7 +7769,6 @@ static inline int __select_idle_cpu(int cpu, struct task_struct *p) return -1; } -#ifdef CONFIG_SCHED_SMT DEFINE_STATIC_KEY_FALSE(sched_smt_present); EXPORT_SYMBOL_GPL(sched_smt_present); @@ -7888,29 +7878,6 @@ static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int t return -1; } -#else /* !CONFIG_SCHED_SMT: */ - -static inline void set_idle_cores(int cpu, int val) -{ -} - -static inline bool test_idle_cores(int cpu) -{ - return false; -} - -static inline int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu) -{ - return __select_idle_cpu(core, p); -} - -static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target) -{ - return -1; -} - -#endif /* !CONFIG_SCHED_SMT */ - /* * Scan the LLC domain for idle CPUs; this is dynamically regulated by * comparing the average scan cost (tracked in sd->avg_scan_cost) against the @@ -12002,9 +11969,7 @@ static int should_we_balance(struct lb_env *env) * idle has been found, then its not needed to check other * SMT siblings for idleness: */ -#ifdef CONFIG_SCHED_SMT cpumask_andnot(swb_cpus, swb_cpus, cpu_smt_mask(cpu)); -#endif continue; } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 9f63b15d309d..e476623a0c2a 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1667,7 +1667,6 @@ do { \ flags = _raw_spin_rq_lock_irqsave(rq); \ } while (0) -#ifdef CONFIG_SCHED_SMT extern void __update_idle_core(struct rq *rq); static inline void update_idle_core(struct rq *rq) @@ -1676,12 +1675,7 @@ static inline void update_idle_core(struct rq *rq) __update_idle_core(rq); } -#else /* !CONFIG_SCHED_SMT: */ -static inline void update_idle_core(struct rq *rq) { } -#endif /* !CONFIG_SCHED_SMT */ - #ifdef CONFIG_FAIR_GROUP_SCHED - static inline struct task_struct *task_of(struct sched_entity *se) { WARN_ON_ONCE(!entity_is_task(se)); diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 5847b83d9d55..a1f46e3f4ede 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -1310,9 +1310,7 @@ static void init_sched_groups_capacity(int cpu, struct sched_domain *sd) cpumask_copy(mask, sched_group_span(sg)); for_each_cpu(cpu, mask) { cores++; -#ifdef CONFIG_SCHED_SMT cpumask_andnot(mask, mask, cpu_smt_mask(cpu)); -#endif } sg->cores = cores; diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c index 3fe6b0c99f3d..773d8e9ae30c 100644 --- a/kernel/stop_machine.c +++ b/kernel/stop_machine.c @@ -633,6 +633,11 @@ int stop_machine(cpu_stop_fn_t fn, void *data, const struct cpumask *cpus) EXPORT_SYMBOL_GPL(stop_machine); #ifdef CONFIG_SCHED_SMT +/* + * INTEL_IFS is the only user of this API. That selftest can + * only be compiled if SMP=y. On x86 it selects SCHED_SMT. + * Keep the ifdefs for now. + */ int stop_core_cpuslocked(unsigned int cpu, cpu_stop_fn_t fn, void *data) { const struct cpumask *smt_mask = cpu_smt_mask(cpu); diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 5f747f241a5f..99ef412f02a6 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -8187,11 +8187,7 @@ static bool __init cpus_dont_share(int cpu0, int cpu1) static bool __init cpus_share_smt(int cpu0, int cpu1) { -#ifdef CONFIG_SCHED_SMT return cpumask_test_cpu(cpu0, cpu_smt_mask(cpu1)); -#else - return false; -#endif } static bool __init cpus_share_numa(int cpu0, int cpu1) -- cgit v1.2.3 From 3dbb362f90f3a8300ed9209d3278e30f8dbfb780 Mon Sep 17 00:00:00 2001 From: Shrikanth Hegde Date: Fri, 15 May 2026 22:54:55 +0530 Subject: sched/fair: Add sched_smt_active check for fastpaths For fastpaths such as wakeup and load balance even minimal code additions can add up. is_core_idle is accessed during load balance. Other callsites of is_core_idle make sched_smt_active() check first. Make the same check in should_we_balance. Rest of access to cpu_smt_mask isn't in fastpath. Note: Remove the stale comment above is_core_idle. Enqueue methods of fair aren't close to it anymore. Suggested-by: K Prateek Nayak Signed-off-by: Shrikanth Hegde Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Valentin Schneider Link: https://patch.msgid.link/20260515172456.542799-4-sshegde@linux.ibm.com --- kernel/sched/fair.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 233bd2ebbb73..14bd31b17c71 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1549,10 +1549,7 @@ update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) se->exec_start = rq_clock_task(rq_of(cfs_rq)); } -/************************************************** - * Scheduling class queueing methods: - */ - +/* Check sched_smt_active before calling this to avoid overheads in fastpaths */ static inline bool is_core_idle(int cpu) { int sibling; @@ -11961,7 +11958,9 @@ static int should_we_balance(struct lb_env *env) * balancing cores, but remember the first idle SMT CPU for * later consideration. Find CPU on an idle core first. */ - if (!(env->sd->flags & SD_SHARE_CPUCAPACITY) && !is_core_idle(cpu)) { + if (sched_smt_active() && + !(env->sd->flags & SD_SHARE_CPUCAPACITY) && + !is_core_idle(cpu)) { if (idle_smt == -1) idle_smt = cpu; /* -- cgit v1.2.3 From acbdbab75ff4b1b87ab3c3d2b6ca86948f472189 Mon Sep 17 00:00:00 2001 From: Shrikanth Hegde Date: Fri, 15 May 2026 22:54:56 +0530 Subject: sched: Unify SMT active check via sched_smt_active() There is a use of sched_smt_active() and explicit use of sched_smt_present. Remove the explicit usage for better code maintenance and readability. Note that this differs slightly for update_idle_core. It used to call static_branch_unlikely earlier and now it will call static_branch_likely. Signed-off-by: Shrikanth Hegde Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Valentin Schneider Link: https://patch.msgid.link/20260515172456.542799-5-sshegde@linux.ibm.com --- kernel/sched/core_sched.c | 2 +- kernel/sched/fair.c | 2 +- kernel/sched/sched.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/core_sched.c b/kernel/sched/core_sched.c index 73b6b2426911..43e0bde3038e 100644 --- a/kernel/sched/core_sched.c +++ b/kernel/sched/core_sched.c @@ -136,7 +136,7 @@ int sched_core_share_pid(unsigned int cmd, pid_t pid, enum pid_type type, struct pid *grp; int err = 0; - if (!static_branch_likely(&sched_smt_present)) + if (!sched_smt_active()) return -ENODEV; BUILD_BUG_ON(PR_SCHED_CORE_SCOPE_THREAD != PIDTYPE_PID); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 14bd31b17c71..bcaadddf8624 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -2247,7 +2247,7 @@ numa_type numa_classify(unsigned int imbalance_pct, static inline bool test_idle_cores(int cpu); static inline int numa_idle_core(int idle_core, int cpu) { - if (!static_branch_likely(&sched_smt_present) || + if (!sched_smt_active() || idle_core >= 0 || !test_idle_cores(cpu)) return idle_core; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index e476623a0c2a..ffe77b2b6296 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1671,7 +1671,7 @@ extern void __update_idle_core(struct rq *rq); static inline void update_idle_core(struct rq *rq) { - if (static_branch_unlikely(&sched_smt_present)) + if (sched_smt_active()) __update_idle_core(rq); } -- cgit v1.2.3 From c9d93a73ce871ca32caf9308562501290b64b955 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Sat, 9 May 2026 20:07:25 +0200 Subject: sched/fair: Drop redundant RCU read lock in NOHZ kick path nohz_balancer_kick() is reached from sched_balance_trigger(), which is called from sched_tick(). sched_tick() runs with IRQs disabled, so the additional rcu_read_lock/unlock() used around sched_domain accesses in this path is redundant. Rely on the existing IRQ-disabled context (and the rcu_dereference_all() checking) instead. The same applies to set_cpu_sd_state_idle(), called from the idle entry path with IRQs disabled, and to set_cpu_sd_state_busy(), reachable via nohz_balance_exit_idle() from two contexts: nohz_balancer_kick() (IRQs disabled, as above) and sched_cpu_deactivate() (the CPUHP_AP_ACTIVE teardown, which runs under cpus_write_lock(), so it cannot race with sched-domain rebuilds). In both cases the rcu_dereference_all() validation is sufficient. No functional change intended. Suggested-by: K Prateek Nayak Signed-off-by: Andrea Righi Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: K Prateek Nayak Reviewed-by: Vincent Guittot Link: https://patch.msgid.link/20260509180955.1840064-2-arighi@nvidia.com --- kernel/sched/fair.c | 38 +++++++++++--------------------------- 1 file changed, 11 insertions(+), 27 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index bcaadddf8624..03f63b094ff9 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -12715,8 +12715,6 @@ static void nohz_balancer_kick(struct rq *rq) goto out; } - rcu_read_lock(); - sd = rcu_dereference_all(rq->sd); if (sd) { /* @@ -12724,8 +12722,8 @@ static void nohz_balancer_kick(struct rq *rq) * capacity, kick the ILB to see if there's a better CPU to run on: */ if (rq->cfs.h_nr_runnable >= 1 && check_cpu_capacity(rq, sd)) { - flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; - goto unlock; + flags |= NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; + goto out; } } @@ -12741,8 +12739,8 @@ static void nohz_balancer_kick(struct rq *rq) */ for_each_cpu_and(i, sched_domain_span(sd), nohz.idle_cpus_mask) { if (sched_asym(sd, i, cpu)) { - flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; - goto unlock; + flags |= NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; + goto out; } } } @@ -12753,10 +12751,8 @@ static void nohz_balancer_kick(struct rq *rq) * When ASYM_CPUCAPACITY; see if there's a higher capacity CPU * to run the misfit task on. */ - if (check_misfit_status(rq)) { - flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; - goto unlock; - } + if (check_misfit_status(rq)) + flags |= NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; /* * For asymmetric systems, we do not want to nicely balance @@ -12765,7 +12761,7 @@ static void nohz_balancer_kick(struct rq *rq) * * Skip the LLC logic because it's not relevant in that case. */ - goto unlock; + goto out; } sds = rcu_dereference_all(per_cpu(sd_llc_shared, cpu)); @@ -12780,13 +12776,9 @@ static void nohz_balancer_kick(struct rq *rq) * like this LLC domain has tasks we could move. */ nr_busy = atomic_read(&sds->nr_busy_cpus); - if (nr_busy > 1) { - flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; - goto unlock; - } + if (nr_busy > 1) + flags |= NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; } -unlock: - rcu_read_unlock(); out: if (READ_ONCE(nohz.needs_update)) flags |= NOHZ_NEXT_KICK; @@ -12798,17 +12790,13 @@ out: static void set_cpu_sd_state_busy(int cpu) { struct sched_domain *sd; - - rcu_read_lock(); sd = rcu_dereference_all(per_cpu(sd_llc, cpu)); if (!sd || !sd->nohz_idle) - goto unlock; + return; sd->nohz_idle = 0; atomic_inc(&sd->shared->nr_busy_cpus); -unlock: - rcu_read_unlock(); } void nohz_balance_exit_idle(struct rq *rq) @@ -12827,17 +12815,13 @@ void nohz_balance_exit_idle(struct rq *rq) static void set_cpu_sd_state_idle(int cpu) { struct sched_domain *sd; - - rcu_read_lock(); sd = rcu_dereference_all(per_cpu(sd_llc, cpu)); if (!sd || sd->nohz_idle) - goto unlock; + return; sd->nohz_idle = 1; atomic_dec(&sd->shared->nr_busy_cpus); -unlock: - rcu_read_unlock(); } /* -- cgit v1.2.3 From fdfe5a8cd8731dd81840f26abfb6527edd27b0cb Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Sat, 16 May 2026 07:58:50 +0200 Subject: sched/fair: Attach sched_domain_shared to sd_asym_cpucapacity On asymmetric CPU capacity systems, the wakeup path uses select_idle_capacity(), which scans the span of sd_asym_cpucapacity rather than sd_llc. The has_idle_cores hint however lives on sd_llc->shared, so the wakeup-time read of has_idle_cores operates on an LLC-scoped blob while the actual scan/decision spans the asym domain; nr_busy_cpus also lives in the same shared sched_domain data, but it's never used in the asym CPU capacity scenario. Therefore, move the sched_domain_shared object to sd_asym_cpucapacity whenever the CPU has a SD_ASYM_CPUCAPACITY_FULL ancestor and that ancestor is non-overlapping (i.e., not built from SD_NUMA). In that case the scope of has_idle_cores matches the scope of the wakeup scan. Fall back to attaching the shared object to sd_llc in three cases: 1) plain symmetric systems (no SD_ASYM_CPUCAPACITY_FULL anywhere); 2) CPUs in an exclusive cpuset that carves out a symmetric capacity island: has_asym is system-wide but those CPUs have no SD_ASYM_CPUCAPACITY_FULL ancestor in their hierarchy and follow the symmetric LLC path in select_idle_sibling(); 3) exotic topologies where SD_ASYM_CPUCAPACITY_FULL lands on an SD_NUMA-built domain. init_sched_domain_shared() keys the shared blob off cpumask_first(span), which on overlapping NUMA domains would alias unrelated spans onto the same blob. Keep the shared object on the LLC there; select_idle_capacity() gracefully skips the has_idle_cores preference when sd->shared is NULL. While at it, also rename the per-CPU sd_llc_shared to sd_balance_shared, as it is no longer strictly tied to the LLC. Co-developed-by: Andrea Righi Signed-off-by: Andrea Righi Signed-off-by: K Prateek Nayak Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Shrikanth Hegde Acked-by: Vincent Guittot Link: https://patch.msgid.link/20260516055850.1345932-1-arighi@nvidia.com --- kernel/sched/fair.c | 22 +++++++----- kernel/sched/sched.h | 2 +- kernel/sched/topology.c | 95 ++++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 97 insertions(+), 22 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 03f63b094ff9..2637a6fe9a87 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -7773,7 +7773,7 @@ static inline void set_idle_cores(int cpu, int val) { struct sched_domain_shared *sds; - sds = rcu_dereference_all(per_cpu(sd_llc_shared, cpu)); + sds = rcu_dereference_all(per_cpu(sd_balance_shared, cpu)); if (sds) WRITE_ONCE(sds->has_idle_cores, val); } @@ -7782,7 +7782,7 @@ static inline bool test_idle_cores(int cpu) { struct sched_domain_shared *sds; - sds = rcu_dereference_all(per_cpu(sd_llc_shared, cpu)); + sds = rcu_dereference_all(per_cpu(sd_balance_shared, cpu)); if (sds) return READ_ONCE(sds->has_idle_cores); @@ -7791,7 +7791,7 @@ static inline bool test_idle_cores(int cpu) /* * Scans the local SMT mask to see if the entire core is idle, and records this - * information in sd_llc_shared->has_idle_cores. + * information in sd_balance_shared->has_idle_cores. * * Since SMT siblings share all cache levels, inspecting this limited remote * state should be fairly cheap. @@ -7821,7 +7821,8 @@ unlock: /* * Scan the entire LLC domain for idle cores; this dynamically switches off if * there are no idle cores left in the system; tracked through - * sd_llc->shared->has_idle_cores and enabled through update_idle_core() above. + * sd_balance_shared->has_idle_cores and enabled through update_idle_core() + * above. */ static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu) { @@ -7885,7 +7886,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask); int i, cpu, idle_cpu = -1, nr = INT_MAX; - if (sched_feat(SIS_UTIL)) { + if (sched_feat(SIS_UTIL) && sd->shared) { /* * Increment because !--nr is the condition to stop scan. * @@ -12764,7 +12765,7 @@ static void nohz_balancer_kick(struct rq *rq) goto out; } - sds = rcu_dereference_all(per_cpu(sd_llc_shared, cpu)); + sds = rcu_dereference_all(per_cpu(sd_balance_shared, cpu)); if (sds) { /* * If there is an imbalance between LLC domains (IOW we could @@ -12792,7 +12793,11 @@ static void set_cpu_sd_state_busy(int cpu) struct sched_domain *sd; sd = rcu_dereference_all(per_cpu(sd_llc, cpu)); - if (!sd || !sd->nohz_idle) + /* + * sd->nohz_idle only pairs with nr_busy_cpus on sd->shared; if this + * domain has no shared object there is nothing to clear or account. + */ + if (!sd || !sd->shared || !sd->nohz_idle) return; sd->nohz_idle = 0; @@ -12817,7 +12822,8 @@ static void set_cpu_sd_state_idle(int cpu) struct sched_domain *sd; sd = rcu_dereference_all(per_cpu(sd_llc, cpu)); - if (!sd || sd->nohz_idle) + /* See set_cpu_sd_state_busy(): nohz_idle is only used with sd->shared. */ + if (!sd || !sd->shared || sd->nohz_idle) return; sd->nohz_idle = 1; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index ffe77b2b6296..bfb4b47c021b 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2164,7 +2164,7 @@ DECLARE_PER_CPU(struct sched_domain __rcu *, sd_llc); DECLARE_PER_CPU(int, sd_llc_size); DECLARE_PER_CPU(int, sd_llc_id); DECLARE_PER_CPU(int, sd_share_id); -DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared); +DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_balance_shared); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_numa); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity); diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index a1f46e3f4ede..f96d50131495 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -665,7 +665,7 @@ DEFINE_PER_CPU(struct sched_domain __rcu *, sd_llc); DEFINE_PER_CPU(int, sd_llc_size); DEFINE_PER_CPU(int, sd_llc_id); DEFINE_PER_CPU(int, sd_share_id); -DEFINE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared); +DEFINE_PER_CPU(struct sched_domain_shared __rcu *, sd_balance_shared); DEFINE_PER_CPU(struct sched_domain __rcu *, sd_numa); DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing); DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity); @@ -680,20 +680,38 @@ static void update_top_cache_domain(int cpu) int id = cpu; int size = 1; + sd = lowest_flag_domain(cpu, SD_ASYM_CPUCAPACITY_FULL); + /* + * The shared object is attached to sd_asym_cpucapacity only when the + * asym domain is non-overlapping (i.e., not built from SD_NUMA). + * On overlapping (NUMA) asym domains we fall back to letting the + * SD_SHARE_LLC path own the shared object, so sd->shared may be NULL + * here. + */ + if (sd && sd->shared) + sds = sd->shared; + + rcu_assign_pointer(per_cpu(sd_asym_cpucapacity, cpu), sd); + sd = highest_flag_domain(cpu, SD_SHARE_LLC); if (sd) { id = cpumask_first(sched_domain_span(sd)); size = cpumask_weight(sched_domain_span(sd)); - /* If sd_llc exists, sd_llc_shared should exist too. */ - WARN_ON_ONCE(!sd->shared); - sds = sd->shared; + /* + * If sd_asym_cpucapacity didn't claim the shared object, + * sd_llc must have one linked. + */ + if (!sds) { + WARN_ON_ONCE(!sd->shared); + sds = sd->shared; + } } rcu_assign_pointer(per_cpu(sd_llc, cpu), sd); per_cpu(sd_llc_size, cpu) = size; per_cpu(sd_llc_id, cpu) = id; - rcu_assign_pointer(per_cpu(sd_llc_shared, cpu), sds); + rcu_assign_pointer(per_cpu(sd_balance_shared, cpu), sds); sd = lowest_flag_domain(cpu, SD_CLUSTER); if (sd) @@ -711,9 +729,6 @@ static void update_top_cache_domain(int cpu) sd = highest_flag_domain(cpu, SD_ASYM_PACKING); rcu_assign_pointer(per_cpu(sd_asym_packing, cpu), sd); - - sd = lowest_flag_domain(cpu, SD_ASYM_CPUCAPACITY_FULL); - rcu_assign_pointer(per_cpu(sd_asym_cpucapacity, cpu), sd); } /* @@ -2648,6 +2663,54 @@ static void adjust_numa_imbalance(struct sched_domain *sd_llc) } } +static void init_sched_domain_shared(struct s_data *d, struct sched_domain *sd) +{ + int sd_id = cpumask_first(sched_domain_span(sd)); + + sd->shared = *per_cpu_ptr(d->sds, sd_id); + /* + * nr_busy_cpus is consumed only by the NOHZ kick path via + * sd_balance_shared; on the asym-capacity path it is initialized but + * never read. + */ + atomic_set(&sd->shared->nr_busy_cpus, sd->span_weight); + atomic_inc(&sd->shared->ref); +} + +/* + * For asymmetric CPU capacity, attach sched_domain_shared on the innermost + * SD_ASYM_CPUCAPACITY_FULL ancestor of @cpu's base domain when that ancestor is + * not an overlapping NUMA-built domain (then LLC should claim shared). + * + * A CPU may lack any FULL ancestor (e.g., exclusive cpuset symmetric island), + * then LLC must claim shared instead. + * + * Note: SD_ASYM_CPUCAPACITY_FULL is only set when all CPU capacity values + * are present in the domain span, so the asym domain we attach to cannot + * degenerate into a single-capacity group. The relevant edge cases are instead + * covered by the caveats above. + * + * Return true if this CPU's asym path claimed sd->shared, false otherwise. + */ +static bool claim_asym_sched_domain_shared(struct s_data *d, int cpu) +{ + struct sched_domain *sd = *per_cpu_ptr(d->sd, cpu); + struct sched_domain *sd_asym; + + if (!sd) + return false; + + sd_asym = sd; + while (sd_asym && !(sd_asym->flags & SD_ASYM_CPUCAPACITY_FULL)) + sd_asym = sd_asym->parent; + + if (!sd_asym || (sd_asym->flags & SD_NUMA)) + return false; + + init_sched_domain_shared(d, sd_asym); + return true; +} + /* * Build sched domains for a given set of CPUs and attach the sched domains * to the individual CPUs @@ -2706,20 +2769,26 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att } for_each_cpu(i, cpu_map) { + bool asym_claimed = false; + sd = *per_cpu_ptr(d.sd, i); if (!sd) continue; + if (has_asym) + asym_claimed = claim_asym_sched_domain_shared(&d, i); + /* First, find the topmost SD_SHARE_LLC domain */ while (sd->parent && (sd->parent->flags & SD_SHARE_LLC)) sd = sd->parent; if (sd->flags & SD_SHARE_LLC) { - int sd_id = cpumask_first(sched_domain_span(sd)); - - sd->shared = *per_cpu_ptr(d.sds, sd_id); - atomic_set(&sd->shared->nr_busy_cpus, sd->span_weight); - atomic_inc(&sd->shared->ref); + /* + * Initialize the sd->shared for SD_SHARE_LLC unless + * the asym path above already claimed it. + */ + if (!asym_claimed) + init_sched_domain_shared(&d, sd); /* * In presence of higher domains, adjust the -- cgit v1.2.3 From 25a32e400a14009601c0a727643057f5515152df Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Mon, 11 May 2026 16:25:02 +0200 Subject: sched/fair: Prefer fully-idle SMT cores in asym-capacity idle selection On systems with asymmetric CPU capacity (e.g., ACPI/CPPC reporting different per-core frequencies), the wakeup path uses select_idle_capacity() and prioritizes idle CPUs with higher capacity for better task placement. However, when those CPUs belong to SMT cores, their effective capacity can be much lower than the nominal capacity when the sibling thread is busy: SMT siblings compete for shared resources, so a "high capacity" CPU that is idle but whose sibling is busy does not deliver its full capacity. This effective capacity reduction cannot be modeled by the static capacity value alone. Introduce SMT awareness in the asym-capacity idle selection policy: when SMT is active, always prefer fully-idle SMT cores over partially-idle ones. Prioritizing fully-idle SMT cores yields better task placement because the effective capacity of partially-idle SMT cores is reduced; always preferring them when available leads to more accurate capacity usage on task wakeup. On an SMT system with asymmetric CPU capacities (NVIDIA Vera Rubin), SMT-aware idle selection has been shown to improve throughput by around 15-18% over NO_ASYM mainline and by around 60% over ASYM mainline, for CPU-bound workloads (NVBLAS) running an amount of tasks equal to the amount of SMT cores. Reported-by: Felix Abecassis Signed-off-by: Andrea Righi Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Vincent Guittot Reviewed-by: K Prateek Nayak Link: https://patch.msgid.link/20260511142502.3873984-1-arighi@nvidia.com --- kernel/sched/fair.c | 120 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 114 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 2637a6fe9a87..8854d4d980b0 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -7950,6 +7950,54 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool return idle_cpu; } +/* + * Idle-capacity scan converts util_fits_cpu() outcomes into preference ranks, + * where lower values indicate a better fit - see select_idle_capacity(). + * + * A CPU that both fits the task and sits on a fully-idle SMT core is returned + * immediately and is never assigned one of these ranks. On !SMT every CPU is + * its own "core", so the early return covers all fits-and-idle cases and the + * core-tier ranks below become unreachable. + * + * Rank Val Tier Meaning + * ------------------------------ --- ------ --------------------------- + * ASYM_IDLE_UCLAMP_MISFIT -4 core Idle core; capacity fits + * util but uclamp_min misses. + * ASYM_IDLE_COMPLETE_MISFIT -3 core Idle core; capacity does + * not fit. Still beats every + * thread-tier rank: a busy + * sibling cuts effective + * capacity more than a + * misfit hurts a quiet core. + * ASYM_IDLE_THREAD_FITS -2 thread Busy SMT sibling; capacity + * fits util + uclamp. + * ASYM_IDLE_THREAD_UCLAMP_MISFIT -1 thread Busy SMT sibling; capacity + * fits but uclamp_min misses + * (native util_fits_cpu() + * return value). + * ASYM_IDLE_THREAD_MISFIT 0 thread Busy SMT sibling; capacity + * does not fit. + * + * ASYM_IDLE_CORE_BIAS (-3) is an offset, not a state. On an idle core, + * fits += ASYM_IDLE_CORE_BIAS rebases thread-tier ranks into the core tier: + * + * ASYM_IDLE_THREAD_UCLAMP_MISFIT (-1) + BIAS -> ASYM_IDLE_UCLAMP_MISFIT (-4) + * ASYM_IDLE_THREAD_MISFIT (0) + BIAS -> ASYM_IDLE_COMPLETE_MISFIT (-3) + * + * ASYM_IDLE_THREAD_FITS (-2) is never rebased because a fully-fitting idle-core + * candidate early-returns from select_idle_capacity(). + */ +enum asym_fits_state { + ASYM_IDLE_UCLAMP_MISFIT = -4, + ASYM_IDLE_COMPLETE_MISFIT, + ASYM_IDLE_THREAD_FITS, + ASYM_IDLE_THREAD_UCLAMP_MISFIT, + ASYM_IDLE_THREAD_MISFIT, + + /* util_fits_cpu() bias for idle core */ + ASYM_IDLE_CORE_BIAS = -3, +}; + /* * Scan the asym_capacity domain for idle CPUs; pick the first idle one on which * the task fits. If no CPU is big enough, but there are idle ones, try to @@ -7958,8 +8006,14 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool static int select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) { + /* + * On !SMT systems, has_idle_core is always false and preferred_core + * is always true (CPU == core), so the SMT preference logic below + * collapses to the plain capacity scan. + */ + bool has_idle_core = sched_smt_active() && test_idle_cores(target); unsigned long task_util, util_min, util_max, best_cap = 0; - int fits, best_fits = 0; + int fits, best_fits = ASYM_IDLE_THREAD_MISFIT; int cpu, best_cpu = -1; struct cpumask *cpus; @@ -7971,6 +8025,7 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) util_max = uclamp_eff_value(p, UCLAMP_MAX); for_each_cpu_wrap(cpu, cpus, target) { + bool preferred_core = !has_idle_core || is_core_idle(cpu); unsigned long cpu_cap = capacity_of(cpu); if (!choose_idle_cpu(cpu, p)) @@ -7978,8 +8033,14 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) fits = util_fits_cpu(task_util, util_min, util_max, cpu); - /* This CPU fits with all requirements */ - if (fits > 0) + /* + * Perfect fit: capacity satisfies util + uclamp and the CPU + * sits on a fully-idle SMT core, this is a !SMT system, or + * there is no idle core to find. + * Short-circuit the rank-based selection and return + * immediately. + */ + if (fits > 0 && preferred_core) return cpu; /* * Only the min performance hint (i.e. uclamp_min) doesn't fit. @@ -7987,9 +8048,33 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) */ else if (fits < 0) cpu_cap = get_actual_cpu_capacity(cpu); + /* + * fits > 0 implies we are not on a preferred core, but the util + * fits CPU capacity. Set fits to ASYM_IDLE_THREAD_FITS + * so the effective range becomes + * [ASYM_IDLE_THREAD_FITS, ASYM_IDLE_THREAD_MISFIT], where: + * ASYM_IDLE_THREAD_MISFIT - does not fit + * ASYM_IDLE_THREAD_UCLAMP_MISFIT - fits with the exception of UCLAMP_MIN + * ASYM_IDLE_THREAD_FITS - fits with the exception of preferred_core + */ + else if (fits > 0) + fits = ASYM_IDLE_THREAD_FITS; /* - * First, select CPU which fits better (-1 being better than 0). + * If we are on a preferred core, translate the range of fits + * of [ASYM_IDLE_THREAD_UCLAMP_MISFIT, ASYM_IDLE_THREAD_MISFIT] to + * [ASYM_IDLE_UCLAMP_MISFIT, ASYM_IDLE_COMPLETE_MISFIT]. + * This ensures that an idle core is always given priority over + * (partially) busy core. + * + * A fully fitting idle core would have returned early and hence + * fits > 0 for preferred_core need not be dealt with. + */ + if (preferred_core) + fits += ASYM_IDLE_CORE_BIAS; + + /* + * First, select CPU which fits better (lower is more preferred). * Then, select the one with best capacity at same level. */ if ((fits < best_fits) || @@ -8000,6 +8085,19 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) } } + /* + * A value in the [ASYM_IDLE_UCLAMP_MISFIT, ASYM_IDLE_COMPLETE_MISFIT] + * range means the chosen CPU is in a fully idle SMT core. Values above + * ASYM_IDLE_COMPLETE_MISFIT mean we never ranked such a CPU best. + * + * The asym-capacity wakeup path returns from select_idle_sibling() + * after this function and never runs select_idle_cpu(), so the usual + * select_idle_cpu() tail that clears idle cores must live here when the + * idle-core preference did not win. + */ + if (has_idle_core && best_fits > ASYM_IDLE_COMPLETE_MISFIT) + set_idle_cores(target, false); + return best_cpu; } @@ -8008,12 +8106,22 @@ static inline bool asym_fits_cpu(unsigned long util, unsigned long util_max, int cpu) { - if (sched_asym_cpucap_active()) + if (sched_asym_cpucap_active()) { /* * Return true only if the cpu fully fits the task requirements * which include the utilization and the performance hints. + * + * When SMT is active, also require that the core has no busy + * siblings. + * + * Note: gating on is_core_idle() also makes the early-bailout + * candidates in select_idle_sibling() (target, prev, + * recent_used_cpu) idle-core-aware on ASYM+SMT, which the + * NO_ASYM path does not do. */ - return (util_fits_cpu(util, util_min, util_max, cpu) > 0); + return (!sched_smt_active() || is_core_idle(cpu)) && + (util_fits_cpu(util, util_min, util_max, cpu) > 0); + } return true; } -- cgit v1.2.3 From bf6aa722198d3c06e4236e8c5a480f30a64e1513 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Sat, 9 May 2026 20:07:28 +0200 Subject: sched/fair: Reject misfit pulls onto busy SMT siblings on asym-capacity When SD_ASYM_CPUCAPACITY load balancing considers pulling a misfit task, capacity_of(dst_cpu) can overstate available compute if the SMT sibling is busy: the core does not deliver its full nominal capacity. If SMT is active and dst_cpu is not on a fully idle core, skip this destination so we do not migrate a misfit expecting a capacity upgrade we cannot actually provide. Reported-by: Felix Abecassis Signed-off-by: Andrea Righi Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Vincent Guittot Link: https://patch.msgid.link/20260509180955.1840064-5-arighi@nvidia.com --- kernel/sched/fair.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 8854d4d980b0..f69ee5ae2b8c 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -9625,6 +9625,7 @@ struct lb_env { int dst_cpu; struct rq *dst_rq; + bool dst_core_idle; struct cpumask *dst_grpmask; int new_dst_cpu; @@ -10850,10 +10851,16 @@ static bool update_sd_pick_busiest(struct lb_env *env, * We can use max_capacity here as reduction in capacity on some * CPUs in the group should either be possible to resolve * internally or be covered by avg_load imbalance (eventually). + * + * When SMT is active, only pull a misfit to dst_cpu if it is on a + * fully idle core; otherwise the effective capacity of the core is + * reduced and we may not actually provide more capacity than the + * source. */ if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && (sgs->group_type == group_misfit_task) && - (!capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) || + (!env->dst_core_idle || + !capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) || sds->local_stat.group_type != group_has_spare)) return false; @@ -11417,6 +11424,8 @@ static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sd unsigned long sum_util = 0; bool sg_overloaded = 0, sg_overutilized = 0; + env->dst_core_idle = !sched_smt_active() || is_core_idle(env->dst_cpu); + do { struct sg_lb_stats *sgs = &tmp_sgs; int local_group; -- cgit v1.2.3 From 61ea17a63719bac51e1bc50eb39fc637f0fdc06e Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Sat, 9 May 2026 20:07:29 +0200 Subject: sched/fair: Add SIS_UTIL support to select_idle_capacity() Add to select_idle_capacity() the same SIS_UTIL-controlled idle-scan mechanism, already used by select_idle_cpu(): when sched_feat(SIS_UTIL) is enabled and the LLC domain has sched_domain_shared data, derive the per-attempt scan limit from sd->shared->nr_idle_scan. That bounds the walk on large LLCs: once nr_idle_scan is exhausted, return the best CPU seen so far. The early exit is gated on !has_idle_core so an active idle-core search (SMT with idle cores reported by test_idle_cores()) isn't cut short before it gets a chance to find one. Co-developed-by: Andrea Righi Signed-off-by: Andrea Righi Signed-off-by: K Prateek Nayak Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Vincent Guittot Link: https://patch.msgid.link/20260509180955.1840064-6-arighi@nvidia.com --- kernel/sched/fair.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index f69ee5ae2b8c..69ba882681c5 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -8016,6 +8016,7 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) int fits, best_fits = ASYM_IDLE_THREAD_MISFIT; int cpu, best_cpu = -1; struct cpumask *cpus; + int nr = INT_MAX; cpus = this_cpu_cpumask_var_ptr(select_rq_mask); cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr); @@ -8024,10 +8025,28 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) util_min = uclamp_eff_value(p, UCLAMP_MIN); util_max = uclamp_eff_value(p, UCLAMP_MAX); + if (sched_feat(SIS_UTIL) && sd->shared) { + /* + * Same nr_idle_scan hint as select_idle_cpu(), nr only limits + * the scan when not preferring an idle core. + */ + nr = READ_ONCE(sd->shared->nr_idle_scan) + 1; + /* overloaded domain is unlikely to have idle cpu/core */ + if (nr == 1) + return -1; + } + for_each_cpu_wrap(cpu, cpus, target) { bool preferred_core = !has_idle_core || is_core_idle(cpu); unsigned long cpu_cap = capacity_of(cpu); + /* + * Stop when the nr_idle_scan is exhausted (mirrors + * select_idle_cpu() logic). + */ + if (!has_idle_core && --nr <= 0) + return best_cpu; + if (!choose_idle_cpu(cpu, p)) continue; -- cgit v1.2.3 From 04f80f8b12a02fa2e0827c8f37eb357adca8ce44 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Thu, 14 May 2026 23:47:17 +0000 Subject: sched: Switch rq->next_class on proxy_resched_idle() K Prateek noticed we weren't setting the rq->next_class in proxy_resched_idle(), when I was debugging an issue seen with CONFIG_SCHED_PROXY_EXEC and some of Peter's new patches, and suggested this fix. So set rq->next_class when we temporarily switch the donor to idle, so we don't accidentally call wakeup_preempt_fair() with idle as the donor. Suggested-by: K Prateek Nayak Signed-off-by: John Stultz Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260514234732.3170197-1-jstultz@google.com --- kernel/sched/core.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 3ae5f19c1b7e..77f4ebe8f5c7 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6653,6 +6653,7 @@ static inline void proxy_set_task_cpu(struct task_struct *p, int cpu) static inline struct task_struct *proxy_resched_idle(struct rq *rq) { put_prev_set_next_task(rq, rq->donor, rq->idle); + rq->next_class = &idle_sched_class; rq_set_donor(rq, rq->idle); set_tsk_need_resched(rq->idle); return rq->idle; -- cgit v1.2.3 From dd29c017aed628076e915fe4cdfb5392fd4c5cab Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Fri, 15 May 2026 10:37:40 -0400 Subject: sched/rt: Have RT_PUSH_IPI be default off for non PREEMPT_RT RT migration is done aggressively. When a CPU schedules out a high priority RT task for a lower priority task, it will look to see if there's any RT tasks that are waiting to run on another CPU that is of higher priority than the task this CPU is about to run. If it finds one, it will pull that task over to the CPU and allow it to run there instead. Normally, this pulling is done by looking at the RT overloaded mask (rto) which contains all the CPUs in the scheduler domain with RT tasks that are waiting to run due to a higher priority RT task currently running on their CPU. The CPU that is about to schedule a lower priority task will grab the rq lock of the overloaded CPU and move the RT task from that CPU's runqueue to the local one and schedule the higher priority RT task. This caused issues when a lot of CPUs would schedule a lower priority task at the same time. They would all try to grab the same runqueue lock of the CPU with the overloaded RT tasks. Only the first CPU that got in will get that task. All the others would wait until they got the runqueue lock and see there's nothing to pull and do nothing. On systems with lots of CPUs, this caused a large latency (up to 500us) which is beyond what PREEMPT_RT is to allow. The solution to that was to create an RT_PUSH_IPI logic. When any CPU wanted to pull a task, instead of grabbing the runqueue lock of the overloaded CPU, it would start by sending an IPI to the overloaded CPU, and that IPI handler would have the CPU with the waiting RT task do a push instead. Then that handler would send an IPI to the next CPU with overloaded RT tasks, and so on. Note, after the first CPU starts this process, if another CPU wanted to do a pull, it would see that the process has already begun and would only increment a counter to have the IPIs continue again. The RT_PUSH_IPI solved the latency problem with PREEMPT_RT but could cause a new issue with non PREEMPT_RT. Namely, softirqs run in a threaded context on PREEMPT_RT but they can run in an interrupt context in non-RT. If an IPI lands on a CPU that has just woken up multiple RT tasks and the current CPU is running a non RT or a low priority RT task, instead of doing a push, it would simply do a schedule on that CPU. But if a softirq was also executing on this CPU, the schedule would need to wait until the softirq finished. Until then, the CPU would still be considered overloaded as there are RT tasks still waiting to run on it. A live lock occurred on a workload that was doing heavy networking traffic on a large machine where the softirqs would run 500us out of 750us. And it would also be waking up RT tasks, causing the RT pull logic to be constantly executed. When a softirq triggered on a CPU with RT tasks queued but not running yet, and the other CPUs would see this CPU as being overloaded, they would send an IPI over to it. The CPU would notice that the waiting RT tasks are of higher priority than the currently running task and simply schedule that CPU instead. But because the softirq was executing, before it could schedule, it would receive another IPI to do the same. The amount of IPIs would slow down the currently running softirq so much that before it could return back to task context, it would execute another softirq never allowing the CPU to schedule. This live locked that CPU. As RT_PUSH_IPI was created to help PREEMPT_RT, make it default off if PREEMPT_RT is not enabled. Fixes: b6366f048e0c ("sched/rt: Use IPI to trigger RT task push migration instead of pulling") Closes: https://lore.kernel.org/all/20260506235716.2530720-1-tj@kernel.org/ Reported-by: Tejun Heo Signed-off-by: Steven Rostedt Signed-off-by: Peter Zijlstra (Intel) Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260515103740.25ccbed8@gandalf.local.home --- kernel/sched/features.h | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/features.h b/kernel/sched/features.h index 84c4fe3abd74..8f0dee8fc475 100644 --- a/kernel/sched/features.h +++ b/kernel/sched/features.h @@ -110,8 +110,16 @@ SCHED_FEAT(WARN_DOUBLE_CLOCK, false) * rq lock and possibly create a large contention, sending an * IPI to that CPU and let that CPU push the RT task to where * it should go may be a better scenario. + * + * This is best for PREEMPT_RT, but for non-RT it can cause issues + * when preemption is disabled for long periods of time. Have + * it only default enabled for PREEMPT_RT. */ +# ifdef CONFIG_PREEMPT_RT SCHED_FEAT(RT_PUSH_IPI, true) +# else +SCHED_FEAT(RT_PUSH_IPI, false) +# endif #endif SCHED_FEAT(RT_RUNTIME_SHARE, false) -- cgit v1.2.3 From 9e005ed21152d4a4bb0ceea71045ff8a642a6feb Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Tue, 19 May 2026 05:14:23 +0000 Subject: sched/topology: Allow multiple domains to claim sched_domain_shared Recent optimizations of sd->shared assignment moved to allocating a single instance of per-CPU sched_domain_shared objects per s_data. Recent optimizations to select_idle_capacity() moved the sd->shared assignments to "sd_asym" domain when ASYM_CPUCAPACITY is detected but cache-aware scheduling mandates the presence of "sd_llc_shared" to compute and cache per-LLC statistics. Use an "alloc_flags" union in sched_domain_shared to claim a sched_domain_shared object per sched_domain. Allocation starts searching for an available / matching sched_domain_shared instance from the first CPU of sched_domain_span(sd) (sd can be sd_llc, or sd_asym). If the shared object is claimed by another domain, the instance corresponding to next CPU in the domain span is explored until a matching / available instance is found. In case of a single CPU in sched_domain_span(), the domain will be degenerated and a temporary overlap of ->shared objects across different domains is acceptable. "alloc_flags" forms a union with "nr_idle_scan" and the stale flags are left as is when the sd->shared is published. The expectation is for the first load balancing instance to correct the value just like the current behavior, except the initial value is no longer 0. Originally-by: Peter Zijlstra Signed-off-by: K Prateek Nayak Signed-off-by: Peter Zijlstra (Intel) Tested-by: Andrea Righi --- kernel/sched/topology.c | 63 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 54 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index dbfd9657f897..df2ceb54c970 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -623,6 +623,12 @@ static void free_sched_groups(struct sched_group *sg, int free_sgc) } while (sg != first); } +static void free_sched_domain_shared(struct sched_domain_shared *sds) +{ + if (sds && atomic_dec_and_test(&sds->ref)) + kfree(sds); +} + static void destroy_sched_domain(struct sched_domain *sd) { /* @@ -631,9 +637,7 @@ static void destroy_sched_domain(struct sched_domain *sd) * dropping group/capacity references, freeing where none remain. */ free_sched_groups(sd->groups, 1); - - if (sd->shared && atomic_dec_and_test(&sd->shared->ref)) - kfree(sd->shared); + free_sched_domain_shared(sd->shared); #ifdef CONFIG_SCHED_CACHE /* only the bottom sd has llc_counts array */ @@ -755,7 +759,14 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) /* Pick reference to parent->shared. */ if (parent->shared) { - WARN_ON_ONCE(tmp->shared); + /* + * It is safe to free a sd->shared that + * has not been published yet. If a + * sd->shared was published, the refcount + * will end up being non-zero and it will + * not be freed here. + */ + free_sched_domain_shared(tmp->shared); tmp->shared = parent->shared; parent->shared = NULL; } @@ -2916,11 +2927,45 @@ static void adjust_numa_imbalance(struct sched_domain *sd_llc) } } -static void init_sched_domain_shared(struct s_data *d, struct sched_domain *sd) +static void +init_sched_domain_shared(struct s_data *d, struct sched_domain *sd, int flags) { - int sd_id = cpumask_first(sched_domain_span(sd)); + struct sched_domain_shared *sds = NULL; + int cpu; + + /* + * Multiple domains can try to claim a shared object like + * SD_ASYM_CPUCAPACITY and SD_SHARE_LLC which can alias to + * same cpumask_first(sched_domain_span(sd)) CPU and can + * cause "nr_idle_scan" to be populated incorrectly during + * load balancing. + * + * Find the first CPU in sched_domain_span(sd) with an + * unclaimed domain (!alloc_flags) or where the alloc_flag + * matches the requested flag (SD_* flag) + * + * If the domain only has single CPU, allow temporary overlap + * in allocation since the domains will be degenerated later. + */ + for_each_cpu(cpu, sched_domain_span(sd)) { + sds = *per_cpu_ptr(d->sds, cpu); + + if (!sds->alloc_flags || + sd->span_weight == 1 || + sds->alloc_flags == flags) { + sds->alloc_flags = flags; + sd->shared = sds; + break; + } + } + + /* + * Use the sd_shared corresponding to the last + * CPU in the span if none are avaialable. + */ + if (WARN_ON_ONCE(!sd->shared)) + sd->shared = sds; - sd->shared = *per_cpu_ptr(d->sds, sd_id); /* * nr_busy_cpus is consumed only by the NOHZ kick path via * sd_balance_shared; on the asym-capacity path it is initialized but @@ -2960,7 +3005,7 @@ static bool claim_asym_sched_domain_shared(struct s_data *d, int cpu) if (!sd_asym || (sd_asym->flags & SD_NUMA)) return false; - init_sched_domain_shared(d, sd_asym); + init_sched_domain_shared(d, sd_asym, SD_ASYM_CPUCAPACITY); return true; } @@ -3115,7 +3160,7 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att sd = sd->parent; if (sd->flags & SD_SHARE_LLC) { - init_sched_domain_shared(&d, sd); + init_sched_domain_shared(&d, sd, SD_SHARE_LLC); /* * In presence of higher domains, adjust the -- cgit v1.2.3 From a9e4e50519e99e5be2d69fa4eebec6a4848ae201 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 8 May 2026 10:45:16 -0700 Subject: locking/rtmutex: Annotate API and implementation Enable context analysis for struct rt_mutex and annotate all functions that accept a struct rt_mutex pointer. In the __rt_mutex_lock_common() callers, instead of adding the __no_context_analysis annotation, emit a runtime warning if the __rt_mutex_lock_common() return value is not zero and add an __acquire() statement. Signed-off-by: Bart Van Assche Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260508174520.1416285-1-bvanassche@acm.org --- kernel/locking/rtmutex.c | 4 ++++ kernel/locking/rtmutex_api.c | 31 ++++++++++++++++++++++++++++--- 2 files changed, 32 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c index 4f386ea6c792..9147d6a31b78 100644 --- a/kernel/locking/rtmutex.c +++ b/kernel/locking/rtmutex.c @@ -484,6 +484,7 @@ static __always_inline bool __waiter_less(struct rb_node *a, const struct rb_nod static __always_inline void rt_mutex_enqueue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter) + __must_hold(&lock->wait_lock) { lockdep_assert_held(&lock->wait_lock); @@ -492,6 +493,7 @@ rt_mutex_enqueue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter) static __always_inline void rt_mutex_dequeue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter) + __must_hold(&lock->wait_lock) { lockdep_assert_held(&lock->wait_lock); @@ -1092,6 +1094,7 @@ static int __sched rt_mutex_adjust_prio_chain(struct task_struct *task, static int __sched try_to_take_rt_mutex(struct rt_mutex_base *lock, struct task_struct *task, struct rt_mutex_waiter *waiter) + __must_hold(&lock->wait_lock) { lockdep_assert_held(&lock->wait_lock); @@ -1319,6 +1322,7 @@ static int __sched task_blocks_on_rt_mutex(struct rt_mutex_base *lock, */ static void __sched mark_wakeup_next_waiter(struct rt_wake_q_head *wqh, struct rt_mutex_base *lock) + __must_hold(&lock->wait_lock) { struct rt_mutex_waiter *waiter; diff --git a/kernel/locking/rtmutex_api.c b/kernel/locking/rtmutex_api.c index 124219aea46e..7c40b91422a0 100644 --- a/kernel/locking/rtmutex_api.c +++ b/kernel/locking/rtmutex_api.c @@ -41,6 +41,7 @@ static __always_inline int __rt_mutex_lock_common(struct rt_mutex *lock, unsigned int state, struct lockdep_map *nest_lock, unsigned int subclass) + __cond_acquires(0, lock) { int ret; @@ -67,13 +68,27 @@ EXPORT_SYMBOL(rt_mutex_base_init); */ void __sched rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass) { - __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, subclass); + if (__rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, subclass) == 0) + return; + /* + * The code below is never reached because __rt_mutex_lock_common() only + * returns an error code if interrupted by a signal or upon a timeout. + */ + WARN_ON_ONCE(true); + __acquire(lock); } EXPORT_SYMBOL_GPL(rt_mutex_lock_nested); void __sched _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map *nest_lock) { - __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, nest_lock, 0); + if (__rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, nest_lock, 0) == 0) + return; + /* + * The code below is never reached because __rt_mutex_lock_common() only + * returns an error code if interrupted by a signal or upon a timeout. + */ + WARN_ON_ONCE(true); + __acquire(lock); } EXPORT_SYMBOL_GPL(_rt_mutex_lock_nest_lock); @@ -86,7 +101,14 @@ EXPORT_SYMBOL_GPL(_rt_mutex_lock_nest_lock); */ void __sched rt_mutex_lock(struct rt_mutex *lock) { - __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, 0); + if (__rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, 0) == 0) + return; + /* + * The code below is never reached because __rt_mutex_lock_common() only + * returns an error code if interrupted by a signal or upon a timeout. + */ + WARN_ON_ONCE(true); + __acquire(lock); } EXPORT_SYMBOL_GPL(rt_mutex_lock); #endif @@ -157,6 +179,7 @@ void __sched rt_mutex_unlock(struct rt_mutex *lock) { mutex_release(&lock->dep_map, _RET_IP_); __rt_mutex_unlock(&lock->rtmutex); + __release(lock); } EXPORT_SYMBOL_GPL(rt_mutex_unlock); @@ -182,6 +205,7 @@ int __sched __rt_mutex_futex_trylock(struct rt_mutex_base *lock) */ bool __sched __rt_mutex_futex_unlock(struct rt_mutex_base *lock, struct rt_wake_q_head *wqh) + __must_hold(&lock->wait_lock) { lockdep_assert_held(&lock->wait_lock); @@ -312,6 +336,7 @@ int __sched __rt_mutex_start_proxy_lock(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter, struct task_struct *task, struct wake_q_head *wake_q) + __must_hold(&lock->wait_lock) { int ret; -- cgit v1.2.3 From 49b18315be4eecfc36b75f4aecb4d40a87d68a20 Mon Sep 17 00:00:00 2001 From: KP Singh Date: Wed, 20 May 2026 04:40:59 +0200 Subject: bpf: Reject NULL data/sig in bpf_verify_pkcs7_signature __bpf_dynptr_data() can return NULL (FILE dynptrs, any non-contiguous backing). bpf_verify_pkcs7_signature() forwards the pointer to verify_pkcs7_signature() unchecked, causing a NULL deref in asn1_ber_decoder() reachable from a sleepable BPF LSM at lsm.s/bpf. NULL-check both pointers and reject with -EINVAL. Mirrors the guards already in kernel/bpf/crypto.c. Fixes: 865b0566d8f1 ("bpf: Add bpf_verify_pkcs7_signature() kfunc") Reported-by: Xianrui Dong Signed-off-by: KP Singh Reviewed-by: Amery Hung Acked-by: Song Liu Acked-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260520024059.313468-1-kpsingh@kernel.org Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/helpers.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 2bb60200c266..b5314c9fed3c 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -4241,8 +4241,13 @@ __bpf_kfunc int bpf_verify_pkcs7_signature(struct bpf_dynptr *data_p, data_len = __bpf_dynptr_size(data_ptr); data = __bpf_dynptr_data(data_ptr, data_len); + if (!data) + return -EINVAL; + sig_len = __bpf_dynptr_size(sig_ptr); sig = __bpf_dynptr_data(sig_ptr, sig_len); + if (!sig) + return -EINVAL; return verify_pkcs7_signature(data, data_len, sig, sig_len, trusted_keyring->key, -- cgit v1.2.3 From 22572dbcd3486e6c4dced877125bbf50e4e24edf Mon Sep 17 00:00:00 2001 From: Cunlong Li Date: Wed, 20 May 2026 11:30:54 +0800 Subject: cgroup: rstat: relax NMI guard after switch to try_cmpxchg Commit 36df6e3dbd7e ("cgroup: make css_rstat_updated nmi safe") used this_cpu_cmpxchg() for the lockless insertion, and therefore required both ARCH_HAVE_NMI_SAFE_CMPXCHG and ARCH_HAS_NMI_SAFE_THIS_CPU_OPS in the NMI guard: on archs without the latter, this_cpu_cmpxchg() falls back to "local_irq_save() + plain cmpxchg", and local_irq_save() cannot mask NMIs. Commit 3309b63a2281 ("cgroup: rstat: use LOCK CMPXCHG in css_rstat_updated") later replaced this_cpu_cmpxchg() with plain try_cmpxchg() to fix cross-CPU lockless-list corruption, but left the NMI guard untouched. After that switch, css_rstat_updated() no longer performs any this_cpu_*() RMW operations and only relies on the arch having NMI-safe cmpxchg, so ARCH_HAS_NMI_SAFE_THIS_CPU_OPS is no longer required in the guard. Relax the guard accordingly so that archs which have HAVE_NMI and ARCH_HAVE_NMI_SAFE_CMPXCHG but not ARCH_HAS_NMI_SAFE_THIS_CPU_OPS (e.g. sparc, powerpc on PPC64/BOOK3S) can benefit from the existing CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC path. Without this, the css is never queued in NMI on those archs, and the atomics staged by account_{slab,kmem}_nmi_safe() are not drained by flush_nmi_stats(). Fixes: 3309b63a2281 ("cgroup: rstat: use LOCK CMPXCHG in css_rstat_updated") Signed-off-by: Cunlong Li Signed-off-by: Tejun Heo --- kernel/cgroup/rstat.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/cgroup/rstat.c b/kernel/cgroup/rstat.c index ed60ba119c68..de816a43db9f 100644 --- a/kernel/cgroup/rstat.c +++ b/kernel/cgroup/rstat.c @@ -81,11 +81,10 @@ void __css_rstat_updated(struct cgroup_subsys_state *css, int cpu) lockdep_assert_preemption_disabled(); /* - * For archs withnot nmi safe cmpxchg or percpu ops support, ignore - * the requests from nmi context. + * The lockless insertion below relies on NMI-safe cmpxchg; + * bail out in NMI on archs that don't provide it. */ - if ((!IS_ENABLED(CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG) || - !IS_ENABLED(CONFIG_ARCH_HAS_NMI_SAFE_THIS_CPU_OPS)) && in_nmi()) + if (!IS_ENABLED(CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG) && in_nmi()) return; rstatc = css_rstat_cpu(css, cpu); -- cgit v1.2.3 From 576ec047d20b368b43c4d5db98c4f2e0f3c101ec Mon Sep 17 00:00:00 2001 From: David Carlier Date: Fri, 8 May 2026 20:57:47 +0100 Subject: tracing: Avoid NULL return from hist_field_name() on truncation hist_field_name() returns "" everywhere except the fully-qualified VAR_REF/EXPR case, where snprintf() truncation returns NULL early and bypasses the bottom NULL->"" guard. Callers don't expect NULL: strcat(expr, hist_field_name(field, 0)) at trace_events_hist.c:1758 and the strcmp() in the sort-key match loop at :4804 both deref it. system and event_name are bounded by MAX_EVENT_NAME_LEN, but the field name on a VAR_REF is kstrdup'd from a histogram variable name parsed out of the trigger string and has no length cap, so a long enough var name in a fully qualified reference can reach the truncation path. Keep the length check but leave field_name as "" on overflow. Link: https://patch.msgid.link/20260508195747.25492-1-devnexen@gmail.com Fixes: 5ec1d1e97de1 ("tracing: Rebuild full_name on each hist_field_name() call") Signed-off-by: David Carlier Signed-off-by: Steven Rostedt --- kernel/trace/trace_events_hist.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index 0dbbf6cca9bc..eb2c2bc8bc3d 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -1369,10 +1369,8 @@ static const char *hist_field_name(struct hist_field *field, len = snprintf(full_name, sizeof(full_name), fmt, field->system, field->event_name, field->name); - if (len >= sizeof(full_name)) - return NULL; - - field_name = full_name; + if (len < sizeof(full_name)) + field_name = full_name; } else field_name = field->name; } else if (field->flags & HIST_FIELD_FL_TIMESTAMP) -- cgit v1.2.3 From 783c8109844503bd1c35dab41b6d5fd074a9f131 Mon Sep 17 00:00:00 2001 From: Matthew Leach Date: Fri, 3 Apr 2026 08:36:30 +0100 Subject: PM: hibernate: call preallocate_image() after freeze prepare Certain drivers release resources (pinned pages, etc.) into system memory during the prepare freeze PM op, making them swappable. Currently, hibernate_preallocate_memory() is called before prepare freeze, so those drivers have no opportunity to release resources first. If a driver is holding a large amount of unswappable system RAM, this can cause hibernate_preallocate_memory() to fail. Move the call to hibernate_preallocate_memory() after prepare freeze. According to the documentation for the prepare callback, devices should be left in a usable state, so storage drivers should still be able to service I/O requests. This allows drivers to release unswappable resources prior to preallocation, so they can be swapped out through hibernate_preallocate_memory()'s reclaim path. Also remove shrink_shmem_memory() since hibernate_preallocate_memory() will have reclaimed enough memory for the hibernation image. Signed-off-by: Matthew Leach Reviewed-by: Mario Limonciello (AMD) [ rjw: Subject and changelog tweaks ] Link: https://patch.msgid.link/20260403-hibernation-fixes-v3-1-31bc9fa3ba2d@collabora.com Signed-off-by: Rafael J. Wysocki --- kernel/power/hibernate.c | 46 +++++++++------------------------------------- 1 file changed, 9 insertions(+), 37 deletions(-) (limited to 'kernel') diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index af8d07bafe02..d2479c69d71a 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c @@ -392,23 +392,6 @@ static int create_image(int platform_mode) return error; } -static void shrink_shmem_memory(void) -{ - struct sysinfo info; - unsigned long nr_shmem_pages, nr_freed_pages; - - si_meminfo(&info); - nr_shmem_pages = info.sharedram; /* current page count used for shmem */ - /* - * The intent is to reclaim all shmem pages. Though shrink_all_memory() can - * only reclaim about half of them, it's enough for creating the hibernation - * image. - */ - nr_freed_pages = shrink_all_memory(nr_shmem_pages); - pr_debug("requested to reclaim %lu shmem pages, actually freed %lu pages\n", - nr_shmem_pages, nr_freed_pages); -} - /** * hibernation_snapshot - Quiesce devices and create a hibernation image. * @platform_mode: If set, use platform driver to prepare for the transition. @@ -425,14 +408,9 @@ int hibernation_snapshot(int platform_mode) if (error) goto Close; - /* Preallocate image memory before shutting down devices. */ - error = hibernate_preallocate_memory(); - if (error) - goto Close; - error = freeze_kernel_threads(); if (error) - goto Cleanup; + goto Close; if (hibernation_test(TEST_FREEZER)) { @@ -445,19 +423,13 @@ int hibernation_snapshot(int platform_mode) } error = dpm_prepare(PMSG_FREEZE); - if (error) { - dpm_complete(PMSG_RECOVER); - goto Thaw; - } + if (error) + goto Complete; - /* - * Device drivers may move lots of data to shmem in dpm_prepare(). The shmem - * pages will use lots of system memory, causing hibernation image creation - * fail due to insufficient free memory. - * This call is to force flush the shmem pages to swap disk and reclaim - * the system memory so that image creation can succeed. - */ - shrink_shmem_memory(); + /* Preallocate image memory before shutting down devices. */ + error = hibernate_preallocate_memory(); + if (error) + goto Complete; console_suspend_all(); pm_restrict_gfp_mask(); @@ -492,10 +464,10 @@ int hibernation_snapshot(int platform_mode) platform_end(platform_mode); return error; + Complete: + dpm_complete(PMSG_RECOVER); Thaw: thaw_kernel_threads(); - Cleanup: - swsusp_free(); goto Close; } -- cgit v1.2.3 From e11c9c8365cd7655e81c4e6611492f1b15850aa5 Mon Sep 17 00:00:00 2001 From: Crystal Wood Date: Mon, 11 May 2026 17:31:43 -0500 Subject: tracing/osnoise: Dump stack on timerlat uret threshold event Dump the saved IRQ stack trace regardless of whether the event was THREAD_CONTEXT or THREAD_URET. In the uret case, the latency presumably had not yet crossed the threshold at IRQ time (or else it would have dumped the stack at thread wakeup time, unless we're racing with a change to the threshold), but it may have at least contributed -- and this is possible with THREAD_CONTEXT as well. In any case, it helps with writing reliable rtla tests if we always get a stack trace on a threshold event. Cc: John Kacur Cc: Tomas Glozar Cc: Costa Shulyupin Cc: Wander Lairson Costa Link: https://patch.msgid.link/20260511223143.1477332-1-crwood@redhat.com Signed-off-by: Crystal Wood Signed-off-by: Steven Rostedt --- kernel/trace/trace_osnoise.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c index 75678053b21c..62c2667d97fa 100644 --- a/kernel/trace/trace_osnoise.c +++ b/kernel/trace/trace_osnoise.c @@ -2544,9 +2544,12 @@ timerlat_fd_read(struct file *file, char __user *ubuf, size_t count, notify_new_max_latency(diff); tlat->tracing_thread = false; - if (osnoise_data.stop_tracing_total) - if (time_to_us(diff) >= osnoise_data.stop_tracing_total) + if (osnoise_data.stop_tracing_total) { + if (time_to_us(diff) >= osnoise_data.stop_tracing_total) { + timerlat_dump_stack(time_to_us(diff)); osnoise_stop_tracing(); + } + } } else { tlat->tracing_thread = false; tlat->kthread = current; -- cgit v1.2.3 From d6236d5b2391cfdfa14d8acf7e29cc48068adc2a Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 18 May 2026 21:53:11 -1000 Subject: sched_ext: Rename scx_cmask.nr_bits to nr_cids struct scx_cmask is a base-windowed bitmap over cid space. Each bit represents one cid, so the count of active bits is the count of cids. The sibling struct scx_cid_shard already uses nr_cids. Rename as a prep so the following patches that grow the cmask API can use the consistent name. v2: Also rename src->nr_bits / dst->nr_bits in cmask_copy_from_kernel(). (sashiko AI) Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext_cid.h | 8 ++++---- kernel/sched/ext_types.h | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext_cid.h b/kernel/sched/ext_cid.h index f41d48afb7d1..e1c44a180bb1 100644 --- a/kernel/sched/ext_cid.h +++ b/kernel/sched/ext_cid.h @@ -138,7 +138,7 @@ static inline bool scx_is_cid_type(void) static inline bool __scx_cmask_contains(const struct scx_cmask *m, u32 cid) { - return likely(cid >= m->base && cid < m->base + m->nr_bits); + return likely(cid >= m->base && cid < m->base + m->nr_cids); } /* Word in bits[] covering @cid. @cid must satisfy __scx_cmask_contains(). */ @@ -147,11 +147,11 @@ static inline u64 *__scx_cmask_word(const struct scx_cmask *m, u32 cid) return (u64 *)&m->bits[cid / 64 - m->base / 64]; } -static inline void scx_cmask_init(struct scx_cmask *m, u32 base, u32 nr_bits) +static inline void scx_cmask_init(struct scx_cmask *m, u32 base, u32 nr_cids) { m->base = base; - m->nr_bits = nr_bits; - memset(m->bits, 0, SCX_CMASK_NR_WORDS(nr_bits) * sizeof(u64)); + m->nr_cids = nr_cids; + memset(m->bits, 0, SCX_CMASK_NR_WORDS(nr_cids) * sizeof(u64)); } static inline void __scx_cmask_set(struct scx_cmask *m, u32 cid) diff --git a/kernel/sched/ext_types.h b/kernel/sched/ext_types.h index ebb8cdf90612..c6c4e3db7311 100644 --- a/kernel/sched/ext_types.h +++ b/kernel/sched/ext_types.h @@ -67,10 +67,10 @@ struct scx_cid_topo { * cmask: variable-length, base-windowed bitmap over cid space * ----------------------------------------------------------- * - * A cmask covers the cid range [base, base + nr_bits). bits[] is aligned to the + * A cmask covers the cid range [base, base + nr_cids). bits[] is aligned to the * global 64-cid grid: bits[0] spans [base & ~63, (base & ~63) + 64), so the * first (base & 63) bits of bits[0] are head padding and any tail past base + - * nr_bits is tail padding. Both must stay zero for the lifetime of the mask; + * nr_cids is tail padding. Both must stay zero for the lifetime of the mask; * all mutating helpers preserve that invariant. * * Grid alignment means two cmasks always address bits[] against the same global @@ -82,21 +82,21 @@ struct scx_cid_topo { */ struct scx_cmask { u32 base; - u32 nr_bits; + u32 nr_cids; DECLARE_FLEX_ARRAY(u64, bits); }; /* - * Number of u64 words of bits[] storage that covers @nr_bits regardless of base + * Number of u64 words of bits[] storage that covers @nr_cids regardless of base * alignment. The +1 absorbs up to 63 bits of head padding when base is not * 64-aligned - always allocating one extra word beats branching on base or * splitting the compute. */ -#define SCX_CMASK_NR_WORDS(nr_bits) (((nr_bits) + 63) / 64 + 1) +#define SCX_CMASK_NR_WORDS(nr_cids) (((nr_cids) + 63) / 64 + 1) /* * Define an on-stack cmask for up to @cap_bits. @name is a struct scx_cmask * - * aliasing zero-initialized storage; call scx_cmask_init() to set base/nr_bits. + * aliasing zero-initialized storage; call scx_cmask_init() to set base/nr_cids. */ #define SCX_CMASK_DEFINE(name, cap_bits) \ DEFINE_RAW_FLEX(struct scx_cmask, name, bits, SCX_CMASK_NR_WORDS(cap_bits)) -- cgit v1.2.3 From a0b48fd7fe2854211eadb5056e72bce3946140c1 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 18 May 2026 21:53:11 -1000 Subject: sched_ext: Track bits[] storage size in struct scx_cmask scx_cmask carries @base and @nr_cids but not the bits[] allocation size, so helpers reshaping the active range have no way to check it fits and later kfuncs taking caller-provided storage can't validate it. Add @alloc_words (u64 word count) annotated with __counted_by, and split the bit-range API into three helpers: - SCX_CMASK_DEFINE() / __SCX_CMASK_DEFINE() define an on-stack cmask, the latter taking an explicit capacity for oversized storage. SCX_CMASK_DEFINE_SHARD() is a thin wrapper that always reserves SCX_CID_SHARD_MAX_CPUS bits of storage. - scx_cmask_init() / __scx_cmask_init() initialize a cmask, with the same tight-vs-explicit split. - scx_cmask_reframe() reshapes the active range without resizing storage. The BPF mirror (cmask_init / __cmask_init / cmask_reframe) gets the same shape. Add scx_cmask_clear() and scx_cmask_fill() to zero and set the active-range bits respectively. scx_cpumask_to_cmask() uses scx_cmask_clear(); scx_cmask_init() would otherwise re-write @alloc_words on every call. A later patch uses @alloc_words in scx_cmask_ref_shard() to refuse output storage that can't hold the requested shard. v2: Init per-CPU scx_set_cmask_scratch (was zero-init, emitted empty cmasks). Add nr_cids/alloc_cids check in BPF __cmask_init(). (sashiko AI) Widen SCX_CMASK_NR_WORDS()/CMASK_NR_WORDS() to compute in u64 so that @nr_cids near U32_MAX no longer wraps to a small value and bypasses the bounds check in cmask_reframe(). (Andrea) Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext_cid.c | 52 +++++++++++++++++++++++++++++++++++++--- kernel/sched/ext_cid.h | 57 +++++++++++++++++++++++++++++++++++++++++++- kernel/sched/ext_types.h | 62 +++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 156 insertions(+), 15 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext_cid.c b/kernel/sched/ext_cid.c index bdd8ef8eae3d..44dd47a87709 100644 --- a/kernel/sched/ext_cid.c +++ b/kernel/sched/ext_cid.c @@ -55,6 +55,7 @@ static s32 scx_cid_arrays_alloc(void) s16 *cid_to_cpu, *cpu_to_cid; struct scx_cid_topo *cid_topo; struct scx_cmask __percpu *set_cmask_scratch; + s32 cpu; if (scx_cid_to_cpu_tbl) return 0; @@ -77,6 +78,9 @@ static s32 scx_cid_arrays_alloc(void) WRITE_ONCE(scx_cid_to_cpu_tbl, cid_to_cpu); WRITE_ONCE(scx_cpu_to_cid_tbl, cpu_to_cid); WRITE_ONCE(scx_cid_topo, cid_topo); + for_each_possible_cpu(cpu) + scx_cmask_init(per_cpu_ptr(set_cmask_scratch, cpu), + 0, npossible); WRITE_ONCE(scx_set_cmask_scratch, set_cmask_scratch); return 0; } @@ -222,19 +226,61 @@ s32 scx_cid_init(struct scx_sched *sch) return 0; } +/** + * scx_cmask_clear - Zero every bit in @m's active range + * @m: cmask to clear + * + * Storage past the active range is left as is. + */ +void scx_cmask_clear(struct scx_cmask *m) +{ + u32 nr_words; + + if (!m->nr_cids) + return; + nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1; + memset(m->bits, 0, nr_words * sizeof(u64)); +} + +/** + * scx_cmask_fill - Set every bit in @m's active range + * @m: cmask to fill + * + * Counterpart to scx_cmask_clear(). Storage past the active range is left as is. + */ +void scx_cmask_fill(struct scx_cmask *m) +{ + u32 nr_words, head_bits, tail_bits; + + if (!m->nr_cids) + return; + nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1; + memset(m->bits, 0xff, nr_words * sizeof(u64)); + + /* clear word-0 bits below base */ + head_bits = m->base & 63; + if (head_bits) + m->bits[0] &= ~((1ULL << head_bits) - 1); + + /* clear last-word bits at or past base + nr_cids */ + tail_bits = (m->base + m->nr_cids) & 63; + if (tail_bits) + m->bits[nr_words - 1] &= (1ULL << tail_bits) - 1; +} + /** * scx_cpumask_to_cmask - Translate a kernel cpumask into a cmask * @src: source cpumask * @dst: cmask to write * - * Initialize @dst to cover the full cid space [0, num_possible_cpus()) and - * set the bit for each cid whose cpu is in @src. + * Clear @dst's active range and set the bit for each cid whose cpu is in + * @src and lies within that range. Out-of-range cids are silently ignored. */ void scx_cpumask_to_cmask(const struct cpumask *src, struct scx_cmask *dst) { s32 cpu; - scx_cmask_init(dst, 0, num_possible_cpus()); + scx_cmask_clear(dst); for_each_cpu(cpu, src) { s32 cid = __scx_cpu_to_cid(cpu); diff --git a/kernel/sched/ext_cid.h b/kernel/sched/ext_cid.h index e1c44a180bb1..223ed0e857ec 100644 --- a/kernel/sched/ext_cid.h +++ b/kernel/sched/ext_cid.h @@ -51,6 +51,8 @@ extern s16 *scx_cpu_to_cid_tbl; extern struct scx_cid_topo *scx_cid_topo; extern struct btf_id_set8 scx_kfunc_ids_init; +void scx_cmask_clear(struct scx_cmask *m); +void scx_cmask_fill(struct scx_cmask *m); s32 scx_cid_init(struct scx_sched *sch); int scx_cid_kfunc_init(void); void scx_cpumask_to_cmask(const struct cpumask *src, struct scx_cmask *dst); @@ -147,11 +149,64 @@ static inline u64 *__scx_cmask_word(const struct scx_cmask *m, u32 cid) return (u64 *)&m->bits[cid / 64 - m->base / 64]; } +/** + * __scx_cmask_init - Initialize @m with explicit storage capacity + * @m: cmask to initialize + * @base: first cid of the active range + * @nr_cids: number of cids in the active range + * @alloc_cids: storage capacity in cids, at least @nr_cids + * + * Use when storage is sized larger than the initial active range. All of + * bits[] is zeroed. + */ +static inline void __scx_cmask_init(struct scx_cmask *m, u32 base, u32 nr_cids, + u32 alloc_cids) +{ + if (WARN_ON_ONCE(alloc_cids < nr_cids)) + nr_cids = alloc_cids; + + m->base = base; + m->nr_cids = nr_cids; + m->alloc_words = SCX_CMASK_NR_WORDS(alloc_cids); + memset(m->bits, 0, m->alloc_words * sizeof(u64)); +} + +/** + * scx_cmask_init - Initialize @m on tight storage + * @m: cmask to initialize + * @base: first cid of the active range + * @nr_cids: number of cids in the active range + * + * All of bits[] is zeroed. + */ static inline void scx_cmask_init(struct scx_cmask *m, u32 base, u32 nr_cids) { + __scx_cmask_init(m, base, nr_cids, nr_cids); +} + +/** + * scx_cmask_reframe - Reshape @m's active range without resizing storage + * @m: cmask to reframe + * @base: new active range base + * @nr_cids: new active range length, must fit within @m->alloc_words + * + * Body bits within the new range become garbage - only the head and tail + * words are zeroed to keep the padding invariant. + */ +static inline void scx_cmask_reframe(struct scx_cmask *m, u32 base, u32 nr_cids) +{ + if (WARN_ON_ONCE(SCX_CMASK_NR_WORDS(nr_cids) > m->alloc_words)) + return; + + if (nr_cids) { + u32 last_word = ((base & 63) + nr_cids - 1) / 64; + + m->bits[0] = 0; + m->bits[last_word] = 0; + } + m->base = base; m->nr_cids = nr_cids; - memset(m->bits, 0, SCX_CMASK_NR_WORDS(nr_cids) * sizeof(u64)); } static inline void __scx_cmask_set(struct scx_cmask *m, u32 cid) diff --git a/kernel/sched/ext_types.h b/kernel/sched/ext_types.h index c6c4e3db7311..8b3527e21fca 100644 --- a/kernel/sched/ext_types.h +++ b/kernel/sched/ext_types.h @@ -69,9 +69,10 @@ struct scx_cid_topo { * * A cmask covers the cid range [base, base + nr_cids). bits[] is aligned to the * global 64-cid grid: bits[0] spans [base & ~63, (base & ~63) + 64), so the - * first (base & 63) bits of bits[0] are head padding and any tail past base + - * nr_cids is tail padding. Both must stay zero for the lifetime of the mask; - * all mutating helpers preserve that invariant. + * first (base & 63) bits of bits[0] are head padding and the trailing bits of + * the last active word past base + nr_cids are tail padding. Both stay zero; + * all mutating helpers preserve that. Words past the last active word are not + * read by any helper and have no constraint. * * Grid alignment means two cmasks always address bits[] against the same global * 64-cid windows, so cross-cmask word ops (AND, OR, ...) reduce to @@ -83,22 +84,61 @@ struct scx_cid_topo { struct scx_cmask { u32 base; u32 nr_cids; - DECLARE_FLEX_ARRAY(u64, bits); + u32 alloc_words; + u64 bits[] __counted_by(alloc_words); }; /* * Number of u64 words of bits[] storage that covers @nr_cids regardless of base * alignment. The +1 absorbs up to 63 bits of head padding when base is not * 64-aligned - always allocating one extra word beats branching on base or - * splitting the compute. + * splitting the compute. The u64 cast keeps the +63 from wrapping when @nr_cids + * is near U32_MAX, so callers bounds-checking the result against @alloc_words + * catch the overflow instead of seeing a small value. */ -#define SCX_CMASK_NR_WORDS(nr_cids) (((nr_cids) + 63) / 64 + 1) +#define SCX_CMASK_NR_WORDS(nr_cids) ((u32)(((u64)(nr_cids) + 63) / 64 + 1)) -/* - * Define an on-stack cmask for up to @cap_bits. @name is a struct scx_cmask * - * aliasing zero-initialized storage; call scx_cmask_init() to set base/nr_cids. +/** + * __SCX_CMASK_DEFINE - Define an on-stack cmask with explicit storage capacity + * @NAME: variable name to define + * @BASE: first cid of the active range + * @NR_CIDS: active range length + * @ALLOC_CIDS: storage capacity in cids, at least @NR_CIDS + * + * @NAME aliases zero-initialized storage with the active range set to + * [BASE, BASE + NR_CIDS). Use scx_cmask_reframe() to reshape later, up to + * @ALLOC_CIDS. + */ +#define __SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, ALLOC_CIDS) \ + _DEFINE_FLEX(struct scx_cmask, NAME, bits, SCX_CMASK_NR_WORDS(ALLOC_CIDS), \ + = { .base = (BASE), \ + .nr_cids = (NR_CIDS), \ + .alloc_words = SCX_CMASK_NR_WORDS(ALLOC_CIDS) }) + +/** + * SCX_CMASK_DEFINE - Define an on-stack cmask on tight storage + * @NAME: variable name to define + * @BASE: first cid of the active range + * @NR_CIDS: active range length, also storage capacity + * + * @NAME aliases zero-initialized storage with the active range and storage + * both [BASE, BASE + NR_CIDS). + */ +#define SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS) \ + __SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, NR_CIDS) + +/** + * SCX_CMASK_DEFINE_SHARD - Define an on-stack cmask sized to one shard + * @NAME: variable name to define + * @BASE: first cid of the active range + * @NR_CIDS: active range length, must be <= SCX_CID_SHARD_MAX_CPUS + * + * Storage is fixed at SCX_CID_SHARD_MAX_CPUS, active range framed by + * (BASE, NR_CIDS). Passing NR_CIDS > SCX_CID_SHARD_MAX_CPUS leaves the + * cmask claiming more bits than storage holds and subsequent cmask + * operations will overrun. */ -#define SCX_CMASK_DEFINE(name, cap_bits) \ - DEFINE_RAW_FLEX(struct scx_cmask, name, bits, SCX_CMASK_NR_WORDS(cap_bits)) +#define SCX_CMASK_DEFINE_SHARD(NAME, BASE, NR_CIDS) \ + __SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, SCX_CID_SHARD_MAX_CPUS) #endif /* _KERNEL_SCHED_EXT_TYPES_H */ -- cgit v1.2.3 From f31e89a8f583cb8b1b68002cedb77ce444d6f7d2 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 18 May 2026 21:53:11 -1000 Subject: sched_ext: Add cmask mask ops Sub-sched cap code and other upcoming consumers need bulk cmask ops, both mutating (and/or/copy/andnot) and predicate (subset/intersects/empty). cmask_walk_op2() walks the intersection of two ranges word by word; cmask_walk_op1() walks one range. Both are __always_inline and dispatched on a compile-time-constant op enum, so each public entry collapses to a specialized loop with the inner switch reduced to one arm. Two-cmask ops only touch bits in the intersection of the two ranges; bits outside are left unchanged. scx_cmask_or_racy() and scx_cmask_copy_racy() mirror the locking forms but read @src word-by-word through data_race(); callers handle ordering with concurrent writers themselves. v2: Add scx_cmask_empty(). Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext_cid.c | 270 +++++++++++++++++++++++++++++++++++++++++++++++++ kernel/sched/ext_cid.h | 9 ++ 2 files changed, 279 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/ext_cid.c b/kernel/sched/ext_cid.c index 44dd47a87709..0c91b951fd33 100644 --- a/kernel/sched/ext_cid.c +++ b/kernel/sched/ext_cid.c @@ -397,6 +397,276 @@ __bpf_kfunc s32 scx_bpf_cpu_to_cid(s32 cpu, const struct bpf_prog_aux *aux) return scx_cpu_to_cid(sch, cpu); } +/* + * Set ops on cmasks. cmask_walk_op2() shares one walk across mutating + * (and/or/copy/andnot) and predicate (subset/intersects) two-cmask forms; + * cmask_walk_op1() does the same shape over a single cmask range. Every public + * entry passes a compile-time-constant @op; cmask_walk_op{1,2}() and + * cmask_word_op{1,2}() are __always_inline so the inner switch collapses to the + * selected op and cmask_op2_is_pred() folds the predicate early-exit out of + * mutating ops. + * + * Two-cmask ops only touch @dst bits inside the intersection of the two ranges; + * bits outside stay untouched. In particular, scx_cmask_copy() does NOT zero + * @dst bits that lie outside @src's range. + * + * The _RACY variants are otherwise identical to their non-racy counterpart but + * read @src word-by-word via data_race(). Memory ordering with concurrent + * writers is the caller's responsibility. + */ +enum cmask_op2 { + /* mutating */ + CMASK_OP2_AND, + CMASK_OP2_OR, + CMASK_OP2_OR_RACY, + CMASK_OP2_COPY, + CMASK_OP2_COPY_RACY, + CMASK_OP2_ANDNOT, + /* predicates - short-circuit when the per-word result is true */ + CMASK_OP2_SUBSET, + CMASK_OP2_INTERSECTS, +}; + +static __always_inline bool cmask_op2_is_pred(const enum cmask_op2 op) +{ + return op == CMASK_OP2_SUBSET || op == CMASK_OP2_INTERSECTS; +} + +static __always_inline bool cmask_word_op2(u64 *av, const u64 *bp, u64 mask, + const enum cmask_op2 op) +{ + switch (op) { + case CMASK_OP2_AND: + *av &= ~mask | *bp; + return false; + case CMASK_OP2_OR: + *av |= *bp & mask; + return false; + case CMASK_OP2_OR_RACY: + *av |= data_race(*bp) & mask; + return false; + case CMASK_OP2_COPY: + *av = (*av & ~mask) | (*bp & mask); + return false; + case CMASK_OP2_COPY_RACY: + *av = (*av & ~mask) | (data_race(*bp) & mask); + return false; + case CMASK_OP2_ANDNOT: + *av &= ~(*bp & mask); + return false; + case CMASK_OP2_SUBSET: + /* stop on the first bit in @sub not set in @super */ + return (*bp & ~*av) & mask; + case CMASK_OP2_INTERSECTS: + return (*av & *bp) & mask; + } + unreachable(); +} + +/* + * Walk the intersection of [@a_base, @a_base + @a_nr_cids) with [@b_base, + * @b_base + @b_nr_cids) word by word, applying @op. Mutating ops walk all words + * and return false; predicates return true on the first word whose per-word + * test is true. Empty intersection returns false (matches "no bits to consider" + * for both mutate and predicate). + * + * Base/nr_cids are taken as parameters so callers with snapshotted bounds can + * drive the walk with values independent of the cmask's header. + */ +static __always_inline bool cmask_walk_op2(u64 *a_bits, u32 a_base, u32 a_nr_cids, + const u64 *b_bits, u32 b_base, u32 b_nr_cids, + const enum cmask_op2 op) +{ + u32 lo = max(a_base, b_base); + u32 hi = min(a_base + a_nr_cids, b_base + b_nr_cids); + u32 a_word_off = a_base / 64; + u32 b_word_off = b_base / 64; + u32 lo_word = lo / 64; + u32 hi_word = (hi - 1) / 64; + u64 head_mask = GENMASK_U64(63, lo & 63); + u64 tail_mask = GENMASK_U64((hi - 1) & 63, 0); + u32 w; + + if (lo >= hi) + return false; + + if (lo_word == hi_word) + return cmask_word_op2(&a_bits[lo_word - a_word_off], + &b_bits[lo_word - b_word_off], + head_mask & tail_mask, op); + + if (cmask_word_op2(&a_bits[lo_word - a_word_off], + &b_bits[lo_word - b_word_off], head_mask, op) && + cmask_op2_is_pred(op)) + return true; + + for (w = lo_word + 1; w < hi_word; w++) + if (cmask_word_op2(&a_bits[w - a_word_off], + &b_bits[w - b_word_off], ~0ULL, op) && + cmask_op2_is_pred(op)) + return true; + + return cmask_word_op2(&a_bits[hi_word - a_word_off], + &b_bits[hi_word - b_word_off], tail_mask, op); +} + +enum cmask_op1 { + CMASK_OP1_ANY_SET, +}; + +static __always_inline bool cmask_word_op1(const u64 *ap, u64 mask, + const enum cmask_op1 op) +{ + switch (op) { + case CMASK_OP1_ANY_SET: + return *ap & mask; + } + unreachable(); +} + +/* + * Walk [@a_base, @a_base + @a_nr_cids) of @a_bits word by word, applying @op. + * Returns true on the first word whose per-word test is true; returns false if + * no word matches or the range is empty. All current op1s short-circuit on + * per-word true; if a non-predicate op1 lands here, add a cmask_op1_is_pred() + * guard analogous to cmask_op2_is_pred(). + */ +static __always_inline bool cmask_walk_op1(const u64 *a_bits, u32 a_base, + u32 a_nr_cids, + const enum cmask_op1 op) +{ + u32 lo = a_base; + u32 hi = a_base + a_nr_cids; + u32 a_word_off = a_base / 64; + u32 lo_word = lo / 64; + u32 hi_word = (hi - 1) / 64; + u64 head_mask = GENMASK_U64(63, lo & 63); + u64 tail_mask = GENMASK_U64((hi - 1) & 63, 0); + u32 w; + + if (lo >= hi) + return false; + + if (lo_word == hi_word) + return cmask_word_op1(&a_bits[lo_word - a_word_off], + head_mask & tail_mask, op); + + if (cmask_word_op1(&a_bits[lo_word - a_word_off], head_mask, op)) + return true; + for (w = lo_word + 1; w < hi_word; w++) + if (cmask_word_op1(&a_bits[w - a_word_off], ~0ULL, op)) + return true; + return cmask_word_op1(&a_bits[hi_word - a_word_off], tail_mask, op); +} + +void scx_cmask_and(struct scx_cmask *dst, const struct scx_cmask *src) +{ + cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, + src->bits, src->base, src->nr_cids, CMASK_OP2_AND); +} + +void scx_cmask_or(struct scx_cmask *dst, const struct scx_cmask *src) +{ + cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, + src->bits, src->base, src->nr_cids, CMASK_OP2_OR); +} + +/** + * scx_cmask_or_racy - OR @src into @dst, reading @src without locking + * + * @src is read word-by-word through data_race(). Same per-bit independence + * rationale as scx_cmask_copy_racy(). Memory ordering with writers is the + * caller's responsibility. + */ +void scx_cmask_or_racy(struct scx_cmask *dst, const struct scx_cmask *src) +{ + cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, + src->bits, src->base, src->nr_cids, CMASK_OP2_OR_RACY); +} + +void scx_cmask_copy(struct scx_cmask *dst, const struct scx_cmask *src) +{ + cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, + src->bits, src->base, src->nr_cids, CMASK_OP2_COPY); +} + +/** + * scx_cmask_copy_racy - Snapshot @src into @dst without locking + * + * @src is read word-by-word through data_race(). Head/tail masking matches + * scx_cmask_copy(). Each bit in a cmask is independent, so partial updates + * just leave some bits fresher than others. Memory ordering with writers is + * the caller's responsibility. + */ +void scx_cmask_copy_racy(struct scx_cmask *dst, const struct scx_cmask *src) +{ + cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, + src->bits, src->base, src->nr_cids, CMASK_OP2_COPY_RACY); +} + +void scx_cmask_andnot(struct scx_cmask *dst, const struct scx_cmask *src) +{ + cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, + src->bits, src->base, src->nr_cids, CMASK_OP2_ANDNOT); +} + +/* + * Return true if @cm has any bit set in [@lo, @hi). Caller must ensure + * [@lo, @hi) is contained in @cm's range. + */ +static bool cmask_any_set_in_range(const struct scx_cmask *cm, u32 lo, u32 hi) +{ + if (lo >= hi) + return false; + return cmask_walk_op1(&cm->bits[lo / 64 - cm->base / 64], lo, hi - lo, + CMASK_OP1_ANY_SET); +} + +/** + * scx_cmask_subset - test whether @sub is a subset of @super + * @sub: cmask to test + * @super: cmask to test against + * + * Return true iff every set bit of @sub is also set in @super. + */ +bool scx_cmask_subset(const struct scx_cmask *sub, const struct scx_cmask *super) +{ + u32 super_end = super->base + super->nr_cids; + u32 sub_end = sub->base + sub->nr_cids; + + /* + * Set bits in @sub outside @super's range can't be in @super, so any + * such bit means not a subset. The walk below only visits words + * common to both ranges, so these need a separate scan. + */ + if (sub->base < super->base && + cmask_any_set_in_range(sub, sub->base, min(super->base, sub_end))) + return false; + if (sub_end > super_end && + cmask_any_set_in_range(sub, max(sub->base, super_end), sub_end)) + return false; + + return !cmask_walk_op2((u64 *)super->bits, super->base, super->nr_cids, + sub->bits, sub->base, sub->nr_cids, CMASK_OP2_SUBSET); +} + +bool scx_cmask_intersects(const struct scx_cmask *a, const struct scx_cmask *b) +{ + return cmask_walk_op2((u64 *)a->bits, a->base, a->nr_cids, + b->bits, b->base, b->nr_cids, CMASK_OP2_INTERSECTS); +} + +/** + * scx_cmask_empty - Test whether @m has no bits set + * @m: cmask to test + * + * Return true iff @m's active range has no bits set. + */ +bool scx_cmask_empty(const struct scx_cmask *m) +{ + return !cmask_any_set_in_range(m, m->base, m->base + m->nr_cids); +} + /** * scx_bpf_cid_topo - Copy out per-cid topology info * @cid: cid to look up diff --git a/kernel/sched/ext_cid.h b/kernel/sched/ext_cid.h index 223ed0e857ec..abea22ba2cc2 100644 --- a/kernel/sched/ext_cid.h +++ b/kernel/sched/ext_cid.h @@ -53,6 +53,15 @@ extern struct btf_id_set8 scx_kfunc_ids_init; void scx_cmask_clear(struct scx_cmask *m); void scx_cmask_fill(struct scx_cmask *m); +void scx_cmask_and(struct scx_cmask *dst, const struct scx_cmask *src); +void scx_cmask_or(struct scx_cmask *dst, const struct scx_cmask *src); +void scx_cmask_or_racy(struct scx_cmask *dst, const struct scx_cmask *src); +void scx_cmask_copy(struct scx_cmask *dst, const struct scx_cmask *src); +void scx_cmask_copy_racy(struct scx_cmask *dst, const struct scx_cmask *src); +void scx_cmask_andnot(struct scx_cmask *dst, const struct scx_cmask *src); +bool scx_cmask_subset(const struct scx_cmask *sub, const struct scx_cmask *super); +bool scx_cmask_intersects(const struct scx_cmask *a, const struct scx_cmask *b); +bool scx_cmask_empty(const struct scx_cmask *m); s32 scx_cid_init(struct scx_sched *sch); int scx_cid_kfunc_init(void); void scx_cpumask_to_cmask(const struct cpumask *src, struct scx_cmask *dst); -- cgit v1.2.3 From cb339ac61d72f7fb7f57bfc0516b7b2b65bc1bad Mon Sep 17 00:00:00 2001 From: Kaitao Cheng Date: Thu, 21 May 2026 11:22:59 +0800 Subject: bpf: refactor __bpf_list_del to take list node pointer Refactor __bpf_list_del to accept (head, struct list_head *n) instead of (head, bool tail). The caller now passes the specific node to remove: bpf_list_pop_front passes h->next, bpf_list_pop_back passes h->prev. Prepares for introducing bpf_list_del(head, node) kfunc to remove an arbitrary node when the user holds ownership. Signed-off-by: Kaitao Cheng Reviewed-by: Eduard Zingerman Link: https://lore.kernel.org/r/20260521032306.97118-2-kaitao.cheng@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/helpers.c | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 07de26e7314c..094457c3e6d3 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -2550,37 +2550,44 @@ __bpf_kfunc int bpf_list_push_back_impl(struct bpf_list_head *head, return bpf_list_push_back(head, node, meta__ign, off); } -static struct bpf_list_node *__bpf_list_del(struct bpf_list_head *head, bool tail) +static struct bpf_list_node *__bpf_list_del(struct bpf_list_head *head, + struct list_head *n) { - struct list_head *n, *h = (void *)head; + struct list_head *h = (void *)head; struct bpf_list_node_kern *node; /* If list_head was 0-initialized by map, bpf_obj_init_field wasn't * called on its fields, so init here */ - if (unlikely(!h->next)) + if (unlikely(!h->next)) { INIT_LIST_HEAD(h); + return NULL; + } if (list_empty(h)) return NULL; - n = tail ? h->prev : h->next; node = container_of(n, struct bpf_list_node_kern, list_head); - if (WARN_ON_ONCE(READ_ONCE(node->owner) != head)) + if (unlikely(READ_ONCE(node->owner) != head)) return NULL; list_del_init(n); - WRITE_ONCE(node->owner, NULL); + /* Ensure __bpf_list_add() sees the node as unlinked. */ + smp_store_release(&node->owner, NULL); return (struct bpf_list_node *)n; } __bpf_kfunc struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) { - return __bpf_list_del(head, false); + struct list_head *h = (void *)head; + + return __bpf_list_del(head, h->next); } __bpf_kfunc struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) { - return __bpf_list_del(head, true); + struct list_head *h = (void *)head; + + return __bpf_list_del(head, h->prev); } __bpf_kfunc struct bpf_list_node *bpf_list_front(struct bpf_list_head *head) -- cgit v1.2.3 From cfa6afa4b931aed08288454943e5077f114fd7f3 Mon Sep 17 00:00:00 2001 From: Kaitao Cheng Date: Thu, 21 May 2026 11:23:00 +0800 Subject: bpf: clear list node owner and unlink before drop The issue only becomes exposed once bpf_list_del() is available: callers can pass an arbitrary bpf_list_head and bpf_list_node pair, including nodes that are not actually linked to the supplied head, or nodes that outlive their original head after refcount-based retention. This was not practically reachable for callers restricted to pop-style helpers alone; bpf_list_del() widens the API surface. A failure mode appears when bpf_list_head_free() runs while a program still holds an independent refcount on a node (for example via bpf_refcount_acquire()). The list head value embedded in map memory can go away while the node object survives. If node->owner is left pointing at the old head address until drop completes, that pointer becomes stale. If a new bpf_list_head is later allocated at the same address and the stale node is passed to bpf_list_del(), the owner comparison can succeed even though the node is not really linked to the new head, and list_del_init() will follow bogus next/prev pointers with the risk of memory corruption. When draining a bpf_list_head, mark each node owner with BPF_PTR_POISON under the map spinlock while moving it to a private drain list, then list_del_init() the node and clear owner to NULL before calling __bpf_obj_drop_impl(). Concurrent readers therefore never observe a node that appears linked to a head while its list_head is inconsistent, and surviving refcounted nodes never retain a stale non-NULL owner. Signed-off-by: Kaitao Cheng Link: https://lore.kernel.org/r/20260521032306.97118-3-kaitao.cheng@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/helpers.c | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 094457c3e6d3..59855b434f0b 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -2247,10 +2247,11 @@ EXPORT_SYMBOL_GPL(bpf_base_func_proto); void bpf_list_head_free(const struct btf_field *field, void *list_head, struct bpf_spin_lock *spin_lock) { - struct list_head *head = list_head, *orig_head = list_head; + struct list_head *head = list_head, drain, *pos, *n; BUILD_BUG_ON(sizeof(struct list_head) > sizeof(struct bpf_list_head)); BUILD_BUG_ON(__alignof__(struct list_head) > __alignof__(struct bpf_list_head)); + INIT_LIST_HEAD(&drain); /* Do the actual list draining outside the lock to not hold the lock for * too long, and also prevent deadlocks if tracing programs end up @@ -2261,20 +2262,30 @@ void bpf_list_head_free(const struct btf_field *field, void *list_head, __bpf_spin_lock_irqsave(spin_lock); if (!head->next || list_empty(head)) goto unlock; - head = head->next; + list_for_each_safe(pos, n, head) { + struct bpf_list_node_kern *node; + + node = container_of(pos, struct bpf_list_node_kern, list_head); + WRITE_ONCE(node->owner, BPF_PTR_POISON); + list_move_tail(pos, &drain); + } unlock: - INIT_LIST_HEAD(orig_head); + INIT_LIST_HEAD(head); __bpf_spin_unlock_irqrestore(spin_lock); - while (head != orig_head) { - void *obj = head; + while (!list_empty(&drain)) { + struct bpf_list_node_kern *node; - obj -= field->graph_root.node_offset; - head = head->next; + pos = drain.next; + node = container_of(pos, struct bpf_list_node_kern, list_head); + list_del_init(pos); + /* Ensure __bpf_list_add() sees the node as unlinked. */ + smp_store_release(&node->owner, NULL); /* The contained type can also have resources, including a * bpf_list_head which needs to be freed. */ - __bpf_obj_drop_impl(obj, field->graph_root.value_rec, false); + __bpf_obj_drop_impl((char *)pos - field->graph_root.node_offset, + field->graph_root.value_rec, false); } } -- cgit v1.2.3 From 7c8c71591b768284caa81e92cb47a6912952d3f2 Mon Sep 17 00:00:00 2001 From: Kaitao Cheng Date: Thu, 21 May 2026 11:23:01 +0800 Subject: bpf: allow non-owning list-node args via __nonown_allowed KF_ARG_PTR_TO_LIST_NODE normally requires an owning reference (PTR_TO_BTF_ID | MEM_ALLOC with ref_obj_id). Introduce the __nonown_allowed annotation on selected list-node arguments so non-owning references with ref_obj_id==0 are accepted as well. This patch only adds the generic verifier support and documents the annotation. Later patches in the series will apply it to bpf_list_add /del(), and bpf_list_is_first/last(), allowing bpf_list_front/back() results to be used as the insertion point, deletion target, or query target for those kfuncs. Verifier keeps existing owning-ref checks by default; only arguments annotated with __nonown_allowed bypass MEM_ALLOC/ref_obj_id checks and then follow the same list-node validation path. Signed-off-by: Kaitao Cheng Reviewed-by: Eduard Zingerman Link: https://lore.kernel.org/r/20260521032306.97118-4-kaitao.cheng@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 8dd79b735a69..f3cf8d85bea0 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -10714,6 +10714,11 @@ static bool is_kfunc_arg_nullable(const struct btf *btf, const struct btf_param return btf_param_match_suffix(btf, arg, "__nullable"); } +static bool is_kfunc_arg_nonown_allowed(const struct btf *btf, const struct btf_param *arg) +{ + return btf_param_match_suffix(btf, arg, "__nonown_allowed"); +} + static bool is_kfunc_arg_const_str(const struct btf *btf, const struct btf_param *arg) { return btf_param_match_suffix(btf, arg, "__str"); @@ -12244,6 +12249,13 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ return ret; break; case KF_ARG_PTR_TO_LIST_NODE: + if (is_kfunc_arg_nonown_allowed(btf, &args[i]) && + type_is_non_owning_ref(reg->type) && !reg->ref_obj_id) { + /* Allow bpf_list_front/back return value for + * __nonown_allowed list-node arguments. + */ + goto check_ok; + } if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { verbose(env, "%s expected pointer to allocated object\n", reg_arg_name(env, argno)); @@ -12253,6 +12265,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ verbose(env, "allocated object must be referenced\n"); return -EINVAL; } +check_ok: ret = process_kf_arg_ptr_to_list_node(env, reg, argno, meta); if (ret < 0) return ret; -- cgit v1.2.3 From 187baa10963ac9f1db5123fa2ab761ab34ea06b9 Mon Sep 17 00:00:00 2001 From: Kaitao Cheng Date: Thu, 21 May 2026 11:23:02 +0800 Subject: bpf: Introduce the bpf_list_del kfunc. Allow users to remove any node from a linked list. We have added an additional parameter bpf_list_head *head to bpf_list_del, as the verifier requires the head parameter to check whether the lock is being held. Signed-off-by: Kaitao Cheng Reviewed-by: Eduard Zingerman Link: https://lore.kernel.org/r/20260521032306.97118-5-kaitao.cheng@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/helpers.c | 10 ++++++++++ kernel/bpf/verifier.c | 6 +++++- 2 files changed, 15 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 59855b434f0b..804c201c28f3 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -2601,6 +2601,15 @@ __bpf_kfunc struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) return __bpf_list_del(head, h->prev); } +__bpf_kfunc struct bpf_list_node *bpf_list_del(struct bpf_list_head *head, + struct bpf_list_node *node__nonown_allowed) +{ + struct bpf_list_node_kern *kn = (void *)node__nonown_allowed; + + /* verifier guarantees node is a list node rather than list head */ + return __bpf_list_del(head, &kn->list_head); +} + __bpf_kfunc struct bpf_list_node *bpf_list_front(struct bpf_list_head *head) { struct list_head *h = (struct list_head *)head; @@ -4733,6 +4742,7 @@ BTF_ID_FLAGS(func, bpf_list_push_back, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_list_push_back_impl) BTF_ID_FLAGS(func, bpf_list_pop_front, KF_ACQUIRE | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_list_pop_back, KF_ACQUIRE | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_list_del, KF_ACQUIRE | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_list_front, KF_RET_NULL) BTF_ID_FLAGS(func, bpf_list_back, KF_RET_NULL) BTF_ID_FLAGS(func, bpf_task_acquire, KF_ACQUIRE | KF_RCU | KF_RET_NULL) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index f3cf8d85bea0..35eebb5e7769 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -10961,6 +10961,7 @@ enum special_kfunc_type { KF_bpf_list_push_back, KF_bpf_list_pop_front, KF_bpf_list_pop_back, + KF_bpf_list_del, KF_bpf_list_front, KF_bpf_list_back, KF_bpf_cast_to_kern_ctx, @@ -11029,6 +11030,7 @@ BTF_ID(func, bpf_list_push_back_impl) BTF_ID(func, bpf_list_push_back) BTF_ID(func, bpf_list_pop_front) BTF_ID(func, bpf_list_pop_back) +BTF_ID(func, bpf_list_del) BTF_ID(func, bpf_list_front) BTF_ID(func, bpf_list_back) BTF_ID(func, bpf_cast_to_kern_ctx) @@ -11549,6 +11551,7 @@ static bool is_bpf_list_api_kfunc(u32 btf_id) return is_bpf_list_push_kfunc(btf_id) || btf_id == special_kfunc_list[KF_bpf_list_pop_front] || btf_id == special_kfunc_list[KF_bpf_list_pop_back] || + btf_id == special_kfunc_list[KF_bpf_list_del] || btf_id == special_kfunc_list[KF_bpf_list_front] || btf_id == special_kfunc_list[KF_bpf_list_back]; } @@ -11671,7 +11674,8 @@ static bool check_kfunc_is_graph_node_api(struct bpf_verifier_env *env, switch (node_field_type) { case BPF_LIST_NODE: - ret = is_bpf_list_push_kfunc(kfunc_btf_id); + ret = is_bpf_list_push_kfunc(kfunc_btf_id) || + kfunc_btf_id == special_kfunc_list[KF_bpf_list_del]; break; case BPF_RB_NODE: ret = (is_bpf_rbtree_add_kfunc(kfunc_btf_id) || -- cgit v1.2.3 From e6919ff67c1e612ec1ce3be51dba6e7ffc47997a Mon Sep 17 00:00:00 2001 From: Kaitao Cheng Date: Thu, 21 May 2026 11:23:03 +0800 Subject: bpf: refactor __bpf_list_add to take insertion point via **prev_ptr Refactor __bpf_list_add to accept (node, head, struct list_head **prev_ptr, ..) instead of (node, head, bool tail, ..). Load prev from *prev_ptr after INIT_LIST_HEAD(h), so we never dereference an uninitialized h->prev when head was 0-initialized (e.g. push_back passes &h->prev). When prev is not the list head, validate that prev is in the list via its owner. Prepares for bpf_list_add(head, new, prev, ..) to insert after a given list node. Signed-off-by: Kaitao Cheng Reviewed-by: Eduard Zingerman Link: https://lore.kernel.org/r/20260521032306.97118-6-kaitao.cheng@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/helpers.c | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 804c201c28f3..1c69476c8a09 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -2478,9 +2478,11 @@ __bpf_kfunc void *bpf_refcount_acquire_impl(void *p__refcounted_kptr, void *meta static int __bpf_list_add(struct bpf_list_node_kern *node, struct bpf_list_head *head, - bool tail, struct btf_record *rec, u64 off) + struct list_head **prev_ptr, + struct btf_record *rec, u64 off) { struct list_head *n = &node->list_head, *h = (void *)head; + struct list_head *prev; /* If list_head was 0-initialized by map, bpf_obj_init_field wasn't * called on its fields, so init here @@ -2488,19 +2490,31 @@ static int __bpf_list_add(struct bpf_list_node_kern *node, if (unlikely(!h->next)) INIT_LIST_HEAD(h); + prev = *prev_ptr; + + /* When prev is not the list head, it must be a node in this list. */ + if (prev != h) { + struct bpf_list_node_kern *prev_kn = + container_of(prev, struct bpf_list_node_kern, list_head); + + if (unlikely(READ_ONCE(prev_kn->owner) != head)) + goto fail; + } + /* node->owner != NULL implies !list_empty(n), no need to separately * check the latter */ - if (cmpxchg(&node->owner, NULL, BPF_PTR_POISON)) { - /* Only called from BPF prog, no need to migrate_disable */ - __bpf_obj_drop_impl((void *)n - off, rec, false); - return -EINVAL; - } + if (cmpxchg(&node->owner, NULL, BPF_PTR_POISON)) + goto fail; - tail ? list_add_tail(n, h) : list_add(n, h); + list_add(n, prev); WRITE_ONCE(node->owner, head); - return 0; + +fail: + /* Only called from BPF prog, no need to migrate_disable */ + __bpf_obj_drop_impl((void *)n - off, rec, false); + return -EINVAL; } /** @@ -2521,8 +2535,9 @@ __bpf_kfunc int bpf_list_push_front(struct bpf_list_head *head, u64 off) { struct bpf_list_node_kern *n = (void *)node; + struct list_head *h = (void *)head; - return __bpf_list_add(n, head, false, meta ? meta->record : NULL, off); + return __bpf_list_add(n, head, &h, meta ? meta->record : NULL, off); } __bpf_kfunc int bpf_list_push_front_impl(struct bpf_list_head *head, @@ -2550,8 +2565,9 @@ __bpf_kfunc int bpf_list_push_back(struct bpf_list_head *head, u64 off) { struct bpf_list_node_kern *n = (void *)node; + struct list_head *h = (void *)head; - return __bpf_list_add(n, head, true, meta ? meta->record : NULL, off); + return __bpf_list_add(n, head, &h->prev, meta ? meta->record : NULL, off); } __bpf_kfunc int bpf_list_push_back_impl(struct bpf_list_head *head, -- cgit v1.2.3 From a3493ca504f16877bf29a123f27835c3f841a05f Mon Sep 17 00:00:00 2001 From: Kaitao Cheng Date: Thu, 21 May 2026 11:23:04 +0800 Subject: bpf: Add bpf_list_add to insert node after a given list node Add a new kfunc bpf_list_add(head, new, prev, meta, off) that inserts 'new' after 'prev' in the BPF linked list. Both must be in the same list; 'prev' must already be in the list. The new node must be an owning reference (e.g. from bpf_obj_new); the kfunc consumes that reference and the node becomes non-owning once inserted. We have added an additional parameter bpf_list_head *head to bpf_list_add, as the verifier requires the head parameter to check whether the lock is being held. Returns 0 on success, -EINVAL if 'prev' is not in a list or 'new' is already in a list (or duplicate insertion). On failure, the kernel drops the passed-in node. Signed-off-by: Kaitao Cheng Reviewed-by: Eduard Zingerman Link: https://lore.kernel.org/r/20260521032306.97118-7-kaitao.cheng@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/helpers.c | 11 +++++++++++ kernel/bpf/verifier.c | 12 +++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 1c69476c8a09..89579165ef4d 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -2577,6 +2577,16 @@ __bpf_kfunc int bpf_list_push_back_impl(struct bpf_list_head *head, return bpf_list_push_back(head, node, meta__ign, off); } +__bpf_kfunc int bpf_list_add(struct bpf_list_head *head, struct bpf_list_node *new, + struct bpf_list_node *prev__nonown_allowed, + struct btf_struct_meta *meta, u64 off) +{ + struct bpf_list_node_kern *n = (void *)new, *p = (void *)prev__nonown_allowed; + struct list_head *prev_ptr = &p->list_head; + + return __bpf_list_add(n, head, &prev_ptr, meta ? meta->record : NULL, off); +} + static struct bpf_list_node *__bpf_list_del(struct bpf_list_head *head, struct list_head *n) { @@ -4756,6 +4766,7 @@ BTF_ID_FLAGS(func, bpf_list_push_front, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_list_push_front_impl) BTF_ID_FLAGS(func, bpf_list_push_back, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_list_push_back_impl) +BTF_ID_FLAGS(func, bpf_list_add, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_list_pop_front, KF_ACQUIRE | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_list_pop_back, KF_ACQUIRE | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_list_del, KF_ACQUIRE | KF_RET_NULL) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 35eebb5e7769..662ad7312697 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -10959,6 +10959,7 @@ enum special_kfunc_type { KF_bpf_list_push_front, KF_bpf_list_push_back_impl, KF_bpf_list_push_back, + KF_bpf_list_add, KF_bpf_list_pop_front, KF_bpf_list_pop_back, KF_bpf_list_del, @@ -11028,6 +11029,7 @@ BTF_ID(func, bpf_list_push_front_impl) BTF_ID(func, bpf_list_push_front) BTF_ID(func, bpf_list_push_back_impl) BTF_ID(func, bpf_list_push_back) +BTF_ID(func, bpf_list_add) BTF_ID(func, bpf_list_pop_front) BTF_ID(func, bpf_list_pop_back) BTF_ID(func, bpf_list_del) @@ -11140,7 +11142,8 @@ static bool is_bpf_list_push_kfunc(u32 func_id) return func_id == special_kfunc_list[KF_bpf_list_push_front] || func_id == special_kfunc_list[KF_bpf_list_push_front_impl] || func_id == special_kfunc_list[KF_bpf_list_push_back] || - func_id == special_kfunc_list[KF_bpf_list_push_back_impl]; + func_id == special_kfunc_list[KF_bpf_list_push_back_impl] || + func_id == special_kfunc_list[KF_bpf_list_add]; } static bool is_bpf_rbtree_add_kfunc(u32 func_id) @@ -19524,8 +19527,11 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, int struct_meta_reg = BPF_REG_3; int node_offset_reg = BPF_REG_4; - /* rbtree_add has extra 'less' arg, so args-to-fixup are in diff regs */ - if (is_bpf_rbtree_add_kfunc(desc->func_id)) { + /* list_add/rbtree_add have an extra arg (prev/less), + * so args-to-fixup are in diff regs. + */ + if (desc->func_id == special_kfunc_list[KF_bpf_list_add] || + is_bpf_rbtree_add_kfunc(desc->func_id)) { struct_meta_reg = BPF_REG_4; node_offset_reg = BPF_REG_5; } -- cgit v1.2.3 From 745515d386eb5e6891d9f91a92ad15dace3a33ef Mon Sep 17 00:00:00 2001 From: Kaitao Cheng Date: Thu, 21 May 2026 11:23:05 +0800 Subject: bpf: add bpf_list_is_first/last/empty kfuncs Add three kfuncs for BPF linked list queries: - bpf_list_is_first(head, node): true if node is the first in the list. - bpf_list_is_last(head, node): true if node is the last in the list. - bpf_list_empty(head): true if the list has no entries. Currently, without these kfuncs, to implement the above functionality it is necessary to first call bpf_list_pop_front/back to retrieve the first or last node before checking whether the passed-in node was the first or last one. After the check, the node had to be pushed back into the list using bpf_list_push_front/back, which was very inefficient. Now, with the bpf_list_is_first/last/empty kfuncs, we can directly check whether a node is the first, last, or whether the list is empty, without having to first retrieve the node. Signed-off-by: Kaitao Cheng Reviewed-by: Emil Tsalapatis Link: https://lore.kernel.org/r/20260521032306.97118-8-kaitao.cheng@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/helpers.c | 40 ++++++++++++++++++++++++++++++++++++++++ kernel/bpf/verifier.c | 15 +++++++++++++-- 2 files changed, 53 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 89579165ef4d..b6c3d02d5593 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -2656,6 +2656,43 @@ __bpf_kfunc struct bpf_list_node *bpf_list_back(struct bpf_list_head *head) return (struct bpf_list_node *)h->prev; } +__bpf_kfunc bool bpf_list_is_first(struct bpf_list_head *head, + struct bpf_list_node *node__nonown_allowed) +{ + struct list_head *h = (struct list_head *)head; + struct bpf_list_node_kern *kn = (struct bpf_list_node_kern *)node__nonown_allowed; + + if (READ_ONCE(kn->owner) != head) + return false; + + return list_is_first(&kn->list_head, h); +} + +__bpf_kfunc bool bpf_list_is_last(struct bpf_list_head *head, + struct bpf_list_node *node__nonown_allowed) +{ + struct list_head *h = (struct list_head *)head; + struct bpf_list_node_kern *kn = (struct bpf_list_node_kern *)node__nonown_allowed; + + if (READ_ONCE(kn->owner) != head) + return false; + + return list_is_last(&kn->list_head, h); +} + +__bpf_kfunc bool bpf_list_empty(struct bpf_list_head *head) +{ + struct list_head *h = (struct list_head *)head; + + /* If list_head was 0-initialized by map, bpf_obj_init_field wasn't + * called on its fields, so init here + */ + if (unlikely(!h->next)) + INIT_LIST_HEAD(h); + + return list_empty(h); +} + __bpf_kfunc struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, struct bpf_rb_node *node) { @@ -4772,6 +4809,9 @@ BTF_ID_FLAGS(func, bpf_list_pop_back, KF_ACQUIRE | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_list_del, KF_ACQUIRE | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_list_front, KF_RET_NULL) BTF_ID_FLAGS(func, bpf_list_back, KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_list_is_first) +BTF_ID_FLAGS(func, bpf_list_is_last) +BTF_ID_FLAGS(func, bpf_list_empty) BTF_ID_FLAGS(func, bpf_task_acquire, KF_ACQUIRE | KF_RCU | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_task_release, KF_RELEASE) BTF_ID_FLAGS(func, bpf_rbtree_remove, KF_ACQUIRE | KF_RET_NULL) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 662ad7312697..d9bdc3b32c05 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -10965,6 +10965,9 @@ enum special_kfunc_type { KF_bpf_list_del, KF_bpf_list_front, KF_bpf_list_back, + KF_bpf_list_is_first, + KF_bpf_list_is_last, + KF_bpf_list_empty, KF_bpf_cast_to_kern_ctx, KF_bpf_rdonly_cast, KF_bpf_rcu_read_lock, @@ -11035,6 +11038,9 @@ BTF_ID(func, bpf_list_pop_back) BTF_ID(func, bpf_list_del) BTF_ID(func, bpf_list_front) BTF_ID(func, bpf_list_back) +BTF_ID(func, bpf_list_is_first) +BTF_ID(func, bpf_list_is_last) +BTF_ID(func, bpf_list_empty) BTF_ID(func, bpf_cast_to_kern_ctx) BTF_ID(func, bpf_rdonly_cast) BTF_ID(func, bpf_rcu_read_lock) @@ -11556,7 +11562,10 @@ static bool is_bpf_list_api_kfunc(u32 btf_id) btf_id == special_kfunc_list[KF_bpf_list_pop_back] || btf_id == special_kfunc_list[KF_bpf_list_del] || btf_id == special_kfunc_list[KF_bpf_list_front] || - btf_id == special_kfunc_list[KF_bpf_list_back]; + btf_id == special_kfunc_list[KF_bpf_list_back] || + btf_id == special_kfunc_list[KF_bpf_list_is_first] || + btf_id == special_kfunc_list[KF_bpf_list_is_last] || + btf_id == special_kfunc_list[KF_bpf_list_empty]; } static bool is_bpf_rbtree_api_kfunc(u32 btf_id) @@ -11678,7 +11687,9 @@ static bool check_kfunc_is_graph_node_api(struct bpf_verifier_env *env, switch (node_field_type) { case BPF_LIST_NODE: ret = is_bpf_list_push_kfunc(kfunc_btf_id) || - kfunc_btf_id == special_kfunc_list[KF_bpf_list_del]; + kfunc_btf_id == special_kfunc_list[KF_bpf_list_del] || + kfunc_btf_id == special_kfunc_list[KF_bpf_list_is_first] || + kfunc_btf_id == special_kfunc_list[KF_bpf_list_is_last]; break; case BPF_RB_NODE: ret = (is_bpf_rbtree_add_kfunc(kfunc_btf_id) || -- cgit v1.2.3 From a254b6d13b0edd6272926674d2afc46d46e496b7 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Wed, 20 May 2026 22:08:01 -0400 Subject: ring-buffer: Fix reporting of missed events in iterator When tracing is active while reading the trace file, if the iterator reading the buffer detects that the writer has passed the iterator head, it will reset and set a "missed events" flag. This flag is passed to the output processing to show the user that events were missed: CPU:4 [LOST EVENTS] The problem is that the flag is reset after it is checked in ring_buffer_iter_dropped(). But the "trace" file iterates over all the CPU ring buffers and it will check if they are dropped when figuring out which buffer to print next. This prematurely clears the missed_events flag if the CPU buffer with the missed events is not the one that is printed next. On the iteration where the CPU buffer with the missed events is printed, the check if it had missed events would return false and the output does not show that events were missed. Do not reset the missed_events flag when checking if there were missed events, but instead clear it when moving the iterator head to the next event. Cc: stable@vger.kernel.org Cc: Mathieu Desnoyers Link: https://patch.msgid.link/20260520220801.4fd09d13@fedora Fixes: c9b7a4a72ff64 ("ring-buffer/tracing: Have iterator acknowledge dropped events") Acked-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 5326924615a4..fcd93d49851e 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -5407,6 +5407,7 @@ static void rb_iter_reset(struct ring_buffer_iter *iter) iter->head_page = cpu_buffer->reader_page; iter->head = cpu_buffer->reader_page->read; iter->next_event = iter->head; + iter->missed_events = 0; iter->cache_reader_page = iter->head_page; iter->cache_read = cpu_buffer->read; @@ -6086,10 +6087,7 @@ ring_buffer_peek(struct trace_buffer *buffer, int cpu, u64 *ts, */ bool ring_buffer_iter_dropped(struct ring_buffer_iter *iter) { - bool ret = iter->missed_events != 0; - - iter->missed_events = 0; - return ret; + return iter->missed_events != 0; } EXPORT_SYMBOL_GPL(ring_buffer_iter_dropped); @@ -6251,7 +6249,7 @@ void ring_buffer_iter_advance(struct ring_buffer_iter *iter) unsigned long flags; raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); - + iter->missed_events = 0; rb_advance_iter(iter); raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); -- cgit v1.2.3 From a494d3c8d5392bcdff83c2a593df0c160ff9f322 Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Thu, 30 Apr 2026 12:28:16 +0900 Subject: ring-buffer: Flush and stop persistent ring buffer on panic On real hardware, panic and machine reboot may not flush hardware cache to memory. This means the persistent ring buffer, which relies on a coherent state of memory, may not have its events written to the buffer and they may be lost. Moreover, there may be inconsistency with the counters which are used for validation of the integrity of the persistent ring buffer which may cause all data to be discarded. To avoid this issue, stop recording of the ring buffer on panic and flush the cache of the ring buffer's memory. Fixes: e645535a954a ("tracing: Add option to use memmapped memory for trace boot instance") Cc: stable@vger.kernel.org Cc: Will Deacon Cc: Mathieu Desnoyers Cc: Ian Rogers Link: https://patch.msgid.link/177751969602.2136606.12031934362587643488.stgit@mhiramat.tok.corp.google.com Signed-off-by: Masami Hiramatsu (Google) Acked-by: Catalin Marinas Acked-by: Geert Uytterhoeven Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'kernel') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index fcd93d49851e..7b07d2004cc6 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,7 @@ #include #include +#include #include #include #include @@ -559,6 +561,7 @@ struct trace_buffer { unsigned long range_addr_start; unsigned long range_addr_end; + struct notifier_block flush_nb; struct ring_buffer_meta *meta; @@ -2521,6 +2524,16 @@ static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer) kfree(cpu_buffer); } +/* Stop recording on a persistent buffer and flush cache if needed. */ +static int rb_flush_buffer_cb(struct notifier_block *nb, unsigned long event, void *data) +{ + struct trace_buffer *buffer = container_of(nb, struct trace_buffer, flush_nb); + + ring_buffer_record_off(buffer); + arch_ring_buffer_flush_range(buffer->range_addr_start, buffer->range_addr_end); + return NOTIFY_DONE; +} + static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, int order, unsigned long start, unsigned long end, @@ -2651,6 +2664,12 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, mutex_init(&buffer->mutex); + /* Persistent ring buffer needs to flush cache before reboot. */ + if (start && end) { + buffer->flush_nb.notifier_call = rb_flush_buffer_cb; + atomic_notifier_chain_register(&panic_notifier_list, &buffer->flush_nb); + } + return_ptr(buffer); fail_free_buffers: @@ -2749,6 +2768,9 @@ ring_buffer_free(struct trace_buffer *buffer) { int cpu; + if (buffer->range_addr_start && buffer->range_addr_end) + atomic_notifier_chain_unregister(&panic_notifier_list, &buffer->flush_nb); + cpuhp_state_remove_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node); irq_work_sync(&buffer->irq_work.work); -- cgit v1.2.3 From c2d2856cf6c9efccdf5e0d2564162ec616ce58cf Mon Sep 17 00:00:00 2001 From: David Carlier Date: Tue, 12 May 2026 14:54:20 +0100 Subject: tracing: Fix nr_subbufs initialization in simple_ring_buffer_init_mm() nr_subbufs in the ring buffer metadata is always initialized to zero because it is assigned from cpu_buffer->nr_pages before the page initialization loop has run. While nr_subbufs is not currently read by the kernel, it should reflect the actual buffer geometry in the meta page for correctness. Move the assignment after the page loop so that cpu_buffer->nr_pages holds the final count. Link: https://patch.msgid.link/20260512135420.99194-1-devnexen@gmail.com Fixes: 34e5b958bdad ("tracing: Introduce simple_ring_buffer") Reviewed-by: Vincent Donnefort Assisted-by: Claude:claude-opus-4-7 Signed-off-by: David Carlier Signed-off-by: Steven Rostedt --- kernel/trace/simple_ring_buffer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/trace/simple_ring_buffer.c b/kernel/trace/simple_ring_buffer.c index 02af2297ae5a..f731f14d0ff7 100644 --- a/kernel/trace/simple_ring_buffer.c +++ b/kernel/trace/simple_ring_buffer.c @@ -395,7 +395,6 @@ int simple_ring_buffer_init_mm(struct simple_rb_per_cpu *cpu_buffer, memset(cpu_buffer->meta, 0, sizeof(*cpu_buffer->meta)); cpu_buffer->meta->meta_page_size = PAGE_SIZE; - cpu_buffer->meta->nr_subbufs = cpu_buffer->nr_pages; /* The reader page is not part of the ring initially */ page = load_page(desc->page_va[0]); @@ -437,6 +436,7 @@ int simple_ring_buffer_init_mm(struct simple_rb_per_cpu *cpu_buffer, return ret; } + cpu_buffer->meta->nr_subbufs = cpu_buffer->nr_pages; /* Close the ring */ bpage->link.next = &cpu_buffer->tail_page->link; cpu_buffer->tail_page->link.prev = &bpage->link; -- cgit v1.2.3 From a0a2f42a37f90b29d8c43374dd9c8bd2f3e7bdcc Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Tue, 12 May 2026 15:16:14 +0100 Subject: tracing: Fix unload_page for simple_ring_buffer init rollback The unload_page callback expects the return value of load_page() as its argument: ret = load_page(va); unload(ret). Fix the rollback code in simple_ring_buffer_init_mm() where the descriptor's VA is used instead of the loaded page address. Link: https://patch.msgid.link/20260512141614.1759430-1-vdonnefort@google.com Fixes: 635923081c79 ("tracing: load/unload page callbacks for simple_ring_buffer") Signed-off-by: Vincent Donnefort Signed-off-by: Steven Rostedt --- kernel/trace/simple_ring_buffer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/trace/simple_ring_buffer.c b/kernel/trace/simple_ring_buffer.c index f731f14d0ff7..f4642f5adda3 100644 --- a/kernel/trace/simple_ring_buffer.c +++ b/kernel/trace/simple_ring_buffer.c @@ -430,7 +430,7 @@ int simple_ring_buffer_init_mm(struct simple_rb_per_cpu *cpu_buffer, if (ret) { for (i--; i >= 0; i--) - unload_page((void *)desc->page_va[i]); + unload_page(bpages[i].page); unload_page(cpu_buffer->meta); return ret; -- cgit v1.2.3 From 057caace5214da3b457bbd295e1a2ad34d3685ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Wed, 20 May 2026 20:01:55 +0200 Subject: tracing: Create output file from cmd_check_undefined MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As the output file is currently never created, the check will run every time, even if the inputs have not changed. Create an empty output file which allows make to skip the execution when it is not necessary. Cc: Masami Hiramatsu Cc: Mathieu Desnoyers Cc: Vincent Donnefort Cc: Marc Zyngier Cc: Arnd Bergmann Link: https://patch.msgid.link/20260520-tracing-ringbuffer-check-v1-1-d979cfab1338@weissschuh.net Fixes: 1211907ac0b5 ("tracing: Generate undef symbols allowlist for simple_ring_buffer") Fixes: 58b4bd18390e ("tracing: Adjust cmd_check_undefined to show unexpected undefined symbols") Reviewed-by: Nathan Chancellor Tested-by: Nathan Chancellor Signed-off-by: Thomas Weißschuh Signed-off-by: Steven Rostedt --- kernel/trace/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile index 9b0834134cae..8d3d96e847d8 100644 --- a/kernel/trace/Makefile +++ b/kernel/trace/Makefile @@ -154,7 +154,8 @@ quiet_cmd_check_undefined = NM $< echo "Unexpected symbols in $<:" >&2; \ echo "$$undefsyms" >&2; \ false; \ - fi + fi; \ + touch $@ $(obj)/%.o.checked: $(obj)/%.o $(obj)/undefsyms_base.o FORCE $(call if_changed,check_undefined) -- cgit v1.2.3 From 8f0f5c4fb9df0e19a341e0c6ed8dc4fda9124f03 Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Thu, 21 May 2026 13:49:14 +0900 Subject: tracing: Do not call map->ops->elt_free() if elt_alloc() fails In paths where tracing_map_elt_alloc() failed to allocate objects, the map->ops->elt_alloc() call was never successful. In this case, map->ops->elt_free() should not be called. Link: https://sashiko.dev/#/patchset/20260520223101.34710-1-rosenp%40gmail.com Cc: stable@vger.kernel.org Cc: Tom Zanussi Cc: Mathieu Desnoyers Cc: Rosen Penev Reported-by: Sashiko Fixes: 2734b629525a ("tracing: Add per-element variable support to tracing_map") Link: https://patch.msgid.link/177933895460.108746.5396070821443932634.stgit@devnote2 Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt --- kernel/trace/tracing_map.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/tracing_map.c b/kernel/trace/tracing_map.c index bf1a507695b6..0dd7927df22a 100644 --- a/kernel/trace/tracing_map.c +++ b/kernel/trace/tracing_map.c @@ -386,13 +386,11 @@ static void tracing_map_elt_init_fields(struct tracing_map_elt *elt) } } -static void tracing_map_elt_free(struct tracing_map_elt *elt) +static void __tracing_map_elt_free(struct tracing_map_elt *elt) { if (!elt) return; - if (elt->map->ops && elt->map->ops->elt_free) - elt->map->ops->elt_free(elt); kfree(elt->fields); kfree(elt->vars); kfree(elt->var_set); @@ -400,6 +398,17 @@ static void tracing_map_elt_free(struct tracing_map_elt *elt) kfree(elt); } +static void tracing_map_elt_free(struct tracing_map_elt *elt) +{ + if (!elt) + return; + + /* Only objects initialized with alloc_elt() should be passed to free_elt().*/ + if (elt->map->ops && elt->map->ops->elt_free) + elt->map->ops->elt_free(elt); + __tracing_map_elt_free(elt); +} + static struct tracing_map_elt *tracing_map_elt_alloc(struct tracing_map *map) { struct tracing_map_elt *elt; @@ -444,7 +453,7 @@ static struct tracing_map_elt *tracing_map_elt_alloc(struct tracing_map *map) } return elt; free: - tracing_map_elt_free(elt); + __tracing_map_elt_free(elt); return ERR_PTR(err); } -- cgit v1.2.3 From 0c1a9dce208b4dc265925898e5da98934f7f9266 Mon Sep 17 00:00:00 2001 From: Samuele Mariotti Date: Thu, 21 May 2026 12:59:11 +0200 Subject: sched_ext: Fix spurious WARN on stale ops_state in ops_dequeue() ops_dequeue() can race with finish_dispatch() and spuriously trigger the "queued task must be in BPF scheduler's custody" warning. ops_dequeue() snapshots p->scx.ops_state via atomic_long_read_acquire() and then, in the SCX_OPSS_QUEUED arm, asserts that SCX_TASK_IN_CUSTODY is set. The two reads are not atomic w.r.t. a concurrent finish_dispatch() running on another CPU: CPU 1 CPU 2 ===== ===== dequeue_task_scx() ops_dequeue() opss = read_acquire(ops_state) = SCX_OPSS_QUEUED finish_dispatch() cmpxchg ops_state: SCX_OPSS_QUEUED -> SCX_OPSS_DISPATCHING [succeeds] dispatch_enqueue(SCX_DSQ_GLOBAL, SCX_ENQ_CLEAR_OPSS) call_task_dequeue() p->scx.flags &= ~SCX_TASK_IN_CUSTODY WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_IN_CUSTODY)) /* opss is stale: QUEUED, * but task already claimed */ set_release(ops_state, SCX_OPSS_NONE) The race has been observed via two distinct call chains: the most common goes through sched_setaffinity(), a rarer variant through sched_change_begin(). For SCX_DSQ_GLOBAL / SCX_DSQ_BYPASS, dispatch_enqueue() clears SCX_TASK_IN_CUSTODY before clearing ops_state to SCX_OPSS_NONE (intentional, to avoid concurrent non-atomic RMW of p->scx.flags against ops_dequeue()). The window between those two writes is exactly what ops_dequeue() observes as "QUEUED without custody". The observed state is not actually inconsistent, it just means CPU 1 has already claimed the task and the QUEUED value held by CPU 2 is stale. Re-read ops_state in that case; the next read is guaranteed to return SCX_OPSS_DISPATCHING or SCX_OPSS_NONE, both of which exit the switch cleanly. The retry is bounded: once IN_CUSTODY is cleared, ops_state has already advanced past QUEUED for this dispatch cycle, and a fresh QUEUED would require re-enqueue under p's rq lock, which CPU 2 holds. Changes in v2: - Use READ_ONCE() for p->scx.flags to ensure fresh reads and prevent compiler reordering in the lockless path - Add cpu_relax() to reduce power consumption and improve performance during the spin-wait - Use unlikely() to optimize branch prediction for the common case - Expand the in-code comment to document the race condition and bounded retry guarantee Fixes: ebf1ccff79c4 ("sched_ext: Fix ops.dequeue() semantics") Suggested-by: Andrea Righi Signed-off-by: Samuele Mariotti Signed-off-by: Paolo Valente Signed-off-by: Tejun Heo --- kernel/sched/ext.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 547ca398f646..c1762420cc35 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -2078,6 +2078,7 @@ static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags) /* dequeue is always temporary, don't reset runnable_at */ clr_task_runnable(p, false); +retry: /* acquire ensures that we see the preceding updates on QUEUED */ opss = atomic_long_read_acquire(&p->scx.ops_state); @@ -2091,8 +2092,20 @@ static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags) */ BUG(); case SCX_OPSS_QUEUED: - /* A queued task must always be in BPF scheduler's custody */ - WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_IN_CUSTODY)); + /* + * A queued task must always be in BPF scheduler's custody. If + * SCX_TASK_IN_CUSTODY is clear, finish_dispatch() on another + * CPU has already passed call_task_dequeue() (which clears the + * flag), but has not yet written SCX_OPSS_NONE. That final + * store does not require this rq's lock, so retrying with + * cpu_relax() is bounded: we will observe NONE (or DISPATCHING, + * handled by the fallthrough) on a subsequent iteration. + */ + if (unlikely(!(READ_ONCE(p->scx.flags) & SCX_TASK_IN_CUSTODY))) { + cpu_relax(); + goto retry; + } + if (atomic_long_try_cmpxchg(&p->scx.ops_state, &opss, SCX_OPSS_NONE)) break; -- cgit v1.2.3 From 3839a8eedde57a9f5b869025ce2626a7a37f5ccb Mon Sep 17 00:00:00 2001 From: Yash Suthar Date: Mon, 20 Apr 2026 15:42:36 +0530 Subject: tracing: Remove redundant IS_ERR() check in trace_pipe_open() in trace_pipe_open() already check the IS_ERR(iter) and return early on error,so iter after will be valid and it is safe to return 0 at end. Link: https://patch.msgid.link/20260420101236.223919-1-yashsuthar983@gmail.com Signed-off-by: Yash Suthar Signed-off-by: Steven Rostedt --- kernel/trace/trace_remote.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/trace/trace_remote.c b/kernel/trace/trace_remote.c index d6c3f94d67cd..2a6cc000ec98 100644 --- a/kernel/trace/trace_remote.c +++ b/kernel/trace/trace_remote.c @@ -602,7 +602,7 @@ static int trace_pipe_open(struct inode *inode, struct file *filp) filp->private_data = iter; - return IS_ERR(iter) ? PTR_ERR(iter) : 0; + return 0; } static int trace_pipe_release(struct inode *inode, struct file *filp) -- cgit v1.2.3 From f07883450eb14d1cf020b55d9f3a7ec5683bcd26 Mon Sep 17 00:00:00 2001 From: Pengpeng Hou Date: Thu, 30 Apr 2026 12:33:50 +0800 Subject: tracing: Bound synthetic-field strings with seq_buf The synthetic field helpers build a prefixed synthetic variable name and a generated hist command in fixed MAX_FILTER_STR_VAL buffers. The current code appends those strings with raw strcat(), so long key lists, field names, or saved filters can run past the end of the staging buffers. Build both strings with seq_buf and propagate -E2BIG if either the synthetic variable name or the generated command exceeds MAX_FILTER_STR_VAL. This keeps the existing tracing-side limit while using the helper intended for bounded command construction. Cc: Mathieu Desnoyers Cc: Tom Zanussi Link: https://patch.msgid.link/20260430043350.57928-1-pengpeng@iscas.ac.cn Fixes: 02205a6752f2 ("tracing: Add support for 'field variables'") Acked-by: Masami Hiramatsu (Google) Reviewed-by: Tom Zanussi Signed-off-by: Pengpeng Hou [ sdr: Moved struct seq_buf *s for upside-down x-mas tree formatting ] Signed-off-by: Steven Rostedt --- kernel/trace/trace_events_hist.c | 41 ++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index eb2c2bc8bc3d..9701650c89b2 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -2967,13 +2968,22 @@ find_synthetic_field_var(struct hist_trigger_data *target_hist_data, { struct hist_field *event_var; char *synthetic_name; + struct seq_buf s; synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); if (!synthetic_name) return ERR_PTR(-ENOMEM); - strcpy(synthetic_name, "synthetic_"); - strcat(synthetic_name, field_name); + seq_buf_init(&s, synthetic_name, MAX_FILTER_STR_VAL); + seq_buf_printf(&s, "synthetic_%s", field_name); + + /* Terminate synthetic_name with a NUL. */ + seq_buf_str(&s); + + if (seq_buf_has_overflowed(&s)) { + kfree(synthetic_name); + return ERR_PTR(-E2BIG); + } event_var = find_event_var(target_hist_data, system, event_name, synthetic_name); @@ -3019,6 +3029,7 @@ create_field_var_hist(struct hist_trigger_data *target_hist_data, struct hist_field *key_field; struct hist_field *event_var; char *saved_filter; + struct seq_buf s; char *cmd; int ret; @@ -3063,28 +3074,34 @@ create_field_var_hist(struct hist_trigger_data *target_hist_data, return ERR_PTR(-ENOMEM); } + seq_buf_init(&s, cmd, MAX_FILTER_STR_VAL); + /* Use the same keys as the compatible histogram */ - strcat(cmd, "keys="); + seq_buf_puts(&s, "keys="); for_each_hist_key_field(i, hist_data) { key_field = hist_data->fields[i]; if (!first) - strcat(cmd, ","); - strcat(cmd, key_field->field->name); + seq_buf_putc(&s, ','); + seq_buf_puts(&s, key_field->field->name); first = false; } /* Create the synthetic field variable specification */ - strcat(cmd, ":synthetic_"); - strcat(cmd, field_name); - strcat(cmd, "="); - strcat(cmd, field_name); + seq_buf_printf(&s, ":synthetic_%s=%s", field_name, field_name); /* Use the same filter as the compatible histogram */ saved_filter = find_trigger_filter(hist_data, file); - if (saved_filter) { - strcat(cmd, " if "); - strcat(cmd, saved_filter); + if (saved_filter) + seq_buf_printf(&s, " if %s", saved_filter); + + /* Terminate cmd with a NUL. */ + seq_buf_str(&s); + + if (seq_buf_has_overflowed(&s)) { + kfree(cmd); + kfree(var_hist); + return ERR_PTR(-E2BIG); } var_hist->cmd = kstrdup(cmd, GFP_KERNEL); -- cgit v1.2.3 From 8a6e9af2fac8cf212f86cff282ae96f6c7d7ca18 Mon Sep 17 00:00:00 2001 From: Yash Suthar Date: Sat, 2 May 2026 23:17:41 +0530 Subject: tracing: Switch trace_recursion_record.c code over to use guard() Switch mutex_lock()/mutex_unlock() to guard(). also drop the ret local variable and return directly. Link: https://patch.msgid.link/20260502174741.39636-1-yashsuthar983@gmail.com Signed-off-by: Yash Suthar Signed-off-by: Steven Rostedt --- kernel/trace/trace_recursion_record.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/trace_recursion_record.c b/kernel/trace/trace_recursion_record.c index 784fe1fbb866..bac4bc844ccd 100644 --- a/kernel/trace/trace_recursion_record.c +++ b/kernel/trace/trace_recursion_record.c @@ -180,9 +180,8 @@ static const struct seq_operations recursed_function_seq_ops = { static int recursed_function_open(struct inode *inode, struct file *file) { - int ret = 0; + guard(mutex)(&recursed_function_lock); - mutex_lock(&recursed_function_lock); /* If this file was opened for write, then erase contents */ if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { /* disable updating records */ @@ -194,10 +193,9 @@ static int recursed_function_open(struct inode *inode, struct file *file) atomic_set(&nr_records, 0); } if (file->f_mode & FMODE_READ) - ret = seq_open(file, &recursed_function_seq_ops); - mutex_unlock(&recursed_function_lock); + return seq_open(file, &recursed_function_seq_ops); - return ret; + return 0; } static ssize_t recursed_function_write(struct file *file, -- cgit v1.2.3 From bb4613842e8033a714bacf3b62cc70638926cdcf Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Wed, 13 May 2026 15:00:07 -0400 Subject: tracing: Allow perf to read synthetic events Currently, perf can not enable synthetic events. When it does, it either causes a warning in the kernel or errors with "no such device". Add the necessary code to allow perf to also attach to synthetic events. Cc: Masami Hiramatsu Cc: Mathieu Desnoyers Cc: Arnaldo Carvalho de Melo Cc: Jiri Olsa Cc: Peter Zijlstra Link: https://patch.msgid.link/20260513150007.3b280e87@gandalf.local.home Reported-by: Ian Rogers Acked-by: Namhyung Kim Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace_events_synth.c | 121 +++++++++++++++++++++++++++++--------- 1 file changed, 94 insertions(+), 27 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/trace_events_synth.c b/kernel/trace/trace_events_synth.c index 39ac4eba0702..e6871230bde9 100644 --- a/kernel/trace/trace_events_synth.c +++ b/kernel/trace/trace_events_synth.c @@ -499,28 +499,19 @@ static unsigned int trace_stack(struct synth_trace_event *entry, return len; } -static void trace_event_raw_event_synth(void *__data, - u64 *var_ref_vals, - unsigned int *var_ref_idx) +static __always_inline int get_field_size(struct synth_event *event, + u64 *var_ref_vals, + unsigned int *var_ref_idx) { - unsigned int i, n_u64, val_idx, len, data_size = 0; - struct trace_event_file *trace_file = __data; - struct synth_trace_event *entry; - struct trace_event_buffer fbuffer; - struct trace_buffer *buffer; - struct synth_event *event; - int fields_size = 0; - - event = trace_file->event_call->data; - - if (trace_trigger_soft_disabled(trace_file)) - return; + int fields_size; fields_size = event->n_u64 * sizeof(u64); - for (i = 0; i < event->n_dynamic_fields; i++) { + for (int i = 0; i < event->n_dynamic_fields; i++) { unsigned int field_pos = event->dynamic_fields[i]->field_pos; char *str_val; + int val_idx; + int len; val_idx = var_ref_idx[field_pos]; str_val = (char *)(long)var_ref_vals[val_idx]; @@ -535,18 +526,18 @@ static void trace_event_raw_event_synth(void *__data, fields_size += len; } + return fields_size; +} - /* - * Avoid ring buffer recursion detection, as this event - * is being performed within another event. - */ - buffer = trace_file->tr->array_buffer.buffer; - guard(ring_buffer_nest)(buffer); - - entry = trace_event_buffer_reserve(&fbuffer, trace_file, - sizeof(*entry) + fields_size); - if (!entry) - return; +static __always_inline void write_synth_entry(struct synth_event *event, + struct synth_trace_event *entry, + u64 *var_ref_vals, + unsigned int *var_ref_idx) +{ + int data_size = 0; + int i, n_u64; + int val_idx; + int len; for (i = 0, n_u64 = 0; i < event->n_fields; i++) { val_idx = var_ref_idx[i]; @@ -587,10 +578,83 @@ static void trace_event_raw_event_synth(void *__data, n_u64++; } } +} + +static void trace_event_raw_event_synth(void *__data, + u64 *var_ref_vals, + unsigned int *var_ref_idx) +{ + struct trace_event_file *trace_file = __data; + struct synth_trace_event *entry; + struct trace_event_buffer fbuffer; + struct trace_buffer *buffer; + struct synth_event *event; + int fields_size; + + event = trace_file->event_call->data; + + if (trace_trigger_soft_disabled(trace_file)) + return; + + fields_size = get_field_size(event, var_ref_vals, var_ref_idx); + + /* + * Avoid ring buffer recursion detection, as this event + * is being performed within another event. + */ + buffer = trace_file->tr->array_buffer.buffer; + guard(ring_buffer_nest)(buffer); + + entry = trace_event_buffer_reserve(&fbuffer, trace_file, + sizeof(*entry) + fields_size); + if (!entry) + return; + + write_synth_entry(event, entry, var_ref_vals, var_ref_idx); trace_event_buffer_commit(&fbuffer); } +#ifdef CONFIG_PERF_EVENTS +static void perf_event_raw_event_synth(void *__data, + u64 *var_ref_vals, + unsigned int *var_ref_idx) +{ + struct trace_event_call *call = __data; + struct synth_trace_event *entry; + struct hlist_head *perf_head; + struct synth_event *event; + struct pt_regs *regs; + int fields_size; + size_t size; + int context; + + event = call->data; + + perf_head = this_cpu_ptr(call->perf_events); + + if (!perf_head || hlist_empty(perf_head)) + return; + + fields_size = get_field_size(event, var_ref_vals, var_ref_idx); + + size = ALIGN(sizeof(*entry) + fields_size, 8); + + entry = perf_trace_buf_alloc(size, ®s, &context); + + if (unlikely(!entry)) + return; + + write_synth_entry(event, entry, var_ref_vals, var_ref_idx); + + perf_fetch_caller_regs(regs); + + perf_trace_buf_submit(entry, size, context, + call->event.type, 1, regs, + perf_head, NULL); +} +#endif + static void free_synth_event_print_fmt(struct trace_event_call *call) { if (call) { @@ -917,6 +981,9 @@ static int register_synth_event(struct synth_event *event) call->flags = TRACE_EVENT_FL_TRACEPOINT; call->class->reg = synth_event_reg; call->class->probe = trace_event_raw_event_synth; +#ifdef CONFIG_PERF_EVENTS + call->class->perf_probe = perf_event_raw_event_synth; +#endif call->data = event; call->tp = event->tp; -- cgit v1.2.3 From 5ec75d6a1b5b8beebea3bcaa85baad295ee09dcb Mon Sep 17 00:00:00 2001 From: Yu Peng Date: Tue, 19 May 2026 16:16:20 +0800 Subject: tracing/branch: Use pr_warn() instead of printk(KERN_WARNING) Use pr_warn() instead of printk(KERN_WARNING ...) for the branch tracer warning messages. Keep the message text unchanged. The change only removes the open-coded log level from these warnings. Cc: Masami Hiramatsu Cc: Mathieu Desnoyers Link: https://patch.msgid.link/20260519081620.3874441-1-pengyu@kylinos.cn Signed-off-by: Yu Peng Signed-off-by: Steven Rostedt --- kernel/trace/trace_branch.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/trace_branch.c b/kernel/trace/trace_branch.c index d1564db95a8f..d8e97ad798f0 100644 --- a/kernel/trace/trace_branch.c +++ b/kernel/trace/trace_branch.c @@ -181,8 +181,7 @@ __init static int init_branch_tracer(void) ret = register_trace_event(&trace_branch_event); if (!ret) { - printk(KERN_WARNING "Warning: could not register " - "branch events\n"); + pr_warn("Warning: could not register branch events\n"); return 1; } return register_tracer(&branch_trace); @@ -374,8 +373,7 @@ __init static int init_annotated_branch_stats(void) ret = register_stat_tracer(&annotated_branch_stats); if (ret) { - printk(KERN_WARNING "Warning: could not register " - "annotated branches stats\n"); + pr_warn("Warning: could not register annotated branches stats\n"); return ret; } return 0; @@ -439,8 +437,7 @@ __init static int all_annotated_branch_stats(void) ret = register_stat_tracer(&all_branch_stats); if (ret) { - printk(KERN_WARNING "Warning: could not register " - "all branches stats\n"); + pr_warn("Warning: could not register all branches stats\n"); return ret; } return 0; -- cgit v1.2.3 From 153498f200d295f3efa6bd37fc6e7ff105ab344f Mon Sep 17 00:00:00 2001 From: Yu Peng Date: Tue, 19 May 2026 16:34:09 +0800 Subject: tracing: Use krealloc_array() for trace option array growth Use krealloc_array() when growing tr->topts instead of open-coding the size calculation in krealloc(). This makes the resize path use the helper intended for array allocations and avoids manual multiplication of the element count and element size. Cc: Masami Hiramatsu Cc: Mathieu Desnoyers Link: https://patch.msgid.link/20260519083409.3885032-1-pengyu@kylinos.cn Signed-off-by: Yu Peng Signed-off-by: Steven Rostedt --- kernel/trace/trace.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 6eb4d3097a4d..aec3f31ed027 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -7928,8 +7928,8 @@ create_trace_option_files(struct trace_array *tr, struct tracer *tracer, if (!topts) return 0; - tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1), - GFP_KERNEL); + tr_topts = krealloc_array(tr->topts, tr->nr_topts + 1, sizeof(*tr->topts), + GFP_KERNEL); if (!tr_topts) { kfree(topts); return -ENOMEM; -- cgit v1.2.3 From 3a5b29657593fe7695337dfcf40c49eb6fa544d4 Mon Sep 17 00:00:00 2001 From: Ao Sun Date: Thu, 21 May 2026 01:52:34 +0000 Subject: tracing: Fix README path for synthetic_events The events/ prefix should be removed, since synthetic_events is now directly under the tracing root directory. Cc: Cc: Link: https://patch.msgid.link/20260521015211.111-1-ao.sun@transsion.com Signed-off-by: Ao Sun Signed-off-by: Steven Rostedt --- kernel/trace/trace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index aec3f31ed027..4218c174460b 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -4474,7 +4474,7 @@ static const char readme_msg[] = "\t snapshot() - snapshot the trace buffer\n\n" #endif #ifdef CONFIG_SYNTH_EVENTS - " events/synthetic_events\t- Create/append/remove/show synthetic events\n" + " synthetic_events\t- Create/append/remove/show synthetic events\n" "\t Write into this file to define/undefine new synthetic events.\n" "\t example: echo 'myevent u64 lat; char name[]; long[] stack' >> synthetic_events\n" #endif -- cgit v1.2.3 From 817e148935dd76ce39bf468f51308ec77de87033 Mon Sep 17 00:00:00 2001 From: Rosen Penev Date: Wed, 20 May 2026 14:50:06 -0700 Subject: tracing: Simplify pages allocation for tracing_map logic Change to a flexible array member to allocate together with the array struct. Simplifies code slightly by removing no longer correct null checks for pages and removing kfrees. Cc: Mathieu Desnoyers Cc: Kees Cook Cc: "Gustavo A. R. Silva" Link: https://patch.msgid.link/20260520215006.12008-1-rosenp@gmail.com Signed-off-by: Rosen Penev Acked-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt --- kernel/trace/tracing_map.c | 30 +++++++++++------------------- kernel/trace/tracing_map.h | 2 +- 2 files changed, 12 insertions(+), 20 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/tracing_map.c b/kernel/trace/tracing_map.c index 0dd7927df22a..d7922f40dbe2 100644 --- a/kernel/trace/tracing_map.c +++ b/kernel/trace/tracing_map.c @@ -288,9 +288,6 @@ static void tracing_map_array_clear(struct tracing_map_array *a) { unsigned int i; - if (!a->pages) - return; - for (i = 0; i < a->n_pages; i++) memset(a->pages[i], 0, PAGE_SIZE); } @@ -302,9 +299,6 @@ static void tracing_map_array_free(struct tracing_map_array *a) if (!a) return; - if (!a->pages) - goto free; - for (i = 0; i < a->n_pages; i++) { if (!a->pages[i]) break; @@ -312,9 +306,6 @@ static void tracing_map_array_free(struct tracing_map_array *a) free_page((unsigned long)a->pages[i]); } - kfree(a->pages); - - free: kfree(a); } @@ -322,24 +313,25 @@ static struct tracing_map_array *tracing_map_array_alloc(unsigned int n_elts, unsigned int entry_size) { struct tracing_map_array *a; + unsigned int entry_size_shift; + unsigned int entries_per_page; + unsigned int n_pages; unsigned int i; - a = kzalloc_obj(*a); + entry_size_shift = fls(roundup_pow_of_two(entry_size) - 1); + entries_per_page = PAGE_SIZE / (1 << entry_size_shift); + n_pages = max(1, n_elts / entries_per_page); + + a = kzalloc_flex(*a, pages, n_pages); if (!a) return NULL; - a->entry_size_shift = fls(roundup_pow_of_two(entry_size) - 1); - a->entries_per_page = PAGE_SIZE / (1 << a->entry_size_shift); - a->n_pages = n_elts / a->entries_per_page; - if (!a->n_pages) - a->n_pages = 1; + a->entry_size_shift = entry_size_shift; + a->entries_per_page = entries_per_page; + a->n_pages = n_pages; a->entry_shift = fls(a->entries_per_page) - 1; a->entry_mask = (1 << a->entry_shift) - 1; - a->pages = kcalloc(a->n_pages, sizeof(void *), GFP_KERNEL); - if (!a->pages) - goto free; - for (i = 0; i < a->n_pages; i++) { a->pages[i] = (void *)get_zeroed_page(GFP_KERNEL); if (!a->pages[i]) diff --git a/kernel/trace/tracing_map.h b/kernel/trace/tracing_map.h index 99c37eeebc16..18a02959d77b 100644 --- a/kernel/trace/tracing_map.h +++ b/kernel/trace/tracing_map.h @@ -167,7 +167,7 @@ struct tracing_map_array { unsigned int entry_shift; unsigned int entry_mask; unsigned int n_pages; - void **pages; + void *pages[] __counted_by(n_pages); }; #define TRACING_MAP_ARRAY_ELT(array, idx) \ -- cgit v1.2.3 From 292c8197e3685aa7ea5a3803ca240b2210fcd021 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Thu, 21 May 2026 09:50:26 -0400 Subject: tracing: Move trace_iterator_increment() into trace_find_next_entry_inc() trace_iterator_increment() is only called from trace_find_next_entry_inc(). It's a small enough function that really doesn't need to be separated. Move the code from trace_iterator_increment() into trace_find_next_entry_inc() and remove trace_iterator_increment(). Cc: Masami Hiramatsu Cc: Mathieu Desnoyers Link: https://patch.msgid.link/20260521095026.20c9799d@gandalf.local.home Signed-off-by: Steven Rostedt --- kernel/trace/trace.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 4218c174460b..ae527c419508 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -2338,15 +2338,6 @@ void trace_last_func_repeats(struct trace_array *tr, __buffer_unlock_commit(buffer, event); } -static void trace_iterator_increment(struct trace_iterator *iter) -{ - struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu); - - iter->idx++; - if (buf_iter) - ring_buffer_iter_advance(buf_iter); -} - static struct trace_entry * peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts, unsigned long *lost_events) @@ -2676,11 +2667,17 @@ struct trace_entry *trace_find_next_entry(struct trace_iterator *iter, /* Find the next real entry, and increment the iterator to the next entry */ void *trace_find_next_entry_inc(struct trace_iterator *iter) { + struct ring_buffer_iter *buf_iter; + iter->ent = __find_next_entry(iter, &iter->cpu, &iter->lost_events, &iter->ts); - if (iter->ent) - trace_iterator_increment(iter); + if (iter->ent) { + iter->idx++; + buf_iter = trace_buffer_iter(iter, iter->cpu); + if (buf_iter) + ring_buffer_iter_advance(buf_iter); + } return iter->ent ? iter : NULL; } -- cgit v1.2.3 From 09e7827e785729f391c8d46dc71becce70d296ab Mon Sep 17 00:00:00 2001 From: Deepanshu Kartikey Date: Mon, 16 Mar 2026 20:49:56 +0530 Subject: kernel/fork: validate exit_signal in kernel_clone() When a child process exits, it sends exit_signal to its parent via do_notify_parent(). The clone() syscall constructs exit_signal as: (lower_32_bits(clone_flags) & CSIGNAL) CSIGNAL is 0xff, so values in the range 65-255 are possible. However, valid_signal() only accepts signals up to _NSIG (64 on x86_64). A non-zero non-valid exit_signal acts the same as exit_signal == 0: the parent process is not signaled when the child terminates. The syzkaller reproducer triggers this by calling clone() with flags=0x80, resulting in exit_signal = (0x80 & CSIGNAL) = 128, which exceeds _NSIG and is not a valid signal. The v1 of this patch added the check only in the clone() syscall handler, which is incomplete. kernel_clone() has other callers such as sys_ia32_clone() which would remain unprotected. Move the check to kernel_clone() to cover all callers. Since the valid_signal() check is now in kernel_clone() and covers all callers including clone3(), the same check in copy_clone_args_from_user() becomes redundant and is removed. The higher 32bits check for clone3() is kept as it is clone3() specific. Note that this is a user-visible change: previously, passing an invalid exit_signal to clone() was silently accepted. The man page for clone() does not document any defined behavior for invalid exit_signal values, so rejecting them with -EINVAL is the correct behavior. It is unlikely that any sane application relies on passing an invalid exit_signal. [oleg@redhat.com: the comment above kernel_clone() should be updated] Link: https://lore.kernel.org/abwvgU17W8wuW2-J@redhat.com Link: https://lore.kernel.org/20260316151956.563558-1-kartikey406@gmail.com Fixes: 3f2c788a1314 ("fork: prevent accidental access to clone3 features") Signed-off-by: Deepanshu Kartikey Signed-off-by: Oleg Nesterov Reported-by: syzbot+bbe6b99feefc3a0842de@syzkaller.appspotmail.com Closes: https://syzkaller.appspot.com/bug?extid=bbe6b99feefc3a0842de Tested-by: syzbot+bbe6b99feefc3a0842de@syzkaller.appspotmail.com Link: https://lore.kernel.org/all/20260307064202.353405-1-kartikey406@gmail.com/T/ [v1] Link: https://lore.kernel.org/all/20260316104536.558108-1-kartikey406@gmail.com/T/ [v2] Acked-by: Oleg Nesterov Acked-by: Michal Hocko Cc: Ben Segall Cc: Christian Brauner Cc: David Hildenbrand Cc: Dietmar Eggemann Cc: Ingo Molnar Cc: Juri Lelli Cc: Kees Cook Cc: Liam Howlett Cc: Lorenzo Stoakes (Oracle) Cc: Mel Gorman Cc: Mike Rapoport Cc: Peter Zijlstra Cc: Steven Rostedt Cc: Suren Baghdasaryan Cc: Valentin Schneider Cc: Vincent Guittot Cc: Vlastimil Babka Cc: Tetsuo Handa Signed-off-by: Andrew Morton --- kernel/fork.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/fork.c b/kernel/fork.c index 5f3fdfdb14c7..8ac38beae360 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -2664,8 +2664,6 @@ struct task_struct *create_io_thread(int (*fn)(void *), void *arg, int node) * * It copies the process, and if successful kick-starts * it and waits for it to finish using the VM if required. - * - * args->exit_signal is expected to be checked for sanity by the caller. */ pid_t kernel_clone(struct kernel_clone_args *args) { @@ -2700,6 +2698,9 @@ pid_t kernel_clone(struct kernel_clone_args *args) (args->pidfd == args->parent_tid)) return -EINVAL; + if (!valid_signal(args->exit_signal)) + return -EINVAL; + /* * Determine whether and which event to report to ptracer. When * called from kernel_thread or CLONE_UNTRACED is explicitly @@ -2898,11 +2899,9 @@ static noinline int copy_clone_args_from_user(struct kernel_clone_args *kargs, return -EINVAL; /* - * Verify that higher 32bits of exit_signal are unset and that - * it is a valid signal + * Verify that higher 32bits of exit_signal are unset */ - if (unlikely((args.exit_signal & ~((u64)CSIGNAL)) || - !valid_signal(args.exit_signal))) + if (unlikely(args.exit_signal & ~((u64)CSIGNAL))) return -EINVAL; if ((args.flags & CLONE_INTO_CGROUP) && -- cgit v1.2.3 From 7e6ace2535d032c908e4d8747d9a7952617c001a Mon Sep 17 00:00:00 2001 From: Feng Tang Date: Tue, 12 May 2026 16:55:09 +0800 Subject: dma-contiguous: add kconfig option to setup numa cma area if not configured explicitly There was a report on a multi-numa-nodes arm64 server that when IOMMU is disabled, the dma_alloc_coherent() function always returns memory from node 0 even for devices attaching to other nodes, while they can get local dma memory when IOMMU is on with the same API. The reason is, when IOMMU is disabled, the dma_alloc_coherent() will go the direct way and call dma_alloc_contiguous(). The system doesn't have any explicit cma setting (like per-numa cma), and only has a default 64MB cma reserved area (on node 0), where kernel will try first to allocate memory from. Robin Murphy suggested to setup pernuma cma or disable cma, which did solve the issue. While there is still concern that for customers which don't have much kernel knowledge, they could still suffer from this silently as some architectures enable cma area by default (not an issue for X86 though, which set CONFIG_CMA_SIZE_MBYTES to 0 by default) for most Linux distributions. One thought is to follow the current cma reserving policy for platform with 'CONFIG_DMA_NUMA_CMA=y', that if the numa cma (either the 'numa cma' or 'cma pernuma' method) is not explicitly configured, and the platform really has multiple NUMA nodes, set it up according to size of default 'dma_contiguous_default_area'. This way, the default behavior of platform with one NUMA node is kept unchanged (say embedded/small devices don't need to allocate extra memory), while the general dma locality is improved. Add a new bool kernel config CONFIG_CMA_SIZE_PERNUMA to control whether to enable it. Even when the config is enabled, user can still disable it by kernel-cmdline setting like "numa_cma=0:0" or "cma_pernuma=0". Reported-by: Changrong Chen Suggested-by: Ying Huang Suggested-by: Robin Murphy Signed-off-by: Feng Tang Link: https://lore.kernel.org/r/20260512085509.83002-1-feng.tang@linux.alibaba.com Link: https://lore.kernel.org/all/20260520222742.GA1607511@ax162/ [mszyprow: squashed changes from both links, added __initdata attribute to the numa_cma_configured variable] Signed-off-by: Marek Szyprowski --- kernel/dma/Kconfig | 10 ++++++++++ kernel/dma/contiguous.c | 12 ++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/dma/Kconfig b/kernel/dma/Kconfig index bfef21b4a9ae..c9fa0a922cba 100644 --- a/kernel/dma/Kconfig +++ b/kernel/dma/Kconfig @@ -181,6 +181,16 @@ config DMA_NUMA_CMA or set the node id and its size of CMA by specifying "numa_cma= :size[,:size]" on the kernel's command line. +config CMA_SIZE_PERNUMA + bool "Default CMA area per NUMA node" + depends on DMA_NUMA_CMA + default y + help + On systems with more than one NUMA node, the selected CMA + area size will be also allocated on each additional node, + so that most devices may have benefit from better DMA + locality without an explicit command-line opt-in. + comment "Default contiguous memory area size:" config CMA_SIZE_MBYTES diff --git a/kernel/dma/contiguous.c b/kernel/dma/contiguous.c index 03f52bd17120..799f6e9c88bd 100644 --- a/kernel/dma/contiguous.c +++ b/kernel/dma/contiguous.c @@ -136,6 +136,7 @@ static struct cma *dma_contiguous_numa_area[MAX_NUMNODES]; static phys_addr_t numa_cma_size[MAX_NUMNODES] __initdata; static struct cma *dma_contiguous_pernuma_area[MAX_NUMNODES]; static phys_addr_t pernuma_size_bytes __initdata; +static bool numa_cma_configured __initdata; static int __init early_numa_cma(char *p) { @@ -164,6 +165,7 @@ static int __init early_numa_cma(char *p) break; } + numa_cma_configured = true; return 0; } early_param("numa_cma", early_numa_cma); @@ -171,6 +173,7 @@ early_param("numa_cma", early_numa_cma); static int __init early_cma_pernuma(char *p) { pernuma_size_bytes = memparse(p, &p); + numa_cma_configured = true; return 0; } early_param("cma_pernuma", early_cma_pernuma); @@ -199,6 +202,11 @@ static void __init dma_numa_cma_reserve(void) { int nid; + if (IS_ENABLED(CONFIG_CMA_SIZE_PERNUMA) && + !numa_cma_configured && dma_contiguous_default_area && + nr_online_nodes > 1) + pernuma_size_bytes = cma_get_size(dma_contiguous_default_area); + for_each_node(nid) { int ret; char name[CMA_MAX_NAME]; @@ -255,8 +263,6 @@ void __init dma_contiguous_reserve(phys_addr_t limit) phys_addr_t selected_limit = limit; bool fixed = false; - dma_numa_cma_reserve(); - pr_debug("%s(limit %08lx)\n", __func__, (unsigned long)limit); if (size_cmdline != -1) { @@ -312,6 +318,8 @@ void __init dma_contiguous_reserve(phys_addr_t limit) if (ret) pr_warn("Couldn't queue default CMA region for heap creation."); } + + dma_numa_cma_reserve(); } void __weak -- cgit v1.2.3 From 90918794a4e2c3b440f8fcf3847765a8b1d81b25 Mon Sep 17 00:00:00 2001 From: Aleksandr Nogikh Date: Thu, 21 May 2026 16:22:40 +0200 Subject: signal: clear JOBCTL_PENDING_MASK for caller in zap_other_threads() When a multi-threaded process receives a stop signal (e.g., SIGSTOP), do_signal_stop() sets JOBCTL_STOP_PENDING and JOBCTL_STOP_CONSUME on all threads and sets signal->group_stop_count to the number of threads. If one of the threads concurrently calls execve(), de_thread() invokes zap_other_threads() to kill all other threads. zap_other_threads() aborts the pending group stop by resetting signal->group_stop_count to 0 and clears the JOBCTL_PENDING_MASK for all other threads. However, it fails to clear the job control flags for the calling thread. When execve() completes, the calling thread returns to user mode and checks for pending signals. Seeing the stale JOBCTL_STOP_PENDING flag, it calls do_signal_stop(), which invokes task_participate_group_stop(). Since JOBCTL_STOP_CONSUME is still set, it attempts to decrement the already-zero signal->group_stop_count, triggering a warning: sig->group_stop_count == 0 WARNING: CPU: 1 PID: 6475 at kernel/signal.c:373 task_participate_group_stop+0x215/0x2d0 Call Trace: do_signal_stop+0x3be/0x5c0 kernel/signal.c:2619 get_signal+0xa8c/0x1330 kernel/signal.c:2884 arch_do_signal_or_restart+0xbc/0x840 arch/x86/kernel/signal.c:337 exit_to_user_mode_loop+0x8c/0x4d0 kernel/entry/common.c:98 do_syscall_64+0x33e/0xf80 arch/x86/entry/syscall_64.c:100 entry_SYSCALL_64_after_hwframe+0x77/0x7f Fix this race condition by clearing the JOBCTL_PENDING_MASK for the calling thread in zap_other_threads(), ensuring it does not retain any stale job control state after the thread group is destroyed. This aligns with other functions that tear down a thread group and abort group stops, such as zap_process() and complete_signal(), which correctly clear these flags for all threads including the current one. Fixes: 39efa3ef3a37 ("signal: Use GROUP_STOP_PENDING to stop once for a single group stop") Assisted-by: Gemini:gemini-3.1-pro-preview Gemini:gemini-3-flash-preview syzbot Reported-by: syzbot+b109633ea805cac54a61@syzkaller.appspotmail.com Closes: https://syzkaller.appspot.com/bug?extid=b109633ea805cac54a61 Link: https://syzkaller.appspot.com/ai_job?id=d70208cc-862b-4fe3-bf02-3031e10cd0b3 Signed-off-by: Aleksandr Nogikh Link: https://patch.msgid.link/20260521142240.2973022-1-nogikh@google.com Signed-off-by: Christian Brauner (Amutable) --- kernel/signal.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/signal.c b/kernel/signal.c index 2d102e025883..9c2b32c4d755 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -1338,6 +1338,7 @@ int zap_other_threads(struct task_struct *p) int count = 0; p->signal->group_stop_count = 0; + task_clear_jobctl_pending(p, JOBCTL_PENDING_MASK); for_other_threads(p, t) { task_clear_jobctl_pending(t, JOBCTL_PENDING_MASK); -- cgit v1.2.3 From dc11a4dba2464e5144c318ffaf7fb16b1a5c74d6 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Fri, 22 May 2026 07:22:13 -1000 Subject: bpf: Recover arena kernel faults with scratch page BPF arena usage is becoming more prevalent, but kernel <-> BPF communication over arena memory is awkward today. Data has to be staged through a trusted kernel pointer with extra code and copying on the BPF side. While reads through arena pointers can use a fault-safe helper, writes don't have a good solution. The in-line alternative would need instruction emulation or asm fixup labels. Enable direct kernel-side reads and writes within GUARD_SZ / 2 of any handed-in arena pointer, without bounds checking. A per-arena scratch page is installed by the arch fault path into empty arena kernel PTEs - x86 from page_fault_oops() for not-present faults, arm64 from __do_kernel_fault() for translation faults, both after the existing exception-table and KFENCE handling. The faulting instruction retries and the access is also reported through the program's BPF stream, preserving error reporting. bpf_prog_find_from_stack() resolves the current BPF program (and its arena) from the kernel stack - no new bpf_run_ctx state is added. Recovery covers the 4 GiB arena plus the upper half-guard (GUARD_SZ / 2). The lower half-guard is excluded because well-behaved kfuncs only access forward from arena pointers. The kfunc-author contract - access at most GUARD_SZ / 2 past a handed-in pointer - is documented in Documentation/bpf/kfuncs.rst. The install is lock-free via ptep_try_set(). On race-loss the winning installer's PTE is already valid, so the access retry succeeds. The arena clear path uses ptep_get_and_clear() so installer and clearer race through atomic accessors. No flush_tlb_kernel_range() afterwards. Stale "not mapped" entries just cause one extra re-fault, cheaper than a global IPI on every install. Scratch exists only to keep the kernel from oopsing on an in-line arena access. Its presence at a PTE means the BPF program has already malfunctioned, and the violation is reported through the program's BPF stream. The only requirement for behavior on a scratched PTE is that the kernel doesn't crash. In particular, any user-side access through such a PTE may segfault. The shared scratch page is freed once during map destruction. BPF instruction faults continue to use the existing JIT exception-table path. This patch changes only the kernel-text fault path. No UAPI flag is added. The new behavior is the default. v2: Use ptep_get_and_clear() in apply_range_clear_cb(). (David) v3: Stub bpf_arena_handle_page_fault() for !CONFIG_BPF_SYSCALL. (lkp) Suggested-by: Alexei Starovoitov Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Tejun Heo Reviewed-by: Emil Tsalapatis Cc: David Hildenbrand Link: https://lore.kernel.org/r/20260522172219.1423324-3-tj@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/bpf/arena.c | 177 +++++++++++++++++++++++++++++++++++++++++------------ kernel/bpf/core.c | 5 ++ 2 files changed, 142 insertions(+), 40 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 49a8f7b1beef..2da2c275cff6 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -53,6 +53,7 @@ struct bpf_arena { u64 user_vm_start; u64 user_vm_end; struct vm_struct *kern_vm; + struct page *scratch_page; struct range_tree rt; /* protects rt */ rqspinlock_t spinlock; @@ -118,6 +119,11 @@ struct apply_range_data { int i; }; +struct clear_range_data { + struct llist_head *free_pages; + struct page *scratch_page; +}; + static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data) { struct apply_range_data *d = data; @@ -144,33 +150,59 @@ static void flush_vmap_cache(unsigned long start, unsigned long size) flush_cache_vmap(start, start + size); } -static int apply_range_clear_cb(pte_t *pte, unsigned long addr, void *free_pages) +static int apply_range_clear_cb(pte_t *pte, unsigned long addr, void *data) { + struct clear_range_data *d = data; pte_t old_pte; struct page *page; - /* sanity check */ - old_pte = ptep_get(pte); + /* + * Pairs with ptep_try_set() in the kernel-fault scratch installer. + * Both sides must be atomic. + */ + old_pte = ptep_get_and_clear(&init_mm, addr, pte); if (pte_none(old_pte) || !pte_present(old_pte)) - return 0; /* nothing to do */ + return 0; page = pte_page(old_pte); if (WARN_ON_ONCE(!page)) return -EINVAL; - pte_clear(&init_mm, addr, pte); + /* + * Skip the per-arena scratch page. A kernel fault on an unallocated uaddr + * scratches its PTE. A later bpf_arena_free_pages() over that range walks + * here. Without the skip, scratch_page would be freed. + */ + if (page == d->scratch_page) + return 0; + + __llist_add(&page->pcp_llist, d->free_pages); + return 0; +} - /* Add page to the list so it is freed later */ - if (free_pages) - __llist_add(&page->pcp_llist, free_pages); +static int apply_range_set_scratch_cb(pte_t *pte, unsigned long addr, void *data) +{ + struct page *scratch_page = data; + if (!pte_none(ptep_get(pte))) + return 0; + /* + * Best-effort install. ptep_try_set() returns false only if another + * installer (real allocation or concurrent fault) won the cmpxchg. + * Their PTE is already valid, so the access retry succeeds. + * + * No flush_tlb_kernel_range() needed. Stale "not mapped" entries just + * cause one extra re-fault through this same path. + */ + ptep_try_set(pte, mk_pte(scratch_page, PAGE_KERNEL)); return 0; } static int populate_pgtable_except_pte(struct bpf_arena *arena) { + /* Populate intermediates for the recovery range (4 GiB + upper half-guard). */ return apply_to_page_range(&init_mm, bpf_arena_get_kern_vm_start(arena), - KERN_VM_SZ - GUARD_SZ, apply_range_set_cb, NULL); + SZ_4G + GUARD_SZ / 2, apply_range_set_cb, NULL); } static struct bpf_map *arena_map_alloc(union bpf_attr *attr) @@ -221,22 +253,29 @@ static struct bpf_map *arena_map_alloc(union bpf_attr *attr) init_irq_work(&arena->free_irq, arena_free_irq); INIT_WORK(&arena->free_work, arena_free_worker); bpf_map_init_from_attr(&arena->map, attr); + + err = bpf_map_alloc_pages(&arena->map, NUMA_NO_NODE, 1, &arena->scratch_page); + if (err) + goto err_free_arena; + range_tree_init(&arena->rt); err = range_tree_set(&arena->rt, 0, attr->max_entries); - if (err) { - bpf_map_area_free(arena); - goto err; - } + if (err) + goto err_free_scratch; mutex_init(&arena->lock); raw_res_spin_lock_init(&arena->spinlock); err = populate_pgtable_except_pte(arena); - if (err) { - range_tree_destroy(&arena->rt); - bpf_map_area_free(arena); - goto err; - } + if (err) + goto err_destroy_rt; return &arena->map; + +err_destroy_rt: + range_tree_destroy(&arena->rt); +err_free_scratch: + __free_page(arena->scratch_page); +err_free_arena: + bpf_map_area_free(arena); err: free_vm_area(kern_vm); return ERR_PTR(err); @@ -244,6 +283,7 @@ err: static int existing_page_cb(pte_t *ptep, unsigned long addr, void *data) { + struct bpf_arena *arena = data; struct page *page; pte_t pte; @@ -251,6 +291,12 @@ static int existing_page_cb(pte_t *ptep, unsigned long addr, void *data) if (!pte_present(pte)) /* sanity check */ return 0; page = pte_page(pte); + /* + * Skip the scratch page. The walk is page-table-driven, not range-tree-driven, + * so it can visit scratch PTEs at uaddrs the BPF program never allocated. + */ + if (page == arena->scratch_page) + return 0; /* * We do not update pte here: * 1. Nobody should be accessing bpf_arena's range outside of a kernel bug @@ -286,9 +332,10 @@ static void arena_map_free(struct bpf_map *map) * free those pages. */ apply_to_existing_page_range(&init_mm, bpf_arena_get_kern_vm_start(arena), - KERN_VM_SZ - GUARD_SZ, existing_page_cb, NULL); + SZ_4G + GUARD_SZ / 2, existing_page_cb, arena); free_vm_area(arena->kern_vm); range_tree_destroy(&arena->rt); + __free_page(arena->scratch_page); bpf_map_area_free(arena); } @@ -384,33 +431,37 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) return VM_FAULT_RETRY; page = vmalloc_to_page((void *)kaddr); - if (page) + if (page) { + if (page == arena->scratch_page) + /* BPF triggered scratch here; don't lazy-alloc over it */ + goto out_sigsegv; /* already have a page vmap-ed */ goto out; + } bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); if (arena->map.map_flags & BPF_F_SEGV_ON_FAULT) /* User space requested to segfault when page is not allocated by bpf prog */ - goto out_unlock_sigsegv; + goto out_sigsegv_memcg; ret = range_tree_clear(&arena->rt, vmf->pgoff, 1); if (ret) - goto out_unlock_sigsegv; + goto out_sigsegv_memcg; struct apply_range_data data = { .pages = &page, .i = 0 }; /* Account into memcg of the process that created bpf_arena */ ret = bpf_map_alloc_pages(map, NUMA_NO_NODE, 1, &page); if (ret) { range_tree_set(&arena->rt, vmf->pgoff, 1); - goto out_unlock_sigsegv; + goto out_sigsegv_memcg; } ret = apply_to_page_range(&init_mm, kaddr, PAGE_SIZE, apply_range_set_cb, &data); if (ret) { range_tree_set(&arena->rt, vmf->pgoff, 1); free_pages_nolock(page, 0); - goto out_unlock_sigsegv; + goto out_sigsegv_memcg; } flush_vmap_cache(kaddr, PAGE_SIZE); bpf_map_memcg_exit(old_memcg, new_memcg); @@ -419,8 +470,9 @@ out: raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); vmf->page = page; return 0; -out_unlock_sigsegv: +out_sigsegv_memcg: bpf_map_memcg_exit(old_memcg, new_memcg); +out_sigsegv: raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); return VM_FAULT_SIGSEGV; } @@ -685,6 +737,7 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, struct llist_head free_pages; struct llist_node *pos, *t; struct arena_free_span *s; + struct clear_range_data cdata; unsigned long flags; int ret = 0; @@ -713,9 +766,11 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, range_tree_set(&arena->rt, pgoff, page_cnt); init_llist_head(&free_pages); + cdata.free_pages = &free_pages; + cdata.scratch_page = arena->scratch_page; /* clear ptes and collect struct pages */ apply_to_existing_page_range(&init_mm, kaddr, page_cnt << PAGE_SHIFT, - apply_range_clear_cb, &free_pages); + apply_range_clear_cb, &cdata); /* drop the lock to do the tlb flush and zap pages */ raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); @@ -805,6 +860,7 @@ static void arena_free_worker(struct work_struct *work) struct arena_free_span *s; u64 arena_vm_start, user_vm_start; struct llist_head free_pages; + struct clear_range_data cdata; struct page *page; unsigned long full_uaddr; long kaddr, page_cnt, pgoff; @@ -818,6 +874,8 @@ static void arena_free_worker(struct work_struct *work) bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); init_llist_head(&free_pages); + cdata.free_pages = &free_pages; + cdata.scratch_page = arena->scratch_page; arena_vm_start = bpf_arena_get_kern_vm_start(arena); user_vm_start = bpf_arena_get_user_vm_start(arena); @@ -830,7 +888,7 @@ static void arena_free_worker(struct work_struct *work) /* clear ptes and collect pages in free_pages llist */ apply_to_existing_page_range(&init_mm, kaddr, page_cnt << PAGE_SHIFT, - apply_range_clear_cb, &free_pages); + apply_range_clear_cb, &cdata); range_tree_set(&arena->rt, pgoff, page_cnt); } @@ -945,23 +1003,12 @@ static int __init kfunc_init(void) } late_initcall(kfunc_init); -void bpf_prog_report_arena_violation(bool write, unsigned long addr, unsigned long fault_ip) +static void __bpf_prog_report_arena_violation(struct bpf_prog *prog, bool write, + unsigned long addr, unsigned long fault_ip) { struct bpf_stream_stage ss; - struct bpf_prog *prog; u64 user_vm_start; - /* - * The RCU read lock is held to safely traverse the latch tree, but we - * don't need its protection when accessing the prog, since it will not - * disappear while we are handling the fault. - */ - rcu_read_lock(); - prog = bpf_prog_ksym_find(fault_ip); - rcu_read_unlock(); - if (!prog) - return; - /* Use main prog for stream access */ prog = prog->aux->main_prog_aux->prog; @@ -974,3 +1021,53 @@ void bpf_prog_report_arena_violation(bool write, unsigned long addr, unsigned lo bpf_stream_dump_stack(ss); })); } + +bool bpf_arena_handle_page_fault(unsigned long addr, bool is_write, unsigned long fault_ip) +{ + struct bpf_arena *arena; + struct bpf_prog *prog; + unsigned long kbase; + unsigned long page_addr = addr & PAGE_MASK; + + prog = bpf_prog_find_from_stack(); + if (!prog) + return false; + + arena = prog->aux->arena; + /* a prog not using arena may be on stack, so arena can be NULL */ + if (!arena) + return false; + + kbase = bpf_arena_get_kern_vm_start(arena); + + /* + * Recovery covers the 4 GiB mappable band plus the upper half-guard. + * Lower guard is unreachable from kfuncs; an address there indicates + * a different bug class - leave it to the regular kernel oops path. + */ + if (page_addr < kbase || page_addr >= kbase + SZ_4G + GUARD_SZ / 2) + return false; + + apply_to_page_range(&init_mm, page_addr, PAGE_SIZE, + apply_range_set_scratch_cb, arena->scratch_page); + flush_vmap_cache(page_addr, PAGE_SIZE); + __bpf_prog_report_arena_violation(prog, is_write, page_addr - kbase, fault_ip); + return true; +} + +void bpf_prog_report_arena_violation(bool write, unsigned long addr, unsigned long fault_ip) +{ + struct bpf_prog *prog; + + /* + * The RCU read lock is held to safely traverse the latch tree, but we + * don't need its protection when accessing the prog, since it will not + * disappear while we are handling the fault. + */ + rcu_read_lock(); + prog = bpf_prog_ksym_find(fault_ip); + rcu_read_unlock(); + if (!prog) + return; + __bpf_prog_report_arena_violation(prog, write, addr, fault_ip); +} diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 8b018ff48875..fc3ee67486ce 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3350,6 +3350,11 @@ __weak u64 bpf_arena_get_kern_vm_start(struct bpf_arena *arena) { return 0; } +__weak bool bpf_arena_handle_page_fault(unsigned long addr, bool is_write, + unsigned long fault_ip) +{ + return false; +} #ifdef CONFIG_BPF_SYSCALL static int __init bpf_global_ma_init(void) -- cgit v1.2.3 From f211c81ddc368e5cc6ad69d171bca0fa52e71ad7 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 22 May 2026 07:22:14 -1000 Subject: bpf: Add sleepable variant of bpf_arena_alloc_pages for kernel callers The existing kernel-side export of bpf_arena_alloc_pages is _non_sleepable only - it's used by the verifier to inline the kfunc when the call site is non-sleepable. There is no sleepable equivalent for kernel callers. The kfunc bpf_arena_alloc_pages itself is BPF-only. sched_ext needs sleepable kernel-side allocs for its arena pool init/grow paths. Add bpf_arena_alloc_pages_sleepable() mirroring the _non_sleepable wrapper but passing sleepable=true to arena_alloc_pages(). Signed-off-by: Tejun Heo Reviewed-by: Emil Tsalapatis Link: https://lore.kernel.org/r/20260522172219.1423324-4-tj@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/bpf/arena.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 2da2c275cff6..9e379ef27d41 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -951,6 +951,19 @@ void *bpf_arena_alloc_pages_non_sleepable(void *p__map, void *addr__ign, u32 pag return (void *)arena_alloc_pages(arena, (long)addr__ign, page_cnt, node_id, false); } + +void *bpf_arena_alloc_pages_sleepable(void *p__map, void *addr__ign, u32 page_cnt, + int node_id, u64 flags) +{ + struct bpf_map *map = p__map; + struct bpf_arena *arena = container_of(map, struct bpf_arena, map); + + if (map->map_type != BPF_MAP_TYPE_ARENA || flags || !page_cnt) + return NULL; + + return (void *)arena_alloc_pages(arena, (long)addr__ign, page_cnt, node_id, true); +} + __bpf_kfunc void bpf_arena_free_pages(void *p__map, void *ptr__ign, u32 page_cnt) { struct bpf_map *map = p__map; -- cgit v1.2.3 From 7c48a28c1bbe26e272bc978a42adb757fc6aa639 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 22 May 2026 07:22:15 -1000 Subject: bpf: Add bpf_struct_ops_for_each_prog() Add a helper that walks the member progs of the struct_ops map containing a given @kdata vmtable. struct_ops ->reg() callbacks (and similar) sometimes need to inspect the loaded BPF programs, e.g. to discover maps they reference via prog->aux->used_maps. The implementation mirrors bpf_struct_ops_id(): container_of @kdata to recover the bpf_struct_ops_map, then iterate st_map->links[i]->prog for i in [0, funcs_cnt). Same access pattern, no new locking - by the time ->reg() fires st_map is fully populated and stable. A sched_ext follow-up walks the member progs of a cid-form scheduler's struct_ops map, reads prog->aux->arena directly, and requires all member progs to reference exactly one arena, without requiring the BPF program to call a registration kfunc. Signed-off-by: Tejun Heo Reviewed-by: Emil Tsalapatis Link: https://lore.kernel.org/r/20260522172219.1423324-5-tj@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/bpf/bpf_struct_ops.c | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 521cb9d7e8c7..5e51c1211673 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -1204,6 +1204,42 @@ u32 bpf_struct_ops_id(const void *kdata) } EXPORT_SYMBOL_GPL(bpf_struct_ops_id); +/** + * bpf_struct_ops_for_each_prog - Invoke @cb for each member prog + * @kdata: kernel-side struct_ops vmtable (the @kdata arg to ->reg/->update/->unreg) + * @cb: callback invoked once per member prog; non-zero return stops iteration + * @data: opaque argument passed to @cb + * + * Walks the struct_ops member progs registered on the map containing @kdata. + * Intended for use from struct_ops ->reg() callbacks (and similar) that need to + * inspect the loaded BPF programs (for example to discover maps they reference + * via @prog->aux->used_maps). + * + * Return 0 if iteration completed, otherwise the first non-zero @cb return. + */ +int bpf_struct_ops_for_each_prog(const void *kdata, + int (*cb)(struct bpf_prog *prog, void *data), + void *data) +{ + struct bpf_struct_ops_value *kvalue; + struct bpf_struct_ops_map *st_map; + u32 i; + int ret; + + kvalue = container_of(kdata, struct bpf_struct_ops_value, data); + st_map = container_of(kvalue, struct bpf_struct_ops_map, kvalue); + + for (i = 0; i < st_map->funcs_cnt; i++) { + if (!st_map->links[i]) + continue; + ret = cb(st_map->links[i]->prog, data); + if (ret) + return ret; + } + return 0; +} +EXPORT_SYMBOL_GPL(bpf_struct_ops_for_each_prog); + static bool bpf_struct_ops_valid_to_reg(struct bpf_map *map) { struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; -- cgit v1.2.3 From 53cc12a2dc88c2c6f62f507548640885a70a56a8 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 22 May 2026 07:22:16 -1000 Subject: bpf/arena: Add bpf_arena_map_kern_vm_start() and bpf_prog_arena() struct bpf_arena is opaque to callers outside arena.c. Add two helpers for struct_ops subsystems that need to reach into an arena: bpf_arena_map_kern_vm_start(struct bpf_map *map) returns @map's kern_vm_start. A sched_ext follow-up needs this to translate kern_va <-> uaddr. bpf_prog_arena(struct bpf_prog *prog) returns the bpf_map of the arena referenced by @prog (NULL if @prog references no arena). The verifier enforces at most one arena per program. Used by struct_ops callers that auto-discover an arena from a member prog and need to take a map reference. Suggested-by: Kumar Kartikeya Dwivedi Signed-off-by: Tejun Heo Reviewed-by: Emil Tsalapatis Link: https://lore.kernel.org/r/20260522172219.1423324-6-tj@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/bpf/arena.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 9e379ef27d41..1727503b25d8 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -84,6 +84,32 @@ u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena) return arena ? arena->user_vm_start : 0; } +/** + * bpf_arena_map_kern_vm_start - kern_vm_start lookup by struct bpf_map * + * @map: a BPF_MAP_TYPE_ARENA map + * + * Return @map's kern_vm_start. + */ +u64 bpf_arena_map_kern_vm_start(struct bpf_map *map) +{ + return bpf_arena_get_kern_vm_start(container_of(map, struct bpf_arena, map)); +} + +/** + * bpf_prog_arena - return the bpf_map of the arena referenced by @prog + * @prog: a loaded BPF program + * + * The verifier enforces at most one arena per program and stores it in + * prog->aux->arena. Return that arena's underlying bpf_map, or NULL if + * @prog does not reference an arena. + */ +struct bpf_map *bpf_prog_arena(struct bpf_prog *prog) +{ + struct bpf_arena *arena = prog->aux->arena; + + return arena ? &arena->map : NULL; +} + static long arena_map_peek_elem(struct bpf_map *map, void *value) { return -EOPNOTSUPP; -- cgit v1.2.3 From aa61e8b4fb2c9c5315228d0c830b5b2ae4df5cca Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 7 May 2026 09:57:15 -0700 Subject: rcutorture: Fully test lazy RCU Currently, rcutorture bypasses lazy RCU by using call_rcu_hurry(). This works, avoiding the dreaded rtort_pipe_count WARN(), but fails to fully test lazy RCU. The rtort_pipe_count WARN() splats because lazy RCU could delay the start of an RCU grace period for a full stutter period, which defaults to only three seconds. This commit therefore reverts the call_rcu_hurry() instances back to call_rcu(), but, in kernels built with CONFIG_RCU_LAZY=y, queues a workqueue handler just before the call to stutter_wait() in rcu_torture_writer(). This workqueue handler invokes rcu_barrier(), which motivates any lingering lazy callbacks, thus avoiding the splat. Reported-by: Saravana Kannan Signed-off-by: Paul E. McKenney Signed-off-by: Uladzislau Rezki (Sony) --- kernel/rcu/rcutorture.c | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 5f2848b828dc..882a158ada7b 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -572,7 +572,7 @@ static unsigned long rcu_no_completed(void) static void rcu_torture_deferred_free(struct rcu_torture *p) { - call_rcu_hurry(&p->rtort_rcu, rcu_torture_cb); + call_rcu(&p->rtort_rcu, rcu_torture_cb); } static void rcu_sync_torture_init(void) @@ -619,7 +619,7 @@ static struct rcu_torture_ops rcu_ops = { .poll_gp_state_exp = poll_state_synchronize_rcu, .cond_sync_exp = cond_synchronize_rcu_expedited, .cond_sync_exp_full = cond_synchronize_rcu_expedited_full, - .call = call_rcu_hurry, + .call = call_rcu, .cb_barrier = rcu_barrier, .fqs = rcu_force_quiescent_state, .gp_kthread_dbg = show_rcu_gp_kthreads, @@ -1145,7 +1145,7 @@ static void rcu_tasks_torture_deferred_free(struct rcu_torture *p) static void synchronize_rcu_mult_test(void) { - synchronize_rcu_mult(call_rcu_tasks, call_rcu_hurry); + synchronize_rcu_mult(call_rcu_tasks, call_rcu); } static struct rcu_torture_ops tasks_ops = { @@ -1631,6 +1631,17 @@ static void do_rtws_sync(struct torture_random_state *trsp, void (*sync)(void)) cpus_read_unlock(); } +/* + * Do an rcu_barrier() to motivate lazy callbacks during a stutter + * pause. Without this, we can get false-positives rtort_pipe_count + * splats. + */ +static void rcu_torture_writer_work(struct work_struct *work) +{ + if (cur_ops->cb_barrier) + cur_ops->cb_barrier(); +} + /* * RCU torture writer kthread. Repeatedly substitutes a new structure * for that pointed to by rcu_torture_current, freeing the old structure @@ -1651,6 +1662,7 @@ rcu_torture_writer(void *arg) int i; int idx; unsigned long j; + struct work_struct lazy_work; int oldnice = task_nice(current); struct rcu_gp_oldstate *rgo = NULL; int rgo_size = 0; @@ -1703,6 +1715,9 @@ rcu_torture_writer(void *arg) pr_alert("%s" TORTURE_FLAG " Waited %lu jiffies for boot to complete.\n", torture_type, jiffies - j); + if (IS_ENABLED(CONFIG_RCU_LAZY)) + INIT_WORK_ONSTACK(&lazy_work, rcu_torture_writer_work); + do { rcu_torture_writer_state = RTWS_FIXED_DELAY; torture_hrtimeout_us(500, 1000, &rand); @@ -1895,6 +1910,8 @@ rcu_torture_writer(void *arg) !rcu_gp_is_normal(); } rcu_torture_writer_state = RTWS_STUTTER; + if (IS_ENABLED(CONFIG_RCU_LAZY)) + queue_work(system_percpu_wq, &lazy_work); stutter_waited = stutter_wait("rcu_torture_writer"); if (stutter_waited && !atomic_read(&rcu_fwd_cb_nodelay) && @@ -1925,6 +1942,12 @@ rcu_torture_writer(void *arg) pr_alert("%s" TORTURE_FLAG " Dynamic grace-period expediting was disabled.\n", torture_type); + + if (IS_ENABLED(CONFIG_RCU_LAZY)) { + cancel_work_sync(&lazy_work); + destroy_work_on_stack(&lazy_work); + } + kfree(ulo); kfree(rgo); rcu_torture_writer_state = RTWS_STOPPING; -- cgit v1.2.3 From 59cf5dbcc95f11a6b03fad9fe9723ff31149459b Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 7 May 2026 09:57:16 -0700 Subject: torture: Add torture_sched_set_normal() for user-specified nice values This new torture_sched_set_normal() function clamps the nice value at the MIN_NICE..MAX_NICE limits, splatting it these limits are exceeded. It then invokes sched_set_normal() to set the new value. This prevents more difficult-to-debug failures within the scheduler. Signed-off-by: Paul E. McKenney Signed-off-by: Uladzislau Rezki (Sony) --- kernel/torture.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'kernel') diff --git a/kernel/torture.c b/kernel/torture.c index 62c1ac777694..77cb3589b19f 100644 --- a/kernel/torture.c +++ b/kernel/torture.c @@ -972,3 +972,19 @@ void _torture_stop_kthread(char *m, struct task_struct **tp) *tp = NULL; } EXPORT_SYMBOL_GPL(_torture_stop_kthread); + +/* + * Set the specified task's niceness value, saturating at limits. + * Saturating noisily, but saturating. + */ +void torture_sched_set_normal(struct task_struct *t, int nice) +{ + int realnice = nice; + + if (WARN_ON_ONCE(realnice > MAX_NICE)) + realnice = MAX_NICE; + if (WARN_ON_ONCE(realnice < MIN_NICE)) + realnice = MIN_NICE; + sched_set_normal(t, realnice); +} +EXPORT_SYMBOL_GPL(torture_sched_set_normal); -- cgit v1.2.3 From 5f136351edd37d779e45704d573f5b6d6cab1e6e Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 8 May 2026 10:43:51 -0700 Subject: rcu: Simplify rcu_do_batch() by applying clamp() This commit replaces a nested ?: sequence with clamp(). This does not reduce the number of lines of code, but it does simplify the line that it modifies. Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney Signed-off-by: Uladzislau Rezki (Sony) --- kernel/rcu/tree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 55df6d37145e..e46a5124c3eb 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2584,7 +2584,7 @@ static void rcu_do_batch(struct rcu_data *rdp) const long npj = NSEC_PER_SEC / HZ; long rrn = READ_ONCE(rcu_resched_ns); - rrn = rrn < NSEC_PER_MSEC ? NSEC_PER_MSEC : rrn > NSEC_PER_SEC ? NSEC_PER_SEC : rrn; + rrn = clamp(rrn, NSEC_PER_MSEC, NSEC_PER_SEC); tlimit = local_clock() + rrn; jlimit = jiffies + (rrn + npj + 1) / npj; jlimit_check = true; -- cgit v1.2.3 From d9b4d36b8c8fb6006e7b62dd0ddc218c87e8196a Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 8 May 2026 10:43:52 -0700 Subject: rcu: Simplify param_set_next_fqs_jiffies() by applying clamp_val() This commit replaces a nested ?: sequence with clamp_val(). This does not reduce the number of lines of code, but it does simplify the line that it modifies. Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney Signed-off-by: Uladzislau Rezki (Sony) --- kernel/rcu/tree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index e46a5124c3eb..09f0cef5014c 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -492,7 +492,7 @@ static int param_set_next_fqs_jiffies(const char *val, const struct kernel_param int ret = kstrtoul(val, 0, &j); if (!ret) { - WRITE_ONCE(*(ulong *)kp->arg, (j > HZ) ? HZ : (j ?: 1)); + WRITE_ONCE(*(ulong *)kp->arg, clamp_val(j, 1, HZ)); adjust_jiffies_till_sched_qs(); } return ret; -- cgit v1.2.3 From eceb65975256d7f7a5e5a5a2f4b7865eb32eaaf7 Mon Sep 17 00:00:00 2001 From: "Uladzislau Rezki (Sony)" Date: Wed, 11 Mar 2026 19:58:11 +0100 Subject: rcu: Latch normal synchronize_rcu() path on flood Currently, rcu_normal_wake_from_gp is only enabled by default on small systems(<= 16 CPUs) or when a user explicitly set it enabled. Introduce an adaptive latching mechanism: * Track the number of in-flight synchronize_rcu() requests using a new rcu_sr_normal_count counter; * If the count reaches/exceeds RCU_SR_NORMAL_LATCH_THR(64), it sets the rcu_sr_normal_latched, reverting new requests onto the scaled wait_rcu_gp() path; * The latch is cleared only when the pending requests are fully drained(nr == 0); * Enables rcu_normal_wake_from_gp by default for all systems, relying on this dynamic throttling instead of static CPU limits. Testing(synthetic flood workload): * Kernel version: 6.19.0-rc6 * Number of CPUs: 1536 * 60K concurrent synchronize_rcu() calls Perf(cycles, system-wide): total cycles: 932020263832 rcu_sr_normal_add_req(): 2650282811 cycles(~0.28%) Perf report excerpt: 0.01% 0.01% sync_test/... [k] rcu_sr_normal_add_req Measured overhead of rcu_sr_normal_add_req() remained ~0.28% of total CPU cycles in this synthetic stress test. Reviewed-by: Frederic Weisbecker Tested-by: Samir M Suggested-by: Joel Fernandes Signed-off-by: Uladzislau Rezki (Sony) --- kernel/rcu/tree.c | 52 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 13 deletions(-) (limited to 'kernel') diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 09f0cef5014c..afb9e7db8f78 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -1632,17 +1632,21 @@ static void rcu_sr_put_wait_head(struct llist_node *node) atomic_set_release(&sr_wn->inuse, 0); } -/* Enable rcu_normal_wake_from_gp automatically on small systems. */ -#define WAKE_FROM_GP_CPU_THRESHOLD 16 - -static int rcu_normal_wake_from_gp = -1; +static int rcu_normal_wake_from_gp = 1; module_param(rcu_normal_wake_from_gp, int, 0644); static struct workqueue_struct *sync_wq; +#define RCU_SR_NORMAL_LATCH_THR 64 + +/* Number of in-flight synchronize_rcu() calls queued on srs_next. */ +static atomic_long_t rcu_sr_normal_count; +static int rcu_sr_normal_latched; /* 0/1 */ + static void rcu_sr_normal_complete(struct llist_node *node) { struct rcu_synchronize *rs = container_of( (struct rcu_head *) node, struct rcu_synchronize, head); + long nr; WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && !poll_state_synchronize_rcu_full(&rs->oldstate), @@ -1650,6 +1654,15 @@ static void rcu_sr_normal_complete(struct llist_node *node) /* Finally. */ complete(&rs->completion); + nr = atomic_long_dec_return(&rcu_sr_normal_count); + WARN_ON_ONCE(nr < 0); + + /* + * Unlatch: switch back to normal path when fully + * drained and if it has been latched. + */ + if (nr == 0) + (void)cmpxchg_relaxed(&rcu_sr_normal_latched, 1, 0); } static void rcu_sr_normal_gp_cleanup_work(struct work_struct *work) @@ -1795,6 +1808,24 @@ static bool rcu_sr_normal_gp_init(void) static void rcu_sr_normal_add_req(struct rcu_synchronize *rs) { + /* + * Increment before publish to avoid a complete + * vs enqueue race on latch. + */ + long nr = atomic_long_inc_return(&rcu_sr_normal_count); + + /* + * Latch when threshold is reached. Checking for an exact match + * restricts cmpxchg() to a single context. + * + * This latch is intentionally relaxed and best-effort. Concurrent + * set/clear can race and temporarily lose the latch, which is OK + * because it only selects between the fast and fallback paths. + */ + if (nr == RCU_SR_NORMAL_LATCH_THR) + (void)cmpxchg_relaxed(&rcu_sr_normal_latched, 0, 1); + + /* Publish for the GP kthread/worker. */ llist_add((struct llist_node *) &rs->head, &rcu_state.srs_next); } @@ -3278,14 +3309,15 @@ static void synchronize_rcu_normal(void) { struct rcu_synchronize rs; + init_rcu_head_on_stack(&rs.head); trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("request")); - if (READ_ONCE(rcu_normal_wake_from_gp) < 1) { + if (READ_ONCE(rcu_normal_wake_from_gp) < 1 || + READ_ONCE(rcu_sr_normal_latched)) { wait_rcu_gp(call_rcu_hurry); goto trace_complete_out; } - init_rcu_head_on_stack(&rs.head); init_completion(&rs.completion); /* @@ -3302,10 +3334,10 @@ static void synchronize_rcu_normal(void) /* Now we can wait. */ wait_for_completion(&rs.completion); - destroy_rcu_head_on_stack(&rs.head); trace_complete_out: trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("complete")); + destroy_rcu_head_on_stack(&rs.head); } /** @@ -4904,12 +4936,6 @@ void __init rcu_init(void) sync_wq = alloc_workqueue("sync_wq", WQ_MEM_RECLAIM | WQ_UNBOUND, 0); WARN_ON(!sync_wq); - /* Respect if explicitly disabled via a boot parameter. */ - if (rcu_normal_wake_from_gp < 0) { - if (num_possible_cpus() <= WAKE_FROM_GP_CPU_THRESHOLD) - rcu_normal_wake_from_gp = 1; - } - /* Fill in default value for rcutree.qovld boot parameter. */ /* -After- the rcu_node ->lock fields are initialized! */ if (qovld < 0) -- cgit v1.2.3 From 42c5468f9cdc0c892fec3c0916b3ac5b670775af Mon Sep 17 00:00:00 2001 From: Zqiang Date: Thu, 23 Apr 2026 19:19:30 +0800 Subject: rcu-tasks: Fix possible boot-time tests failed for the call_rcu_tasks() The following scenarios will cause the call_rcu_tasks() boot-time tests failed: CPU0 CPU1 rcu_init_tasks_generic() ->rcu_tasks_initiate_self_tests() ->call_rcu_tasks_trace(&tests[1].rh, test_rcu_tasks_callback) ->call_rcu_tasks_generic() ->havekthread = smp_load_acquire(&rtp->kthread_ptr) "The havekthread is false" .... rcu_tasks_kthread() ->smp_store_release(&rtp->kthread_ptr, current) ->rcu_tasks_one_gp() ->rcuwait_wait_event() ->rcu_tasks_need_gpcb() ->for (cpu = 0; cpu < dequeue_limit; cpu++) ->rcu_segcblist_n_cbs(&rtpcp->cblist) == 0 ->schedule() ->raw_spin_trylock_rcu_node() ->needwake = (func == wakeme_after_rcu) || (rcu_segcblist_n_cbs(&rtpcp->cblist) == rcu_task_lazy_lim) "the rcu_task_lazy_lim default value is 32, and the func pointer is test_rcu_tasks_callback, lead to needwake is false." ->if (havekthread && !needwake && !timer_pending(&rtpcp->lazy_timer)) "the havekthread is false, will not enter here." .... "the needwake is false lead to rtp_irq_work can not queue, even if the rtp->kthread_ptr already exists at this point." ->if (needwake && READ_ONCE(rtp->kthread_ptr)) ->irq_work_queue(&rtpcp->rtp_irq_work) For the above scenarios, if the call_rcu_tasks() is not called again afterward, the rcu_tasks_kthread will not have a chance to be wakeup, the test_rcu_tasks_callback() will never be called, the boot-time tests failed can happen, this commit therefore check havekthread variable, if it's false and the rtpcp->cblist is empty, set needwake variable is true, if the rtp->kthread_ptr exist, the rtpcp->rtp_irq_work can be queued to wakeup rcu_tasks_kthread. Signed-off-by: Zqiang Signed-off-by: Uladzislau Rezki (Sony) --- kernel/rcu/tasks.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 48f0d803c8e2..f4da5fad70f5 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -373,7 +373,8 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, // Queuing callbacks before initialization not yet supported. if (WARN_ON_ONCE(!rcu_segcblist_is_enabled(&rtpcp->cblist))) rcu_segcblist_init(&rtpcp->cblist); - needwake = (func == wakeme_after_rcu) || + needwake = (!havekthread && rcu_segcblist_empty(&rtpcp->cblist)) || + (func == wakeme_after_rcu) || (rcu_segcblist_n_cbs(&rtpcp->cblist) == rcu_task_lazy_lim); if (havekthread && !needwake && !timer_pending(&rtpcp->lazy_timer)) { if (rtp->lazy_jiffies) -- cgit v1.2.3 From 0e2819cba977f910c7eeaf77c705e28787c3385d Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 22 May 2026 07:06:00 -1000 Subject: sched_ext: Require an arena for cid-form schedulers Upcoming patches will let the kernel place arena-resident scratch shared with the BPF program (e.g. per-CPU set_cmask cmask) so the BPF side can dereference it directly via __arena pointers, replacing the current cmask_copy_from_kernel() probe-read loop. That requires each cid-form scheduler to expose its arena to the kernel. Kernel- side accesses are recovered by the per-arena scratch-page mechanism. bpf_scx_reg_cid() walks the struct_ops member progs via bpf_struct_ops_for_each_prog() and reads each prog's arena via bpf_prog_arena(). The verifier enforces one arena per program, so each member prog contributes at most one arena. All non-NULL contributions must match and at least one member prog must use an arena. The map ref is held on scx_sched and dropped on sched destroy. cpu-form schedulers (bpf_scx_reg) are unchanged - no arena requirement. Signed-off-by: Tejun Heo --- kernel/sched/ext.c | 56 ++++++++++++++++++++++++++++++++++++++++++++- kernel/sched/ext_internal.h | 8 +++++++ 2 files changed, 63 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 295df262ee5a..53708fd3cc34 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -5003,6 +5003,8 @@ static void scx_sched_free_rcu_work(struct work_struct *work) rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL); free_exit_info(sch->exit_info); + if (sch->arena_map) + bpf_map_put(sch->arena_map); kfree(sch); } @@ -6746,6 +6748,7 @@ struct scx_enable_cmd { struct sched_ext_ops_cid *ops_cid; }; bool is_cid_type; + struct bpf_map *arena_map; /* arena ref to transfer to sch */ int ret; }; @@ -6913,6 +6916,15 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, return ERR_PTR(ret); } #endif /* CONFIG_EXT_SUB_SCHED */ + + /* + * Consume the arena_map ref bpf_scx_reg_cid() took. Defer to here so + * earlier failure paths leave cmd->arena_map set and bpf_scx_reg_cid + * drops the ref. After this point, sch owns the ref and any cleanup + * runs through scx_sched_free_rcu_work() which puts it. + */ + sch->arena_map = cmd->arena_map; + cmd->arena_map = NULL; return sch; #ifdef CONFIG_EXT_SUB_SCHED @@ -7898,11 +7910,53 @@ static int bpf_scx_reg(void *kdata, struct bpf_link *link) return scx_enable(&cmd, link); } +struct scx_arena_scan { + struct bpf_map *arena; + int err; +}; + +/* + * The verifier enforces one arena per BPF program, so each struct_ops + * member prog contributes at most one arena via bpf_prog_arena(). + * Require all non-NULL contributions to match. + */ +static int scx_arena_scan_prog(struct bpf_prog *prog, void *data) +{ + struct scx_arena_scan *s = data; + struct bpf_map *arena = bpf_prog_arena(prog); + + if (!arena) + return 0; + if (s->arena && s->arena != arena) { + s->err = -EINVAL; + return 1; + } + s->arena = arena; + return 0; +} + static int bpf_scx_reg_cid(void *kdata, struct bpf_link *link) { struct scx_enable_cmd cmd = { .ops_cid = kdata, .is_cid_type = true }; + struct scx_arena_scan scan = {}; + int ret; - return scx_enable(&cmd, link); + bpf_struct_ops_for_each_prog(kdata, scx_arena_scan_prog, &scan); + if (scan.err) { + pr_err("sched_ext: cid-form scheduler uses multiple arena maps\n"); + return scan.err; + } + if (!scan.arena) { + pr_err("sched_ext: cid-form scheduler must use a BPF arena map\n"); + return -EINVAL; + } + + bpf_map_inc(scan.arena); + cmd.arena_map = scan.arena; + ret = scx_enable(&cmd, link); + if (cmd.arena_map) /* not consumed by scx_alloc_and_add_sched() */ + bpf_map_put(cmd.arena_map); + return ret; } static void bpf_scx_unreg(void *kdata, struct bpf_link *link) diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext_internal.h index 7258aea94b9f..d40cfd29ddaa 100644 --- a/kernel/sched/ext_internal.h +++ b/kernel/sched/ext_internal.h @@ -1111,6 +1111,14 @@ struct scx_sched { struct sched_ext_ops_cid ops_cid; }; bool is_cid_type; /* true if registered via bpf_sched_ext_ops_cid */ + + /* + * Arena map auto-discovered from member progs at struct_ops attach. + * cid-form schedulers must use exactly one arena across all member + * progs. NULL on cpu-form. + */ + struct bpf_map *arena_map; + DECLARE_BITMAP(has_op, SCX_OPI_END); /* -- cgit v1.2.3 From 9eca087deb0b35f3170109a9630a6c5c06c2e222 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 22 May 2026 07:06:01 -1000 Subject: sched_ext: Sub-allocator over kernel-claimed BPF arena pages Build a per-scheduler sub-allocator on top of pages claimed from the BPF arena registered in the previous patch. Subsequent kernel-managed arena-resident structures (e.g. per-CPU set_cmask cmask) carve their storage from this pool. scx_arena_pool_init() creates a gen_pool. scx_arena_alloc() returns the kernel VA. On exhaustion, the pool grows by claiming more pages via bpf_arena_alloc_pages_sleepable(). Chunks are added at the kernel-side mapping address. Callers translate to the BPF-arena form themselves if needed. Allocations sleep (GFP_KERNEL) - they may grow the pool through vzalloc and arena page allocation. All current consumers run from the enable path (after ops.init() and the kernel-side arena auto-discovery, before validate_ops()), where sleeping is fine. scx_arena_pool_destroy() walks each chunk, returns outstanding ranges to the gen_pool with gen_pool_free() and then calls gen_pool_destroy(). The underlying arena pages are released when the arena map itself is torn down, so the pool destroy doesn't free them explicitly. v2: Switch scx_arena_alloc() to a loop. (Andrea) Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/build_policy.c | 4 ++ kernel/sched/ext.c | 11 ++++ kernel/sched/ext_arena.c | 126 ++++++++++++++++++++++++++++++++++++++++++++ kernel/sched/ext_arena.h | 18 +++++++ kernel/sched/ext_internal.h | 5 ++ 5 files changed, 164 insertions(+) create mode 100644 kernel/sched/ext_arena.c create mode 100644 kernel/sched/ext_arena.h (limited to 'kernel') diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c index 5e76c9177d54..067979a7b69e 100644 --- a/kernel/sched/build_policy.c +++ b/kernel/sched/build_policy.c @@ -59,12 +59,16 @@ #ifdef CONFIG_SCHED_CLASS_EXT # include +# include +# include # include "ext_types.h" # include "ext_internal.h" # include "ext_cid.h" +# include "ext_arena.h" # include "ext_idle.h" # include "ext.c" # include "ext_cid.c" +# include "ext_arena.c" # include "ext_idle.c" #endif diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 53708fd3cc34..f5c67e3ff075 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -5003,6 +5003,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work) rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL); free_exit_info(sch->exit_info); + scx_arena_pool_destroy(sch); if (sch->arena_map) bpf_map_put(sch->arena_map); kfree(sch); @@ -7155,6 +7156,12 @@ static void scx_root_enable_workfn(struct kthread_work *work) sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; } + ret = scx_arena_pool_init(sch); + if (ret) { + cpus_read_unlock(); + goto err_disable; + } + for (i = SCX_OPI_CPU_HOTPLUG_BEGIN; i < SCX_OPI_CPU_HOTPLUG_END; i++) if (((void (**)(void))ops)[i]) set_bit(i, sch->has_op); @@ -7473,6 +7480,10 @@ static void scx_sub_enable_workfn(struct kthread_work *work) sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; } + ret = scx_arena_pool_init(sch); + if (ret) + goto err_disable; + if (validate_ops(sch, ops)) goto err_disable; diff --git a/kernel/sched/ext_arena.c b/kernel/sched/ext_arena.c new file mode 100644 index 000000000000..b413e155ba6b --- /dev/null +++ b/kernel/sched/ext_arena.c @@ -0,0 +1,126 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * scx_arena_pool: kernel-side sub-allocator over BPF-arena pages. + * + * Each chunk added to @sch->arena_pool comes from one + * bpf_arena_alloc_pages_sleepable() call and is registered at the + * kernel-side mapping address. Callers translate to the BPF-arena form + * themselves if needed. + * + * Allocations grow the pool on demand. Underlying arena pages are released + * when the arena map itself is torn down. + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2026 Tejun Heo + */ + +enum scx_arena_consts { + SCX_ARENA_MIN_ORDER = 3, /* 8-byte minimum sub-allocation */ + SCX_ARENA_GROW_PAGES = 4, /* per growth */ +}; + +s32 scx_arena_pool_init(struct scx_sched *sch) +{ + if (!sch->arena_map) + return 0; + + sch->arena_pool = gen_pool_create(SCX_ARENA_MIN_ORDER, NUMA_NO_NODE); + if (!sch->arena_pool) + return -ENOMEM; + return 0; +} + +static void scx_arena_clear_chunk(struct gen_pool *pool, struct gen_pool_chunk *chunk, + void *data) +{ + int order = pool->min_alloc_order; + size_t chunk_sz = chunk->end_addr - chunk->start_addr + 1; + unsigned long end_bit = chunk_sz >> order; + unsigned long b, e; + + for_each_set_bitrange(b, e, chunk->bits, end_bit) + gen_pool_free(pool, chunk->start_addr + (b << order), + (e - b) << order); +} + +/* + * Tear down the pool. Outstanding gen_pool allocations are freed via + * scx_arena_clear_chunk() so gen_pool_destroy() doesn't BUG. The underlying + * arena pages are released when the arena map itself is torn down. + */ +void scx_arena_pool_destroy(struct scx_sched *sch) +{ + if (!sch->arena_pool) + return; + gen_pool_for_each_chunk(sch->arena_pool, scx_arena_clear_chunk, NULL); + gen_pool_destroy(sch->arena_pool); + sch->arena_pool = NULL; +} + +/* + * Grow the pool by @page_cnt pages. bpf_arena_alloc_pages_sleepable() and + * gen_pool_add() (which calls vzalloc(GFP_KERNEL)) require a sleepable + * context. + */ +static int scx_arena_grow(struct scx_sched *sch, u32 page_cnt) +{ + u64 kern_vm_start; + u32 uaddr32; + void *p; + int ret; + + if (!sch->arena_map || !sch->arena_pool) + return -EINVAL; + + p = bpf_arena_alloc_pages_sleepable(sch->arena_map, NULL, + page_cnt, NUMA_NO_NODE, 0); + if (!p) + return -ENOMEM; + + uaddr32 = (u32)(unsigned long)p; + kern_vm_start = bpf_arena_map_kern_vm_start(sch->arena_map); + + ret = gen_pool_add(sch->arena_pool, kern_vm_start + uaddr32, + page_cnt * PAGE_SIZE, NUMA_NO_NODE); + if (ret) { + bpf_arena_free_pages_non_sleepable(sch->arena_map, p, page_cnt); + return ret; + } + return 0; +} + +/* + * Allocate @size bytes from the arena pool. Returns kernel VA on success, NULL + * on failure. May grow the pool via scx_arena_grow() which sleeps. Caller must + * be in a GFP_KERNEL context. + */ +void *scx_arena_alloc(struct scx_sched *sch, size_t size) +{ + unsigned long kern_va; + u32 page_cnt; + + might_sleep(); + + if (!sch->arena_pool) + return NULL; + + while (true) { + kern_va = gen_pool_alloc(sch->arena_pool, size); + if (kern_va) + break; + page_cnt = max_t(u32, SCX_ARENA_GROW_PAGES, + (size + PAGE_SIZE - 1) >> PAGE_SHIFT); + if (scx_arena_grow(sch, page_cnt)) + return NULL; + } + + return (void *)kern_va; +} + +void scx_arena_free(struct scx_sched *sch, void *kern_va, size_t size) +{ + if (sch->arena_pool && kern_va) + gen_pool_free(sch->arena_pool, (unsigned long)kern_va, size); +} diff --git a/kernel/sched/ext_arena.h b/kernel/sched/ext_arena.h new file mode 100644 index 000000000000..4f3610160102 --- /dev/null +++ b/kernel/sched/ext_arena.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Copyright (c) 2025 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2025 Tejun Heo + */ +#ifndef _KERNEL_SCHED_EXT_ARENA_H +#define _KERNEL_SCHED_EXT_ARENA_H + +struct scx_sched; + +s32 scx_arena_pool_init(struct scx_sched *sch); +void scx_arena_pool_destroy(struct scx_sched *sch); +void *scx_arena_alloc(struct scx_sched *sch, size_t size); +void scx_arena_free(struct scx_sched *sch, void *kern_va, size_t size); + +#endif /* _KERNEL_SCHED_EXT_ARENA_H */ diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext_internal.h index d40cfd29ddaa..ff7e882bd67a 100644 --- a/kernel/sched/ext_internal.h +++ b/kernel/sched/ext_internal.h @@ -1116,8 +1116,13 @@ struct scx_sched { * Arena map auto-discovered from member progs at struct_ops attach. * cid-form schedulers must use exactly one arena across all member * progs. NULL on cpu-form. + * + * @arena_pool sub-allocates @arena_map. Each gen_pool chunk is added + * at the kernel-side mapping address. Grows on demand and pages are + * not released until sched destroy. */ struct bpf_map *arena_map; + struct gen_pool *arena_pool; DECLARE_BITMAP(has_op, SCX_OPI_END); -- cgit v1.2.3 From abdc2516f100d8f9e637a49e4fdfd2d09a318680 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 22 May 2026 07:06:01 -1000 Subject: sched_ext: Convert ops.set_cmask() to arena-resident cmask ops_cid.set_cmask() expects a cmask. The kernel couldn't write into the arena, so it translated cpumask -> cmask in kernel memory and passed the result as a trusted pointer. The BPF cmask helpers all operate on arena cmasks though, so the BPF side had to word-by-word probe-read the kernel cmask into an arena cmask via cmask_copy_from_kernel() before any helper could touch it. It works, but is clumsy. With direct kernel-side arena access now in place, build the cmask in the arena. The kernel writes to it through the kern_va side of the dual mapping. BPF directly dereferences it via an __arena pointer like any other arena struct. Signed-off-by: Tejun Heo Reviewed-by: Emil Tsalapatis --- kernel/sched/ext.c | 68 +++++++++++++++++++++++++++++++++++++++++---- kernel/sched/ext_cid.c | 20 +------------ kernel/sched/ext_internal.h | 10 +++++-- 3 files changed, 72 insertions(+), 26 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index f5c67e3ff075..83272acf1763 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -621,11 +621,16 @@ static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, update_locked_rq(rq); if (scx_is_cid_type()) { - struct scx_cmask *cmask = this_cpu_ptr(scx_set_cmask_scratch); - - lockdep_assert_irqs_disabled(); - scx_cpumask_to_cmask(cpumask, cmask); - sch->ops_cid.set_cmask(task, cmask); + struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch); + unsigned long uaddr = (unsigned long)kern_va - + bpf_arena_map_kern_vm_start(sch->arena_map); + /* + * Build the per-CPU arena cmask and hand BPF the uaddr. Caller + * holds the rq lock with IRQs disabled, which makes us the sole + * user of the scratch area. + */ + scx_cpumask_to_cmask(cpumask, kern_va); + sch->ops_cid.set_cmask(task, (struct scx_cmask *)uaddr); } else { sch->ops.set_cpumask(task, cpumask); } @@ -4949,6 +4954,48 @@ static const struct attribute_group scx_global_attr_group = { static void free_pnode(struct scx_sched_pnode *pnode); static void free_exit_info(struct scx_exit_info *ei); +static s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch) +{ + size_t size = struct_size_t(struct scx_cmask, bits, + SCX_CMASK_NR_WORDS(num_possible_cpus())); + int cpu; + + if (!sch->is_cid_type || !sch->arena_pool) + return 0; + + sch->set_cmask_scratch = alloc_percpu(struct scx_cmask *); + if (!sch->set_cmask_scratch) + return -ENOMEM; + + for_each_possible_cpu(cpu) { + struct scx_cmask **slot = per_cpu_ptr(sch->set_cmask_scratch, cpu); + + *slot = scx_arena_alloc(sch, size); + if (!*slot) + return -ENOMEM; + scx_cmask_init(*slot, 0, num_possible_cpus()); + } + return 0; +} + +static void scx_set_cmask_scratch_free(struct scx_sched *sch) +{ + size_t size = struct_size_t(struct scx_cmask, bits, + SCX_CMASK_NR_WORDS(num_possible_cpus())); + int cpu; + + if (!sch->set_cmask_scratch) + return; + + for_each_possible_cpu(cpu) { + struct scx_cmask **slot = per_cpu_ptr(sch->set_cmask_scratch, cpu); + + scx_arena_free(sch, *slot, size); + } + free_percpu(sch->set_cmask_scratch); + sch->set_cmask_scratch = NULL; +} + static void scx_sched_free_rcu_work(struct work_struct *work) { struct rcu_work *rcu_work = to_rcu_work(work); @@ -5003,6 +5050,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work) rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL); free_exit_info(sch->exit_info); + scx_set_cmask_scratch_free(sch); scx_arena_pool_destroy(sch); if (sch->arena_map) bpf_map_put(sch->arena_map); @@ -7162,6 +7210,12 @@ static void scx_root_enable_workfn(struct kthread_work *work) goto err_disable; } + ret = scx_set_cmask_scratch_alloc(sch); + if (ret) { + cpus_read_unlock(); + goto err_disable; + } + for (i = SCX_OPI_CPU_HOTPLUG_BEGIN; i < SCX_OPI_CPU_HOTPLUG_END; i++) if (((void (**)(void))ops)[i]) set_bit(i, sch->has_op); @@ -7484,6 +7538,10 @@ static void scx_sub_enable_workfn(struct kthread_work *work) if (ret) goto err_disable; + ret = scx_set_cmask_scratch_alloc(sch); + if (ret) + goto err_disable; + if (validate_ops(sch, ops)) goto err_disable; diff --git a/kernel/sched/ext_cid.c b/kernel/sched/ext_cid.c index 0c91b951fd33..808c6390da5a 100644 --- a/kernel/sched/ext_cid.c +++ b/kernel/sched/ext_cid.c @@ -7,14 +7,6 @@ */ #include -/* - * Per-cpu scratch cmask used by scx_call_op_set_cpumask() to synthesize a - * cmask from a cpumask. Allocated alongside the cid arrays on first enable - * and never freed. Sized to the full cid space. Caller holds rq lock so - * this_cpu_ptr is safe. - */ -struct scx_cmask __percpu *scx_set_cmask_scratch; - /* * cid tables. * @@ -54,8 +46,6 @@ static s32 scx_cid_arrays_alloc(void) u32 npossible = num_possible_cpus(); s16 *cid_to_cpu, *cpu_to_cid; struct scx_cid_topo *cid_topo; - struct scx_cmask __percpu *set_cmask_scratch; - s32 cpu; if (scx_cid_to_cpu_tbl) return 0; @@ -63,25 +53,17 @@ static s32 scx_cid_arrays_alloc(void) cid_to_cpu = kzalloc_objs(*scx_cid_to_cpu_tbl, npossible, GFP_KERNEL); cpu_to_cid = kzalloc_objs(*scx_cpu_to_cid_tbl, nr_cpu_ids, GFP_KERNEL); cid_topo = kmalloc_objs(*scx_cid_topo, npossible, GFP_KERNEL); - set_cmask_scratch = __alloc_percpu(struct_size(set_cmask_scratch, bits, - SCX_CMASK_NR_WORDS(npossible)), - sizeof(u64)); - if (!cid_to_cpu || !cpu_to_cid || !cid_topo || !set_cmask_scratch) { + if (!cid_to_cpu || !cpu_to_cid || !cid_topo) { kfree(cid_to_cpu); kfree(cpu_to_cid); kfree(cid_topo); - free_percpu(set_cmask_scratch); return -ENOMEM; } WRITE_ONCE(scx_cid_to_cpu_tbl, cid_to_cpu); WRITE_ONCE(scx_cpu_to_cid_tbl, cpu_to_cid); WRITE_ONCE(scx_cid_topo, cid_topo); - for_each_possible_cpu(cpu) - scx_cmask_init(per_cpu_ptr(set_cmask_scratch, cpu), - 0, npossible); - WRITE_ONCE(scx_set_cmask_scratch, set_cmask_scratch); return 0; } diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext_internal.h index ff7e882bd67a..9bb65367f510 100644 --- a/kernel/sched/ext_internal.h +++ b/kernel/sched/ext_internal.h @@ -1124,6 +1124,14 @@ struct scx_sched { struct bpf_map *arena_map; struct gen_pool *arena_pool; + /* + * Per-CPU arena cmask used by scx_call_op_set_cpumask() to hand a cmask + * to ops_cid.set_cmask(). The kernel writes through the stored kern_va; + * the BPF-arena uaddr handed to BPF is recovered by subtracting the + * arena's kern_vm_start. + */ + struct scx_cmask * __percpu *set_cmask_scratch; + DECLARE_BITMAP(has_op, SCX_OPI_END); /* @@ -1480,8 +1488,6 @@ enum scx_ops_state { extern struct scx_sched __rcu *scx_root; DECLARE_PER_CPU(struct rq *, scx_locked_rq_state); -extern struct scx_cmask __percpu *scx_set_cmask_scratch; - /* * True when the currently loaded scheduler hierarchy is cid-form. All scheds * in a hierarchy share one form, so this single key tells callsites which -- cgit v1.2.3 From 8fd2f26fa2a33cfe8ac043f976137ecf5b567f03 Mon Sep 17 00:00:00 2001 From: "Pratyush Yadav (Google)" Date: Tue, 19 May 2026 15:33:30 +0200 Subject: kho: fix order calculation for kho_unpreserve_pages() Commit 91e74fa8b1bc ("kho: make sure preservations do not span multiple NUMA nodes") made sure preservations from kho_preserve_pages() do not span multiple NUMA nodes. If they do, the order is reduced and tried again. The same logic was not implemented for kho_unpreserve_pages(). This can result in unpreserve calculating a different order than preserve, and thus not actually unpreserving the pages. Fix this by moving the order calculation logic to __kho_preserve_pages_order() and use it from both preserve and unpreserve paths. Move __kho_unpreserve() down to avoid having a forward declaration. Its users are further down in the file anyway. Also, it results in grouping for all the page-level preservation and unpreservation functions. This unfortunately makes the diff hard to read, but the main change in __kho_unpreserve() is to call __kho_preserve_pages_order() instead of open-coding the order calculation. Fixes: 91e74fa8b1bc ("kho: make sure preservations do not span multiple NUMA nodes") Cc: stable@vger.kernel.org Signed-off-by: Pratyush Yadav (Google) Reviewed-by: Samiullah Khawaja Reviewed-by: Pasha Tatashin Link: https://patch.msgid.link/20260519133332.2498092-1-pratyush@kernel.org Signed-off-by: Mike Rapoport (Microsoft) --- kernel/liveupdate/kexec_handover.c | 56 ++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 24 deletions(-) (limited to 'kernel') diff --git a/kernel/liveupdate/kexec_handover.c b/kernel/liveupdate/kexec_handover.c index 2592f7ca16e2..1b592d86dc48 100644 --- a/kernel/liveupdate/kexec_handover.c +++ b/kernel/liveupdate/kexec_handover.c @@ -357,20 +357,6 @@ int kho_radix_walk_tree(struct kho_radix_tree *tree, } EXPORT_SYMBOL_GPL(kho_radix_walk_tree); -static void __kho_unpreserve(struct kho_radix_tree *tree, - unsigned long pfn, unsigned long end_pfn) -{ - unsigned int order; - - while (pfn < end_pfn) { - order = min(count_trailing_zeros(pfn), ilog2(end_pfn - pfn)); - - kho_radix_del_page(tree, pfn, order); - - pfn += 1 << order; - } -} - /* For physically contiguous 0-order pages. */ static void kho_init_pages(struct page *page, unsigned long nr_pages) { @@ -860,6 +846,37 @@ void kho_unpreserve_folio(struct folio *folio) } EXPORT_SYMBOL_GPL(kho_unpreserve_folio); +static unsigned int __kho_preserve_pages_order(unsigned long start_pfn, + unsigned long end_pfn) +{ + unsigned int order = min(count_trailing_zeros(start_pfn), + ilog2(end_pfn - start_pfn)); + + /* + * Make sure all the pages in a single preservation are in the same NUMA + * node. The restore machinery can not cope with a preservation spanning + * multiple NUMA nodes. + */ + while (pfn_to_nid(start_pfn) != pfn_to_nid(start_pfn + (1UL << order) - 1)) + order--; + + return order; +} + +static void __kho_unpreserve(struct kho_radix_tree *tree, + unsigned long pfn, unsigned long end_pfn) +{ + unsigned int order; + + while (pfn < end_pfn) { + order = __kho_preserve_pages_order(pfn, end_pfn); + + kho_radix_del_page(tree, pfn, order); + + pfn += 1 << order; + } +} + /** * kho_preserve_pages - preserve contiguous pages across kexec * @page: first page in the list. @@ -885,16 +902,7 @@ int kho_preserve_pages(struct page *page, unsigned long nr_pages) } while (pfn < end_pfn) { - unsigned int order = - min(count_trailing_zeros(pfn), ilog2(end_pfn - pfn)); - - /* - * Make sure all the pages in a single preservation are in the - * same NUMA node. The restore machinery can not cope with a - * preservation spanning multiple NUMA nodes. - */ - while (pfn_to_nid(pfn) != pfn_to_nid(pfn + (1UL << order) - 1)) - order--; + unsigned int order = __kho_preserve_pages_order(pfn, end_pfn); err = kho_radix_add_page(tree, pfn, order); if (err) { -- cgit v1.2.3 From 4f365e7a5d448dab7e0bb56ed32ff2bfddd134bd Mon Sep 17 00:00:00 2001 From: "Christian Brauner (Amutable)" Date: Wed, 20 May 2026 23:48:52 +0200 Subject: sched/coredump: introduce enum task_dumpable Replace the SUID_DUMP_DISABLE/USER/ROOT preprocessor constants with enum task_dumpable. Numeric values are preserved (kernel.suid_dumpable sysctl and prctl(PR_SET_DUMPABLE) ABI), so this is a pure rename with no behavioral change. Subsequent commits relocate dumpability onto a per-task structure where the enum type will allow stronger type-checking on the new API. Reviewed-by: Jann Horn Reviewed-by: David Hildenbrand (arm) Link: https://patch.msgid.link/20260520-work-task_exec_state-v3-1-69f895bc1385@kernel.org Signed-off-by: Christian Brauner (Amutable) --- kernel/exit.c | 2 +- kernel/ptrace.c | 4 ++-- kernel/sys.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/exit.c b/kernel/exit.c index f50d73c272d6..507eda655e8d 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -571,7 +571,7 @@ static void exit_mm(void) */ smp_mb__after_spinlock(); local_irq_disable(); - current->user_dumpable = (get_dumpable(mm) == SUID_DUMP_USER); + current->user_dumpable = (get_dumpable(mm) == TASK_DUMPABLE_OWNER); current->mm = NULL; membarrier_update_current_mm(NULL); enter_lazy_tlb(mm, current); diff --git a/kernel/ptrace.c b/kernel/ptrace.c index 130043bfc209..07398c9c8fe3 100644 --- a/kernel/ptrace.c +++ b/kernel/ptrace.c @@ -53,7 +53,7 @@ int ptrace_access_vm(struct task_struct *tsk, unsigned long addr, if (!tsk->ptrace || (current != tsk->parent) || - ((get_dumpable(mm) != SUID_DUMP_USER) && + ((get_dumpable(mm) != TASK_DUMPABLE_OWNER) && !ptracer_capable(tsk, mm->user_ns))) { mmput(mm); return 0; @@ -276,7 +276,7 @@ static bool task_still_dumpable(struct task_struct *task, unsigned int mode) { struct mm_struct *mm = task->mm; if (mm) { - if (get_dumpable(mm) == SUID_DUMP_USER) + if (get_dumpable(mm) == TASK_DUMPABLE_OWNER) return true; return ptrace_has_cap(mm->user_ns, mode); } diff --git a/kernel/sys.c b/kernel/sys.c index 62e842055cc9..f1189f719db5 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -2568,7 +2568,7 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, error = get_dumpable(me->mm); break; case PR_SET_DUMPABLE: - if (arg2 != SUID_DUMP_DISABLE && arg2 != SUID_DUMP_USER) { + if (arg2 != TASK_DUMPABLE_OFF && arg2 != TASK_DUMPABLE_OWNER) { error = -EINVAL; break; } -- cgit v1.2.3 From b092062cb6d799fa3504c5975cbb1b05c8b67d6d Mon Sep 17 00:00:00 2001 From: "Christian Brauner (Amutable)" Date: Wed, 20 May 2026 23:48:53 +0200 Subject: exec: introduce struct task_exec_state Introduce struct task_exec_state, a per-task RCU-protected structure that holds the dumpable mode and the user namespace and stays attached to the task for its full lifetime. task_exec_state_rcu() is the canonical reader: asserts RCU or task_lock is held, WARNs on a NULL state, returns the rcu_dereference()'d pointer. Reviewed-by: Jann Horn Link: https://patch.msgid.link/20260520-work-task_exec_state-v3-2-69f895bc1385@kernel.org Signed-off-by: Christian Brauner (Amutable) --- kernel/Makefile | 2 +- kernel/exec_state.c | 116 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 kernel/exec_state.c (limited to 'kernel') diff --git a/kernel/Makefile b/kernel/Makefile index 6785982013dc..1e1a31673577 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -3,7 +3,7 @@ # Makefile for the linux kernel. # -obj-y = fork.o exec_domain.o panic.o \ +obj-y = fork.o exec_domain.o exec_state.o panic.o \ cpu.o exit.o softirq.o resource.o \ sysctl.o capability.o ptrace.o user.o \ signal.o sys.o umh.o workqueue.o pid.o task_work.o \ diff --git a/kernel/exec_state.c b/kernel/exec_state.c new file mode 100644 index 000000000000..1e0b59ff8fa8 --- /dev/null +++ b/kernel/exec_state.c @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Christian Brauner */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static struct kmem_cache *task_exec_state_cachep; + +static void __free_task_exec_state(struct rcu_head *rcu) +{ + struct task_exec_state *exec_state = container_of(rcu, struct task_exec_state, rcu); + + put_user_ns(exec_state->user_ns); + kmem_cache_free(task_exec_state_cachep, exec_state); +} + +void put_task_exec_state(struct task_exec_state *exec_state) +{ + if (exec_state && refcount_dec_and_test(&exec_state->count)) + call_rcu(&exec_state->rcu, __free_task_exec_state); +} + +struct task_exec_state *alloc_task_exec_state(struct user_namespace *user_ns) +{ + struct task_exec_state *exec_state; + + exec_state = kmem_cache_alloc(task_exec_state_cachep, GFP_KERNEL); + if (!exec_state) + return NULL; + refcount_set(&exec_state->count, 1); + exec_state->dumpable = TASK_DUMPABLE_OFF; + exec_state->user_ns = get_user_ns(user_ns); + return exec_state; +} + +struct task_exec_state *task_exec_state_rcu(const struct task_struct *tsk) +{ + struct task_exec_state *exec_state; + + exec_state = rcu_dereference_check(tsk->exec_state, + lockdep_is_held(&tsk->alloc_lock)); + WARN_ON_ONCE(!exec_state); + return exec_state; +} + +struct task_exec_state *task_exec_state_replace(struct task_struct *tsk, + struct task_exec_state *exec_state) +{ + /* + * Updates must hold both locks so callers needing a consistent + * snapshot of mm + dumpability are covered. + */ + lockdep_assert_held(&tsk->alloc_lock); + lockdep_assert_held_write(&tsk->signal->exec_update_lock); + + return rcu_replace_pointer(tsk->exec_state, exec_state, true); +} + +/* + * The non-CLONE_VM clone path: allocate a fresh exec_state and + * inherit the parent's dumpable mode and user_ns reference. CLONE_VM + * siblings refcount-share via copy_exec_state() in fork.c; only this + * path and execve() ever allocate. + */ +int task_exec_state_copy(struct task_struct *tsk) +{ + struct task_exec_state *src, *dst; + + src = rcu_dereference_protected(current->exec_state, true); + dst = alloc_task_exec_state(src->user_ns); + if (!dst) + return -ENOMEM; + dst->dumpable = READ_ONCE(src->dumpable); + rcu_assign_pointer(tsk->exec_state, dst); + return 0; +} + +/* + * Store TASK_DUMPABLE_* on current->exec_state. All callers + * (commit_creds, begin_new_exec, prctl(PR_SET_DUMPABLE)) act on the + * running task, which guarantees ->exec_state is allocated and cannot + * be replaced under us. + */ +void task_exec_state_set_dumpable(enum task_dumpable value) +{ + struct task_exec_state *exec_state; + + if (WARN_ON_ONCE(value > TASK_DUMPABLE_ROOT)) + value = TASK_DUMPABLE_OFF; + + exec_state = rcu_dereference_protected(current->exec_state, true); + WRITE_ONCE(exec_state->dumpable, value); +} + +enum task_dumpable task_exec_state_get_dumpable(struct task_struct *task) +{ + struct task_exec_state *exec_state; + + guard(rcu)(); + exec_state = rcu_dereference(task->exec_state); + return READ_ONCE(exec_state->dumpable); +} + +void __init exec_state_init(void) +{ + task_exec_state_cachep = kmem_cache_create("task_exec_state", + sizeof(struct task_exec_state), 0, + SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT, + NULL); +} -- cgit v1.2.3 From 92f829f6b2faad37a5582df73df01c93d4429821 Mon Sep 17 00:00:00 2001 From: "Christian Brauner (Amutable)" Date: Wed, 20 May 2026 23:48:54 +0200 Subject: ptrace: add ptracer_access_allowed() Add a helper that encapsulates all of the logic for checking ptrace access and remove open-coded versions in follow-up patches. Reviewed-by: Jann Horn Reviewed-by: David Hildenbrand (arm) Link: https://patch.msgid.link/20260520-work-task_exec_state-v3-3-69f895bc1385@kernel.org Signed-off-by: Christian Brauner (Amutable) --- kernel/ptrace.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) (limited to 'kernel') diff --git a/kernel/ptrace.c b/kernel/ptrace.c index 07398c9c8fe3..4be5e718db03 100644 --- a/kernel/ptrace.c +++ b/kernel/ptrace.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -36,6 +37,30 @@ #include /* for syscall_get_* */ +/** + * ptracer_access_allowed - may current peek/poke @tsk's address space? + * @tsk: tracee + * + * Per-access check used by ptrace_access_vm() and architecture-specific + * tag/register accessors. Returns true iff current is the registered + * ptracer of @tsk and either @tsk is owner-dumpable or current holds + * CAP_SYS_PTRACE in @tsk's exec namespace. Lighter than + * __ptrace_may_access(): it re-validates only dumpability and + * capability on every access, without re-running LSM hooks or + * cred_cap_issubset() checks performed at attach time. + */ +bool ptracer_access_allowed(struct task_struct *tsk) +{ + const struct task_exec_state *es; + + guard(rcu)(); + if (ptrace_parent(tsk) != current) + return false; + es = task_exec_state_rcu(tsk); + return READ_ONCE(es->dumpable) == TASK_DUMPABLE_OWNER || + ptracer_capable(tsk, es->user_ns); +} + /* * Access another process' address space via ptrace. * Source/target buffer must be kernel space, -- cgit v1.2.3 From 6b1c66c9cca99bf00386481c7b2aa7394c26d8b8 Mon Sep 17 00:00:00 2001 From: "Christian Brauner (Amutable)" Date: Wed, 20 May 2026 23:48:55 +0200 Subject: exec_state: relocate dumpable information The dumpable flag captured at execve() is consulted by __ptrace_may_access() and several /proc owner / visibility checks. It lives on mm_struct today, which exit_mm() clears from the task long before the task itself is reaped. exec_state is anchored to the execve() that established the current privilege domain. CLONE_VM siblings refcount-share the parent's exec_state via copy_exec_state(); non-CLONE_VM clones allocate a fresh exec_state inheriting the parent's dumpable mode and user_ns reference via task_exec_state_copy(). execve() allocates a fresh instance (via alloc_task_exec_state() in begin_new_exec()) and installs it under task_lock + exec_update_lock with task_exec_state_replace(). init_task uses a static instance. The dumpable mode now lives on task->exec_state->dumpable. task->mm->flags no longer carries dumpability; MMF_DUMPABLE_MASK is removed, but MMF_DUMPABLE_BITS is reserved so MMF_DUMP_FILTER_* bit positions remain stable for the /proc//coredump_filter ABI. The task->user_dumpable cache bit and its assignment in exit_mm() are removed; readers go through get_dumpable(task) directly. coredump_params gains a snapshot field cprm.dumpable, populated from get_dumpable(current) at vfs_coredump() entry, replacing the previous __get_dumpable(cprm->mm_flags) consumers in fs/coredump.c and fs/pidfs.c. The user namespace recorded at execve() is consulted by __ptrace_may_access() and by /proc/PID/* owner derivation. Move the captured user_ns onto task_exec_state, which stays attached to the task past exit_mm() and across exit_files(). bprm grows a user_ns field staged in bprm_mm_init() with the caller's user_ns, narrowed by would_dump() to the closest privileged ancestor, and consumed by exec_mmap() via alloc_task_exec_state(bprm->user_ns). free_bprm() releases the staging reference. mm_struct loses ->user_ns entirely. Initializers in init-mm, efi_mm, and the implicit one in mm_init()/dup_mm()/mm_alloc() are removed; __mmdrop() drops the matching put_user_ns(). The kthread_use_mm() WARN_ON_ONCE(!mm->user_ns) is no longer meaningful and goes too. Reviewed-by: Jann Horn Link: https://patch.msgid.link/20260520-work-task_exec_state-v3-4-69f895bc1385@kernel.org Signed-off-by: Christian Brauner (Amutable) --- kernel/cred.c | 3 ++- kernel/exec_state.c | 3 +++ kernel/exit.c | 1 - kernel/fork.c | 33 +++++++++++++++++++++++++++------ kernel/kthread.c | 1 - kernel/ptrace.c | 26 ++++++++------------------ kernel/sys.c | 4 ++-- 7 files changed, 42 insertions(+), 29 deletions(-) (limited to 'kernel') diff --git a/kernel/cred.c b/kernel/cred.c index 12a7b1ce5131..3df4e15bd67f 100644 --- a/kernel/cred.c +++ b/kernel/cred.c @@ -384,8 +384,9 @@ int commit_creds(struct cred *new) !uid_eq(old->fsuid, new->fsuid) || !gid_eq(old->fsgid, new->fsgid) || !cred_cap_issubset(old, new)) { + /* mm-less tasks share init_task's exec_state */ if (task->mm) - set_dumpable(task->mm, suid_dumpable); + task_exec_state_set_dumpable(suid_dumpable); task->pdeath_signal = 0; /* * If a task drops privileges and becomes nondumpable, diff --git a/kernel/exec_state.c b/kernel/exec_state.c index 1e0b59ff8fa8..6034f4b4808f 100644 --- a/kernel/exec_state.c +++ b/kernel/exec_state.c @@ -95,6 +95,9 @@ void task_exec_state_set_dumpable(enum task_dumpable value) value = TASK_DUMPABLE_OFF; exec_state = rcu_dereference_protected(current->exec_state, true); + /* mm-less tasks share init_task's exec_state; never mutate it */ + if (WARN_ON_ONCE(exec_state == &init_task_exec_state)) + return; WRITE_ONCE(exec_state->dumpable, value); } diff --git a/kernel/exit.c b/kernel/exit.c index 507eda655e8d..9a909993ab1d 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -571,7 +571,6 @@ static void exit_mm(void) */ smp_mb__after_spinlock(); local_irq_disable(); - current->user_dumpable = (get_dumpable(mm) == TASK_DUMPABLE_OWNER); current->mm = NULL; membarrier_update_current_mm(NULL); enter_lazy_tlb(mm, current); diff --git a/kernel/fork.c b/kernel/fork.c index 5f3fdfdb14c7..91545ed6463f 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -555,6 +556,7 @@ void free_task(struct task_struct *tsk) if (tsk->flags & PF_KTHREAD) free_kthread_struct(tsk); bpf_task_storage_free(tsk); + put_task_exec_state(rcu_access_pointer(tsk->exec_state)); free_task_struct(tsk); } EXPORT_SYMBOL(free_task); @@ -731,7 +733,6 @@ void __mmdrop(struct mm_struct *mm) destroy_context(mm); mmu_notifier_subscriptions_destroy(mm); check_mm(mm); - put_user_ns(mm->user_ns); mm_pasid_drop(mm); mm_destroy_cid(mm); percpu_counter_destroy_many(mm->rss_stat, NR_MM_COUNTERS); @@ -946,6 +947,8 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) tsk->seccomp.filter = NULL; #endif + RCU_INIT_POINTER(tsk->exec_state, NULL); + setup_thread_stack(tsk, orig); clear_user_return_notifier(tsk); clear_tsk_need_resched(tsk); @@ -1072,8 +1075,7 @@ static void mmap_init_lock(struct mm_struct *mm) #endif } -static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, - struct user_namespace *user_ns) +static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p) { mt_init_flags(&mm->mm_mt, MM_MT_FLAGS); mt_set_external_lock(&mm->mm_mt, &mm->mmap_lock); @@ -1132,7 +1134,6 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, NR_MM_COUNTERS)) goto fail_pcpu; - mm->user_ns = get_user_ns(user_ns); lru_gen_init_mm(mm); return mm; @@ -1163,7 +1164,7 @@ struct mm_struct *mm_alloc(void) return NULL; memset(mm, 0, sizeof(*mm)); - return mm_init(mm, current, current_user_ns()); + return mm_init(mm, current); } EXPORT_SYMBOL_IF_KUNIT(mm_alloc); @@ -1527,7 +1528,7 @@ static struct mm_struct *dup_mm(struct task_struct *tsk, memcpy(mm, oldmm, sizeof(*mm)); - if (!mm_init(mm, tsk, mm->user_ns)) + if (!mm_init(mm, tsk)) goto fail_nomem; uprobe_start_dup_mmap(); @@ -1593,6 +1594,22 @@ static int copy_mm(u64 clone_flags, struct task_struct *tsk) return 0; } +static int copy_exec_state(u64 clone_flags, struct task_struct *tsk) +{ + struct task_exec_state *exec_state; + + /* CLONE_VM siblings refcount-share the parent's exec_state. */ + if (clone_flags & CLONE_VM) { + exec_state = rcu_dereference_protected(current->exec_state, true); + refcount_inc(&exec_state->count); + rcu_assign_pointer(tsk->exec_state, exec_state); + return 0; + } + + /* Everyone else inherits a fresh copy. */ + return task_exec_state_copy(tsk); +} + static int copy_fs(u64 clone_flags, struct task_struct *tsk) { struct fs_struct *fs = current->fs; @@ -2090,6 +2107,9 @@ __latent_entropy struct task_struct *copy_process( p = dup_task_struct(current, node); if (!p) goto fork_out; + retval = copy_exec_state(clone_flags, p); + if (retval) + goto bad_fork_free; p->flags &= ~PF_KTHREAD; if (args->kthread) p->flags |= PF_KTHREAD; @@ -3098,6 +3118,7 @@ void __init proc_caches_init(void) sizeof(struct signal_struct), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL); + exec_state_init(); files_cachep = kmem_cache_create("files_cache", sizeof(struct files_struct), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, diff --git a/kernel/kthread.c b/kernel/kthread.c index 791210daf8b4..63beb59b7a3d 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -1619,7 +1619,6 @@ void kthread_use_mm(struct mm_struct *mm) WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD)); WARN_ON_ONCE(tsk->mm); - WARN_ON_ONCE(!mm->user_ns); /* * It is possible for mm to be the same as tsk->active_mm, but diff --git a/kernel/ptrace.c b/kernel/ptrace.c index 4be5e718db03..d041645d9d17 100644 --- a/kernel/ptrace.c +++ b/kernel/ptrace.c @@ -70,21 +70,14 @@ int ptrace_access_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, unsigned int gup_flags) { struct mm_struct *mm; - int ret; + int ret = 0; mm = get_task_mm(tsk); if (!mm) return 0; - if (!tsk->ptrace || - (current != tsk->parent) || - ((get_dumpable(mm) != TASK_DUMPABLE_OWNER) && - !ptracer_capable(tsk, mm->user_ns))) { - mmput(mm); - return 0; - } - - ret = access_remote_vm(mm, addr, buf, len, gup_flags); + if (ptracer_access_allowed(tsk)) + ret = access_remote_vm(mm, addr, buf, len, gup_flags); mmput(mm); return ret; @@ -299,16 +292,13 @@ static bool ptrace_has_cap(struct user_namespace *ns, unsigned int mode) static bool task_still_dumpable(struct task_struct *task, unsigned int mode) { - struct mm_struct *mm = task->mm; - if (mm) { - if (get_dumpable(mm) == TASK_DUMPABLE_OWNER) - return true; - return ptrace_has_cap(mm->user_ns, mode); - } + const struct task_exec_state *exec_state; - if (task->user_dumpable) + guard(rcu)(); + exec_state = task_exec_state_rcu(task); + if (READ_ONCE(exec_state->dumpable) == TASK_DUMPABLE_OWNER) return true; - return ptrace_has_cap(&init_user_ns, mode); + return ptrace_has_cap(exec_state->user_ns, mode); } /* Returns 0 on success, -errno on denial. */ diff --git a/kernel/sys.c b/kernel/sys.c index f1189f719db5..df69bd71de03 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -2565,14 +2565,14 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, error = put_user(me->pdeath_signal, (int __user *)arg2); break; case PR_GET_DUMPABLE: - error = get_dumpable(me->mm); + error = task_exec_state_get_dumpable(me); break; case PR_SET_DUMPABLE: if (arg2 != TASK_DUMPABLE_OFF && arg2 != TASK_DUMPABLE_OWNER) { error = -EINVAL; break; } - set_dumpable(me->mm, arg2); + task_exec_state_set_dumpable(arg2); break; case PR_SET_UNALIGN: -- cgit v1.2.3 From 2068d7715e947f0321bc676a44215d3983af4bbc Mon Sep 17 00:00:00 2001 From: l1rox3 Date: Wed, 20 May 2026 10:12:54 +0200 Subject: PM: hibernate: make LZ4 available for hibernation compression Without this, CRYPTO_LZ4 had to be manually enabled in the config to use LZ4 for hibernation compression. Add the select so it gets pulled in automatically when hibernation is enabled, just like CRYPTO_LZO already does. Tested-by: l1rox3 Signed-off-by: l1rox3 Link: https://patch.msgid.link/20260520081254.13493-1-l1rox3.developer@gmail.com Signed-off-by: Rafael J. Wysocki --- kernel/power/Kconfig | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/power/Kconfig b/kernel/power/Kconfig index 05337f437cca..530c897311d4 100644 --- a/kernel/power/Kconfig +++ b/kernel/power/Kconfig @@ -42,6 +42,7 @@ config HIBERNATION select CRC32 select CRYPTO select CRYPTO_LZO + select CRYPTO_LZ4 help Enable the suspend to disk (STD) functionality, which is usually called "hibernation" in user interfaces. STD checkpoints the -- cgit v1.2.3 From 25139c11693afed894db46d1a44e2b6e015b804d Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Fri, 22 May 2026 11:25:23 +0200 Subject: sched/fair: Fix RCU usage in NOHZ exit path on CPU offline Commit c9d93a73ce87 ("sched/fair: Drop redundant RCU read lock in NOHZ kick path") removed the rcu_read_lock()/unlock() pair from set_cpu_sd_state_busy() and set_cpu_sd_state_idle() on the assumption that all callers run in a safe context for rcu_dereference_all(): IRQs disabled or cpus_write_lock() held. That assumption is wrong for the CPU hotplug teardown path. When CPUs are taken offline, set_cpu_sd_state_busy() is invoked via: cpuhp/N kthread cpuhp_thread_fun() cpuhp_invoke_callback() sched_cpu_deactivate() nohz_balance_exit_idle() set_cpu_sd_state_busy() rcu_dereference_all(per_cpu(sd_llc, cpu)) The cpuhp kthread holds cpu_hotplug_lock (percpu-rwsem) but runs with preemption and IRQs enabled. As a result, lockdep correctly reports a suspicious RCU usage on CPU offline, e.g.: # echo 0 > /sys/devices/system/cpu/cpu1/online ============================= WARNING: suspicious RCU usage ----------------------------- kernel/sched/fair.c:12793 suspicious rcu_dereference_check() usage! ... 2 locks held by cpuhp/1/20: #0: (cpu_hotplug_lock){++++}-{0:0}, at: cpuhp_thread_fun+0x42/0x1ae #1: (cpuhp_state-down){+.+.}-{0:0}, at: cpuhp_thread_fun+0x72/0x1ae Call Trace: lockdep_rcu_suspicious nohz_balance_exit_idle sched_cpu_deactivate cpuhp_invoke_callback cpuhp_thread_fun smpboot_thread_fn Fix this by adding RCU read lock coverage to the one caller that lacks it: nohz_balance_exit_idle() in the CPU hotplug teardown. The other callers (nohz_balancer_kick() and nohz_balance_enter_idle()) genuinely run with IRQs disabled, so they remain unchanged. Fixes: c9d93a73ce87 ("sched/fair: Drop redundant RCU read lock in NOHZ kick path") Closes: https://lore.kernel.org/all/38fe0a1d-1a48-435a-910a-c278024d9ac9@samsung.com/ Reported-by: Marek Szyprowski Suggested-by: Peter Zijlstra (Intel) Signed-off-by: Andrea Righi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260522092523.2046095-1-arighi@nvidia.com --- kernel/sched/core.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 7fb3f5f2d48c..b3a416b1c251 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -8681,7 +8681,8 @@ int sched_cpu_deactivate(unsigned int cpu) * Remove CPU from nohz.idle_cpus_mask to prevent participating in * load balancing when not active */ - nohz_balance_exit_idle(rq); + scoped_guard (rcu) + nohz_balance_exit_idle(rq); set_cpu_active(cpu, false); -- cgit v1.2.3 From 333f6f0e11acc20d036f94f94709874f76d0b430 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Mon, 11 May 2026 13:31:05 +0200 Subject: sched/debug: Use char * instead of char (*)[] Some of the fancy AI robots are getting 'upset'. Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260511120627.065013766@infradead.org --- kernel/sched/debug.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index ed3a0d65da0c..af13a896858c 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -136,7 +136,7 @@ sched_feat_write(struct file *filp, const char __user *ubuf, if (cnt > 63) cnt = 63; - if (copy_from_user(&buf, ubuf, cnt)) + if (copy_from_user(buf, ubuf, cnt)) return -EFAULT; buf[cnt] = 0; @@ -263,7 +263,7 @@ static ssize_t sched_dynamic_write(struct file *filp, const char __user *ubuf, if (cnt > 15) cnt = 15; - if (copy_from_user(&buf, ubuf, cnt)) + if (copy_from_user(buf, ubuf, cnt)) return -EFAULT; buf[cnt] = 0; -- cgit v1.2.3 From 77557002234546a4fb46ebf517d6cb1d515535a9 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Mon, 11 May 2026 13:31:06 +0200 Subject: sched: Use {READ,WRITE}_ONCE() for preempt_dynamic_mode Robots figured out you can read and write this concurrently and got 'upset'. Gemini even noted sched_dynamic_show() can generate 'confusing' output if it observed different values during the printing. Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260511120627.176946327@infradead.org --- kernel/sched/core.c | 15 ++++++++------- kernel/sched/debug.c | 5 +++-- 2 files changed, 11 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index b3a416b1c251..83202f090f0c 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -7979,7 +7979,7 @@ static void __sched_dynamic_update(int mode) break; } - preempt_dynamic_mode = mode; + WRITE_ONCE(preempt_dynamic_mode, mode); } void sched_dynamic_update(int mode) @@ -8020,12 +8020,13 @@ static void __init preempt_dynamic_init(void) } } -# define PREEMPT_MODEL_ACCESSOR(mode) \ - bool preempt_model_##mode(void) \ - { \ - WARN_ON_ONCE(preempt_dynamic_mode == preempt_dynamic_undefined); \ - return preempt_dynamic_mode == preempt_dynamic_##mode; \ - } \ +# define PREEMPT_MODEL_ACCESSOR(mode) \ + bool preempt_model_##mode(void) \ + { \ + int mode = READ_ONCE(preempt_dynamic_mode); \ + WARN_ON_ONCE(mode == preempt_dynamic_undefined); \ + return mode == preempt_dynamic_##mode; \ + } \ EXPORT_SYMBOL_GPL(preempt_model_##mode) PREEMPT_MODEL_ACCESSOR(none); diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index af13a896858c..4cdf2f9c5d9e 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -281,6 +281,7 @@ static ssize_t sched_dynamic_write(struct file *filp, const char __user *ubuf, static int sched_dynamic_show(struct seq_file *m, void *v) { int i = (IS_ENABLED(CONFIG_PREEMPT_RT) || IS_ENABLED(CONFIG_ARCH_HAS_PREEMPT_LAZY)) * 2; + int mode = READ_ONCE(preempt_dynamic_mode); int j; /* Count entries in NULL terminated preempt_modes */ @@ -289,10 +290,10 @@ static int sched_dynamic_show(struct seq_file *m, void *v) j -= !IS_ENABLED(CONFIG_ARCH_HAS_PREEMPT_LAZY); for (; i < j; i++) { - if (preempt_dynamic_mode == i) + if (mode == i) seq_puts(m, "("); seq_puts(m, preempt_modes[i]); - if (preempt_dynamic_mode == i) + if (mode == i) seq_puts(m, ")"); seq_puts(m, " "); -- cgit v1.2.3 From e05777c44e53df8ab41d930510a384d65a34aafa Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Mon, 11 May 2026 13:31:07 +0200 Subject: sched/debug: Collapse subsequent CONFIG_SCHED_CLASS_EXT sections Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260511120627.281160085@infradead.org --- kernel/sched/debug.c | 92 +++++++++++++++++++++++++--------------------------- 1 file changed, 44 insertions(+), 48 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 4cdf2f9c5d9e..4809e1d23081 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -488,6 +488,8 @@ static const struct file_operations fair_server_runtime_fops = { .release = single_release, }; +static struct dentry *debugfs_sched; + #ifdef CONFIG_SCHED_CLASS_EXT static ssize_t sched_ext_server_runtime_write(struct file *filp, const char __user *ubuf, @@ -520,75 +522,92 @@ static const struct file_operations ext_server_runtime_fops = { .llseek = seq_lseek, .release = single_release, }; -#endif /* CONFIG_SCHED_CLASS_EXT */ static ssize_t -sched_fair_server_period_write(struct file *filp, const char __user *ubuf, - size_t cnt, loff_t *ppos) +sched_ext_server_period_write(struct file *filp, const char __user *ubuf, + size_t cnt, loff_t *ppos) { long cpu = (long) ((struct seq_file *) filp->private_data)->private; struct rq *rq = cpu_rq(cpu); return sched_server_write_common(filp, ubuf, cnt, ppos, DL_PERIOD, - &rq->fair_server); + &rq->ext_server); } -static int sched_fair_server_period_show(struct seq_file *m, void *v) +static int sched_ext_server_period_show(struct seq_file *m, void *v) { unsigned long cpu = (unsigned long) m->private; struct rq *rq = cpu_rq(cpu); - return sched_server_show_common(m, v, DL_PERIOD, &rq->fair_server); + return sched_server_show_common(m, v, DL_PERIOD, &rq->ext_server); } -static int sched_fair_server_period_open(struct inode *inode, struct file *filp) +static int sched_ext_server_period_open(struct inode *inode, struct file *filp) { - return single_open(filp, sched_fair_server_period_show, inode->i_private); + return single_open(filp, sched_ext_server_period_show, inode->i_private); } -static const struct file_operations fair_server_period_fops = { - .open = sched_fair_server_period_open, - .write = sched_fair_server_period_write, +static const struct file_operations ext_server_period_fops = { + .open = sched_ext_server_period_open, + .write = sched_ext_server_period_write, .read = seq_read, .llseek = seq_lseek, .release = single_release, }; -#ifdef CONFIG_SCHED_CLASS_EXT +static void debugfs_ext_server_init(void) +{ + struct dentry *d_ext; + unsigned long cpu; + + d_ext = debugfs_create_dir("ext_server", debugfs_sched); + if (!d_ext) + return; + + for_each_possible_cpu(cpu) { + struct dentry *d_cpu; + char buf[32]; + + snprintf(buf, sizeof(buf), "cpu%lu", cpu); + d_cpu = debugfs_create_dir(buf, d_ext); + + debugfs_create_file("runtime", 0644, d_cpu, (void *) cpu, &ext_server_runtime_fops); + debugfs_create_file("period", 0644, d_cpu, (void *) cpu, &ext_server_period_fops); + } +} +#endif /* CONFIG_SCHED_CLASS_EXT */ + static ssize_t -sched_ext_server_period_write(struct file *filp, const char __user *ubuf, - size_t cnt, loff_t *ppos) +sched_fair_server_period_write(struct file *filp, const char __user *ubuf, + size_t cnt, loff_t *ppos) { long cpu = (long) ((struct seq_file *) filp->private_data)->private; struct rq *rq = cpu_rq(cpu); return sched_server_write_common(filp, ubuf, cnt, ppos, DL_PERIOD, - &rq->ext_server); + &rq->fair_server); } -static int sched_ext_server_period_show(struct seq_file *m, void *v) +static int sched_fair_server_period_show(struct seq_file *m, void *v) { unsigned long cpu = (unsigned long) m->private; struct rq *rq = cpu_rq(cpu); - return sched_server_show_common(m, v, DL_PERIOD, &rq->ext_server); + return sched_server_show_common(m, v, DL_PERIOD, &rq->fair_server); } -static int sched_ext_server_period_open(struct inode *inode, struct file *filp) +static int sched_fair_server_period_open(struct inode *inode, struct file *filp) { - return single_open(filp, sched_ext_server_period_show, inode->i_private); + return single_open(filp, sched_fair_server_period_show, inode->i_private); } -static const struct file_operations ext_server_period_fops = { - .open = sched_ext_server_period_open, - .write = sched_ext_server_period_write, +static const struct file_operations fair_server_period_fops = { + .open = sched_fair_server_period_open, + .write = sched_fair_server_period_write, .read = seq_read, .llseek = seq_lseek, .release = single_release, }; -#endif /* CONFIG_SCHED_CLASS_EXT */ - -static struct dentry *debugfs_sched; static void debugfs_fair_server_init(void) { @@ -611,29 +630,6 @@ static void debugfs_fair_server_init(void) } } -#ifdef CONFIG_SCHED_CLASS_EXT -static void debugfs_ext_server_init(void) -{ - struct dentry *d_ext; - unsigned long cpu; - - d_ext = debugfs_create_dir("ext_server", debugfs_sched); - if (!d_ext) - return; - - for_each_possible_cpu(cpu) { - struct dentry *d_cpu; - char buf[32]; - - snprintf(buf, sizeof(buf), "cpu%lu", cpu); - d_cpu = debugfs_create_dir(buf, d_ext); - - debugfs_create_file("runtime", 0644, d_cpu, (void *) cpu, &ext_server_runtime_fops); - debugfs_create_file("period", 0644, d_cpu, (void *) cpu, &ext_server_period_fops); - } -} -#endif /* CONFIG_SCHED_CLASS_EXT */ - static __init int sched_init_debug(void) { struct dentry __maybe_unused *numa, *llc; -- cgit v1.2.3 From b3a2dfa8b42e5b97dd144aa59374f4e045725cac Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Mon, 11 May 2026 13:31:12 +0200 Subject: sched/fair: Add newidle balance to pick_task_fair() With commit 50653216e4ff ("sched: Add support to pick functions to take rf") removing the balance callback, the pick_task() callback is in charge of newidle balancing. This means pick_task_fair() should do so too. This hasn't been a problem in practise because pick_next_task_fair() is used. However, since we'll be removing that one shortly, make sure pick_next_task() is up to scratch. Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Vincent Guittot Link: https://patch.msgid.link/20260511120627.944705718@infradead.org --- kernel/sched/fair.c | 38 +++++++++++++++----------------------- 1 file changed, 15 insertions(+), 23 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 8e858ca6bcd0..5f48af700fd4 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -9863,16 +9863,18 @@ preempt: } static struct task_struct *pick_task_fair(struct rq *rq, struct rq_flags *rf) + __must_hold(__rq_lockp(rq)) { struct sched_entity *se; struct cfs_rq *cfs_rq; struct task_struct *p; bool throttled; + int new_tasks; again: cfs_rq = &rq->cfs; if (!cfs_rq->nr_queued) - return NULL; + goto idle; throttled = false; @@ -9893,6 +9895,14 @@ again: if (unlikely(throttled)) task_throttle_setup_work(p); return p; + +idle: + new_tasks = sched_balance_newidle(rq, rf); + if (new_tasks < 0) + return RETRY_TASK; + if (new_tasks > 0) + goto again; + return NULL; } static void __set_next_task_fair(struct rq *rq, struct task_struct *p, bool first); @@ -9904,12 +9914,12 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf { struct sched_entity *se; struct task_struct *p; - int new_tasks; -again: p = pick_task_fair(rq, rf); + if (unlikely(p == RETRY_TASK)) + return p; if (!p) - goto idle; + return p; se = &p->se; #ifdef CONFIG_FAIR_GROUP_SCHED @@ -9959,29 +9969,11 @@ simple: #endif /* CONFIG_FAIR_GROUP_SCHED */ put_prev_set_next_task(rq, prev, p); return p; - -idle: - if (rf) { - new_tasks = sched_balance_newidle(rq, rf); - - /* - * Because sched_balance_newidle() releases (and re-acquires) - * rq->lock, it is possible for any higher priority task to - * appear. In that case we must re-start the pick_next_entity() - * loop. - */ - if (new_tasks < 0) - return RETRY_TASK; - - if (new_tasks > 0) - goto again; - } - - return NULL; } static struct task_struct * fair_server_pick_task(struct sched_dl_entity *dl_se, struct rq_flags *rf) + __must_hold(__rq_lockp(dl_se->rq)) { return pick_task_fair(dl_se->rq, rf); } -- cgit v1.2.3 From 5ad278dd20bdf59714443894d7b3044471af97d0 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Mon, 11 May 2026 13:31:13 +0200 Subject: sched: Remove sched_class::pick_next_task() The reason for pick_next_task_fair() is the put/set optimization that avoids touching the common ancestors. However, it is possible to implement this in the put_prev_task() and set_next_task() calls as used in put_prev_set_next_task(). Notably, put_prev_set_next_task() is the only site that: - calls put_prev_task() with a .next argument; - calls set_next_task() with .first = true. This means that put_prev_task() can determine the common hierarchy and stop there, and then set_next_task() can terminate where put_prev_task stopped. Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Vincent Guittot Link: https://patch.msgid.link/20260511120628.057634261@infradead.org --- kernel/sched/core.c | 27 ++++------ kernel/sched/fair.c | 139 +++++++++++++++++---------------------------------- kernel/sched/sched.h | 14 +----- 3 files changed, 57 insertions(+), 123 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 83202f090f0c..3c8bfd697e2c 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6030,16 +6030,15 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) if (likely(!sched_class_above(prev->sched_class, &fair_sched_class) && rq->nr_running == rq->cfs.h_nr_queued)) { - p = pick_next_task_fair(rq, prev, rf); + p = pick_task_fair(rq, rf); if (unlikely(p == RETRY_TASK)) goto restart; /* Assume the next prioritized class is idle_sched_class */ - if (!p) { + if (!p) p = pick_task_idle(rq, rf); - put_prev_set_next_task(rq, prev, p); - } + put_prev_set_next_task(rq, prev, p); return p; } @@ -6047,20 +6046,12 @@ restart: prev_balance(rq, prev, rf); for_each_active_class(class) { - if (class->pick_next_task) { - p = class->pick_next_task(rq, prev, rf); - if (unlikely(p == RETRY_TASK)) - goto restart; - if (p) - return p; - } else { - p = class->pick_task(rq, rf); - if (unlikely(p == RETRY_TASK)) - goto restart; - if (p) { - put_prev_set_next_task(rq, prev, p); - return p; - } + p = class->pick_task(rq, rf); + if (unlikely(p == RETRY_TASK)) + goto restart; + if (p) { + put_prev_set_next_task(rq, prev, p); + return p; } } diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 5f48af700fd4..62a2dcb0d03e 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -9862,7 +9862,7 @@ preempt: resched_curr_lazy(rq); } -static struct task_struct *pick_task_fair(struct rq *rq, struct rq_flags *rf) +struct task_struct *pick_task_fair(struct rq *rq, struct rq_flags *rf) __must_hold(__rq_lockp(rq)) { struct sched_entity *se; @@ -9905,72 +9905,6 @@ idle: return NULL; } -static void __set_next_task_fair(struct rq *rq, struct task_struct *p, bool first); -static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first); - -struct task_struct * -pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) - __must_hold(__rq_lockp(rq)) -{ - struct sched_entity *se; - struct task_struct *p; - - p = pick_task_fair(rq, rf); - if (unlikely(p == RETRY_TASK)) - return p; - if (!p) - return p; - se = &p->se; - -#ifdef CONFIG_FAIR_GROUP_SCHED - if (prev->sched_class != &fair_sched_class) - goto simple; - - __put_prev_set_next_dl_server(rq, prev, p); - - /* - * Because of the set_next_buddy() in dequeue_task_fair() it is rather - * likely that a next task is from the same cgroup as the current. - * - * Therefore attempt to avoid putting and setting the entire cgroup - * hierarchy, only change the part that actually changes. - * - * Since we haven't yet done put_prev_entity and if the selected task - * is a different task than we started out with, try and touch the - * least amount of cfs_rqs. - */ - if (prev != p) { - struct sched_entity *pse = &prev->se; - struct cfs_rq *cfs_rq; - - while (!(cfs_rq = is_same_group(se, pse))) { - int se_depth = se->depth; - int pse_depth = pse->depth; - - if (se_depth <= pse_depth) { - put_prev_entity(cfs_rq_of(pse), pse); - pse = parent_entity(pse); - } - if (se_depth >= pse_depth) { - set_next_entity(cfs_rq_of(se), se, true); - se = parent_entity(se); - } - } - - put_prev_entity(cfs_rq, pse); - set_next_entity(cfs_rq, se, true); - - __set_next_task_fair(rq, p, true); - } - - return p; - -simple: -#endif /* CONFIG_FAIR_GROUP_SCHED */ - put_prev_set_next_task(rq, prev, p); - return p; -} - static struct task_struct * fair_server_pick_task(struct sched_dl_entity *dl_se, struct rq_flags *rf) __must_hold(__rq_lockp(dl_se->rq)) @@ -9994,10 +9928,33 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct t { struct sched_entity *se = &prev->se; struct cfs_rq *cfs_rq; + struct sched_entity *nse = NULL; - for_each_sched_entity(se) { +#ifdef CONFIG_FAIR_GROUP_SCHED + if (next && next->sched_class == &fair_sched_class) + nse = &next->se; +#endif + + while (se) { cfs_rq = cfs_rq_of(se); - put_prev_entity(cfs_rq, se); + if (!nse || cfs_rq->curr) + put_prev_entity(cfs_rq, se); +#ifdef CONFIG_FAIR_GROUP_SCHED + if (nse) { + if (is_same_group(se, nse)) + break; + + int d = nse->depth - se->depth; + if (d >= 0) { + /* nse has equal or greater depth, ascend */ + nse = parent_entity(nse); + /* if nse is the deeper, do not ascend se */ + if (d > 0) + continue; + } + } +#endif + se = parent_entity(se); } } @@ -15021,10 +14978,30 @@ static void switched_to_fair(struct rq *rq, struct task_struct *p) } } -static void __set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) +/* + * Account for a task changing its policy or group. + * + * This routine is mostly called to set cfs_rq->curr field when a task + * migrates between groups/classes. + */ +static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) { struct sched_entity *se = &p->se; + for_each_sched_entity(se) { + struct cfs_rq *cfs_rq = cfs_rq_of(se); + + if (IS_ENABLED(CONFIG_FAIR_GROUP_SCHED) && + first && cfs_rq->curr) + break; + + set_next_entity(cfs_rq, se, first); + /* ensure bandwidth has been allocated on our new cfs_rq */ + account_cfs_rq_runtime(cfs_rq, 0); + } + + se = &p->se; + if (task_on_rq_queued(p)) { /* * Move the next running task to the front of the list, so our @@ -15044,27 +15021,6 @@ static void __set_next_task_fair(struct rq *rq, struct task_struct *p, bool firs sched_fair_update_stop_tick(rq, p); } -/* - * Account for a task changing its policy or group. - * - * This routine is mostly called to set cfs_rq->curr field when a task - * migrates between groups/classes. - */ -static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) -{ - struct sched_entity *se = &p->se; - - for_each_sched_entity(se) { - struct cfs_rq *cfs_rq = cfs_rq_of(se); - - set_next_entity(cfs_rq, se, first); - /* ensure bandwidth has been allocated on our new cfs_rq */ - account_cfs_rq_runtime(cfs_rq, 0); - } - - __set_next_task_fair(rq, p, first); -} - void init_cfs_rq(struct cfs_rq *cfs_rq) { cfs_rq->tasks_timeline = RB_ROOT_CACHED; @@ -15376,7 +15332,6 @@ DEFINE_SCHED_CLASS(fair) = { .wakeup_preempt = wakeup_preempt_fair, .pick_task = pick_task_fair, - .pick_next_task = pick_next_task_fair, .put_prev_task = put_prev_task_fair, .set_next_task = set_next_task_fair, diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 8eb8f83db6b0..6b48bb3074fe 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2589,17 +2589,6 @@ struct sched_class { * schedule/pick_next_task: rq->lock */ struct task_struct *(*pick_task)(struct rq *rq, struct rq_flags *rf); - /* - * Optional! When implemented pick_next_task() should be equivalent to: - * - * next = pick_task(); - * if (next) { - * put_prev_task(prev); - * set_next_task_first(next); - * } - */ - struct task_struct *(*pick_next_task)(struct rq *rq, struct task_struct *prev, - struct rq_flags *rf); /* * sched_change: @@ -2823,8 +2812,7 @@ static inline bool sched_fair_runnable(struct rq *rq) return rq->cfs.nr_queued > 0; } -extern struct task_struct *pick_next_task_fair(struct rq *rq, struct task_struct *prev, - struct rq_flags *rf); +extern struct task_struct *pick_task_fair(struct rq *rq, struct rq_flags *rf); extern struct task_struct *pick_task_idle(struct rq *rq, struct rq_flags *rf); #define SCA_CHECK 0x01 -- cgit v1.2.3 From 3da1dbf936d2fb25ef9925550bb276861d803e57 Mon Sep 17 00:00:00 2001 From: Rosen Penev Date: Sun, 10 May 2026 14:39:48 -0700 Subject: PM: hibernate: Use flexible array for CRC uncompressed buffers The CRC uncompressed buffer pointer array has the same lifetime as struct crc_data, but it is currently allocated separately. That adds another allocation failure path and a matching cleanup branch without providing any extra flexibility. Store the pointer array as a flexible array member and allocate it together with the crc_data using kzalloc_flex(). The array remains zero-initialized, while the allocation and error handling become simpler. Assisted-by: Codex:GPT-5.5 Signed-off-by: Rosen Penev Link: https://patch.msgid.link/20260510213948.41750-1-rosenp@gmail.com Signed-off-by: Rafael J. Wysocki --- kernel/power/swap.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/power/swap.c b/kernel/power/swap.c index 2e64869bb5a0..b28233b8d00e 100644 --- a/kernel/power/swap.c +++ b/kernel/power/swap.c @@ -570,29 +570,23 @@ struct crc_data { wait_queue_head_t done; /* crc update done */ u32 *crc32; /* points to handle's crc32 */ size_t **unc_len; /* uncompressed lengths */ - unsigned char **unc; /* uncompressed data */ + unsigned char *unc[]; /* uncompressed data */ }; static struct crc_data *alloc_crc_data(int nr_threads) { struct crc_data *crc; - crc = kzalloc_obj(*crc); + crc = kzalloc_flex(*crc, unc, nr_threads); if (!crc) return NULL; - crc->unc = kcalloc(nr_threads, sizeof(*crc->unc), GFP_KERNEL); - if (!crc->unc) - goto err_free_crc; - crc->unc_len = kzalloc_objs(*crc->unc_len, nr_threads); if (!crc->unc_len) - goto err_free_unc; + goto err_free_crc; return crc; -err_free_unc: - kfree(crc->unc); err_free_crc: kfree(crc); return NULL; @@ -607,7 +601,6 @@ static void free_crc_data(struct crc_data *crc) kthread_stop(crc->thr); kfree(crc->unc_len); - kfree(crc->unc); kfree(crc); } -- cgit v1.2.3 From 95c33a64f203be444954a1e1d855a4820c4f0efa Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sun, 17 May 2026 22:01:38 +0200 Subject: genirq/proc: Avoid formatting zero counts in /proc/interrupts A large portion of interrupt count entries are zero. There is no point in formatting the zero value as it is way cheeper to just emit a constant string. Collect the number of consecutive zero counts and emit them in one go before a non-zero count and at the end of the line. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Reviewed-by: Dmitry Ilvokhin Reviewed-by: Radu Rendec Reviewed-by: Shrikanth Hegde Link: https://patch.msgid.link/20260517194931.034728540@kernel.org --- kernel/irq/proc.c | 43 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index dfa0b0723642..378b523a1b00 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c @@ -451,6 +451,43 @@ int __weak arch_show_interrupts(struct seq_file *p, int prec) # define ACTUAL_NR_IRQS irq_get_nr_irqs() #endif +/* Same as seq_put_decimal_ull_width(p, " ", cnt, 10) */ +#define ZSTR1 " 0" +#define ZSTR1_LEN (sizeof(ZSTR1) - 1) +#define ZSTR16 ZSTR1 ZSTR1 ZSTR1 ZSTR1 ZSTR1 ZSTR1 ZSTR1 ZSTR1 \ + ZSTR1 ZSTR1 ZSTR1 ZSTR1 ZSTR1 ZSTR1 ZSTR1 ZSTR1 +#define ZSTR256 ZSTR16 ZSTR16 ZSTR16 ZSTR16 ZSTR16 ZSTR16 ZSTR16 ZSTR16 \ + ZSTR16 ZSTR16 ZSTR16 ZSTR16 ZSTR16 ZSTR16 ZSTR16 ZSTR16 + +static inline void irq_proc_emit_zero_counts(struct seq_file *p, unsigned int zeros) +{ + if (!zeros) + return; + + for (unsigned int n = min(zeros, 256); n; zeros -= n, n = min(zeros, 256)) + seq_write(p, ZSTR256, n * ZSTR1_LEN); +} + +static inline unsigned int irq_proc_emit_count(struct seq_file *p, unsigned int cnt, + unsigned int zeros) +{ + if (!cnt) + return zeros + 1; + + irq_proc_emit_zero_counts(p, zeros); + seq_put_decimal_ull_width(p, " ", cnt, 10); + return 0; +} + +void irq_proc_emit_counts(struct seq_file *p, unsigned int __percpu *cnts) +{ + unsigned int cpu, zeros = 0; + + for_each_online_cpu(cpu) + zeros = irq_proc_emit_count(p, per_cpu(*cnts, cpu), zeros); + irq_proc_emit_zero_counts(p, zeros); +} + int show_interrupts(struct seq_file *p, void *v) { const unsigned int nr_irqs = irq_get_nr_irqs(); @@ -486,11 +523,7 @@ int show_interrupts(struct seq_file *p, void *v) return 0; seq_printf(p, "%*d:", prec, i); - for_each_online_cpu(j) { - unsigned int cnt = desc->kstat_irqs ? per_cpu(desc->kstat_irqs->cnt, j) : 0; - - seq_put_decimal_ull_width(p, " ", cnt, 10); - } + irq_proc_emit_counts(p, &desc->kstat_irqs->cnt); seq_putc(p, ' '); guard(raw_spinlock_irq)(&desc->lock); -- cgit v1.2.3 From 0179464391af9a01b911f441d2dda42ea253dfbd Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sun, 17 May 2026 22:01:43 +0200 Subject: genirq/proc: Utilize irq_desc::tot_count to avoid evaluation Interrupts which are not marked per CPU increment not only the per CPU statistics, but also the accumulation counter irq_desc::tot_count. Change the counter to type unsigned long so it does not produce sporadic zeros due to wrap arounds on 64-bit machines and do a quick check for non per CPU interrupts. If the counter is zero, then simply emit a full set of zero strings. That spares the evaluation of the per CPU counters completely for interrupts with zero events. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Reviewed-by: Dmitry Ilvokhin Reviewed-by: Radu Rendec Link: https://patch.msgid.link/20260517194931.115522199@kernel.org --- kernel/irq/proc.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index 378b523a1b00..5a1805b0b8a9 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c @@ -523,7 +523,16 @@ int show_interrupts(struct seq_file *p, void *v) return 0; seq_printf(p, "%*d:", prec, i); - irq_proc_emit_counts(p, &desc->kstat_irqs->cnt); + + /* + * Always output per CPU interrupts. Output device interrupts only when + * desc::tot_count is not zero. + */ + if (irq_settings_is_per_cpu(desc) || irq_settings_is_per_cpu_devid(desc) || + data_race(desc->tot_count)) + irq_proc_emit_counts(p, &desc->kstat_irqs->cnt); + else + irq_proc_emit_zero_counts(p, num_online_cpus()); seq_putc(p, ' '); guard(raw_spinlock_irq)(&desc->lock); -- cgit v1.2.3 From b99dc723b12ea587fc8b2e07bd8401433eec58d8 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sun, 17 May 2026 22:02:09 +0200 Subject: genirq: Expose nr_irqs in core code ... to avoid function calls in the core code to retrieve the maximum number of interrupts. Rename it to 'total_nr_irqs' as 'nr_irqs' is too generic and fix up the 'nr_irqs' reference in the related GDB script as well. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Reviewed-by: Dmitry Ilvokhin Reviewed-by: Radu Rendec Reviewed-by: Shrikanth Hegde Link: https://patch.msgid.link/20260517194931.522168332@kernel.org --- kernel/irq/internals.h | 1 + kernel/irq/irqdesc.c | 28 ++++++++++++++-------------- kernel/irq/proc.c | 4 ++-- 3 files changed, 17 insertions(+), 16 deletions(-) (limited to 'kernel') diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h index 9412e57056f5..db359acdf27f 100644 --- a/kernel/irq/internals.h +++ b/kernel/irq/internals.h @@ -21,6 +21,7 @@ extern bool noirqdebug; extern int irq_poll_cpu; +extern unsigned int total_nr_irqs; extern struct irqaction chained_action; diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c index 7173b8b634f2..a7ce1e8165a6 100644 --- a/kernel/irq/irqdesc.c +++ b/kernel/irq/irqdesc.c @@ -140,14 +140,14 @@ static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node, desc_smp_init(desc, node, affinity); } -static unsigned int nr_irqs = NR_IRQS; +unsigned int total_nr_irqs __read_mostly = NR_IRQS; /** * irq_get_nr_irqs() - Number of interrupts supported by the system. */ unsigned int irq_get_nr_irqs(void) { - return nr_irqs; + return total_nr_irqs; } EXPORT_SYMBOL_GPL(irq_get_nr_irqs); @@ -159,7 +159,7 @@ EXPORT_SYMBOL_GPL(irq_get_nr_irqs); */ unsigned int irq_set_nr_irqs(unsigned int nr) { - nr_irqs = nr; + total_nr_irqs = nr; return nr; } @@ -187,9 +187,9 @@ static unsigned int irq_find_at_or_after(unsigned int offset) struct irq_desc *desc; guard(rcu)(); - desc = mt_find(&sparse_irqs, &index, nr_irqs); + desc = mt_find(&sparse_irqs, &index, total_nr_irqs); - return desc ? irq_desc_get_irq(desc) : nr_irqs; + return desc ? irq_desc_get_irq(desc) : total_nr_irqs; } static void irq_insert_desc(unsigned int irq, struct irq_desc *desc) @@ -543,7 +543,7 @@ static bool irq_expand_nr_irqs(unsigned int nr) { if (nr > MAX_SPARSE_IRQS) return false; - nr_irqs = nr; + total_nr_irqs = nr; return true; } @@ -557,16 +557,16 @@ int __init early_irq_init(void) /* Let arch update nr_irqs and return the nr of preallocated irqs */ initcnt = arch_probe_nr_irqs(); printk(KERN_INFO "NR_IRQS: %d, nr_irqs: %d, preallocated irqs: %d\n", - NR_IRQS, nr_irqs, initcnt); + NR_IRQS, total_nr_irqs, initcnt); - if (WARN_ON(nr_irqs > MAX_SPARSE_IRQS)) - nr_irqs = MAX_SPARSE_IRQS; + if (WARN_ON(total_nr_irqs > MAX_SPARSE_IRQS)) + total_nr_irqs = MAX_SPARSE_IRQS; if (WARN_ON(initcnt > MAX_SPARSE_IRQS)) initcnt = MAX_SPARSE_IRQS; - if (initcnt > nr_irqs) - nr_irqs = initcnt; + if (initcnt > total_nr_irqs) + total_nr_irqs = initcnt; for (i = 0; i < initcnt; i++) { desc = alloc_desc(i, node, 0, NULL, NULL); @@ -862,7 +862,7 @@ void irq_free_descs(unsigned int from, unsigned int cnt) { int i; - if (from >= nr_irqs || (from + cnt) > nr_irqs) + if (from >= total_nr_irqs || (from + cnt) > total_nr_irqs) return; guard(mutex)(&sparse_irq_lock); @@ -911,7 +911,7 @@ int __ref __irq_alloc_descs(int irq, unsigned int from, unsigned int cnt, int no if (irq >=0 && start != irq) return -EEXIST; - if (start + cnt > nr_irqs) { + if (start + cnt > total_nr_irqs) { if (!irq_expand_nr_irqs(start + cnt)) return -ENOMEM; } @@ -923,7 +923,7 @@ EXPORT_SYMBOL_GPL(__irq_alloc_descs); * irq_get_next_irq - get next allocated irq number * @offset: where to start the search * - * Returns next irq number after offset or nr_irqs if none is found. + * Returns next irq number after offset or total_nr_irqs if none is found. */ unsigned int irq_get_next_irq(unsigned int offset) { diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index 5a1805b0b8a9..d5b812680f5c 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c @@ -448,7 +448,7 @@ int __weak arch_show_interrupts(struct seq_file *p, int prec) } #ifndef ACTUAL_NR_IRQS -# define ACTUAL_NR_IRQS irq_get_nr_irqs() +# define ACTUAL_NR_IRQS total_nr_irqs #endif /* Same as seq_put_decimal_ull_width(p, " ", cnt, 10) */ @@ -490,7 +490,7 @@ void irq_proc_emit_counts(struct seq_file *p, unsigned int __percpu *cnts) int show_interrupts(struct seq_file *p, void *v) { - const unsigned int nr_irqs = irq_get_nr_irqs(); + const unsigned int nr_irqs = total_nr_irqs; static int prec; int i = *(loff_t *) v, j; -- cgit v1.2.3 From 3ba92f6a28203e30d0b2c7d75b59f48d5ff9fbcc Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sun, 17 May 2026 22:02:14 +0200 Subject: genirq/manage: Make NMI cleanup RT safe Eventually blocking functions cannot be invoked with interrupts disabled and a raw spin lock held. Restructure the code so this happens outside of the descriptor lock held region. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Link: https://patch.msgid.link/20260517194931.601972758@kernel.org --- kernel/irq/manage.c | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) (limited to 'kernel') diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index 2e8072437826..8863b2d4b8ca 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -2026,24 +2026,30 @@ const void *free_irq(unsigned int irq, void *dev_id) } EXPORT_SYMBOL(free_irq); -/* This function must be called with desc->lock held */ static const void *__cleanup_nmi(unsigned int irq, struct irq_desc *desc) { + struct irqaction *action = NULL; const char *devname = NULL; - desc->istate &= ~IRQS_NMI; + scoped_guard(raw_spinlock_irqsave, &desc->lock) { + irq_nmi_teardown(desc); - if (!WARN_ON(desc->action == NULL)) { - irq_pm_remove_action(desc, desc->action); - devname = desc->action->name; - unregister_handler_proc(irq, desc->action); + desc->istate &= ~IRQS_NMI; - kfree(desc->action); + if (!WARN_ON(desc->action == NULL)) { + action = desc->action; + irq_pm_remove_action(desc, action); + devname = action->name; + } desc->action = NULL; + + irq_settings_clr_disable_unlazy(desc); + irq_shutdown_and_deactivate(desc); } - irq_settings_clr_disable_unlazy(desc); - irq_shutdown_and_deactivate(desc); + if (action) + unregister_handler_proc(irq, action); + kfree(action); irq_release_resources(desc); @@ -2067,8 +2073,6 @@ const void *free_nmi(unsigned int irq, void *dev_id) if (WARN_ON(desc->depth == 0)) disable_nmi_nosync(irq); - guard(raw_spinlock_irqsave)(&desc->lock); - irq_nmi_teardown(desc); return __cleanup_nmi(irq, desc); } @@ -2318,13 +2322,14 @@ int request_nmi(unsigned int irq, irq_handler_t handler, /* Setup NMI state */ desc->istate |= IRQS_NMI; retval = irq_nmi_setup(desc); - if (retval) { - __cleanup_nmi(irq, desc); - return -EINVAL; - } - return 0; } + if (retval) { + __cleanup_nmi(irq, desc); + return -EINVAL; + } + return 0; + err_irq_setup: irq_chip_pm_put(&desc->irq_data); err_out: -- cgit v1.2.3 From 4892e5e71ec9c942293cea7fd26e0c76179211ed Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sun, 17 May 2026 22:02:19 +0200 Subject: genirq: Cache the condition for /proc/interrupts exposure show_interrupts() evaluates a boatload of conditions to establish whether it should expose an interrupt in /proc/interrupts or not. That can be simplified by caching the condition in an internal status flag, which is updated when one of the relevant inputs changes. The irq_desc::kstat_irq check is dropped because visible interrupt descriptors always have a valid pointer. As a result the number of instructions and branches for reading /proc/interrupts is reduced significantly. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Reviewed-by: Dmitry Ilvokhin Reviewed-by: Radu Rendec Link: https://patch.msgid.link/20260517194931.680943749@kernel.org --- kernel/irq/chip.c | 2 ++ kernel/irq/internals.h | 2 ++ kernel/irq/manage.c | 8 +++++++- kernel/irq/proc.c | 15 +++++++++++---- kernel/irq/settings.h | 13 +++++++++++++ 5 files changed, 35 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c index 6c9b1dc4e7d4..2d283d1390dc 100644 --- a/kernel/irq/chip.c +++ b/kernel/irq/chip.c @@ -1007,6 +1007,7 @@ __irq_do_set_handler(struct irq_desc *desc, irq_flow_handler_t handle, WARN_ON(irq_chip_pm_get(irq_desc_get_irq_data(desc))); irq_activate_and_startup(desc, IRQ_RESEND); } + irq_proc_update_valid(desc); } void __irq_set_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained, @@ -1067,6 +1068,7 @@ void irq_modify_status(unsigned int irq, unsigned long clr, unsigned long set) trigger = tmp; irqd_set(&desc->irq_data, trigger); + irq_proc_update_valid(desc); } } EXPORT_SYMBOL_GPL(irq_modify_status); diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h index db359acdf27f..e5f3d310a825 100644 --- a/kernel/irq/internals.h +++ b/kernel/irq/internals.h @@ -123,6 +123,7 @@ extern void register_irq_proc(unsigned int irq, struct irq_desc *desc); extern void unregister_irq_proc(unsigned int irq, struct irq_desc *desc); extern void register_handler_proc(unsigned int irq, struct irqaction *action); extern void unregister_handler_proc(unsigned int irq, struct irqaction *action); +void irq_proc_update_valid(struct irq_desc *desc); #else static inline void register_irq_proc(unsigned int irq, struct irq_desc *desc) { } static inline void unregister_irq_proc(unsigned int irq, struct irq_desc *desc) { } @@ -130,6 +131,7 @@ static inline void register_handler_proc(unsigned int irq, struct irqaction *action) { } static inline void unregister_handler_proc(unsigned int irq, struct irqaction *action) { } +static inline void irq_proc_update_valid(struct irq_desc *desc) { } #endif extern bool irq_can_set_affinity_usr(unsigned int irq); diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index 8863b2d4b8ca..7eb07e3bdb4c 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -1802,6 +1802,7 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new) __enable_irq(desc); } + irq_proc_update_valid(desc); raw_spin_unlock_irqrestore(&desc->lock, flags); chip_bus_sync_unlock(desc); mutex_unlock(&desc->request_mutex); @@ -1906,6 +1907,7 @@ static struct irqaction *__free_irq(struct irq_desc *desc, void *dev_id) desc->affinity_hint = NULL; #endif + irq_proc_update_valid(desc); raw_spin_unlock_irqrestore(&desc->lock, flags); /* * Drop bus_lock here so the changes which were done in the chip @@ -2047,6 +2049,8 @@ static const void *__cleanup_nmi(unsigned int irq, struct irq_desc *desc) irq_shutdown_and_deactivate(desc); } + irq_proc_update_valid(desc); + if (action) unregister_handler_proc(irq, action); kfree(action); @@ -2433,8 +2437,10 @@ static struct irqaction *__free_percpu_irq(unsigned int irq, void __percpu *dev_ *action_ptr = action->next; /* Demote from NMI if we killed the last action */ - if (!desc->action) + if (!desc->action) { desc->istate &= ~IRQS_NMI; + irq_proc_update_valid(desc); + } } unregister_handler_proc(irq, action); diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index d5b812680f5c..cc9c7c907dd2 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c @@ -440,6 +440,16 @@ void init_irq_proc(void) register_irq_proc(irq, desc); } +void irq_proc_update_valid(struct irq_desc *desc) +{ + u32 set = _IRQ_PROC_VALID; + + if (irq_settings_is_hidden(desc) || irq_desc_is_chained(desc) || !desc->action) + set = 0; + + irq_settings_update_proc_valid(desc, set); +} + #ifdef CONFIG_GENERIC_IRQ_SHOW int __weak arch_show_interrupts(struct seq_file *p, int prec) @@ -516,10 +526,7 @@ int show_interrupts(struct seq_file *p, void *v) guard(rcu)(); desc = irq_to_desc(i); - if (!desc || irq_settings_is_hidden(desc)) - return 0; - - if (!desc->action || irq_desc_is_chained(desc) || !desc->kstat_irqs) + if (!desc || !irq_settings_proc_valid(desc)) return 0; seq_printf(p, "%*d:", prec, i); diff --git a/kernel/irq/settings.h b/kernel/irq/settings.h index 00b3bd127692..0a0c027a5d34 100644 --- a/kernel/irq/settings.h +++ b/kernel/irq/settings.h @@ -18,6 +18,7 @@ enum { _IRQ_DISABLE_UNLAZY = IRQ_DISABLE_UNLAZY, _IRQ_HIDDEN = IRQ_HIDDEN, _IRQ_NO_DEBUG = IRQ_NO_DEBUG, + _IRQ_PROC_VALID = IRQ_RESERVED, _IRQF_MODIFY_MASK = IRQF_MODIFY_MASK, }; @@ -34,6 +35,7 @@ enum { #define IRQ_DISABLE_UNLAZY GOT_YOU_MORON #define IRQ_HIDDEN GOT_YOU_MORON #define IRQ_NO_DEBUG GOT_YOU_MORON +#define IRQ_RESERVED GOT_YOU_MORON #undef IRQF_MODIFY_MASK #define IRQF_MODIFY_MASK GOT_YOU_MORON @@ -180,3 +182,14 @@ static inline bool irq_settings_no_debug(struct irq_desc *desc) { return desc->status_use_accessors & _IRQ_NO_DEBUG; } + +static inline bool irq_settings_proc_valid(struct irq_desc *desc) +{ + return desc->status_use_accessors & _IRQ_PROC_VALID; +} + +static inline void irq_settings_update_proc_valid(struct irq_desc *desc, u32 set) +{ + desc->status_use_accessors &= ~_IRQ_PROC_VALID; + desc->status_use_accessors |= (set & _IRQ_PROC_VALID); +} -- cgit v1.2.3 From 2d62735f1d4a2832af367c9e3de04bfb280a945c Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sun, 17 May 2026 22:02:24 +0200 Subject: genirq: Calculate precision only when required Calculating the precision of the interrupt number column on every initial show_interrupt() invocation is a pointless exercise as the underlying maximum number of interrupts rarely changes. Calculate it only when that number is modified and let show_interrupts() use the cached value. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Reviewed-by: Dmitry Ilvokhin Reviewed-by: Radu Rendec Link: https://patch.msgid.link/20260517194931.760664517@kernel.org --- kernel/irq/internals.h | 6 ++++++ kernel/irq/irqdesc.c | 10 ++++++---- kernel/irq/proc.c | 33 +++++++++++++++++++++++---------- 3 files changed, 35 insertions(+), 14 deletions(-) (limited to 'kernel') diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h index e5f3d310a825..be702dd64422 100644 --- a/kernel/irq/internals.h +++ b/kernel/irq/internals.h @@ -134,6 +134,12 @@ static inline void unregister_handler_proc(unsigned int irq, static inline void irq_proc_update_valid(struct irq_desc *desc) { } #endif +#if defined(CONFIG_PROC_FS) && defined(CONFIG_GENERIC_IRQ_SHOW) +void irq_proc_calc_prec(void); +#else +static inline void irq_proc_calc_prec(void) { } +#endif + extern bool irq_can_set_affinity_usr(unsigned int irq); extern int irq_do_set_affinity(struct irq_data *data, diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c index a7ce1e8165a6..90595eefb477 100644 --- a/kernel/irq/irqdesc.c +++ b/kernel/irq/irqdesc.c @@ -157,13 +157,12 @@ EXPORT_SYMBOL_GPL(irq_get_nr_irqs); * * Return: @nr. */ -unsigned int irq_set_nr_irqs(unsigned int nr) +unsigned int __init irq_set_nr_irqs(unsigned int nr) { total_nr_irqs = nr; - + irq_proc_calc_prec(); return nr; } -EXPORT_SYMBOL_GPL(irq_set_nr_irqs); static DEFINE_MUTEX(sparse_irq_lock); static struct maple_tree sparse_irqs = MTREE_INIT_EXT(sparse_irqs, @@ -544,6 +543,7 @@ static bool irq_expand_nr_irqs(unsigned int nr) if (nr > MAX_SPARSE_IRQS) return false; total_nr_irqs = nr; + irq_proc_calc_prec(); return true; } @@ -572,6 +572,7 @@ int __init early_irq_init(void) desc = alloc_desc(i, node, 0, NULL, NULL); irq_insert_desc(i, desc); } + irq_proc_calc_prec(); return arch_early_irq_init(); } @@ -592,7 +593,7 @@ int __init early_irq_init(void) init_irq_default_affinity(); - printk(KERN_INFO "NR_IRQS: %d\n", NR_IRQS); + pr_info("NR_IRQS: %d\n", NR_IRQS); count = ARRAY_SIZE(irq_desc); @@ -602,6 +603,7 @@ int __init early_irq_init(void) goto __free_desc_res; } + irq_proc_calc_prec(); return arch_early_irq_init(); __free_desc_res: diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index cc9c7c907dd2..c635fb62c783 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c @@ -457,10 +457,25 @@ int __weak arch_show_interrupts(struct seq_file *p, int prec) return 0; } +static struct irq_proc_constraints { + unsigned int num_prec; +} irq_proc_constraints __read_mostly = { + .num_prec = 3, +}; + #ifndef ACTUAL_NR_IRQS # define ACTUAL_NR_IRQS total_nr_irqs #endif +void irq_proc_calc_prec(void) +{ + unsigned int prec, n; + + for (prec = 3, n = 1000; prec < 10 && n <= total_nr_irqs; ++prec) + n *= 10; + WRITE_ONCE(irq_proc_constraints.num_prec, prec); +} + /* Same as seq_put_decimal_ull_width(p, " ", cnt, 10) */ #define ZSTR1 " 0" #define ZSTR1_LEN (sizeof(ZSTR1) - 1) @@ -500,9 +515,7 @@ void irq_proc_emit_counts(struct seq_file *p, unsigned int __percpu *cnts) int show_interrupts(struct seq_file *p, void *v) { - const unsigned int nr_irqs = total_nr_irqs; - static int prec; - + unsigned int prec = READ_ONCE(irq_proc_constraints.num_prec); int i = *(loff_t *) v, j; struct irqaction *action; struct irq_desc *desc; @@ -515,9 +528,6 @@ int show_interrupts(struct seq_file *p, void *v) /* print header and calculate the width of the first column */ if (i == 0) { - for (prec = 3, j = 1000; prec < 10 && j <= nr_irqs; ++prec) - j *= 10; - seq_printf(p, "%*s", prec + 8, ""); for_each_online_cpu(j) seq_printf(p, "CPU%-8d", j); @@ -553,13 +563,16 @@ int show_interrupts(struct seq_file *p, void *v) } else { seq_printf(p, "%8s", "None"); } + + seq_putc(p, ' '); if (desc->irq_data.domain) - seq_printf(p, " %*lu", prec, desc->irq_data.hwirq); + seq_put_decimal_ull_width(p, "", desc->irq_data.hwirq, prec); else seq_printf(p, " %*s", prec, ""); -#ifdef CONFIG_GENERIC_IRQ_SHOW_LEVEL - seq_printf(p, " %-8s", irqd_is_level_type(&desc->irq_data) ? "Level" : "Edge"); -#endif + + if (IS_ENABLED(CONFIG_GENERIC_IRQ_SHOW_LEVEL)) + seq_printf(p, " %-8s", irqd_is_level_type(&desc->irq_data) ? "Level" : "Edge"); + if (desc->name) seq_printf(p, "-%-8s", desc->name); -- cgit v1.2.3 From 34594da7650d3ea67f96c0f4034ff6b2453f67c3 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sun, 17 May 2026 22:02:29 +0200 Subject: genirq/proc: Increase default interrupt number precision to four Quite some architectures have four character wide acronyms for architecture specific interrupts like IPI, NMI, etc. The default precision of printing the Linux device interrupt numbers is three, which causes quite some code to play games with adding or omitting space after the acronym and the colon in order to keep the per CPU numbers properly aligned. Increase the default number precision to four in the core code and get rid of the space games all over the place. At the same time align all architecture specific descriptor texts left so that they show up in the same column as the interrupt chip names, which makes the output more uniform accross architectures. Fix up the GDB script to this new scheme as well. Signed-off-by: Thomas Gleixner Link: https://patch.msgid.link/20260517194931.839482411@kernel.org --- kernel/irq/proc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index c635fb62c783..1cb47a8eab49 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c @@ -460,7 +460,7 @@ int __weak arch_show_interrupts(struct seq_file *p, int prec) static struct irq_proc_constraints { unsigned int num_prec; } irq_proc_constraints __read_mostly = { - .num_prec = 3, + .num_prec = 4, }; #ifndef ACTUAL_NR_IRQS @@ -471,7 +471,7 @@ void irq_proc_calc_prec(void) { unsigned int prec, n; - for (prec = 3, n = 1000; prec < 10 && n <= total_nr_irqs; ++prec) + for (prec = 4, n = 10000; prec < 10 && n <= total_nr_irqs; ++prec) n *= 10; WRITE_ONCE(irq_proc_constraints.num_prec, prec); } -- cgit v1.2.3 From 1d9c4745bfb6773712751f5068c23ebad9cd30a0 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sun, 17 May 2026 22:02:34 +0200 Subject: genirq: Add rcuref count to struct irq_desc Prepare for a smarter iterator for /proc/interrupts so that the next interrupt descriptor can be cached after lookup. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Reviewed-by: Dmitry Ilvokhin Link: https://patch.msgid.link/20260517194931.917415190@kernel.org --- kernel/irq/internals.h | 17 ++++++++++++++++- kernel/irq/irqdesc.c | 21 +++++++++++++-------- 2 files changed, 29 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h index be702dd64422..bacbf4020242 100644 --- a/kernel/irq/internals.h +++ b/kernel/irq/internals.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #ifdef CONFIG_SPARSE_IRQ @@ -101,9 +102,23 @@ extern void unmask_irq(struct irq_desc *desc); extern void unmask_threaded_irq(struct irq_desc *desc); #ifdef CONFIG_SPARSE_IRQ -static inline void irq_mark_irq(unsigned int irq) { } +static __always_inline void irq_mark_irq(unsigned int irq) { } +void irq_desc_free_rcu(struct irq_desc *desc); + +static __always_inline bool irq_desc_get_ref(struct irq_desc *desc) +{ + return rcuref_get(&desc->refcnt); +} + +static __always_inline void irq_desc_put_ref(struct irq_desc *desc) +{ + if (rcuref_put(&desc->refcnt)) + irq_desc_free_rcu(desc); +} #else extern void irq_mark_irq(unsigned int irq); +static __always_inline bool irq_desc_get_ref(struct irq_desc *desc) { return true; } +static __always_inline void irq_desc_put_ref(struct irq_desc *desc) { } #endif irqreturn_t __handle_irq_event_percpu(struct irq_desc *desc); diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c index 90595eefb477..23f21763bfed 100644 --- a/kernel/irq/irqdesc.c +++ b/kernel/irq/irqdesc.c @@ -137,6 +137,7 @@ static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node, desc->tot_count = 0; desc->name = NULL; desc->owner = owner; + rcuref_init(&desc->refcnt, 1); desc_smp_init(desc, node, affinity); } @@ -465,6 +466,17 @@ static void delayed_free_desc(struct rcu_head *rhp) kobject_put(&desc->kobj); } +void irq_desc_free_rcu(struct irq_desc *desc) +{ + /* + * We free the descriptor, masks and stat fields via RCU. That + * allows demultiplex interrupts to do rcu based management of + * the child interrupts. + * This also allows us to use rcu in kstat_irqs_usr(). + */ + call_rcu(&desc->rcu, delayed_free_desc); +} + static void free_desc(unsigned int irq) { struct irq_desc *desc = irq_to_desc(irq); @@ -483,14 +495,7 @@ static void free_desc(unsigned int irq) */ irq_sysfs_del(desc); delete_irq_desc(irq); - - /* - * We free the descriptor, masks and stat fields via RCU. That - * allows demultiplex interrupts to do rcu based management of - * the child interrupts. - * This also allows us to use rcu in kstat_irqs_usr(). - */ - call_rcu(&desc->rcu, delayed_free_desc); + irq_desc_put_ref(desc); } static int alloc_descs(unsigned int start, unsigned int cnt, int node, -- cgit v1.2.3 From 7603e0575d8a92318bd4695917fce7ec2c5825a1 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sun, 17 May 2026 22:02:39 +0200 Subject: genirq: Expose irq_find_desc_at_or_after() in core code ... in preparation for a smarter iterator for /proc/interrupts. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Reviewed-by: Dmitry Ilvokhin Link: https://patch.msgid.link/20260517194932.005787611@kernel.org --- kernel/irq/internals.h | 2 ++ kernel/irq/irqdesc.c | 15 ++++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h index bacbf4020242..37eec0337867 100644 --- a/kernel/irq/internals.h +++ b/kernel/irq/internals.h @@ -155,6 +155,8 @@ void irq_proc_calc_prec(void); static inline void irq_proc_calc_prec(void) { } #endif +struct irq_desc *irq_find_desc_at_or_after(unsigned int offset); + extern bool irq_can_set_affinity_usr(unsigned int irq); extern int irq_do_set_affinity(struct irq_data *data, diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c index 23f21763bfed..80ef4e27dcf4 100644 --- a/kernel/irq/irqdesc.c +++ b/kernel/irq/irqdesc.c @@ -181,15 +181,12 @@ static int irq_find_free_area(unsigned int from, unsigned int cnt) return mas.index; } -static unsigned int irq_find_at_or_after(unsigned int offset) +struct irq_desc *irq_find_desc_at_or_after(unsigned int offset) { unsigned long index = offset; - struct irq_desc *desc; - - guard(rcu)(); - desc = mt_find(&sparse_irqs, &index, total_nr_irqs); - return desc ? irq_desc_get_irq(desc) : total_nr_irqs; + lockdep_assert_in_rcu_read_lock(); + return mt_find(&sparse_irqs, &index, total_nr_irqs); } static void irq_insert_desc(unsigned int irq, struct irq_desc *desc) @@ -934,7 +931,11 @@ EXPORT_SYMBOL_GPL(__irq_alloc_descs); */ unsigned int irq_get_next_irq(unsigned int offset) { - return irq_find_at_or_after(offset); + struct irq_desc *desc; + + guard(rcu)(); + desc = irq_find_desc_at_or_after(offset); + return desc ? irq_desc_get_irq(desc) : total_nr_irqs; } struct irq_desc *__irq_get_desc_lock(unsigned int irq, unsigned long *flags, bool bus, -- cgit v1.2.3 From 61b51a167c524b65a59b1342e70c2008d514a796 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sun, 17 May 2026 22:02:44 +0200 Subject: genirq/proc: Runtime size the chip name The chip name column in the /proc/interrupt output is 8 characters and right aligned, which causes visual clutter due to the fixed length and the alignment. Many interrupt chips, e.g. PCI/MSI[X] have way longer names. Update the length when a chip is assigned to an interrupt and utilize this information for the output. Align it left so all chip names start at the begin of the column. Update the GDB script as well and disentangle the header maze so it actually works with all .config combinations. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Reviewed-by: Dmitry Ilvokhin Link: https://patch.msgid.link/20260517194932.085786035@kernel.org --- kernel/irq/chip.c | 6 ++++-- kernel/irq/debugfs.h | 44 ++++++++++++++++++++++++++++++++++++++++++++ kernel/irq/internals.h | 48 +++--------------------------------------------- kernel/irq/irqdomain.c | 5 ++++- kernel/irq/proc.c | 33 ++++++++++++++++++++++++++++----- kernel/irq/proc.h | 13 +++++++++++++ 6 files changed, 96 insertions(+), 53 deletions(-) create mode 100644 kernel/irq/debugfs.h create mode 100644 kernel/irq/proc.h (limited to 'kernel') diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c index 2d283d1390dc..934777925f5c 100644 --- a/kernel/irq/chip.c +++ b/kernel/irq/chip.c @@ -47,9 +47,11 @@ int irq_set_chip(unsigned int irq, const struct irq_chip *chip) scoped_irqdesc->irq_data.chip = (struct irq_chip *)(chip ?: &no_irq_chip); ret = 0; } - /* For !CONFIG_SPARSE_IRQ make the irq show up in allocated_irqs. */ - if (!ret) + if (!ret) { + /* For !CONFIG_SPARSE_IRQ make the irq show up in allocated_irqs. */ irq_mark_irq(irq); + irq_proc_update_chip(chip); + } return ret; } EXPORT_SYMBOL(irq_set_chip); diff --git a/kernel/irq/debugfs.h b/kernel/irq/debugfs.h new file mode 100644 index 000000000000..8a9360d5fefb --- /dev/null +++ b/kernel/irq/debugfs.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _KERNEL_IRQ_DEBUGFS_H +#define _KERNEL_IRQ_DEBUGFS_H + +#ifdef CONFIG_GENERIC_IRQ_DEBUGFS +#include + +struct irq_bit_descr { + unsigned int mask; + char *name; +}; + +#define BIT_MASK_DESCR(m) { .mask = m, .name = #m } + +void irq_debug_show_bits(struct seq_file *m, int ind, unsigned int state, + const struct irq_bit_descr *sd, int size); + +void irq_add_debugfs_entry(unsigned int irq, struct irq_desc *desc); +static inline void irq_remove_debugfs_entry(struct irq_desc *desc) +{ + debugfs_remove(desc->debugfs_file); + kfree(desc->dev_name); +} +void irq_debugfs_copy_devname(int irq, struct device *dev); +# ifdef CONFIG_IRQ_DOMAIN +void irq_domain_debugfs_init(struct dentry *root); +# else +static inline void irq_domain_debugfs_init(struct dentry *root) +{ +} +# endif +#else /* CONFIG_GENERIC_IRQ_DEBUGFS */ +static inline void irq_add_debugfs_entry(unsigned int irq, struct irq_desc *d) +{ +} +static inline void irq_remove_debugfs_entry(struct irq_desc *d) +{ +} +static inline void irq_debugfs_copy_devname(int irq, struct device *dev) +{ +} +#endif /* CONFIG_GENERIC_IRQ_DEBUGFS */ + +#endif diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h index 37eec0337867..f9c099d45a64 100644 --- a/kernel/irq/internals.h +++ b/kernel/irq/internals.h @@ -12,6 +12,9 @@ #include #include +#include "debugfs.h" +#include "proc.h" + #ifdef CONFIG_SPARSE_IRQ # define MAX_SPARSE_IRQS INT_MAX #else @@ -149,12 +152,6 @@ static inline void unregister_handler_proc(unsigned int irq, static inline void irq_proc_update_valid(struct irq_desc *desc) { } #endif -#if defined(CONFIG_PROC_FS) && defined(CONFIG_GENERIC_IRQ_SHOW) -void irq_proc_calc_prec(void); -#else -static inline void irq_proc_calc_prec(void) { } -#endif - struct irq_desc *irq_find_desc_at_or_after(unsigned int offset); extern bool irq_can_set_affinity_usr(unsigned int irq); @@ -398,42 +395,3 @@ static inline struct irq_data *irqd_get_parent_data(struct irq_data *irqd) return NULL; #endif } - -#ifdef CONFIG_GENERIC_IRQ_DEBUGFS -#include - -struct irq_bit_descr { - unsigned int mask; - char *name; -}; - -#define BIT_MASK_DESCR(m) { .mask = m, .name = #m } - -void irq_debug_show_bits(struct seq_file *m, int ind, unsigned int state, - const struct irq_bit_descr *sd, int size); - -void irq_add_debugfs_entry(unsigned int irq, struct irq_desc *desc); -static inline void irq_remove_debugfs_entry(struct irq_desc *desc) -{ - debugfs_remove(desc->debugfs_file); - kfree(desc->dev_name); -} -void irq_debugfs_copy_devname(int irq, struct device *dev); -# ifdef CONFIG_IRQ_DOMAIN -void irq_domain_debugfs_init(struct dentry *root); -# else -static inline void irq_domain_debugfs_init(struct dentry *root) -{ -} -# endif -#else /* CONFIG_GENERIC_IRQ_DEBUGFS */ -static inline void irq_add_debugfs_entry(unsigned int irq, struct irq_desc *d) -{ -} -static inline void irq_remove_debugfs_entry(struct irq_desc *d) -{ -} -static inline void irq_debugfs_copy_devname(int irq, struct device *dev) -{ -} -#endif /* CONFIG_GENERIC_IRQ_DEBUGFS */ diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index cc93abf009e8..f15c9f1223bb 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -20,6 +20,8 @@ #include #include +#include "proc.h" + static LIST_HEAD(irq_domain_list); static DEFINE_MUTEX(irq_domain_mutex); @@ -1532,6 +1534,7 @@ int irq_domain_set_hwirq_and_chip(struct irq_domain *domain, unsigned int virq, irq_data->chip = (struct irq_chip *)(chip ? chip : &no_irq_chip); irq_data->chip_data = chip_data; + irq_proc_update_chip(chip); return 0; } EXPORT_SYMBOL_GPL(irq_domain_set_hwirq_and_chip); @@ -2081,7 +2084,7 @@ static void irq_domain_free_one_irq(struct irq_domain *domain, unsigned int virq #endif /* CONFIG_IRQ_DOMAIN_HIERARCHY */ #ifdef CONFIG_GENERIC_IRQ_DEBUGFS -#include "internals.h" +#include "debugfs.h" static struct dentry *domain_dir; diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index 1cb47a8eab49..9a968007eb2d 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c @@ -457,10 +457,14 @@ int __weak arch_show_interrupts(struct seq_file *p, int prec) return 0; } +static DEFINE_RAW_SPINLOCK(irq_proc_constraints_lock); + static struct irq_proc_constraints { unsigned int num_prec; + unsigned int chip_width; } irq_proc_constraints __read_mostly = { .num_prec = 4, + .chip_width = 8, }; #ifndef ACTUAL_NR_IRQS @@ -473,7 +477,23 @@ void irq_proc_calc_prec(void) for (prec = 4, n = 10000; prec < 10 && n <= total_nr_irqs; ++prec) n *= 10; - WRITE_ONCE(irq_proc_constraints.num_prec, prec); + + guard(raw_spinlock_irqsave)(&irq_proc_constraints_lock); + if (prec > irq_proc_constraints.num_prec) + WRITE_ONCE(irq_proc_constraints.num_prec, prec); +} + +void irq_proc_update_chip(const struct irq_chip *chip) +{ + unsigned int len = chip && chip->name ? strlen(chip->name) : 0; + + if (!len || len <= READ_ONCE(irq_proc_constraints.chip_width)) + return; + + /* Can be invoked from interrupt disabled contexts */ + guard(raw_spinlock_irqsave)(&irq_proc_constraints_lock); + if (len > irq_proc_constraints.chip_width) + WRITE_ONCE(irq_proc_constraints.chip_width, len); } /* Same as seq_put_decimal_ull_width(p, " ", cnt, 10) */ @@ -515,6 +535,7 @@ void irq_proc_emit_counts(struct seq_file *p, unsigned int __percpu *cnts) int show_interrupts(struct seq_file *p, void *v) { + unsigned int chip_width = READ_ONCE(irq_proc_constraints.chip_width); unsigned int prec = READ_ONCE(irq_proc_constraints.num_prec); int i = *(loff_t *) v, j; struct irqaction *action; @@ -550,18 +571,20 @@ int show_interrupts(struct seq_file *p, void *v) irq_proc_emit_counts(p, &desc->kstat_irqs->cnt); else irq_proc_emit_zero_counts(p, num_online_cpus()); - seq_putc(p, ' '); + + /* Enforce a visual gap */ + seq_write(p, " ", 2); guard(raw_spinlock_irq)(&desc->lock); if (desc->irq_data.chip) { if (desc->irq_data.chip->irq_print_chip) desc->irq_data.chip->irq_print_chip(&desc->irq_data, p); else if (desc->irq_data.chip->name) - seq_printf(p, "%8s", desc->irq_data.chip->name); + seq_printf(p, "%-*s", chip_width, desc->irq_data.chip->name); else - seq_printf(p, "%8s", "-"); + seq_printf(p, "%-*s", chip_width, "-"); } else { - seq_printf(p, "%8s", "None"); + seq_printf(p, "%-*s", chip_width, "None"); } seq_putc(p, ' '); diff --git a/kernel/irq/proc.h b/kernel/irq/proc.h new file mode 100644 index 000000000000..0631d57fbfb7 --- /dev/null +++ b/kernel/irq/proc.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _KERNEL_IRQ_PROC_H +#define _KERNEL_IRQ_PROC_H + +#if defined(CONFIG_PROC_FS) && defined(CONFIG_GENERIC_IRQ_SHOW) +void irq_proc_calc_prec(void); +void irq_proc_update_chip(const struct irq_chip *chip); +#else +static inline void irq_proc_calc_prec(void) { } +static inline void irq_proc_update_chip(const struct irq_chip *chip) { } +#endif + +#endif -- cgit v1.2.3 From 171cc0d9eed1cad5de7ce6a212efbeda390edb0f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sun, 17 May 2026 22:02:49 +0200 Subject: genirq/proc: Speed up /proc/interrupts iteration Reading /proc/interrupts iterates over the interrupt number space one by one and looks up the descriptors one by one. That's just a waste of time. When CONFIG_GENERIC_IRQ_SHOW is enabled this can utilize the maple tree and cache the descriptor pointer efficiently for the sequence file operations. Implement a CONFIG_GENERIC_IRQ_SHOW specific version in the core code and leave the fs/proc/ variant for the legacy architectures which ignore generic code. This reduces the time wasted for looking up the next record significantly. Signed-off-by: Thomas Gleixner Tested-by: Michael Kelley Reviewed-by: Dmitry Ilvokhin Link: https://patch.msgid.link/20260517194932.165280601@kernel.org --- kernel/irq/proc.c | 116 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 91 insertions(+), 25 deletions(-) (limited to 'kernel') diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index 9a968007eb2d..1b835725f7b1 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c @@ -452,6 +452,8 @@ void irq_proc_update_valid(struct irq_desc *desc) #ifdef CONFIG_GENERIC_IRQ_SHOW +#define ARCH_PROC_IRQDESC ((void *)0x00001111) + int __weak arch_show_interrupts(struct seq_file *p, int prec) { return 0; @@ -460,6 +462,7 @@ int __weak arch_show_interrupts(struct seq_file *p, int prec) static DEFINE_RAW_SPINLOCK(irq_proc_constraints_lock); static struct irq_proc_constraints { + bool print_header; unsigned int num_prec; unsigned int chip_width; } irq_proc_constraints __read_mostly = { @@ -533,34 +536,28 @@ void irq_proc_emit_counts(struct seq_file *p, unsigned int __percpu *cnts) irq_proc_emit_zero_counts(p, zeros); } -int show_interrupts(struct seq_file *p, void *v) +static int irq_seq_show(struct seq_file *p, void *v) { - unsigned int chip_width = READ_ONCE(irq_proc_constraints.chip_width); - unsigned int prec = READ_ONCE(irq_proc_constraints.num_prec); - int i = *(loff_t *) v, j; + struct irq_proc_constraints *constr = p->private; + struct irq_desc *desc = v; struct irqaction *action; - struct irq_desc *desc; - if (i > ACTUAL_NR_IRQS) - return 0; + /* Print header for the first interrupt? */ + if (constr->print_header) { + unsigned int cpu; - if (i == ACTUAL_NR_IRQS) - return arch_show_interrupts(p, prec); - - /* print header and calculate the width of the first column */ - if (i == 0) { - seq_printf(p, "%*s", prec + 8, ""); - for_each_online_cpu(j) - seq_printf(p, "CPU%-8d", j); + seq_printf(p, "%*s", constr->num_prec + 8, ""); + for_each_online_cpu(cpu) + seq_printf(p, "CPU%-8d", cpu); seq_putc(p, '\n'); + constr->print_header = false; } - guard(rcu)(); - desc = irq_to_desc(i); - if (!desc || !irq_settings_proc_valid(desc)) - return 0; + if (desc == ARCH_PROC_IRQDESC) + return arch_show_interrupts(p, constr->num_prec); - seq_printf(p, "%*d:", prec, i); + seq_put_decimal_ull_width(p, "", irq_desc_get_irq(desc), constr->num_prec); + seq_putc(p, ':'); /* * Always output per CPU interrupts. Output device interrupts only when @@ -580,18 +577,18 @@ int show_interrupts(struct seq_file *p, void *v) if (desc->irq_data.chip->irq_print_chip) desc->irq_data.chip->irq_print_chip(&desc->irq_data, p); else if (desc->irq_data.chip->name) - seq_printf(p, "%-*s", chip_width, desc->irq_data.chip->name); + seq_printf(p, "%-*s", constr->chip_width, desc->irq_data.chip->name); else - seq_printf(p, "%-*s", chip_width, "-"); + seq_printf(p, "%-*s", constr->chip_width, "-"); } else { - seq_printf(p, "%-*s", chip_width, "None"); + seq_printf(p, "%-*s", constr->chip_width, "None"); } seq_putc(p, ' '); if (desc->irq_data.domain) - seq_put_decimal_ull_width(p, "", desc->irq_data.hwirq, prec); + seq_put_decimal_ull_width(p, "", desc->irq_data.hwirq, constr->num_prec); else - seq_printf(p, " %*s", prec, ""); + seq_printf(p, " %*s", constr->num_prec, ""); if (IS_ENABLED(CONFIG_GENERIC_IRQ_SHOW_LEVEL)) seq_printf(p, " %-8s", irqd_is_level_type(&desc->irq_data) ? "Level" : "Edge"); @@ -609,4 +606,73 @@ int show_interrupts(struct seq_file *p, void *v) seq_putc(p, '\n'); return 0; } + +static void *irq_seq_next_desc(loff_t *pos) +{ + if (*pos > total_nr_irqs) + return NULL; + + guard(rcu)(); + for (;;) { + struct irq_desc *desc = irq_find_desc_at_or_after((unsigned int) *pos); + + if (desc) { + *pos = irq_desc_get_irq(desc); + /* + * If valid for output then try to acquire a reference + * count on the descriptor so that it can't be freed + * after dropping RCU read lock on return. + */ + if (irq_settings_proc_valid(desc) && irq_desc_get_ref(desc)) + return desc; + (*pos)++; + } else { + *pos = total_nr_irqs; + return ARCH_PROC_IRQDESC; + } + } +} + +static void *irq_seq_start(struct seq_file *f, loff_t *pos) +{ + if (!*pos) { + struct irq_proc_constraints *constr = f->private; + + constr->num_prec = READ_ONCE(irq_proc_constraints.num_prec); + constr->chip_width = READ_ONCE(irq_proc_constraints.chip_width); + constr->print_header = true; + } + return irq_seq_next_desc(pos); +} + +static void *irq_seq_next(struct seq_file *f, void *v, loff_t *pos) +{ + if (v && v != ARCH_PROC_IRQDESC) + irq_desc_put_ref(v); + + (*pos)++; + return irq_seq_next_desc(pos); +} + +static void irq_seq_stop(struct seq_file *f, void *v) +{ + if (v && v != ARCH_PROC_IRQDESC) + irq_desc_put_ref(v); +} + +static const struct seq_operations irq_seq_ops = { + .start = irq_seq_start, + .next = irq_seq_next, + .stop = irq_seq_stop, + .show = irq_seq_show, +}; + +static int __init irq_proc_init(void) +{ + proc_create_seq_private("interrupts", 0, NULL, &irq_seq_ops, + sizeof(irq_proc_constraints), NULL); + return 0; +} +fs_initcall(irq_proc_init); + #endif -- cgit v1.2.3 From 8b226771014beab1292081151a99530886ce54b4 Mon Sep 17 00:00:00 2001 From: Ricardo Robaina Date: Thu, 14 May 2026 12:13:20 -0300 Subject: audit: use 'unsigned int' instead of 'unsigned' Address checkpatch.pl warning below, across the audit subsystem: WARNING: Prefer 'unsigned int' to bare use of 'unsigned' Minor cleanup, no functional changes. Signed-off-by: Ricardo Robaina Signed-off-by: Paul Moore --- kernel/audit.c | 2 +- kernel/audit.h | 2 +- kernel/audit_tree.c | 2 +- kernel/audit_watch.c | 2 +- kernel/auditfilter.c | 8 ++++---- kernel/auditsc.c | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/audit.c b/kernel/audit.c index e1d489bc2dff..a6f367a0d424 100644 --- a/kernel/audit.c +++ b/kernel/audit.c @@ -2030,7 +2030,7 @@ void audit_log_vformat(struct audit_buffer *ab, const char *fmt, va_list args) * here and AUDIT_BUFSIZ is at least 1024, then we can * log everything that printk could have logged. */ avail = audit_expand(ab, - max_t(unsigned, AUDIT_BUFSIZ, 1+len-avail)); + max_t(unsigned int, AUDIT_BUFSIZ, 1+len-avail)); if (!avail) goto out_va_end; len = vsnprintf(skb_tail_pointer(skb), avail, fmt, args2); diff --git a/kernel/audit.h b/kernel/audit.h index ac81fa02bcd7..a5926c83e61b 100644 --- a/kernel/audit.h +++ b/kernel/audit.h @@ -233,7 +233,7 @@ static inline int audit_hash_ino(u64 ino) /* Indicates that audit should log the full pathname. */ #define AUDIT_NAME_FULL -1 -extern int audit_match_class(int class, unsigned syscall); +extern int audit_match_class(int class, unsigned int syscall); extern int audit_comparator(const u32 left, const u32 op, const u32 right); extern int audit_uid_comparator(kuid_t left, u32 op, kuid_t right); extern int audit_gid_comparator(kgid_t left, u32 op, kgid_t right); diff --git a/kernel/audit_tree.c b/kernel/audit_tree.c index ee84777fdfad..1ed19b775912 100644 --- a/kernel/audit_tree.c +++ b/kernel/audit_tree.c @@ -33,7 +33,7 @@ struct audit_chunk { struct audit_node { struct list_head list; struct audit_tree *owner; - unsigned index; /* index; upper bit indicates 'will prune' */ + unsigned int index; /* index; upper bit indicates 'will prune' */ } owners[] __counted_by(count); }; diff --git a/kernel/audit_watch.c b/kernel/audit_watch.c index 33577f0f54ef..c9fe64f73ffc 100644 --- a/kernel/audit_watch.c +++ b/kernel/audit_watch.c @@ -244,7 +244,7 @@ static void audit_watch_log_rule_change(struct audit_krule *r, struct audit_watc /* Update inode info in audit rules based on filesystem event. */ static void audit_update_watch(struct audit_parent *parent, const struct qstr *dname, dev_t dev, - u64 ino, unsigned invalidating) + u64 ino, unsigned int invalidating) { struct audit_watch *owatch, *nwatch, *nextw; struct audit_krule *r, *nextr; diff --git a/kernel/auditfilter.c b/kernel/auditfilter.c index 093425123f6c..b97c97495807 100644 --- a/kernel/auditfilter.c +++ b/kernel/auditfilter.c @@ -165,13 +165,13 @@ static inline int audit_to_inode(struct audit_krule *krule, static __u32 *classes[AUDIT_SYSCALL_CLASSES]; -int __init audit_register_class(int class, unsigned *list) +int __init audit_register_class(int class, unsigned int *list) { __u32 *p = kcalloc(AUDIT_BITMASK_SIZE, sizeof(__u32), GFP_KERNEL); if (!p) return -ENOMEM; while (*list != ~0U) { - unsigned n = *list++; + unsigned int n = *list++; if (n >= AUDIT_BITMASK_SIZE * 32 - AUDIT_SYSCALL_CLASSES) { kfree(p); return -EINVAL; @@ -186,7 +186,7 @@ int __init audit_register_class(int class, unsigned *list) return 0; } -int audit_match_class(int class, unsigned syscall) +int audit_match_class(int class, unsigned int syscall) { if (unlikely(syscall >= AUDIT_BITMASK_SIZE * 32)) return 0; @@ -237,7 +237,7 @@ static int audit_match_signal(struct audit_entry *entry) /* Common user-space to kernel rule translation. */ static inline struct audit_entry *audit_to_entry_common(struct audit_rule_data *rule) { - unsigned listnr; + unsigned int listnr; struct audit_entry *entry; int i, err; diff --git a/kernel/auditsc.c b/kernel/auditsc.c index ab54fccba215..b0e809d4d2f2 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -150,7 +150,7 @@ static const struct audit_nfcfgop_tab audit_nfcfgs[] = { static int audit_match_perm(struct audit_context *ctx, int mask) { - unsigned n; + unsigned int n; if (unlikely(!ctx)) return 0; -- cgit v1.2.3 From 888a0396e154524f4027f27da84bdbec9eb68916 Mon Sep 17 00:00:00 2001 From: Ricardo Robaina Date: Wed, 13 May 2026 18:47:59 -0300 Subject: audit: fix removal of dangling executable rules When an audited executable is deleted from the disk, its dentry becomes negative. Any later attempt to delete the associated audit rule will lead to audit_alloc_mark() encountering this negative dentry and immediately aborting, returning -ENOENT. This early abort prevents the subsystem from allocating the temporary fsnotify mark needed to construct the search key, meaning the kernel cannot find the existing rule in its own lists to delete it. This leaves a dangling rule in memory, resulting in the following error while attempting to delete the rule: # ./audit-dupe-exe-deadlock.sh No rules Error deleting rule (No such file or directory) There was an error while processing parameters # auditctl -l -a always,exit -S all -F exe=/tmp/file -F path=/tmp/file -F key=dr # auditctl -D Error deleting rule (No such file or directory) There was an error while processing parameters This patch fixes this issue by removing the d_really_is_negative() check. By doing so, a dummy mark can be successfully generated for the deleted path, which allows the audit subsystem to properly match and flush the dangling rule. Cc: stable@kernel.org Fixes: 76a53de6f7ff ("VFS/audit: introduce kern_path_parent() for audit") Acked-by: Waiman Long Acked-by: Richard Guy Briggs Signed-off-by: Ricardo Robaina Signed-off-by: Paul Moore --- kernel/audit_fsnotify.c | 4 ---- 1 file changed, 4 deletions(-) (limited to 'kernel') diff --git a/kernel/audit_fsnotify.c b/kernel/audit_fsnotify.c index 711454f9f724..ae0e75403f76 100644 --- a/kernel/audit_fsnotify.c +++ b/kernel/audit_fsnotify.c @@ -84,10 +84,6 @@ struct audit_fsnotify_mark *audit_alloc_mark(struct audit_krule *krule, char *pa dentry = kern_path_parent(pathname, &path); if (IS_ERR(dentry)) return ERR_CAST(dentry); /* returning an error */ - if (d_really_is_negative(dentry)) { - audit_mark = ERR_PTR(-ENOENT); - goto out; - } audit_mark = kzalloc_obj(*audit_mark); if (unlikely(!audit_mark)) { -- cgit v1.2.3 From 6bf8973ddeeec0ead87b0dfe9a3ae6072432e334 Mon Sep 17 00:00:00 2001 From: Cheng-Yang Chou Date: Tue, 26 May 2026 01:22:31 +0800 Subject: sched_ext: idle: Fix errno loss in scx_idle_init() || is a boolean operator, any nonzero (error) return short-circuits to 1 rather than the actual errno. The caller in scx_init() logs and propagates this value, so the wrong code reaches upper layers. Signed-off-by: Cheng-Yang Chou Signed-off-by: Tejun Heo --- kernel/sched/ext_idle.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext_idle.c b/kernel/sched/ext_idle.c index bbb845f36a0a..6ff80722f645 100644 --- a/kernel/sched/ext_idle.c +++ b/kernel/sched/ext_idle.c @@ -1509,13 +1509,9 @@ static const struct btf_kfunc_id_set scx_kfunc_set_select_cpu = { int scx_idle_init(void) { - int ret; - - ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_idle) || - register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &scx_kfunc_set_idle) || - register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_idle) || - register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_select_cpu) || - register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_select_cpu); - - return ret; + return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_idle) ?: + register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &scx_kfunc_set_idle) ?: + register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_idle) ?: + register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_select_cpu) ?: + register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_select_cpu); } -- cgit v1.2.3 From 611583a76ea97991b0f65ec1ff099eac7fe0bae4 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Sun, 24 May 2026 08:19:56 -0700 Subject: workqueue: drop spurious '*' from print_worker_info() fn declaration print_worker_info() declares its local 'fn' as work_func_t * but worker->current_func has type work_func_t (a function pointer). The extra level of indirection is wrong and only happens to be harmless today because every supported Linux architecture has sizeof(work_func_t) == sizeof(work_func_t *): copy_from_kernel_nofault() reads the correct number of bytes by accident, and %ps still resolves the printed address because the stored value is the function address regardless of declared type. On any future ABI where sizeof(void (*)()) differs from sizeof(void *), the nofault copy would transfer the wrong number of bytes and the subsequent %ps would print an incorrect address. Match the field type so the intent is explicit and the code does not silently rely on equal pointer sizes. Fixes: 3d1cb2059d93 ("workqueue: include workqueue info when printing debug dump of a worker task") Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo --- kernel/workqueue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 9adee917e2bb..35b0c8f4f10f 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -6286,7 +6286,7 @@ EXPORT_SYMBOL_GPL(set_worker_desc); */ void print_worker_info(const char *log_lvl, struct task_struct *task) { - work_func_t *fn = NULL; + work_func_t fn = NULL; char name[WQ_NAME_LEN] = { }; char desc[WORKER_DESC_LEN] = { }; struct pool_workqueue *pwq = NULL; -- cgit v1.2.3 From 0a68853de27b522bca2b9934127277185374a24f Mon Sep 17 00:00:00 2001 From: Sun Shaojie Date: Wed, 27 May 2026 14:43:28 +0800 Subject: cgroup/cpuset: Use effective_xcpus in partcmd_update add/del mask calculation When sibling CPU exclusion occurs, a partition's user_xcpus may contain CPUs that were never actually granted to it. These CPUs are present in user_xcpus(cs) but not in cs->effective_xcpus. The partcmd_update path in update_parent_effective_cpumask() uses user_xcpus(cs) (via the local variable xcpus) to compute the addmask (CPUs to return to parent) and delmask (CPUs to request from parent). This is incorrect: 1) When newmask removes a CPU that was previously excluded by a sibling, addmask incorrectly includes that CPU and tries to return it to the parent even though the partition never actually owned it, causing CPU overlap with sibling partitions and triggering warnings in generate_sched_domains(). 2) When newmask adds a previously excluded CPU that is now available, delmask fails to request it from the parent because user_xcpus(cs) already includes it. Fix this by using cs->effective_xcpus instead of user_xcpus(cs) in all partcmd_update paths that calculate addmask or delmask, including the PERR_NOCPUS error handling paths. Reproducers: Example 1 - Removing a sibling-excluded CPU incorrectly returns it: # cd /sys/fs/cgroup # echo "0-1" > a1/cpuset.cpus # echo "root" > a1/cpuset.cpus.partition # echo "0-2" > b1/cpuset.cpus # echo "root" > b1/cpuset.cpus.partition # echo "2" > b1/cpuset.cpus # cat cpuset.cpus.effective # Actual: 0-1,3 Expected: 3 Example 2 - Expanding to a previously excluded CPU fails to request it: # cd /sys/fs/cgroup # echo "0-1" > a1/cpuset.cpus # echo "root" > a1/cpuset.cpus.partition # echo "0-2" > b1/cpuset.cpus # echo "root" > b1/cpuset.cpus.partition # echo "member" > a1/cpuset.cpus.partition # echo "1-2" > b1/cpuset.cpus # cat cpuset.cpus.effective # Actual: 0-1,3 Expected: 0,3 Fixes: 2a3602030d80 ("cgroup/cpuset: Don't invalidate sibling partitions on cpuset.cpus conflict") Cc: stable@vger.kernel.org # v7.0+ Suggested-by: Zhang Guopeng Signed-off-by: Sun Shaojie Reviewed-by: Waiman Long Signed-off-by: Tejun Heo --- kernel/cgroup/cpuset.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 5c33ab20cc20..c9e14fda3d6f 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -1811,9 +1811,9 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd, * Compute add/delete mask to/from effective_cpus * * For valid partition: - * addmask = exclusive_cpus & ~newmask + * addmask = effective_xcpus & ~newmask * & parent->effective_xcpus - * delmask = newmask & ~exclusive_cpus + * delmask = newmask & ~effective_xcpus * & parent->effective_xcpus * * For invalid partition: @@ -1825,11 +1825,11 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd, deleting = cpumask_and(tmp->delmask, newmask, parent->effective_xcpus); } else { - cpumask_andnot(tmp->addmask, xcpus, newmask); + cpumask_andnot(tmp->addmask, cs->effective_xcpus, newmask); adding = cpumask_and(tmp->addmask, tmp->addmask, parent->effective_xcpus); - cpumask_andnot(tmp->delmask, newmask, xcpus); + cpumask_andnot(tmp->delmask, newmask, cs->effective_xcpus); deleting = cpumask_and(tmp->delmask, tmp->delmask, parent->effective_xcpus); } @@ -1868,7 +1868,7 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd, part_error = PERR_NOCPUS; deleting = false; adding = cpumask_and(tmp->addmask, - xcpus, parent->effective_xcpus); + cs->effective_xcpus, parent->effective_xcpus); } } else { /* @@ -1890,7 +1890,8 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd, part_error = PERR_NOCPUS; if (is_partition_valid(cs)) adding = cpumask_and(tmp->addmask, - xcpus, parent->effective_xcpus); + cs->effective_xcpus, + parent->effective_xcpus); } else if (is_partition_invalid(cs) && !cpumask_empty(xcpus) && cpumask_subset(xcpus, parent->effective_xcpus)) { struct cgroup_subsys_state *css; -- cgit v1.2.3 From e42e53ae23b7d41df22ccd7788192bf578f24da2 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 27 May 2026 09:26:32 -1000 Subject: bpf: Fix bpf_arena_handle_page_fault() redefinition without CONFIG_BPF_SYSCALL On configs with CONFIG_BPF=y but CONFIG_BPF_SYSCALL=n (e.g. arm multi_v7_defconfig), kernel/bpf/core.c defines a __weak bpf_arena_handle_page_fault() while bpf_defs.h already supplies a static inline stub for it, causing a redefinition error. Build the __weak definition only under CONFIG_BPF_SYSCALL, matching the bpf_defs.h declaration and the CONFIG_BPF_SYSCALL-gated strong definition in arena.c. Fixes: dc11a4dba246 ("bpf: Recover arena kernel faults with scratch page") Reported-by: Mark Brown Signed-off-by: Tejun Heo Acked-by: Song Liu Link: https://lore.kernel.org/r/20260527192632.2109419-1-tj@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/bpf/core.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 8ecba2989d88..a656a8572bdb 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3376,13 +3376,14 @@ __weak u64 bpf_arena_get_kern_vm_start(struct bpf_arena *arena) { return 0; } + +#ifdef CONFIG_BPF_SYSCALL __weak bool bpf_arena_handle_page_fault(unsigned long addr, bool is_write, unsigned long fault_ip) { return false; } -#ifdef CONFIG_BPF_SYSCALL static int __init bpf_global_ma_init(void) { int ret; -- cgit v1.2.3 From 81905b5acbe77284734438df3fbec1158e6429a3 Mon Sep 17 00:00:00 2001 From: Ricardo Robaina Date: Wed, 27 May 2026 19:15:34 -0400 Subject: audit: fix recursive locking deadlock in audit_dupe_exe() A deadlock occurs in the audit subsystem when duplicating executable-related rules. When a file is moved (e.g., via do_renameat2()), the VFS layer locks the parent directory (I_MUTEX_PARENT), which synchronously triggers an fsnotify_move event. If an existing executable audit rule matches the file being moved, the audit subsystem catches this event and calls audit_dupe_exe() to duplicate the watch and update the rule. Then, audit_alloc_mark() would call kern_path_parent() to resolve the path, leading to a blind attempt to acquire the exact same I_MUTEX_PARENT lock already held by the task, resulting in the following recursive locking deadlock: ============================================ WARNING: possible recursive locking detected 6.12.0-55.27.1.el10_0.x86_64+debug #1 Not tainted -------------------------------------------- mv/5099 is trying to acquire lock: ffff888132845358 (&inode->i_sb->s_type->i_mutex_dir_key/1){+.+.}-{3:3}, at: __kern_path_locked+0x10a/0x2f0 but task is already holding lock: ffff888132846b58 (&inode->i_sb->s_type->i_mutex_dir_key/1){+.+.}-{3:3}, at: lock_two_directories+0x13f/0x2b0 other info that might help us debug this: Possible unsafe locking scenario: CPU0 ---- lock(&inode->i_sb->s_type->i_mutex_dir_key/1); lock(&inode->i_sb->s_type->i_mutex_dir_key/1); *** DEADLOCK *** May be due to missing lock nesting notation 6 locks held by mv/5099: #0: ffff888112a9c440 (sb_writers#13) at: do_renameat2+0x34c/0xbc0 #1: ffff888112a9c790 (&type->s_vfs_rename_key#3) at: do_renameat2+0x415/0xbc0 #2: ffff888132846b58 (&inode->i_sb->s_type->i_mutex_dir_key/1) at: lock_two_directories+0x13f/0x2b0 #3: ffff888132845358 (&inode->i_sb->s_type->i_mutex_dir_key/5) at: lock_two_directories+0x175/0x2b0 #4: ffffffffb3a1fb10 (&fsnotify_mark_srcu) at: fsnotify+0x454/0x28a0 #5: ffffffffaf886230 (audit_filter_mutex) at: audit_update_watch+0x36/0x11e0 stack backtrace: Call Trace: dump_stack_lvl+0x6f/0xb0 print_deadlock_bug.cold+0xbd/0xca validate_chain+0x83a/0xf00 __lock_acquire+0xcac/0x1d20 lock_acquire.part.0+0x11b/0x360 down_write_nested+0x9f/0x230 __kern_path_locked+0x10a/0x2f0 kern_path_locked+0x26/0x40 audit_alloc_mark+0xfb/0x4f0 audit_dupe_exe+0x6c/0xe0 audit_dupe_rule+0x6c2/0xc00 audit_update_watch+0x4cc/0x11e0 audit_watch_handle_event+0x12c/0x1b0 send_to_group+0x5d0/0x8b0 fsnotify+0x615/0x28a0 fsnotify_move+0x1d8/0x630 vfs_rename+0xdcd/0x1df0 do_renameat2+0x9d4/0xbc0 __x64_sys_renameat+0x192/0x260 do_syscall_64+0x92/0x180 entry_SYSCALL_64_after_hwframe+0x76/0x7e RIP: 0033:0x7f0491fe8c4e Code: 0f 1f 40 00 48 8b 15 c1 e1 16 00 f7 d8 64 89 02 b8 ff ff ff ff c3 66 0f 1f 44 00 00 f3 0f 1e fa 49 89 ca b8 08 01 00 00 0f 05 <48> 3d 00 f0 ff ff 77 0a c3 66 0f 1f 84 00 00 00 00 00 48 8b 15 89 RSP: 002b:00007ffc7210bf38 EFLAGS: 00000246 ORIG_RAX: 0000000000000108 RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007f0491fe8c4e RDX: 0000000000000003 RSI: 00007ffc7210e6c8 RDI: 00000000ffffff9c RBP: 0000000000000000 R08: 0000000000000000 R09: 0000000000000001 R10: 00005575eb2dae2a R11: 0000000000000246 R12: 00005575eb2dae2a R13: 00007ffc7210e6c8 R14: 0000000000000003 R15: 00000000ffffff9c The aforementioned deadlock can be consistently reproduced by running the script below: audit-dupe-exe-deadlock.sh -------------------------- #!/bin/bash auditctl -D mkdir -p /tmp/foo touch /tmp/file auditctl -a always,exit -F exe=/tmp/file -F path=/tmp/file -S all -k dr mv /tmp/file /tmp/foo/file rm -Rf /tmp/foo This patch fixes the issue by introducing struct audit_watch_ctx to pass the fsnotify event context down to audit_alloc_mark(). By utilizing the already-resolved directory inode provided by the event, we bypass the kern_path_parent() path resolution entirely, safely avoiding the recursive lock. Furthermore, it explicitly allows duplicate fsnotify marks (allow_dups = 1) during the rename update, allowing the new rule's mark to safely coexist with the old rule's mark until the old rule is freed. P.S.: This issue was identified and reproduced during a comprehensive code coverage analysis of the audit subsystem. The full report is available at the link below: https://people.redhat.com/rrobaina/audit-code-coverage-analysis.pdf P.P.S: With the permission of both Ricardo and Nathan, I've squashed a fixup patch from Nathan that addresses a compile time error when CONFIG_AUDITSYSCALL=n. Cc: stable@kernel.org Fixes: 34d99af52ad4 ("audit: implement audit by executable") Acked-by: Waiman Long Acked-by: Richard Guy Briggs Signed-off-by: Nathan Chancellor Signed-off-by: Ricardo Robaina [PM: move link metadata into the msg, apply fix from NC] Signed-off-by: Paul Moore --- kernel/audit.h | 17 ++++++++++++----- kernel/audit_fsnotify.c | 32 +++++++++++++++++++++++--------- kernel/audit_watch.c | 25 +++++++++++++++++-------- kernel/auditfilter.c | 9 +++++---- 4 files changed, 57 insertions(+), 26 deletions(-) (limited to 'kernel') diff --git a/kernel/audit.h b/kernel/audit.h index a5926c83e61b..92d5e723d570 100644 --- a/kernel/audit.h +++ b/kernel/audit.h @@ -256,8 +256,13 @@ extern int audit_del_rule(struct audit_entry *entry); extern void audit_free_rule_rcu(struct rcu_head *head); extern struct list_head audit_filter_list[]; -extern struct audit_entry *audit_dupe_rule(struct audit_krule *old); +struct audit_watch_ctx { + struct inode *dir; + struct inode *child; +}; +extern struct audit_entry *audit_dupe_rule(struct audit_krule *old, + struct audit_watch_ctx *ctx); extern void audit_log_d_path_exe(struct audit_buffer *ab, struct mm_struct *mm); @@ -280,13 +285,15 @@ extern char *audit_watch_path(struct audit_watch *watch); extern int audit_watch_compare(struct audit_watch *watch, u64 ino, dev_t dev); extern struct audit_fsnotify_mark *audit_alloc_mark(struct audit_krule *krule, - char *pathname, int len); + char *pathname, int len, + struct audit_watch_ctx *ctx); extern char *audit_mark_path(struct audit_fsnotify_mark *mark); extern void audit_remove_mark(struct audit_fsnotify_mark *audit_mark); extern void audit_remove_mark_rule(struct audit_krule *krule); extern int audit_mark_compare(struct audit_fsnotify_mark *mark, u64 ino, dev_t dev); -extern int audit_dupe_exe(struct audit_krule *new, struct audit_krule *old); +extern int audit_dupe_exe(struct audit_krule *new, struct audit_krule *old, + struct audit_watch_ctx *ctx); extern int audit_exe_compare(struct task_struct *tsk, struct audit_fsnotify_mark *mark); @@ -317,13 +324,13 @@ extern struct list_head *audit_killed_trees(void); #define audit_watch_path(w) "" #define audit_watch_compare(w, i, d) 0 -#define audit_alloc_mark(k, p, l) (ERR_PTR(-EINVAL)) +#define audit_alloc_mark(k, p, l, c) (ERR_PTR(-EINVAL)) #define audit_mark_path(m) "" #define audit_remove_mark(m) do { } while (0) #define audit_remove_mark_rule(k) do { } while (0) #define audit_mark_compare(m, i, d) 0 #define audit_exe_compare(t, m) (-EINVAL) -#define audit_dupe_exe(n, o) (-EINVAL) +#define audit_dupe_exe(n, o, c) (-EINVAL) #define audit_remove_tree_rule(rule) BUG() #define audit_add_tree_rule(rule) -EINVAL diff --git a/kernel/audit_fsnotify.c b/kernel/audit_fsnotify.c index ae0e75403f76..fa33d57e4320 100644 --- a/kernel/audit_fsnotify.c +++ b/kernel/audit_fsnotify.c @@ -71,19 +71,30 @@ static void audit_update_mark(struct audit_fsnotify_mark *audit_mark, audit_mark->ino = inode ? inode->i_ino : AUDIT_INO_UNSET; } -struct audit_fsnotify_mark *audit_alloc_mark(struct audit_krule *krule, char *pathname, int len) +struct audit_fsnotify_mark *audit_alloc_mark(struct audit_krule *krule, char *pathname, + int len, struct audit_watch_ctx *ctx) { struct audit_fsnotify_mark *audit_mark; struct path path; struct dentry *dentry; - int ret; + struct inode *dir, *child; + int ret, allow_dups; if (pathname[0] != '/' || pathname[len-1] == '/') return ERR_PTR(-EINVAL); - dentry = kern_path_parent(pathname, &path); - if (IS_ERR(dentry)) - return ERR_CAST(dentry); /* returning an error */ + if (!ctx) { + dentry = kern_path_parent(pathname, &path); + if (IS_ERR(dentry)) + return ERR_CAST(dentry); /* returning an error */ + dir = d_inode(path.dentry); + child = d_inode(dentry); + allow_dups = 0; + } else { + dir = ctx->dir; + child = ctx->child; + allow_dups = 1; + } audit_mark = kzalloc_obj(*audit_mark); if (unlikely(!audit_mark)) { @@ -94,18 +105,21 @@ struct audit_fsnotify_mark *audit_alloc_mark(struct audit_krule *krule, char *pa fsnotify_init_mark(&audit_mark->mark, audit_fsnotify_group); audit_mark->mark.mask = AUDIT_FS_EVENTS; audit_mark->path = pathname; - audit_update_mark(audit_mark, dentry->d_inode); audit_mark->rule = krule; - ret = fsnotify_add_inode_mark(&audit_mark->mark, path.dentry->d_inode, 0); + audit_update_mark(audit_mark, child); + ret = fsnotify_add_inode_mark(&audit_mark->mark, dir, allow_dups); + if (ret < 0) { audit_mark->path = NULL; fsnotify_put_mark(&audit_mark->mark); audit_mark = ERR_PTR(ret); } out: - dput(dentry); - path_put(&path); + if (!ctx) { + dput(dentry); + path_put(&path); + } return audit_mark; } diff --git a/kernel/audit_watch.c b/kernel/audit_watch.c index c9fe64f73ffc..06dd0ebe73e2 100644 --- a/kernel/audit_watch.c +++ b/kernel/audit_watch.c @@ -244,7 +244,8 @@ static void audit_watch_log_rule_change(struct audit_krule *r, struct audit_watc /* Update inode info in audit rules based on filesystem event. */ static void audit_update_watch(struct audit_parent *parent, const struct qstr *dname, dev_t dev, - u64 ino, unsigned int invalidating) + u64 ino, unsigned int invalidating, + struct audit_watch_ctx *ctx) { struct audit_watch *owatch, *nwatch, *nextw; struct audit_krule *r, *nextr; @@ -280,7 +281,7 @@ static void audit_update_watch(struct audit_parent *parent, list_del(&oentry->rule.rlist); list_del_rcu(&oentry->list); - nentry = audit_dupe_rule(&oentry->rule); + nentry = audit_dupe_rule(&oentry->rule, ctx); if (IS_ERR(nentry)) { list_del(&oentry->rule.list); audit_panic("error updating watch, removing"); @@ -479,10 +480,17 @@ static int audit_watch_handle_event(struct fsnotify_mark *inode_mark, u32 mask, if (WARN_ON_ONCE(inode_mark->group != audit_watch_group)) return 0; - if (mask & (FS_CREATE|FS_MOVED_TO) && inode) - audit_update_watch(parent, dname, inode->i_sb->s_dev, inode->i_ino, 0); - else if (mask & (FS_DELETE|FS_MOVED_FROM)) - audit_update_watch(parent, dname, AUDIT_DEV_UNSET, AUDIT_INO_UNSET, 1); + if (mask & (FS_CREATE|FS_MOVED_TO) && inode) { + struct audit_watch_ctx ctx = { .dir = dir, .child = inode }; + + audit_update_watch(parent, dname, inode->i_sb->s_dev, inode->i_ino, 0, + &ctx); + } else if (mask & (FS_DELETE|FS_MOVED_FROM)) { + struct audit_watch_ctx ctx = { .dir = dir, .child = NULL }; + + audit_update_watch(parent, dname, AUDIT_DEV_UNSET, AUDIT_INO_UNSET, 1, + &ctx); + } else if (mask & (FS_DELETE_SELF|FS_UNMOUNT|FS_MOVE_SELF)) audit_remove_parent_watches(parent); @@ -505,7 +513,8 @@ static int __init audit_watch_init(void) } device_initcall(audit_watch_init); -int audit_dupe_exe(struct audit_krule *new, struct audit_krule *old) +int audit_dupe_exe(struct audit_krule *new, struct audit_krule *old, + struct audit_watch_ctx *ctx) { struct audit_fsnotify_mark *audit_mark; char *pathname; @@ -514,7 +523,7 @@ int audit_dupe_exe(struct audit_krule *new, struct audit_krule *old) if (!pathname) return -ENOMEM; - audit_mark = audit_alloc_mark(new, pathname, strlen(pathname)); + audit_mark = audit_alloc_mark(new, pathname, strlen(pathname), ctx); if (IS_ERR(audit_mark)) { kfree(pathname); return PTR_ERR(audit_mark); diff --git a/kernel/auditfilter.c b/kernel/auditfilter.c index b97c97495807..4401119b5275 100644 --- a/kernel/auditfilter.c +++ b/kernel/auditfilter.c @@ -589,7 +589,7 @@ static struct audit_entry *audit_data_to_entry(struct audit_rule_data *data, err = PTR_ERR(str); goto exit_free; } - audit_mark = audit_alloc_mark(&entry->rule, str, f_val); + audit_mark = audit_alloc_mark(&entry->rule, str, f_val, NULL); if (IS_ERR(audit_mark)) { kfree(str); err = PTR_ERR(audit_mark); @@ -816,7 +816,8 @@ static inline int audit_dupe_lsm_field(struct audit_field *df, * rule with the new rule in the filterlist, then free the old rule. * The rlist element is undefined; list manipulations are handled apart from * the initial copy. */ -struct audit_entry *audit_dupe_rule(struct audit_krule *old) +struct audit_entry *audit_dupe_rule(struct audit_krule *old, + struct audit_watch_ctx *ctx) { u32 fcount = old->field_count; struct audit_entry *entry; @@ -875,7 +876,7 @@ struct audit_entry *audit_dupe_rule(struct audit_krule *old) new->filterkey = fk; break; case AUDIT_EXE: - err = audit_dupe_exe(new, old); + err = audit_dupe_exe(new, old, ctx); break; } if (err) { @@ -1414,7 +1415,7 @@ static int update_lsm_rule(struct audit_krule *r) if (!security_audit_rule_known(r)) return 0; - nentry = audit_dupe_rule(r); + nentry = audit_dupe_rule(r, NULL); if (entry->rule.exe) audit_remove_mark(entry->rule.exe); if (IS_ERR(nentry)) { -- cgit v1.2.3 From d5cae2261b86913e602452ce4a07e6aefc0f603b Mon Sep 17 00:00:00 2001 From: Feng Tang Date: Mon, 25 May 2026 09:51:11 +0800 Subject: dma-contiguous: simplify numa cma area handling Currently, there are 2 kernel cmdline ways to setup numa cma area: "cma_pernuma=" and "numa_cma=", and there are 2 cma arrays as well, while they have no difference technically. Robin suggested to cleanup the code and only use one array [1], as "the apparent intent that users only want one _or_ the other". Simplify the code by only using one array to save the numa cma area. And in rare case that a user really setup the 2 cmdline parameters at the same time, let the per-node specific size setting 'numa_cma=' take priority over the global numa cma setting. Link[1]: https://lore.kernel.org/lkml/43c5301c-fe6a-41e4-9482-ccfc7b62f2a7@arm.com/ Suggested-by: Robin Murphy Signed-off-by: Feng Tang Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/20260525015111.6267-1-feng.tang@linux.alibaba.com --- kernel/dma/Kconfig | 4 +++- kernel/dma/contiguous.c | 46 ++++++++++++---------------------------------- 2 files changed, 15 insertions(+), 35 deletions(-) (limited to 'kernel') diff --git a/kernel/dma/Kconfig b/kernel/dma/Kconfig index c9fa0a922cba..0a4ba21a57a7 100644 --- a/kernel/dma/Kconfig +++ b/kernel/dma/Kconfig @@ -179,7 +179,9 @@ config DMA_NUMA_CMA You can set the size of pernuma CMA by specifying "cma_pernuma=size" or set the node id and its size of CMA by specifying "numa_cma= - :size[,:size]" on the kernel's command line. + :size[,:size]" on the kernel's command line. And in + rare case that the above 2 are both setup, then the "numa_cma=" + takes priority for the specified numa nodes. config CMA_SIZE_PERNUMA bool "Default CMA area per NUMA node" diff --git a/kernel/dma/contiguous.c b/kernel/dma/contiguous.c index 799f6e9c88bd..f754079a287d 100644 --- a/kernel/dma/contiguous.c +++ b/kernel/dma/contiguous.c @@ -134,7 +134,6 @@ EXPORT_SYMBOL_GPL(dev_get_cma_area); static struct cma *dma_contiguous_numa_area[MAX_NUMNODES]; static phys_addr_t numa_cma_size[MAX_NUMNODES] __initdata; -static struct cma *dma_contiguous_pernuma_area[MAX_NUMNODES]; static phys_addr_t pernuma_size_bytes __initdata; static bool numa_cma_configured __initdata; @@ -208,7 +207,7 @@ static void __init dma_numa_cma_reserve(void) pernuma_size_bytes = cma_get_size(dma_contiguous_default_area); for_each_node(nid) { - int ret; + int size, ret; char name[CMA_MAX_NAME]; struct cma **cma; @@ -218,27 +217,17 @@ static void __init dma_numa_cma_reserve(void) continue; } - if (pernuma_size_bytes) { - - cma = &dma_contiguous_pernuma_area[nid]; - snprintf(name, sizeof(name), "pernuma%d", nid); - ret = cma_declare_contiguous_nid(0, pernuma_size_bytes, 0, 0, - 0, false, name, cma, nid); - if (ret) - pr_warn("%s: reservation failed: err %d, node %d", __func__, - ret, nid); - } - - if (numa_cma_size[nid]) { + /* per-node numa setting has the priority */ + size = numa_cma_size[nid] ?: pernuma_size_bytes; + if (!size) + continue; - cma = &dma_contiguous_numa_area[nid]; - snprintf(name, sizeof(name), "numa%d", nid); - ret = cma_declare_contiguous_nid(0, numa_cma_size[nid], 0, 0, 0, false, - name, cma, nid); - if (ret) - pr_warn("%s: reservation failed: err %d, node %d", __func__, - ret, nid); - } + cma = &dma_contiguous_numa_area[nid]; + snprintf(name, sizeof(name), "numa%d", nid); + ret = cma_declare_contiguous_nid(0, size, 0, 0, 0, false, name, cma, nid); + if (ret) + pr_warn("%s: reservation failed: err %d, node %d", __func__, + ret, nid); } } #else @@ -437,16 +426,8 @@ struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp) #ifdef CONFIG_DMA_NUMA_CMA if (nid != NUMA_NO_NODE && !(gfp & (GFP_DMA | GFP_DMA32))) { - struct cma *cma = dma_contiguous_pernuma_area[nid]; + struct cma *cma = dma_contiguous_numa_area[nid]; struct page *page; - - if (cma) { - page = cma_alloc_aligned(cma, size, gfp); - if (page) - return page; - } - - cma = dma_contiguous_numa_area[nid]; if (cma) { page = cma_alloc_aligned(cma, size, gfp); if (page) @@ -484,9 +465,6 @@ void dma_free_contiguous(struct device *dev, struct page *page, size_t size) * otherwise, page is from either per-numa cma or default cma */ #ifdef CONFIG_DMA_NUMA_CMA - if (cma_release(dma_contiguous_pernuma_area[page_to_nid(page)], - page, count)) - return; if (cma_release(dma_contiguous_numa_area[page_to_nid(page)], page, count)) return; -- cgit v1.2.3 From 336f87d742a616236006bb77275f79a3ac101637 Mon Sep 17 00:00:00 2001 From: Ren Tamura Date: Thu, 28 May 2026 13:28:39 +0900 Subject: cgroup: pair max limit READ_ONCE() with WRITE_ONCE() cgroup.max.descendants and cgroup.max.depth are shown through seq_file. Their show callbacks read cgrp->max_descendants and cgrp->max_depth with READ_ONCE(), respectively. The corresponding write callbacks update the same scalar fields while holding the cgroup lock, but the seq_file show path does not serialize against those stores. This leaves the lockless show-side loads annotated with READ_ONCE(), while the corresponding stores remain plain stores. Use WRITE_ONCE() for the updates so the intended lockless access is marked consistently on both sides. This does not change locking, ordering, or user-visible semantics. Assisted-by: OpenAI-Codex:gpt-5.5 Signed-off-by: Ren Tamura Signed-off-by: Tejun Heo --- kernel/cgroup/cgroup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index bdc8deedb4f7..6e92791d279e 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -3734,7 +3734,7 @@ static ssize_t cgroup_max_descendants_write(struct kernfs_open_file *of, if (!cgrp) return -ENOENT; - cgrp->max_descendants = descendants; + WRITE_ONCE(cgrp->max_descendants, descendants); cgroup_kn_unlock(of->kn); @@ -3777,7 +3777,7 @@ static ssize_t cgroup_max_depth_write(struct kernfs_open_file *of, if (!cgrp) return -ENOENT; - cgrp->max_depth = depth; + WRITE_ONCE(cgrp->max_depth, depth); cgroup_kn_unlock(of->kn); -- cgit v1.2.3 From 45a13ba52c82dbec9715222c51e629e85daa37d7 Mon Sep 17 00:00:00 2001 From: Zhan Xusheng Date: Tue, 26 May 2026 10:21:06 +0800 Subject: timers/migration: Update stale @online doc to @available Commit 8312cab5ff47 ("timers/migration: Rename 'online' bit to 'available'") renamed the 'online' field of struct tmigr_cpu to 'available'. The kernel doc comment above the struct still describes the old field name. Update it to reflect the actual field name and use the 'available' wording in the description. Fixes: 8312cab5ff47 ("timers/migration: Rename 'online' bit to 'available'") Signed-off-by: Zhan Xusheng Signed-off-by: Thomas Gleixner Link: https://patch.msgid.link/20260526022106.1302279-1-zhanxusheng@xiaomi.com --- kernel/time/timer_migration.h | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timer_migration.h b/kernel/time/timer_migration.h index 291bfb6adfc3..ea8db95fc63e 100644 --- a/kernel/time/timer_migration.h +++ b/kernel/time/timer_migration.h @@ -94,15 +94,17 @@ struct tmigr_group { /** * struct tmigr_cpu - timer migration per CPU group * @lock: Lock protecting the tmigr_cpu group information - * @online: Indicates whether the CPU is online; In deactivate path - * it is required to know whether the migrator in the top - * level group is to be set offline, while a timer is - * pending. Then another online CPU needs to be notified to - * take over the migrator role. Furthermore the information - * is required in CPU hotplug path as the CPU is able to go - * idle before the timer migration hierarchy hotplug AP is - * reached. During this phase, the CPU has to handle the + * @available: Indicates whether the CPU is available for handling + * global timers. In the deactivate path it is required to + * know whether the migrator in the top level group is to + * be set offline, while a timer is pending. Then another + * available CPU needs to be notified to take over the + * migrator role. Furthermore the information is required + * in the CPU hotplug path as the CPU is able to go idle + * before the timer migration hierarchy hotplug callback is + * reached. During this phase, the CPU has to handle the * global timers on its own and must not act as a migrator. + * @idle: Indicates whether the CPU is idle in the timer migration * hierarchy * @remote: Is set when timers of the CPU are expired remotely -- cgit v1.2.3 From fc99547a8bda22a6a489284641385d8dcfb3ecd8 Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Mon, 25 May 2026 15:39:46 -0700 Subject: bpf: Factor out stack_map build ID helpers Factor out helpers from stack_map_get_build_id_offset() in preparation for adding a sleepable build ID resolution path: stack_map_build_id_set_ip(), stack_map_build_id_offset(), and stack_map_build_id_set_valid(). While here, refactor stack_map_get_build_id_offset(): * use continue-driven control flow in the main loop and remove build_id_valid label * update prev_vma and prev_build_id on the fall-back-to-IP branch so the cache reflects the actual VMA seen on the previous IP [1] * guard fetch_build_id() with vma_is_anonymous() [2] to skip parse attempts that would otherwise fail the ELF magic check [1] https://lore.kernel.org/bpf/CAEf4Bzac9uWWqBvzH0iFzKvJcq3vxscZ3pKm0sUHmN-F-z9wVQ@mail.gmail.com/ [2] https://lore.kernel.org/bpf/226398c1ff3f2b686c0aeb010408d85fb15df13f9ff60a045bee31e79b9e41e9@mail.kernel.org/ Signed-off-by: Ihor Solodrai Signed-off-by: Andrii Nakryiko Acked-by: Mykyta Yatsenko Link: https://lore.kernel.org/bpf/20260525223948.1920986-2-ihor.solodrai@linux.dev --- kernel/bpf/stackmap.c | 57 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 17 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index da3d328f5c15..e23be7d44503 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -152,6 +152,28 @@ static int fetch_build_id(struct vm_area_struct *vma, unsigned char *build_id, b : build_id_parse_nofault(vma, build_id, NULL); } +static inline void stack_map_build_id_set_ip(struct bpf_stack_build_id *id) +{ + id->status = BPF_STACK_BUILD_ID_IP; + memset(id->build_id, 0, BUILD_ID_SIZE_MAX); +} + +static inline u64 stack_map_build_id_offset(unsigned long vm_pgoff, + unsigned long vm_start, u64 ip) +{ + return (vm_pgoff << PAGE_SHIFT) + ip - vm_start; +} + +static inline void stack_map_build_id_set_valid(struct bpf_stack_build_id *id, + u64 offset, + const unsigned char *build_id) +{ + id->status = BPF_STACK_BUILD_ID_VALID; + id->offset = offset; + if (id->build_id != build_id) + memcpy(id->build_id, build_id, BUILD_ID_SIZE_MAX); +} + /* * Expects all id_offs[i].ip values to be set to correct initial IPs. * They will be subsequently: @@ -165,44 +187,45 @@ static int fetch_build_id(struct vm_area_struct *vma, unsigned char *build_id, b static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, u32 trace_nr, bool user, bool may_fault) { - int i; struct mmap_unlock_irq_work *work = NULL; bool irq_work_busy = bpf_mmap_unlock_get_irq_work(&work); + bool has_user_ctx = user && current && current->mm; struct vm_area_struct *vma, *prev_vma = NULL; - const char *prev_build_id; + const unsigned char *prev_build_id = NULL; + int i; /* If the irq_work is in use, fall back to report ips. Same * fallback is used for kernel stack (!user) on a stackmap with * build_id. */ - if (!user || !current || !current->mm || irq_work_busy || - !mmap_read_trylock(current->mm)) { + if (!has_user_ctx || irq_work_busy || !mmap_read_trylock(current->mm)) { /* cannot access current->mm, fall back to ips */ - for (i = 0; i < trace_nr; i++) { - id_offs[i].status = BPF_STACK_BUILD_ID_IP; - memset(id_offs[i].build_id, 0, BUILD_ID_SIZE_MAX); - } + for (i = 0; i < trace_nr; i++) + stack_map_build_id_set_ip(&id_offs[i]); return; } for (i = 0; i < trace_nr; i++) { u64 ip = READ_ONCE(id_offs[i].ip); + u64 offset; - if (range_in_vma(prev_vma, ip, ip)) { + if (prev_build_id && range_in_vma(prev_vma, ip, ip)) { vma = prev_vma; - memcpy(id_offs[i].build_id, prev_build_id, BUILD_ID_SIZE_MAX); - goto build_id_valid; + offset = stack_map_build_id_offset(vma->vm_pgoff, vma->vm_start, ip); + stack_map_build_id_set_valid(&id_offs[i], offset, prev_build_id); + continue; } vma = find_vma(current->mm, ip); - if (!vma || fetch_build_id(vma, id_offs[i].build_id, may_fault)) { + if (!vma || vma_is_anonymous(vma) || + fetch_build_id(vma, id_offs[i].build_id, may_fault)) { /* per entry fall back to ips */ - id_offs[i].status = BPF_STACK_BUILD_ID_IP; - memset(id_offs[i].build_id, 0, BUILD_ID_SIZE_MAX); + stack_map_build_id_set_ip(&id_offs[i]); + prev_vma = vma; + prev_build_id = NULL; continue; } -build_id_valid: - id_offs[i].offset = (vma->vm_pgoff << PAGE_SHIFT) + ip - vma->vm_start; - id_offs[i].status = BPF_STACK_BUILD_ID_VALID; + offset = stack_map_build_id_offset(vma->vm_pgoff, vma->vm_start, ip); + stack_map_build_id_set_valid(&id_offs[i], offset, id_offs[i].build_id); prev_vma = vma; prev_build_id = id_offs[i].build_id; } -- cgit v1.2.3 From fad3021faf7b0b64e9daea41c5662b65c8ad7379 Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Mon, 25 May 2026 15:39:47 -0700 Subject: bpf: Avoid faultable build ID reads under mm locks Sleepable build ID parsing can block in __kernel_read() [1], so the stackmap sleepable path must not call it while holding mmap_lock or a per-VMA read lock. The issue and the fix are conceptually similar to a recent procfs patch [2]. A similar VMA locking pattern has already been used in PROCMAP_QUERY [3]. Resolve each covered VMA with a stable read-side reference, preferring lock_vma_under_rcu() and falling back to mmap_read_trylock() only long enough to acquire the VMA read lock. Take a reference to the backing file, drop the VMA lock, and then parse the build ID through (sleepable) build_id_parse_file(). We have to use mmap_read_trylock() (and give up on failure) in this context because taking mmap_read_lock() is generally unsafe on code paths reachable from BPF programs [4], and may lead to deadlocks. [1] https://lore.kernel.org/all/20251218005818.614819-1-shakeel.butt@linux.dev/ [2] https://lore.kernel.org/all/20260128183232.2854138-1-andrii@kernel.org/ [3] https://lore.kernel.org/all/20250808152850.2580887-1-surenb@google.com/ [4] https://lore.kernel.org/bpf/2895ecd8-df1e-4cc0-b9f9-aef893dc2360@linux.dev/ Fixes: d4dd9775ec24 ("bpf: wire up sleepable bpf_get_stack() and bpf_get_task_stack() helpers") Suggested-by: Puranjay Mohan Signed-off-by: Ihor Solodrai Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260525223948.1920986-3-ihor.solodrai@linux.dev --- kernel/bpf/stackmap.c | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index e23be7d44503..c53cfd9a67cf 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -9,6 +9,7 @@ #include #include #include +#include #include "percpu_freelist.h" #include "mmap_unlock_work.h" @@ -174,6 +175,109 @@ static inline void stack_map_build_id_set_valid(struct bpf_stack_build_id *id, memcpy(id->build_id, build_id, BUILD_ID_SIZE_MAX); } +struct stack_map_vma_lock { + struct vm_area_struct *vma; + struct mm_struct *mm; +}; + +/* + * Acquire a stable read-side reference on the VMA covering @ip. + * + * With CONFIG_PER_VMA_LOCK=y this returns a VMA with its per-VMA read + * lock held and mmap_lock dropped, so the caller may sleep. + * + * With CONFIG_PER_VMA_LOCK=n it returns a VMA with mmap_lock still + * held; the caller must snapshot any fields it needs and pin vm_file + * with get_file() before stack_map_unlock_vma() drops mmap_lock, as + * the VMA may be split, merged, or freed after that. + * + * Returns NULL on failure, in which case no lock is held. + */ +static struct vm_area_struct * +stack_map_lock_vma(struct stack_map_vma_lock *lock, unsigned long ip) +{ + struct mm_struct *mm = lock->mm; + struct vm_area_struct *vma; + + /* noop under !CONFIG_PER_VMA_LOCK */ + vma = lock_vma_under_rcu(mm, ip); + if (vma) { + lock->vma = vma; + return vma; + } + + /* + * Taking mmap_read_lock() is unsafe here, because the caller BPF + * program might already hold it, causing a deadlock. + */ + if (!mmap_read_trylock(mm)) + return NULL; + + vma = vma_lookup(mm, ip); + if (!vma) { + mmap_read_unlock(mm); + return NULL; + } + +#ifdef CONFIG_PER_VMA_LOCK + if (!vma_start_read_locked(vma)) { + mmap_read_unlock(mm); + return NULL; + } + mmap_read_unlock(mm); +#endif + + lock->vma = vma; + return vma; +} + +static void stack_map_unlock_vma(struct stack_map_vma_lock *lock) +{ +#ifdef CONFIG_PER_VMA_LOCK + vma_end_read(lock->vma); +#else + mmap_read_unlock(lock->mm); +#endif + lock->vma = NULL; +} + +static void stack_map_get_build_id_offset_sleepable(struct bpf_stack_build_id *id_offs, + u32 trace_nr) +{ + struct mm_struct *mm = current->mm; + struct stack_map_vma_lock lock = { .mm = mm }; + struct vm_area_struct *vma; + struct file *file; + u64 offset; + u64 ip; + + for (u32 i = 0; i < trace_nr; i++) { + ip = READ_ONCE(id_offs[i].ip); + + vma = stack_map_lock_vma(&lock, ip); + if (!vma) { + stack_map_build_id_set_ip(&id_offs[i]); + continue; + } + if (vma_is_anonymous(vma) || !vma->vm_file) { + stack_map_build_id_set_ip(&id_offs[i]); + stack_map_unlock_vma(&lock); + continue; + } + + file = get_file(vma->vm_file); + offset = stack_map_build_id_offset(vma->vm_pgoff, vma->vm_start, ip); + stack_map_unlock_vma(&lock); + + /* build_id_parse_file() may block on filesystem reads */ + if (build_id_parse_file(file, id_offs[i].build_id, NULL)) + stack_map_build_id_set_ip(&id_offs[i]); + else + stack_map_build_id_set_valid(&id_offs[i], offset, id_offs[i].build_id); + fput(file); + } +} + /* * Expects all id_offs[i].ip values to be set to correct initial IPs. * They will be subsequently: @@ -194,6 +298,11 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, const unsigned char *prev_build_id = NULL; int i; + if (may_fault && has_user_ctx) { + stack_map_get_build_id_offset_sleepable(id_offs, trace_nr); + return; + } + /* If the irq_work is in use, fall back to report ips. Same * fallback is used for kernel stack (!user) on a stackmap with * build_id. -- cgit v1.2.3 From 5e9099d8ff24ec32aa5af872a4d84d086bd70579 Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Mon, 25 May 2026 15:39:48 -0700 Subject: bpf: Cache build IDs in sleepable stackmap path Stack traces often contain adjacent IPs from the same VMA or from different VMAs backed by the same ELF file. Cache the last successfully parsed build id together with the resolved VMA range and backing file so the sleepable build id path can avoid repeated VMA locking and file parsing in common cases. Suggested-by: Mykyta Yatsenko Signed-off-by: Ihor Solodrai Signed-off-by: Andrii Nakryiko Acked-by: Mykyta Yatsenko Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260525223948.1920986-4-ihor.solodrai@linux.dev --- kernel/bpf/stackmap.c | 61 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 55 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index c53cfd9a67cf..77ba03216c09 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -246,6 +246,14 @@ static void stack_map_get_build_id_offset_sleepable(struct bpf_stack_build_id *i { struct mm_struct *mm = current->mm; struct stack_map_vma_lock lock = { .mm = mm }; + struct { + struct file *file; + const unsigned char *build_id; + unsigned long vm_start; + unsigned long vm_end; + unsigned long vm_pgoff; + } cache = {}; + unsigned long vm_pgoff, vm_start, vm_end; struct vm_area_struct *vma; struct file *file; u64 offset; @@ -254,6 +262,17 @@ static void stack_map_get_build_id_offset_sleepable(struct bpf_stack_build_id *i for (u32 i = 0; i < trace_nr; i++) { ip = READ_ONCE(id_offs[i].ip); + /* + * Range cache fast path: if ip falls within the previously + * resolved VMA range, reuse the cache build_id without + * re-acquiring the VMA lock. + */ + if (cache.build_id && ip >= cache.vm_start && ip < cache.vm_end) { + offset = stack_map_build_id_offset(cache.vm_pgoff, cache.vm_start, ip); + stack_map_build_id_set_valid(&id_offs[i], offset, cache.build_id); + continue; + } + vma = stack_map_lock_vma(&lock, ip); if (!vma) { stack_map_build_id_set_ip(&id_offs[i]); @@ -265,17 +284,47 @@ static void stack_map_get_build_id_offset_sleepable(struct bpf_stack_build_id *i continue; } - file = get_file(vma->vm_file); - offset = stack_map_build_id_offset(vma->vm_pgoff, vma->vm_start, ip); + file = vma->vm_file; + vm_pgoff = vma->vm_pgoff; + vm_start = vma->vm_start; + vm_end = vma->vm_end; + offset = stack_map_build_id_offset(vm_pgoff, vm_start, ip); + + /* + * Same backing file as previous (e.g. different VMAs + * of the same ELF binary). Reuse the cache build_id. + */ + if (file == cache.file) { + stack_map_unlock_vma(&lock); + stack_map_build_id_set_valid(&id_offs[i], offset, cache.build_id); + cache.vm_start = vm_start; + cache.vm_end = vm_end; + cache.vm_pgoff = vm_pgoff; + continue; + } + + file = get_file(file); stack_map_unlock_vma(&lock); /* build_id_parse_file() may block on filesystem reads */ - if (build_id_parse_file(file, id_offs[i].build_id, NULL)) + if (build_id_parse_file(file, id_offs[i].build_id, NULL)) { stack_map_build_id_set_ip(&id_offs[i]); - else - stack_map_build_id_set_valid(&id_offs[i], offset, id_offs[i].build_id); - fput(file); + fput(file); + continue; + } + + stack_map_build_id_set_valid(&id_offs[i], offset, id_offs[i].build_id); + if (cache.file) + fput(cache.file); + cache.file = file; + cache.build_id = id_offs[i].build_id; + cache.vm_start = vm_start; + cache.vm_end = vm_end; + cache.vm_pgoff = vm_pgoff; } + + if (cache.file) + fput(cache.file); } /* -- cgit v1.2.3 From eb3bd277b37cd435d26a44a40d8f7c87ff16feb6 Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Fri, 22 May 2026 13:08:58 -0400 Subject: ring-buffer: Skip invalid sub-buffers when validating persistent ring buffer Skip invalid sub-buffers when validating the persistent ring buffer instead of discarding the entire ring buffer. Only skipped buffers are invalidated (cleared). If the cache data in memory fails to be synchronized during a reboot, the persistent ring buffer may become partially corrupted, but other sub-buffers may still contain readable event data. Only discard the subbuffers that are found to be corrupted. Link: https://lore.kernel.org/all/20260520185018.051228084@kernel.org/ Link: https://patch.msgid.link/20260522171050.914418536@kernel.org Signed-off-by: Masami Hiramatsu (Google) [SDR: Fixed max_loops in rb_iter_peek() as well ] Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 120 +++++++++++++++++++++++++++------------------ 1 file changed, 73 insertions(+), 47 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 7b07d2004cc6..6afd8ea5081a 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -370,6 +370,12 @@ static __always_inline unsigned int rb_page_commit(struct buffer_page *bpage) return local_read(&bpage->page->commit); } +/* Size is determined by what has been committed */ +static __always_inline unsigned int rb_page_size(struct buffer_page *bpage) +{ + return rb_page_commit(bpage) & ~RB_MISSED_MASK; +} + static void free_buffer_page(struct buffer_page *bpage) { /* Range pages are not to be freed */ @@ -1762,7 +1768,6 @@ static bool rb_cpu_meta_valid(struct ring_buffer_cpu_meta *meta, int cpu, unsigned long *subbuf_mask) { int subbuf_size = PAGE_SIZE; - struct buffer_data_page *subbuf; unsigned long buffers_start; unsigned long buffers_end; int i; @@ -1770,6 +1775,11 @@ static bool rb_cpu_meta_valid(struct ring_buffer_cpu_meta *meta, int cpu, if (!subbuf_mask) return false; + if (meta->subbuf_size != PAGE_SIZE) { + pr_info("Ring buffer boot meta [%d] invalid subbuf_size\n", cpu); + return false; + } + buffers_start = meta->first_buffer; buffers_end = meta->first_buffer + (subbuf_size * meta->nr_subbufs); @@ -1786,11 +1796,12 @@ static bool rb_cpu_meta_valid(struct ring_buffer_cpu_meta *meta, int cpu, return false; } - subbuf = rb_subbufs_from_meta(meta); - bitmap_clear(subbuf_mask, 0, meta->nr_subbufs); - /* Is the meta buffers and the subbufs themselves have correct data? */ + /* + * Ensure the meta::buffers array has correct data. The data in each subbufs + * are checked later in rb_meta_validate_events(). + */ for (i = 0; i < meta->nr_subbufs; i++) { if (meta->buffers[i] < 0 || meta->buffers[i] >= meta->nr_subbufs) { @@ -1798,18 +1809,12 @@ static bool rb_cpu_meta_valid(struct ring_buffer_cpu_meta *meta, int cpu, return false; } - if ((unsigned)local_read(&subbuf->commit) > subbuf_size) { - pr_info("Ring buffer boot meta [%d] buffer invalid commit\n", cpu); - return false; - } - if (test_bit(meta->buffers[i], subbuf_mask)) { pr_info("Ring buffer boot meta [%d] array has duplicates\n", cpu); return false; } set_bit(meta->buffers[i], subbuf_mask); - subbuf = (void *)subbuf + subbuf_size; } return true; @@ -1873,13 +1878,22 @@ static int rb_read_data_buffer(struct buffer_data_page *dpage, int tail, int cpu return events; } -static int rb_validate_buffer(struct buffer_data_page *dpage, int cpu) +static int rb_validate_buffer(struct buffer_data_page *dpage, int cpu, + struct ring_buffer_cpu_meta *meta) { unsigned long long ts; + unsigned long tail; u64 delta; - int tail; - tail = local_read(&dpage->commit); + /* + * When a sub-buffer is recovered from a read, the commit value may + * have RB_MISSED_* bits set, as these bits are reset on reuse. + * Even after clearing these bits, a commit value greater than the + * subbuf_size is considered invalid. + */ + tail = local_read(&dpage->commit) & ~RB_MISSED_MASK; + if (tail > meta->subbuf_size - BUF_PAGE_HDR_SIZE) + return -1; return rb_read_data_buffer(dpage, tail, cpu, &ts, &delta); } @@ -1890,6 +1904,7 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) struct buffer_page *head_page, *orig_head, *orig_reader; unsigned long entry_bytes = 0; unsigned long entries = 0; + int discarded = 0; int ret; u64 ts; int i; @@ -1901,14 +1916,19 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) orig_reader = cpu_buffer->reader_page; /* Do the reader page first */ - ret = rb_validate_buffer(orig_reader->page, cpu_buffer->cpu); + ret = rb_validate_buffer(orig_reader->page, cpu_buffer->cpu, meta); if (ret < 0) { - pr_info("Ring buffer reader page is invalid\n"); - goto invalid; + pr_info("Ring buffer meta [%d] invalid reader page detected\n", + cpu_buffer->cpu); + discarded++; + /* Instead of discard whole ring buffer, discard only this sub-buffer. */ + local_set(&orig_reader->entries, 0); + local_set(&orig_reader->page->commit, 0); + } else { + entries += ret; + entry_bytes += rb_page_size(orig_reader); + local_set(&orig_reader->entries, ret); } - entries += ret; - entry_bytes += local_read(&orig_reader->page->commit); - local_set(&orig_reader->entries, ret); ts = head_page->page->time_stamp; @@ -1936,7 +1956,7 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) break; /* Stop rewind if the page is invalid. */ - ret = rb_validate_buffer(head_page->page, cpu_buffer->cpu); + ret = rb_validate_buffer(head_page->page, cpu_buffer->cpu, meta); if (ret < 0) break; @@ -1945,7 +1965,7 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) if (ret) local_inc(&cpu_buffer->pages_touched); entries += ret; - entry_bytes += rb_page_commit(head_page); + entry_bytes += rb_page_size(head_page); } if (i) pr_info("Ring buffer [%d] rewound %d pages\n", cpu_buffer->cpu, i); @@ -2015,21 +2035,24 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) if (head_page == orig_reader) continue; - ret = rb_validate_buffer(head_page->page, cpu_buffer->cpu); + ret = rb_validate_buffer(head_page->page, cpu_buffer->cpu, meta); if (ret < 0) { - pr_info("Ring buffer meta [%d] invalid buffer page\n", - cpu_buffer->cpu); - goto invalid; - } - - /* If the buffer has content, update pages_touched */ - if (ret) - local_inc(&cpu_buffer->pages_touched); - - entries += ret; - entry_bytes += local_read(&head_page->page->commit); - local_set(&head_page->entries, ret); + if (!discarded) + pr_info("Ring buffer meta [%d] invalid buffer page detected\n", + cpu_buffer->cpu); + discarded++; + /* Instead of discard whole ring buffer, discard only this sub-buffer. */ + local_set(&head_page->entries, 0); + local_set(&head_page->page->commit, 0); + } else { + /* If the buffer has content, update pages_touched */ + if (ret) + local_inc(&cpu_buffer->pages_touched); + entries += ret; + entry_bytes += rb_page_size(head_page); + local_set(&head_page->entries, ret); + } if (head_page == cpu_buffer->commit_page) break; } @@ -2043,7 +2066,10 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) local_set(&cpu_buffer->entries, entries); local_set(&cpu_buffer->entries_bytes, entry_bytes); - pr_info("Ring buffer meta [%d] is from previous boot!\n", cpu_buffer->cpu); + pr_info("Ring buffer meta [%d] is from previous boot!", cpu_buffer->cpu); + if (discarded) + pr_cont(" (%d pages discarded)", discarded); + pr_cont("\n"); return; invalid: @@ -3330,12 +3356,6 @@ rb_iter_head_event(struct ring_buffer_iter *iter) return NULL; } -/* Size is determined by what has been committed */ -static __always_inline unsigned rb_page_size(struct buffer_page *bpage) -{ - return rb_page_commit(bpage) & ~RB_MISSED_MASK; -} - static __always_inline unsigned rb_commit_index(struct ring_buffer_per_cpu *cpu_buffer) { @@ -5636,8 +5656,9 @@ __rb_get_reader_page_from_remote(struct ring_buffer_per_cpu *cpu_buffer) static struct buffer_page * __rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) { - struct buffer_page *reader = NULL; + int max_loops = cpu_buffer->ring_meta ? cpu_buffer->nr_pages : 3; unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size); + struct buffer_page *reader = NULL; unsigned long overwrite; unsigned long flags; int nr_loops = 0; @@ -5649,11 +5670,14 @@ __rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) again: /* * This should normally only loop twice. But because the - * start of the reader inserts an empty page, it causes - * a case where we will loop three times. There should be no - * reason to loop four times (that I know of). + * start of the reader inserts an empty page, it causes a + * case where we will loop three times. There should be no + * reason to loop four times unless the ring buffer is a + * recovered persistent ring buffer. For persistent ring buffers, + * invalid pages are reset during recovery, so there may be more + * than 3 contiguous pages can be empty, but less than nr_pages. */ - if (RB_WARN_ON(cpu_buffer, ++nr_loops > 3)) { + if (RB_WARN_ON(cpu_buffer, ++nr_loops > max_loops)) { reader = NULL; goto out; } @@ -5950,12 +5974,14 @@ rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts) struct ring_buffer_per_cpu *cpu_buffer; struct ring_buffer_event *event; int nr_loops = 0; + int max_loops; if (ts) *ts = 0; cpu_buffer = iter->cpu_buffer; buffer = cpu_buffer->buffer; + max_loops = cpu_buffer->ring_meta ? cpu_buffer->nr_pages : 3; /* * Check if someone performed a consuming read to the buffer @@ -5978,7 +6004,7 @@ rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts) * the ring buffer with an active write as the consumer is. * Do not warn if the three failures is reached. */ - if (++nr_loops > 3) + if (++nr_loops > max_loops) return NULL; if (rb_per_cpu_empty(cpu_buffer)) -- cgit v1.2.3 From c8a7d4b4723a21e7464efe86dcf80627e0b4df33 Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Fri, 22 May 2026 13:08:59 -0400 Subject: ring-buffer: Skip invalid sub-buffers when rewinding persistent ring buffer Skip invalid sub-buffers when rewinding the persistent ring buffer instead of stopping the rewinding the ring buffer. The skipped buffers are cleared. To ensure the rewinding stops at the unused page, this also clears buffer_data_page::time_stamp when tracing resets the buffer. This allows us to identify unused pages and empty pages. Link: https://patch.msgid.link/20260522171051.091265852@kernel.org Signed-off-by: Masami Hiramatsu (Google) [ SDR: Have reader_page still get evaluated if header_page fails ] Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 107 +++++++++++++++++++++++++++++---------------- 1 file changed, 69 insertions(+), 38 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 6afd8ea5081a..c10cf4ba91d6 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -363,6 +363,7 @@ struct buffer_page { static void rb_init_page(struct buffer_data_page *bpage) { local_set(&bpage->commit, 0); + bpage->time_stamp = 0; } static __always_inline unsigned int rb_page_commit(struct buffer_page *bpage) @@ -1878,12 +1879,14 @@ static int rb_read_data_buffer(struct buffer_data_page *dpage, int tail, int cpu return events; } -static int rb_validate_buffer(struct buffer_data_page *dpage, int cpu, - struct ring_buffer_cpu_meta *meta) +static int rb_validate_buffer(struct buffer_page *bpage, int cpu, + struct ring_buffer_cpu_meta *meta, u64 prev_ts, u64 next_ts) { + struct buffer_data_page *dpage = bpage->page; unsigned long long ts; unsigned long tail; u64 delta; + int ret; /* * When a sub-buffer is recovered from a read, the commit value may @@ -1892,9 +1895,27 @@ static int rb_validate_buffer(struct buffer_data_page *dpage, int cpu, * subbuf_size is considered invalid. */ tail = local_read(&dpage->commit) & ~RB_MISSED_MASK; - if (tail > meta->subbuf_size - BUF_PAGE_HDR_SIZE) - return -1; - return rb_read_data_buffer(dpage, tail, cpu, &ts, &delta); + if (tail <= meta->subbuf_size - BUF_PAGE_HDR_SIZE) + ret = rb_read_data_buffer(dpage, tail, cpu, &ts, &delta); + else + ret = -1; + + /* + * The timestamp must be greater than @prev_ts and smaller than @next_ts. + * Since this function works in both forward (verify) and reverse (unwind) + * loop, we don't know both @prev_ts and @next_ts at the same time. + * So use the known boundary as the boundary. + */ + if (ret < 0 || (prev_ts && prev_ts > ts) || (next_ts && ts > next_ts)) { + local_set(&bpage->entries, 0); + local_set(&dpage->commit, 0); + dpage->time_stamp = prev_ts ? prev_ts : next_ts; + ret = -1; + } else { + local_set(&bpage->entries, ret); + } + + return ret; } /* If the meta data has been validated, now validate the events */ @@ -1905,6 +1926,7 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) unsigned long entry_bytes = 0; unsigned long entries = 0; int discarded = 0; + bool skip = false; int ret; u64 ts; int i; @@ -1915,25 +1937,35 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) orig_head = head_page = cpu_buffer->head_page; orig_reader = cpu_buffer->reader_page; - /* Do the reader page first */ - ret = rb_validate_buffer(orig_reader->page, cpu_buffer->cpu, meta); + /* Do the head page first */ + ret = rb_validate_buffer(head_page, cpu_buffer->cpu, meta, 0, 0); + if (ret < 0) { + pr_info("Ring buffer meta [%d] invalid head page detected\n", + cpu_buffer->cpu); + /* Don't bother rewinding */ + skip = true; + ts = 0; + } else { + ts = head_page->page->time_stamp; + } + + /* Do the reader page - reader must be previous to head. */ + ret = rb_validate_buffer(orig_reader, cpu_buffer->cpu, meta, 0, ts); if (ret < 0) { pr_info("Ring buffer meta [%d] invalid reader page detected\n", cpu_buffer->cpu); discarded++; - /* Instead of discard whole ring buffer, discard only this sub-buffer. */ - local_set(&orig_reader->entries, 0); - local_set(&orig_reader->page->commit, 0); } else { entries += ret; entry_bytes += rb_page_size(orig_reader); - local_set(&orig_reader->entries, ret); + ts = orig_reader->page->time_stamp; } - ts = head_page->page->time_stamp; + if (skip) + goto skip_rewind; /* - * Try to rewind the head so that we can read the pages which already + * Try to rewind the head so that we can read the pages which are already * read in the previous boot. */ if (head_page == cpu_buffer->tail_page) @@ -1946,26 +1978,27 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) if (head_page == cpu_buffer->tail_page) break; - /* Ensure the page has older data than head. */ - if (ts < head_page->page->time_stamp) + /* Rewind until unused page (no timestamp, no commit). */ + if (!head_page->page->time_stamp && rb_page_commit(head_page) == 0) break; - ts = head_page->page->time_stamp; - /* Ensure the page has correct timestamp and some data. */ - if (!ts || rb_page_commit(head_page) == 0) - break; - - /* Stop rewind if the page is invalid. */ - ret = rb_validate_buffer(head_page->page, cpu_buffer->cpu, meta); - if (ret < 0) - break; - - /* Recover the number of entries and update stats. */ - local_set(&head_page->entries, ret); - if (ret) - local_inc(&cpu_buffer->pages_touched); - entries += ret; - entry_bytes += rb_page_size(head_page); + /* + * Skip if the page is invalid, or its timestamp is newer than the + * previous valid page. + */ + ret = rb_validate_buffer(head_page, cpu_buffer->cpu, meta, 0, ts); + if (ret < 0) { + if (!discarded) + pr_info("Ring buffer meta [%d] invalid buffer page detected\n", + cpu_buffer->cpu); + discarded++; + } else { + entries += ret; + entry_bytes += rb_page_size(head_page); + if (ret > 0) + local_inc(&cpu_buffer->pages_touched); + ts = head_page->page->time_stamp; + } } if (i) pr_info("Ring buffer [%d] rewound %d pages\n", cpu_buffer->cpu, i); @@ -2027,6 +2060,7 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) /* Nothing more to do, the only page is the reader page */ goto done; } + ts = head_page->page->time_stamp; /* Iterate until finding the commit page */ for (i = 0; i < meta->nr_subbufs + 1; i++, rb_inc_page(&head_page)) { @@ -2035,15 +2069,12 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) if (head_page == orig_reader) continue; - ret = rb_validate_buffer(head_page->page, cpu_buffer->cpu, meta); + ret = rb_validate_buffer(head_page, cpu_buffer->cpu, meta, ts, 0); if (ret < 0) { if (!discarded) pr_info("Ring buffer meta [%d] invalid buffer page detected\n", cpu_buffer->cpu); discarded++; - /* Instead of discard whole ring buffer, discard only this sub-buffer. */ - local_set(&head_page->entries, 0); - local_set(&head_page->page->commit, 0); } else { /* If the buffer has content, update pages_touched */ if (ret) @@ -2051,7 +2082,7 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) entries += ret; entry_bytes += rb_page_size(head_page); - local_set(&head_page->entries, ret); + ts = head_page->page->time_stamp; } if (head_page == cpu_buffer->commit_page) break; @@ -2079,12 +2110,12 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) /* Reset the reader page */ local_set(&cpu_buffer->reader_page->entries, 0); - local_set(&cpu_buffer->reader_page->page->commit, 0); + rb_init_page(cpu_buffer->reader_page->page); /* Reset all the subbuffers */ for (i = 0; i < meta->nr_subbufs - 1; i++, rb_inc_page(&head_page)) { local_set(&head_page->entries, 0); - local_set(&head_page->page->commit, 0); + rb_init_page(head_page->page); } } -- cgit v1.2.3 From 68413a36f0051bd7ba7ea37ae2167951aeae5bd2 Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Fri, 22 May 2026 13:09:00 -0400 Subject: ring-buffer: Add persistent ring buffer invalid-page inject test Add a self-corrupting test for the persistent ring buffer. This will inject an erroneous value to some sub-buffer pages (where the index is even or multiples of 5) in the persistent ring buffer when the kernel panics, and checks whether the number of detected invalid pages and the total entry_bytes are the same as the recorded values after reboot. This ensures that the kernel can correctly recover a partially corrupted persistent ring buffer after a reboot or panic. The test only runs on the persistent ring buffer whose name is "ptracingtest". The user has to fill it with events before a kernel panic. To run the test, enable CONFIG_RING_BUFFER_PERSISTENT_INJECT and add the following kernel cmdline: reserve_mem=20M:2M:trace trace_instance=ptracingtest^traceoff@trace panic=1 Run the following commands after the 1st boot: cd /sys/kernel/tracing/instances/ptracingtest echo 1 > tracing_on echo 1 > events/enable sleep 3 echo c > /proc/sysrq-trigger After panic message, the kernel will reboot and run the verification on the persistent ring buffer, e.g. Ring buffer meta [2] invalid buffer page detected Ring buffer meta [2] is from previous boot! (318 pages discarded) Ring buffer testing [2] invalid pages: PASSED (318/318) Ring buffer testing [2] entry_bytes: PASSED (1300476/1300476) Link: https://patch.msgid.link/20260522171051.260140328@kernel.org Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt --- kernel/trace/Kconfig | 34 ++++++++++++++++++++ kernel/trace/ring_buffer.c | 79 ++++++++++++++++++++++++++++++++++++++++++++++ kernel/trace/trace.c | 4 +++ 3 files changed, 117 insertions(+) (limited to 'kernel') diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index e130da35808f..084f34dc6c9f 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig @@ -1202,6 +1202,40 @@ config RING_BUFFER_VALIDATE_TIME_DELTAS Only say Y if you understand what this does, and you still want it enabled. Otherwise say N +config RING_BUFFER_PERSISTENT_INJECT + bool "Enable persistent ring buffer error injection test" + depends on RING_BUFFER + help + This option will have the kernel check if the persistent ring + buffer is named "ptracingtest". and if so, it will corrupt some + of its pages on a kernel panic. This is used to test if the + persistent ring buffer can recover from some of its sub-buffers + being corrupted. + To use this, boot a kernel with a "ptracingtest" persistent + ring buffer, e.g. + + reserve_mem=20M:2M:trace trace_instance=ptracingtest@trace panic=1 + + And after the 1st boot, run the following commands: + + cd /sys/kernel/tracing/instances/ptracingtest + echo 1 > events/enable + echo 1 > tracing_on + sleep 3 + echo c > /proc/sysrq-trigger + + After the panic message, the kernel will reboot and will show + the test results in the console output. + + Note that events for the test ring buffer needs to be enabled + prior to crashing the kernel so that the ring buffer has content + that the test will corrupt. + As the test will corrupt events in the "ptracingtest" persistent + ring buffer, it should not be used for any other purpose other + than this test. + + If unsure, say N + config MMIOTRACE_TEST tristate "Test module for mmiotrace" depends on MMIOTRACE && m diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index c10cf4ba91d6..dc603d9c9414 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -64,6 +64,10 @@ struct ring_buffer_cpu_meta { unsigned long commit_buffer; __u32 subbuf_size; __u32 nr_subbufs; +#ifdef CONFIG_RING_BUFFER_PERSISTENT_INJECT + __u32 nr_invalid; + __u32 entry_bytes; +#endif int buffers[]; }; @@ -2101,6 +2105,21 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) if (discarded) pr_cont(" (%d pages discarded)", discarded); pr_cont("\n"); + +#ifdef CONFIG_RING_BUFFER_PERSISTENT_INJECT + if (meta->nr_invalid) + pr_warn("Ring buffer testing [%d] invalid pages: %s (%d/%d)\n", + cpu_buffer->cpu, + (discarded == meta->nr_invalid) ? "PASSED" : "FAILED", + discarded, meta->nr_invalid); + if (meta->entry_bytes) + pr_warn("Ring buffer testing [%d] entry_bytes: %s (%ld/%ld)\n", + cpu_buffer->cpu, + (entry_bytes == meta->entry_bytes) ? "PASSED" : "FAILED", + (long)entry_bytes, (long)meta->entry_bytes); + meta->nr_invalid = 0; + meta->entry_bytes = 0; +#endif return; invalid: @@ -2581,12 +2600,72 @@ static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer) kfree(cpu_buffer); } +#ifdef CONFIG_RING_BUFFER_PERSISTENT_INJECT +static void rb_test_inject_invalid_pages(struct trace_buffer *buffer) +{ + struct ring_buffer_per_cpu *cpu_buffer; + struct ring_buffer_cpu_meta *meta; + struct buffer_data_page *dpage; + unsigned long entry_bytes = 0; + unsigned long ptr; + int subbuf_size; + int invalid = 0; + int cpu; + int i; + + if (!(buffer->flags & RB_FL_TESTING)) + return; + + guard(preempt)(); + cpu = smp_processor_id(); + + cpu_buffer = buffer->buffers[cpu]; + if (!cpu_buffer) + return; + meta = cpu_buffer->ring_meta; + if (!meta) + return; + + ptr = (unsigned long)rb_subbufs_from_meta(meta); + subbuf_size = meta->subbuf_size; + + for (i = 0; i < meta->nr_subbufs; i++) { + unsigned long idx = meta->buffers[i]; + + dpage = (void *)(ptr + idx * subbuf_size); + /* Skip unused pages */ + if (!local_read(&dpage->commit)) + continue; + + /* + * Invalidate even pages or multiples of 5. This will cause 3 + * contiguous invalidated(empty) pages. + */ + if (!(i & 0x1) || !(i % 5)) { + local_add(subbuf_size + 1, &dpage->commit); + invalid++; + } else { + /* Count total commit bytes. */ + entry_bytes += local_read(&dpage->commit) & ~RB_MISSED_MASK; + } + } + + pr_info("Inject invalidated %d pages on CPU%d, total size: %ld\n", + invalid, cpu, (long)entry_bytes); + meta->nr_invalid = invalid; + meta->entry_bytes = entry_bytes; +} +#else /* !CONFIG_RING_BUFFER_PERSISTENT_INJECT */ +#define rb_test_inject_invalid_pages(buffer) do { } while (0) +#endif + /* Stop recording on a persistent buffer and flush cache if needed. */ static int rb_flush_buffer_cb(struct notifier_block *nb, unsigned long event, void *data) { struct trace_buffer *buffer = container_of(nb, struct trace_buffer, flush_nb); ring_buffer_record_off(buffer); + rb_test_inject_invalid_pages(buffer); arch_ring_buffer_flush_range(buffer->range_addr_start, buffer->range_addr_end); return NOTIFY_DONE; } diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 6eb4d3097a4d..4573f65d68ce 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -8383,6 +8383,8 @@ static void setup_trace_scratch(struct trace_array *tr, memset(tscratch, 0, size); } +#define TRACE_TEST_PTRACING_NAME "ptracingtest" + int allocate_trace_buffer(struct trace_array *tr, struct array_buffer *buf, int size) { enum ring_buffer_flags rb_flags; @@ -8394,6 +8396,8 @@ int allocate_trace_buffer(struct trace_array *tr, struct array_buffer *buf, int buf->tr = tr; if (tr->range_addr_start && tr->range_addr_size) { + if (tr->name && !strcmp(tr->name, TRACE_TEST_PTRACING_NAME)) + rb_flags |= RB_FL_TESTING; /* Add scratch buffer to handle 128 modules */ buf->buffer = ring_buffer_alloc_range(size, rb_flags, 0, tr->range_addr_start, -- cgit v1.2.3 From e500acc34b556df7774213d95ff130801b9d6512 Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Fri, 22 May 2026 13:09:01 -0400 Subject: ring-buffer: Show commit numbers in buffer_meta file In addition to the index number, show the commit numbers of each data page in the per_cpu buffer_meta file. This is useful for understanding the current status of the persistent ring buffer. (Note that this file is shown only for persistent ring buffer and its backup instance) Link: https://patch.msgid.link/20260522171051.424411323@kernel.org Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index dc603d9c9414..88e613e78632 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -2231,6 +2231,7 @@ static int rbm_show(struct seq_file *m, void *v) struct ring_buffer_per_cpu *cpu_buffer = m->private; struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta; unsigned long val = (unsigned long)v; + struct buffer_data_page *dpage; if (val == 1) { seq_printf(m, "head_buffer: %d\n", @@ -2243,7 +2244,9 @@ static int rbm_show(struct seq_file *m, void *v) } val -= 2; - seq_printf(m, "buffer[%ld]: %d\n", val, meta->buffers[val]); + dpage = rb_range_buffer(cpu_buffer, val); + seq_printf(m, "buffer[%ld]: %d (commit: %ld)\n", + val, meta->buffers[val], dpage ? local_read(&dpage->commit) : -1); return 0; } -- cgit v1.2.3 From ea250b7ef9ff2af4241b4d7f078594e51d155b81 Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Fri, 22 May 2026 13:09:02 -0400 Subject: ring-buffer: Cleanup persistent ring buffer validation Cleanup rb_meta_validate_events() function to make it easier to read. This includes the following cleanups: - Introduce rb_validatation_state to hold working variables in validation. - Move repleated validation state updates into rb_validate_buffer(). - Move reader_page injection code outside of rb_meta_validate_events(). Link: https://patch.msgid.link/20260522171051.577231395@kernel.org Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 200 ++++++++++++++++++++++++--------------------- 1 file changed, 108 insertions(+), 92 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 88e613e78632..73f453ba12ce 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -1883,8 +1883,16 @@ static int rb_read_data_buffer(struct buffer_data_page *dpage, int tail, int cpu return events; } -static int rb_validate_buffer(struct buffer_page *bpage, int cpu, - struct ring_buffer_cpu_meta *meta, u64 prev_ts, u64 next_ts) +struct rb_validation_state { + unsigned long entries; + unsigned long entry_bytes; + int discarded; + u64 ts; +}; + +static int __rb_validate_buffer(struct buffer_page *bpage, int cpu, + struct ring_buffer_cpu_meta *meta, + u64 prev_ts, u64 next_ts) { struct buffer_data_page *dpage = bpage->page; unsigned long long ts; @@ -1922,17 +1930,95 @@ static int rb_validate_buffer(struct buffer_page *bpage, int cpu, return ret; } +/** + * rb_validate_buffer - validates a single buffer page and updates the state. + * @bpage: buffer page to validate + * @cpu_buffer: cpu_buffer this page belongs to + * @meta: meta of the cpu_buffer + * @state: validation state + * @prev_ts: previous buffer's timestamp (optional) + * @next_ts: next buffer's timestamp (optional) + * + * If the page is invalid (wrong event length or timestamp), it increments the + * discarded counter and warns it. Otherwise, it updates the validation state. + */ +static void rb_validate_buffer(struct buffer_page *bpage, + struct ring_buffer_per_cpu *cpu_buffer, + struct ring_buffer_cpu_meta *meta, + struct rb_validation_state *state, + u64 prev_ts, u64 next_ts) +{ + int ret; + + ret = __rb_validate_buffer(bpage, cpu_buffer->cpu, meta, prev_ts, next_ts); + if (ret < 0) { + if (!state->discarded) + pr_info("Ring buffer meta [%d] invalid buffer page detected\n", + cpu_buffer->cpu); + state->discarded++; + } else { + /* If the buffer has content, update pages_touched */ + if (ret) + local_inc(&cpu_buffer->pages_touched); + + state->entries += ret; + state->entry_bytes += rb_page_size(bpage); + state->ts = bpage->page->time_stamp; + } +} + +static void rb_meta_inject_reader_page(struct ring_buffer_per_cpu *cpu_buffer, + struct ring_buffer_cpu_meta *meta, + struct buffer_page *orig_head, + struct buffer_page *head_page) +{ + struct buffer_page *bpage = orig_head; + int i; + + rb_dec_page(&bpage); + /* + * Insert the reader_page before the original head page. + * Since the list encode RB_PAGE flags, general list + * operations should be avoided. + */ + cpu_buffer->reader_page->list.next = &orig_head->list; + cpu_buffer->reader_page->list.prev = orig_head->list.prev; + orig_head->list.prev = &cpu_buffer->reader_page->list; + bpage->list.next = &cpu_buffer->reader_page->list; + + /* Make the head_page the reader page */ + cpu_buffer->reader_page = head_page; + bpage = head_page; + rb_inc_page(&head_page); + head_page->list.prev = bpage->list.prev; + rb_dec_page(&bpage); + bpage->list.next = &head_page->list; + rb_set_list_to_head(&bpage->list); + cpu_buffer->pages = &head_page->list; + + cpu_buffer->head_page = head_page; + meta->head_buffer = (unsigned long)head_page->page; + + /* Reset all the indexes */ + bpage = cpu_buffer->reader_page; + meta->buffers[0] = rb_meta_subbuf_idx(meta, bpage->page); + bpage->id = 0; + + for (i = 1, bpage = head_page; i < meta->nr_subbufs; + i++, rb_inc_page(&bpage)) { + meta->buffers[i] = rb_meta_subbuf_idx(meta, bpage->page); + bpage->id = i; + } +} + /* If the meta data has been validated, now validate the events */ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) { struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta; struct buffer_page *head_page, *orig_head, *orig_reader; - unsigned long entry_bytes = 0; - unsigned long entries = 0; - int discarded = 0; + struct rb_validation_state state = { 0 }; bool skip = false; int ret; - u64 ts; int i; if (!meta || !meta->head_buffer) @@ -1942,28 +2028,19 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) orig_reader = cpu_buffer->reader_page; /* Do the head page first */ - ret = rb_validate_buffer(head_page, cpu_buffer->cpu, meta, 0, 0); + ret = __rb_validate_buffer(head_page, cpu_buffer->cpu, meta, 0, 0); if (ret < 0) { pr_info("Ring buffer meta [%d] invalid head page detected\n", cpu_buffer->cpu); /* Don't bother rewinding */ skip = true; - ts = 0; + state.ts = 0; } else { - ts = head_page->page->time_stamp; + state.ts = head_page->page->time_stamp; } /* Do the reader page - reader must be previous to head. */ - ret = rb_validate_buffer(orig_reader, cpu_buffer->cpu, meta, 0, ts); - if (ret < 0) { - pr_info("Ring buffer meta [%d] invalid reader page detected\n", - cpu_buffer->cpu); - discarded++; - } else { - entries += ret; - entry_bytes += rb_page_size(orig_reader); - ts = orig_reader->page->time_stamp; - } + rb_validate_buffer(orig_reader, cpu_buffer, meta, &state, 0, state.ts); if (skip) goto skip_rewind; @@ -1990,19 +2067,7 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) * Skip if the page is invalid, or its timestamp is newer than the * previous valid page. */ - ret = rb_validate_buffer(head_page, cpu_buffer->cpu, meta, 0, ts); - if (ret < 0) { - if (!discarded) - pr_info("Ring buffer meta [%d] invalid buffer page detected\n", - cpu_buffer->cpu); - discarded++; - } else { - entries += ret; - entry_bytes += rb_page_size(head_page); - if (ret > 0) - local_inc(&cpu_buffer->pages_touched); - ts = head_page->page->time_stamp; - } + rb_validate_buffer(head_page, cpu_buffer, meta, &state, 0, state.ts); } if (i) pr_info("Ring buffer [%d] rewound %d pages\n", cpu_buffer->cpu, i); @@ -2016,43 +2081,7 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) * into the location just before the original head page. */ if (head_page != orig_head) { - struct buffer_page *bpage = orig_head; - - rb_dec_page(&bpage); - /* - * Insert the reader_page before the original head page. - * Since the list encode RB_PAGE flags, general list - * operations should be avoided. - */ - cpu_buffer->reader_page->list.next = &orig_head->list; - cpu_buffer->reader_page->list.prev = orig_head->list.prev; - orig_head->list.prev = &cpu_buffer->reader_page->list; - bpage->list.next = &cpu_buffer->reader_page->list; - - /* Make the head_page the reader page */ - cpu_buffer->reader_page = head_page; - bpage = head_page; - rb_inc_page(&head_page); - head_page->list.prev = bpage->list.prev; - rb_dec_page(&bpage); - bpage->list.next = &head_page->list; - rb_set_list_to_head(&bpage->list); - cpu_buffer->pages = &head_page->list; - - cpu_buffer->head_page = head_page; - meta->head_buffer = (unsigned long)head_page->page; - - /* Reset all the indexes */ - bpage = cpu_buffer->reader_page; - meta->buffers[0] = rb_meta_subbuf_idx(meta, bpage->page); - bpage->id = 0; - - for (i = 1, bpage = head_page; i < meta->nr_subbufs; - i++, rb_inc_page(&bpage)) { - meta->buffers[i] = rb_meta_subbuf_idx(meta, bpage->page); - bpage->id = i; - } - + rb_meta_inject_reader_page(cpu_buffer, meta, orig_head, head_page); /* We'll restart verifying from orig_head */ head_page = orig_head; } @@ -2064,7 +2093,7 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) /* Nothing more to do, the only page is the reader page */ goto done; } - ts = head_page->page->time_stamp; + state.ts = head_page->page->time_stamp; /* Iterate until finding the commit page */ for (i = 0; i < meta->nr_subbufs + 1; i++, rb_inc_page(&head_page)) { @@ -2073,21 +2102,8 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) if (head_page == orig_reader) continue; - ret = rb_validate_buffer(head_page, cpu_buffer->cpu, meta, ts, 0); - if (ret < 0) { - if (!discarded) - pr_info("Ring buffer meta [%d] invalid buffer page detected\n", - cpu_buffer->cpu); - discarded++; - } else { - /* If the buffer has content, update pages_touched */ - if (ret) - local_inc(&cpu_buffer->pages_touched); + rb_validate_buffer(head_page, cpu_buffer, meta, &state, state.ts, 0); - entries += ret; - entry_bytes += rb_page_size(head_page); - ts = head_page->page->time_stamp; - } if (head_page == cpu_buffer->commit_page) break; } @@ -2098,25 +2114,25 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) goto invalid; } done: - local_set(&cpu_buffer->entries, entries); - local_set(&cpu_buffer->entries_bytes, entry_bytes); + local_set(&cpu_buffer->entries, state.entries); + local_set(&cpu_buffer->entries_bytes, state.entry_bytes); pr_info("Ring buffer meta [%d] is from previous boot!", cpu_buffer->cpu); - if (discarded) - pr_cont(" (%d pages discarded)", discarded); + if (state.discarded) + pr_cont(" (%d pages discarded)", state.discarded); pr_cont("\n"); #ifdef CONFIG_RING_BUFFER_PERSISTENT_INJECT if (meta->nr_invalid) pr_warn("Ring buffer testing [%d] invalid pages: %s (%d/%d)\n", cpu_buffer->cpu, - (discarded == meta->nr_invalid) ? "PASSED" : "FAILED", - discarded, meta->nr_invalid); + (state.discarded == meta->nr_invalid) ? "PASSED" : "FAILED", + state.discarded, meta->nr_invalid); if (meta->entry_bytes) pr_warn("Ring buffer testing [%d] entry_bytes: %s (%ld/%ld)\n", cpu_buffer->cpu, - (entry_bytes == meta->entry_bytes) ? "PASSED" : "FAILED", - (long)entry_bytes, (long)meta->entry_bytes); + (state.entry_bytes == meta->entry_bytes) ? "PASSED" : "FAILED", + (long)state.entry_bytes, (long)meta->entry_bytes); meta->nr_invalid = 0; meta->entry_bytes = 0; #endif -- cgit v1.2.3 From 7cf02d0aa6bd4a36f784f6e5dabf225e10b31f3a Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Fri, 22 May 2026 13:09:03 -0400 Subject: ring-buffer: Cleanup buffer_data_page related code Code cleanup related to buffer_data_page for readability, which includes: - Introduce rb_data_page_commit() and rb_data_page_size() - Use 'dpage' for buffer_data_page, instead of 'bpage' because 'bpage' is used for buffer_page. Link: https://patch.msgid.link/20260522171051.722645963@kernel.org Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 112 ++++++++++++++++++++++++--------------------- 1 file changed, 60 insertions(+), 52 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 73f453ba12ce..fd4a8dc5484e 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -364,21 +364,30 @@ struct buffer_page { #define RB_WRITE_MASK 0xfffff #define RB_WRITE_INTCNT (1 << 20) -static void rb_init_page(struct buffer_data_page *bpage) +static void rb_init_data_page(struct buffer_data_page *bpage) { local_set(&bpage->commit, 0); bpage->time_stamp = 0; } +static __always_inline long rb_data_page_commit(struct buffer_data_page *dpage) +{ + return local_read(&dpage->commit); +} + +static __always_inline long rb_data_page_size(struct buffer_data_page *dpage) +{ + return rb_data_page_commit(dpage) & ~RB_MISSED_MASK; +} + static __always_inline unsigned int rb_page_commit(struct buffer_page *bpage) { - return local_read(&bpage->page->commit); + return rb_data_page_commit(bpage->page); } -/* Size is determined by what has been committed */ static __always_inline unsigned int rb_page_size(struct buffer_page *bpage) { - return rb_page_commit(bpage) & ~RB_MISSED_MASK; + return rb_data_page_size(bpage->page); } static void free_buffer_page(struct buffer_page *bpage) @@ -419,7 +428,7 @@ static struct buffer_data_page *alloc_cpu_data(int cpu, int order) return NULL; dpage = page_address(page); - rb_init_page(dpage); + rb_init_data_page(dpage); return dpage; } @@ -659,7 +668,7 @@ static void verify_event(struct ring_buffer_per_cpu *cpu_buffer, do { if (page == tail_page || WARN_ON_ONCE(stop++ > 100)) done = true; - commit = local_read(&page->page->commit); + commit = rb_page_commit(page); write = local_read(&page->write); if (addr >= (unsigned long)&page->page->data[commit] && addr < (unsigned long)&page->page->data[write]) @@ -1906,7 +1915,7 @@ static int __rb_validate_buffer(struct buffer_page *bpage, int cpu, * Even after clearing these bits, a commit value greater than the * subbuf_size is considered invalid. */ - tail = local_read(&dpage->commit) & ~RB_MISSED_MASK; + tail = rb_data_page_size(dpage); if (tail <= meta->subbuf_size - BUF_PAGE_HDR_SIZE) ret = rb_read_data_buffer(dpage, tail, cpu, &ts, &delta); else @@ -2145,12 +2154,12 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) /* Reset the reader page */ local_set(&cpu_buffer->reader_page->entries, 0); - rb_init_page(cpu_buffer->reader_page->page); + rb_init_data_page(cpu_buffer->reader_page->page); /* Reset all the subbuffers */ for (i = 0; i < meta->nr_subbufs - 1; i++, rb_inc_page(&head_page)) { local_set(&head_page->entries, 0); - rb_init_page(head_page->page); + rb_init_data_page(head_page->page); } } @@ -2210,7 +2219,7 @@ static void rb_range_meta_init(struct trace_buffer *buffer, int nr_pages, int sc */ for (i = 0; i < meta->nr_subbufs; i++) { meta->buffers[i] = i; - rb_init_page(subbuf); + rb_init_data_page(subbuf); subbuf += meta->subbuf_size; } } @@ -2262,7 +2271,7 @@ static int rbm_show(struct seq_file *m, void *v) val -= 2; dpage = rb_range_buffer(cpu_buffer, val); seq_printf(m, "buffer[%ld]: %d (commit: %ld)\n", - val, meta->buffers[val], dpage ? local_read(&dpage->commit) : -1); + val, meta->buffers[val], dpage ? rb_data_page_commit(dpage) : -1); return 0; } @@ -2653,7 +2662,7 @@ static void rb_test_inject_invalid_pages(struct trace_buffer *buffer) dpage = (void *)(ptr + idx * subbuf_size); /* Skip unused pages */ - if (!local_read(&dpage->commit)) + if (!rb_data_page_commit(dpage)) continue; /* @@ -2665,7 +2674,7 @@ static void rb_test_inject_invalid_pages(struct trace_buffer *buffer) invalid++; } else { /* Count total commit bytes. */ - entry_bytes += local_read(&dpage->commit) & ~RB_MISSED_MASK; + entry_bytes += rb_data_page_size(dpage); } } @@ -4194,8 +4203,7 @@ rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer) local_set(&cpu_buffer->commit_page->page->commit, rb_page_write(cpu_buffer->commit_page)); RB_WARN_ON(cpu_buffer, - local_read(&cpu_buffer->commit_page->page->commit) & - ~RB_WRITE_MASK); + rb_page_commit(cpu_buffer->commit_page) & ~RB_WRITE_MASK); barrier(); } @@ -4567,7 +4575,7 @@ static const char *show_interrupt_level(void) return show_irq_str(level); } -static void dump_buffer_page(struct buffer_data_page *bpage, +static void dump_buffer_page(struct buffer_data_page *dpage, struct rb_event_info *info, unsigned long tail) { @@ -4575,12 +4583,12 @@ static void dump_buffer_page(struct buffer_data_page *bpage, u64 ts, delta; int e; - ts = bpage->time_stamp; + ts = dpage->time_stamp; pr_warn(" [%lld] PAGE TIME STAMP\n", ts); for (e = 0; e < tail; e += rb_event_length(event)) { - event = (struct ring_buffer_event *)(bpage->data + e); + event = (struct ring_buffer_event *)(dpage->data + e); switch (event->type_len) { @@ -4630,7 +4638,7 @@ static atomic_t ts_dump; } \ atomic_inc(&cpu_buffer->record_disabled); \ pr_warn(fmt, ##__VA_ARGS__); \ - dump_buffer_page(bpage, info, tail); \ + dump_buffer_page(dpage, info, tail); \ atomic_dec(&ts_dump); \ /* There's some cases in boot up that this can happen */ \ if (WARN_ON_ONCE(system_state != SYSTEM_BOOTING)) \ @@ -4646,16 +4654,16 @@ static void check_buffer(struct ring_buffer_per_cpu *cpu_buffer, struct rb_event_info *info, unsigned long tail) { - struct buffer_data_page *bpage; + struct buffer_data_page *dpage; u64 ts, delta; bool full = false; int ret; - bpage = info->tail_page->page; + dpage = info->tail_page->page; if (tail == CHECK_FULL_PAGE) { full = true; - tail = local_read(&bpage->commit); + tail = rb_data_page_commit(dpage); } else if (info->add_timestamp & (RB_ADD_STAMP_FORCE | RB_ADD_STAMP_ABSOLUTE)) { /* Ignore events with absolute time stamps */ @@ -4666,7 +4674,7 @@ static void check_buffer(struct ring_buffer_per_cpu *cpu_buffer, * Do not check the first event (skip possible extends too). * Also do not check if previous events have not been committed. */ - if (tail <= 8 || tail > local_read(&bpage->commit)) + if (tail <= 8 || tail > rb_data_page_commit(dpage)) return; /* @@ -4675,7 +4683,7 @@ static void check_buffer(struct ring_buffer_per_cpu *cpu_buffer, if (atomic_inc_return(this_cpu_ptr(&checking)) != 1) goto out; - ret = rb_read_data_buffer(bpage, tail, cpu_buffer->cpu, &ts, &delta); + ret = rb_read_data_buffer(dpage, tail, cpu_buffer->cpu, &ts, &delta); if (ret < 0) { if (delta < ts) { buffer_warn_return("[CPU: %d]ABSOLUTE TIME WENT BACKWARDS: last ts: %lld absolute ts: %lld clock:%pS\n", @@ -6466,7 +6474,7 @@ static void rb_clear_buffer_page(struct buffer_page *page) { local_set(&page->write, 0); local_set(&page->entries, 0); - rb_init_page(page->page); + rb_init_data_page(page->page); page->read = 0; } @@ -6951,7 +6959,7 @@ ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu) local_irq_restore(flags); if (bpage->data) { - rb_init_page(bpage->data); + rb_init_data_page(bpage->data); } else { bpage->data = alloc_cpu_data(cpu, cpu_buffer->buffer->subbuf_order); if (!bpage->data) { @@ -6976,8 +6984,8 @@ void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, struct buffer_data_read_page *data_page) { struct ring_buffer_per_cpu *cpu_buffer; - struct buffer_data_page *bpage = data_page->data; - struct page *page = virt_to_page(bpage); + struct buffer_data_page *dpage = data_page->data; + struct page *page = virt_to_page(dpage); unsigned long flags; if (!buffer || !buffer->buffers || !buffer->buffers[cpu]) @@ -6997,15 +7005,15 @@ void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, arch_spin_lock(&cpu_buffer->lock); if (!cpu_buffer->free_page) { - cpu_buffer->free_page = bpage; - bpage = NULL; + cpu_buffer->free_page = dpage; + dpage = NULL; } arch_spin_unlock(&cpu_buffer->lock); local_irq_restore(flags); out: - free_pages((unsigned long)bpage, data_page->order); + free_pages((unsigned long)dpage, data_page->order); kfree(data_page); } EXPORT_SYMBOL_GPL(ring_buffer_free_read_page); @@ -7050,7 +7058,7 @@ int ring_buffer_read_page(struct trace_buffer *buffer, { struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu]; struct ring_buffer_event *event; - struct buffer_data_page *bpage; + struct buffer_data_page *dpage; struct buffer_page *reader; unsigned long missed_events; unsigned int commit; @@ -7076,8 +7084,8 @@ int ring_buffer_read_page(struct trace_buffer *buffer, if (data_page->order != buffer->subbuf_order) return -1; - bpage = data_page->data; - if (!bpage) + dpage = data_page->data; + if (!dpage) return -1; guard(raw_spinlock_irqsave)(&cpu_buffer->reader_lock); @@ -7143,7 +7151,7 @@ int ring_buffer_read_page(struct trace_buffer *buffer, * We have already ensured there's enough space if this * is a time extend. */ size = rb_event_length(event); - memcpy(bpage->data + pos, rpage->data + rpos, size); + memcpy(dpage->data + pos, rpage->data + rpos, size); len -= size; @@ -7159,9 +7167,9 @@ int ring_buffer_read_page(struct trace_buffer *buffer, size = rb_event_ts_length(event); } while (len >= size); - /* update bpage */ - local_set(&bpage->commit, pos); - bpage->time_stamp = save_timestamp; + /* update dpage */ + local_set(&dpage->commit, pos); + dpage->time_stamp = save_timestamp; /* we copied everything to the beginning */ read = 0; @@ -7171,13 +7179,13 @@ int ring_buffer_read_page(struct trace_buffer *buffer, cpu_buffer->read_bytes += rb_page_size(reader); /* swap the pages */ - rb_init_page(bpage); - bpage = reader->page; + rb_init_data_page(dpage); + dpage = reader->page; reader->page = data_page->data; local_set(&reader->write, 0); local_set(&reader->entries, 0); reader->read = 0; - data_page->data = bpage; + data_page->data = dpage; /* * Use the real_end for the data size, @@ -7185,12 +7193,12 @@ int ring_buffer_read_page(struct trace_buffer *buffer, * on the page. */ if (reader->real_end) - local_set(&bpage->commit, reader->real_end); + local_set(&dpage->commit, reader->real_end); } cpu_buffer->lost_events = 0; - commit = local_read(&bpage->commit); + commit = rb_data_page_commit(dpage); /* * Set a flag in the commit field if we lost events */ @@ -7199,19 +7207,19 @@ int ring_buffer_read_page(struct trace_buffer *buffer, * missed events, then record it there. */ if (buffer->subbuf_size - commit >= sizeof(missed_events)) { - memcpy(&bpage->data[commit], &missed_events, + memcpy(&dpage->data[commit], &missed_events, sizeof(missed_events)); - local_add(RB_MISSED_STORED, &bpage->commit); + local_add(RB_MISSED_STORED, &dpage->commit); commit += sizeof(missed_events); } - local_add(RB_MISSED_EVENTS, &bpage->commit); + local_add(RB_MISSED_EVENTS, &dpage->commit); } /* * This page may be off to user land. Zero it out here. */ if (commit < buffer->subbuf_size) - memset(&bpage->data[commit], 0, buffer->subbuf_size - commit); + memset(&dpage->data[commit], 0, buffer->subbuf_size - commit); return read; } @@ -7842,7 +7850,7 @@ consume: if (missed_events) { if (cpu_buffer->reader_page != cpu_buffer->commit_page) { - struct buffer_data_page *bpage = reader->page; + struct buffer_data_page *dpage = reader->page; unsigned int commit; /* * Use the real_end for the data size, @@ -7850,18 +7858,18 @@ consume: * on the page. */ if (reader->real_end) - local_set(&bpage->commit, reader->real_end); + local_set(&dpage->commit, reader->real_end); /* * If there is room at the end of the page to save the * missed events, then record it there. */ commit = rb_page_size(reader); if (buffer->subbuf_size - commit >= sizeof(missed_events)) { - memcpy(&bpage->data[commit], &missed_events, + memcpy(&dpage->data[commit], &missed_events, sizeof(missed_events)); - local_add(RB_MISSED_STORED, &bpage->commit); + local_add(RB_MISSED_STORED, &dpage->commit); } - local_add(RB_MISSED_EVENTS, &bpage->commit); + local_add(RB_MISSED_EVENTS, &dpage->commit); } else if (!WARN_ONCE(cpu_buffer->reader_page == cpu_buffer->tail_page, "Reader on commit with %ld missed events", missed_events)) { -- cgit v1.2.3 From 8913e2a48b8d3c1ef9c99481e5725afca841762e Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Fri, 22 May 2026 13:09:04 -0400 Subject: ring-buffer: Have dropped subbuffers be persistent across reboots When the persistent ring buffer detects a corrupted subbuffer, it will zero its size and report dropped pages in the dmesg, then it continues normally. But if a reboot happens without clearing or restarting tracing on the persistent ring buffer, the next boot will show no pages are dropped. If the persistent ring buffer is still the same, then it should still report dropped pages so the user knows that the buffer has missing events. Add the RB_MISSED_EVENTS flag to the commit value of the subbuffer so that the next boot will still show that pages were dropped. Link: https://patch.msgid.link/20260522171051.860780286@kernel.org Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index fd4a8dc5484e..db72969aefa9 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -1915,7 +1915,7 @@ static int __rb_validate_buffer(struct buffer_page *bpage, int cpu, * Even after clearing these bits, a commit value greater than the * subbuf_size is considered invalid. */ - tail = rb_data_page_size(dpage); + tail = rb_data_page_commit(dpage); if (tail <= meta->subbuf_size - BUF_PAGE_HDR_SIZE) ret = rb_read_data_buffer(dpage, tail, cpu, &ts, &delta); else @@ -1929,7 +1929,7 @@ static int __rb_validate_buffer(struct buffer_page *bpage, int cpu, */ if (ret < 0 || (prev_ts && prev_ts > ts) || (next_ts && ts > next_ts)) { local_set(&bpage->entries, 0); - local_set(&dpage->commit, 0); + local_set(&dpage->commit, RB_MISSED_EVENTS); dpage->time_stamp = prev_ts ? prev_ts : next_ts; ret = -1; } else { @@ -3451,7 +3451,7 @@ rb_iter_head_event(struct ring_buffer_iter *iter) * is a mb(), which will synchronize with the rmb here. * (see rb_tail_page_update() and __rb_reserve_next()) */ - commit = rb_page_commit(iter_head_page); + commit = rb_page_size(iter_head_page); smp_rmb(); /* An event needs to be at least 8 bytes in size */ @@ -3480,7 +3480,7 @@ rb_iter_head_event(struct ring_buffer_iter *iter) /* Make sure the page didn't change since we read this */ if (iter->page_stamp != iter_head_page->page->time_stamp || - commit > rb_page_commit(iter_head_page)) + commit > rb_page_size(iter_head_page)) goto reset; iter->next_event = iter->head + length; @@ -5651,7 +5651,7 @@ int ring_buffer_iter_empty(struct ring_buffer_iter *iter) * (see rb_tail_page_update()) */ smp_rmb(); - commit = rb_page_commit(commit_page); + commit = rb_page_size(commit_page); /* We want to make sure that the commit page doesn't change */ smp_rmb(); @@ -5844,6 +5844,7 @@ __rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) */ local_set(&cpu_buffer->reader_page->write, 0); local_set(&cpu_buffer->reader_page->entries, 0); + rb_init_data_page(cpu_buffer->reader_page->page); cpu_buffer->reader_page->real_end = 0; spin: -- cgit v1.2.3 From 97628ff073a8235b401b99e8619c121040f39251 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Fri, 22 May 2026 13:09:05 -0400 Subject: ring-buffer: Show persistent buffer dropped events in trace file When the persistent ring buffer is validated on boot up, if a subbuffer is deemed invalid, it resets the buffer and continues. Currently, these lost events are not shown in the trace file output. Have the trace iterator look for subbuffers that have the RB_MISSED_EVENTS set and set the iter->missed_events flag when it is detected. This will then have the trace file shows "LOST EVENTS" when it reads across a subbuffer that was corrupted and invalidated. For example: <...>-1016 [005] ...1. 6230.660403: preempt_disable: caller=__mod_memcg_state+0x1c8/0x200 parent=__mod_memcg_state+0x1c8/0x200 CPU:5 [LOST EVENTS] <...>-1016 [005] ..... 6230.660673: kmem_cache_alloc: call_site=__anon_vma_prepare+0x1ad/0x1e0 ptr=000000006e40294c name=anon_vma bytes_req=200 bytes_alloc=208 gfp_flags=GFP_KERNEL node=-1 accounted=true Link: https://patch.msgid.link/20260522171052.006276604@kernel.org Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index db72969aefa9..988915f035c7 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -3525,6 +3525,9 @@ static void rb_inc_iter(struct ring_buffer_iter *iter) else rb_inc_page(&iter->head_page); + if (rb_page_commit(iter->head_page) & RB_MISSED_EVENTS) + iter->missed_events = -1; + iter->page_stamp = iter->read_stamp = iter->head_page->page->time_stamp; iter->head = 0; iter->next_event = 0; @@ -7061,7 +7064,7 @@ int ring_buffer_read_page(struct trace_buffer *buffer, struct ring_buffer_event *event; struct buffer_data_page *dpage; struct buffer_page *reader; - unsigned long missed_events; + long missed_events; unsigned int commit; unsigned int read; u64 save_timestamp; @@ -7187,6 +7190,8 @@ int ring_buffer_read_page(struct trace_buffer *buffer, local_set(&reader->entries, 0); reader->read = 0; data_page->data = dpage; + if (!missed_events && rb_data_page_commit(dpage) & RB_MISSED_EVENTS) + missed_events = -1; /* * Use the real_end for the data size, @@ -7204,10 +7209,12 @@ int ring_buffer_read_page(struct trace_buffer *buffer, * Set a flag in the commit field if we lost events */ if (missed_events) { - /* If there is room at the end of the page to save the + /* + * If there is room at the end of the page to save the * missed events, then record it there. */ - if (buffer->subbuf_size - commit >= sizeof(missed_events)) { + if (missed_events > 0 && + buffer->subbuf_size - commit >= sizeof(missed_events)) { memcpy(&dpage->data[commit], &missed_events, sizeof(missed_events)); local_add(RB_MISSED_STORED, &dpage->commit); -- cgit v1.2.3 From 8928e4a3be34bf053f9ef1cad67263604bf4f05e Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Fri, 22 May 2026 13:09:06 -0400 Subject: ring-buffer: Show persistent buffer dropped events in trace_pipe file When the persistent ring buffer is validated on boot up, if a subbuffer is deemed invalid, it resets the buffer and continues. Have the code preserve the RB_MISSED_EVENTS flag in the commit portion of the subbuffer header and pass that back so that the trace_pipe file can show the missed events like the trace file does. For example: <...>-1242 [005] d.... 4429.120116: page_fault_user: address=0x7ffaebb6e728 ip=0x7ffaeb9d4960 error_code=0x7 <...>-1242 [005] ..... 4429.120124: mm_page_alloc: page=00000000055254f3 pfn=0x1373bd order=0 migratetype=1 gfp_flags=GFP_HIGHUSER_MOVABLE|__GFP_COMP <...>-1242 [005] d..2. 4429.120132: tlb_flush: pages:1 reason:local MM shootdown (3) CPU:5 [LOST EVENTS] <...>-1242 [005] d.... 4429.120661: page_fault_user: address=0x55ba7c2d0944 ip=0x55ba7c20cd02 error_code=0x7 <...>-1242 [005] ..... 4429.120669: mm_page_alloc: page=0000000005a02500 pfn=0x12b6e4 order=0 migratetype=1 gfp_flags=GFP_HIGHUSER_MOVABLE|__GFP_COMP <...>-1242 [005] d..2. 4429.120680: tlb_flush: pages:1 reason:local MM shootdown (3) Link: https://patch.msgid.link/20260522171052.156419479@kernel.org Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 56 ++++++++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 22 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 988915f035c7..910f6b3adf74 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -5801,6 +5801,7 @@ __rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) struct buffer_page *reader = NULL; unsigned long overwrite; unsigned long flags; + int missed_events = 0; int nr_loops = 0; bool ret; @@ -5901,6 +5902,9 @@ __rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) if (!ret) goto spin; + if (rb_page_commit(reader) & RB_MISSED_EVENTS) + missed_events = -1; + if (cpu_buffer->ring_meta) rb_update_meta_reader(cpu_buffer, reader); @@ -5965,6 +5969,8 @@ __rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) */ smp_rmb(); + if (!cpu_buffer->lost_events) + cpu_buffer->lost_events = missed_events; return reader; } @@ -7066,6 +7072,7 @@ int ring_buffer_read_page(struct trace_buffer *buffer, struct buffer_page *reader; long missed_events; unsigned int commit; + unsigned int size; unsigned int read; u64 save_timestamp; bool force_memcpy; @@ -7101,7 +7108,8 @@ int ring_buffer_read_page(struct trace_buffer *buffer, event = rb_reader_event(cpu_buffer); read = reader->read; - commit = rb_page_size(reader); + commit = rb_page_commit(reader); + size = rb_page_size(reader); /* Check if any events were dropped */ missed_events = cpu_buffer->lost_events; @@ -7115,13 +7123,14 @@ int ring_buffer_read_page(struct trace_buffer *buffer, * we must copy the data from the page to the buffer. * Otherwise, we can simply swap the page with the one passed in. */ - if (read || (len < (commit - read)) || + if (read || (len < (size - read)) || cpu_buffer->reader_page == cpu_buffer->commit_page || force_memcpy) { struct buffer_data_page *rpage = cpu_buffer->reader_page->page; unsigned int rpos = read; unsigned int pos = 0; - unsigned int size; + unsigned int event_size; + unsigned int flags = 0; /* * If a full page is expected, this can still be returned @@ -7130,19 +7139,22 @@ int ring_buffer_read_page(struct trace_buffer *buffer, * the reader page. */ if (full && - (!read || (len < (commit - read)) || + (!read || (len < (size - read)) || cpu_buffer->reader_page == cpu_buffer->commit_page)) return -1; - if (len > (commit - read)) - len = (commit - read); + if (len > (size - read)) + len = (size - read); /* Always keep the time extend and data together */ - size = rb_event_ts_length(event); + event_size = rb_event_ts_length(event); - if (len < size) + if (len < event_size) return -1; + if (commit & RB_MISSED_EVENTS) + flags = RB_MISSED_EVENTS; + /* save the current timestamp, since the user will need it */ save_timestamp = cpu_buffer->read_stamp; @@ -7154,25 +7166,25 @@ int ring_buffer_read_page(struct trace_buffer *buffer, * one or two events. * We have already ensured there's enough space if this * is a time extend. */ - size = rb_event_length(event); - memcpy(dpage->data + pos, rpage->data + rpos, size); + event_size = rb_event_length(event); + memcpy(dpage->data + pos, rpage->data + rpos, event_size); - len -= size; + len -= event_size; rb_advance_reader(cpu_buffer); rpos = reader->read; - pos += size; + pos += event_size; - if (rpos >= commit) + if (rpos >= event_size) break; event = rb_reader_event(cpu_buffer); /* Always keep the time extend and data together */ - size = rb_event_ts_length(event); - } while (len >= size); + event_size = rb_event_ts_length(event); + } while (len >= event_size); /* update dpage */ - local_set(&dpage->commit, pos); + local_set(&dpage->commit, pos | flags); dpage->time_stamp = save_timestamp; /* we copied everything to the beginning */ @@ -7204,7 +7216,7 @@ int ring_buffer_read_page(struct trace_buffer *buffer, cpu_buffer->lost_events = 0; - commit = rb_data_page_commit(dpage); + size = rb_data_page_size(dpage); /* * Set a flag in the commit field if we lost events */ @@ -7214,11 +7226,11 @@ int ring_buffer_read_page(struct trace_buffer *buffer, * missed events, then record it there. */ if (missed_events > 0 && - buffer->subbuf_size - commit >= sizeof(missed_events)) { - memcpy(&dpage->data[commit], &missed_events, + buffer->subbuf_size - size >= sizeof(missed_events)) { + memcpy(&dpage->data[size], &missed_events, sizeof(missed_events)); local_add(RB_MISSED_STORED, &dpage->commit); - commit += sizeof(missed_events); + size += sizeof(missed_events); } local_add(RB_MISSED_EVENTS, &dpage->commit); } @@ -7226,8 +7238,8 @@ int ring_buffer_read_page(struct trace_buffer *buffer, /* * This page may be off to user land. Zero it out here. */ - if (commit < buffer->subbuf_size) - memset(&dpage->data[commit], 0, buffer->subbuf_size - commit); + if (size < buffer->subbuf_size) + memset(&dpage->data[size], 0, buffer->subbuf_size - size); return read; } -- cgit v1.2.3 From 9b435d23f51e55b62dda3a345a9f8931248ca514 Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Thu, 21 May 2026 22:29:09 +0800 Subject: bpf: Fix race between bpf_map_new_fd() and close_fd() Because there is time gap between bpf_map_new_fd() and close_fd(), a concurrent thread is able to close the new fd and opens a new, unrelated file with the exact same fd number. Thereafter, this close_fd() might inadvertently close the unrelated file. To avoid such regression, do finalize log before security_bpf_map_create(). However, in order to achieve it, move bpf_get_file_flag(), security_bpf_map_create(), bpf_map_alloc_id(), and bpf_map_new_fd() from __map_create() to map_create(). And, rename __map_create() to map_create_alloc() meanwhile. Then, in order to reuse the map and token when all checks pass in map_create_alloc(), pass "struct bpf_map **" and "struct bpf_token **" to map_create_alloc(). Fixes: 49f9b2b2a18c ("bpf: Add syscall common attributes support for map_create") Signed-off-by: Leon Hwang Link: https://lore.kernel.org/r/20260521142909.95818-1-leon.hwang@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/syscall.c | 80 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 33 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 8d4ea700aac6..9e91fb2fb492 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1359,7 +1359,8 @@ free_map_tab: #define BPF_MAP_CREATE_LAST_FIELD excl_prog_hash_size /* called via syscall */ -static int __map_create(union bpf_attr *attr, bpfptr_t uattr, struct bpf_verifier_log *log) +static int map_create_alloc(union bpf_attr *attr, bpfptr_t uattr, struct bpf_verifier_log *log, + struct bpf_map **mapp, struct bpf_token **tokenp) { const struct bpf_map_ops *ops; struct bpf_token *token = NULL; @@ -1367,7 +1368,6 @@ static int __map_create(union bpf_attr *attr, bpfptr_t uattr, struct bpf_verifie u32 map_type = attr->map_type; struct bpf_map *map; bool token_flag; - int f_flags; int err; err = CHECK_ATTR(BPF_MAP_CREATE); @@ -1403,10 +1403,6 @@ static int __map_create(union bpf_attr *attr, bpfptr_t uattr, struct bpf_verifie return -EINVAL; } - f_flags = bpf_get_file_flag(attr->map_flags); - if (f_flags < 0) - return f_flags; - if (numa_node != NUMA_NO_NODE && ((unsigned int)numa_node >= nr_node_ids || !node_online(numa_node))) { @@ -1598,6 +1594,49 @@ static int __map_create(union bpf_attr *attr, bpfptr_t uattr, struct bpf_verifie goto free_map; } + *mapp = map; + *tokenp = token; + return 0; + +free_map: + bpf_map_free(map); +put_token: + bpf_token_put(token); + return err; +} + +static int map_create(union bpf_attr *attr, bpfptr_t uattr, struct bpf_common_attr *attr_common, + bpfptr_t uattr_common, u32 size_common) +{ + struct bpf_token *token = NULL; + struct bpf_verifier_log *log; + struct bpf_log_attr attr_log; + struct bpf_map *map = NULL; + int err, ret; + int f_flags; + + log = bpf_log_attr_create_vlog(&attr_log, attr_common, uattr_common, size_common); + if (IS_ERR(log)) + return PTR_ERR(log); + + err = map_create_alloc(attr, uattr, log, &map, &token); + + /* preserve original error even if log finalization is successful */ + ret = bpf_log_attr_finalize(&attr_log, log); + if (ret) + err = ret; + + kfree(log); + + if (err) + goto free_map; + + f_flags = bpf_get_file_flag(attr->map_flags); + if (f_flags < 0) { + err = f_flags; + goto free_map; + } + err = security_bpf_map_create(map, attr, token, uattr.is_kernel); if (err) goto free_map_sec; @@ -1626,37 +1665,12 @@ static int __map_create(union bpf_attr *attr, bpfptr_t uattr, struct bpf_verifie free_map_sec: security_bpf_map_free(map); free_map: - bpf_map_free(map); -put_token: + if (map) + bpf_map_free(map); bpf_token_put(token); return err; } -static int map_create(union bpf_attr *attr, bpfptr_t uattr, struct bpf_common_attr *attr_common, - bpfptr_t uattr_common, u32 size_common) -{ - struct bpf_verifier_log *log; - struct bpf_log_attr attr_log; - int err, ret; - - log = bpf_log_attr_create_vlog(&attr_log, attr_common, uattr_common, size_common); - if (IS_ERR(log)) - return PTR_ERR(log); - - err = __map_create(attr, uattr, log); - - /* preserve original error even if log finalization is successful */ - ret = bpf_log_attr_finalize(&attr_log, log); - if (ret) { - if (err >= 0) - close_fd(err); - err = ret; - } - - kfree(log); - return err; -} - void bpf_map_inc(struct bpf_map *map) { atomic64_inc(&map->refcnt); -- cgit v1.2.3 From 88692f0c33a788072abfa1888b28bc6d7d7d1165 Mon Sep 17 00:00:00 2001 From: "David Hildenbrand (Arm)" Date: Mon, 27 Apr 2026 13:43:15 +0200 Subject: bpf: arena: use page_ref_count() instead of page_mapped() in arena_free_pages() Pages that BPF arena code maps are allocated through bpf_map_alloc_pages(), which does not allocate folios but pages. In the future, pages will not have a mapcount, only folios will. Converting the code to use folios and rely on folio_mapped() sounds like the wrong approach. Should BPF arena code allocate folios and use folio_mapped() here? But likely we would not want to use folios here longterm, as we don't really need folio information. Hard to tell. But in the meantime, we can simply use the page refcount instead, as a heuristic whether the page might be mapped to user space and we would want to try zapping it, so we can get rid of page_mapped(). Page allocation will give us a page with a refcount of 1. Any user space mapping adds a page reference. While there can be references from other subsystems (e.g., GUP), in the common case for this test here relying on the page count is good enough. Link: https://lore.kernel.org/20260427-page_mapped-v1-2-e89c3592c74c@kernel.org Signed-off-by: David Hildenbrand (Arm) Reviewed-by: Matthew Wilcox (Oracle) Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Eduard Zingerman Cc: Harry Yoo Cc: Jann Horn Cc: Jiri Olsa Cc: John Paul Adrian Glaubitz Cc: Kumar Kartikeya Dwivedi Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Martin KaFai Lau Cc: Michal Hocko Cc: Mike Rapoport Cc: Rich Felker Cc: Rik van Riel Cc: Song Liu Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Yonghong Song Cc: Yoshinori Sato Signed-off-by: Andrew Morton --- kernel/bpf/arena.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 49a8f7b1beef..a497c5913bd4 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -729,7 +729,7 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, llist_for_each_safe(pos, t, __llist_del_all(&free_pages)) { page = llist_entry(pos, struct page, pcp_llist); - if (page_cnt == 1 && page_mapped(page)) /* mapped by some user process */ + if (page_cnt == 1 && page_ref_count(page) > 1) /* maybe mapped by user space */ /* Optimization for the common case of page_cnt==1: * If page wasn't mapped into some user vma there * is no need to call zap_pages which is slow. When -- cgit v1.2.3 From 6ae51adb084a9d87a8b9501d2231e20271dece87 Mon Sep 17 00:00:00 2001 From: Muhammad Usama Anjum Date: Wed, 29 Apr 2026 15:57:03 +0530 Subject: kasan: skip HW tagging for all kernel thread stacks HW-tag KASAN never checks kernel stacks because stack pointers carry the match-all tag, so setting/poisoning tags is pure overhead. - Add __GFP_SKIP_KASAN to THREADINFO_GFP so every stack allocator that uses it skips tagging (fork path plus arch users) - Add __GFP_SKIP_KASAN to GFP_VMAP_STACK for the fork-specific vmap stacks. - When reusing cached vmap stacks, skip kasan_unpoison_range() if HW tags are enabled. Software KASAN is unchanged; this only affects tag-based KASAN. Link: https://lore.kernel.org/20260429102704.680174-3-dev.jain@arm.com Signed-off-by: Muhammad Usama Anjum Signed-off-by: Dev Jain Reviewed-by: Catalin Marinas Cc: Arnd Bergmann Cc: Ben Segall Cc: David Hildenbrand (Arm) Cc: Dietmar Eggemann Cc: Ingo Molnar Cc: Juri Lelli Cc: Kees Cook Cc: K Prateek Nayak Cc: Liam Howlett Cc: Lorenzo Stoakes Cc: Mathieu Desnoyers Cc: Mel Gorman Cc: Michal Hocko Cc: Mike Rapoport Cc: Peter Zijlstra Cc: Ryan Roberts Cc: Steven Rostedt Cc: Suren Baghdasaryan Cc: "Uladzislau Rezki (Sony)" Cc: Valentin Schneider Cc: Vincent Guittot Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- kernel/fork.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/fork.c b/kernel/fork.c index 8ac38beae360..ec6a120291e5 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -204,7 +204,7 @@ static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]); * accounting is performed by the code assigning/releasing stacks to tasks. * We need a zeroed memory without __GFP_ACCOUNT. */ -#define GFP_VMAP_STACK (GFP_KERNEL | __GFP_ZERO) +#define GFP_VMAP_STACK (GFP_KERNEL | __GFP_ZERO | __GFP_SKIP_KASAN) struct vm_stack { struct rcu_head rcu; @@ -342,7 +342,8 @@ static int alloc_thread_stack_node(struct task_struct *tsk, int node) } /* Reset stack metadata. */ - kasan_unpoison_range(vm_area->addr, THREAD_SIZE); + if (!kasan_hw_tags_enabled()) + kasan_unpoison_range(vm_area->addr, THREAD_SIZE); stack = kasan_reset_tag(vm_area->addr); -- cgit v1.2.3 From b3e4fbb04220efc3bc022bcf31b5689d39c6b111 Mon Sep 17 00:00:00 2001 From: Yiyang Chen Date: Mon, 13 Apr 2026 23:45:44 +0800 Subject: taskstats: retain dead thread stats in TGID queries Patch series "taskstats: fix TGID dead-thread stat retention", v3. This series fixes a taskstats TGID aggregation bug where fields added in the TGID query path were not preserved after thread exit, and adds a kselftest covering the regression. The first patch keeps the cached TGID aggregate used for dead threads in step with the fields already accumulated for live threads, and also fixes the final TGID exit notification emitted when group_dead is true. The second patch adds a kselftest that verifies TGID CPU stats do not regress after a worker thread exits and has been reaped. This patch (of 2): fill_stats_for_tgid() builds TGID stats from two sources: the cached aggregate in signal->stats and a scan of the live threads in the group. However, fill_tgid_exit() only accumulates delay accounting into signal->stats. This means that once a thread exits, TGID queries lose the fields that fill_stats_for_tgid() adds for live threads. This gap was introduced incrementally by two earlier changes that extended fill_stats_for_tgid() but did not make the corresponding update to fill_tgid_exit(): - commit 8c733420bdd5 ("taskstats: add e/u/stime for TGID command") added ac_etime, ac_utime, and ac_stime to the TGID query path. - commit b663a79c1915 ("taskstats: add context-switch counters") added nvcsw and nivcsw to the TGID query path. As a result, those fields were accounted for live threads in TGID queries, but were dropped from the cached TGID aggregate after thread exit. The final TGID exit notification emitted when group_dead is true also copies that cached aggregate, so it loses the same fields. Factor the per-task TGID accumulation into tgid_stats_add_task() and use it in both fill_stats_for_tgid() and fill_tgid_exit(). This keeps the cached aggregate used for dead threads aligned with the live-thread accumulation used by TGID queries. Link: https://lore.kernel.org/cover.1776094300.git.cyyzero16@gmail.com Link: https://lore.kernel.org/abd2a15d33343636ab5ba43d540bcfe508bd66c7.1776094300.git.cyyzero16@gmail.com Fixes: 8c733420bdd5 ("taskstats: add e/u/stime for TGID command") Fixes: b663a79c1915 ("taskstats: add context-switch counters") Signed-off-by: Yiyang Chen Acked-by: Balbir Singh Cc: Dr. Thomas Orgis Cc: Oleg Nesterov Cc: Wang Yaxin Cc: Yang Yang Cc: Signed-off-by: Andrew Morton --- kernel/taskstats.c | 62 +++++++++++++++++++++++++++++------------------------- 1 file changed, 33 insertions(+), 29 deletions(-) (limited to 'kernel') diff --git a/kernel/taskstats.c b/kernel/taskstats.c index 73bd6a6a7893..2cd0172d0516 100644 --- a/kernel/taskstats.c +++ b/kernel/taskstats.c @@ -210,13 +210,39 @@ static int fill_stats_for_pid(pid_t pid, struct taskstats *stats) return 0; } +static void tgid_stats_add_task(struct taskstats *stats, + struct task_struct *tsk, u64 now_ns) +{ + u64 delta, utime, stime; + + /* + * Each accounting subsystem calls its functions here to + * accumulate its per-task stats for tsk, into the per-tgid structure + * + * per-task-foo(stats, tsk); + */ + delayacct_add_tsk(stats, tsk); + + /* calculate task elapsed time in nsec */ + delta = now_ns - tsk->start_time; + /* Convert to micro seconds */ + do_div(delta, NSEC_PER_USEC); + stats->ac_etime += delta; + + task_cputime(tsk, &utime, &stime); + stats->ac_utime += div_u64(utime, NSEC_PER_USEC); + stats->ac_stime += div_u64(stime, NSEC_PER_USEC); + + stats->nvcsw += tsk->nvcsw; + stats->nivcsw += tsk->nivcsw; +} + static int fill_stats_for_tgid(pid_t tgid, struct taskstats *stats) { struct task_struct *tsk, *first; unsigned long flags; int rc = -ESRCH; - u64 delta, utime, stime; - u64 start_time; + u64 now_ns; /* * Add additional stats from live tasks except zombie thread group @@ -233,30 +259,12 @@ static int fill_stats_for_tgid(pid_t tgid, struct taskstats *stats) else memset(stats, 0, sizeof(*stats)); - start_time = ktime_get_ns(); + now_ns = ktime_get_ns(); for_each_thread(first, tsk) { if (tsk->exit_state) continue; - /* - * Accounting subsystem can call its functions here to - * fill in relevant parts of struct taskstsats as follows - * - * per-task-foo(stats, tsk); - */ - delayacct_add_tsk(stats, tsk); - - /* calculate task elapsed time in nsec */ - delta = start_time - tsk->start_time; - /* Convert to micro seconds */ - do_div(delta, NSEC_PER_USEC); - stats->ac_etime += delta; - task_cputime(tsk, &utime, &stime); - stats->ac_utime += div_u64(utime, NSEC_PER_USEC); - stats->ac_stime += div_u64(stime, NSEC_PER_USEC); - - stats->nvcsw += tsk->nvcsw; - stats->nivcsw += tsk->nivcsw; + tgid_stats_add_task(stats, tsk, now_ns); } unlock_task_sighand(first, &flags); @@ -275,18 +283,14 @@ out: static void fill_tgid_exit(struct task_struct *tsk) { unsigned long flags; + u64 now_ns; spin_lock_irqsave(&tsk->sighand->siglock, flags); if (!tsk->signal->stats) goto ret; - /* - * Each accounting subsystem calls its functions here to - * accumalate its per-task stats for tsk, into the per-tgid structure - * - * per-task-foo(tsk->signal->stats, tsk); - */ - delayacct_add_tsk(tsk->signal->stats, tsk); + now_ns = ktime_get_ns(); + tgid_stats_add_task(tsk->signal->stats, tsk, now_ns); ret: spin_unlock_irqrestore(&tsk->sighand->siglock, flags); return; -- cgit v1.2.3 From 5a13e296a3e32a70db2a520d8611a53d7bde10e8 Mon Sep 17 00:00:00 2001 From: Jann Horn Date: Thu, 30 Apr 2026 16:15:33 +0200 Subject: kcov: refactor common handle ID into kcov_common_handle_id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Store common handle IDs in "struct kcov_common_handle_id", which consumes no space in non-KCOV builds. This cleanup removes #ifdef boilerplate code from subsystems that integrate with KCOV (in particular in usbip_common.h and skbuff.h, see the diffstat). This should also make it easier to add KCOV remote coverage to more subsystems in the future. Link: https://lore.kernel.org/20260430-kcov-refactor-common-handle-v1-1-23a0c7a0ba38@google.com Signed-off-by: Jann Horn Acked-by: Greg Kroah-Hartman Reviewed-by: Dmitry Vyukov Acked-by: Jakub Kicinski Cc: Alexander Potapenko Cc: Andrey Konovalov Cc: Eugenio Pérez Cc: Hongren (Zenithal) Zheng Cc: Jann Horn Cc: Jason Wang Cc: "Michael S. Tsirkin" Cc: Shuah Khan Cc: Valentina Manea Signed-off-by: Andrew Morton --- kernel/kcov.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/kcov.c b/kernel/kcov.c index 0b369e88c7c9..a43e33a28adb 100644 --- a/kernel/kcov.c +++ b/kernel/kcov.c @@ -1083,11 +1083,11 @@ void kcov_remote_stop(void) EXPORT_SYMBOL(kcov_remote_stop); /* See the comment before kcov_remote_start() for usage details. */ -u64 kcov_common_handle(void) +struct kcov_common_handle_id kcov_common_handle(void) { if (!in_task()) - return 0; - return current->kcov_handle; + return (struct kcov_common_handle_id){ .val = 0 }; + return (struct kcov_common_handle_id){ .val = current->kcov_handle }; } EXPORT_SYMBOL(kcov_common_handle); -- cgit v1.2.3 From 1d9c9493692eccd16b47610f1d21cc7a100199e9 Mon Sep 17 00:00:00 2001 From: Jann Horn Date: Tue, 5 May 2026 11:00:46 +0200 Subject: kcov: allow simultaneous KCOV_ENABLE/KCOV_REMOTE_ENABLE Allow the same userspace thread to simultaneously collect normal coverage in syscall context (KCOV_ENABLE) and remote coverage of asynchronous work created by the thread (KCOV_REMOTE_ENABLE). With this, remote KCOV coverage becomes useful for generic fuzzing and not just fuzzing of specific data injection interfaces. This requires that the task_struct::kcov_* fields are separated into ones that are used by the task that generates coverage, and ones that are used by the task that requested remote coverage. To split this up: - Split task_struct::kcov into kcov and kcov_remote. kcov_task_exit() now has to clean up both separately. - Only use task_struct::kcov_mode on the task that generates coverage. - Only reset task_struct::kcov_handle on the task that requested remote coverage. After this change, fields used by the task that generates coverage are: - kcov_mode - kcov_size - kcov_area - kcov - kcov_sequence - kcov_softirq Fields used by the task that requested remote coverage are: - kcov_remote - kcov_handle [jannh@google.com: remove unused constant KCOV_MODE_REMOTE, per Dmitry] Link: https://lore.kernel.org/20260515-kcov-simultaneous-remote-v2-1-56fde1cfa509@google.com [jannh@google.com: update documentation on remote coverage collection] Link: https://lore.kernel.org/20260519-kcov-docs-v1-1-5bb22f4cb20c@google.com [jannh@google.com: move and reword sentence on simultaneous normal/remote collection Link: https://lore.kernel.org/20260520-kcov-docs-v2-1-819f78778763@google.com Link: https://lore.kernel.org/20260505-kcov-simultaneous-remote-v1-1-a670ba7cefd2@google.com Signed-off-by: Jann Horn Reviewed-by: Dmitry Vyukov Cc: Alexander Potapenko Cc: Andrey Konovalov Cc: Marco Elver Signed-off-by: Andrew Morton --- kernel/kcov.c | 94 +++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 52 insertions(+), 42 deletions(-) (limited to 'kernel') diff --git a/kernel/kcov.c b/kernel/kcov.c index a43e33a28adb..fd2503030729 100644 --- a/kernel/kcov.c +++ b/kernel/kcov.c @@ -368,6 +368,7 @@ static void kcov_start(struct task_struct *t, struct kcov *kcov, WRITE_ONCE(t->kcov_mode, mode); } +/* operates on coverage-generator-owned fields */ static void kcov_stop(struct task_struct *t) { WRITE_ONCE(t->kcov_mode, KCOV_MODE_DISABLED); @@ -377,16 +378,17 @@ static void kcov_stop(struct task_struct *t) t->kcov_area = NULL; } +/* operates on coverage-generator-owned fields */ static void kcov_task_reset(struct task_struct *t) { kcov_stop(t); t->kcov_sequence = 0; - t->kcov_handle = 0; } void kcov_task_init(struct task_struct *t) { kcov_task_reset(t); + t->kcov_remote = NULL; t->kcov_handle = current->kcov_handle; } @@ -423,11 +425,14 @@ static void kcov_remote_reset(struct kcov *kcov) static void kcov_disable(struct task_struct *t, struct kcov *kcov) __must_hold(&kcov->lock) { - kcov_task_reset(t); - if (kcov->remote) + if (kcov->remote) { + t->kcov_handle = 0; + t->kcov_remote = NULL; kcov_remote_reset(kcov); - else + } else { + kcov_task_reset(t); kcov_reset(kcov); + } } static void kcov_get(struct kcov *kcov) @@ -453,41 +458,47 @@ void kcov_task_exit(struct task_struct *t) unsigned long flags; kcov = t->kcov; - if (kcov == NULL) - return; - - spin_lock_irqsave(&kcov->lock, flags); - kcov_debug("t = %px, kcov->t = %px\n", t, kcov->t); - /* - * For KCOV_ENABLE devices we want to make sure that t->kcov->t == t, - * which comes down to: - * WARN_ON(!kcov->remote && kcov->t != t); - * - * For KCOV_REMOTE_ENABLE devices, the exiting task is either: - * - * 1. A remote task between kcov_remote_start() and kcov_remote_stop(). - * In this case we should print a warning right away, since a task - * shouldn't be exiting when it's in a kcov coverage collection - * section. Here t points to the task that is collecting remote - * coverage, and t->kcov->t points to the thread that created the - * kcov device. Which means that to detect this case we need to - * check that t != t->kcov->t, and this gives us the following: - * WARN_ON(kcov->remote && kcov->t != t); - * - * 2. The task that created kcov exiting without calling KCOV_DISABLE, - * and then again we make sure that t->kcov->t == t: - * WARN_ON(kcov->remote && kcov->t != t); - * - * By combining all three checks into one we get: - */ - if (WARN_ON(kcov->t != t)) { + if (kcov) { + spin_lock_irqsave(&kcov->lock, flags); + kcov_debug("t = %px, kcov->t = %px\n", t, kcov->t); + /* + * This could be a remote task between kcov_remote_start() and + * kcov_remote_stop(). + * In this case we should print a warning right away, since a + * task shouldn't be exiting when it's in a kcov coverage + * collection section. + * + * Otherwise, this should be a task that created a local + * kcov instance and hasn't called KCOV_DISABLE. + * Make sure that t->kcov->t is consistent. + */ + if (WARN_ON(kcov->remote) || WARN_ON(kcov->t != t)) { + spin_unlock_irqrestore(&kcov->lock, flags); + return; + } + /* Just to not leave dangling references behind. */ + kcov_disable(t, kcov); spin_unlock_irqrestore(&kcov->lock, flags); - return; + kcov_put(kcov); + } + kcov = t->kcov_remote; + if (kcov) { + spin_lock_irqsave(&kcov->lock, flags); + kcov_debug("t = %px, kcov->t = %px\n", t, kcov->t); + /* + * This is a KCOV_REMOTE_ENABLE device, and the task is the + * user task which has requested remote coverage collection. + * Make sure that t->kcov->t is consistent. + */ + if (WARN_ON(!kcov->remote) || WARN_ON(kcov->t != t)) { + spin_unlock_irqrestore(&kcov->lock, flags); + return; + } + /* Just to not leave dangling references behind. */ + kcov_disable(t, kcov); + spin_unlock_irqrestore(&kcov->lock, flags); + kcov_put(kcov); } - /* Just to not leave dangling references behind. */ - kcov_disable(t, kcov); - spin_unlock_irqrestore(&kcov->lock, flags); - kcov_put(kcov); } static int kcov_mmap(struct file *filep, struct vm_area_struct *vma) @@ -629,9 +640,9 @@ static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd, case KCOV_DISABLE: /* Disable coverage for the current task. */ unused = arg; - if (unused != 0 || current->kcov != kcov) - return -EINVAL; t = current; + if (unused != 0 || (kcov != t->kcov && kcov != t->kcov_remote)) + return -EINVAL; if (WARN_ON(kcov->t != t)) return -EINVAL; kcov_disable(t, kcov); @@ -641,7 +652,7 @@ static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd, if (kcov->mode != KCOV_MODE_INIT || !kcov->area) return -EINVAL; t = current; - if (kcov->t != NULL || t->kcov != NULL) + if (kcov->t != NULL || t->kcov_remote != NULL) return -EBUSY; remote_arg = (struct kcov_remote_arg *)arg; mode = kcov_get_mode(remote_arg->trace_mode); @@ -651,8 +662,7 @@ static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd, LONG_MAX / sizeof(unsigned long)) return -EINVAL; kcov->mode = mode; - t->kcov = kcov; - t->kcov_mode = KCOV_MODE_REMOTE; + t->kcov_remote = kcov; kcov->t = t; kcov->remote = true; kcov->remote_size = remote_arg->area_size; -- cgit v1.2.3 From 1eae219ea0e80eed83c129e8cae0f007843f1893 Mon Sep 17 00:00:00 2001 From: Shrikanth Hegde Date: Fri, 29 May 2026 13:27:12 +0530 Subject: sched/topology: Provide arch_llc_mask for cache aware scheduling Venkat Reported a boot kernel panic next-20260522. Git bisect pointed to b5ea300a17e3 ("sched/cache: Make LLC id continuous") Stacktrace points to llc_mask being null. NIP [c000000000e58504] _find_first_bit+0x44/0x130 LR [c000000000e58500] _find_first_bit+0x40/0x130 Call Trace: build_sched_domains+0xad8/0xe50 sched_init_smp+0xa8/0x164 kernel_init_freeable+0x250/0x370 ret_from_kernel_user_thread+0x14/0x1c On powerpc, cpu_coregroup_mask is available only when the underlying hardware support coregroup. In shared LPAR, QEMU guest or power9 etc coregroup isn't supported. In such cases llc_mask was being referenced when it was null leading to panic. On powerpc, LLC is at SMT core level. So assumption that coregroup(MC) domain point to LLC is wrong. Provide a way for archs to say where its LLC is if it not at MC domain. Fixes: b5ea300a17e3 ("sched/cache: Make LLC id continuous") Closes: https://lore.kernel.org/all/51154de7-3700-4cb4-82f2-1b3a8fa427f7@linux.ibm.com/ Reported-by: Venkat Rao Bagalkote Co-developed-by: Chen, Yu C Signed-off-by: Shrikanth Hegde Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Chen Yu Tested-by: Venkat Rao Bagalkote Tested-by: Ritesh Harjani (IBM) Link: https://patch.msgid.link/20260529075712.1181039-1-sshegde@linux.ibm.com --- kernel/sched/topology.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index df2ceb54c970..622e2e01974c 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -2063,12 +2063,21 @@ const struct cpumask *tl_mc_mask(struct sched_domain_topology_level *tl, int cpu return cpu_coregroup_mask(cpu); } -#define llc_mask(cpu) cpu_coregroup_mask(cpu) +/* + * Majority of architectures have LLC at MC domain level with exception + * such as powerpc. Provide a way for arch to specify where its LLC is + * if it falls in exception category + */ +# ifndef arch_llc_mask +#define arch_llc_mask(cpu) cpu_coregroup_mask(cpu) +# endif #else -#define llc_mask(cpu) cpumask_of(cpu) +#define arch_llc_mask(cpu) cpumask_of(cpu) #endif +#define llc_mask(cpu) arch_llc_mask(cpu) + const struct cpumask *tl_pkg_mask(struct sched_domain_topology_level *tl, int cpu) { return cpu_node_mask(cpu); -- cgit v1.2.3 From 4043f549841619a01999bf5d4e0b7931ef87f6cc Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 26 May 2026 12:05:02 +0200 Subject: sched/deadline: Reject debugfs dl_server writes for offline CPUs Writing runtime or period via the per-CPU dl_server debugfs files (/sys/kernel/debug/sched/{fair,ext}_server/cpu*/{runtime,period}) on an offline CPU can trigger two distinct kernel issues: 1) Divide-by-zero in dl_server_apply_params(): Oops: divide error: 0000 [#1] SMP NOPTI RIP: 0010:dl_server_apply_params+0x239/0x3a0 Call Trace: sched_server_write_common.isra.0+0x21a/0x3c0 full_proxy_write+0x78/0xd0 vfs_write+0xe7/0x6e0 Both __dl_sub() and __dl_add() divide by cpus internally, which can be 0 once the CPU has been removed from any active root-domain span (this has been latent since the debugfs interface was introduced). 2) WARN_ON_ONCE in dl_server_start(): WARNING: kernel/sched/deadline.c:1805 at dl_server_start+0x232/0x270 Commit ee6e44dfe6e5 ("sched/deadline: Stop dl_server before CPU goes offline") added this check to catch enqueueing the server on an offline rq. There's no meaningful semantics for re-configuring the per-CPU dl_server bandwidth while the CPU is offline, so simply reject the write with -EBUSY so userspace gets a clear error. Closes: https://lore.kernel.org/all/20260526092228.3B6891F00A3A@smtp.kernel.org/ Fixes: d741f297bcea ("sched/fair: Fair server interface") Reported-by: Sashiko Signed-off-by: Andrea Righi Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Juri Lelli Tested-by: abaci-kreproducer Link: https://patch.msgid.link/20260526100502.575774-1-arighi@nvidia.com --- kernel/sched/debug.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 4809e1d23081..5e09cf9fae3e 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -416,6 +416,9 @@ static ssize_t sched_server_write_common(struct file *filp, const char __user *u return -EINVAL; } + if (!cpu_online(cpu_of(rq))) + return -EBUSY; + update_rq_clock(rq); dl_server_stop(dl_se); retval = dl_server_apply_params(dl_se, runtime, period, 0); -- cgit v1.2.3 From e7b63427fdb4977621d69085a97272c8856644fe Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 26 May 2026 18:42:48 +0200 Subject: sched_ext: Auto-register/unregister dl_server reservations Commit cd959a3562050d ("sched_ext: Add a DL server for sched_ext tasks") introduced an ext_server deadline server to protect sched_ext tasks from fair/RT starvation, mirroring the existing fair_server. Currently, both servers reserve their 50ms/1000ms bandwidth at boot, regardless of whether a BPF scheduler is loaded. Unused bandwidth is still reclaimed at runtime by other classes, but the static reservation prevents the RT class from implicitly using that headroom when one of the two classes is guaranteed to be empty. A sysadmin can work around this by writing /sys/kernel/debug/sched/{fair,ext}_server/cpu*/runtime, but that requires manual action and not all systems expose debugfs. A better approach is to make server bandwidth reservations dynamic: only the scheduling policy that is currently active should register its reservation, while the inactive one should not artificially hold capacity (keeping both reservations only when the BPF scheduler is running in partial mode): +---------------------------------------------+-------------+------------+ | BPF scheduler state | fair server | ext server | +---------------------------------------------+-------------+------------+ | not loaded (default boot) | reserved | none | | loaded full mode (!SCX_OPS_SWITCH_PARTIAL) | none | reserved | | loaded partial mode (SCX_OPS_SWITCH_PARTIAL)| reserved | reserved | +---------------------------------------------+-------------+------------+ To achieve this, introduce an "attached/detached" state for each deadline server, so the kernel can decide whether a server's bandwidth should be accounted in global bandwidth tracking. At boot, the system starts with only the fair server contributing to bandwidth accounting. When a BPF scheduler is enabled, the ext server is attached and may replace or complement the fair server depending on whether full or partial mode is used. When sched_ext is disabled, the system restores the previous deadline bandwidth values and behavior. The transition logic ensures that switching between scheduling modes is consistent and reversible, without losing runtime configuration or requiring manual intervention. Signed-off-by: Andrea Righi Signed-off-by: Peter Zijlstra (Intel) Acked-by: Juri Lelli Link: https://patch.msgid.link/20260526164420.638711-2-arighi@nvidia.com --- kernel/sched/deadline.c | 204 ++++++++++++++++++++++++++++++++++++++++++++++-- kernel/sched/ext.c | 71 +++++++++++++++++ kernel/sched/sched.h | 4 + 3 files changed, 272 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index b60e2df8ff9d..f9e62ed08d77 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -1797,7 +1797,8 @@ void dl_server_start(struct sched_dl_entity *dl_se) struct rq *rq = dl_se->rq; dl_se->dl_defer_idle = 0; - if (!dl_server(dl_se) || dl_se->dl_server_active || !dl_se->dl_runtime) + if (!dl_server(dl_se) || dl_se->dl_server_active || !dl_se->dl_runtime || + !dl_se->dl_bw_attached) return; /* @@ -1872,6 +1873,13 @@ void sched_init_dl_servers(void) dl_se->dl_server = 1; dl_se->dl_defer = 1; setup_new_dl_entity(dl_se); + + /* + * No BPF scheduler is loaded at boot, so the ext_server has no + * tasks to protect. Detach its bandwidth reservation, it will + * be attached when a BPF scheduler is loaded. + */ + dl_server_detach_bw(dl_se); #endif } } @@ -1882,6 +1890,9 @@ void __dl_server_attach_root(struct sched_dl_entity *dl_se, struct rq *rq) int cpu = cpu_of(rq); struct dl_bw *dl_b; + if (!dl_se->dl_bw_attached) + return; + dl_b = dl_bw_of(cpu_of(rq)); guard(raw_spinlock)(&dl_b->lock); @@ -1893,7 +1904,8 @@ void __dl_server_attach_root(struct sched_dl_entity *dl_se, struct rq *rq) int dl_server_apply_params(struct sched_dl_entity *dl_se, u64 runtime, u64 period, bool init) { - u64 old_bw = init ? 0 : to_ratio(dl_se->dl_period, dl_se->dl_runtime); + u64 old_bw = (init || !dl_se->dl_bw_attached) ? 0 : + to_ratio(dl_se->dl_period, dl_se->dl_runtime); u64 new_bw = to_ratio(period, runtime); struct rq *rq = dl_se->rq; int cpu = cpu_of(rq); @@ -1913,7 +1925,8 @@ int dl_server_apply_params(struct sched_dl_entity *dl_se, u64 runtime, u64 perio if (init) { __add_rq_bw(new_bw, &rq->dl); __dl_add(dl_b, new_bw, cpus); - } else { + dl_se->dl_bw_attached = 1; + } else if (dl_se->dl_bw_attached) { __dl_sub(dl_b, dl_se->dl_bw, cpus); __dl_add(dl_b, new_bw, cpus); @@ -1933,6 +1946,181 @@ int dl_server_apply_params(struct sched_dl_entity *dl_se, u64 runtime, u64 perio return 0; } +/* + * Add @dl_se's bw to the root-domain accounting. + * + * Return -EBUSY if attaching would overflow root domain capacity. + */ +static int __dl_server_attach_bw_locked(struct sched_dl_entity *dl_se, + struct dl_bw *dl_b, int cpus) +{ + struct rq *rq = dl_se->rq; + unsigned long cap; + + /* + * Always update @rq->dl.this_bw, but only update @dl_b->total_bw + * (and run the overflow check it gates) while this CPU is active. + * + * This mirrors dl_server_add_bw() during root-domain rebuilds, which + * only publishes bandwidth from active CPUs into @dl_b. + */ + if (cpu_active(cpu_of(rq))) { + cap = dl_bw_capacity(cpu_of(rq)); + if (__dl_overflow(dl_b, cap, 0, dl_se->dl_bw)) + return -EBUSY; + __dl_add(dl_b, dl_se->dl_bw, cpus); + } + __add_rq_bw(dl_se->dl_bw, &rq->dl); + dl_se->dl_bw_attached = 1; + + return 0; +} + +/* + * Drain @dl_se and remove its bw from the root-domain accounting. + */ +static void __dl_server_detach_bw_locked(struct sched_dl_entity *dl_se, + struct dl_bw *dl_b, int cpus) +{ + struct rq *rq = dl_se->rq; + + /* + * If the server is still active (on_rq), dequeue it via + * dl_server_stop(); task_non_contending() will either subtract + * @dl_bw from running_bw immediately (0-lag passed) or set + * dl_non_contending and arm the inactive_timer. + */ + if (dl_se->dl_server_active) + dl_server_stop(dl_se); + + /* + * Drop @dl_se's contribution from this rq's bandwidth accounting, + * mirroring the __add_rq_bw() done at attach time. + */ + dl_rq_change_utilization(rq, dl_se, 0); + + /* + * Update @dl_b only while this CPU is active, matching + * dl_server_add_bw() during root-domain rebuilds. + * + * If this CPU is inactive, its bandwidth is not currently accounted in + * @dl_b->total_bw: either attach skipped adding it, or a rebuild + * already dropped it while re-publishing active CPUs only. + * + * In that case there is nothing to subtract from @dl_b. Just clear + * @dl_se->dl_bw_attached; if the CPU becomes active again, the next + * rebuild will re-publish its bandwidth. + */ + if (cpu_active(cpu_of(rq))) + __dl_sub(dl_b, dl_se->dl_bw, cpus); + dl_se->dl_bw_attached = 0; +} + +/* + * Attach @dl_se's bandwidth to the root domain's total_bw accounting. + * + * Use to dynamically register a dl_server's bandwidth reservation while + * preserving its configured @dl_runtime / @dl_period. No-op if @dl_se is + * already attached. + * + * Returns -EBUSY if attaching would overflow the root domain capacity. + */ +int dl_server_attach_bw(struct sched_dl_entity *dl_se) +{ + struct rq *rq = dl_se->rq; + int cpu = cpu_of(rq); + struct dl_bw *dl_b; + int cpus, ret; + + if (dl_se->dl_bw_attached) + return 0; + + scoped_guard (raw_spinlock, &dl_bw_of(cpu)->lock) { + dl_b = dl_bw_of(cpu); + cpus = dl_bw_cpus(cpu); + ret = __dl_server_attach_bw_locked(dl_se, dl_b, cpus); + } + if (ret) + return ret; + + /* + * The natural 0->nr_running transition that triggers dl_server_start() + * may have happened while @dl_se was still detached (e.g., between + * scx_bypass(false) and the scx_enable() re-balance loop), so kick a + * start here. + * + * dl_server_start() bails out cleanly if there's nothing to schedule or + * it's already active. Skip if @cpu is offline; the server will be + * started naturally on the first enqueue once @cpu comes back. + */ + if (cpu_online(cpu)) + dl_server_start(dl_se); + + return 0; +} + +/* + * Detach @dl_se's bandwidth from the root domain's total_bw accounting. + * + * Use to dynamically unregister a dl_server's bandwidth reservation while + * preserving its configured @dl_runtime / @dl_period. No-op if @dl_se is + * not currently attached. + */ +void dl_server_detach_bw(struct sched_dl_entity *dl_se) +{ + int cpu = cpu_of(dl_se->rq); + struct dl_bw *dl_b; + int cpus; + + if (!dl_se->dl_bw_attached) + return; + + dl_b = dl_bw_of(cpu); + guard(raw_spinlock)(&dl_b->lock); + cpus = dl_bw_cpus(cpu); + __dl_server_detach_bw_locked(dl_se, dl_b, cpus); +} + +/* + * Atomically detach @detach_se and attach @attach_se on the same rq, holding + * @dl_b->lock across both operations so a concurrent sched_setattr() cannot + * steal the bandwidth freed by the detach before the attach can claim it. + * + * Both entities must live on the same rq (same root domain). Returns the + * result of the attach: -EBUSY if attaching @attach_se would overflow root + * domain capacity (in which case both servers end up detached). + */ +int dl_server_swap_bw(struct sched_dl_entity *detach_se, + struct sched_dl_entity *attach_se) +{ + struct rq *rq = detach_se->rq; + int cpu = cpu_of(rq); + struct dl_bw *dl_b; + int cpus, ret; + + WARN_ON_ONCE(attach_se->rq != rq); + + scoped_guard (raw_spinlock, &dl_bw_of(cpu)->lock) { + dl_b = dl_bw_of(cpu); + cpus = dl_bw_cpus(cpu); + + if (detach_se->dl_bw_attached) + __dl_server_detach_bw_locked(detach_se, dl_b, cpus); + + if (attach_se->dl_bw_attached) + ret = 0; + else + ret = __dl_server_attach_bw_locked(attach_se, dl_b, cpus); + } + if (ret) + return ret; + + if (cpu_online(cpu)) + dl_server_start(attach_se); + + return 0; +} + /* * Update the current task's runtime statistics (provided it is still * a -deadline task and has not been removed from the dl_rq). @@ -3233,12 +3421,12 @@ static void dl_server_add_bw(struct root_domain *rd, int cpu) struct sched_dl_entity *dl_se; dl_se = &cpu_rq(cpu)->fair_server; - if (dl_server(dl_se) && cpu_active(cpu)) + if (dl_server(dl_se) && dl_se->dl_bw_attached && cpu_active(cpu)) __dl_add(&rd->dl_bw, dl_se->dl_bw, dl_bw_cpus(cpu)); #ifdef CONFIG_SCHED_CLASS_EXT dl_se = &cpu_rq(cpu)->ext_server; - if (dl_server(dl_se) && cpu_active(cpu)) + if (dl_server(dl_se) && dl_se->dl_bw_attached && cpu_active(cpu)) __dl_add(&rd->dl_bw, dl_se->dl_bw, dl_bw_cpus(cpu)); #endif } @@ -3247,11 +3435,13 @@ static u64 dl_server_read_bw(int cpu) { u64 dl_bw = 0; - if (cpu_rq(cpu)->fair_server.dl_server) + if (cpu_rq(cpu)->fair_server.dl_server && + cpu_rq(cpu)->fair_server.dl_bw_attached) dl_bw += cpu_rq(cpu)->fair_server.dl_bw; #ifdef CONFIG_SCHED_CLASS_EXT - if (cpu_rq(cpu)->ext_server.dl_server) + if (cpu_rq(cpu)->ext_server.dl_server && + cpu_rq(cpu)->ext_server.dl_bw_attached) dl_bw += cpu_rq(cpu)->ext_server.dl_bw; #endif diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 345aa11b84b2..f412c4bb21c3 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -5814,6 +5814,7 @@ static void scx_root_disable(struct scx_sched *sch) struct scx_exit_info *ei = sch->exit_info; struct scx_task_iter sti; struct task_struct *p; + bool was_switched_all; int cpu; /* guarantee forward progress and wait for descendants to be disabled */ @@ -5840,6 +5841,8 @@ static void scx_root_disable(struct scx_sched *sch) */ mutex_lock(&scx_enable_mutex); + was_switched_all = scx_switched_all(); + static_branch_disable(&__scx_switched_all); WRITE_ONCE(scx_switching_all, false); @@ -5889,10 +5892,34 @@ static void scx_root_disable(struct scx_sched *sch) /* * Invalidate all the rq clocks to prevent getting outdated * rq clocks from a previous scx scheduler. + * + * Also re-balance the dl_server bandwidth reservations: detach + * ext_server (no more sched_ext tasks) and reinstate fair_server if it + * was previously detached because we were running in full mode. + * + * Unlike the enable path, this runs on a recovery path that cannot + * fail, so we use dl_server_swap_bw() to atomically free ext_server's + * bandwidth and reclaim it for fair_server under the same dl_b lock. + * + * The swap can still fail with -EBUSY if someone bumped ext_server's + * runtime via debugfs between enable and disable; in that narrow case + * both servers end up detached and we just WARN. */ for_each_possible_cpu(cpu) { struct rq *rq = cpu_rq(cpu); + scx_rq_clock_invalidate(rq); + + scoped_guard(rq_lock_irqsave, rq) { + update_rq_clock(rq); + if (was_switched_all) { + if (WARN_ON_ONCE(dl_server_swap_bw(&rq->ext_server, + &rq->fair_server))) + pr_warn("failed to re-attach fair_server on CPU %d\n", cpu); + } else { + dl_server_detach_bw(&rq->ext_server); + } + } } /* no task is on scx, turn off all the switches and flush in-progress calls */ @@ -6810,6 +6837,31 @@ static void scx_root_enable_workfn(struct kthread_work *work) if (ret) goto err_disable; + /* + * Attach the ext_server bandwidth reservation before anything is + * committed so that we can fail the enable if the root domain cannot + * accommodate it. The matching fair_server detach is deferred to the + * tail of this function, after the switch is fully committed and can no + * longer fail. + * + * On failure, err_disable funnels into scx_root_disable() which + * detaches ext_server, so partially-attached state is cleaned up + * automatically. + */ + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + scoped_guard(rq_lock_irqsave, rq) { + update_rq_clock(rq); + ret = dl_server_attach_bw(&rq->ext_server); + } + if (ret) { + pr_warn("sched_ext: failed to attach ext_server on CPU %d (%d)\n", + cpu, ret); + goto err_disable; + } + } + /* * Once __scx_enabled is set, %current can be switched to SCX anytime. * This can lead to stalls as some BPF schedulers (e.g. userspace @@ -6926,6 +6978,25 @@ static void scx_root_enable_workfn(struct kthread_work *work) if (!(ops->flags & SCX_OPS_SWITCH_PARTIAL)) static_branch_enable(&__scx_switched_all); + /* + * Detach the fair_server bandwidth reservation now that the switch + * is fully committed. In full mode (!SCX_OPS_SWITCH_PARTIAL) no + * task will ever run in the fair class, so give that bandwidth + * back to the RT class. The matching ext_server attach already + * happened earlier; this only releases bandwidth and cannot fail. + * + * In partial mode keep fair_server attached. + */ + if (scx_switched_all()) { + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + guard(rq_lock_irqsave)(rq); + update_rq_clock(rq); + dl_server_detach_bw(&rq->fair_server); + } + } + pr_info("sched_ext: BPF scheduler \"%s\" enabled%s\n", sch->ops.name, scx_switched_all() ? "" : " (partial)"); kobject_uevent(&sch->kobj, KOBJ_ADD); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 6b48bb3074fe..332ecf8930b4 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -421,6 +421,10 @@ extern void ext_server_init(struct rq *rq); extern void __dl_server_attach_root(struct sched_dl_entity *dl_se, struct rq *rq); extern int dl_server_apply_params(struct sched_dl_entity *dl_se, u64 runtime, u64 period, bool init); +extern int dl_server_attach_bw(struct sched_dl_entity *dl_se); +extern void dl_server_detach_bw(struct sched_dl_entity *dl_se); +extern int dl_server_swap_bw(struct sched_dl_entity *detach_se, + struct sched_dl_entity *attach_se); static inline bool dl_server_active(struct sched_dl_entity *dl_se) { -- cgit v1.2.3 From 3b7be8e7fa698359616c3276e005f08c3b6070e4 Mon Sep 17 00:00:00 2001 From: Rik van Riel Date: Tue, 26 May 2026 12:43:29 -0700 Subject: sched/fair: Use rq_clock() in update_tg_load_avg() rate-limit update_tg_load_avg() is called once per leaf cfs_rq from the __update_blocked_fair() walk that runs inside the NOHZ idle-balance softirq, and again from update_load_avg() with UPDATE_TG. Its first operation after the trivial early-outs is unconditionally: now = sched_clock_cpu(cpu_of(rq_of(cfs_rq))); if (now - cfs_rq->last_update_tg_load_avg < NSEC_PER_MSEC) return; Jakub ran into a system where nohz_idle_balance() was taking 75% of a CPU (which is handling network traffic and doing many irq_exit_cpu calls), with 35% of that CPU spent in update_load_avg, and 17% of the CPU in sched_clock_cpu(), reading the TSC. In a quick synthetic test, it looks like this patch reduces the CPU use of sched_balance_update_blocked_averages by about 20%. Switch the rate-limit to read rq_clock(rq_of(cfs_rq)) instead. This eliminates the rdtsc, and uses a fairly fresh timestamp, because all callers of update_tg_load_avg() and clear_tg_load_avg() hold rq->lock and have called update_rq_clock(rq) within microseconds: caller pre-state __update_blocked_fair encloser did update_rq_clock(rq) update_load_avg's three UPDATE_TG sites under rq->lock after enqueue/dequeue/update_curr attach_/detach_entity_cfs_rq preceded by update_load_avg(...) clear_tg_load_avg via offline path rq_clock_start_loop_update(rq) upfront so rq->clock is fresh at every call. Since cfs_rqs are per-CPU per-task_group, cfs_rq->last_update_tg_load_avg is always compared against the same rq's clock; no cross-rq drift. Signed-off-by: Rik van Riel Assisted-by: Claude (Anthropic) Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Vincent Guittot Link: https://patch.msgid.link/20260527110250.6a91718d@fangorn --- kernel/sched/fair.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 62a2dcb0d03e..b5819c4899f1 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -4962,7 +4962,7 @@ static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) * For migration heavy workloads, access to tg->load_avg can be * unbound. Limit the update rate to at most once per ms. */ - now = sched_clock_cpu(cpu_of(rq_of(cfs_rq))); + now = rq_clock(rq_of(cfs_rq)); if (now - cfs_rq->last_update_tg_load_avg < NSEC_PER_MSEC) return; @@ -4985,7 +4985,7 @@ static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq) if (cfs_rq->tg == &root_task_group) return; - now = sched_clock_cpu(cpu_of(rq_of(cfs_rq))); + now = rq_clock(rq_of(cfs_rq)); delta = 0 - cfs_rq->tg_load_avg_contrib; atomic_long_add(delta, &cfs_rq->tg->load_avg); cfs_rq->tg_load_avg_contrib = 0; -- cgit v1.2.3 From 9cb99c598643ba78638dfd668cf020544159cf70 Mon Sep 17 00:00:00 2001 From: Crystal Wood Date: Mon, 11 May 2026 17:30:35 -0500 Subject: tracing/osnoise: Array printk init and cleanup None of the calls to trace_array_printk_buf() will do anything if we don't initialize the buffer on instance creation (unless some other tracer called it), so do that. Add an osnoise_print() function to facilitate adding debug prints (without tainting). Use trace_array_printk() instead of trace_array_printk_buf(), as we're only writing to the main buffer (of a non-main instance) anyway -- and trace_array_printk_buf() skips the check to make sure we're not printing to the global instance. Link: https://patch.msgid.link/20260511223035.1475676-1-crwood@redhat.com Signed-off-by: Crystal Wood Signed-off-by: Steven Rostedt --- kernel/trace/trace_osnoise.c | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c index 62c2667d97fa..5e83c4f6f2b4 100644 --- a/kernel/trace/trace_osnoise.c +++ b/kernel/trace/trace_osnoise.c @@ -83,6 +83,22 @@ struct osnoise_instance { static struct list_head osnoise_instances; +static void osnoise_print(const char *fmt, ...) +{ + struct osnoise_instance *inst; + struct trace_array *tr; + va_list ap; + + rcu_read_lock(); + list_for_each_entry_rcu(inst, &osnoise_instances, list) { + tr = inst->tr; + va_start(ap, fmt); + trace_array_vprintk(tr, _RET_IP_, fmt, ap); + va_end(ap); + } + rcu_read_unlock(); +} + static bool osnoise_has_registered_instances(void) { return !!list_first_or_null_rcu(&osnoise_instances, @@ -123,6 +139,7 @@ static int osnoise_register_instance(struct trace_array *tr) * trace_types_lock. */ lockdep_assert_held(&trace_types_lock); + trace_array_init_printk(tr); inst = kmalloc_obj(*inst); if (!inst) @@ -471,15 +488,7 @@ static void print_osnoise_headers(struct seq_file *s) * osnoise_taint - report an osnoise error. */ #define osnoise_taint(msg) ({ \ - struct osnoise_instance *inst; \ - struct trace_buffer *buffer; \ - \ - rcu_read_lock(); \ - list_for_each_entry_rcu(inst, &osnoise_instances, list) { \ - buffer = inst->tr->array_buffer.buffer; \ - trace_array_printk_buf(buffer, _THIS_IP_, msg); \ - } \ - rcu_read_unlock(); \ + osnoise_print(msg); \ osnoise_data.tainted = true; \ }) @@ -1189,10 +1198,10 @@ static __always_inline void osnoise_stop_exception(char *msg, int cpu) rcu_read_lock(); list_for_each_entry_rcu(inst, &osnoise_instances, list) { tr = inst->tr; - trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_, - "stop tracing hit on cpu %d due to exception: %s\n", - smp_processor_id(), - msg); + trace_array_printk(tr, _THIS_IP_, + "stop tracing hit on cpu %d due to exception: %s\n", + smp_processor_id(), + msg); if (test_bit(OSN_PANIC_ON_STOP, &osnoise_options)) panic("tracer hit on cpu %d due to exception: %s\n", @@ -1362,8 +1371,8 @@ static __always_inline void osnoise_stop_tracing(void) rcu_read_lock(); list_for_each_entry_rcu(inst, &osnoise_instances, list) { tr = inst->tr; - trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_, - "stop tracing hit on cpu %d\n", smp_processor_id()); + trace_array_printk(tr, _THIS_IP_, + "stop tracing hit on cpu %d\n", smp_processor_id()); if (test_bit(OSN_PANIC_ON_STOP, &osnoise_options)) panic("tracer hit stop condition on CPU %d\n", smp_processor_id()); -- cgit v1.2.3 From bfda3b9424e02c6b2afac74410457c9c1743ba4a Mon Sep 17 00:00:00 2001 From: Rosen Penev Date: Fri, 22 May 2026 14:44:07 -0700 Subject: tracing: Turn hist_elt_data field_var_str into a flexible array The field_var_str array was allocated separately via kcalloc() with its length already known at elt_data allocation time. Convert it to a flexible array member and fold the two allocations into a single kzalloc_flex(), reordering hist_trigger_elt_data_alloc() so n_str is computed and bounds-checked before the struct allocation. hist_elt_data is only reached through tracing_map_elt::private_data (a void *), never embedded, so adding a FAM imposes no tail-position constraint on any enclosing struct. Added __counted_by for extra runtime analysis. Link: https://patch.msgid.link/20260522214407.18120-1-rosenp@gmail.com Assisted-by: Claude:Opus-4.7 Signed-off-by: Rosen Penev Signed-off-by: Steven Rostedt --- kernel/trace/trace_events_hist.c | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index 9701650c89b2..82ce492ab268 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -683,8 +683,8 @@ struct track_data { struct hist_elt_data { char *comm; u64 *var_ref_vals; - char **field_var_str; int n_field_var_str; + char *field_var_str[] __counted_by(n_field_var_str); }; struct snapshot_context { @@ -1629,8 +1629,6 @@ static void hist_elt_data_free(struct hist_elt_data *elt_data) for (i = 0; i < elt_data->n_field_var_str; i++) kfree(elt_data->field_var_str[i]); - kfree(elt_data->field_var_str); - kfree(elt_data->comm); kfree(elt_data); } @@ -1650,10 +1648,19 @@ static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt) struct hist_field *hist_field; unsigned int i, n_str; - elt_data = kzalloc_obj(*elt_data); + BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1)); + + n_str = hist_data->n_field_var_str + hist_data->n_save_var_str + + hist_data->n_var_str; + if (n_str > SYNTH_FIELDS_MAX) + return -EINVAL; + + elt_data = kzalloc_flex(*elt_data, field_var_str, n_str); if (!elt_data) return -ENOMEM; + elt_data->n_field_var_str = n_str; + for_each_hist_field(i, hist_data) { hist_field = hist_data->fields[i]; @@ -1667,24 +1674,8 @@ static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt) } } - n_str = hist_data->n_field_var_str + hist_data->n_save_var_str + - hist_data->n_var_str; - if (n_str > SYNTH_FIELDS_MAX) { - hist_elt_data_free(elt_data); - return -EINVAL; - } - - BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1)); - size = STR_VAR_LEN_MAX; - elt_data->field_var_str = kcalloc(n_str, sizeof(char *), GFP_KERNEL); - if (!elt_data->field_var_str) { - hist_elt_data_free(elt_data); - return -EINVAL; - } - elt_data->n_field_var_str = n_str; - for (i = 0; i < n_str; i++) { elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL); if (!elt_data->field_var_str[i]) { -- cgit v1.2.3 From 01046072880b654dbadf71be2f645aad4a7b5d87 Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Mon, 25 May 2026 19:04:28 +0200 Subject: tracing: Disable KCOV instrumentation for trace_irqsoff.o When KCOV runs its boot selftest with whole-kernel instrumentation enabled, it sets current->kcov_mode to KCOV_MODE_TRACE_PC without installing a coverage area. Any instrumented code accepted as task-context coverage in that window dereferences current->kcov_area and crashes. On ARMv5 Versatile PB with CONFIG_KCOV_SELFTEST=y, CONFIG_KCOV_INSTRUMENT_ALL=y and CONFIG_IRQSOFF_TRACER=y, boot hits a NULL pointer fault during the selftest: kcov: running self test Internal error: Oops: 5 [#1] ARM PC is at __sanitizer_cov_trace_pc+0x4c/0x90 Kernel panic - not syncing: Fatal exception A diagnostic run showed the unwanted coverage comes from the IRQs-off tracer callbacks reached from ARM IRQ entry before hardirq context is visible to KCOV: __sanitizer_cov_trace_pc from tracer_hardirqs_off+0x18/0x1cc tracer_hardirqs_off from trace_hardirqs_off+0x34/0x54 trace_hardirqs_off from __irq_svc+0x58/0xb0 __irq_svc from kcov_init+0x7c/0xdc and similarly through tracer_hardirqs_on(). trace_preemptirq.o is already excluded because this tracing path can run from early interrupt code and produce coverage unrelated to syscall inputs. Exclude trace_irqsoff.o as well, instead of requiring users to turn off CONFIG_KCOV_INSTRUMENT_ALL=y, which is the default whole-kernel KCOV mode. With the exclusion in place, the same ARMv5 Versatile PB QEMU test boots through the KCOV selftest and reaches userspace. Tested on ARMv5 Versatile PB QEMU with CONFIG_KCOV_SELFTEST=y, CONFIG_KCOV_INSTRUMENT_ALL=y and CONFIG_IRQSOFF_TRACER=y. Link: https://patch.msgid.link/20260525170428.67211-1-kmehltretter@gmail.com Assisted-by: Codex:gpt-5 Signed-off-by: Karl Mehltretter Signed-off-by: Steven Rostedt --- kernel/trace/Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile index 9b0834134cae..660675e1d426 100644 --- a/kernel/trace/Makefile +++ b/kernel/trace/Makefile @@ -48,9 +48,10 @@ ifdef CONFIG_GCOV_PROFILE_FTRACE GCOV_PROFILE := y endif -# Functions in this file could be invoked from early interrupt -# code and produce random code coverage. +# Functions in these files can run from IRQ entry before hardirq context +# is visible to KCOV, and produce coverage unrelated to syscall inputs. KCOV_INSTRUMENT_trace_preemptirq.o := n +KCOV_INSTRUMENT_trace_irqsoff.o := n CFLAGS_bpf_trace.o := -I$(src) -- cgit v1.2.3 From 9581123304b23049437324038698af9fb56ee663 Mon Sep 17 00:00:00 2001 From: Rik van Riel Date: Wed, 27 May 2026 11:13:01 -0400 Subject: perf/ftrace: Fix WARNING in __unregister_ftrace_function perf_ftrace_function_unregister() unconditionally calls unregister_ftrace_function() without checking whether the ftrace_ops was ever successfully registered. This triggers a WARN_ON in __unregister_ftrace_function() when the ops doesn't have FTRACE_OPS_FL_ENABLED set. This can happen during perf_event_alloc() error cleanup when perf_trace_destroy() is called via __free_event() on an event whose ftrace_ops registration failed or was already torn down by perf_try_init_event()'s err_destroy path. The call path is: perf_event_alloc() error cleanup -> __free_event() -> event->destroy() [tp_perf_event_destroy] -> perf_trace_destroy() -> perf_trace_event_close() -> TRACE_REG_PERF_CLOSE -> perf_ftrace_function_unregister() -> unregister_ftrace_function() -> __unregister_ftrace_function() -> WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)) Fix this by checking FTRACE_OPS_FL_ENABLED before attempting to unregister. If the ops is not enabled, just free the filter and return success. Link: https://patch.msgid.link/20260527111301.2d0d8256@fangorn Signed-off-by: Rik van Riel Signed-off-by: Steven Rostedt --- kernel/trace/trace_event_perf.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/trace/trace_event_perf.c b/kernel/trace/trace_event_perf.c index a6bb7577e8c5..5b272856e5ab 100644 --- a/kernel/trace/trace_event_perf.c +++ b/kernel/trace/trace_event_perf.c @@ -497,7 +497,17 @@ static int perf_ftrace_function_register(struct perf_event *event) static int perf_ftrace_function_unregister(struct perf_event *event) { struct ftrace_ops *ops = &event->ftrace_ops; - int ret = unregister_ftrace_function(ops); + int ret = 0; + + /* + * Perf will call this unconditionally even if the ops is not + * enabled. The unregister_ftrace_function() will warn if called + * when not enabled. Just bypass the unregistering if ops isn't + * enabled here. + */ + if (ops->flags & FTRACE_OPS_FL_ENABLED) + ret = unregister_ftrace_function(ops); + ftrace_free_filter(ops); return ret; } -- cgit v1.2.3 From 64d8eae3f89583821f599c0aa398dd2e69b31a89 Mon Sep 17 00:00:00 2001 From: Marco Crivellari Date: Fri, 29 May 2026 15:06:39 +0200 Subject: workqueue: Add warnings and fallback if system_{unbound}_wq is used Currently many users transitioned already to the new introduced workqueue (system_percpu_wq, system_dfl_wq), but there are new users who still use the older system_wq and system_unbound_wq. This change try to push this transition forward, by warning whether the old workqueues are used. Link: https://lore.kernel.org/all/20250221112003.1dSuoGyc@linutronix.de/ Suggested-by: Tejun Heo Signed-off-by: Marco Crivellari Signed-off-by: Tejun Heo --- kernel/workqueue.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 35b0c8f4f10f..088cd9b70fca 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -2280,6 +2280,14 @@ static void __queue_work(int cpu, struct workqueue_struct *wq, unsigned int work_flags; unsigned int req_cpu = cpu; + /* + * NOTE: Check whether the used workqueue is deprecated and warn + */ + if (unlikely(wq->flags & __WQ_DEPRECATED)) + pr_warn_once("workqueue: work func %ps enqueued on deprecated workqueue. " + "Use system_{percpu|dfl}_wq instead.\n", + work->func); + /* * While a work item is PENDING && off queue, a task trying to * steal the PENDING will busy-loop waiting for it to either get @@ -8013,12 +8021,12 @@ void __init workqueue_init_early(void) ordered_wq_attrs[i] = attrs; } - system_wq = alloc_workqueue("events", WQ_PERCPU, 0); + system_wq = alloc_workqueue("events", WQ_PERCPU | __WQ_DEPRECATED, 0); system_percpu_wq = alloc_workqueue("events", WQ_PERCPU, 0); system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI | WQ_PERCPU, 0); system_long_wq = alloc_workqueue("events_long", WQ_PERCPU, 0); - system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND, WQ_MAX_ACTIVE); + system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND | __WQ_DEPRECATED, WQ_MAX_ACTIVE); system_dfl_wq = alloc_workqueue("events_unbound", WQ_UNBOUND, WQ_MAX_ACTIVE); system_freezable_wq = alloc_workqueue("events_freezable", WQ_FREEZABLE | WQ_PERCPU, 0); -- cgit v1.2.3 From 21c05ca88a548ca1353cbef189c97d4f03b90692 Mon Sep 17 00:00:00 2001 From: Marco Crivellari Date: Fri, 29 May 2026 15:06:40 +0200 Subject: workqueue: Add warnings and ensure one among WQ_PERCPU or WQ_UNBOUND is present Currently there are no checks in order to enforce the use of one between WQ_PERCPU or WQ_UNBOUND. So act as following: - if neither of them is present, set WQ_PERCPU - if both are present, remove WQ_PERCPU Along with this change, WARN_ONCE(), so that the code still uses both or neither of them, can be changed. Link: https://lore.kernel.org/all/20250221112003.1dSuoGyc@linutronix.de/ Suggested-by: Tejun Heo Signed-off-by: Marco Crivellari Signed-off-by: Tejun Heo --- kernel/workqueue.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 088cd9b70fca..34c7c61c69aa 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -5802,7 +5802,7 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt, /* see the comment above the definition of WQ_POWER_EFFICIENT */ if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient) - flags |= WQ_UNBOUND; + flags = (flags & ~WQ_PERCPU) | WQ_UNBOUND; /* allocate wq and format name */ if (flags & WQ_UNBOUND) @@ -5826,6 +5826,23 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt, pr_warn_once("workqueue: name exceeds WQ_NAME_LEN. Truncating to: %s\n", wq->name); + /* + * One among WQ_PERCPU and WQ_UNBOUND must be set, but not both. + * - If neither is set, default to WQ_PERCPU + * - If both are set, default to WQ_UNBOUND + * + * This code can be removed after workqueue are unbound by default + */ + if (unlikely(!(flags & (WQ_UNBOUND | WQ_PERCPU)))) { + WARN_ONCE(1, "workqueue: %s is using neither WQ_PERCPU or WQ_UNBOUND. " + "Setting WQ_PERCPU.\n", wq->name); + flags |= WQ_PERCPU; + } else if (unlikely((flags & WQ_PERCPU) && (flags & WQ_UNBOUND))) { + WARN_ONCE(1, "workqueue: %s uses both WQ_PERCPU and WQ_UNBOUND. " + "Dropped WQ_PERCPU, keeping WQ_UNBOUND.\n", wq->name); + flags &= ~WQ_PERCPU; + } + if (flags & WQ_BH) { /* * BH workqueues always share a single execution context per CPU -- cgit v1.2.3 From 390f2d73bc99a888469f789f274c162da33bafe5 Mon Sep 17 00:00:00 2001 From: Guopeng Zhang Date: Thu, 28 May 2026 17:37:42 +0800 Subject: cgroup/cpuset: Free sched domains on rebuild guard failure generate_sched_domains() returns sched-domain masks and optional attributes that are normally handed to partition_sched_domains(), which takes ownership of them. rebuild_sched_domains_locked() has a WARN guard after generate_sched_domains() and before partition_sched_domains() to avoid passing offline CPUs into the scheduler domain rebuild path. If that guard fires, the function currently returns directly without freeing the generated doms and attr. Free the generated sched-domain masks and attributes before returning from the guard failure path. Signed-off-by: Guopeng Zhang Reviewed-by: Waiman Long Signed-off-by: Tejun Heo --- kernel/cgroup/cpuset.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 8500e4341c60..2a4122b8db29 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -1004,8 +1004,11 @@ void rebuild_sched_domains_locked(void) * prevent the panic. */ for (i = 0; doms && i < ndoms; i++) { - if (WARN_ON_ONCE(!cpumask_subset(doms[i], cpu_active_mask))) + if (WARN_ON_ONCE(!cpumask_subset(doms[i], cpu_active_mask))) { + free_sched_domains(doms, ndoms); + kfree(attr); return; + } } /* Have scheduler rebuild the domains */ -- cgit v1.2.3 From 85e0f27dd1396307913ffc5745b0c05137e9beac Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Mon, 25 May 2026 11:21:14 +0900 Subject: tracing/probes: Point the error offset correctly for eprobe argument error Fix to point the error offset correctly for eprobe argument error. In the cleanup commit 1b8b0cd754cd ("tracing/probes: Move event parameter fetching code to common parser"), due to incorrect backward compatibility aimed at conforming to the test specifications, the error location was set to 0 when a non-existent formal parameter was specified for Eprobe. However, this should be corrected in both the test and the implementation to point correct error position. Link: https://lore.kernel.org/all/177967567399.209006.1451571244515632097.stgit@devnote2/ Fixes: 1b8b0cd754cd ("tracing/probes: Move event parameter fetching code to common parser") Cc: stable@vger.kernel.org Signed-off-by: Masami Hiramatsu (Google) Reviewed-by: Steven Rostedt --- kernel/trace/trace_probe.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index e0d3a0da26af..44c22d4e7881 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -962,8 +962,6 @@ static int parse_probe_vars(char *orig_arg, const struct fetch_type *t, code->op = FETCH_OP_COMM; return 0; } - /* backward compatibility */ - ctx->offset = 0; goto inval; } -- cgit v1.2.3 From 21c4b99b27f3f85b89256e81b3e997dec0a460d0 Mon Sep 17 00:00:00 2001 From: Yuyang Huang Date: Sun, 31 May 2026 15:55:59 +0800 Subject: bpf: fix BPF_PROG_QUERY OOB write and cgroup backward compat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BPF_PROG_QUERY writes back the 'query.revision' field unconditionally to userspace. If userspace passes a smaller 'bpf_attr' structure (e.g. 40 bytes, which was the layout before the addition of 'query.revision'), the kernel performs an out-of-bounds write. Fix this by propagating the user-provided attribute size 'uattr_size' down to the cgroup query handlers, and conditionally skipping writing the revision field to userspace when the provided buffer size is insufficient. query.revision in bpf_mprog_query is structurally identical to the cgroup case: a late tail field, written unconditionally. But the backward-compat hazard is not the same. The min-historical-size test is per command, and bpf_mprog_query only serves attach types that were born with revision in the struct: - tcx_prog_query -> BPF_TCX_INGRESS/EGRESS - netkit_prog_query -> BPF_NETKIT_PRIMARY/PEER tcx, netkit, the revision field, and bpf_mprog_query itself all landed in the same v6.6 merge window (053c8e1f235d added the mprog query API + revision; tcx in e420bed02507, netkit in 35dfaad7188c). There has never been a tcx/netkit BPF_PROG_QUERY userspace that doesn't know about revision. So for these commands the minimum legitimate struct already covers offset 56-64 — no old binary can be broken here. Contrast with cgroup: BPF_PROG_QUERY on cgroup attach types shipped in 2017; revision write-back was bolted on years later (120933984460). That path has a real population of pre-revision callers. Fixes: 120933984460 ("bpf: Implement mprog API on top of existing cgroup progs") Cc: Maciej Żenczykowski Cc: Lorenzo Colitti Signed-off-by: Yuyang Huang Link: https://lore.kernel.org/r/20260531075600.4058207-2-yuyanghuang@google.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/cgroup.c | 13 +++++++------ kernel/bpf/syscall.c | 6 +++--- 2 files changed, 10 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 876f6a81a9b6..2c2bdaa86aa7 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -1208,7 +1208,7 @@ static int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, /* Must be called with cgroup_mutex held to avoid races. */ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, - union bpf_attr __user *uattr) + union bpf_attr __user *uattr, u32 uattr_size) { __u32 __user *prog_attach_flags = u64_to_user_ptr(attr->query.prog_attach_flags); bool effective_query = attr->query.query_flags & BPF_F_QUERY_EFFECTIVE; @@ -1259,7 +1259,8 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, return -EFAULT; if (!effective_query && from_atype == to_atype) revision = cgrp->bpf.revisions[from_atype]; - if (copy_to_user(&uattr->query.revision, &revision, sizeof(revision))) + if (uattr_size >= offsetofend(union bpf_attr, query.revision) && + copy_to_user(&uattr->query.revision, &revision, sizeof(revision))) return -EFAULT; if (attr->query.prog_cnt == 0 || !prog_ids || !total_cnt) /* return early if user requested only program count + flags */ @@ -1312,12 +1313,12 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, } static int cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, - union bpf_attr __user *uattr) + union bpf_attr __user *uattr, u32 uattr_size) { int ret; cgroup_lock(); - ret = __cgroup_bpf_query(cgrp, attr, uattr); + ret = __cgroup_bpf_query(cgrp, attr, uattr, uattr_size); cgroup_unlock(); return ret; } @@ -1520,7 +1521,7 @@ out_put_cgroup: } int cgroup_bpf_prog_query(const union bpf_attr *attr, - union bpf_attr __user *uattr) + union bpf_attr __user *uattr, u32 uattr_size) { struct cgroup *cgrp; int ret; @@ -1529,7 +1530,7 @@ int cgroup_bpf_prog_query(const union bpf_attr *attr, if (IS_ERR(cgrp)) return PTR_ERR(cgrp); - ret = cgroup_bpf_query(cgrp, attr, uattr); + ret = cgroup_bpf_query(cgrp, attr, uattr, uattr_size); cgroup_put(cgrp); return ret; diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 9e91fb2fb492..93bbbe610a7a 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -4719,7 +4719,7 @@ static int bpf_prog_detach(const union bpf_attr *attr) #define BPF_PROG_QUERY_LAST_FIELD query.revision static int bpf_prog_query(const union bpf_attr *attr, - union bpf_attr __user *uattr) + union bpf_attr __user *uattr, u32 uattr_size) { if (!bpf_net_capable()) return -EPERM; @@ -4758,7 +4758,7 @@ static int bpf_prog_query(const union bpf_attr *attr, case BPF_CGROUP_GETSOCKOPT: case BPF_CGROUP_SETSOCKOPT: case BPF_LSM_CGROUP: - return cgroup_bpf_prog_query(attr, uattr); + return cgroup_bpf_prog_query(attr, uattr, uattr_size); case BPF_LIRC_MODE2: return lirc_prog_query(attr, uattr); case BPF_FLOW_DISSECTOR: @@ -6376,7 +6376,7 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size, err = bpf_prog_detach(&attr); break; case BPF_PROG_QUERY: - err = bpf_prog_query(&attr, uattr.user); + err = bpf_prog_query(&attr, uattr.user, size); break; case BPF_PROG_TEST_RUN: err = bpf_prog_test_run(&attr, uattr.user); -- cgit v1.2.3 From a73aa3a5685e648e55787b461f6ee0558db4b0c8 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sun, 31 May 2026 08:01:47 -1000 Subject: sched_ext: Guard BPF arena helper calls to fix 32-bit build BPF arena (kernel/bpf/arena.c) is compiled only on MMU && 64BIT, while SCHED_CLASS_EXT depends on BPF_SYSCALL && BPF_JIT && DEBUG_INFO_BTF with no 64BIT requirement. On a 32-bit arch with a BPF JIT, SCX builds while the arena helpers are absent, so the cid-form code's unconditional calls to bpf_prog_arena() and bpf_arena_map_kern_vm_start() fail to link: build_policy.o: undefined reference to `bpf_prog_arena' build_policy.o: undefined reference to `bpf_arena_map_kern_vm_start' Guard the three call sites with the same MMU && 64BIT condition that gates arena.o. A cid-form scheduler needs a BPF arena, which isn't available on such builds, so it can't run there regardless. cpu-form schedulers don't touch the arena and are unaffected. This is a quick workaround to get past the build errors. A fuller fix may make the whole cid-form path conditional on the same condition, or drop 32-bit support outright. Fixes: 0e2819cba977 ("sched_ext: Require an arena for cid-form schedulers") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202605310454.U9iByL2n-lkp@intel.com/ Closes: https://lore.kernel.org/oe-kbuild-all/202605310926.APXMc0RJ-lkp@intel.com/ Signed-off-by: Tejun Heo --- kernel/sched/ext.c | 14 +++++++++++--- kernel/sched/ext_arena.c | 5 +++++ 2 files changed, 16 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 83272acf1763..32ebbc351564 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -622,8 +622,12 @@ static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, if (scx_is_cid_type()) { struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch); - unsigned long uaddr = (unsigned long)kern_va - - bpf_arena_map_kern_vm_start(sch->arena_map); + unsigned long uaddr = (unsigned long)kern_va; + + /* arena.o, which defines these, is built only on MMU && 64BIT */ +#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) + uaddr -= bpf_arena_map_kern_vm_start(sch->arena_map); +#endif /* * Build the per-CPU arena cmask and hand BPF the uaddr. Caller * holds the rq lock with IRQs disabled, which makes us the sole @@ -7992,8 +7996,12 @@ struct scx_arena_scan { static int scx_arena_scan_prog(struct bpf_prog *prog, void *data) { struct scx_arena_scan *s = data; - struct bpf_map *arena = bpf_prog_arena(prog); + struct bpf_map *arena = NULL; + /* arena.o, which defines these, is built only on MMU && 64BIT */ +#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) + arena = bpf_prog_arena(prog); +#endif if (!arena) return 0; if (s->arena && s->arena != arena) { diff --git a/kernel/sched/ext_arena.c b/kernel/sched/ext_arena.c index b413e155ba6b..493c2424f842 100644 --- a/kernel/sched/ext_arena.c +++ b/kernel/sched/ext_arena.c @@ -80,7 +80,12 @@ static int scx_arena_grow(struct scx_sched *sch, u32 page_cnt) return -ENOMEM; uaddr32 = (u32)(unsigned long)p; + /* arena.o, which defines these, is built only on MMU && 64BIT */ +#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) kern_vm_start = bpf_arena_map_kern_vm_start(sch->arena_map); +#else + kern_vm_start = 0; +#endif ret = gen_pool_add(sch->arena_pool, kern_vm_start + uaddr32, page_cnt * PAGE_SIZE, NUMA_NO_NODE); -- cgit v1.2.3 From bf480c6133461a60e8a4486e560550a20e40ac11 Mon Sep 17 00:00:00 2001 From: Michal Clapinski Date: Thu, 23 Apr 2026 14:25:36 +0200 Subject: kho: fix deferred initialization of scratch areas Currently, if CONFIG_DEFERRED_STRUCT_PAGE_INIT is enabled, kho_release_scratch() will initialize the struct pages and set migratetype of KHO scratch. Unless the whole scratch fits below first_deferred_pfn, some of that will be overwritten either by deferred_init_pages() or memmap_init_reserved_range(). To fix it, make memmap_init_range(), deferred_init_memmap_chunk() and __init_page_from_nid() recognize KHO scratch regions and set migratetype of pageblocks in those regions to MIGRATE_CMA. Co-developed-by: Mike Rapoport (Microsoft) Signed-off-by: Michal Clapinski Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Pasha Tatashin Reviewed-by: Pratyush Yadav (Google) Link: https://patch.msgid.link/20260423122538.140993-2-mclapinski@google.com Signed-off-by: Mike Rapoport (Microsoft) --- kernel/liveupdate/kexec_handover.c | 25 ------------------------- 1 file changed, 25 deletions(-) (limited to 'kernel') diff --git a/kernel/liveupdate/kexec_handover.c b/kernel/liveupdate/kexec_handover.c index 1b592d86dc48..a0aa8281dba1 100644 --- a/kernel/liveupdate/kexec_handover.c +++ b/kernel/liveupdate/kexec_handover.c @@ -1584,35 +1584,10 @@ err_free_scratch: } fs_initcall(kho_init); -static void __init kho_release_scratch(void) -{ - phys_addr_t start, end; - u64 i; - - memmap_init_kho_scratch_pages(); - - /* - * Mark scratch mem as CMA before we return it. That way we - * ensure that no kernel allocations happen on it. That means - * we can reuse it as scratch memory again later. - */ - __for_each_mem_range(i, &memblock.memory, NULL, NUMA_NO_NODE, - MEMBLOCK_KHO_SCRATCH, &start, &end, NULL) { - ulong start_pfn = pageblock_start_pfn(PFN_DOWN(start)); - ulong end_pfn = pageblock_align(PFN_UP(end)); - ulong pfn; - - for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) - init_pageblock_migratetype(pfn_to_page(pfn), - MIGRATE_CMA, false); - } -} - void __init kho_memory_init(void) { if (kho_in.scratch_phys) { kho_scratch = phys_to_virt(kho_in.scratch_phys); - kho_release_scratch(); if (kho_mem_retrieve(kho_get_fdt())) kho_in.fdt_phys = 0; -- cgit v1.2.3 From c6073743d0c7f011461bc542c9112180471affad Mon Sep 17 00:00:00 2001 From: Evangelos Petrongonas Date: Thu, 23 Apr 2026 14:25:37 +0200 Subject: kho: make preserved pages compatible with deferred struct page init When CONFIG_DEFERRED_STRUCT_PAGE_INIT is enabled, struct page initialization is deferred to parallel kthreads that run later in the boot process. During KHO restoration, kho_preserved_memory_reserve() writes metadata for each preserved memory region. However, if the struct page has not been initialized, this write targets uninitialized memory, potentially leading to errors like: BUG: unable to handle page fault for address: ... Fix this by introducing kho_get_preserved_page(), which ensures all struct pages in a preserved region are initialized by calling init_deferred_page() which is a no-op when the struct page is already initialized. Signed-off-by: Evangelos Petrongonas Co-developed-by: Michal Clapinski Signed-off-by: Michal Clapinski Reviewed-by: Pratyush Yadav (Google) Reviewed-by: Pasha Tatashin Reviewed-by: Mike Rapoport (Microsoft) Link: https://patch.msgid.link/20260423122538.140993-3-mclapinski@google.com Signed-off-by: Mike Rapoport (Microsoft) --- kernel/liveupdate/Kconfig | 2 -- kernel/liveupdate/kexec_handover.c | 27 ++++++++++++++++++++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/liveupdate/Kconfig b/kernel/liveupdate/Kconfig index 1a8513f16ef7..c13af38ba23a 100644 --- a/kernel/liveupdate/Kconfig +++ b/kernel/liveupdate/Kconfig @@ -1,12 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-only menu "Live Update and Kexec HandOver" - depends on !DEFERRED_STRUCT_PAGE_INIT config KEXEC_HANDOVER bool "kexec handover" depends on ARCH_SUPPORTS_KEXEC_HANDOVER && ARCH_SUPPORTS_KEXEC_FILE - depends on !DEFERRED_STRUCT_PAGE_INIT select MEMBLOCK_KHO_SCRATCH select KEXEC_FILE select LIBFDT diff --git a/kernel/liveupdate/kexec_handover.c b/kernel/liveupdate/kexec_handover.c index a0aa8281dba1..eb8abd04e7ff 100644 --- a/kernel/liveupdate/kexec_handover.c +++ b/kernel/liveupdate/kexec_handover.c @@ -459,6 +459,31 @@ struct page *kho_restore_pages(phys_addr_t phys, unsigned long nr_pages) } EXPORT_SYMBOL_GPL(kho_restore_pages); +/* + * With CONFIG_DEFERRED_STRUCT_PAGE_INIT, struct pages in higher memory regions + * may not be initialized yet at the time KHO deserializes preserved memory. + * KHO uses the struct page to store metadata and a later initialization would + * overwrite it. + * Ensure all the struct pages in the preservation are + * initialized. kho_preserved_memory_reserve() marks the reservation as noinit + * to make sure they don't get re-initialized later. + */ +static struct page *__init kho_get_preserved_page(phys_addr_t phys, + unsigned int order) +{ + unsigned long pfn = PHYS_PFN(phys); + int nid; + + if (!IS_ENABLED(CONFIG_DEFERRED_STRUCT_PAGE_INIT)) + return pfn_to_page(pfn); + + nid = early_pfn_to_nid(pfn); + for (unsigned long i = 0; i < (1UL << order); i++) + init_deferred_page(pfn + i, nid); + + return pfn_to_page(pfn); +} + static int __init kho_preserved_memory_reserve(phys_addr_t phys, unsigned int order) { @@ -467,7 +492,7 @@ static int __init kho_preserved_memory_reserve(phys_addr_t phys, u64 sz; sz = 1 << (order + PAGE_SHIFT); - page = phys_to_page(phys); + page = kho_get_preserved_page(phys, order); /* Reserve the memory preserved in KHO in memblock */ memblock_reserve(phys, sz); -- cgit v1.2.3 From e947433cd0d2b95a277757451b9b9c2714136dc2 Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Wed, 29 Apr 2026 22:21:14 +0100 Subject: liveupdate: reject LIVEUPDATE_IOCTL_CREATE_SESSION with invalid name length A session name must not be an empty string, and must not exceed the maximum size define in the uapi header, including null termination. Fixes: 0153094d03df ("liveupdate: luo_session: add sessions support") Cc: stable@vger.kernel.org Signed-off-by: Luca Boccassi Reviewed-by: Pasha Tatashin Reviewed-by: Pratyush Yadav Acked-by: Mike Rapoport (Microsoft) Link: https://lore.kernel.org/r/20260429212221.814107-2-luca.boccassi@gmail.com Signed-off-by: Pasha Tatashin Signed-off-by: Mike Rapoport (Microsoft) --- kernel/liveupdate/luo_session.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'kernel') diff --git a/kernel/liveupdate/luo_session.c b/kernel/liveupdate/luo_session.c index 7a42385dabe2..ec7aebc15a80 100644 --- a/kernel/liveupdate/luo_session.c +++ b/kernel/liveupdate/luo_session.c @@ -382,9 +382,13 @@ static int luo_session_getfile(struct luo_session *session, struct file **filep) int luo_session_create(const char *name, struct file **filep) { + size_t len = strnlen(name, LIVEUPDATE_SESSION_NAME_LENGTH); struct luo_session *session; int err; + if (len == 0 || len > LIVEUPDATE_SESSION_NAME_LENGTH - 1) + return -EINVAL; + session = luo_session_alloc(name); if (IS_ERR(session)) return PTR_ERR(session); -- cgit v1.2.3 From 4c08a9f38c50c0df7091b5567be7a3bbac92de0b Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Wed, 29 Apr 2026 22:21:16 +0100 Subject: liveupdate: add LIVEUPDATE_SESSION_GET_NAME ioctl Userspace when requesting a session via the ioctl specifies a name and gets a FD, but then there is no ioctl to go back the other way and get the name given a LUO session FD. This is problematic especially when there is a userspace orchestrator that wants to check what FDs it is handling for clients without having to do manual string scraping of procfs, or without procfs at all. Add a ioctl to simply get the name from an FD. Signed-off-by: Luca Boccassi Reviewed-by: Pasha Tatashin Reviewed-by: Pratyush Yadav Link: https://lore.kernel.org/r/20260429212221.814107-4-luca.boccassi@gmail.com Signed-off-by: Pasha Tatashin Signed-off-by: Mike Rapoport (Microsoft) --- kernel/liveupdate/luo_session.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'kernel') diff --git a/kernel/liveupdate/luo_session.c b/kernel/liveupdate/luo_session.c index ec7aebc15a80..e7f27815c051 100644 --- a/kernel/liveupdate/luo_session.c +++ b/kernel/liveupdate/luo_session.c @@ -289,10 +289,24 @@ static int luo_session_finish(struct luo_session *session, return luo_ucmd_respond(ucmd, sizeof(*argp)); } +static int luo_session_get_name(struct luo_session *session, + struct luo_ucmd *ucmd) +{ + struct liveupdate_session_get_name *argp = ucmd->cmd; + + if (argp->reserved != 0) + return -EINVAL; + + strscpy((char *)argp->name, session->name, sizeof(argp->name)); + + return luo_ucmd_respond(ucmd, sizeof(*argp)); +} + union ucmd_buffer { struct liveupdate_session_finish finish; struct liveupdate_session_preserve_fd preserve; struct liveupdate_session_retrieve_fd retrieve; + struct liveupdate_session_get_name get_name; }; struct luo_ioctl_op { @@ -319,6 +333,8 @@ static const struct luo_ioctl_op luo_session_ioctl_ops[] = { struct liveupdate_session_preserve_fd, token), IOCTL_OP(LIVEUPDATE_SESSION_RETRIEVE_FD, luo_session_retrieve_fd, struct liveupdate_session_retrieve_fd, token), + IOCTL_OP(LIVEUPDATE_SESSION_GET_NAME, luo_session_get_name, + struct liveupdate_session_get_name, name), }; static long luo_session_ioctl(struct file *filep, unsigned int cmd, -- cgit v1.2.3 From d2850ff2f8c5acda4c1097aa53c006a75b091f2c Mon Sep 17 00:00:00 2001 From: David Matlack Date: Thu, 23 Apr 2026 17:40:28 +0000 Subject: liveupdate: Use refcount_t for FLB reference counts Use refcount_t instead of a raw integer to keep track of references on incoming and outgoing FLBs. Using refcount_t provides protection from overflow, underflow, and other issues. Fixes: cab056f2aae7 ("liveupdate: luo_flb: introduce File-Lifecycle-Bound global state") Signed-off-by: David Matlack Reviewed-by: Samiullah Khawaja Reviewed-by: Pasha Tatashin Link: https://lore.kernel.org/r/20260423174032.3140399-2-dmatlack@google.com Signed-off-by: Pasha Tatashin Signed-off-by: Mike Rapoport (Microsoft) --- kernel/liveupdate/luo_flb.c | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) (limited to 'kernel') diff --git a/kernel/liveupdate/luo_flb.c b/kernel/liveupdate/luo_flb.c index 00f5494812c4..59c5f31ab767 100644 --- a/kernel/liveupdate/luo_flb.c +++ b/kernel/liveupdate/luo_flb.c @@ -111,7 +111,7 @@ static int luo_flb_file_preserve_one(struct liveupdate_flb *flb) struct luo_flb_private *private = luo_flb_get_private(flb); scoped_guard(mutex, &private->outgoing.lock) { - if (!private->outgoing.count) { + if (!refcount_read(&private->outgoing.count)) { struct liveupdate_flb_op_args args = {0}; int err; @@ -126,8 +126,10 @@ static int luo_flb_file_preserve_one(struct liveupdate_flb *flb) } private->outgoing.data = args.data; private->outgoing.obj = args.obj; + refcount_set(&private->outgoing.count, 1); + } else { + refcount_inc(&private->outgoing.count); } - private->outgoing.count++; } return 0; @@ -138,8 +140,7 @@ static void luo_flb_file_unpreserve_one(struct liveupdate_flb *flb) struct luo_flb_private *private = luo_flb_get_private(flb); scoped_guard(mutex, &private->outgoing.lock) { - private->outgoing.count--; - if (!private->outgoing.count) { + if (refcount_dec_and_test(&private->outgoing.count)) { struct liveupdate_flb_op_args args = {0}; args.flb = flb; @@ -178,7 +179,7 @@ static int luo_flb_retrieve_one(struct liveupdate_flb *flb) for (int i = 0; i < fh->header_ser->count; i++) { if (!strcmp(fh->ser[i].name, flb->compatible)) { private->incoming.data = fh->ser[i].data; - private->incoming.count = fh->ser[i].count; + refcount_set(&private->incoming.count, fh->ser[i].count); found = true; break; } @@ -208,12 +209,8 @@ static int luo_flb_retrieve_one(struct liveupdate_flb *flb) static void luo_flb_file_finish_one(struct liveupdate_flb *flb) { struct luo_flb_private *private = luo_flb_get_private(flb); - u64 count; - scoped_guard(mutex, &private->incoming.lock) - count = --private->incoming.count; - - if (!count) { + if (refcount_dec_and_test(&private->incoming.count)) { struct liveupdate_flb_op_args args = {0}; if (!private->incoming.retrieved) { @@ -652,12 +649,13 @@ void luo_flb_serialize(void) guard(rwsem_read)(&luo_register_rwlock); list_private_for_each_entry(gflb, &luo_flb_global.list, private.list) { struct luo_flb_private *private = luo_flb_get_private(gflb); + long count = refcount_read(&private->outgoing.count); - if (private->outgoing.count > 0) { + if (count > 0) { strscpy(fh->ser[i].name, gflb->compatible, sizeof(fh->ser[i].name)); fh->ser[i].data = private->outgoing.data; - fh->ser[i].count = private->outgoing.count; + fh->ser[i].count = count; i++; } } -- cgit v1.2.3 From d8e47bd066d7e626f9f45d416182d585b7e18b9b Mon Sep 17 00:00:00 2001 From: David Matlack Date: Thu, 23 Apr 2026 17:40:29 +0000 Subject: liveupdate: Reference count incoming FLB data Increment the incoming FLB refcount in liveupdate_flb_get_incoming() so that the FLB structure cannot be freed while the caller is actively using it. Add an additional liveupdate_flb_put_incoming() function so the caller can explicitly indicate when it is done using the FLB data. During a Live Update, a subsystem might need to hold onto the incoming File-Lifecycle-Bound (FLB) data for an extended period, such as during device enumeration. Incrementing the reference count guarantees that the data remains valid and accessible until the subsystem releases it, preventing future use-after-free bugs. Fixes: cab056f2aae7 ("liveupdate: luo_flb: introduce File-Lifecycle-Bound global state") Signed-off-by: David Matlack Reviewed-by: Samiullah Khawaja Reviewed-by: Pasha Tatashin Link: https://lore.kernel.org/r/20260423174032.3140399-3-dmatlack@google.com Signed-off-by: Pasha Tatashin Signed-off-by: Mike Rapoport (Microsoft) --- kernel/liveupdate/luo_flb.c | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) (limited to 'kernel') diff --git a/kernel/liveupdate/luo_flb.c b/kernel/liveupdate/luo_flb.c index 59c5f31ab767..8f5c5dd01cd0 100644 --- a/kernel/liveupdate/luo_flb.c +++ b/kernel/liveupdate/luo_flb.c @@ -165,7 +165,7 @@ static int luo_flb_retrieve_one(struct liveupdate_flb *flb) bool found = false; int err; - guard(mutex)(&private->incoming.lock); + lockdep_assert_held(&private->incoming.lock); if (private->incoming.finished) return -ENODATA; @@ -206,12 +206,14 @@ static int luo_flb_retrieve_one(struct liveupdate_flb *flb) return 0; } -static void luo_flb_file_finish_one(struct liveupdate_flb *flb) +void liveupdate_flb_put_incoming(struct liveupdate_flb *flb) { struct luo_flb_private *private = luo_flb_get_private(flb); + struct liveupdate_flb_op_args args = {0}; - if (refcount_dec_and_test(&private->incoming.count)) { - struct liveupdate_flb_op_args args = {0}; + scoped_guard(mutex, &private->incoming.lock) { + if (!refcount_dec_and_test(&private->incoming.count)) + return; if (!private->incoming.retrieved) { int err = luo_flb_retrieve_one(flb); @@ -220,16 +222,14 @@ static void luo_flb_file_finish_one(struct liveupdate_flb *flb) return; } - scoped_guard(mutex, &private->incoming.lock) { - args.flb = flb; - args.obj = private->incoming.obj; - flb->ops->finish(&args); + args.flb = flb; + args.obj = private->incoming.obj; + flb->ops->finish(&args); - private->incoming.data = 0; - private->incoming.obj = NULL; - private->incoming.finished = true; - module_put(flb->ops->owner); - } + private->incoming.data = 0; + private->incoming.obj = NULL; + private->incoming.finished = true; + module_put(flb->ops->owner); } } @@ -312,7 +312,7 @@ void luo_flb_file_finish(struct liveupdate_file_handler *fh) guard(rwsem_read)(&luo_register_rwlock); list_for_each_entry_reverse(iter, flb_list, list) - luo_flb_file_finish_one(iter->flb); + liveupdate_flb_put_incoming(iter->flb); } static void luo_flb_unregister_one(struct liveupdate_file_handler *fh, @@ -509,6 +509,8 @@ int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, void **objp) if (!liveupdate_enabled()) return -EOPNOTSUPP; + guard(mutex)(&private->incoming.lock); + if (!private->incoming.obj) { int err = luo_flb_retrieve_one(flb); @@ -516,7 +518,7 @@ int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, void **objp) return err; } - guard(mutex)(&private->incoming.lock); + refcount_inc(&private->incoming.count); *objp = private->incoming.obj; return 0; -- cgit v1.2.3 From 4e8729e6598ba0d10021dbf48b308cd53a06bbc4 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Thu, 28 May 2026 22:37:38 -0400 Subject: ring-buffer: Better comment the use of RB_MISSED_EVENTS If the persistent ring buffer is detected on boot up to have a corrupted sub-buffer, that sub-buffer is cleared to zero and its commit value has the RB_MISSED_EVENTS bit set. That bit is to allow the "trace", "trace_pipe" and "trace_pipe_raw" files know that events were dropped by outputting "[LOST EVENTS]". Only in this case does that bit get set in the writeable portion of the ring buffer. When events are dropped in the normal ring buffer, that information is stored in the cpu_buffer descriptor and the RB_MISSED_EVENTS is set in the buffer page at the time the page is consumed. It is never set in the writeable portion of the buffer. Add comments to describe this better as it can be confusing to know when the RB_MISSED_EVENTS are set in the commit portion of the buffer page. Link: https://lore.kernel.org/all/20260529001500.14178455a046a5cbc6180861@kernel.org/ Acked-by: Masami Hiramatsu (Google) Link: https://patch.msgid.link/20260528223738.41276c0e@fedora Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'kernel') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 910f6b3adf74..06fb365bb86e 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -1929,6 +1929,12 @@ static int __rb_validate_buffer(struct buffer_page *bpage, int cpu, */ if (ret < 0 || (prev_ts && prev_ts > ts) || (next_ts && ts > next_ts)) { local_set(&bpage->entries, 0); + /* + * Note, the RB_MISSED_EVENTS is only set inside the main write + * buffer by this verification logic. The normal ring buffer + * has this bit set when the page is read and passed to the + * consumers. + */ local_set(&dpage->commit, RB_MISSED_EVENTS); dpage->time_stamp = prev_ts ? prev_ts : next_ts; ret = -1; @@ -7232,6 +7238,14 @@ int ring_buffer_read_page(struct trace_buffer *buffer, local_add(RB_MISSED_STORED, &dpage->commit); size += sizeof(missed_events); } + /* + * Note, for the persistent ring buffer, the RB_MISSED_EVENTS + * may have been set in the main buffer via the verification code. + * But here, dpage is a copy of that page and has not yet had + * the RB_MISSED_EVENTS set. As for the normal buffers, + * the main write buffer does not set these bits and it needs + * to be set here. + */ local_add(RB_MISSED_EVENTS, &dpage->commit); } -- cgit v1.2.3 From de36adca634634c205a9eb8b56a28175ab7abf5f Mon Sep 17 00:00:00 2001 From: Taegu Ha Date: Thu, 28 May 2026 15:21:55 +0900 Subject: bpf: reject overlarge global subprog argument sizes Global subprogram argument checking derives generic pointer sizes from BTF and passes the resolved size to check_mem_reg() as a u32. The access-size validation path then uses a signed int, and stack pointers negate the value before calling check_helper_mem_access(). This creates a wrap when BTF describes a pointee size larger than S32_MAX. For example, a global subprogram argument of type: int (*p)[0x3fffffff] has a BTF-resolved pointee size of 0xfffffffc bytes. At a call site the caller can pass a pointer to a 4-byte stack slot at fp-4. The current PTR_TO_STACK path computes: size = -(int)mem_size so 0xfffffffc becomes -4 as a signed int and the negation validates only a 4-byte stack range. That range is covered by the caller's stack slot, so the call is accepted. The callee is then verified independently with R1 as PTR_TO_MEM and mem_size 0xfffffffc. A small instruction such as: r0 = *(u32 *)(r1 + 4) is accepted as being inside that BTF-described memory region. At run time, however, the actual argument value is still fp-4, so r1 + 4 addresses fp+0, outside the 4-byte object that the caller provided. Reject sizes that cannot be represented by the verifier's signed access-size API before the stack-specific negation. Add a verifier regression test for the oversized BTF argument. Fixes: 2cb27158adb3 ("bpf: poison dead stack slots") Signed-off-by: Taegu Ha Acked-by: Yonghong Song Link: https://lore.kernel.org/r/20260528062155.3988156-1-hataegu0826@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index c8d980fdd709..3a270bc485c2 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6927,6 +6927,12 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg if (bpf_register_is_null(reg)) return 0; + if (mem_size > S32_MAX) { + verbose(env, "%s memory size %u is too large\n", + reg_arg_name(env, argno), mem_size); + return -EACCES; + } + /* Assuming that the register contains a value check if the memory * access is safe. Temporarily save and restore the register's state as * the conversion shouldn't be visible to a caller. -- cgit v1.2.3 From 868d43cf8f970b456fd93334bee40f792cf27e4d Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Sat, 23 May 2026 12:00:26 -0400 Subject: bpf: Fix security_bpf_prog_load() error handling If security_bpf_prog_load() fails there is no need to call into security_bpf_prog_free() as the LSM will handle the cleanup of any partial LSM state before returning to the caller with an error. Thankfully this isn't an issue with any of the existing code as the LSMs which currently provide BPF hook callback implementations don't allocate any internal state, but this is something we want to fix for potential future users. Signed-off-by: Paul Moore Link: https://lore.kernel.org/r/20260523160025.16363-2-paul@paul-moore.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/syscall.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 93bbbe610a7a..2aafd2131983 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -3136,7 +3136,7 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at err = security_bpf_prog_load(prog, attr, token, uattr.is_kernel); if (err) - goto free_prog_sec; + goto free_prog; /* run eBPF verifier */ err = bpf_check(&prog, attr, uattr, attr_log); @@ -3182,8 +3182,6 @@ free_used_maps: __bpf_prog_put_noref(prog, prog->aux->real_func_cnt); return err; -free_prog_sec: - security_bpf_prog_free(prog); free_prog: free_uid(prog->aux->user); if (prog->aux->attach_btf) -- cgit v1.2.3 From 507e3b479f9c6d85135eb5e1a77fb3fddb259ad8 Mon Sep 17 00:00:00 2001 From: "Pratyush Yadav (Google)" Date: Tue, 19 May 2026 14:24:26 +0200 Subject: liveupdate: validate session type before performing operation The sessions ioctls are not applicable to all session types. PRESERVE_FD is only applicable to outgoing sessions. RETRIEVE_FD and FINISH are only valid for incoming session. Calling a incoming ioctl on an outgoing session is invalid and can cause file handlers to run into unexpected errors. For example, a user can create a (outgoing) session, preserve a memfd, and then immediately do a retrieve without doing a kexec in between. This would result in memfd's retrieve handler to run. The handlers expects to be called from a post-kexec context, and will try to do a kho_restore_vmalloc() or kho_restore_folio() to try and restore memory. KHO catches this (thanks to KHO_PAGE_MAGIC) and returns an error, but since this is considered an internal error and KHO throws out a bunch of WARN()s. Associate a type with each ioctl op and validate the type in luo_session_ioctl() before dispatching the ioctl handler to make sure the op is being called for the right session type. Fixes: 16cec0d26521 ("liveupdate: luo_session: add ioctls for file preservation") Cc: stable@vger.kernel.org Signed-off-by: Pratyush Yadav (Google) Link: https://patch.msgid.link/20260519122428.2378446-1-pratyush@kernel.org Signed-off-by: Mike Rapoport (Microsoft) --- kernel/liveupdate/luo_session.c | 39 +++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/liveupdate/luo_session.c b/kernel/liveupdate/luo_session.c index e7f27815c051..099db679bdc5 100644 --- a/kernel/liveupdate/luo_session.c +++ b/kernel/liveupdate/luo_session.c @@ -309,34 +309,60 @@ union ucmd_buffer { struct liveupdate_session_get_name get_name; }; +/* Type of sessions the ioctl applies to. */ +enum luo_ioctl_type { + LUO_IOCTL_INCOMING, + LUO_IOCTL_OUTGOING, + LUO_IOCTL_ALL, +}; + struct luo_ioctl_op { unsigned int size; unsigned int min_size; unsigned int ioctl_num; + enum luo_ioctl_type type; int (*execute)(struct luo_session *session, struct luo_ucmd *ucmd); }; -#define IOCTL_OP(_ioctl, _fn, _struct, _last) \ +#define IOCTL_OP(_ioctl, _fn, _struct, _last, _type) \ [_IOC_NR(_ioctl) - LIVEUPDATE_CMD_SESSION_BASE] = { \ .size = sizeof(_struct) + \ BUILD_BUG_ON_ZERO(sizeof(union ucmd_buffer) < \ sizeof(_struct)), \ .min_size = offsetofend(_struct, _last), \ .ioctl_num = _ioctl, \ + .type = _type, \ .execute = _fn, \ } static const struct luo_ioctl_op luo_session_ioctl_ops[] = { IOCTL_OP(LIVEUPDATE_SESSION_FINISH, luo_session_finish, - struct liveupdate_session_finish, reserved), + struct liveupdate_session_finish, reserved, LUO_IOCTL_INCOMING), IOCTL_OP(LIVEUPDATE_SESSION_PRESERVE_FD, luo_session_preserve_fd, - struct liveupdate_session_preserve_fd, token), + struct liveupdate_session_preserve_fd, token, LUO_IOCTL_OUTGOING), IOCTL_OP(LIVEUPDATE_SESSION_RETRIEVE_FD, luo_session_retrieve_fd, - struct liveupdate_session_retrieve_fd, token), + struct liveupdate_session_retrieve_fd, token, LUO_IOCTL_INCOMING), IOCTL_OP(LIVEUPDATE_SESSION_GET_NAME, luo_session_get_name, - struct liveupdate_session_get_name, name), + struct liveupdate_session_retrieve_fd, token, LUO_IOCTL_ALL), }; +static bool luo_ioctl_type_valid(struct luo_session *session, + const struct luo_ioctl_op *op) +{ + switch (op->type) { + case LUO_IOCTL_INCOMING: + /* Retrieved is only set on incoming sessions */ + return session->retrieved; + case LUO_IOCTL_OUTGOING: + return !session->retrieved; + case LUO_IOCTL_ALL: + return true; + } + + /* Catch-all. */ + return false; +} + static long luo_session_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) { @@ -361,6 +387,8 @@ static long luo_session_ioctl(struct file *filep, unsigned int cmd, op = &luo_session_ioctl_ops[nr - LIVEUPDATE_CMD_SESSION_BASE]; if (op->ioctl_num != cmd) return -ENOIOCTLCMD; + if (!luo_ioctl_type_valid(session, op)) + return -EINVAL; if (ucmd.user_size < op->min_size) return -EINVAL; @@ -631,4 +659,3 @@ err_undo: return err; } - -- cgit v1.2.3 From 6bd280c7feebd922144fd74869ee222c22ddd042 Mon Sep 17 00:00:00 2001 From: "Pratyush Yadav (Google)" Date: Tue, 19 May 2026 14:57:06 +0200 Subject: liveupdate: document liveupdate=on While the liveupdate= parameter is documented in kernel-parameters.txt, it is not listed in LUO's user facing documentation. This can make it hard for users to figure out how to enable the subsystem, since enabling just the config isn't enough. Note the need for the kernel parameter in LUO core documentation, which gets exported to Documentation/core-api/liveupdate.rst. Suggested-by: Luca Boccassi Signed-off-by: Pratyush Yadav (Google) Acked-by: Luca Boccassi Link: https://patch.msgid.link/20260519125714.2435640-1-pratyush@kernel.org Signed-off-by: Pasha Tatashin Signed-off-by: Mike Rapoport (Microsoft) --- kernel/liveupdate/luo_core.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'kernel') diff --git a/kernel/liveupdate/luo_core.c b/kernel/liveupdate/luo_core.c index 803f51c84275..5d5827ced73c 100644 --- a/kernel/liveupdate/luo_core.c +++ b/kernel/liveupdate/luo_core.c @@ -36,6 +36,10 @@ * * LUO uses Kexec Handover to transfer memory state from the current kernel to * the next kernel. For more details see Documentation/core-api/kho/index.rst. + * + * .. note:: + * To enable LUO, boot the kernel with the ``liveupdate=on`` command line + * parameter. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -- cgit v1.2.3 From 0e39380a7316122e1b00012b3f3cd3e318b3e7d3 Mon Sep 17 00:00:00 2001 From: "Pratyush Yadav (Google)" Date: Tue, 19 May 2026 18:05:49 +0200 Subject: kho: make sure scratch size is always aligned by CMA_MIN_ALIGNMENT_BYTES When using scratch_scale, the scratch sizes are rounded up to CMA_MIN_ALIGNMENT_BYTES since they will be released as MIGRATE_CMA. This is not done when using fixed scratch sizes via command line. This can result in user specifying a size which is not aligned, and thus kernel releasing a pageblock that is only partially scratch. Do the rounding up for both cases in scratch_size_update(). Fixes: 3dc92c311498 ("kexec: add Kexec HandOver (KHO) generation helpers") Cc: stable@kernel.org Signed-off-by: Pratyush Yadav (Google) Link: https://patch.msgid.link/20260519160554.2713361-1-pratyush@kernel.org Signed-off-by: Pasha Tatashin Signed-off-by: Mike Rapoport (Microsoft) --- kernel/liveupdate/kexec_handover.c | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/kernel/liveupdate/kexec_handover.c b/kernel/liveupdate/kexec_handover.c index eb8abd04e7ff..4834a809985a 100644 --- a/kernel/liveupdate/kexec_handover.c +++ b/kernel/liveupdate/kexec_handover.c @@ -618,20 +618,30 @@ early_param("kho_scratch", kho_parse_scratch_size); static void __init scratch_size_update(void) { - phys_addr_t size; + /* + * If fixed sizes are not provided via command line, calculate them + * now. + */ + if (scratch_scale) { + phys_addr_t size; - if (!scratch_scale) - return; + size = memblock_reserved_kern_size(ARCH_LOW_ADDRESS_LIMIT, + NUMA_NO_NODE); + size = size * scratch_scale / 100; + scratch_size_lowmem = size; - size = memblock_reserved_kern_size(ARCH_LOW_ADDRESS_LIMIT, - NUMA_NO_NODE); - size = size * scratch_scale / 100; - scratch_size_lowmem = round_up(size, CMA_MIN_ALIGNMENT_BYTES); + size = memblock_reserved_kern_size(MEMBLOCK_ALLOC_ANYWHERE, + NUMA_NO_NODE); + size = size * scratch_scale / 100 - scratch_size_lowmem; + scratch_size_global = size; + } - size = memblock_reserved_kern_size(MEMBLOCK_ALLOC_ANYWHERE, - NUMA_NO_NODE); - size = size * scratch_scale / 100 - scratch_size_lowmem; - scratch_size_global = round_up(size, CMA_MIN_ALIGNMENT_BYTES); + /* + * Scratch areas are released as MIGRATE_CMA. Round them up to the right + * size. + */ + scratch_size_lowmem = round_up(scratch_size_lowmem, CMA_MIN_ALIGNMENT_BYTES); + scratch_size_global = round_up(scratch_size_global, CMA_MIN_ALIGNMENT_BYTES); } static phys_addr_t __init scratch_size_node(int nid) -- cgit v1.2.3 From 5eff62b051fbdb686e885c1468301d964f2e3d66 Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Wed, 27 May 2026 20:27:33 +0000 Subject: liveupdate: skip serialization for context-preserving kexec A preserve_context kexec returns to the current kernel, which is unrelated to live update where the state is passed to the next kernel. Skip liveupdate_reboot() in this case to avoid serialization and prevent sessions from being left in a frozen state upon return. Fixes: db8bed8082dc ("kexec: call liveupdate_reboot() before kexec") Reported-by: Oskar Gerlicz Kowalczuk Reviewed-by: Pratyush Yadav (Google) Acked-by: Mike Rapoport (Microsoft) Signed-off-by: Pasha Tatashin Link: https://patch.msgid.link/20260527202737.1345192-2-pasha.tatashin@soleen.com Signed-off-by: Mike Rapoport (Microsoft) --- kernel/kexec_core.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index a43d2da0fe3e..dc770b9a6d05 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -1146,9 +1146,11 @@ int kernel_kexec(void) goto Unlock; } - error = liveupdate_reboot(); - if (error) - goto Unlock; + if (!kexec_image->preserve_context) { + error = liveupdate_reboot(); + if (error) + goto Unlock; + } #ifdef CONFIG_KEXEC_JUMP if (kexec_image->preserve_context) { -- cgit v1.2.3 From d3ae9e7fddb4036f50003d7fa1ef52801fdb961b Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Wed, 27 May 2026 20:27:34 +0000 Subject: liveupdate: fix TOCTOU race in luo_session_retrieve() Extend the scope of the rwsem_read lock in luo_session_retrieve() to overlap with the acquisition of the session mutex. This prevents a concurrent thread from releasing and freeing the session between the lookup and the mutex lock. Fixes: 0153094d03df ("liveupdate: luo_session: add sessions support") Acked-by: Mike Rapoport (Microsoft) Reviewed-by: Pratyush Yadav (Google) Signed-off-by: Pasha Tatashin Link: https://patch.msgid.link/20260527202737.1345192-3-pasha.tatashin@soleen.com Signed-off-by: Mike Rapoport (Microsoft) --- kernel/liveupdate/luo_session.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/liveupdate/luo_session.c b/kernel/liveupdate/luo_session.c index 099db679bdc5..a1c742eeb444 100644 --- a/kernel/liveupdate/luo_session.c +++ b/kernel/liveupdate/luo_session.c @@ -463,12 +463,11 @@ int luo_session_retrieve(const char *name, struct file **filep) struct luo_session *it; int err; - scoped_guard(rwsem_read, &sh->rwsem) { - list_for_each_entry(it, &sh->list, list) { - if (!strncmp(it->name, name, sizeof(it->name))) { - session = it; - break; - } + guard(rwsem_read)(&sh->rwsem); + list_for_each_entry(it, &sh->list, list) { + if (!strncmp(it->name, name, sizeof(it->name))) { + session = it; + break; } } -- cgit v1.2.3 From bb1328be35bf43c88288c5c31ceb45181b574c0c Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Wed, 27 May 2026 20:27:35 +0000 Subject: liveupdate: block session mutations during reboot During the reboot() syscall, user processes may still be running concurrently and attempting to mutate sessions (e.g., creating, retrieving, or releasing sessions). To prevent this, introduce luo_session_serialize_rwsem to synchronize mutations with the serialization process. All session mutation operations (create, retrieve, release, ioctl) take the read lock. The serialization process (luo_session_serialize) takes the write lock and holds it indefinitely on success. This effectively freezes the LUO session subsystem during the transition to the new kernel. If serialization fails, the lock is released to allow recovery. Fixes: 0153094d03df ("liveupdate: luo_session: add sessions support") Reported-by: Oskar Gerlicz Kowalczuk Acked-by: Mike Rapoport (Microsoft) Signed-off-by: Pasha Tatashin Reviewed-by: Pratyush Yadav (Google) Link: https://patch.msgid.link/20260527202737.1345192-4-pasha.tatashin@soleen.com Signed-off-by: Mike Rapoport (Microsoft) --- kernel/liveupdate/luo_session.c | 51 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/liveupdate/luo_session.c b/kernel/liveupdate/luo_session.c index a1c742eeb444..5c6cebc6e326 100644 --- a/kernel/liveupdate/luo_session.c +++ b/kernel/liveupdate/luo_session.c @@ -46,6 +46,38 @@ * 4. Retrieval: A userspace agent in the new kernel can then call * `luo_session_retrieve()` with a session name to get a new file * descriptor and access the preserved state. + * + * Locking: + * + * The LUO session subsystem uses a three-tier locking hierarchy to ensure thread + * safety and prevent deadlocks during concurrent session mutations and kexec + * serialization: + * + * 1. `luo_session_serialize_rwsem` (global rwsem): + * Protects session mutations (creation, retrieval, release, and ioctls) + * against the serialization process during reboot. + * + * - Readers: Taken by any path modifying or accessing session state (e.g., + * `luo_session_create()`, `luo_session_retrieve()`, `luo_session_release()`, + * and `luo_session_ioctl()`). + * - Writer: Taken by the serialization process (`luo_session_serialize()`) + * during reboot. On success, the write lock is held indefinitely to freeze + * the subsystem. On failure, it is released to allow recovery. + * + * 2. `luo_session_header->rwsem` (per-list rwsem): + * Synchronizes list-level operations for the incoming and outgoing session headers. + * + * - Writer: Taken during list mutation operations (inserting or removing a + * session from the list). + * - Reader: Taken when traversing the list (e.g., retrieving a session by name). + * + * 3. `luo_session->mutex` (per-session mutex): + * Protects the internal state and file sets of an individual session. It is + * acquired during per-session operations such as preserving, retrieving, + * or freezing files. + * + * Lock Hierarchy: + * `luo_session_serialize_rwsem` -> `luo_session_header->rwsem` -> `luo_session->mutex` */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -75,6 +107,8 @@ sizeof(struct luo_session_header_ser)) / \ sizeof(struct luo_session_ser)) +static DECLARE_RWSEM(luo_session_serialize_rwsem); + /** * struct luo_session_header - Header struct for managing LUO sessions. * @count: The number of sessions currently tracked in the @list. @@ -205,6 +239,7 @@ static int luo_session_release(struct inode *inodep, struct file *filep) struct luo_session *session = filep->private_data; struct luo_session_header *sh; + guard(rwsem_read)(&luo_session_serialize_rwsem); /* If retrieved is set, it means this session is from incoming list */ if (session->retrieved) { int err = luo_session_finish_one(session); @@ -398,6 +433,7 @@ static long luo_session_ioctl(struct file *filep, unsigned int cmd, if (ret) return ret; + guard(rwsem_read)(&luo_session_serialize_rwsem); return op->execute(session, &ucmd); } @@ -437,14 +473,17 @@ int luo_session_create(const char *name, struct file **filep) if (IS_ERR(session)) return PTR_ERR(session); + down_read(&luo_session_serialize_rwsem); err = luo_session_insert(&luo_session_global.outgoing, session); if (err) goto err_free; - scoped_guard(mutex, &session->mutex) - err = luo_session_getfile(session, filep); + mutex_lock(&session->mutex); + err = luo_session_getfile(session, filep); + mutex_unlock(&session->mutex); if (err) goto err_remove; + up_read(&luo_session_serialize_rwsem); return 0; @@ -452,6 +491,7 @@ err_remove: luo_session_remove(&luo_session_global.outgoing, session); err_free: luo_session_free(session); + up_read(&luo_session_serialize_rwsem); return err; } @@ -463,6 +503,7 @@ int luo_session_retrieve(const char *name, struct file **filep) struct luo_session *it; int err; + guard(rwsem_read)(&luo_session_serialize_rwsem); guard(rwsem_read)(&sh->rwsem); list_for_each_entry(it, &sh->list, list) { if (!strncmp(it->name, name, sizeof(it->name))) { @@ -635,7 +676,8 @@ int luo_session_serialize(void) int i = 0; int err; - guard(rwsem_write)(&sh->rwsem); + down_write(&luo_session_serialize_rwsem); + down_write(&sh->rwsem); list_for_each_entry(session, &sh->list, list) { err = luo_session_freeze_one(session, &sh->ser[i]); if (err) @@ -646,6 +688,7 @@ int luo_session_serialize(void) i++; } sh->header_ser->count = sh->count; + up_write(&sh->rwsem); return 0; @@ -655,6 +698,8 @@ err_undo: luo_session_unfreeze_one(session, &sh->ser[i]); memset(sh->ser[i].name, 0, sizeof(sh->ser[i].name)); } + up_write(&sh->rwsem); + up_write(&luo_session_serialize_rwsem); return err; } -- cgit v1.2.3 From 291dcd37c8c8f8f8e1bccc92228f44bf371762a8 Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Wed, 27 May 2026 20:27:36 +0000 Subject: liveupdate: fix u-a-f in luo_file_unpreserve_files() and luo_file_finish() In luo_file_unpreserve_files() and luo_file_finish(), reorder module_put() and xa_erase() to ensure the file handler module remains pinned while its operations are being accessed. Specifically, luo_get_id() dereferences fh->ops->get_id, so the module reference must be held until after xa_erase() (which calls luo_get_id) completes. For luo_file_finish(), this requires moving the module_put() call out of the luo_file_finish_one() helper and into the main loop of luo_file_finish() itself. Fixes: 00d0b372374f ("liveupdate: prevent double management of files") Acked-by: Mike Rapoport (Microsoft) Reviewed-by: Pratyush Yadav (Google) Signed-off-by: Pasha Tatashin Link: https://patch.msgid.link/20260527202737.1345192-5-pasha.tatashin@soleen.com Signed-off-by: Mike Rapoport (Microsoft) --- kernel/liveupdate/luo_file.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/liveupdate/luo_file.c b/kernel/liveupdate/luo_file.c index a0a419085e28..208987502f73 100644 --- a/kernel/liveupdate/luo_file.c +++ b/kernel/liveupdate/luo_file.c @@ -385,10 +385,11 @@ void luo_file_unpreserve_files(struct luo_file_set *file_set) args.private_data = luo_file->private_data; luo_file->fh->ops->unpreserve(&args); luo_flb_file_unpreserve(luo_file->fh); - module_put(luo_file->fh->ops->owner); xa_erase(&luo_preserved_files, luo_get_id(luo_file->fh, luo_file->file)); + module_put(luo_file->fh->ops->owner); + list_del(&luo_file->list); file_set->count--; @@ -677,7 +678,6 @@ static void luo_file_finish_one(struct luo_file_set *file_set, luo_file->fh->ops->finish(&args); luo_flb_file_finish(luo_file->fh); - module_put(luo_file->fh->ops->owner); } /** @@ -738,6 +738,7 @@ int luo_file_finish(struct luo_file_set *file_set) luo_get_id(luo_file->fh, luo_file->file)); fput(luo_file->file); } + module_put(luo_file->fh->ops->owner); list_del(&luo_file->list); file_set->count--; mutex_destroy(&luo_file->mutex); -- cgit v1.2.3 From 2935777b418d2bfcbfe96705bb2c0fa6c0d94e18 Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Wed, 27 May 2026 20:27:37 +0000 Subject: liveupdate: Remove unused ser field from struct luo_session The ser field in struct luo_session was intended to point to the serialized data for a session, but it was never actually utilized in the implementation. All serialization and deserialization logic consistently uses the pointers maintained in struct luo_session_header. Remove the dead field to clean up the structure. Acked-by: Mike Rapoport (Microsoft) Reviewed-by: Pratyush Yadav (Google) Signed-off-by: Pasha Tatashin Link: https://patch.msgid.link/20260527202737.1345192-6-pasha.tatashin@soleen.com Signed-off-by: Mike Rapoport (Microsoft) --- kernel/liveupdate/luo_internal.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'kernel') diff --git a/kernel/liveupdate/luo_internal.h b/kernel/liveupdate/luo_internal.h index 875844d7a41d..dd53d4a7277e 100644 --- a/kernel/liveupdate/luo_internal.h +++ b/kernel/liveupdate/luo_internal.h @@ -59,7 +59,6 @@ struct luo_file_set { * struct luo_session - Represents an active or incoming Live Update session. * @name: A unique name for this session, used for identification and * retrieval. - * @ser: Pointer to the serialized data for this session. * @list: A list_head member used to link this session into a global list * of either outgoing (to be preserved) or incoming (restored from * previous kernel) sessions. @@ -70,7 +69,6 @@ struct luo_file_set { */ struct luo_session { char name[LIVEUPDATE_SESSION_NAME_LENGTH]; - struct luo_session_ser *ser; struct list_head list; bool retrieved; struct luo_file_set file_set; -- cgit v1.2.3 From 4304c8165281fe730b6861d37a30b567d3c57832 Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Mon, 1 Jun 2026 23:35:21 +0900 Subject: tracing/probes: Ensure the uprobe buffer size is bigger than event size Add BUILD_BUG_ON() to ensure the uprobe per-CPU working buffer size is bigger than the event size. Link: https://lore.kernel.org/all/177849383209.8038.1902170479780501237.stgit@devnote2/ Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/trace_uprobe.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c index 2cabf8a23ec5..c5ee7920dec6 100644 --- a/kernel/trace/trace_uprobe.c +++ b/kernel/trace/trace_uprobe.c @@ -979,6 +979,7 @@ static struct uprobe_cpu_buffer *prepare_uprobe_buffer(struct trace_uprobe *tu, ucb = uprobe_buffer_get(); ucb->dsize = tu->tp.size + dsize; + BUILD_BUG_ON(MAX_UCB_BUFFER_SIZE < MAX_PROBE_EVENT_SIZE); if (WARN_ON_ONCE(ucb->dsize > MAX_UCB_BUFFER_SIZE)) { ucb->dsize = MAX_UCB_BUFFER_SIZE; dsize = MAX_UCB_BUFFER_SIZE - tu->tp.size; -- cgit v1.2.3 From cf24cbb4e5861caacfdb5bface90b80eaa26e649 Mon Sep 17 00:00:00 2001 From: Rosen Penev Date: Mon, 1 Jun 2026 23:35:21 +0900 Subject: tracing: Use flexible array for entry fetch code Store probe entry fetch instructions in the probe_entry_arg allocation instead of allocating a separate instruction array. This keeps the entry fetch code tied to the entry argument lifetime while leaving regular probe_arg instruction arrays separately allocated and freed. Assisted-by: Codex:GPT-5.5 Link: https://lore.kernel.org/all/20260520215817.16560-1-rosenp@gmail.com/ Signed-off-by: Rosen Penev Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/trace_probe.c | 8 +------- kernel/trace/trace_probe.h | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index 44c22d4e7881..695310571b08 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -838,15 +838,10 @@ static int __store_entry_arg(struct trace_probe *tp, int argnum) int i, offset, last_offset = 0; if (!earg) { - earg = kzalloc_obj(*tp->entry_arg); + earg = kzalloc_flex(*earg, code, 2 * tp->nr_args + 1); if (!earg) return -ENOMEM; earg->size = 2 * tp->nr_args + 1; - earg->code = kzalloc_objs(struct fetch_insn, earg->size); - if (!earg->code) { - kfree(earg); - return -ENOMEM; - } /* Fill the code buffer with 'end' to simplify it */ for (i = 0; i < earg->size; i++) earg->code[i].op = FETCH_OP_END; @@ -2049,7 +2044,6 @@ void trace_probe_cleanup(struct trace_probe *tp) traceprobe_free_probe_arg(&tp->args[i]); if (tp->entry_arg) { - kfree(tp->entry_arg->code); kfree(tp->entry_arg); tp->entry_arg = NULL; } diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h index 262d8707a3df..1076f1df347b 100644 --- a/kernel/trace/trace_probe.h +++ b/kernel/trace/trace_probe.h @@ -238,8 +238,8 @@ struct probe_arg { }; struct probe_entry_arg { - struct fetch_insn *code; unsigned int size; /* The entry data size */ + struct fetch_insn code[] __counted_by(size); }; struct trace_uprobe_filter { -- cgit v1.2.3 From 585abc02be3d3ab82fbcc4dbcbbf0ceb61a02129 Mon Sep 17 00:00:00 2001 From: Yash Suthar Date: Mon, 1 Jun 2026 23:35:21 +0900 Subject: tracing: Replace BUG_ON with lockdep_assert_held in uprobe_buffer functions Replace BUG_ON(!mutex_is_locked(&event_mutex)) with lockdep_assert_held(&event_mutex) in uprobe_buffer_enable() and uprobe_buffer_disable(). BUG_ON() will crash the kernel. mutex_is_locked() only checks if any task holds lock,but not the caller task. lockdep_assert_held() also check current task for lock and no crash on true condition. Link: https://lore.kernel.org/all/20260521192846.8306-1-yashsuthar983@gmail.com/ Signed-off-by: Yash Suthar Acked-by: Steven Rostedt Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/trace_uprobe.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c index c5ee7920dec6..c274346853d1 100644 --- a/kernel/trace/trace_uprobe.c +++ b/kernel/trace/trace_uprobe.c @@ -912,7 +912,7 @@ static int uprobe_buffer_enable(void) { int ret = 0; - BUG_ON(!mutex_is_locked(&event_mutex)); + lockdep_assert_held(&event_mutex); if (uprobe_buffer_refcnt++ == 0) { ret = uprobe_buffer_init(); @@ -927,7 +927,7 @@ static void uprobe_buffer_disable(void) { int cpu; - BUG_ON(!mutex_is_locked(&event_mutex)); + lockdep_assert_held(&event_mutex); if (--uprobe_buffer_refcnt == 0) { for_each_possible_cpu(cpu) -- cgit v1.2.3 From bab08d1f70438cf06de9ab438313b7ef3099514c Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Thu, 28 May 2026 18:49:24 -0700 Subject: bpf: Simplify mark_stack_slot_obj_read() and callers Rename mark_stack_slot_obj_read() as mark_stack_slots_scratched() and directly call it from functions processing iter, dynptr and irq_flag. Commit 6762e3a0bce5 ("bpf: simplify liveness to use (callsite, depth) keyed func_instances") has removed the dynamic liveness component in mark_stack_slot_obj_read(). The function effectively only marks stack slots as scratched and always succeed. Therefore, return void, drop the unused bpf_reg_state argument and rename it to mark_stack_slots_scratched() to reflect what it does now. In addition, to prepare for unifying dynptr handling, dynptr_get_spi() will be moved out of mark_dynptr_read(). As mark_dynptr_read() would join mark_iter_read() as a thin wrapper of mark_stack_slots_scratched(), just open code these helpers. Acked-by: Eduard Zingerman Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260529014936.2811085-2-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 69 ++++++++++++++++----------------------------------- 1 file changed, 21 insertions(+), 48 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 3a270bc485c2..d048f3033220 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3006,50 +3006,13 @@ out: return ret; } -static int mark_stack_slot_obj_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - int spi, int nr_slots) +static void mark_stack_slots_scratched(struct bpf_verifier_env *env, + int spi, int nr_slots) { int i; for (i = 0; i < nr_slots; i++) mark_stack_slot_scratched(env, spi - i); - return 0; -} - -static int mark_dynptr_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg) -{ - int spi; - - /* For CONST_PTR_TO_DYNPTR, it must have already been done by - * check_reg_arg in check_helper_call and mark_btf_func_reg_size in - * check_kfunc_call. - */ - if (reg->type == CONST_PTR_TO_DYNPTR) - return 0; - spi = dynptr_get_spi(env, reg); - if (spi < 0) - return spi; - /* Caller ensures dynptr is valid and initialized, which means spi is in - * bounds and spi is the first dynptr slot. Simply mark stack slot as - * read. - */ - return mark_stack_slot_obj_read(env, reg, spi, BPF_DYNPTR_NR_SLOTS); -} - -static int mark_iter_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - int spi, int nr_slots) -{ - return mark_stack_slot_obj_read(env, reg, spi, nr_slots); -} - -static int mark_irq_flag_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg) -{ - int spi; - - spi = irq_flag_get_spi(env, reg); - if (spi < 0) - return spi; - return mark_stack_slot_obj_read(env, reg, spi, 1); } /* This function is supposed to be used by the following 32-bit optimization @@ -7261,7 +7224,7 @@ static int process_kptr_func(struct bpf_verifier_env *env, int regno, static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int insn_idx, enum bpf_arg_type arg_type, int clone_ref_obj_id) { - int err; + int spi, err = 0; if (reg->type != PTR_TO_STACK && reg->type != CONST_PTR_TO_DYNPTR) { verbose(env, @@ -7323,7 +7286,17 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat return -EINVAL; } - err = mark_dynptr_read(env, reg); + if (reg->type != CONST_PTR_TO_DYNPTR) { + spi = dynptr_get_spi(env, reg); + if (spi < 0) + return spi; + + /* + * For CONST_PTR_TO_DYNPTR, reg is already scratched by check_reg_arg + * in check_helper_call and mark_btf_func_reg_size in check_kfunc_call. + */ + mark_stack_slots_scratched(env, spi, BPF_DYNPTR_NR_SLOTS); + } } return err; } @@ -7433,9 +7406,7 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * if (spi < 0) return spi; - err = mark_iter_read(env, reg, spi, nr_slots); - if (err) - return err; + mark_stack_slots_scratched(env, spi, nr_slots); /* remember meta->iter info for process_iter_next_call() */ meta->iter.spi = spi; @@ -11399,7 +11370,7 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, struct bpf_kfunc_call_arg_meta *meta) { - int err, kfunc_class = IRQ_NATIVE_KFUNC; + int err, spi, kfunc_class = IRQ_NATIVE_KFUNC; bool irq_save; if (meta->func_id == special_kfunc_list[KF_bpf_local_irq_save] || @@ -11440,9 +11411,11 @@ static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state * return err; } - err = mark_irq_flag_read(env, reg); - if (err) - return err; + spi = irq_flag_get_spi(env, reg); + if (spi < 0) + return spi; + + mark_stack_slots_scratched(env, spi, 1); err = unmark_stack_slot_irq_flag(env, reg, kfunc_class); if (err) -- cgit v1.2.3 From b5c0a07eb2c23bfd0c42ad6b461e6881b4b0995b Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Thu, 28 May 2026 18:49:25 -0700 Subject: bpf: Unify dynptr handling in the verifier Simplify dynptr checking for helper and kfunc by unifying it. Remember the initialized dynptr (i.e.,g !(arg_type |= MEM_UNINIT)) pass to a dynptr kfunc during process_dynptr_func() so that we can easily retrieve the information for verification later. By saving it in meta->dynptr, there is no need to call dynptr helpers such as dynptr_id(), dynptr_ref_obj_id() and dynptr_type() in check_func_arg(). Remove and open code the helpers in process_dynptr_func() when saving id, ref_obj_id, and type. Besides, since dynptr ref_obj_id information is now pass around in meta->bpf_dynptr_desc, drop the check in helper_multiple_ref_obj_use. Acked-by: Eduard Zingerman Acked-by: Mykyta Yatsenko Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260529014936.2811085-3-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 178 +++++++++----------------------------------------- 1 file changed, 32 insertions(+), 146 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d048f3033220..0c9792d42668 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -233,6 +233,7 @@ static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state) struct bpf_call_arg_meta { struct bpf_map_desc map; + struct bpf_dynptr_desc dynptr; bool raw_mode; bool pkt_access; u8 release_regno; @@ -241,7 +242,6 @@ struct bpf_call_arg_meta { int mem_size; u64 msize_max_value; int ref_obj_id; - int dynptr_id; int func_id; struct btf *btf; u32 btf_id; @@ -470,11 +470,6 @@ static bool is_ptr_cast_function(enum bpf_func_id func_id) func_id == BPF_FUNC_skc_to_tcp_request_sock; } -static bool is_dynptr_ref_function(enum bpf_func_id func_id) -{ - return func_id == BPF_FUNC_dynptr_data; -} - static bool is_sync_callback_calling_kfunc(u32 btf_id); static bool is_async_callback_calling_kfunc(u32 btf_id); static bool is_callback_calling_kfunc(u32 btf_id); @@ -542,8 +537,6 @@ static bool helper_multiple_ref_obj_use(enum bpf_func_id func_id, ref_obj_uses++; if (is_acquire_function(func_id, map)) ref_obj_uses++; - if (is_dynptr_ref_function(func_id)) - ref_obj_uses++; return ref_obj_uses > 1; } @@ -7221,8 +7214,9 @@ static int process_kptr_func(struct bpf_verifier_env *env, int regno, * use case. The second level is tracked using the upper bit of bpf_dynptr->size * and checked dynamically during runtime. */ -static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int insn_idx, - enum bpf_arg_type arg_type, int clone_ref_obj_id) +static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, int insn_idx, enum bpf_arg_type arg_type, + int clone_ref_obj_id, struct bpf_dynptr_desc *dynptr) { int spi, err = 0; @@ -7287,6 +7281,8 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat } if (reg->type != CONST_PTR_TO_DYNPTR) { + struct bpf_func_state *state = bpf_func(env, reg); + spi = dynptr_get_spi(env, reg); if (spi < 0) return spi; @@ -7296,6 +7292,14 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat * in check_helper_call and mark_btf_func_reg_size in check_kfunc_call. */ mark_stack_slots_scratched(env, spi, BPF_DYNPTR_NR_SLOTS); + + reg = &state->stack[spi].spilled_ptr; + } + + if (dynptr) { + dynptr->type = reg->dynptr.type; + dynptr->id = reg->id; + dynptr->ref_obj_id = reg->ref_obj_id; } } return err; @@ -8065,72 +8069,6 @@ static int check_func_arg_reg_off(struct bpf_verifier_env *env, } } -static struct bpf_reg_state *get_dynptr_arg_reg(struct bpf_verifier_env *env, - const struct bpf_func_proto *fn, - struct bpf_reg_state *regs) -{ - struct bpf_reg_state *state = NULL; - int i; - - for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) - if (arg_type_is_dynptr(fn->arg_type[i])) { - if (state) { - verbose(env, "verifier internal error: multiple dynptr args\n"); - return NULL; - } - state = ®s[BPF_REG_1 + i]; - } - - if (!state) - verbose(env, "verifier internal error: no dynptr arg found\n"); - - return state; -} - -static int dynptr_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg) -{ - struct bpf_func_state *state = bpf_func(env, reg); - int spi; - - if (reg->type == CONST_PTR_TO_DYNPTR) - return reg->id; - spi = dynptr_get_spi(env, reg); - if (spi < 0) - return spi; - return state->stack[spi].spilled_ptr.id; -} - -static int dynptr_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg) -{ - struct bpf_func_state *state = bpf_func(env, reg); - int spi; - - if (reg->type == CONST_PTR_TO_DYNPTR) - return reg->ref_obj_id; - spi = dynptr_get_spi(env, reg); - if (spi < 0) - return spi; - return state->stack[spi].spilled_ptr.ref_obj_id; -} - -static enum bpf_dynptr_type dynptr_get_type(struct bpf_verifier_env *env, - struct bpf_reg_state *reg) -{ - struct bpf_func_state *state = bpf_func(env, reg); - int spi; - - if (reg->type == CONST_PTR_TO_DYNPTR) - return reg->dynptr.type; - - spi = bpf_get_spi(reg->var_off.value); - if (spi < 0) { - verbose(env, "verifier internal error: invalid spi when querying dynptr type\n"); - return BPF_DYNPTR_TYPE_INVALID; - } - - return state->stack[spi].spilled_ptr.dynptr.type; -} - static int check_arg_const_str(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno) { @@ -8488,7 +8426,8 @@ skip_type_check: true, meta); break; case ARG_PTR_TO_DYNPTR: - err = process_dynptr_func(env, reg, argno_from_reg(regno), insn_idx, arg_type, 0); + err = process_dynptr_func(env, reg, argno_from_reg(regno), insn_idx, arg_type, 0, + &meta->dynptr); if (err) return err; break; @@ -9170,7 +9109,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, if (ret) return ret; - ret = process_dynptr_func(env, reg, argno, -1, arg->arg_type, 0); + ret = process_dynptr_func(env, reg, argno, -1, arg->arg_type, 0, NULL); if (ret) return ret; } else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) { @@ -10278,52 +10217,10 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn } } break; - case BPF_FUNC_dynptr_data: - { - struct bpf_reg_state *reg; - int id, ref_obj_id; - - reg = get_dynptr_arg_reg(env, fn, regs); - if (!reg) - return -EFAULT; - - - if (meta.dynptr_id) { - verifier_bug(env, "meta.dynptr_id already set"); - return -EFAULT; - } - if (meta.ref_obj_id) { - verifier_bug(env, "meta.ref_obj_id already set"); - return -EFAULT; - } - - id = dynptr_id(env, reg); - if (id < 0) { - verifier_bug(env, "failed to obtain dynptr id"); - return id; - } - - ref_obj_id = dynptr_ref_obj_id(env, reg); - if (ref_obj_id < 0) { - verifier_bug(env, "failed to obtain dynptr ref_obj_id"); - return ref_obj_id; - } - - meta.dynptr_id = id; - meta.ref_obj_id = ref_obj_id; - - break; - } case BPF_FUNC_dynptr_write: { - enum bpf_dynptr_type dynptr_type; - struct bpf_reg_state *reg; + enum bpf_dynptr_type dynptr_type = meta.dynptr.type; - reg = get_dynptr_arg_reg(env, fn, regs); - if (!reg) - return -EFAULT; - - dynptr_type = dynptr_get_type(env, reg); if (dynptr_type == BPF_DYNPTR_TYPE_INVALID) return -EFAULT; @@ -10515,10 +10412,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return -EFAULT; } - if (is_dynptr_ref_function(func_id)) - regs[BPF_REG_0].dynptr_id = meta.dynptr_id; - - if (is_ptr_cast_function(func_id) || is_dynptr_ref_function(func_id)) { + if (is_ptr_cast_function(func_id)) { /* For release_reference() */ regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id; } else if (is_acquire_function(func_id, meta.map.ptr)) { @@ -10532,6 +10426,11 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn regs[BPF_REG_0].ref_obj_id = id; } + if (func_id == BPF_FUNC_dynptr_data) { + regs[BPF_REG_0].dynptr_id = meta.dynptr.id; + regs[BPF_REG_0].ref_obj_id = meta.dynptr.ref_obj_id; + } + err = do_refine_retval_range(env, regs, fn->ret_type, func_id, &meta); if (err) return err; @@ -12187,7 +12086,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ meta->release_regno = regno; } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_clone] && (dynptr_arg_type & MEM_UNINIT)) { - enum bpf_dynptr_type parent_type = meta->initialized_dynptr.type; + enum bpf_dynptr_type parent_type = meta->dynptr.type; if (parent_type == BPF_DYNPTR_TYPE_INVALID) { verifier_bug(env, "no dynptr type for parent of clone"); @@ -12195,30 +12094,17 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ } dynptr_arg_type |= (unsigned int)get_dynptr_type_flag(parent_type); - clone_ref_obj_id = meta->initialized_dynptr.ref_obj_id; + clone_ref_obj_id = meta->dynptr.ref_obj_id; if (dynptr_type_refcounted(parent_type) && !clone_ref_obj_id) { verifier_bug(env, "missing ref obj id for parent of clone"); return -EFAULT; } } - ret = process_dynptr_func(env, reg, argno, insn_idx, - dynptr_arg_type, clone_ref_obj_id); + ret = process_dynptr_func(env, reg, argno, insn_idx, dynptr_arg_type, + clone_ref_obj_id, &meta->dynptr); if (ret < 0) return ret; - - if (!(dynptr_arg_type & MEM_UNINIT)) { - int id = dynptr_id(env, reg); - - if (id < 0) { - verifier_bug(env, "failed to obtain dynptr id"); - return id; - } - meta->initialized_dynptr.id = id; - meta->initialized_dynptr.type = dynptr_get_type(env, reg); - meta->initialized_dynptr.ref_obj_id = dynptr_ref_obj_id(env, reg); - } - break; } case KF_ARG_PTR_TO_ITER: @@ -12849,7 +12735,7 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_ca } } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_slice] || meta->func_id == special_kfunc_list[KF_bpf_dynptr_slice_rdwr]) { - enum bpf_type_flag type_flag = get_dynptr_type_flag(meta->initialized_dynptr.type); + enum bpf_type_flag type_flag = get_dynptr_type_flag(meta->dynptr.type); mark_reg_known_zero(env, regs, BPF_REG_0); @@ -12873,11 +12759,11 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_ca } } - if (!meta->initialized_dynptr.id) { + if (!meta->dynptr.id) { verifier_bug(env, "no dynptr id"); return -EFAULT; } - regs[BPF_REG_0].dynptr_id = meta->initialized_dynptr.id; + regs[BPF_REG_0].dynptr_id = meta->dynptr.id; /* we don't need to set BPF_REG_0's ref obj id * because packet slices are not refcounted (see @@ -13063,7 +12949,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (meta.release_regno) { struct bpf_reg_state *reg = ®s[meta.release_regno]; - if (meta.initialized_dynptr.ref_obj_id) { + if (meta.dynptr.ref_obj_id) { err = unmark_stack_slots_dynptr(env, reg); } else { err = release_reference(env, reg->ref_obj_id); -- cgit v1.2.3 From 94ac7553361f3f20b0a60c13f9e7d0a859c73c12 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Thu, 28 May 2026 18:49:26 -0700 Subject: bpf: Assign reg->id when getting referenced kptr from ctx Assign reg->id when getting referenced kptr from read program context to be consistent with R0 of KF_ACQUIRE kfunc. skb dynptr will track the referenced skb in qdisc programs using a new field reg->parent_id in a later patch. Acked-by: Andrii Nakryiko Acked-by: Eduard Zingerman Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260529014936.2811085-4-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 0c9792d42668..a6974ff50514 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6208,8 +6208,6 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b } else { mark_reg_known_zero(env, regs, value_regno); - if (type_may_be_null(info.reg_type)) - regs[value_regno].id = ++env->id_gen; /* A load of ctx field could have different * actual load size with the one encoded in the * insn. When the dst is PTR, it is for sure not @@ -6219,8 +6217,11 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b if (base_type(info.reg_type) == PTR_TO_BTF_ID) { regs[value_regno].btf = info.btf; regs[value_regno].btf_id = info.btf_id; + regs[value_regno].id = info.ref_obj_id; regs[value_regno].ref_obj_id = info.ref_obj_id; } + if (type_may_be_null(info.reg_type) && !regs[value_regno].id) + regs[value_regno].id = ++env->id_gen; } regs[value_regno].type = info.reg_type; } -- cgit v1.2.3 From 06d518a5583fa681dc5c204d578a894f5b11ffa5 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Thu, 28 May 2026 18:49:27 -0700 Subject: bpf: Preserve reg->id of pointer objects after null-check Preserve reg->id of pointer objects after null-checking the register so that children objects derived from it can still refer to it in the new object relationship tracking mechanism introduced in a later patch. This change incurs a slight increase in the number of states in one selftest bpf object, rbtree_search.bpf.o. For Meta bpf objects, the increase of states is also negligible. Selftest BPF objects with insns_diff > 0 Program Insns (A) Insns (B) Insns (DIFF) States (A) States (B) States (DIFF) ------------------------ --------- --------- -------------- ---------- ---------- ------------- rbtree_search 6820 7326 +506 (+7.42%) 379 398 +19 (+5.01%) Meta BPF objects with insns_diff > 0 Program Insns (A) Insns (B) Insns (DIFF) States (A) States (B) States (DIFF) ------------------------ --------- --------- -------------- ---------- ---------- ------------- ned_imex_be_tclass 52 57 +5 (+9.62%) 5 6 +1 (+20.00%) ned_imex_be_tclass 52 57 +5 (+9.62%) 5 6 +1 (+20.00%) ned_skop_auto_flowlabel 523 526 +3 (+0.57%) 39 40 +1 (+2.56%) ned_skop_mss 289 292 +3 (+1.04%) 20 20 +0 (+0.00%) ned_skopt_bet_classifier 78 82 +4 (+5.13%) 8 8 +0 (+0.00%) dctcp_update_alpha 252 320 +68 (+26.98%) 21 27 +6 (+28.57%) dctcp_update_alpha 252 320 +68 (+26.98%) 21 27 +6 (+28.57%) ned_ts_func 119 126 +7 (+5.88%) 6 7 +1 (+16.67%) tw_egress 1119 1128 +9 (+0.80%) 95 96 +1 (+1.05%) tw_ingress 1128 1137 +9 (+0.80%) 95 96 +1 (+1.05%) tw_tproxy_router 4380 4465 +85 (+1.94%) 114 118 +4 (+3.51%) tw_tproxy_router4 3093 3170 +77 (+2.49%) 83 88 +5 (+6.02%) ttls_tc_ingress 34656 35717 +1061 (+3.06%) 936 970 +34 (+3.63%) tw_twfw_egress 222327 222338 +11 (+0.00%) 10563 10564 +1 (+0.01%) tw_twfw_ingress 78295 78299 +4 (+0.01%) 3825 3826 +1 (+0.03%) tw_twfw_tc_eg 222839 222859 +20 (+0.01%) 10584 10585 +1 (+0.01%) tw_twfw_tc_in 78295 78299 +4 (+0.01%) 3825 3826 +1 (+0.03%) tw_twfw_egress 8080 8085 +5 (+0.06%) 456 456 +0 (+0.00%) tw_twfw_ingress 8053 8056 +3 (+0.04%) 454 454 +0 (+0.00%) tw_twfw_tc_eg 8154 8174 +20 (+0.25%) 456 457 +1 (+0.22%) tw_twfw_tc_in 8060 8063 +3 (+0.04%) 455 455 +0 (+0.00%) tw_twfw_egress 222327 222338 +11 (+0.00%) 10563 10564 +1 (+0.01%) tw_twfw_ingress 78295 78299 +4 (+0.01%) 3825 3826 +1 (+0.03%) tw_twfw_tc_eg 222839 222859 +20 (+0.01%) 10584 10585 +1 (+0.01%) tw_twfw_tc_in 78295 78299 +4 (+0.01%) 3825 3826 +1 (+0.03%) tw_twfw_egress 8080 8085 +5 (+0.06%) 456 456 +0 (+0.00%) tw_twfw_ingress 8053 8056 +3 (+0.04%) 454 454 +0 (+0.00%) tw_twfw_tc_eg 8154 8174 +20 (+0.25%) 456 457 +1 (+0.22%) tw_twfw_tc_in 8060 8063 +3 (+0.04%) 455 455 +0 (+0.00%) Looking into rbtree_search, the reason for such increase is that the verifier has to explore the main loop shown below for one more iteration until state pruning decides the current state is safe. long rbtree_search(void *ctx) { ... bpf_spin_lock(&glock0); rb_n = bpf_rbtree_root(&groot0); while (can_loop) { if (!rb_n) { bpf_spin_unlock(&glock0); return __LINE__; } n = rb_entry(rb_n, struct node_data, r0); if (lookup_key == n->key0) break; if (nr_gc < NR_NODES) gc_ns[nr_gc++] = rb_n; if (lookup_key < n->key0) rb_n = bpf_rbtree_left(&groot0, rb_n); else rb_n = bpf_rbtree_right(&groot0, rb_n); } ... } Below is what the verifier sees at the start of each iteration (65: may_goto) after preserving id of rb_n. Without id of rb_n, the verifier stops exploring the loop at iter 16. rb_n gc_ns[15] iter 15 257 257 iter 16 290 257 rb_n: idmap add 257->290 gc_ns[15]: check 257 != 290 --> state not equal iter 17 325 257 rb_n: idmap add 290->325 gc_ns[15]: idmap add 257->257 --> state safe Acked-by: Andrii Nakryiko Acked-by: Eduard Zingerman Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260529014936.2811085-5-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a6974ff50514..0d8be0b68bd8 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -15576,15 +15576,10 @@ static void mark_ptr_or_null_reg(struct bpf_func_state *state, mark_ptr_not_null_reg(reg); - if (!reg_may_point_to_spin_lock(reg)) { - /* For not-NULL ptr, reg->ref_obj_id will be reset - * in release_reference(). - * - * reg->id is still used by spin_lock ptr. Other - * than spin_lock ptr type, reg->id can be reset. - */ - reg->id = 0; - } + /* + * reg->id is preserved for object relationship tracking + * and spin_lock lock state tracking + */ } } -- cgit v1.2.3 From 308c7a0ae8859b34d9d90a3dff953b2d14242145 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Thu, 28 May 2026 18:49:28 -0700 Subject: bpf: Refactor object relationship tracking and fix dynptr UAF bug Refactor object relationship tracking in the verifier and fix a dynptr use-after-free bug where file/skb dynptrs are not invalidated when the parent referenced object is freed. Add parent_id to bpf_reg_state to precisely track child-parent relationships. A child object's parent_id points to the parent object's id. This replaces the PTR_TO_MEM-specific dynptr_id. Remove ref_obj_id from bpf_reg_state by folding its role into the existing id field. Previously, id tracked pointer identity for null checking while ref_obj_id tracked the owning reference for lifetime management. These are now unified: acquire helpers and kfuncs set id to the acquired reference id, and release paths use id directly. Add reg_is_referenced() which checks if a register is referenced by looking up its id in the reference array. This replaces all former ref_obj_id checks. For release_reference(), invalidating an object now also invalidates all descendants by traversing the object tree. This is done using stack-based DFS to avoid recursive call chains of release_reference() -> unmark_stack_slots_dynptr() -> release_reference(). Referenced objects encountered during tree traversal are reported as leaked references. Add parent_id to bpf_reference_state to enable hierarchical reference tracking. When acquiring a reference, a parent_id can be specified to link the new reference to an existing one (e.g., referenced dynptrs acquire a reference with parent_id linking to the parent object's reference). Pointer casting: For pointer casting helpers (bpf_sk_fullsock, bpf_tcp_sock), instead of propagating ref_obj_id, the cast result reuses the same reference id as the source pointer. Since the cast may return NULL for a non-NULL input, the NULL case is explored as a separate verifier branch. This allows releasing any of the original or cast pointers to invalidate all others. Referenced dynptrs: When constructing a referenced dynptr, acquire a intermediate reference with parent_id linking to the parent referenced object. The dynptr and all clones share the same parent_id (pointing to the intermediate ref) but get unique ids for independent slice tracking. Releasing a referenced dynptr releases the parent reference, which in turn invalidates all clones and their derived slices. Owning to non-owning reference conversion: After converting owning to non-owning by clearing id (e.g., object(id=1) -> object(id=0)), the verifier releases the reference state via release_reference_nomark(). Note that the error message "reference has not been acquired before" in the helper and kfunc release paths is removed. This message was already unreachable. The verifier only calls release_reference() after confirming the reference is valid, so the condition could never trigger in practice. Fixes: 870c28588afa ("bpf: net_sched: Add basic bpf qdisc kfuncs") Signed-off-by: Amery Hung Acked-by: Eduard Zingerman Link: https://lore.kernel.org/r/20260529014936.2811085-6-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/btf.c | 2 +- kernel/bpf/fixups.c | 2 +- kernel/bpf/log.c | 18 +- kernel/bpf/states.c | 11 +- kernel/bpf/verifier.c | 560 +++++++++++++++++++++++++------------------------- 5 files changed, 296 insertions(+), 297 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 17d4ab0a8206..f429f6f58cb2 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6957,7 +6957,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, info->reg_type = ctx_arg_info->reg_type; info->btf = ctx_arg_info->btf ? : btf_vmlinux; info->btf_id = ctx_arg_info->btf_id; - info->ref_obj_id = ctx_arg_info->ref_obj_id; + info->ref_id = ctx_arg_info->ref_id; return true; } } diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 12739add2dda..5aa3f7d99ac9 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -870,7 +870,7 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) case PTR_TO_BTF_ID: case PTR_TO_BTF_ID | PTR_UNTRUSTED: /* PTR_TO_BTF_ID | MEM_ALLOC always has a valid lifetime, unlike - * PTR_TO_BTF_ID, and an active ref_obj_id, but the same cannot + * PTR_TO_BTF_ID, and an active referenced id, but the same cannot * be said once it is marked PTR_UNTRUSTED, hence we must handle * any faults for loads into such types. BPF_WRITE is disallowed * for this case. diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index 62fe6ed18374..b740fa73ee26 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c @@ -665,8 +665,8 @@ static void print_reg_state(struct bpf_verifier_env *env, verbose_a("id=%d", reg->id & ~BPF_ADD_CONST); if (reg->id & BPF_ADD_CONST) verbose(env, "%+d", reg->delta); - if (reg->ref_obj_id) - verbose_a("ref_obj_id=%d", reg->ref_obj_id); + if (reg->parent_id) + verbose_a("parent_id=%d", reg->parent_id); if (type_is_non_owning_ref(reg->type)) verbose_a("%s", "non_own_ref"); if (type_is_map_ptr(t)) { @@ -768,21 +768,19 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie verbose(env, "=dynptr_%s(", dynptr_type_str(reg->dynptr.type)); if (reg->id) verbose_a("id=%d", reg->id); - if (reg->ref_obj_id) - verbose_a("ref_id=%d", reg->ref_obj_id); - if (reg->dynptr_id) - verbose_a("dynptr_id=%d", reg->dynptr_id); + if (reg->parent_id) + verbose_a("parent_id=%d", reg->parent_id); verbose(env, ")"); break; case STACK_ITER: - /* only main slot has ref_obj_id set; skip others */ - if (!reg->ref_obj_id) + /* only main slot has id set; skip others */ + if (!reg->id) continue; - verbose(env, " fp%d=iter_%s(ref_id=%d,state=%s,depth=%u)", + verbose(env, " fp%d=iter_%s(id=%d,state=%s,depth=%u)", (-i - 1) * BPF_REG_SIZE, iter_type_str(reg->iter.btf, reg->iter.btf_id), - reg->ref_obj_id, iter_state_str(reg->iter.state), + reg->id, iter_state_str(reg->iter.state), reg->iter.depth); break; case STACK_MISC: diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 877338136009..5945956a7573 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -489,7 +489,7 @@ static bool regs_exact(const struct bpf_reg_state *rold, { return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 && check_ids(rold->id, rcur->id, idmap) && - check_ids(rold->ref_obj_id, rcur->ref_obj_id, idmap); + check_ids(rold->parent_id, rcur->parent_id, idmap); } enum exact_level { @@ -614,7 +614,7 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, range_within(rold, rcur) && tnum_in(rold->var_off, rcur->var_off) && check_ids(rold->id, rcur->id, idmap) && - check_ids(rold->ref_obj_id, rcur->ref_obj_id, idmap); + check_ids(rold->parent_id, rcur->parent_id, idmap); case PTR_TO_PACKET_META: case PTR_TO_PACKET: /* We must have at least as much range as the old ptr @@ -794,7 +794,8 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, cur_reg = &cur->stack[spi].spilled_ptr; if (old_reg->dynptr.type != cur_reg->dynptr.type || old_reg->dynptr.first_slot != cur_reg->dynptr.first_slot || - !check_ids(old_reg->ref_obj_id, cur_reg->ref_obj_id, idmap)) + !check_ids(old_reg->id, cur_reg->id, idmap) || + !check_ids(old_reg->parent_id, cur_reg->parent_id, idmap)) return false; break; case STACK_ITER: @@ -810,13 +811,13 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, old_reg->iter.btf_id != cur_reg->iter.btf_id || old_reg->iter.state != cur_reg->iter.state || /* ignore {old_reg,cur_reg}->iter.depth, see above */ - !check_ids(old_reg->ref_obj_id, cur_reg->ref_obj_id, idmap)) + !check_ids(old_reg->id, cur_reg->id, idmap)) return false; break; case STACK_IRQ_FLAG: old_reg = &old->stack[spi].spilled_ptr; cur_reg = &cur->stack[spi].spilled_ptr; - if (!check_ids(old_reg->ref_obj_id, cur_reg->ref_obj_id, idmap) || + if (!check_ids(old_reg->id, cur_reg->id, idmap) || old_reg->irq.kfunc_class != cur_reg->irq.kfunc_class) return false; break; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 0d8be0b68bd8..6d82ca5acacb 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -200,14 +200,14 @@ struct bpf_verifier_stack_elem { #define BPF_PRIV_STACK_MIN_SIZE 64 -static int acquire_reference(struct bpf_verifier_env *env, int insn_idx); -static int release_reference_nomark(struct bpf_verifier_state *state, int ref_obj_id); -static int release_reference(struct bpf_verifier_env *env, int ref_obj_id); +static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int parent_id); +static int release_reference_nomark(struct bpf_verifier_state *state, int id); +static int release_reference(struct bpf_verifier_env *env, int id); static void invalidate_non_owning_refs(struct bpf_verifier_env *env); static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env); static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg); -static bool is_trusted_reg(const struct bpf_reg_state *reg); +static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg); static inline bool in_sleepable_context(struct bpf_verifier_env *env); static const char *non_sleepable_context_description(struct bpf_verifier_env *env); static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg); @@ -241,7 +241,7 @@ struct bpf_call_arg_meta { int access_size; int mem_size; u64 msize_max_value; - int ref_obj_id; + u32 id; int func_id; struct btf *btf; u32 btf_id; @@ -339,7 +339,7 @@ static void verbose_invalid_scalar(struct bpf_verifier_env *env, verbose(env, " should have been in [%d, %d]\n", range.minval, range.maxval); } -static bool reg_not_null(const struct bpf_reg_state *reg) +static bool reg_not_null(struct bpf_verifier_env *env, const struct bpf_reg_state *reg) { enum bpf_reg_type type; @@ -353,7 +353,7 @@ static bool reg_not_null(const struct bpf_reg_state *reg) type == PTR_TO_MAP_VALUE || type == PTR_TO_MAP_KEY || type == PTR_TO_SOCK_COMMON || - (type == PTR_TO_BTF_ID && is_trusted_reg(reg)) || + (type == PTR_TO_BTF_ID && is_trusted_reg(env, reg)) || (type == PTR_TO_MEM && !(reg->type & PTR_UNTRUSTED)) || type == CONST_PTR_TO_MAP; } @@ -638,43 +638,44 @@ static enum bpf_type_flag get_dynptr_type_flag(enum bpf_dynptr_type type) } } -static bool dynptr_type_refcounted(enum bpf_dynptr_type type) +static bool dynptr_type_referenced(enum bpf_dynptr_type type) { return type == BPF_DYNPTR_TYPE_RINGBUF || type == BPF_DYNPTR_TYPE_FILE; } static void __mark_dynptr_reg(struct bpf_reg_state *reg, enum bpf_dynptr_type type, - bool first_slot, int dynptr_id); + bool first_slot, int id, int parent_id); static void mark_dynptr_stack_regs(struct bpf_verifier_env *env, struct bpf_reg_state *sreg1, struct bpf_reg_state *sreg2, - enum bpf_dynptr_type type) + enum bpf_dynptr_type type, int parent_id) { int id = ++env->id_gen; - __mark_dynptr_reg(sreg1, type, true, id); - __mark_dynptr_reg(sreg2, type, false, id); + __mark_dynptr_reg(sreg1, type, true, id, parent_id); + __mark_dynptr_reg(sreg2, type, false, id, parent_id); } static void mark_dynptr_cb_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, enum bpf_dynptr_type type) { - __mark_dynptr_reg(reg, type, true, ++env->id_gen); + __mark_dynptr_reg(reg, type, true, ++env->id_gen, 0); } static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi); static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - enum bpf_arg_type arg_type, int insn_idx, int clone_ref_obj_id) + enum bpf_arg_type arg_type, int insn_idx, int parent_id, + struct bpf_dynptr_desc *dynptr) { struct bpf_func_state *state = bpf_func(env, reg); - enum bpf_dynptr_type type; int spi, i, err; + enum bpf_dynptr_type type; spi = dynptr_get_spi(env, reg); if (spi < 0) @@ -705,85 +706,62 @@ static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_ if (type == BPF_DYNPTR_TYPE_INVALID) return -EINVAL; - mark_dynptr_stack_regs(env, &state->stack[spi].spilled_ptr, - &state->stack[spi - 1].spilled_ptr, type); - - if (dynptr_type_refcounted(type)) { - /* The id is used to track proper releasing */ - int id; + if (dynptr->type == BPF_DYNPTR_TYPE_INVALID) { /* dynptr constructors */ + if (dynptr_type_referenced(type)) { + int id; - if (clone_ref_obj_id) - id = clone_ref_obj_id; - else - id = acquire_reference(env, insn_idx); - - if (id < 0) - return id; + /* + * Create an intermediate reference that tracks the referenced + * object for the referenced dynptr. Freeing a referenced dynptr + * through helpers/kfuncs will invalidate all clones. + */ + id = acquire_reference(env, insn_idx, parent_id); + if (id < 0) + return id; - state->stack[spi].spilled_ptr.ref_obj_id = id; - state->stack[spi - 1].spilled_ptr.ref_obj_id = id; + parent_id = id; + } + } else { /* bpf_dynptr_clone() */ + parent_id = dynptr->parent_id; } + mark_dynptr_stack_regs(env, &state->stack[spi].spilled_ptr, + &state->stack[spi - 1].spilled_ptr, type, parent_id); + return 0; } -static void invalidate_dynptr(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi) +static void invalidate_dynptr(struct bpf_verifier_env *env, struct bpf_stack_state *stack) { int i; for (i = 0; i < BPF_REG_SIZE; i++) { - state->stack[spi].slot_type[i] = STACK_INVALID; - state->stack[spi - 1].slot_type[i] = STACK_INVALID; + stack[0].slot_type[i] = STACK_INVALID; + stack[1].slot_type[i] = STACK_INVALID; } - bpf_mark_reg_not_init(env, &state->stack[spi].spilled_ptr); - bpf_mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr); + bpf_mark_reg_not_init(env, &stack[0].spilled_ptr); + bpf_mark_reg_not_init(env, &stack[1].spilled_ptr); } static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg) { struct bpf_func_state *state = bpf_func(env, reg); - int spi, ref_obj_id, i; + int spi; spi = dynptr_get_spi(env, reg); if (spi < 0) return spi; - if (!dynptr_type_refcounted(state->stack[spi].spilled_ptr.dynptr.type)) { - invalidate_dynptr(env, state, spi); - return 0; - } - - ref_obj_id = state->stack[spi].spilled_ptr.ref_obj_id; - - /* If the dynptr has a ref_obj_id, then we need to invalidate - * two things: - * - * 1) Any dynptrs with a matching ref_obj_id (clones) - * 2) Any slices derived from this dynptr. + /* + * For referenced dynptr, release the parent ref which cascades to + * all clones and derived slices. For non-referenced dynptr, only + * the dynptr and slices derived from it will be invalidated. */ - - /* Invalidate any slices associated with this dynptr */ - WARN_ON_ONCE(release_reference(env, ref_obj_id)); - - /* Invalidate any dynptr clones */ - for (i = 1; i < state->allocated_stack / BPF_REG_SIZE; i++) { - if (state->stack[i].spilled_ptr.ref_obj_id != ref_obj_id) - continue; - - /* it should always be the case that if the ref obj id - * matches then the stack slot also belongs to a - * dynptr - */ - if (state->stack[i].slot_type[0] != STACK_DYNPTR) { - verifier_bug(env, "misconfigured ref_obj_id"); - return -EFAULT; - } - if (state->stack[i].spilled_ptr.dynptr.first_slot) - invalidate_dynptr(env, state, i); - } - - return 0; + reg = &state->stack[spi].spilled_ptr; + return release_reference(env, dynptr_type_referenced(reg->dynptr.type) + ? reg->parent_id + : reg->id); } static void __mark_reg_unknown(const struct bpf_verifier_env *env, @@ -800,9 +778,7 @@ static void mark_reg_invalid(const struct bpf_verifier_env *env, struct bpf_reg_ static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi) { - struct bpf_func_state *fstate; - struct bpf_reg_state *dreg; - int i, dynptr_id; + int i, err = 0; /* We always ensure that STACK_DYNPTR is never set partially, * hence just checking for slot_type[0] is enough. This is @@ -816,13 +792,13 @@ static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, if (!state->stack[spi].spilled_ptr.dynptr.first_slot) spi = spi + 1; - if (dynptr_type_refcounted(state->stack[spi].spilled_ptr.dynptr.type)) { - int ref_obj_id = state->stack[spi].spilled_ptr.ref_obj_id; + if (dynptr_type_referenced(state->stack[spi].spilled_ptr.dynptr.type)) { + int v_parent_id = state->stack[spi].spilled_ptr.parent_id; int ref_cnt = 0; /* * A referenced dynptr can be overwritten only if there is at - * least one other dynptr sharing the same ref_obj_id, + * least one other dynptr sharing the same virtual ref parent, * ensuring the reference can still be properly released. */ for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { @@ -830,7 +806,7 @@ static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, continue; if (!state->stack[i].spilled_ptr.dynptr.first_slot) continue; - if (state->stack[i].spilled_ptr.ref_obj_id == ref_obj_id) + if (state->stack[i].spilled_ptr.parent_id == v_parent_id) ref_cnt++; } @@ -840,32 +816,14 @@ static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, } } - mark_stack_slot_scratched(env, spi); - mark_stack_slot_scratched(env, spi - 1); - - /* Writing partially to one dynptr stack slot destroys both. */ - for (i = 0; i < BPF_REG_SIZE; i++) { - state->stack[spi].slot_type[i] = STACK_INVALID; - state->stack[spi - 1].slot_type[i] = STACK_INVALID; + /* Invalidate the dynptr and any derived slices */ + err = release_reference(env, state->stack[spi].spilled_ptr.id); + if (!err) { + mark_stack_slot_scratched(env, spi); + mark_stack_slot_scratched(env, spi - 1); } - dynptr_id = state->stack[spi].spilled_ptr.id; - /* Invalidate any slices associated with this dynptr */ - bpf_for_each_reg_in_vstate(env->cur_state, fstate, dreg, ({ - /* Dynptr slices are only PTR_TO_MEM_OR_NULL and PTR_TO_MEM */ - if (dreg->type != (PTR_TO_MEM | PTR_MAYBE_NULL) && dreg->type != PTR_TO_MEM) - continue; - if (dreg->dynptr_id == dynptr_id) - mark_reg_invalid(env, dreg); - })); - - /* Do not release reference state, we are destroying dynptr on stack, - * not using some helper to release it. Just reset register. - */ - bpf_mark_reg_not_init(env, &state->stack[spi].spilled_ptr); - bpf_mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr); - - return 0; + return err; } static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg) @@ -965,7 +923,7 @@ static int mark_stack_slots_iter(struct bpf_verifier_env *env, if (spi < 0) return spi; - id = acquire_reference(env, insn_idx); + id = acquire_reference(env, insn_idx, 0); if (id < 0) return id; @@ -981,7 +939,7 @@ static int mark_stack_slots_iter(struct bpf_verifier_env *env, else st->type |= PTR_UNTRUSTED; } - st->ref_obj_id = i == 0 ? id : 0; + st->id = i == 0 ? id : 0; st->iter.btf = btf; st->iter.btf_id = btf_id; st->iter.state = BPF_ITER_STATE_ACTIVE; @@ -1011,7 +969,7 @@ static int unmark_stack_slots_iter(struct bpf_verifier_env *env, struct bpf_reg_state *st = &slot->spilled_ptr; if (i == 0) - WARN_ON_ONCE(release_reference(env, st->ref_obj_id)); + WARN_ON_ONCE(release_reference(env, st->id)); bpf_mark_reg_not_init(env, st); @@ -1067,10 +1025,10 @@ static int is_iter_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_s if (st->type & PTR_UNTRUSTED) return -EPROTO; - /* only main (first) slot has ref_obj_id set */ - if (i == 0 && !st->ref_obj_id) + /* only main (first) slot has id set */ + if (i == 0 && !st->id) return -EINVAL; - if (i != 0 && st->ref_obj_id) + if (i != 0 && st->id) return -EINVAL; if (st->iter.btf != btf || st->iter.btf_id != btf_id) return -EINVAL; @@ -1109,7 +1067,7 @@ static int mark_stack_slot_irq_flag(struct bpf_verifier_env *env, __mark_reg_known_zero(st); st->type = PTR_TO_STACK; /* we don't have dedicated reg type */ - st->ref_obj_id = id; + st->id = id; st->irq.kfunc_class = kfunc_class; for (i = 0; i < BPF_REG_SIZE; i++) @@ -1143,7 +1101,7 @@ static int unmark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_r return -EINVAL; } - err = release_irq_state(env->cur_state, st->ref_obj_id); + err = release_irq_state(env->cur_state, st->id); WARN_ON_ONCE(err && err != -EACCES); if (err) { int insn_idx = 0; @@ -1207,7 +1165,7 @@ static int is_irq_flag_reg_valid_init(struct bpf_verifier_env *env, struct bpf_r slot = &state->stack[spi]; st = &slot->spilled_ptr; - if (!st->ref_obj_id) + if (!st->id) return -EINVAL; for (i = 0; i < BPF_REG_SIZE; i++) @@ -1448,7 +1406,7 @@ static struct bpf_reference_state *acquire_reference_state(struct bpf_verifier_e return &state->refs[new_ofs]; } -static int acquire_reference(struct bpf_verifier_env *env, int insn_idx) +static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int parent_id) { struct bpf_reference_state *s; @@ -1457,6 +1415,7 @@ static int acquire_reference(struct bpf_verifier_env *env, int insn_idx) return -ENOMEM; s->type = REF_TYPE_PTR; s->id = ++env->id_gen; + s->parent_id = parent_id; return s->id; } @@ -1513,17 +1472,25 @@ static void release_reference_state(struct bpf_verifier_state *state, int idx) return; } -static bool find_reference_state(struct bpf_verifier_state *state, int ptr_id) +static bool find_reference_state(struct bpf_verifier_state *state, int id) { int i; - for (i = 0; i < state->acquired_refs; i++) - if (state->refs[i].id == ptr_id) + for (i = 0; i < state->acquired_refs; i++) { + if (state->refs[i].type != REF_TYPE_PTR) + continue; + if (state->refs[i].id == id) return true; + } return false; } +static bool reg_is_referenced(struct bpf_verifier_env *env, const struct bpf_reg_state *reg) +{ + return find_reference_state(env->cur_state, reg->id); +} + static int release_lock_state(struct bpf_verifier_state *state, int type, int id, void *ptr) { void *prev_ptr = NULL; @@ -1837,7 +1804,7 @@ static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm) memset(((u8 *)reg) + sizeof(reg->type), 0, offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type)); reg->id = 0; - reg->ref_obj_id = 0; + reg->parent_id = 0; ___mark_reg_known(reg, imm); } @@ -1872,7 +1839,7 @@ static void mark_reg_known_zero(struct bpf_verifier_env *env, } static void __mark_dynptr_reg(struct bpf_reg_state *reg, enum bpf_dynptr_type type, - bool first_slot, int dynptr_id) + bool first_slot, int id, int parent_id) { /* reg->type has no meaning for STACK_DYNPTR, but when we set reg for * callback arguments, it does need to be CONST_PTR_TO_DYNPTR, so simply @@ -1881,7 +1848,8 @@ static void __mark_dynptr_reg(struct bpf_reg_state *reg, enum bpf_dynptr_type ty __mark_reg_known_zero(reg); reg->type = CONST_PTR_TO_DYNPTR; /* Give each dynptr a unique id to uniquely associate slices to it. */ - reg->id = dynptr_id; + reg->id = id; + reg->parent_id = parent_id; reg->dynptr.type = type; reg->dynptr.first_slot = first_slot; } @@ -2161,17 +2129,12 @@ out: /* Mark a register as having a completely unknown (scalar) value. */ void bpf_mark_reg_unknown_imprecise(struct bpf_reg_state *reg) { - /* - * Clear type, off, and union(map_ptr, range) and - * padding between 'type' and union - */ - memset(reg, 0, offsetof(struct bpf_reg_state, var_off)); + s32 subreg_def = reg->subreg_def; + + memset(reg, 0, sizeof(*reg)); reg->type = SCALAR_VALUE; - reg->id = 0; - reg->ref_obj_id = 0; reg->var_off = tnum_unknown; - reg->frameno = 0; - reg->precise = false; + reg->subreg_def = subreg_def; __mark_reg_unbounded(reg); } @@ -4330,7 +4293,7 @@ static int map_kptr_match_type(struct bpf_verifier_env *env, * referenced PTR_TO_BTF_ID, and that its fixed offset is 0. For the * normal store of unreferenced kptr, we must ensure var_off is zero. * Since ref_ptr cannot be accessed directly by BPF insns, check for - * reg->ref_obj_id is not needed here. + * reg->id is not needed here. */ if (__check_ptr_off_reg(env, reg, argno_from_reg(regno), true)) return -EACCES; @@ -4703,8 +4666,8 @@ static int __check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int of * type of narrower access. */ if (base_type(info->reg_type) == PTR_TO_BTF_ID) { - if (info->ref_obj_id && - !find_reference_state(env->cur_state, info->ref_obj_id)) { + if (info->ref_id && + !find_reference_state(env->cur_state, info->ref_id)) { verbose(env, "invalid bpf_context access off=%d. Reference may already be released\n", off); return -EACCES; @@ -4873,10 +4836,10 @@ static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = { [CONST_PTR_TO_MAP] = btf_bpf_map_id, }; -static bool is_trusted_reg(const struct bpf_reg_state *reg) +static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg) { /* A referenced register is always trusted. */ - if (reg->ref_obj_id) + if (reg_is_referenced(env, reg)) return true; /* Types listed in the reg2btf_ids are always trusted */ @@ -5790,7 +5753,7 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, ret = env->ops->btf_struct_access(&env->log, reg, off, size); } else { /* Writes are permitted with default btf_struct_access for - * program allocated objects (which always have ref_obj_id > 0), + * program allocated objects (which always have id > 0), * but not for untrusted PTR_TO_BTF_ID | MEM_ALLOC. */ if (atype != BPF_READ && !type_is_ptr_alloc_obj(reg->type)) { @@ -5799,8 +5762,8 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, } if (type_is_alloc(reg->type) && !type_is_non_owning_ref(reg->type) && - !(reg->type & MEM_RCU) && !reg->ref_obj_id) { - verifier_bug(env, "ref_obj_id for allocated object must be non-zero"); + !(reg->type & MEM_RCU) && !reg_is_referenced(env, reg)) { + verifier_bug(env, "allocated object must have a referenced id"); return -EFAULT; } @@ -5819,7 +5782,7 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, */ flag = PTR_UNTRUSTED; - } else if (is_trusted_reg(reg) || is_rcu_reg(reg)) { + } else if (is_trusted_reg(env, reg) || is_rcu_reg(reg)) { /* By default any pointer obtained from walking a trusted pointer is no * longer trusted, unless the field being accessed has explicitly been * marked as inheriting its parent's state of trust (either full or RCU). @@ -6217,8 +6180,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b if (base_type(info.reg_type) == PTR_TO_BTF_ID) { regs[value_regno].btf = info.btf; regs[value_regno].btf_id = info.btf_id; - regs[value_regno].id = info.ref_obj_id; - regs[value_regno].ref_obj_id = info.ref_obj_id; + regs[value_regno].id = info.ref_id; } if (type_may_be_null(info.reg_type) && !regs[value_regno].id) regs[value_regno].id = ++env->id_gen; @@ -7201,7 +7163,16 @@ static int process_kptr_func(struct bpf_verifier_env *env, int regno, return 0; } -/* There are two register types representing a bpf_dynptr, one is PTR_TO_STACK +/* + * Validate dynptr arguments for helper, kfunc and subprog. + * + * @dynptr is both input and output. It is populated when the argument is + * tagged with MEM_UNINIT (i.e., the dynptr argument that will be constructed) + * and consumed when the argument is expecting to be an initialized dynptr. + * @parent_id is used to track the referenced parent object (e.g., file or skb in + * qdisc program) when constructing a dynptr. + * + * There are two register types representing a bpf_dynptr, one is PTR_TO_STACK * which points to a stack slot, and the other is CONST_PTR_TO_DYNPTR. * * In both cases we deal with the first 8 bytes, but need to mark the next 8 @@ -7217,7 +7188,7 @@ static int process_kptr_func(struct bpf_verifier_env *env, int regno, */ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int insn_idx, enum bpf_arg_type arg_type, - int clone_ref_obj_id, struct bpf_dynptr_desc *dynptr) + int parent_id, struct bpf_dynptr_desc *dynptr) { int spi, err = 0; @@ -7258,7 +7229,7 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat return err; } - err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, clone_ref_obj_id); + err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, parent_id, dynptr); } else /* OBJ_RELEASE and None case from above */ { /* For the reg->type == PTR_TO_STACK case, bpf_dynptr is never const */ if (reg->type == CONST_PTR_TO_DYNPTR && (arg_type & OBJ_RELEASE)) { @@ -7300,17 +7271,17 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat if (dynptr) { dynptr->type = reg->dynptr.type; dynptr->id = reg->id; - dynptr->ref_obj_id = reg->ref_obj_id; + dynptr->parent_id = reg->parent_id; } } return err; } -static u32 iter_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int spi) +static u32 iter_ref_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int spi) { struct bpf_func_state *state = bpf_func(env, reg); - return state->stack[spi].spilled_ptr.ref_obj_id; + return state->stack[spi].spilled_ptr.id; } static bool is_iter_kfunc(struct bpf_kfunc_call_arg_meta *meta) @@ -7416,7 +7387,7 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * /* remember meta->iter info for process_iter_next_call() */ meta->iter.spi = spi; meta->iter.frameno = reg->frameno; - meta->ref_obj_id = iter_ref_obj_id(env, reg, spi); + meta->id = iter_ref_id(env, reg, spi); if (is_iter_destroy_kfunc(meta)) { err = unmark_stack_slots_iter(env, reg, nr_slots); @@ -7999,7 +7970,7 @@ static int check_func_arg_reg_off(struct bpf_verifier_env *env, /* When referenced register is passed to release function, its fixed * offset must be 0. * - * We will check arg_type_is_release reg has ref_obj_id when storing + * We will check arg_type_is_release reg has id when storing * meta->release_regno. */ if (arg_type_is_release(arg_type)) { @@ -8260,7 +8231,7 @@ skip_type_check: */ if (reg->type == PTR_TO_STACK) { spi = dynptr_get_spi(env, reg); - if (spi < 0 || !state->stack[spi].spilled_ptr.ref_obj_id) { + if (spi < 0 || !state->stack[spi].spilled_ptr.id) { verbose(env, "arg %d is an unacquired reference\n", regno); return -EINVAL; } @@ -8268,7 +8239,7 @@ skip_type_check: verbose(env, "cannot release unowned const bpf_dynptr\n"); return -EINVAL; } - } else if (!reg->ref_obj_id && !bpf_register_is_null(reg)) { + } else if (!reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { verbose(env, "R%d must be referenced when passed to release function\n", regno); return -EINVAL; @@ -8280,14 +8251,14 @@ skip_type_check: meta->release_regno = regno; } - if (reg->ref_obj_id && base_type(arg_type) != ARG_KPTR_XCHG_DEST) { - if (meta->ref_obj_id) { - verbose(env, "more than one arg with ref_obj_id R%d %u %u", - regno, reg->ref_obj_id, - meta->ref_obj_id); + if (reg_is_referenced(env, reg) && base_type(arg_type) != ARG_KPTR_XCHG_DEST) { + if (meta->id) { + verbose(env, "more than one arg with referenced id R%d %u %u", + regno, reg->id, + meta->id); return -EACCES; } - meta->ref_obj_id = reg->ref_obj_id; + meta->id = reg->id; } switch (base_type(arg_type)) { @@ -8898,14 +8869,14 @@ static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range reg->range = AT_PKT_END; } -static int release_reference_nomark(struct bpf_verifier_state *state, int ref_obj_id) +static int release_reference_nomark(struct bpf_verifier_state *state, int id) { int i; for (i = 0; i < state->acquired_refs; i++) { if (state->refs[i].type != REF_TYPE_PTR) continue; - if (state->refs[i].id == ref_obj_id) { + if (state->refs[i].id == id) { release_reference_state(state, i); return 0; } @@ -8913,26 +8884,83 @@ static int release_reference_nomark(struct bpf_verifier_state *state, int ref_ob return -EINVAL; } -/* The pointer with the specified id has released its reference to kernel - * resources. Identify all copies of the same pointer and clear the reference. - * - * This is the release function corresponding to acquire_reference(). Idempotent. - */ -static int release_reference(struct bpf_verifier_env *env, int ref_obj_id) +static int idstack_push(struct bpf_idmap *idmap, u32 id) +{ + int i; + + if (!id) + return 0; + + for (i = 0; i < idmap->cnt; i++) + if (idmap->map[i].old == id) + return 0; + + if (WARN_ON_ONCE(idmap->cnt >= BPF_ID_MAP_SIZE)) + return -EFAULT; + + idmap->map[idmap->cnt++].old = id; + return 0; +} + +static int idstack_pop(struct bpf_idmap *idmap) { + if (!idmap->cnt) + return 0; + + return idmap->map[--idmap->cnt].old; +} + +/* Release id and objects derived from it iteratively in a DFS manner */ +static int release_reference(struct bpf_verifier_env *env, int id) +{ + u32 mask = (1 << STACK_SPILL) | (1 << STACK_DYNPTR); struct bpf_verifier_state *vstate = env->cur_state; + struct bpf_idmap *idstack = &env->idmap_scratch; + struct bpf_stack_state *stack; struct bpf_func_state *state; struct bpf_reg_state *reg; - int err; + int i, err; - err = release_reference_nomark(vstate, ref_obj_id); + idstack->cnt = 0; + err = idstack_push(idstack, id); if (err) return err; - bpf_for_each_reg_in_vstate(vstate, state, reg, ({ - if (reg->ref_obj_id == ref_obj_id) - mark_reg_invalid(env, reg); - })); + if (find_reference_state(vstate, id)) + WARN_ON_ONCE(release_reference_nomark(vstate, id)); + + while ((id = idstack_pop(idstack))) { + /* + * Child references are inaccessible after parent is released, + * any child references that exist at this point are a leak. + */ + for (i = 0; i < vstate->acquired_refs; i++) { + if (vstate->refs[i].type != REF_TYPE_PTR) + continue; + if (vstate->refs[i].parent_id != id) + continue; + verbose(env, "Leaking reference id=%d alloc_insn=%d. Release it first.\n", + vstate->refs[i].id, vstate->refs[i].insn_idx); + return -EINVAL; + } + + bpf_for_each_reg_in_vstate_mask(vstate, state, reg, stack, mask, ({ + if (reg->id != id && reg->parent_id != id) + continue; + + /* Free objects derived from the current object */ + if (reg->parent_id == id) { + err = idstack_push(idstack, reg->id); + if (err) + return err; + } + + if (!stack || stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL) + mark_reg_invalid(env, reg); + else if (stack->slot_type[BPF_REG_SIZE - 1] == STACK_DYNPTR) + invalidate_dynptr(env, stack); + })); + } return 0; } @@ -9833,7 +9861,7 @@ static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exi * kernel. Type checks are performed later in check_return_code. */ if (type == BPF_PROG_TYPE_STRUCT_OPS && !exception_exit && - reg->ref_obj_id == state->refs[i].id) + reg->id == state->refs[i].id) continue; verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", state->refs[i].id, state->refs[i].insn_idx); @@ -10116,18 +10144,18 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn err = -EINVAL; if (arg_type_is_dynptr(fn->arg_type[meta.release_regno - BPF_REG_1])) { err = unmark_stack_slots_dynptr(env, ®s[meta.release_regno]); - } else if (func_id == BPF_FUNC_kptr_xchg && meta.ref_obj_id) { - u32 ref_obj_id = meta.ref_obj_id; + } else if (func_id == BPF_FUNC_kptr_xchg && meta.id) { + u32 id = meta.id; bool in_rcu = in_rcu_cs(env); struct bpf_func_state *state; struct bpf_reg_state *reg; - err = release_reference_nomark(env->cur_state, ref_obj_id); + err = release_reference_nomark(env->cur_state, id); if (!err) { bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ - if (reg->ref_obj_id == ref_obj_id) { + if (reg->id == id) { if (in_rcu && (reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU)) { - reg->ref_obj_id = 0; + reg->id = 0; reg->type &= ~MEM_ALLOC; reg->type |= MEM_RCU; } else { @@ -10136,19 +10164,16 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn } })); } - } else if (meta.ref_obj_id) { - err = release_reference(env, meta.ref_obj_id); + } else if (meta.id) { + err = release_reference(env, meta.id); } else if (bpf_register_is_null(®s[meta.release_regno])) { - /* meta.ref_obj_id can only be 0 if register that is meant to be + /* meta.id can only be 0 if register that is meant to be * released is NULL, which must be > R0. */ err = 0; } - if (err) { - verbose(env, "func %s#%d reference has not been acquired before\n", - func_id_name(func_id), func_id); + if (err) return err; - } } switch (func_id) { @@ -10413,24 +10438,40 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return -EFAULT; } - if (is_ptr_cast_function(func_id)) { - /* For release_reference() */ - regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id; + if (is_ptr_cast_function(func_id) && + find_reference_state(env->cur_state, meta.id)) { + struct bpf_verifier_state *branch; + struct bpf_reg_state *r0; + + /* + * In order for a release of any of the original or cast pointers + * to invalidate all other pointers, reuse the same reference id for + * the cast result. + * This reference id can't be used for nullness propagation, + * as cast might return NULL for a non-NULL input. + * Hence, explore the NULL case as a separate branch. + */ + branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false); + if (IS_ERR(branch)) + return PTR_ERR(branch); + + r0 = &branch->frame[branch->curframe]->regs[BPF_REG_0]; + __mark_reg_known_zero(r0); + r0->type = SCALAR_VALUE; + + regs[BPF_REG_0].type &= ~PTR_MAYBE_NULL; + regs[BPF_REG_0].id = meta.id; } else if (is_acquire_function(func_id, meta.map.ptr)) { - int id = acquire_reference(env, insn_idx); + int id = acquire_reference(env, insn_idx, 0); if (id < 0) return id; - /* For mark_ptr_or_null_reg() */ + regs[BPF_REG_0].id = id; - /* For release_reference() */ - regs[BPF_REG_0].ref_obj_id = id; } - if (func_id == BPF_FUNC_dynptr_data) { - regs[BPF_REG_0].dynptr_id = meta.dynptr.id; - regs[BPF_REG_0].ref_obj_id = meta.dynptr.ref_obj_id; - } + if (func_id == BPF_FUNC_dynptr_data) + regs[BPF_REG_0].parent_id = meta.dynptr.id; err = do_refine_retval_range(env, regs, fn->ret_type, func_id, &meta); if (err) @@ -11242,7 +11283,7 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, * btf_struct_ids_match() to walk the struct at the 0th offset, and * resolve types. */ - if ((is_kfunc_release(meta) && reg->ref_obj_id) || + if ((is_kfunc_release(meta) && reg_is_referenced(env, reg)) || btf_type_ids_nocast_alias(&env->log, reg_btf, reg_ref_id, meta->btf, ref_id)) strict_type_match = true; @@ -11346,36 +11387,21 @@ static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state return 0; } -static int ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 ref_obj_id) +static void ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 id) { - struct bpf_verifier_state *state = env->cur_state; struct bpf_func_state *unused; struct bpf_reg_state *reg; - int i; - if (!ref_obj_id) { - verifier_bug(env, "ref_obj_id is zero for owning -> non-owning conversion"); - return -EFAULT; - } + WARN_ON_ONCE(release_reference_nomark(env->cur_state, id)); - for (i = 0; i < state->acquired_refs; i++) { - if (state->refs[i].id != ref_obj_id) - continue; - - /* Clear ref_obj_id here so release_reference doesn't clobber - * the whole reg - */ - bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ - if (reg->ref_obj_id == ref_obj_id) { - reg->ref_obj_id = 0; - ref_set_non_owning(env, reg); - } - })); - return 0; - } + bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ + if (reg->id == id) { + reg->id = 0; + ref_set_non_owning(env, reg); + } + })); - verifier_bug(env, "ref state missing for ref_obj_id"); - return -EFAULT; + return; } /* Implementation details: @@ -11907,14 +11933,14 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ return -EACCES; } - if (reg->ref_obj_id) { - if (is_kfunc_release(meta) && meta->ref_obj_id) { - verifier_bug(env, "more than one arg with ref_obj_id %s %u %u", - reg_arg_name(env, argno), reg->ref_obj_id, - meta->ref_obj_id); + if (reg_is_referenced(env, reg)) { + if (is_kfunc_release(meta) && meta->id) { + verifier_bug(env, "more than one arg with referenced id %s %u %u", + reg_arg_name(env, argno), reg->id, + meta->id); return -EFAULT; } - meta->ref_obj_id = reg->ref_obj_id; + meta->id = reg->id; if (is_kfunc_release(meta)) { if (regno < 0) { verbose(env, "%s release arg cannot be a stack argument\n", @@ -11975,7 +12001,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ fallthrough; case KF_ARG_PTR_TO_ALLOC_BTF_ID: case KF_ARG_PTR_TO_BTF_ID: - if (!is_trusted_reg(reg)) { + if (!is_trusted_reg(env, reg)) { if (!is_kfunc_rcu(meta)) { verbose(env, "%s must be referenced or trusted\n", reg_arg_name(env, argno)); @@ -12013,7 +12039,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ return -EFAULT; } - if (is_kfunc_release(meta) && reg->ref_obj_id) + if (is_kfunc_release(meta) && reg_is_referenced(env, reg)) arg_type |= OBJ_RELEASE; ret = check_func_arg_reg_off(env, reg, argno, arg_type); if (ret < 0) @@ -12052,7 +12078,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ reg_arg_name(env, argno)); return -EINVAL; } - if (!reg->ref_obj_id) { + if (!reg_is_referenced(env, reg)) { verbose(env, "allocated object must be referenced\n"); return -EINVAL; } @@ -12064,7 +12090,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ case KF_ARG_PTR_TO_DYNPTR: { enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR; - int clone_ref_obj_id = 0; if (is_kfunc_arg_uninit(btf, &args[i])) dynptr_arg_type |= MEM_UNINIT; @@ -12095,15 +12120,10 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ } dynptr_arg_type |= (unsigned int)get_dynptr_type_flag(parent_type); - clone_ref_obj_id = meta->dynptr.ref_obj_id; - if (dynptr_type_refcounted(parent_type) && !clone_ref_obj_id) { - verifier_bug(env, "missing ref obj id for parent of clone"); - return -EFAULT; - } } ret = process_dynptr_func(env, reg, argno, insn_idx, dynptr_arg_type, - clone_ref_obj_id, &meta->dynptr); + meta->id, &meta->dynptr); if (ret < 0) return ret; break; @@ -12126,7 +12146,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ reg_arg_name(env, argno)); return -EINVAL; } - if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg->ref_obj_id) { + if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && + !reg_is_referenced(env, reg)) { verbose(env, "allocated object must be referenced\n"); return -EINVAL; } @@ -12141,7 +12162,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ reg_arg_name(env, argno)); return -EINVAL; } - if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg->ref_obj_id) { + if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && + !reg_is_referenced(env, reg)) { verbose(env, "allocated object must be referenced\n"); return -EINVAL; } @@ -12151,7 +12173,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ break; case KF_ARG_PTR_TO_LIST_NODE: if (is_kfunc_arg_nonown_allowed(btf, &args[i]) && - type_is_non_owning_ref(reg->type) && !reg->ref_obj_id) { + type_is_non_owning_ref(reg->type) && !reg_is_referenced(env, reg)) { /* Allow bpf_list_front/back return value for * __nonown_allowed list-node arguments. */ @@ -12162,7 +12184,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ reg_arg_name(env, argno)); return -EINVAL; } - if (!reg->ref_obj_id) { + if (!reg_is_referenced(env, reg)) { verbose(env, "allocated object must be referenced\n"); return -EINVAL; } @@ -12178,12 +12200,13 @@ check_ok: reg_arg_name(env, argno)); return -EINVAL; } - if (!reg->ref_obj_id) { + if (!reg_is_referenced(env, reg)) { verbose(env, "allocated object must be referenced\n"); return -EINVAL; } } else { - if (!type_is_non_owning_ref(reg->type) && !reg->ref_obj_id) { + if (!type_is_non_owning_ref(reg->type) && + !reg_is_referenced(env, reg)) { verbose(env, "%s can only take non-owning or refcounted bpf_rb_node pointer\n", func_name); return -EINVAL; } @@ -12764,12 +12787,7 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_ca verifier_bug(env, "no dynptr id"); return -EFAULT; } - regs[BPF_REG_0].dynptr_id = meta->dynptr.id; - - /* we don't need to set BPF_REG_0's ref obj id - * because packet slices are not refcounted (see - * dynptr_type_refcounted) - */ + regs[BPF_REG_0].parent_id = meta->dynptr.id; } else { return 0; } @@ -12783,13 +12801,13 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, int *insn_idx_p) { bool sleepable, rcu_lock, rcu_unlock, preempt_disable, preempt_enable; - u32 i, nargs, ptr_type_id, release_ref_obj_id; struct bpf_reg_state *regs = cur_regs(env); const char *func_name, *ptr_type_name; const struct btf_type *t, *ptr_type; struct bpf_kfunc_call_arg_meta meta; struct bpf_insn_aux_data *insn_aux; int err, insn_idx = *insn_idx_p; + u32 i, nargs, ptr_type_id, id; const struct btf_param *args; struct btf *desc_btf; @@ -12902,6 +12920,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (rcu_lock) { env->cur_state->active_rcu_locks++; } else if (rcu_unlock) { + struct bpf_stack_state *stack; struct bpf_func_state *state; struct bpf_reg_state *reg; u32 clear_mask = (1 << STACK_SPILL) | (1 << STACK_ITER); @@ -12911,7 +12930,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return -EINVAL; } if (--env->cur_state->active_rcu_locks == 0) { - bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, clear_mask, ({ + bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, stack, clear_mask, ({ if (reg->type & MEM_RCU) { reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL); reg->type |= PTR_UNTRUSTED; @@ -12950,35 +12969,20 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (meta.release_regno) { struct bpf_reg_state *reg = ®s[meta.release_regno]; - if (meta.dynptr.ref_obj_id) { + if (meta.dynptr.id) { err = unmark_stack_slots_dynptr(env, reg); } else { - err = release_reference(env, reg->ref_obj_id); - if (err) - verbose(env, "kfunc %s#%d reference has not been acquired before\n", - func_name, meta.func_id); + err = release_reference(env, reg->id); } if (err) return err; } if (is_bpf_list_push_kfunc(meta.func_id) || is_bpf_rbtree_add_kfunc(meta.func_id)) { - release_ref_obj_id = regs[BPF_REG_2].ref_obj_id; + id = regs[BPF_REG_2].id; insn_aux->insert_off = regs[BPF_REG_2].var_off.value; insn_aux->kptr_struct_meta = btf_find_struct_meta(meta.arg_btf, meta.arg_btf_id); - err = ref_convert_owning_non_owning(env, release_ref_obj_id); - if (err) { - verbose(env, "kfunc %s#%d conversion of owning ref to non-owning failed\n", - func_name, meta.func_id); - return err; - } - - err = release_reference(env, release_ref_obj_id); - if (err) { - verbose(env, "kfunc %s#%d reference has not been acquired before\n", - func_name, meta.func_id); - return err; - } + ref_convert_owning_non_owning(env, id); } if (meta.func_id == special_kfunc_list[KF_bpf_throw]) { @@ -13063,8 +13067,8 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, regs[BPF_REG_0].type |= MEM_RDONLY; /* Ensures we don't access the memory after a release_reference() */ - if (meta.ref_obj_id) - regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id; + if (meta.id) + regs[BPF_REG_0].parent_id = meta.id; if (is_kfunc_rcu_protected(&meta)) regs[BPF_REG_0].type |= MEM_RCU; @@ -13110,13 +13114,10 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *)); if (is_kfunc_acquire(&meta)) { - int id = acquire_reference(env, insn_idx); - + id = acquire_reference(env, insn_idx, 0); if (id < 0) return id; - if (is_kfunc_ret_null(&meta)) - regs[BPF_REG_0].id = id; - regs[BPF_REG_0].ref_obj_id = id; + regs[BPF_REG_0].id = id; } else if (is_rbtree_node_type(ptr_type) || is_list_node_type(ptr_type)) { ref_set_non_owning(env, ®s[BPF_REG_0]); } @@ -15347,7 +15348,7 @@ static int is_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_state *r if (!is_reg_const(reg2, is_jmp32)) return -1; - if (!reg_not_null(reg1)) + if (!reg_not_null(env, reg1)) return -1; /* If pointer is valid tests against zero will fail so we can @@ -15564,7 +15565,7 @@ static void mark_ptr_or_null_reg(struct bpf_func_state *state, WARN_ON_ONCE(!tnum_equals_const(reg->var_off, 0))) return; if (is_null) { - /* We don't need id and ref_obj_id from this point + /* We don't need id from this point * onwards anymore, thus we should better reset it, * so that state pruning has chances to take effect. */ @@ -15591,10 +15592,9 @@ static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno, { struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = state->regs, *reg; - u32 ref_obj_id = regs[regno].ref_obj_id; u32 id = regs[regno].id; - if (ref_obj_id && ref_obj_id == id && is_null) + if (is_null && find_reference_state(vstate, id)) /* regs[regno] is in the " == NULL" branch. * No one could have freed the reference state before * doing the NULL check. @@ -16433,7 +16433,7 @@ static int check_return_code(struct bpf_verifier_env *env, int regno, const char ret_type = btf_type_resolve_ptr(prog->aux->attach_btf, prog->aux->attach_func_proto->type, NULL); - if (ret_type && ret_type == reg_type && reg->ref_obj_id) + if (ret_type && ret_type == reg_type && reg_is_referenced(env, reg)) return __check_ptr_off_reg(env, reg, argno_from_reg(regno), false); } @@ -18302,7 +18302,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) mark_reg_unknown(env, regs, i); } else if (arg->arg_type == ARG_PTR_TO_DYNPTR) { /* assume unspecial LOCAL dynptr type */ - __mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_LOCAL, true, ++env->id_gen); + __mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_LOCAL, true, ++env->id_gen, 0); } else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) { reg->type = PTR_TO_MEM; reg->type |= arg->arg_type & @@ -18361,8 +18361,8 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) /* Acquire references for struct_ops program arguments tagged with "__ref" */ if (!subprog && env->prog->type == BPF_PROG_TYPE_STRUCT_OPS) { for (i = 0; i < aux->ctx_arg_info_size; i++) - aux->ctx_arg_info[i].ref_obj_id = aux->ctx_arg_info[i].refcounted ? - acquire_reference(env, 0) : 0; + aux->ctx_arg_info[i].ref_id = aux->ctx_arg_info[i].refcounted ? + acquire_reference(env, 0, 0) : 0; } ret = do_check(env); -- cgit v1.2.3 From 92d681b42746d4497dcc8afb45edd4af5737542f Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Thu, 28 May 2026 18:49:29 -0700 Subject: bpf: Remove redundant dynptr arg check for helper unmark_stack_slots_dynptr() already makes sure that CONST_PTR_TO_DYNPTR cannot be released. process_dynptr_func() also prevents passing uninitialized dynptr to helpers expecting initialized dynptr. Now that unmark_stack_slots_dynptr() also reports error returned from release_reference(), there should be no reason to keep these redundant checks. Acked-by: Eduard Zingerman Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260529014936.2811085-7-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 6d82ca5acacb..4f75e5f95d27 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8220,26 +8220,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, skip_type_check: if (arg_type_is_release(arg_type)) { - if (arg_type_is_dynptr(arg_type)) { - struct bpf_func_state *state = bpf_func(env, reg); - int spi; - - /* Only dynptr created on stack can be released, thus - * the get_spi and stack state checks for spilled_ptr - * should only be done before process_dynptr_func for - * PTR_TO_STACK. - */ - if (reg->type == PTR_TO_STACK) { - spi = dynptr_get_spi(env, reg); - if (spi < 0 || !state->stack[spi].spilled_ptr.id) { - verbose(env, "arg %d is an unacquired reference\n", regno); - return -EINVAL; - } - } else { - verbose(env, "cannot release unowned const bpf_dynptr\n"); - return -EINVAL; - } - } else if (!reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { + if (!arg_type_is_dynptr(arg_type) && !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { verbose(env, "R%d must be referenced when passed to release function\n", regno); return -EINVAL; -- cgit v1.2.3 From b7dd2b388657d99689161e82ed13515505838232 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Thu, 28 May 2026 18:49:30 -0700 Subject: bpf: Unify referenced object tracking in verifier Helpers and kfuncs independently tracked referenced object metadata using standalone id fields in their respective arg_meta structs. This led to duplicated logic and inconsistent error handling between the two paths. Introduce struct ref_obj_desc to consolidate id and parent_id along with a count of how many arguments carry a reference. Add update_ref_obj() to populate it from a bpf_reg_state, replacing open-coded assignments in check_func_arg(), check_kfunc_args(), and process_iter_arg(). Add validate_ref_obj() to check for ambiguous ref_obj before using it. For ref_obj releasing helpers and kfuncs, keep checking it before calling update_ref_obj() for now. A later patch will make these functions not depending on ref_obj. For other users of ref_obj, move the checks to the use locations. For helper, this means moving the checks inside helper_multiple_ref_obj_use() to use locations. is_acquire_function() is dropped as ref_obj is never used. Pass ref_obj_desc into process_dynptr_func()/mark_stack_slots_dynptr() instead of a bare parent_id to make it less confusing. Drop the selftest introduced in 7ec899ac90a2 ("selftests/bpf: Negative test case for ref_obj_id in args") since the verifier no longer complains about ambiguous ref_obj if it is not used. Acked-by: Eduard Zingerman Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260529014936.2811085-8-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 122 +++++++++++++++++++++++++------------------------- 1 file changed, 62 insertions(+), 60 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 4f75e5f95d27..bc8a09c858d8 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -231,9 +231,28 @@ static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state) (poisoned ? BPF_MAP_KEY_POISON : 0ULL); } +static void update_ref_obj(struct ref_obj_desc *ref_obj, struct bpf_reg_state *reg) +{ + ref_obj->id = reg->id; + ref_obj->parent_id = reg->parent_id; + ref_obj->cnt++; +} + +static int validate_ref_obj(struct bpf_verifier_env *env, struct ref_obj_desc *ref_obj) +{ + if (ref_obj->cnt > 1) { + verifier_bug(env, "function expects only one referenced object but got %d\n", + ref_obj->cnt); + return -EFAULT; + } + + return 0; +} + struct bpf_call_arg_meta { struct bpf_map_desc map; struct bpf_dynptr_desc dynptr; + struct ref_obj_desc ref_obj; bool raw_mode; bool pkt_access; u8 release_regno; @@ -241,7 +260,6 @@ struct bpf_call_arg_meta { int access_size; int mem_size; u64 msize_max_value; - u32 id; int func_id; struct btf *btf; u32 btf_id; @@ -528,20 +546,6 @@ bool bpf_is_may_goto_insn(struct bpf_insn *insn) return insn->code == (BPF_JMP | BPF_JCOND) && insn->src_reg == BPF_MAY_GOTO; } -static bool helper_multiple_ref_obj_use(enum bpf_func_id func_id, - const struct bpf_map *map) -{ - int ref_obj_uses = 0; - - if (is_ptr_cast_function(func_id)) - ref_obj_uses++; - if (is_acquire_function(func_id, map)) - ref_obj_uses++; - - return ref_obj_uses > 1; -} - - static bool is_spi_bounds_valid(struct bpf_func_state *state, int spi, int nr_slots) { int allocated_slots = state->allocated_stack / BPF_REG_SIZE; @@ -670,11 +674,11 @@ static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi); static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - enum bpf_arg_type arg_type, int insn_idx, int parent_id, - struct bpf_dynptr_desc *dynptr) + enum bpf_arg_type arg_type, int insn_idx, + struct ref_obj_desc *ref_obj, struct bpf_dynptr_desc *dynptr) { struct bpf_func_state *state = bpf_func(env, reg); - int spi, i, err; + int spi, i, err, parent_id = 0; enum bpf_dynptr_type type; spi = dynptr_get_spi(env, reg); @@ -707,6 +711,13 @@ static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_ return -EINVAL; if (dynptr->type == BPF_DYNPTR_TYPE_INVALID) { /* dynptr constructors */ + err = validate_ref_obj(env, ref_obj); + if (err) + return err; + + /* Track parent's id if the parent is a referenced object */ + parent_id = ref_obj->id; + if (dynptr_type_referenced(type)) { int id; @@ -7188,7 +7199,7 @@ static int process_kptr_func(struct bpf_verifier_env *env, int regno, */ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int insn_idx, enum bpf_arg_type arg_type, - int parent_id, struct bpf_dynptr_desc *dynptr) + struct ref_obj_desc *ref_obj, struct bpf_dynptr_desc *dynptr) { int spi, err = 0; @@ -7229,7 +7240,7 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat return err; } - err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, parent_id, dynptr); + err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, ref_obj, dynptr); } else /* OBJ_RELEASE and None case from above */ { /* For the reg->type == PTR_TO_STACK case, bpf_dynptr is never const */ if (reg->type == CONST_PTR_TO_DYNPTR && (arg_type & OBJ_RELEASE)) { @@ -7277,13 +7288,6 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat return err; } -static u32 iter_ref_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int spi) -{ - struct bpf_func_state *state = bpf_func(env, reg); - - return state->stack[spi].spilled_ptr.id; -} - static bool is_iter_kfunc(struct bpf_kfunc_call_arg_meta *meta) { return meta->kfunc_flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY); @@ -7316,6 +7320,7 @@ static bool is_kfunc_arg_iter(struct bpf_kfunc_call_arg_meta *meta, int arg_idx, static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int insn_idx, struct bpf_kfunc_call_arg_meta *meta) { + struct bpf_func_state *state = bpf_func(env, reg); const struct btf_type *t; u32 arg_idx = arg_idx_from_argno(argno); int spi, err, i, nr_slots, btf_id; @@ -7387,7 +7392,7 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * /* remember meta->iter info for process_iter_next_call() */ meta->iter.spi = spi; meta->iter.frameno = reg->frameno; - meta->id = iter_ref_id(env, reg, spi); + update_ref_obj(&meta->ref_obj, &state->stack[spi].spilled_ptr); if (is_iter_destroy_kfunc(meta)) { err = unmark_stack_slots_iter(env, reg, nr_slots); @@ -8166,6 +8171,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 regno = BPF_REG_1 + arg; struct bpf_reg_state *reg = reg_state(env, regno); enum bpf_arg_type arg_type = fn->arg_type[arg]; + argno_t argno = argno_from_arg(arg + 1); enum bpf_reg_type type = reg->type; u32 *arg_btf_id = NULL; u32 key_size; @@ -8232,15 +8238,8 @@ skip_type_check: meta->release_regno = regno; } - if (reg_is_referenced(env, reg) && base_type(arg_type) != ARG_KPTR_XCHG_DEST) { - if (meta->id) { - verbose(env, "more than one arg with referenced id R%d %u %u", - regno, reg->id, - meta->id); - return -EACCES; - } - meta->id = reg->id; - } + if (reg_is_referenced(env, reg)) + update_ref_obj(&meta->ref_obj, reg); switch (base_type(arg_type)) { case ARG_CONST_MAP_PTR: @@ -8379,7 +8378,7 @@ skip_type_check: true, meta); break; case ARG_PTR_TO_DYNPTR: - err = process_dynptr_func(env, reg, argno_from_reg(regno), insn_idx, arg_type, 0, + err = process_dynptr_func(env, reg, argno_from_reg(regno), insn_idx, arg_type, &meta->ref_obj, &meta->dynptr); if (err) return err; @@ -9042,6 +9041,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, struct bpf_subprog_info *sub = subprog_info(env, subprog); struct bpf_func_state *caller = cur_func(env); struct bpf_verifier_log *log = &env->log; + struct ref_obj_desc ref_obj = {}; u32 i; int ret, err; @@ -9119,7 +9119,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, if (ret) return ret; - ret = process_dynptr_func(env, reg, argno, -1, arg->arg_type, 0, NULL); + ret = process_dynptr_func(env, reg, argno, -1, arg->arg_type, &ref_obj, NULL); if (ret) return ret; } else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) { @@ -10125,8 +10125,8 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn err = -EINVAL; if (arg_type_is_dynptr(fn->arg_type[meta.release_regno - BPF_REG_1])) { err = unmark_stack_slots_dynptr(env, ®s[meta.release_regno]); - } else if (func_id == BPF_FUNC_kptr_xchg && meta.id) { - u32 id = meta.id; + } else if (func_id == BPF_FUNC_kptr_xchg && meta.ref_obj.id) { + u32 id = meta.ref_obj.id; bool in_rcu = in_rcu_cs(env); struct bpf_func_state *state; struct bpf_reg_state *reg; @@ -10145,10 +10145,10 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn } })); } - } else if (meta.id) { - err = release_reference(env, meta.id); + } else if (meta.ref_obj.id) { + err = release_reference(env, meta.ref_obj.id); } else if (bpf_register_is_null(®s[meta.release_regno])) { - /* meta.id can only be 0 if register that is meant to be + /* meta.ref_obj.id can only be 0 if register that is meant to be * released is NULL, which must be > R0. */ err = 0; @@ -10413,17 +10413,15 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (type_may_be_null(regs[BPF_REG_0].type)) regs[BPF_REG_0].id = ++env->id_gen; - if (helper_multiple_ref_obj_use(func_id, meta.map.ptr)) { - verifier_bug(env, "func %s#%d sets ref_obj_id more than once", - func_id_name(func_id), func_id); - return -EFAULT; - } - if (is_ptr_cast_function(func_id) && - find_reference_state(env->cur_state, meta.id)) { + find_reference_state(env->cur_state, meta.ref_obj.id)) { struct bpf_verifier_state *branch; struct bpf_reg_state *r0; + err = validate_ref_obj(env, &meta.ref_obj); + if (err) + return err; + /* * In order for a release of any of the original or cast pointers * to invalidate all other pointers, reuse the same reference id for @@ -10441,7 +10439,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn r0->type = SCALAR_VALUE; regs[BPF_REG_0].type &= ~PTR_MAYBE_NULL; - regs[BPF_REG_0].id = meta.id; + regs[BPF_REG_0].id = meta.ref_obj.id; } else if (is_acquire_function(func_id, meta.map.ptr)) { int id = acquire_reference(env, insn_idx, 0); @@ -11915,13 +11913,13 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ } if (reg_is_referenced(env, reg)) { - if (is_kfunc_release(meta) && meta->id) { - verifier_bug(env, "more than one arg with referenced id %s %u %u", - reg_arg_name(env, argno), reg->id, - meta->id); + if (is_kfunc_release(meta) && meta->ref_obj.cnt) { + verbose(env, "more than one arg with referenced id %s %u %u", + reg_arg_name(env, argno), reg->id, + meta->ref_obj.id); return -EFAULT; } - meta->id = reg->id; + update_ref_obj(&meta->ref_obj, reg); if (is_kfunc_release(meta)) { if (regno < 0) { verbose(env, "%s release arg cannot be a stack argument\n", @@ -12104,7 +12102,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ } ret = process_dynptr_func(env, reg, argno, insn_idx, dynptr_arg_type, - meta->id, &meta->dynptr); + &meta->ref_obj, &meta->dynptr); if (ret < 0) return ret; break; @@ -13048,8 +13046,12 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, regs[BPF_REG_0].type |= MEM_RDONLY; /* Ensures we don't access the memory after a release_reference() */ - if (meta.id) - regs[BPF_REG_0].parent_id = meta.id; + if (meta.ref_obj.id) { + err = validate_ref_obj(env, &meta.ref_obj); + if (err) + return err; + regs[BPF_REG_0].parent_id = meta.ref_obj.id; + } if (is_kfunc_rcu_protected(&meta)) regs[BPF_REG_0].type |= MEM_RCU; -- cgit v1.2.3 From bcfcb15fde94ed39068eb1d6e4b9b37d27111965 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Thu, 28 May 2026 18:49:31 -0700 Subject: bpf: Unify release handling for helpers and kfuncs Introduce release_reg() to consolidate the release logic shared by both helpers and kfuncs: dynptr release, kptr_xchg percpu-to-RCU conversion, regular reference release, and NULL pass-through. NULL pass-through is only allowed if the prototype indicates the argument may be null. Determine release_regno from the function prototype/metadata before argument checking, rather than discovering it dynamically during argument processing. For helpers, scan the arg_type array in check_func_proto() via check_proto_release_reg(). For kfuncs, set release_regno to BPF_REG_1 in bpf_fetch_kfunc_arg_meta() when KF_RELEASE is set. In the future when we start adding decl_tag to kfunc arguments, we can just look at the function prototype instead of a release_regno. Extract ref_convert_alloc_rcu_protected() and invalidate_rcu_protected_refs() to make it more clear what the code is doing. For ref_convert_alloc_rcu_protected(), it pre-converts MEM_ALLOC | MEM_PERCPU registers to MEM_RCU (clearing id so they survive), then calls release_reference() to invalidate the remaining registers and release the reference state. Add KF_RELEASE to bpf_dynptr_file_discard() so its release_regno is set via fetch_kfunc_meta rather than being assigned manually in the dynptr argument processing. Set arg_type to ARG_PTR_TO_DYNPTR for KF_ARG_PTR_TO_DYNPTR so that check_func_arg_reg_off() correctly allows non-zero stack offsets for dynptr release arguments same as helper. Acked-by: Eduard Zingerman Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260529014936.2811085-9-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/helpers.c | 2 +- kernel/bpf/verifier.c | 198 ++++++++++++++++++++++++++------------------------ 2 files changed, 104 insertions(+), 96 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 9ca195104667..03004e4451f5 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -4957,7 +4957,7 @@ BTF_ID_FLAGS(func, bpf_stream_print_stack, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_task_work_schedule_signal, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_task_work_schedule_resume, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_dynptr_from_file) -BTF_ID_FLAGS(func, bpf_dynptr_file_discard) +BTF_ID_FLAGS(func, bpf_dynptr_file_discard, KF_RELEASE) BTF_ID_FLAGS(func, bpf_timer_cancel_async) BTF_KFUNCS_END(common_btf_ids) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index bc8a09c858d8..caa455fad877 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8225,17 +8225,11 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, return err; skip_type_check: - if (arg_type_is_release(arg_type)) { - if (!arg_type_is_dynptr(arg_type) && !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { - verbose(env, "R%d must be referenced when passed to release function\n", - regno); - return -EINVAL; - } - if (meta->release_regno) { - verifier_bug(env, "more than one release argument"); - return -EFAULT; - } - meta->release_regno = regno; + if (arg_type_is_release(arg_type) && !arg_type_is_dynptr(arg_type) && + !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { + verbose(env, "release helper %s expects referenced PTR_TO_BTF_ID passed to %s\n", + func_id_name(meta->func_id), reg_arg_name(env, argno)); + return -EINVAL; } if (reg_is_referenced(env, reg)) @@ -8798,11 +8792,29 @@ static bool check_mem_arg_rw_flag_ok(const struct bpf_func_proto *fn) return true; } -static int check_func_proto(const struct bpf_func_proto *fn) +static bool check_proto_release_reg(const struct bpf_func_proto *fn, struct bpf_call_arg_meta *meta) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { + enum bpf_arg_type arg_type = fn->arg_type[i]; + + if (arg_type_is_release(arg_type)) { + if (meta->release_regno) + return false; + meta->release_regno = i + 1; + } + } + + return true; +} + +static int check_func_proto(const struct bpf_func_proto *fn, struct bpf_call_arg_meta *meta) { return check_raw_mode_ok(fn) && check_arg_pair_ok(fn) && check_mem_arg_rw_flag_ok(fn) && + check_proto_release_reg(fn, meta) && check_btf_id_ok(fn) ? 0 : -EINVAL; } @@ -8956,6 +8968,42 @@ static void invalidate_non_owning_refs(struct bpf_verifier_env *env) })); } +static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env) +{ + struct bpf_stack_state *stack; + struct bpf_func_state *state; + struct bpf_reg_state *reg; + u32 clear_mask = (1 << STACK_SPILL) | (1 << STACK_ITER); + + bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, stack, clear_mask, ({ + if (reg->type & MEM_RCU) { + reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL); + reg->type |= PTR_UNTRUSTED; + } + })); +} + +static int ref_convert_alloc_rcu_protected(struct bpf_verifier_env *env, u32 id) +{ + struct bpf_func_state *state; + struct bpf_reg_state *reg; + int err; + + err = release_reference_nomark(env->cur_state, id); + + bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ + if (reg->id != id) + continue; + if ((reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU)) { + reg->id = 0; + reg->type &= ~MEM_ALLOC; + reg->type |= MEM_RCU; + } + })); + + return err; +} + static void clear_caller_saved_regs(struct bpf_verifier_env *env, struct bpf_reg_state *regs) { @@ -10028,6 +10076,24 @@ static const char *non_sleepable_context_description(struct bpf_verifier_env *en return "non-sleepable prog"; } +static int release_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + bool convert_rcu, bool release_dynptr) +{ + int err = -EINVAL; + + if (bpf_register_is_null(reg)) + return 0; + + if (release_dynptr) + err = unmark_stack_slots_dynptr(env, reg); + else if (convert_rcu) + err = ref_convert_alloc_rcu_protected(env, reg->id); + else if (reg_is_referenced(env, reg)) + err = release_reference(env, reg->id); + + return err; +} + static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn, int *insn_idx_p) { @@ -10077,7 +10143,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn memset(&meta, 0, sizeof(meta)); meta.pkt_access = fn->pkt_access; - err = check_func_proto(fn); + err = check_func_proto(fn, &meta); if (err) { verifier_bug(env, "incorrect func proto %s#%d", func_id_name(func_id), func_id); return err; @@ -10122,37 +10188,11 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn } if (meta.release_regno) { - err = -EINVAL; - if (arg_type_is_dynptr(fn->arg_type[meta.release_regno - BPF_REG_1])) { - err = unmark_stack_slots_dynptr(env, ®s[meta.release_regno]); - } else if (func_id == BPF_FUNC_kptr_xchg && meta.ref_obj.id) { - u32 id = meta.ref_obj.id; - bool in_rcu = in_rcu_cs(env); - struct bpf_func_state *state; - struct bpf_reg_state *reg; - - err = release_reference_nomark(env->cur_state, id); - if (!err) { - bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ - if (reg->id == id) { - if (in_rcu && (reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU)) { - reg->id = 0; - reg->type &= ~MEM_ALLOC; - reg->type |= MEM_RCU; - } else { - mark_reg_invalid(env, reg); - } - } - })); - } - } else if (meta.ref_obj.id) { - err = release_reference(env, meta.ref_obj.id); - } else if (bpf_register_is_null(®s[meta.release_regno])) { - /* meta.ref_obj.id can only be 0 if register that is meant to be - * released is NULL, which must be > R0. - */ - err = 0; - } + struct bpf_reg_state *reg = ®s[meta.release_regno]; + bool convert_rcu = (func_id == BPF_FUNC_kptr_xchg) && in_rcu_cs(env) && + (reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU); + + err = release_reg(env, reg, convert_rcu, !!meta.dynptr.id); if (err) return err; } @@ -10547,7 +10587,6 @@ static bool is_kfunc_release(struct bpf_kfunc_call_arg_meta *meta) return meta->kfunc_flags & KF_RELEASE; } - static bool is_kfunc_destructive(struct bpf_kfunc_call_arg_meta *meta) { return meta->kfunc_flags & KF_DESTRUCTIVE; @@ -11912,24 +11951,16 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ return -EACCES; } - if (reg_is_referenced(env, reg)) { - if (is_kfunc_release(meta) && meta->ref_obj.cnt) { - verbose(env, "more than one arg with referenced id %s %u %u", - reg_arg_name(env, argno), reg->id, - meta->ref_obj.id); - return -EFAULT; - } - update_ref_obj(&meta->ref_obj, reg); - if (is_kfunc_release(meta)) { - if (regno < 0) { - verbose(env, "%s release arg cannot be a stack argument\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - meta->release_regno = regno; - } + if (regno == meta->release_regno && !is_kfunc_arg_dynptr(meta->btf, &args[i]) && + !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { + verbose(env, "release kfunc %s expects referenced PTR_TO_BTF_ID passed to %s\n", + func_name, reg_arg_name(env, argno)); + return -EINVAL; } + if (reg_is_referenced(env, reg)) + update_ref_obj(&meta->ref_obj, reg); + ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); ref_tname = btf_name_by_offset(btf, ref_t->name_off); @@ -11993,7 +12024,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ } } fallthrough; - case KF_ARG_PTR_TO_DYNPTR: case KF_ARG_PTR_TO_ITER: case KF_ARG_PTR_TO_LIST_HEAD: case KF_ARG_PTR_TO_LIST_NODE: @@ -12010,6 +12040,9 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ case KF_ARG_PTR_TO_IRQ_FLAG: case KF_ARG_PTR_TO_RES_SPIN_LOCK: break; + case KF_ARG_PTR_TO_DYNPTR: + arg_type = ARG_PTR_TO_DYNPTR; + break; case KF_ARG_PTR_TO_CTX: arg_type = ARG_PTR_TO_CTX; break; @@ -12018,7 +12051,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ return -EFAULT; } - if (is_kfunc_release(meta) && reg_is_referenced(env, reg)) + if (regno == meta->release_regno) arg_type |= OBJ_RELEASE; ret = check_func_arg_reg_off(env, reg, argno, arg_type); if (ret < 0) @@ -12083,12 +12116,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ dynptr_arg_type |= DYNPTR_TYPE_FILE; } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_file_discard]) { dynptr_arg_type |= DYNPTR_TYPE_FILE | OBJ_RELEASE; - if (regno < 0) { - verbose(env, "%s release arg cannot be a stack argument\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - meta->release_regno = regno; } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_clone] && (dynptr_arg_type & MEM_UNINIT)) { enum bpf_dynptr_type parent_type = meta->dynptr.type; @@ -12377,12 +12404,6 @@ check_ok: } } - if (is_kfunc_release(meta) && !meta->release_regno) { - verbose(env, "release kernel function %s expects refcounted PTR_TO_BTF_ID\n", - func_name); - return -EINVAL; - } - return 0; } @@ -12409,6 +12430,10 @@ int bpf_fetch_kfunc_arg_meta(struct bpf_verifier_env *env, meta->kfunc_flags = *kfunc.flags; + /* Only support release referenced argument passed by register */ + if (is_kfunc_release(meta)) + meta->release_regno = BPF_REG_1; + return 0; } @@ -12899,23 +12924,12 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (rcu_lock) { env->cur_state->active_rcu_locks++; } else if (rcu_unlock) { - struct bpf_stack_state *stack; - struct bpf_func_state *state; - struct bpf_reg_state *reg; - u32 clear_mask = (1 << STACK_SPILL) | (1 << STACK_ITER); - if (env->cur_state->active_rcu_locks == 0) { verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name); return -EINVAL; } - if (--env->cur_state->active_rcu_locks == 0) { - bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, stack, clear_mask, ({ - if (reg->type & MEM_RCU) { - reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL); - reg->type |= PTR_UNTRUSTED; - } - })); - } + if (--env->cur_state->active_rcu_locks == 0) + invalidate_rcu_protected_refs(env); } else if (preempt_disable) { env->cur_state->active_preempt_locks++; } else if (preempt_enable) { @@ -12946,13 +12960,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, * PTR_TO_BTF_ID in bpf_kfunc_arg_meta, do the release now. */ if (meta.release_regno) { - struct bpf_reg_state *reg = ®s[meta.release_regno]; - - if (meta.dynptr.id) { - err = unmark_stack_slots_dynptr(env, reg); - } else { - err = release_reference(env, reg->id); - } + err = release_reg(env, ®s[meta.release_regno], false, !!meta.dynptr.id); if (err) return err; } -- cgit v1.2.3 From 2eee6fe8ac2cd21e931c009d475e5a8407d42d76 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Thu, 28 May 2026 18:49:32 -0700 Subject: bpf: Fix dynptr ref counting to scan all call frames When checking whether a referenced dynptr can be overwritten, destroy_if_dynptr_stack_slot only counted sibling dynptrs in the current call frame. If a clone sharing the same virtual ref parent existed in a different frame (e.g., passed to a subprog), it would not be counted, causing the verifier to incorrectly reject the overwrite with "cannot overwrite referenced dynptr". Fix by extracting the counting into dynptr_ref_cnt() which uses bpf_for_each_reg_in_vstate_mask() to scan dynptr stack slots across all call frames. Fixes: 017f5c4ef73c ("bpf: Allow overwriting referenced dynptr when refcnt > 1") Reported-by: Eduard Zingerman Acked-by: Eduard Zingerman Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260529014936.2811085-10-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 52 ++++++++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 23 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index caa455fad877..5d8f2656dbfd 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -786,10 +786,29 @@ static void mark_reg_invalid(const struct bpf_verifier_env *env, struct bpf_reg_ __mark_reg_unknown(env, reg); } +static int dynptr_ref_cnt(struct bpf_verifier_env *env, int v_parent_id) +{ + struct bpf_stack_state *stack; + struct bpf_func_state *state; + struct bpf_reg_state *reg; + int ref_cnt = 0; + + bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, stack, 1 << STACK_DYNPTR, ({ + if (!stack || stack->slot_type[0] != STACK_DYNPTR) + continue; + if (!stack->spilled_ptr.dynptr.first_slot) + continue; + if (stack->spilled_ptr.parent_id == v_parent_id) + ref_cnt++; + })); + + return ref_cnt; +} + static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi) { - int i, err = 0; + int err = 0; /* We always ensure that STACK_DYNPTR is never set partially, * hence just checking for slot_type[0] is enough. This is @@ -803,28 +822,15 @@ static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, if (!state->stack[spi].spilled_ptr.dynptr.first_slot) spi = spi + 1; - if (dynptr_type_referenced(state->stack[spi].spilled_ptr.dynptr.type)) { - int v_parent_id = state->stack[spi].spilled_ptr.parent_id; - int ref_cnt = 0; - - /* - * A referenced dynptr can be overwritten only if there is at - * least one other dynptr sharing the same virtual ref parent, - * ensuring the reference can still be properly released. - */ - for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { - if (state->stack[i].slot_type[0] != STACK_DYNPTR) - continue; - if (!state->stack[i].spilled_ptr.dynptr.first_slot) - continue; - if (state->stack[i].spilled_ptr.parent_id == v_parent_id) - ref_cnt++; - } - - if (ref_cnt <= 1) { - verbose(env, "cannot overwrite referenced dynptr\n"); - return -EINVAL; - } + /* + * A referenced dynptr can be overwritten only if there is at + * least one other dynptr sharing the same virtual ref parent, + * ensuring the reference can still be properly released. + */ + if (dynptr_type_referenced(state->stack[spi].spilled_ptr.dynptr.type) && + dynptr_ref_cnt(env, state->stack[spi].spilled_ptr.parent_id) <= 1) { + verbose(env, "cannot overwrite referenced dynptr\n"); + return -EINVAL; } /* Invalidate the dynptr and any derived slices */ -- cgit v1.2.3 From 9a3c3c49c333760c8944dadacbe114c1884546ef Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Mon, 1 Jun 2026 17:02:42 +0200 Subject: bpf: Reject exclusive maps as inner maps in map-in-map An exclusive map (created with excl_prog_hash) is bound to a single program by hash: check_map_prog_compatibility() refuses to load any program whose digest does not match map->excl_prog_sha. That check only runs for maps a program references directly, i.e. its used_maps. A map reached at runtime through a map-of-maps is never in used_maps, and bpf_map_meta_equal() does not consider excl_prog_sha, so an exclusive map can be inserted into a non-exclusive outer map and then looked up and mutated by an unrelated program, bypassing the exclusivity guarantee. For the signed loader this defeats the metadata map exclusivity check added in the signed loader: the cached map->sha[] is validated against the signed hash while another program on a hostile host rewrites the frozen map's contents through the outer map. Fixes: baefdbdf6812 ("bpf: Implement exclusive map creation") Reported-by: sashiko Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260601150248.394863-2-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- kernel/bpf/map_in_map.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c index 645bd30bc9a9..d2cbab4bdf64 100644 --- a/kernel/bpf/map_in_map.c +++ b/kernel/bpf/map_in_map.c @@ -20,7 +20,8 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd) /* Does not support >1 level map-in-map */ if (inner_map->inner_map_meta) return ERR_PTR(-EINVAL); - + if (inner_map->excl_prog_sha) + return ERR_PTR(-ENOTSUPP); if (!inner_map->ops->map_meta_equal) return ERR_PTR(-ENOTSUPP); @@ -101,6 +102,8 @@ void *bpf_map_fd_get_ptr(struct bpf_map *map, inner_map = __bpf_map_get(f); if (IS_ERR(inner_map)) return inner_map; + if (inner_map->excl_prog_sha) + return ERR_PTR(-ENOTSUPP); inner_map_meta = map->inner_map_meta; if (inner_map_meta->ops->map_meta_equal(inner_map_meta, inner_map)) -- cgit v1.2.3 From c48c3a7e7d5bed644208ed443d63bb6a6f411676 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Mon, 1 Jun 2026 17:02:43 +0200 Subject: bpf: Drop redundant hash_buf from map_get_hash operation bpf_map_get_info_by_fd() is the only caller of the ->map_get_hash and always invokes it with hash_buf == map->sha and hash_buf_size of SHA256_DIGEST_SIZE. array_map_get_hash() in turn lets sha256() write the digest directly into that buffer (map->sha) and then performs a trailing memcpy(), which evaluates to memcpy(map->sha, map->sha, 32): a redundant self-copy. The hash_buf_size argument was never used at all. Simplify this a bit, no functional change. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260601150248.394863-3-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- kernel/bpf/arraymap.c | 6 ++---- kernel/bpf/syscall.c | 8 +++----- 2 files changed, 5 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index dfb2110ab733..e6271a2bf6d6 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -175,14 +175,12 @@ static void *array_map_lookup_elem(struct bpf_map *map, void *key) return array->value + (u64)array->elem_size * (index & array->index_mask); } -static int array_map_get_hash(struct bpf_map *map, u32 hash_buf_size, - void *hash_buf) +static int array_map_get_hash(struct bpf_map *map) { struct bpf_array *array = container_of(map, struct bpf_array, map); sha256(array->value, (u64)array->elem_size * array->map.max_entries, - hash_buf); - memcpy(array->map.sha, hash_buf, sizeof(array->map.sha)); + array->map.sha); return 0; } diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 2aafd2131983..a27fa2b9b405 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -5434,18 +5434,16 @@ static int bpf_map_get_info_by_fd(struct file *file, if (!map->ops->map_get_hash) return -EINVAL; - - if (info.hash_size != SHA256_DIGEST_SIZE) + if (info.hash_size != sizeof(map->sha)) return -EINVAL; - if (!READ_ONCE(map->frozen)) return -EPERM; - err = map->ops->map_get_hash(map, SHA256_DIGEST_SIZE, map->sha); + err = map->ops->map_get_hash(map); if (err != 0) return err; - if (copy_to_user(uhash, map->sha, SHA256_DIGEST_SIZE) != 0) + if (copy_to_user(uhash, map->sha, sizeof(map->sha)) != 0) return -EFAULT; } else if (info.hash_size) { return -EINVAL; -- cgit v1.2.3 From 0fb6c9ed6493b4af01be8bb0a384574eba7df636 Mon Sep 17 00:00:00 2001 From: KP Singh Date: Mon, 1 Jun 2026 17:02:44 +0200 Subject: libbpf: Reject non-exclusive metadata maps in the signed loader The loader verifies map->sha against the metadata hash in its instructions. map->sha is calculated when BPF_OBJ_GET_INFO_BY_FD is called on the frozen map. While the map is frozen, the /signed loader/ must also ensure the map is exclusive, as, without exclusivity (which a hostile host could just omit when loading the loader), another BPF program with map access can mutate the contents afterwards, so the check passes on stale data. With the extra check as part of the signed loader, it now refuses to move on with map->sha validation if the host set it up wrongly. Fixes: fb2b0e290147 ("libbpf: Update light skeleton for signing") Signed-off-by: KP Singh Co-developed-by: Daniel Borkmann Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260601150248.394863-4-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- kernel/bpf/syscall.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index a27fa2b9b405..625a4366fe6d 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1588,6 +1588,13 @@ static int map_create_alloc(union bpf_attr *attr, bpfptr_t uattr, struct bpf_ver err = -EFAULT; goto free_map; } + + /* See libbpf: emit_signature_match() */ + BUILD_BUG_ON(offsetof(struct bpf_map, excl) != SHA256_DIGEST_SIZE); + BUILD_BUG_ON(!__same_type(map->excl, u32)); + BUILD_BUG_ON(offsetof(struct bpf_map, sha) != 0); + BUILD_BUG_ON(!__same_type(map->sha, u8[SHA256_DIGEST_SIZE])); + map->excl = 1; } else if (attr->excl_prog_hash_size) { bpf_log(log, "Invalid excl_prog_hash_size.\n"); err = -EINVAL; -- cgit v1.2.3 From 5ab4bc67d818ba27388d64cb9c52cb0c3bdac254 Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Mon, 1 Jun 2026 20:41:16 -0400 Subject: verifier: parse BTF type tags for function arguments The BTF parsing logic for function arguments goes through the arguments' decl tags, but does not go into their type tags. Add type tag parsing for function arguments. Acked-by: Eduard Zingerman Signed-off-by: Emil Tsalapatis Link: https://lore.kernel.org/r/20260602004120.17087-3-emil@etsalapatis.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/btf.c | 120 +++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 85 insertions(+), 35 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index f429f6f58cb2..6fb3461e3ac2 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7802,6 +7802,84 @@ enum btf_arg_tag { ARG_TAG_ARENA = BIT_ULL(5), }; +static int btf_scan_decl_tags(struct bpf_verifier_env *env, + const struct btf *btf, + const struct btf_type *fn_t, + u32 arg_idx, bool is_global, u32 *tags) +{ + int id = btf_named_start_id(btf, false) - 1; + + /* + * The 'arg:' decl_tag takes precedence over the derivation + * of the register type from the BTF type itself. + */ + while ((id = btf_find_next_decl_tag(btf, fn_t, arg_idx, "arg:", id)) > 0) { + const struct btf_type *tag_t = btf_type_by_id(btf, id); + const char *tag = __btf_name_by_offset(btf, tag_t->name_off) + 4; + + /* disallow arg tags in static subprogs */ + if (!is_global) { + bpf_log(&env->log, + "arg#%d type tag is not supported in static functions\n", + arg_idx); + return -EOPNOTSUPP; + } + + if (strcmp(tag, "ctx") == 0) { + *tags |= ARG_TAG_CTX; + } else if (strcmp(tag, "trusted") == 0) { + *tags |= ARG_TAG_TRUSTED; + } else if (strcmp(tag, "untrusted") == 0) { + *tags |= ARG_TAG_UNTRUSTED; + } else if (strcmp(tag, "nonnull") == 0) { + *tags |= ARG_TAG_NONNULL; + } else if (strcmp(tag, "nullable") == 0) { + *tags |= ARG_TAG_NULLABLE; + } else if (strcmp(tag, "arena") == 0) { + *tags |= ARG_TAG_ARENA; + } else { + bpf_log(&env->log, "arg#%d has unsupported set of tags\n", arg_idx); + return -EOPNOTSUPP; + } + } + if (id != -ENOENT) { + bpf_log(&env->log, "arg#%d type tag fetching failure: %d\n", arg_idx, id); + return id; + } + + return 0; +} + +static int btf_scan_type_tags(struct bpf_verifier_env *env, + const struct btf *btf, u32 type_id, + u32 *tags) +{ + const struct btf_type *t; + + /* Find the first pointer type in the chain. */ + t = btf_type_skip_modifiers(btf, type_id, NULL); + if (!t || !btf_type_is_ptr(t)) + return 0; + + /* We got a pointer, get all associated type tags. */ + t = btf_type_by_id(btf, t->type); + while (t && btf_type_is_type_tag(t)) { + const char *tag = __btf_name_by_offset(btf, t->name_off); + + if (strcmp(tag, "arena") == 0) { + *tags |= ARG_TAG_ARENA; + } else { + bpf_log(&env->log, "function signature member has unsupported type tag '%s'\n", + tag); + return -EOPNOTSUPP; + } + + t = btf_type_by_id(btf, t->type); + } + + return 0; +} + /* Process BTF of a function to produce high-level expectation of function * arguments (like ARG_PTR_TO_CTX, or ARG_PTR_TO_MEM, etc). This information * is cached in subprog info for reuse. @@ -7820,6 +7898,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) struct btf *btf = prog->aux->btf; const struct btf_param *args; const struct btf_type *t, *ref_t, *fn_t; + int err; u32 i, nargs, btf_id; const char *tname; @@ -7903,42 +7982,13 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) */ for (i = 0; i < nargs; i++) { u32 tags = 0; - int id = btf_named_start_id(btf, false) - 1; - - /* 'arg:' decl_tag takes precedence over derivation of - * register type from BTF type itself - */ - while ((id = btf_find_next_decl_tag(btf, fn_t, i, "arg:", id)) > 0) { - const struct btf_type *tag_t = btf_type_by_id(btf, id); - const char *tag = __btf_name_by_offset(btf, tag_t->name_off) + 4; - - /* disallow arg tags in static subprogs */ - if (!is_global) { - bpf_log(log, "arg#%d type tag is not supported in static functions\n", i); - return -EOPNOTSUPP; - } + err = btf_scan_decl_tags(env, btf, fn_t, i, is_global, &tags); + if (err) + return err; - if (strcmp(tag, "ctx") == 0) { - tags |= ARG_TAG_CTX; - } else if (strcmp(tag, "trusted") == 0) { - tags |= ARG_TAG_TRUSTED; - } else if (strcmp(tag, "untrusted") == 0) { - tags |= ARG_TAG_UNTRUSTED; - } else if (strcmp(tag, "nonnull") == 0) { - tags |= ARG_TAG_NONNULL; - } else if (strcmp(tag, "nullable") == 0) { - tags |= ARG_TAG_NULLABLE; - } else if (strcmp(tag, "arena") == 0) { - tags |= ARG_TAG_ARENA; - } else { - bpf_log(log, "arg#%d has unsupported set of tags\n", i); - return -EOPNOTSUPP; - } - } - if (id != -ENOENT) { - bpf_log(log, "arg#%d type tag fetching failure: %d\n", i, id); - return id; - } + err = btf_scan_type_tags(env, btf, args[i].type, &tags); + if (err) + return err; t = btf_type_by_id(btf, args[i].type); while (btf_type_is_modifier(t)) -- cgit v1.2.3 From 3e924e9272c80939677aa6902aced311c85fe48c Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Mon, 1 Jun 2026 20:41:17 -0400 Subject: bpf: Allow subprogs to return arena pointers BPF subprogs currently only return void or scalar values. However, it is also safe to return arena pointers between subprogs in the same BPF program: Arena pointers are guaranteed to be safe for both programs at any point. Expand the verifier to permit returning an arena pointer to the caller. The main subprog is still not allowed to return an arena pointer because arena pointers are internal to the BPF program, and the return values permitted for each main subprog depend on the program type anyway. Acked-by: Eduard Zingerman Signed-off-by: Emil Tsalapatis Link: https://lore.kernel.org/r/20260602004120.17087-4-emil@etsalapatis.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/btf.c | 66 ++++++++++++++++++++++++++++++++++++++------------- kernel/bpf/verifier.c | 4 ++++ 2 files changed, 54 insertions(+), 16 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 6fb3461e3ac2..68921d9172b5 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7858,12 +7858,22 @@ static int btf_scan_type_tags(struct bpf_verifier_env *env, /* Find the first pointer type in the chain. */ t = btf_type_skip_modifiers(btf, type_id, NULL); + + /* + * We currently reject type tags on non-pointer types, + * which neither LLVM nor GCC support anyway. + */ if (!t || !btf_type_is_ptr(t)) return 0; /* We got a pointer, get all associated type tags. */ - t = btf_type_by_id(btf, t->type); - while (t && btf_type_is_type_tag(t)) { + for (t = btf_type_by_id(btf, t->type); t && btf_type_is_modifier(t); + t = btf_type_by_id(btf, t->type)) { + + /* Skip non-type tag modifiers. */ + if (!btf_type_is_type_tag(t)) + continue; + const char *tag = __btf_name_by_offset(btf, t->name_off); if (strcmp(tag, "arena") == 0) { @@ -7873,13 +7883,39 @@ static int btf_scan_type_tags(struct bpf_verifier_env *env, tag); return -EOPNOTSUPP; } - - t = btf_type_by_id(btf, t->type); } return 0; } +/* Check whether the type is a valid return type. */ +static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *btf, + const struct btf_type *t, int subprog) +{ + u32 tags = 0; + int err; + + err = btf_scan_type_tags(env, btf, t->type, &tags); + if (err) + return err; + + t = btf_type_skip_modifiers(btf, t->type, NULL); + + /* + * We allow all subprogs except for the main one to return any kind of arena pointer. + * General arena variables are not allowed, since it makes no sense to return by value + * a variable that's on the heap in the first place. + */ + if (subprog && (tags & ARG_TAG_ARENA) && btf_type_is_ptr(t)) + return 0; + + /* We always accept void or scalars. */ + if (btf_type_is_void(t) || btf_type_is_int(t) || btf_is_any_enum(t)) + return 0; + + return -EOPNOTSUPP; +} + /* Process BTF of a function to produce high-level expectation of function * arguments (like ARG_PTR_TO_CTX, or ARG_PTR_TO_MEM, etc). This information * is cached in subprog info for reuse. @@ -7963,18 +7999,16 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) tname, nargs, MAX_BPF_FUNC_REG_ARGS); return -EINVAL; } - /* check that function is void or returns int, exception cb also requires this */ - t = btf_type_by_id(btf, t->type); - while (btf_type_is_modifier(t)) - t = btf_type_by_id(btf, t->type); - if (!btf_type_is_void(t) && !btf_type_is_int(t) && !btf_is_any_enum(t)) { - if (!is_global) - return -EINVAL; - bpf_log(log, - "Global function %s() return value not void or scalar. " - "Only those are supported.\n", - tname); - return -EINVAL; + + err = btf_validate_return_type(env, btf, t, subprog); + if (err) { + if (is_global) { + bpf_log(log, + "Global function %s() return value not void or scalar. " + "Only those are supported.\n", + tname); + } + return err; } /* Convert BTF function arguments into verifier types. diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5d8f2656dbfd..8ed484cb1a8a 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -16503,6 +16503,10 @@ static int check_global_subprog_return_code(struct bpf_verifier_env *env) if (err) return err; + /* Pointers to arena are safe to pass between subprograms. */ + if (is_arena_reg(env, BPF_REG_0)) + return 0; + if (is_pointer_value(env, BPF_REG_0)) { verbose(env, "R%d leaks addr as return value\n", BPF_REG_0); return -EACCES; -- cgit v1.2.3 From b93c55b4932dd7e32dca8cf34a3443cc87a02906 Mon Sep 17 00:00:00 2001 From: Deepanshu Kartikey Date: Tue, 2 Jun 2026 08:22:49 +0530 Subject: bpf: fix UAF by restoring RCU-delayed inode freeing in bpffs commit 4f375ade6aa9 ("bpf: Avoid RCU context warning when unpinning htab with internal structs") moved inode cleanup from ->free_inode() into ->destroy_inode() to avoid sleeping in RCU context when calling bpf_any_put(). However this removed the RCU delay on freeing the inode itself and the cached symlink body (i_link), both of which can be accessed by RCU pathwalk (pick_link, may_lookup etc.). This causes a use-after-free when a concurrent unlinkat() drops the last inode reference and destroy_inode() frees the inode immediately, while another task is still walking the path in RCU mode and reads inode->i_opflags (offset +2) inside current_time() -> is_mgtime(). KASAN reports: BUG: KASAN: slab-use-after-free in is_mgtime include/linux/fs.h:2313 Read of size 2 at addr ffff8880407e4282 (offset +2 = i_opflags) The rules (per Al Viro): ->destroy_inode() called immediately, can sleep, use for blocking cleanup e.g. bpf_any_put() ->free_inode() called after RCU grace period, use for freeing inode and anything RCU-accessible e.g. i_link Fix: split the two concerns properly: - keep bpf_any_put() in bpf_destroy_inode() since it is blocking and needs to run promptly - introduce bpf_free_inode() to handle kfree(i_link) and free_inode_nonrcu() with proper RCU delay, preventing the UAF Fixes: 4f375ade6aa9 ("bpf: Avoid RCU context warning when unpinning htab with internal structs") Reported-by: syzbot+36e50496c8ac4bcde3f9@syzkaller.appspotmail.com Closes: https://syzkaller.appspot.com/bug?extid=36e50496c8ac4bcde3f9 Suggested-by: Al Viro Link: https://lore.kernel.org/all/20260423043906.GN3518998@ZenIV/ Link: https://lore.kernel.org/all/20260602002607.110866-1-kartikey406@gmail.com/T/ [v1] Signed-off-by: Deepanshu Kartikey Acked-by: Al Viro Link: https://lore.kernel.org/r/20260602025249.113828-1-kartikey406@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/inode.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c index 25c06a011825..188c774a469c 100644 --- a/kernel/bpf/inode.c +++ b/kernel/bpf/inode.c @@ -766,10 +766,18 @@ static void bpf_destroy_inode(struct inode *inode) { enum bpf_type type; - if (S_ISLNK(inode->i_mode)) - kfree(inode->i_link); if (!bpf_inode_type(inode, &type)) bpf_any_put(inode->i_private, type); +} + +/* + * Called after RCU grace period - safe to free inode and anything + * that might be accessed by RCU pathwalk (inode fields, i_link). + */ +static void bpf_free_inode(struct inode *inode) +{ + if (S_ISLNK(inode->i_mode)) + kfree(inode->i_link); free_inode_nonrcu(inode); } @@ -778,6 +786,7 @@ const struct super_operations bpf_super_ops = { .drop_inode = inode_just_drop, .show_options = bpf_show_options, .destroy_inode = bpf_destroy_inode, + .free_inode = bpf_free_inode, }; enum { -- cgit v1.2.3 From eba302268a019275fd6ff452d4ff0b94fef11c76 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 29 May 2026 21:59:51 +0200 Subject: timekeeping: Provide ktime_get_snapshot_id() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ktime_get_snapshot() provides a snapshot of the underlying clocksource counter value and the corresponding CLOCK_MONOTONIC_RAW, CLOCK_REALTIME and CLOCK_BOOTTIME timestamps. There is no usage of CLOCK_REALTIME and CLOCK_BOOTTIME at the same time and CLOCK_BOOTTIME support was just added for the ARM64 KVM tracing mechanism, which needs CLOCK_BOOTTIME and the underlying clocksource counter value. ktime_get_snapshot() is also not suitable for usage with CLOCK_AUX, but that's a prerequisite to support PTP hardware timestamping for CLOCK_AUX steering. As a first step, rename ktime_get_snapshot() to ktime_get_snapshot_id(), which now takes a clockid argument to select the clock which needs to be captured. The result is stored in system_time_snapshot::systime, which will replace the system_time_snapshot::real/boot members once all usage sites have been converted. ktime_get_snapshot() is a simple wrapper which hands in CLOCK_REALTIME as clockid argument for the conversion period. That means CLOCK_REALTIME is now captured twice, but that redunancy is only temporary. As all usage sites of struct system_time_snapshot has to be updated anyway, rename the 'raw' member to 'monoraw' for clarity. No functional change vs. current users of ktime_get_snapshot() Signed-off-by: Thomas Gleixner Tested-by: David Woodhouse Tested-by: Arthur Kiyanovski Reviewed-by: David Woodhouse Reviewed-by: Thomas Weißschuh Reviewed-by: Jacob Keller Link: https://patch.msgid.link/20260529195556.971591633@kernel.org --- kernel/time/timekeeping.c | 89 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 66 insertions(+), 23 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index c493a4010305..0053dc0010f4 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -1183,43 +1183,86 @@ noinstr time64_t __ktime_get_real_seconds(void) } /** - * ktime_get_snapshot - snapshots the realtime/monotonic raw clocks with counter - * @systime_snapshot: pointer to struct receiving the system time snapshot - */ -void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot) -{ - struct timekeeper *tk = &tk_core.timekeeper; + * ktime_get_snapshot_id - Simultaneously snapshot a given clock ID with + * CLOCK_MONOTONIC_RAW and the underlying + * clocksource counter value. + * @clock_id: The clock ID to snapshot + * @systime_snapshot: Pointer to struct receiving the system time snapshot + */ +void ktime_get_snapshot_id(clockid_t clock_id, struct system_time_snapshot *systime_snapshot) +{ + ktime_t base_raw, base_sys, offs_sys, *offs, offs_zero = 0; + u64 nsec_raw, nsec_sys, now; + struct timekeeper *tk; + struct tk_data *tkd; unsigned int seq; - ktime_t base_raw; ktime_t base_real; ktime_t base_boot; - u64 nsec_raw; - u64 nsec_real; - u64 now; - WARN_ON_ONCE(timekeeping_suspended); + /* Invalidate the snapshot for all failure cases */ + systime_snapshot->valid = false; + + if (WARN_ON_ONCE(timekeeping_suspended)) + return; + + switch (clock_id) { + case CLOCK_REALTIME: + tkd = &tk_core; + offs = &tk_core.timekeeper.offs_real; + break; + /* Map RAW to MONOTONIC so the loop below is trivial */ + case CLOCK_MONOTONIC_RAW: + case CLOCK_MONOTONIC: + tkd = &tk_core; + offs = &offs_zero; + break; + case CLOCK_BOOTTIME: + tkd = &tk_core; + offs = &tk_core.timekeeper.offs_boot; + break; + default: + WARN_ON_ONCE(1); + return; + } + + tk = &tkd->timekeeper; do { - seq = read_seqcount_begin(&tk_core.seq); + seq = read_seqcount_begin(&tkd->seq); + now = tk_clock_read(&tk->tkr_mono); systime_snapshot->cs_id = tk->tkr_mono.clock->id; systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq; systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq; - base_real = ktime_add(tk->tkr_mono.base, - tk_core.timekeeper.offs_real); - base_boot = ktime_add(tk->tkr_mono.base, - tk_core.timekeeper.offs_boot); + + base_sys = tk->tkr_mono.base; + offs_sys = *offs; base_raw = tk->tkr_raw.base; - nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, now); - nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, now); - } while (read_seqcount_retry(&tk_core.seq, seq)); + + /* Kept around until the callers are fixed up */ + base_real = ktime_add(base_sys, tk_core.timekeeper.offs_real); + base_boot = ktime_add(base_sys, tk_core.timekeeper.offs_boot); + + nsec_sys = timekeeping_cycles_to_ns(&tk->tkr_mono, now); + nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, now); + } while (read_seqcount_retry(&tkd->seq, seq)); systime_snapshot->cycles = now; - systime_snapshot->real = ktime_add_ns(base_real, nsec_real); - systime_snapshot->boot = ktime_add_ns(base_boot, nsec_real); - systime_snapshot->raw = ktime_add_ns(base_raw, nsec_raw); + systime_snapshot->systime = ktime_add_ns(base_sys, offs_sys + nsec_sys); + systime_snapshot->real = ktime_add_ns(base_real, nsec_sys); + systime_snapshot->boot = ktime_add_ns(base_boot, nsec_sys); + systime_snapshot->monoraw = ktime_add_ns(base_raw, nsec_raw); + + /* + * Special case for PTP. Just transfer the raw time into sys, + * so the call sites can consistently use snap::systime. + */ + if (clock_id == CLOCK_MONOTONIC_RAW) + systime_snapshot->systime = systime_snapshot->monoraw; + /* Tell the consumer that this snapshot is valid */ + systime_snapshot->valid = true; } -EXPORT_SYMBOL_GPL(ktime_get_snapshot); +EXPORT_SYMBOL_GPL(ktime_get_snapshot_id); /* Scale base by mult/div checking for overflow */ static int scale64_check_overflow(u64 mult, u64 div, u64 *base) -- cgit v1.2.3 From ef22786707e3967b539c3b1e6b5c7ea8b408430f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 29 May 2026 21:59:55 +0200 Subject: timekeeping: Use system_time_snapshot::systime/monoraw instead of ::real/raw MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit system_time_snapshot::systime provides the same information as system_time_snapshot::real when the snapshot was taken with ktime_get_snapshot_id(CLOCK_REALTIME). Convert the history interpolation over to use 'systime' and 'monoraw' as 'real/raw' are going away once all users are converted. As a side effect this is the first step to support CLOCK_AUX with get_device_crosstime_stamp() and the history interpolation. Signed-off-by: Thomas Gleixner Tested-by: David Woodhouse Tested-by: Arthur Kiyanovski Reviewed-by: David Woodhouse Reviewed-by: Thomas Weißschuh Reviewed-by: Jacob Keller Link: https://patch.msgid.link/20260529195557.024415766@kernel.org --- kernel/time/timekeeping.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 0053dc0010f4..ccd04addb021 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -1323,7 +1323,7 @@ static int adjust_historical_crosststamp(struct system_time_snapshot *history, * partial_history_cycles / total_history_cycles */ corr_raw = (u64)ktime_to_ns( - ktime_sub(ts->sys_monoraw, history->raw)); + ktime_sub(ts->sys_monoraw, history->monoraw)); ret = scale64_check_overflow(partial_history_cycles, total_history_cycles, &corr_raw); if (ret) @@ -1341,7 +1341,7 @@ static int adjust_historical_crosststamp(struct system_time_snapshot *history, (corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult); } else { corr_real = (u64)ktime_to_ns( - ktime_sub(ts->sys_realtime, history->real)); + ktime_sub(ts->sys_realtime, history->systime)); ret = scale64_check_overflow(partial_history_cycles, total_history_cycles, &corr_real); if (ret) @@ -1350,8 +1350,8 @@ static int adjust_historical_crosststamp(struct system_time_snapshot *history, /* Fixup monotonic raw and real time time values */ if (interp_forward) { - ts->sys_monoraw = ktime_add_ns(history->raw, corr_raw); - ts->sys_realtime = ktime_add_ns(history->real, corr_real); + ts->sys_monoraw = ktime_add_ns(history->monoraw, corr_raw); + ts->sys_realtime = ktime_add_ns(history->systime, corr_real); } else { ts->sys_monoraw = ktime_sub_ns(ts->sys_monoraw, corr_raw); ts->sys_realtime = ktime_sub_ns(ts->sys_realtime, corr_real); -- cgit v1.2.3 From bdaf235913e1f31453c6e0e109d797269f9f0a37 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Thu, 30 Apr 2026 21:50:47 +0000 Subject: locking: mutex: Fix proxy-exec potentially deactivating tasks marked TASK_RUNNING Vineeth found came up with a test driver that could trip up workqueue stalls. After fixing one issue this test found, Vineeth reported the test was still failing. Greatly simplified, a task that tries to take a mutex already owned by another task that is sleeping, can hit a edge case in the mutex_lock_common() case. If the task fails to get the lock, calls into schedule, but gets a spurious wakeup, it will find that it is first waiter, and go into the mutex_optimistic_spin() logic. Though before calling mutex_optimistic_spin(), we clear task blocked_on state, since mutex_optimistic_spin() may call schedule() if need_resched() is set. After mutex_optimistic_spin() fails, we set blocked_on again, restart the main mutex loop, try to take the lock and call into schedule_preempt_disabled(). From there, with proxy-execution, we'll see the task is blocked_on, follow the chain, see the owner is sleeping and dequeue the waiting task from the runqueue. This all sounds fine and reasonable. But what I had missed is that in mutex_optimistic_spin(), not only do we call schedule() but we set TASK_RUNNABLE right before doing so. This is ok for that invocation of schedule(). But when we come back we re-set the blocked_on we had just cleared, but we do not re-set the task state to TASK_INTERRUPTIBLE/UNINTERRUPTIBLE. This means we have a task that is blocked_on & TASK_RUNNABLE, so when the proxy execution code dequeues the task, we are in trouble since future wakeups will be shortcut by the ttwu_state_match() check. Thus, to avoid this, after mutex_optimistic_spin(), set the task state back when we set blocked_on. Many many thanks again to Vineeth for his very useful testing driver that uncovered this long hidden bug, that I hadn't tripped in all my testing! Very impressed with the problems he's uncovered! Reported-by: Vineeth Pillai Signed-off-by: John Stultz Signed-off-by: Peter Zijlstra (Intel) Tested-by: Vineeth Pillai Link: https://patch.msgid.link/20260430215103.2978955-3-jstultz@google.com --- kernel/locking/mutex.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c index 09534628dc01..a93d4c6bee1a 100644 --- a/kernel/locking/mutex.c +++ b/kernel/locking/mutex.c @@ -763,6 +763,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas raw_spin_lock_irqsave(&lock->wait_lock, flags); raw_spin_lock(¤t->blocked_lock); __set_task_blocked_on(current, lock); + set_current_state(state); if (opt_acquired) break; -- cgit v1.2.3 From 7a3a6bfbd62a2ba3e0ef1e92d6b71abb66890825 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Tue, 12 May 2026 02:56:11 +0000 Subject: sched: Rework prev_balance() to avoid stale prev references Historically, the prev value from __schedule() was the rq->curr. This prev value is passed down through numerous functions, and used in the class scheduler implementations. The fact that prev was on_cpu until the end of __schedule(), meant it was stable across the rq lock drops that the class->balance() implementations often do. However, with proxy-exec, the prev passed to functions called by __schedule() is rq->donor, which may not be the same as rq->curr and may not be on_cpu, this makes the prev value potentially unstable across rq lock drops. A recently found issue with proxy-exec, is when we begin doing return migration from try_to_wake_up(), its possible we may be waking up the rq->donor. When we do this, we proxy_resched_idle() to put_prev_set_next() setting the rq->donor to rq->idle, allowing the rq->donor to be return migrated and allowed to run. This however runs into trouble, as on another cpu we might be in the middle of calling __schedule(). Conceptually the rq lock is held for the majority of the time, but in calling prev_balance() its possible the class->balance() handler call may briefly drop the rq lock. This opens a window for try_to_wake_up() to wake and return migrate the rq->donor before the class logic reacquires the rq lock. Unfortunately prev_balance() pass in a prev argument, to which we pass rq->donor. However this prev value can now become stale and incorrect across a rq lock drop. So, to correct this, rework the prev_balance() call so that it does not take a "prev" argument. Signed-off-by: John Stultz Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260512025635.2840817-2-jstultz@google.com --- kernel/sched/core.c | 33 ++++++++++++++++----------------- kernel/sched/deadline.c | 8 +++++++- kernel/sched/idle.c | 2 +- kernel/sched/rt.c | 8 +++++++- kernel/sched/sched.h | 2 +- kernel/sched/stop_task.c | 2 +- 6 files changed, 33 insertions(+), 22 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 3c8bfd697e2c..a9c9b89260cd 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5986,10 +5986,9 @@ static inline void schedule_debug(struct task_struct *prev, bool preempt) schedstat_inc(this_rq()->sched_count); } -static void prev_balance(struct rq *rq, struct task_struct *prev, - struct rq_flags *rf) +static void prev_balance(struct rq *rq, struct rq_flags *rf) { - const struct sched_class *start_class = prev->sched_class; + const struct sched_class *start_class = rq->donor->sched_class; const struct sched_class *class; /* @@ -6001,7 +6000,7 @@ static void prev_balance(struct rq *rq, struct task_struct *prev, * a runnable task of @class priority or higher. */ for_active_class_range(class, start_class, &idle_sched_class) { - if (class->balance && class->balance(rq, prev, rf)) + if (class->balance && class->balance(rq, rf)) break; } } @@ -6010,7 +6009,7 @@ static void prev_balance(struct rq *rq, struct task_struct *prev, * Pick up the highest-prio task: */ static inline struct task_struct * -__pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) +__pick_next_task(struct rq *rq, struct rq_flags *rf) __must_hold(__rq_lockp(rq)) { const struct sched_class *class; @@ -6027,7 +6026,7 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) * higher scheduling class, because otherwise those lose the * opportunity to pull in more work from other CPUs. */ - if (likely(!sched_class_above(prev->sched_class, &fair_sched_class) && + if (likely(!sched_class_above(rq->donor->sched_class, &fair_sched_class) && rq->nr_running == rq->cfs.h_nr_queued)) { p = pick_task_fair(rq, rf); @@ -6038,19 +6037,19 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) if (!p) p = pick_task_idle(rq, rf); - put_prev_set_next_task(rq, prev, p); + put_prev_set_next_task(rq, rq->donor, p); return p; } restart: - prev_balance(rq, prev, rf); + prev_balance(rq, rf); for_each_active_class(class) { p = class->pick_task(rq, rf); if (unlikely(p == RETRY_TASK)) goto restart; if (p) { - put_prev_set_next_task(rq, prev, p); + put_prev_set_next_task(rq, rq->donor, p); return p; } } @@ -6102,7 +6101,7 @@ extern void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_f static void queue_core_balance(struct rq *rq); static struct task_struct * -pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) +pick_next_task(struct rq *rq, struct rq_flags *rf) __must_hold(__rq_lockp(rq)) { struct task_struct *next, *p, *max; @@ -6115,7 +6114,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) bool need_sync; if (!sched_core_enabled(rq)) - return __pick_next_task(rq, prev, rf); + return __pick_next_task(rq, rf); cpu = cpu_of(rq); @@ -6128,7 +6127,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) */ rq->core_pick = NULL; rq->core_dl_server = NULL; - return __pick_next_task(rq, prev, rf); + return __pick_next_task(rq, rf); } /* @@ -6152,7 +6151,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) goto out_set_next; } - prev_balance(rq, prev, rf); + prev_balance(rq, rf); smt_mask = cpu_smt_mask(cpu); need_sync = !!rq->core->core_cookie; @@ -6334,7 +6333,7 @@ restart_multi: } out_set_next: - put_prev_set_next_task(rq, prev, next); + put_prev_set_next_task(rq, rq->donor, next); if (rq->core->core_forceidle_count && next == rq->idle) queue_core_balance(rq); @@ -6557,10 +6556,10 @@ static inline void sched_core_cpu_deactivate(unsigned int cpu) {} static inline void sched_core_cpu_dying(unsigned int cpu) {} static struct task_struct * -pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) +pick_next_task(struct rq *rq, struct rq_flags *rf) __must_hold(__rq_lockp(rq)) { - return __pick_next_task(rq, prev, rf); + return __pick_next_task(rq, rf); } #endif /* !CONFIG_SCHED_CORE */ @@ -7108,7 +7107,7 @@ static void __sched notrace __schedule(int sched_mode) pick_again: assert_balance_callbacks_empty(rq); - next = pick_next_task(rq, rq->donor, &rf); + next = pick_next_task(rq, &rf); rq->next_class = next->sched_class; if (sched_proxy_exec()) { struct task_struct *prev_donor = rq->donor; diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index f9e62ed08d77..6ef5a808e13e 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -2698,8 +2698,14 @@ static void check_preempt_equal_dl(struct rq *rq, struct task_struct *p) resched_curr(rq); } -static int balance_dl(struct rq *rq, struct task_struct *p, struct rq_flags *rf) +static int balance_dl(struct rq *rq, struct rq_flags *rf) { + /* + * Note, rq->donor may change during rq lock drops, + * so don't re-use prev across lock drops + */ + struct task_struct *p = rq->donor; + if (!on_dl_rq(&p->dl) && need_pull_dl_task(rq, p)) { /* * This is OK, because current is on_cpu, which avoids it being diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index a83be0c834dd..ff39120d723a 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -462,7 +462,7 @@ select_task_rq_idle(struct task_struct *p, int cpu, int flags) } static int -balance_idle(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) +balance_idle(struct rq *rq, struct rq_flags *rf) { return WARN_ON_ONCE(1); } diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index e6ea728f519e..e474c31d8fe6 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -1596,8 +1596,14 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p) resched_curr(rq); } -static int balance_rt(struct rq *rq, struct task_struct *p, struct rq_flags *rf) +static int balance_rt(struct rq *rq, struct rq_flags *rf) { + /* + * Note, rq->donor may change during rq lock drops, + * so don't re-use p across lock drops + */ + struct task_struct *p = rq->donor; + if (!on_rt_rq(&p->rt) && need_pull_rt_task(rq, p)) { /* * This is OK, because current is on_cpu, which avoids it being diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 332ecf8930b4..ef715f2acbaa 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2587,7 +2587,7 @@ struct sched_class { /* * schedule/pick_next_task/prev_balance: rq->lock */ - int (*balance)(struct rq *rq, struct task_struct *prev, struct rq_flags *rf); + int (*balance)(struct rq *rq, struct rq_flags *rf); /* * schedule/pick_next_task: rq->lock diff --git a/kernel/sched/stop_task.c b/kernel/sched/stop_task.c index f95798baddeb..c909ca0d8c87 100644 --- a/kernel/sched/stop_task.c +++ b/kernel/sched/stop_task.c @@ -16,7 +16,7 @@ select_task_rq_stop(struct task_struct *p, int cpu, int flags) } static int -balance_stop(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) +balance_stop(struct rq *rq, struct rq_flags *rf) { return sched_stop_runnable(rq); } -- cgit v1.2.3 From 96a6988fb595ab1d77f60b33ea392b2e15b68605 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Tue, 12 May 2026 02:56:12 +0000 Subject: sched: deadline: Add some helper variables to cleanup deadline logic As part of an improvement to handling pushable deadline tasks, Peter suggested this cleanup[1], to use helper values for dl_entity and dl_rq in the enqueue_task_dl() and put_prev_task_dl() functions. There should be no functional change from this patch. To make sure this cleanup change doesn't obscure later logic changes, I've split it into its own patch. [1]: https://lore.kernel.org/lkml/20260304095123.GP606826@noisy.programming.kicks-ass.net/ Suggested-by: Peter Zijlstra Signed-off-by: John Stultz Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260512025635.2840817-3-jstultz@google.com --- kernel/sched/deadline.c | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 6ef5a808e13e..0b7ac4c12797 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -2484,7 +2484,10 @@ static void dequeue_dl_entity(struct sched_dl_entity *dl_se, int flags) static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags) { - if (is_dl_boosted(&p->dl)) { + struct sched_dl_entity *dl_se = &p->dl; + struct dl_rq *dl_rq = &rq->dl; + + if (is_dl_boosted(dl_se)) { /* * Because of delays in the detection of the overrun of a * thread's runtime, it might be the case that a thread @@ -2497,14 +2500,14 @@ static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags) * * In this case, the boost overrides the throttle. */ - if (p->dl.dl_throttled) { + if (dl_se->dl_throttled) { /* * The replenish timer needs to be canceled. No * problem if it fires concurrently: boosted threads * are ignored in dl_task_timer(). */ - cancel_replenish_timer(&p->dl); - p->dl.dl_throttled = 0; + cancel_replenish_timer(dl_se); + dl_se->dl_throttled = 0; } } else if (!dl_prio(p->normal_prio)) { /* @@ -2516,7 +2519,7 @@ static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags) * being boosted again with no means to replenish the runtime and clear * the throttle. */ - p->dl.dl_throttled = 0; + dl_se->dl_throttled = 0; if (!(flags & ENQUEUE_REPLENISH)) printk_deferred_once("sched: DL de-boosted task PID %d: REPLENISH flag missing\n", task_pid_nr(p)); @@ -2525,20 +2528,20 @@ static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags) } check_schedstat_required(); - update_stats_wait_start_dl(dl_rq_of_se(&p->dl), &p->dl); + update_stats_wait_start_dl(dl_rq, dl_se); if (p->on_rq == TASK_ON_RQ_MIGRATING) flags |= ENQUEUE_MIGRATING; - enqueue_dl_entity(&p->dl, flags); + enqueue_dl_entity(dl_se, flags); - if (dl_server(&p->dl)) + if (dl_server(dl_se)) return; if (task_is_blocked(p)) return; - if (!task_current(rq, p) && !p->dl.dl_throttled && p->nr_cpus_allowed > 1) + if (!task_current(rq, p) && !dl_se->dl_throttled && p->nr_cpus_allowed > 1) enqueue_pushable_dl_task(rq, p); } @@ -2835,7 +2838,7 @@ static void put_prev_task_dl(struct rq *rq, struct task_struct *p, struct task_s struct sched_dl_entity *dl_se = &p->dl; struct dl_rq *dl_rq = &rq->dl; - if (on_dl_rq(&p->dl)) + if (on_dl_rq(dl_se)) update_stats_wait_start_dl(dl_rq, dl_se); update_curr_dl(rq); @@ -2845,7 +2848,7 @@ static void put_prev_task_dl(struct rq *rq, struct task_struct *p, struct task_s if (task_is_blocked(p)) return; - if (on_dl_rq(&p->dl) && p->nr_cpus_allowed > 1) + if (on_dl_rq(dl_se) && p->nr_cpus_allowed > 1) enqueue_pushable_dl_task(rq, p); } -- cgit v1.2.3 From cd8e62c85861bcfbbefedce11a6f8eb00c774312 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Tue, 12 May 2026 02:56:13 +0000 Subject: sched: deadline: Add dl_rq->curr pointer to address issues with Proxy Exec The DL scheduler keeps the current task in the rbtree, since the deadline value isn't usually chagned while the task is runnable. This results in set_next_task() and put_prev_task() being simpler, but unfortunately this causes complexity elsewhere. Specifically when update_curr_dl() updates the deadline, it has to dequeue and then enqueue the task. From put_prev_task_dl(), we first call update_curr_dl(), and then call enqueue_pushable_dl_task(). However, with Proxy Exec this goes awry. Since when a mutex is released, we might wake the waiting rq->donor. This will cause put_prev_task() to be called on the donor to take it off the cpu for return migration. At that point, from put_prev_task_dl() the update_curr_dl() logic will dequeue & enqueue the task, and the enqueue function will call enqueue_pushable_dl_task() (since the task_current() check won't prevent it). Then back up the callstack in put_prev_task_dl() we'll end up calling enqueue_pushable_dl_task() again, tripping the !RB_EMPTY_NODE(&p->pushable_dl_tasks) warning. So to avoid this, use Peter's suggested[1] approach, and add a dl_rq->curr pointer that is set/cleared from set_next_task()/ put_prev_task(), which effectively tracks the rq->donor. We can then use this to avoid adding the active donor to the pushable list from enqueue_task_dl(). [1]: https://lore.kernel.org/lkml/20260304095123.GP606826@noisy.programming.kicks-ass.net/ Suggested-by: Peter Zijlstra Signed-off-by: John Stultz Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260512025635.2840817-4-jstultz@google.com --- kernel/sched/deadline.c | 13 +++++++++++++ kernel/sched/sched.h | 1 + 2 files changed, 14 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 0b7ac4c12797..4754dbe4232d 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -2541,6 +2541,9 @@ static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags) if (task_is_blocked(p)) return; + if (dl_rq->curr == dl_se) + return; + if (!task_current(rq, p) && !dl_se->dl_throttled && p->nr_cpus_allowed > 1) enqueue_pushable_dl_task(rq, p); } @@ -2763,6 +2766,10 @@ static void start_hrtick_dl(struct rq *rq, struct sched_dl_entity *dl_se) } #endif /* !CONFIG_SCHED_HRTICK */ +/* + * DL keeps current in tree, because ->deadline is not typically changed while + * a task is runnable. + */ static void set_next_task_dl(struct rq *rq, struct task_struct *p, bool first) { struct sched_dl_entity *dl_se = &p->dl; @@ -2775,6 +2782,9 @@ static void set_next_task_dl(struct rq *rq, struct task_struct *p, bool first) /* You can't push away the running task */ dequeue_pushable_dl_task(rq, p); + WARN_ON_ONCE(dl_rq->curr); + dl_rq->curr = dl_se; + if (!first) return; @@ -2845,6 +2855,9 @@ static void put_prev_task_dl(struct rq *rq, struct task_struct *p, struct task_s update_dl_rq_load_avg(rq_clock_pelt(rq), rq, 1); + WARN_ON_ONCE(dl_rq->curr != dl_se); + dl_rq->curr = NULL; + if (task_is_blocked(p)) return; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index ef715f2acbaa..b3aff26dbb13 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -893,6 +893,7 @@ struct dl_rq { bool overloaded; + struct sched_dl_entity *curr; /* * Tasks on this rq that can be pushed away. They are kept in * an rb-tree, ordered by tasks' deadlines, with caching -- cgit v1.2.3 From f0c1ecde6447079505f1d4557d30401136218ae0 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Tue, 12 May 2026 02:56:14 +0000 Subject: sched: Rework block_task so it can be directly called Pull most of the logic out of try_to_block_task() and put it into block_task() directly, so that we can call block_task() and not have to worry about the failing cases in try_to_block_task() Suggested-by: Peter Zijlstra Signed-off-by: John Stultz Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260512025635.2840817-5-jstultz@google.com --- kernel/sched/core.c | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index a9c9b89260cd..2f1e85d09b94 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2236,8 +2236,29 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags) dequeue_task(rq, p, flags); } -static void block_task(struct rq *rq, struct task_struct *p, int flags) +static void block_task(struct rq *rq, struct task_struct *p, unsigned long task_state) { + int flags = DEQUEUE_NOCLOCK; + + p->sched_contributes_to_load = + (task_state & TASK_UNINTERRUPTIBLE) && + !(task_state & TASK_NOLOAD) && + !(task_state & TASK_FROZEN); + + if (unlikely(is_special_task_state(task_state))) + flags |= DEQUEUE_SPECIAL; + + /* + * __schedule() ttwu() + * prev_state = prev->state; if (p->on_rq && ...) + * if (prev_state) goto out; + * p->on_rq = 0; smp_acquire__after_ctrl_dep(); + * p->state = TASK_WAKING + * + * Where __schedule() and ttwu() have matching control dependencies. + * + * After this, schedule() must not care about p->state any more. + */ if (dequeue_task(rq, p, DEQUEUE_SLEEP | flags)) __block_task(rq, p); } @@ -6587,7 +6608,6 @@ static bool try_to_block_task(struct rq *rq, struct task_struct *p, unsigned long *task_state_p, bool should_block) { unsigned long task_state = *task_state_p; - int flags = DEQUEUE_NOCLOCK; if (signal_pending_state(task_state, p)) { WRITE_ONCE(p->__state, TASK_RUNNING); @@ -6607,26 +6627,7 @@ static bool try_to_block_task(struct rq *rq, struct task_struct *p, if (!should_block) return false; - p->sched_contributes_to_load = - (task_state & TASK_UNINTERRUPTIBLE) && - !(task_state & TASK_NOLOAD) && - !(task_state & TASK_FROZEN); - - if (unlikely(is_special_task_state(task_state))) - flags |= DEQUEUE_SPECIAL; - - /* - * __schedule() ttwu() - * prev_state = prev->state; if (p->on_rq && ...) - * if (prev_state) goto out; - * p->on_rq = 0; smp_acquire__after_ctrl_dep(); - * p->state = TASK_WAKING - * - * Where __schedule() and ttwu() have matching control dependencies. - * - * After this, schedule() must not care about p->state any more. - */ - block_task(rq, p, flags); + block_task(rq, p, task_state); return true; } -- cgit v1.2.3 From f13beb010e4ab0735c9e46802cbcc820a8bd6467 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Tue, 12 May 2026 02:56:15 +0000 Subject: sched: Have try_to_wake_up() handle return-migration for PROXY_WAKING case This patch adds logic so try_to_wake_up() will notice if we are waking a task where blocked_on == PROXY_WAKING, and if necessary dequeue the task so the wakeup will naturally return-migrate the donor task back to a cpu it can run on. This helps performance as we do the dequeue and wakeup under the locks normally taken in the try_to_wake_up() and avoids having to do proxy_force_return() from __schedule(), which has to re-take similar locks and then force a pick again loop. This was split out from the larger proxy patch, and significantly reworked. Credits for the original patch go to: Peter Zijlstra (Intel) Juri Lelli Valentin Schneider Connor O'Brien Signed-off-by: John Stultz Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260512025635.2840817-6-jstultz@google.com --- kernel/sched/core.c | 195 ++++++++++++++++++++++++++-------------------------- 1 file changed, 96 insertions(+), 99 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 2f1e85d09b94..3f71dd9c1063 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3735,6 +3735,53 @@ void update_rq_avg_idle(struct rq *rq) rq->idle_stamp = 0; } +#ifdef CONFIG_SCHED_PROXY_EXEC +static void zap_balance_callbacks(struct rq *rq); + +static inline void proxy_reset_donor(struct rq *rq) +{ + WARN_ON_ONCE(rq->donor == rq->curr); + + put_prev_set_next_task(rq, rq->donor, rq->curr); + rq_set_donor(rq, rq->curr); + zap_balance_callbacks(rq); + resched_curr(rq); +} + +/* + * Checks to see if task p has been proxy-migrated to another rq + * and needs to be returned. If so, we deactivate the task here + * so that it can be properly woken up on the p->wake_cpu + * (or whichever cpu select_task_rq() picks at the bottom of + * try_to_wake_up() + */ +static inline bool proxy_needs_return(struct rq *rq, struct task_struct *p) +{ + if (!task_is_blocked(p)) + return false; + + scoped_guard(raw_spinlock, &p->blocked_lock) { + /* Task is waking up; clear any blocked_on relationship */ + __clear_task_blocked_on(p, NULL); + + /* If already current, don't need to return migrate */ + if (task_current(rq, p)) + return false; + + /* If we're return migrating the rq->donor, switch it out for idle */ + if (task_current_donor(rq, p)) + proxy_reset_donor(rq); + } + block_task(rq, p, TASK_WAKING); + return true; +} +#else /* !CONFIG_SCHED_PROXY_EXEC */ +static inline bool proxy_needs_return(struct rq *rq, struct task_struct *p) +{ + return false; +} +#endif /* CONFIG_SCHED_PROXY_EXEC */ + static void ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags, struct rq_flags *rf) @@ -3799,28 +3846,26 @@ ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags, */ static int ttwu_runnable(struct task_struct *p, int wake_flags) { - struct rq_flags rf; - struct rq *rq; - int ret = 0; + ACQUIRE(__task_rq_lock, guard)(p); + struct rq *rq = guard.rq; - rq = __task_rq_lock(p, &rf); - if (task_on_rq_queued(p)) { - update_rq_clock(rq); - if (p->se.sched_delayed) - enqueue_task(rq, p, ENQUEUE_NOCLOCK | ENQUEUE_DELAYED); - if (!task_on_cpu(rq, p)) { - /* - * When on_rq && !on_cpu the task is preempted, see if - * it should preempt the task that is current now. - */ - wakeup_preempt(rq, p, wake_flags); - } - ttwu_do_wakeup(p); - ret = 1; - } - __task_rq_unlock(rq, p, &rf); + if (!task_on_rq_queued(p)) + return 0; - return ret; + update_rq_clock(rq); + if (p->se.sched_delayed) + enqueue_task(rq, p, ENQUEUE_NOCLOCK | ENQUEUE_DELAYED); + if (proxy_needs_return(rq, p)) + return 0; + if (!task_on_cpu(rq, p)) { + /* + * When on_rq && !on_cpu the task is preempted, see if + * it should preempt the task that is current now. + */ + wakeup_preempt(rq, p, wake_flags); + } + ttwu_do_wakeup(p); + return 1; } void sched_ttwu_pending(void *arg) @@ -4207,6 +4252,8 @@ int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) * it disabling IRQs (this allows not taking ->pi_lock). */ WARN_ON_ONCE(p->se.sched_delayed); + /* If p is current, we know we can run here, so clear blocked_on */ + clear_task_blocked_on(p, NULL); if (!ttwu_state_match(p, state, &success)) goto out; @@ -4223,6 +4270,7 @@ int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) */ scoped_guard (raw_spinlock_irqsave, &p->pi_lock) { smp_mb__after_spinlock(); + if (!ttwu_state_match(p, state, &success)) break; @@ -4287,6 +4335,14 @@ int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) */ WRITE_ONCE(p->__state, TASK_WAKING); + /* + * We never clear the blocked_on relation on proxy_deactivate. + * If we don't clear it here, we have TASK_RUNNING + p->blocked_on + * when waking up. Since this is a fully blocked, off CPU task + * waking up, it should be safe to clear the blocked_on relation. + */ + if (task_is_blocked(p)) + clear_task_blocked_on(p, NULL); /* * If the owning (remote) CPU is still in the middle of schedule() with * this task as prev, considering queueing p on the remote CPUs wake_list @@ -4331,6 +4387,16 @@ int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) wake_flags |= WF_MIGRATED; psi_ttwu_dequeue(p); set_task_cpu(p, cpu); + } else if (cpu != p->wake_cpu) { + /* + * If we were proxy-migrated to cpu, then + * select_task_rq() picks cpu instead of wake_cpu + * to return to, we won't call set_task_cpu(), + * leaving a stale wake_cpu pointing to where we + * proxy-migrated from. So just fixup wake_cpu here + * if its not correct + */ + p->wake_cpu = cpu; } ttwu_queue(p, cpu, wake_flags); @@ -6612,7 +6678,7 @@ static bool try_to_block_task(struct rq *rq, struct task_struct *p, if (signal_pending_state(task_state, p)) { WRITE_ONCE(p->__state, TASK_RUNNING); *task_state_p = TASK_RUNNING; - set_task_blocked_on_waking(p, NULL); + clear_task_blocked_on(p, NULL); return false; } @@ -6656,13 +6722,11 @@ static inline struct task_struct *proxy_resched_idle(struct rq *rq) return rq->idle; } -static bool proxy_deactivate(struct rq *rq, struct task_struct *donor) +static void proxy_deactivate(struct rq *rq, struct task_struct *donor) { unsigned long state = READ_ONCE(donor->__state); - /* Don't deactivate if the state has been changed to TASK_RUNNING */ - if (state == TASK_RUNNING) - return false; + WARN_ON_ONCE(state == TASK_RUNNING); /* * Because we got donor from pick_next_task(), it is *crucial* * that we call proxy_resched_idle() before we deactivate it. @@ -6673,7 +6737,7 @@ static bool proxy_deactivate(struct rq *rq, struct task_struct *donor) * need to be changed from next *before* we deactivate. */ proxy_resched_idle(rq); - return try_to_block_task(rq, donor, &state, true); + block_task(rq, donor, state); } static inline void proxy_release_rq_lock(struct rq *rq, struct rq_flags *rf) @@ -6747,71 +6811,6 @@ static void proxy_migrate_task(struct rq *rq, struct rq_flags *rf, proxy_reacquire_rq_lock(rq, rf); } -static void proxy_force_return(struct rq *rq, struct rq_flags *rf, - struct task_struct *p) - __must_hold(__rq_lockp(rq)) -{ - struct rq *task_rq, *target_rq = NULL; - int cpu, wake_flag = WF_TTWU; - - lockdep_assert_rq_held(rq); - WARN_ON(p == rq->curr); - - if (p == rq->donor) - proxy_resched_idle(rq); - - proxy_release_rq_lock(rq, rf); - /* - * We drop the rq lock, and re-grab task_rq_lock to get - * the pi_lock (needed for select_task_rq) as well. - */ - scoped_guard (task_rq_lock, p) { - task_rq = scope.rq; - - /* - * Since we let go of the rq lock, the task may have been - * woken or migrated to another rq before we got the - * task_rq_lock. So re-check we're on the same RQ. If - * not, the task has already been migrated and that CPU - * will handle any futher migrations. - */ - if (task_rq != rq) - break; - - /* - * Similarly, if we've been dequeued, someone else will - * wake us - */ - if (!task_on_rq_queued(p)) - break; - - /* - * Since we should only be calling here from __schedule() - * -> find_proxy_task(), no one else should have - * assigned current out from under us. But check and warn - * if we see this, then bail. - */ - if (task_current(task_rq, p) || task_on_cpu(task_rq, p)) { - WARN_ONCE(1, "%s rq: %i current/on_cpu task %s %d on_cpu: %i\n", - __func__, cpu_of(task_rq), - p->comm, p->pid, p->on_cpu); - break; - } - - update_rq_clock(task_rq); - deactivate_task(task_rq, p, DEQUEUE_NOCLOCK); - cpu = select_task_rq(p, p->wake_cpu, &wake_flag); - set_task_cpu(p, cpu); - target_rq = cpu_rq(cpu); - clear_task_blocked_on(p, NULL); - } - - if (target_rq) - attach_one_task(target_rq, p); - - proxy_reacquire_rq_lock(rq, rf); -} - /* * Find runnable lock owner to proxy for mutex blocked donor * @@ -6847,7 +6846,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) clear_task_blocked_on(p, PROXY_WAKING); return p; } - goto force_return; + goto deactivate; } /* @@ -6882,7 +6881,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) __clear_task_blocked_on(p, NULL); return p; } - goto force_return; + goto deactivate; } if (!READ_ONCE(owner->on_rq) || owner->se.sched_delayed) { @@ -6961,12 +6960,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) return owner; deactivate: - if (proxy_deactivate(rq, donor)) - return NULL; - /* If deactivate fails, force return */ - p = donor; -force_return: - proxy_force_return(rq, rf, p); + proxy_deactivate(rq, p); return NULL; migrate_task: proxy_migrate_task(rq, rf, p, owner_cpu); @@ -7113,6 +7107,9 @@ pick_again: if (sched_proxy_exec()) { struct task_struct *prev_donor = rq->donor; + if (!prev_state && prev->blocked_on) + clear_task_blocked_on(prev, NULL); + rq_set_donor(rq, next); if (unlikely(next->blocked_on)) { next = find_proxy_task(rq, next, &rf); -- cgit v1.2.3 From 4c2a20413d7fb3fc3dd7adf233a4f82bb203fb58 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Tue, 12 May 2026 02:56:16 +0000 Subject: sched: Add is_blocked task flag Add a new is_blocked flag to the task struct. This flag is set by try_to_block_task() and cleared by ttwu_do_wakeup() and tracks if the task is blocked. Traditionally this would mirror !p->on_rq, however due things like DELAY_DEQUEUE and PROXY_EXEC, this can diverge, so its useful to manage separately. Additionally with this, we might be able to get rid of the p->se.sched_delayed (ab)use in the core code (eventually). Taken whole cloth from Peter's email: https://lore.kernel.org/lkml/20260501132143.GC1026330@noisy.programming.kicks-ass.net/ With a few additional p->is_blocked = 0 in a few cases where we return current if blocked_on gets zeroed or there is no owner. This may hint that these current special cases might be dropped eventually. This change also helps resolve wait-queue stalls seen with proxy-execution. See previous patch attempts for details: https://lore.kernel.org/lkml/20260430215103.2978955-2-jstultz@google.com/ Reported-by: Vineeth Pillai Suggested-by: Peter Zijlstra Signed-off-by: John Stultz Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260512025635.2840817-7-jstultz@google.com --- kernel/sched/core.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 3f71dd9c1063..c7552869d5c4 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -624,6 +624,12 @@ int task_llc(const struct task_struct *p) * [ The astute reader will observe that it is possible for two tasks on one * CPU to have ->on_cpu = 1 at the same time. ] * + * p->is_blocked <- { 0, 1 }: + * + * is set by try_to_block_task() and cleared by ttwu_do_wakeup() and tracks + * if the task is blocked. Traditionally this would mirror p->on_rq, however + * due things like DELAY_DEQUEUE and PROXY_EXEC, this can diverge. + * * task_cpu(p): is changed by set_task_cpu(), the rules are: * * - Don't call set_task_cpu() on a blocked task: @@ -3719,6 +3725,7 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags) */ static inline void ttwu_do_wakeup(struct task_struct *p) { + p->is_blocked = 0; WRITE_ONCE(p->__state, TASK_RUNNING); trace_sched_wakeup(p); } @@ -4252,6 +4259,7 @@ int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) * it disabling IRQs (this allows not taking ->pi_lock). */ WARN_ON_ONCE(p->se.sched_delayed); + WARN_ON_ONCE(p->is_blocked); /* If p is current, we know we can run here, so clear blocked_on */ clear_task_blocked_on(p, NULL); if (!ttwu_state_match(p, state, &success)) @@ -4563,6 +4571,7 @@ static void __sched_fork(u64 clone_flags, struct task_struct *p) /* A delayed task cannot be in clone(). */ WARN_ON_ONCE(p->se.sched_delayed); + WARN_ON_ONCE(p->is_blocked); #ifdef CONFIG_FAIR_GROUP_SCHED p->se.cfs_rq = NULL; @@ -6676,6 +6685,7 @@ static bool try_to_block_task(struct rq *rq, struct task_struct *p, unsigned long task_state = *task_state_p; if (signal_pending_state(task_state, p)) { + p->is_blocked = 0; WRITE_ONCE(p->__state, TASK_RUNNING); *task_state_p = TASK_RUNNING; clear_task_blocked_on(p, NULL); @@ -6683,6 +6693,8 @@ static bool try_to_block_task(struct rq *rq, struct task_struct *p, return false; } + p->is_blocked = 1; + /* * We check should_block after signal_pending because we * will want to wake the task in that case. But if @@ -6843,6 +6855,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) /* if its PROXY_WAKING, do return migration or run if current */ if (mutex == PROXY_WAKING) { if (task_current(rq, p)) { + p->is_blocked = 0; clear_task_blocked_on(p, PROXY_WAKING); return p; } @@ -6878,6 +6891,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) * just run on this rq), or return-migrate the task. */ if (task_current(rq, p)) { + p->is_blocked = 0; __clear_task_blocked_on(p, NULL); return p; } @@ -7111,7 +7125,7 @@ pick_again: clear_task_blocked_on(prev, NULL); rq_set_donor(rq, next); - if (unlikely(next->blocked_on)) { + if (unlikely(next->is_blocked && next->blocked_on)) { next = find_proxy_task(rq, next, &rf); if (!next) { zap_balance_callbacks(rq); -- cgit v1.2.3 From 1628b25248d0742b2ce9c7cfa59cd183e35f37e1 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 12 May 2026 02:56:17 +0000 Subject: sched: Add blocked_donor link to task for smarter mutex handoffs Add link to the task this task is proxying for, and use it so the mutex owner can do an intelligent hand-off of the mutex to the task that the owner is running on behalf. [jstultz: This patch was split out from larger proxy patch] Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Juri Lelli Signed-off-by: Valentin Schneider Signed-off-by: Connor O'Brien Signed-off-by: John Stultz Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260512025635.2840817-8-jstultz@google.com --- kernel/fork.c | 1 + kernel/locking/mutex.c | 60 ++++++++++++++++++++++++++++++++++++++++++++------ kernel/sched/core.c | 14 +++++++++++- 3 files changed, 67 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/fork.c b/kernel/fork.c index a679b2448234..6fcca1db0af3 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -2224,6 +2224,7 @@ __latent_entropy struct task_struct *copy_process( lockdep_init_task(p); p->blocked_on = NULL; /* not blocked yet */ + p->blocked_donor = NULL; /* nobody is boosting p yet */ #ifdef CONFIG_BCACHE p->sequential_io = 0; diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c index a93d4c6bee1a..28677165785f 100644 --- a/kernel/locking/mutex.c +++ b/kernel/locking/mutex.c @@ -981,15 +981,22 @@ EXPORT_SYMBOL_GPL(ww_mutex_lock_interruptible); static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip) __releases(lock) { - struct task_struct *next = NULL; + struct task_struct *donor, *next = NULL; struct mutex_waiter *waiter; - DEFINE_WAKE_Q(wake_q); unsigned long owner; unsigned long flags; mutex_release(&lock->dep_map, ip); __release(lock); + /* + * Ensures the proxy donor stack is stable across unlock and handoff. + * Specifically, it avoids the case where current->blocked_donor is + * NULL when it is inspected while doing the unlock, but a preemption + * before taking the wake_lock would make it set and a hand-off is + * missed. + */ + guard(preempt)(); /* * Release the lock before (potentially) taking the spinlock such that * other contenders can get on with things ASAP. @@ -1002,6 +1009,12 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne MUTEX_WARN_ON(__owner_task(owner) != current); MUTEX_WARN_ON(owner & MUTEX_FLAG_PICKUP); + if (sched_proxy_exec() && current->blocked_donor) { + /* force handoff if we have a blocked_donor */ + owner = MUTEX_FLAG_HANDOFF; + break; + } + if (owner & MUTEX_FLAG_HANDOFF) break; @@ -1014,20 +1027,53 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne } raw_spin_lock_irqsave(&lock->wait_lock, flags); + raw_spin_lock(¤t->blocked_lock); debug_mutex_unlock(lock); + + if (sched_proxy_exec()) { + /* + * If we have a task boosting current, and that task was boosting + * current through this lock, hand the lock to that task, as that + * is the highest waiter, as selected by the scheduling function. + */ + donor = current->blocked_donor; + if (donor) { + struct mutex *next_lock; + + raw_spin_lock_nested(&donor->blocked_lock, SINGLE_DEPTH_NESTING); + next_lock = __get_task_blocked_on(donor); + if (next_lock == lock) { + next = get_task_struct(donor); + __set_task_blocked_on_waking(donor, next_lock); + current->blocked_donor = NULL; + } + raw_spin_unlock(&donor->blocked_lock); + } + } + + /* + * Failing that, pick first on the wait list. + */ waiter = lock->first_waiter; - if (waiter) { - next = waiter->task; + if (!next && waiter) { + next = get_task_struct(waiter->task); + raw_spin_lock_nested(&next->blocked_lock, SINGLE_DEPTH_NESTING); debug_mutex_wake_waiter(lock, waiter); - set_task_blocked_on_waking(next, lock); - wake_q_add(&wake_q, next); + __set_task_blocked_on_waking(next, lock); + raw_spin_unlock(&next->blocked_lock); + } if (owner & MUTEX_FLAG_HANDOFF) __mutex_handoff(lock, next); - raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q); + raw_spin_unlock(¤t->blocked_lock); + raw_spin_unlock_irqrestore(&lock->wait_lock, flags); + if (next) { + wake_up_process(next); + put_task_struct(next); + } } #ifndef CONFIG_DEBUG_LOCK_ALLOC diff --git a/kernel/sched/core.c b/kernel/sched/core.c index c7552869d5c4..4c6ceff3855e 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6827,7 +6827,17 @@ static void proxy_migrate_task(struct rq *rq, struct rq_flags *rf, * Find runnable lock owner to proxy for mutex blocked donor * * Follow the blocked-on relation: - * task->blocked_on -> mutex->owner -> task... + * + * ,-> task + * | | blocked-on + * | v + * blocked_donor | mutex + * | | owner + * | v + * `-- task + * + * and set the blocked_donor relation, this latter is used by the mutex + * code to find which (blocked) task to hand-off to. * * Lock order: * @@ -6969,6 +6979,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) * rq, therefore holding @rq->lock is sufficient to * guarantee its existence, as per ttwu_remote(). */ + owner->blocked_donor = p; } WARN_ON_ONCE(owner && !owner->on_rq); return owner; @@ -7125,6 +7136,7 @@ pick_again: clear_task_blocked_on(prev, NULL); rq_set_donor(rq, next); + next->blocked_donor = NULL; if (unlikely(next->is_blocked && next->blocked_on)) { next = find_proxy_task(rq, next, &rf); if (!next) { -- cgit v1.2.3 From abc40cca0efdf5ba28b7bc37f1db445a8cc840bd Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 26 May 2026 11:28:46 +0200 Subject: sched/proxy: Optimize try_to_wake_up() The reason for the clause in try_to_wake_up() is, per its comment, that find_proxy_task()'s proxy_deactivate() is not always called with a cleared p->blocked_on. However, that seems silly and easily cured. Make sure to always call proxy_deactivate() with a cleared p->blocked_on such that we might remove this clause from the common wake-up path. Signed-off-by: Peter Zijlstra (Intel) Acked-by: John Stultz Link: https://patch.msgid.link/20260526113322.244729903%40infradead.org --- kernel/sched/core.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 4c6ceff3855e..a06d5a57adc8 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4343,14 +4343,6 @@ int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) */ WRITE_ONCE(p->__state, TASK_WAKING); - /* - * We never clear the blocked_on relation on proxy_deactivate. - * If we don't clear it here, we have TASK_RUNNING + p->blocked_on - * when waking up. Since this is a fully blocked, off CPU task - * waking up, it should be safe to clear the blocked_on relation. - */ - if (task_is_blocked(p)) - clear_task_blocked_on(p, NULL); /* * If the owning (remote) CPU is still in the middle of schedule() with * this task as prev, considering queueing p on the remote CPUs wake_list @@ -6739,6 +6731,7 @@ static void proxy_deactivate(struct rq *rq, struct task_struct *donor) unsigned long state = READ_ONCE(donor->__state); WARN_ON_ONCE(state == TASK_RUNNING); + WARN_ON_ONCE(donor->blocked_on); /* * Because we got donor from pick_next_task(), it is *crucial* * that we call proxy_resched_idle() before we deactivate it. @@ -6864,9 +6857,9 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) for (p = donor; (mutex = p->blocked_on); p = owner) { /* if its PROXY_WAKING, do return migration or run if current */ if (mutex == PROXY_WAKING) { + clear_task_blocked_on(p, PROXY_WAKING); if (task_current(rq, p)) { p->is_blocked = 0; - clear_task_blocked_on(p, PROXY_WAKING); return p; } goto deactivate; @@ -6900,9 +6893,9 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) * and return p (if it is current and safe to * just run on this rq), or return-migrate the task. */ + __clear_task_blocked_on(p, NULL); if (task_current(rq, p)) { p->is_blocked = 0; - __clear_task_blocked_on(p, NULL); return p; } goto deactivate; @@ -6912,6 +6905,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) /* XXX Don't handle blocked owners/delayed dequeue yet */ if (curr_in_chain) return proxy_resched_idle(rq); + __clear_task_blocked_on(p, NULL); goto deactivate; } -- cgit v1.2.3 From 708024b575b4ea58c5956e7c09f2d2f48facd478 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 26 May 2026 11:32:34 +0200 Subject: sched: Be more strict about p->is_blocked Upon entry to try_to_block_task(), p->is_blocked should be false. After all, the prior wakeup would have made it so per ttwu_do_wakeup(). Ensure this is the case, rather than clearing it in the path that doesn't set it. Signed-off-by: Peter Zijlstra (Intel) Acked-by: John Stultz Link: https://patch.msgid.link/20260526113322.364017314%40infradead.org --- kernel/sched/core.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index a06d5a57adc8..8b7eb126c254 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6676,8 +6676,9 @@ static bool try_to_block_task(struct rq *rq, struct task_struct *p, { unsigned long task_state = *task_state_p; + WARN_ON_ONCE(p->is_blocked); + if (signal_pending_state(task_state, p)) { - p->is_blocked = 0; WRITE_ONCE(p->__state, TASK_RUNNING); *task_state_p = TASK_RUNNING; clear_task_blocked_on(p, NULL); -- cgit v1.2.3 From 7918cf3693614c9f96bc9e43daff6fc72c01b81a Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 27 May 2026 09:58:02 +0200 Subject: sched/proxy: Only return migrate when needed Current code will 'unconditionally' return migrate on PROXY_WAKING, even if the task is (still) on the original CPU. Check task_cpu(p) against p->waking_cpu, which per proxy_set_task_cpu() preserves the original CPU the task was on. If they do not mis-match, there is no need to go through the more expensive wakeup path. Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260527082916.GP3126523%40noisy.programming.kicks-ass.net --- kernel/sched/core.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 8b7eb126c254..b007b65d9c88 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3767,6 +3767,21 @@ static inline bool proxy_needs_return(struct rq *rq, struct task_struct *p) if (!task_is_blocked(p)) return false; + /* + * Typically per __set_task_cpu(), task_cpu(p) == p->wake_cpu. + * + * However, proxy_set_task_cpu() is such that it preserves the + * original cpu in p->wake_cpu while migrating p for proxy reasons + * (possibly outside of the allowed p->cpus_ptr). + * + * Furthermore, migration_cpu_stop() / __migrate_swap_task(), will + * only set p->wake_cpu when !p->on_rq, and since here p->on_rq, this + * will not apply. But if it did, this check is the safe way around + * and would migrate. + */ + if (task_cpu(p) == p->wake_cpu) + return false; + scoped_guard(raw_spinlock, &p->blocked_lock) { /* Task is waking up; clear any blocked_on relationship */ __clear_task_blocked_on(p, NULL); -- cgit v1.2.3 From be365ce2bc20b8970bed350f82c3b760256b6945 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 26 May 2026 11:42:29 +0200 Subject: sched/proxy: Switch proxy to use p->is_blocked Rather than gate the proxy paths with p->blocked_on, use p->is_blocked. This opens up the state: '->is_blocked && !->blocked_on' for future use. Notably, only proxy and delayed tasks can be ->on_rq && ->is_blocked, and it is guaranteed that sched_class::pick_task() will never return a delayed task. Therefore any task returned from pick_next_task() that has ->is_blocked set, must be a proxy task. Suggested-by: K Prateek Nayak Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260526113322.477954312%40infradead.org --- kernel/sched/core.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index b007b65d9c88..9b710313dfb3 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3764,7 +3764,7 @@ static inline void proxy_reset_donor(struct rq *rq) */ static inline bool proxy_needs_return(struct rq *rq, struct task_struct *p) { - if (!task_is_blocked(p)) + if (!p->is_blocked) return false; /* @@ -6866,14 +6866,14 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) bool curr_in_chain = false; int this_cpu = cpu_of(rq); struct task_struct *p; - struct mutex *mutex; int owner_cpu; /* Follow blocked_on chain. */ - for (p = donor; (mutex = p->blocked_on); p = owner) { + for (p = donor; p->is_blocked; p = owner) { /* if its PROXY_WAKING, do return migration or run if current */ - if (mutex == PROXY_WAKING) { - clear_task_blocked_on(p, PROXY_WAKING); + struct mutex *mutex = p->blocked_on; + if (!mutex || mutex == PROXY_WAKING) { + clear_task_blocked_on(p, mutex); if (task_current(rq, p)) { p->is_blocked = 0; return p; @@ -7147,7 +7147,7 @@ pick_again: rq_set_donor(rq, next); next->blocked_donor = NULL; - if (unlikely(next->is_blocked && next->blocked_on)) { + if (unlikely(next->is_blocked)) { next = find_proxy_task(rq, next, &rf); if (!next) { zap_balance_callbacks(rq); -- cgit v1.2.3 From ec9d4f1c424134bbf30965075df78d02a5d021dc Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Tue, 26 May 2026 11:43:02 +0200 Subject: sched/proxy: Remove PROXY_WAKING Now that the proxy path uses ->is_blocked, use the '->is_blocked && !->blocked_on' state instead of PROXY_WAKING. Notably, this is where a blocked_on relation is broken but the donor task might still need a return migration. Signed-off-by: K Prateek Nayak Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260526113322.596522894%40infradead.org --- kernel/locking/mutex.c | 4 ++-- kernel/locking/ww_mutex.h | 4 ++-- kernel/sched/core.c | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c index 28677165785f..89d01f788973 100644 --- a/kernel/locking/mutex.c +++ b/kernel/locking/mutex.c @@ -1044,7 +1044,7 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne next_lock = __get_task_blocked_on(donor); if (next_lock == lock) { next = get_task_struct(donor); - __set_task_blocked_on_waking(donor, next_lock); + __clear_task_blocked_on(next, lock); current->blocked_donor = NULL; } raw_spin_unlock(&donor->blocked_lock); @@ -1060,7 +1060,7 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne raw_spin_lock_nested(&next->blocked_lock, SINGLE_DEPTH_NESTING); debug_mutex_wake_waiter(lock, waiter); - __set_task_blocked_on_waking(next, lock); + __clear_task_blocked_on(next, lock); raw_spin_unlock(&next->blocked_lock); } diff --git a/kernel/locking/ww_mutex.h b/kernel/locking/ww_mutex.h index 6c12452097e1..d62b49b53ec3 100644 --- a/kernel/locking/ww_mutex.h +++ b/kernel/locking/ww_mutex.h @@ -324,7 +324,7 @@ __ww_mutex_die(struct MUTEX *lock, struct MUTEX_WAITER *waiter, * blocked_on to PROXY_WAKING. Otherwise we can see * circular blocked_on relationships that can't resolve. */ - set_task_blocked_on_waking(waiter->task, lock); + clear_task_blocked_on(waiter->task, lock); wake_q_add(wake_q, waiter->task); } @@ -383,7 +383,7 @@ static bool __ww_mutex_wound(struct MUTEX *lock, * are waking the mutex owner, who may be currently * blocked on a different mutex. */ - set_task_blocked_on_waking(owner, NULL); + clear_task_blocked_on(owner, NULL); wake_q_add(wake_q, owner); } return true; diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 9b710313dfb3..cec2c164fab1 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6872,7 +6872,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) for (p = donor; p->is_blocked; p = owner) { /* if its PROXY_WAKING, do return migration or run if current */ struct mutex *mutex = p->blocked_on; - if (!mutex || mutex == PROXY_WAKING) { + if (!mutex) { clear_task_blocked_on(p, mutex); if (task_current(rq, p)) { p->is_blocked = 0; -- cgit v1.2.3 From c0404dd88d124714351f7a961d3313ee0f2f036b Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 26 May 2026 11:22:30 +0200 Subject: sched/proxy: Remove superfluous clear_task_blocked_in() Per the discussion here: https://lore.kernel.org/all/20260403112810.GG3738786@noisy.programming.kicks-ass.net/ The reason for this condition is that the signal condition in try_to_block_task() would set_task_blocked_in_waking(). However, it no longer does that, in fact, that path does clear_task_blocked_on(). Further, per the discussions here: https://lore.kernel.org/r/dc61cf77-e541-441d-a708-c40e19aa0db2%40amd.com https://lore.kernel.org/r//9dd1d24d-45d3-4ee2-8e67-8305b34bfb6d%40amd.com there are a few other edge cases that needed this. But they're all variants of PROXY_WAKING leaking out. And since PROXY_WAKING is now gone, this is no longer needed either. Signed-off-by: Peter Zijlstra (Intel) Acked-by: John Stultz Link: https://patch.msgid.link/20260526113322.120970670%40infradead.org --- kernel/sched/core.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index cec2c164fab1..d5795188d5b6 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -7142,9 +7142,6 @@ pick_again: if (sched_proxy_exec()) { struct task_struct *prev_donor = rq->donor; - if (!prev_state && prev->blocked_on) - clear_task_blocked_on(prev, NULL); - rq_set_donor(rq, next); next->blocked_donor = NULL; if (unlikely(next->is_blocked)) { -- cgit v1.2.3 From 56e50ff567810db208cc37d9e17b8df044a9158c Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 26 May 2026 12:00:59 +0200 Subject: sched: Simplify ttwu_runnable() Note that both proxy and delayed tasks have ->is_blocked set. Use this one condition to guard both paths. Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260526113322.714832584%40infradead.org --- kernel/sched/core.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index d5795188d5b6..5a317f62a516 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3764,9 +3764,6 @@ static inline void proxy_reset_donor(struct rq *rq) */ static inline bool proxy_needs_return(struct rq *rq, struct task_struct *p) { - if (!p->is_blocked) - return false; - /* * Typically per __set_task_cpu(), task_cpu(p) == p->wake_cpu. * @@ -3875,10 +3872,12 @@ static int ttwu_runnable(struct task_struct *p, int wake_flags) return 0; update_rq_clock(rq); - if (p->se.sched_delayed) - enqueue_task(rq, p, ENQUEUE_NOCLOCK | ENQUEUE_DELAYED); - if (proxy_needs_return(rq, p)) - return 0; + if (p->is_blocked) { + if (p->se.sched_delayed) + enqueue_task(rq, p, ENQUEUE_NOCLOCK | ENQUEUE_DELAYED); + if (proxy_needs_return(rq, p)) + return 0; + } if (!task_on_cpu(rq, p)) { /* * When on_rq && !on_cpu the task is preempted, see if -- cgit v1.2.3 From 63c1a12bc0e09af7dee919c4fb4a300a719d5125 Mon Sep 17 00:00:00 2001 From: "Guanyou.Chen" Date: Fri, 22 May 2026 21:09:59 +0800 Subject: sched: restore timer_slack_ns when resetting RT policy on fork Commit ed4fb6d7ef68 ("hrtimer: Use and report correct timerslack values for realtime tasks") sets timer_slack_ns to 0 for RT tasks in __setscheduler_params(). However, when an RT task with SCHED_RESET_ON_FORK creates child threads, the children inherit timer_slack_ns=0 from the parent. sched_fork() resets the child's policy to SCHED_NORMAL but does not restore timer_slack_ns, leaving the child permanently running with zero slack. Fix this by restoring timer_slack_ns from default_timer_slack_ns in sched_fork() when resetting from RT/DL to NORMAL policy, matching the existing behavior in __setscheduler_params(). Note: this fix alone requires a correct default_timer_slack_ns to be effective. See the following patch for that fix. Fixes: ed4fb6d7ef68 ("hrtimer: Use and report correct timerslack values for realtime tasks") Reported-by: Qiaoting.Lin Signed-off-by: Guanyou.Chen Signed-off-by: Chunhui.Li Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260522131000.1664983-2-chenguanyou@xiaomi.com --- kernel/sched/core.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 5a317f62a516..2cfe8932d5db 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4826,6 +4826,7 @@ int sched_fork(u64 clone_flags, struct task_struct *p) p->policy = SCHED_NORMAL; p->static_prio = NICE_TO_PRIO(0); p->rt_priority = 0; + p->timer_slack_ns = p->default_timer_slack_ns; } else if (PRIO_TO_NICE(p->static_prio) < 0) p->static_prio = NICE_TO_PRIO(0); -- cgit v1.2.3 From dfcfc97b6df0ea8e1b7d3b590022782abbec3389 Mon Sep 17 00:00:00 2001 From: Zecheng Li Date: Fri, 22 May 2026 10:15:48 -0400 Subject: sched/fair: Co-locate cfs_rq and sched_entity in cfs_tg_state Improve data locality and reduce pointer chasing by allocating struct cfs_rq and struct sched_entity together for non-root task groups. This is achieved by introducing a new combined struct cfs_tg_state that holds both objects in a single allocation. This patch: - Introduces struct cfs_tg_state that embeds cfs_rq, sched_entity, and sched_statistics together in a single structure. - Updates __schedstats_from_se() in stats.h to use cfs_tg_state for accessing sched_statistics from a group sched_entity. - Modifies alloc_fair_sched_group() and free_fair_sched_group() to allocate and free the new struct as a single unit. - Modifies the per-CPU pointers in task_group->se and task_group->cfs_rq to point to the members in the new combined structure. Signed-off-by: Zecheng Li Signed-off-by: Zecheng Li Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: K Prateek Nayak Reviewed-by: Josh Don Link: https://patch.msgid.link/20260522141623.600235-2-zli94@ncsu.edu --- kernel/sched/fair.c | 18 ++++++------------ kernel/sched/sched.h | 12 ++++++++++++ kernel/sched/stats.h | 9 +-------- 3 files changed, 19 insertions(+), 20 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 78162707a9d5..e7d7d47ef7b2 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -15083,8 +15083,6 @@ void free_fair_sched_group(struct task_group *tg) for_each_possible_cpu(i) { if (tg->cfs_rq) kfree(tg->cfs_rq[i]); - if (tg->se) - kfree(tg->se[i]); } kfree(tg->cfs_rq); @@ -15093,6 +15091,7 @@ void free_fair_sched_group(struct task_group *tg) int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) { + struct cfs_tg_state *state; struct sched_entity *se; struct cfs_rq *cfs_rq; int i; @@ -15109,16 +15108,13 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) init_cfs_bandwidth(tg_cfs_bandwidth(tg), tg_cfs_bandwidth(parent)); for_each_possible_cpu(i) { - cfs_rq = kzalloc_node(sizeof(struct cfs_rq), - GFP_KERNEL, cpu_to_node(i)); - if (!cfs_rq) + state = kzalloc_node(sizeof(*state), + GFP_KERNEL, cpu_to_node(i)); + if (!state) goto err; - se = kzalloc_node(sizeof(struct sched_entity_stats), - GFP_KERNEL, cpu_to_node(i)); - if (!se) - goto err_free_rq; - + cfs_rq = &state->cfs_rq; + se = &state->se; init_cfs_rq(cfs_rq); init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]); init_entity_runnable_average(se); @@ -15126,8 +15122,6 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) return 1; -err_free_rq: - kfree(cfs_rq); err: return 0; } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index b3aff26dbb13..585aba9f63b4 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2294,6 +2294,18 @@ static inline struct task_group *task_group(struct task_struct *p) return p->sched_task_group; } +#ifdef CONFIG_FAIR_GROUP_SCHED +/* + * Defined here to be available before stats.h is included, since + * stats.h has dependencies on things defined later in this file. + */ +struct cfs_tg_state { + struct cfs_rq cfs_rq; + struct sched_entity se; + struct sched_statistics stats; +} __no_randomize_layout; +#endif + /* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */ static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { diff --git a/kernel/sched/stats.h b/kernel/sched/stats.h index a612cf253c87..ebe0a7765f98 100644 --- a/kernel/sched/stats.h +++ b/kernel/sched/stats.h @@ -89,19 +89,12 @@ static inline void rq_sched_info_depart (struct rq *rq, unsigned long long delt #endif /* CONFIG_SCHEDSTATS */ -#ifdef CONFIG_FAIR_GROUP_SCHED -struct sched_entity_stats { - struct sched_entity se; - struct sched_statistics stats; -} __no_randomize_layout; -#endif - static inline struct sched_statistics * __schedstats_from_se(struct sched_entity *se) { #ifdef CONFIG_FAIR_GROUP_SCHED if (!entity_is_task(se)) - return &container_of(se, struct sched_entity_stats, se)->stats; + return &container_of(se, struct cfs_tg_state, se)->stats; #endif return &task_of(se)->stats; } -- cgit v1.2.3 From 89e1f67186baca353b68115bb98bd0bfed9f80c8 Mon Sep 17 00:00:00 2001 From: Zecheng Li Date: Fri, 22 May 2026 10:15:49 -0400 Subject: sched/fair: Remove task_group->se pointer array Now that struct sched_entity is co-located with struct cfs_rq for non-root task groups, the task_group->se pointer array is redundant. The associated sched_entity can be loaded directly from the cfs_rq. This patch performs the access conversion with the helpers: - is_root_task_group(tg): checks if a task group is the root task group. It compares the task group's address with the global root_task_group variable. - tg_se(tg, cpu): retrieves the cfs_rq and returns the address of the co-located se. This function checks if tg is the root task group to ensure behaving the same of previous tg->se[cpu]. Replaces all accesses that use the tg->se[cpu] pointer array with calls to the new tg_se(tg, cpu) accessor. - cfs_rq_se(cfs_rq): simplifies access paths like cfs_rq->tg->se[...] to use the co-located sched_entity. This function also checks if tg is the root task group to ensure same behavior. Since tg_se is not in very hot code paths, and the branch is a register comparison with an immediate value (`&root_task_group`), the performance impact is expected to be negligible. Signed-off-by: Zecheng Li Signed-off-by: Zecheng Li Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: K Prateek Nayak Reviewed-by: Josh Don Link: https://patch.msgid.link/20260522141623.600235-3-zli94@ncsu.edu --- kernel/sched/core.c | 7 ++----- kernel/sched/debug.c | 2 +- kernel/sched/fair.c | 25 +++++++++---------------- kernel/sched/sched.h | 31 ++++++++++++++++++++++++++----- 4 files changed, 38 insertions(+), 27 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 2cfe8932d5db..39cea012c230 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -8923,7 +8923,7 @@ void __init sched_init(void) wait_bit_init(); #ifdef CONFIG_FAIR_GROUP_SCHED - ptr += 2 * nr_cpu_ids * sizeof(void **); + ptr += nr_cpu_ids * sizeof(void **); #endif #ifdef CONFIG_RT_GROUP_SCHED ptr += 2 * nr_cpu_ids * sizeof(void **); @@ -8932,9 +8932,6 @@ void __init sched_init(void) ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT); #ifdef CONFIG_FAIR_GROUP_SCHED - root_task_group.se = (struct sched_entity **)ptr; - ptr += nr_cpu_ids * sizeof(void **); - root_task_group.cfs_rq = (struct cfs_rq **)ptr; ptr += nr_cpu_ids * sizeof(void **); @@ -10016,7 +10013,7 @@ static int cpu_cfs_stat_show(struct seq_file *sf, void *v) int i; for_each_possible_cpu(i) { - stats = __schedstats_from_se(tg->se[i]); + stats = __schedstats_from_se(tg_se(tg, i)); ws += schedstat_val(stats->wait_sum); } diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 5e09cf9fae3e..40584b27ea0c 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -808,7 +808,7 @@ void dirty_sched_domain_sysctl(int cpu) #ifdef CONFIG_FAIR_GROUP_SCHED static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_group *tg) { - struct sched_entity *se = tg->se[cpu]; + struct sched_entity *se = tg_se(tg, cpu); #define P(F) SEQ_printf(m, " .%-30s: %lld\n", #F, (long long)F) #define P_SCHEDSTAT(F) SEQ_printf(m, " .%-30s: %lld\n", \ diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index e7d7d47ef7b2..447b0ac426d1 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6876,7 +6876,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) { struct rq *rq = rq_of(cfs_rq); struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); - struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)]; + struct sched_entity *se = cfs_rq_se(cfs_rq); /* * It's possible we are called with runtime_remaining < 0 due to things @@ -11102,7 +11102,6 @@ static bool __update_blocked_fair(struct rq *rq, bool *done) { struct cfs_rq *cfs_rq, *pos; bool decayed = false; - int cpu = cpu_of(rq); /* * Iterates the task_group tree in a bottom up fashion, see @@ -11122,7 +11121,7 @@ static bool __update_blocked_fair(struct rq *rq, bool *done) } /* Propagate pending load changes to the parent, if any: */ - se = cfs_rq->tg->se[cpu]; + se = cfs_rq_se(cfs_rq); if (se && !skip_blocked_update(se)) update_load_avg(cfs_rq_of(se), se, UPDATE_TG); @@ -11148,8 +11147,7 @@ static bool __update_blocked_fair(struct rq *rq, bool *done) */ static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq) { - struct rq *rq = rq_of(cfs_rq); - struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)]; + struct sched_entity *se = cfs_rq_se(cfs_rq); unsigned long now = jiffies; unsigned long load; @@ -15086,7 +15084,6 @@ void free_fair_sched_group(struct task_group *tg) } kfree(tg->cfs_rq); - kfree(tg->se); } int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) @@ -15099,9 +15096,6 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) tg->cfs_rq = kzalloc_objs(cfs_rq, nr_cpu_ids); if (!tg->cfs_rq) goto err; - tg->se = kzalloc_objs(se, nr_cpu_ids); - if (!tg->se) - goto err; tg->shares = NICE_0_LOAD; @@ -15116,7 +15110,7 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) cfs_rq = &state->cfs_rq; se = &state->se; init_cfs_rq(cfs_rq); - init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]); + init_tg_cfs_entry(tg, cfs_rq, se, i, tg_se(parent, i)); init_entity_runnable_average(se); } @@ -15135,7 +15129,7 @@ void online_fair_sched_group(struct task_group *tg) for_each_possible_cpu(i) { rq = cpu_rq(i); - se = tg->se[i]; + se = tg_se(tg, i); rq_lock_irq(rq, &rf); update_rq_clock(rq); attach_entity_cfs_rq(se); @@ -15152,7 +15146,7 @@ void unregister_fair_sched_group(struct task_group *tg) for_each_possible_cpu(cpu) { struct cfs_rq *cfs_rq = tg->cfs_rq[cpu]; - struct sched_entity *se = tg->se[cpu]; + struct sched_entity *se = tg_se(tg, cpu); struct rq *rq = cpu_rq(cpu); if (se) { @@ -15189,7 +15183,6 @@ void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq, init_cfs_rq_runtime(cfs_rq); tg->cfs_rq[cpu] = cfs_rq; - tg->se[cpu] = se; /* se could be NULL for root_task_group */ if (!se) @@ -15220,7 +15213,7 @@ static int __sched_group_set_shares(struct task_group *tg, unsigned long shares) /* * We can't change the weight of the root cgroup. */ - if (!tg->se[0]) + if (is_root_task_group(tg)) return -EINVAL; shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES)); @@ -15231,7 +15224,7 @@ static int __sched_group_set_shares(struct task_group *tg, unsigned long shares) tg->shares = shares; for_each_possible_cpu(i) { struct rq *rq = cpu_rq(i); - struct sched_entity *se = tg->se[i]; + struct sched_entity *se = tg_se(tg, i); struct rq_flags rf; /* Propagate contribution to hierarchy */ @@ -15282,7 +15275,7 @@ int sched_group_set_idle(struct task_group *tg, long idle) for_each_possible_cpu(i) { struct rq *rq = cpu_rq(i); - struct sched_entity *se = tg->se[i]; + struct sched_entity *se = tg_se(tg, i); struct cfs_rq *grp_cfs_rq = tg->cfs_rq[i]; bool was_idle = cfs_rq_is_idle(grp_cfs_rq); long idle_task_delta; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 585aba9f63b4..823ba40cf098 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -484,8 +484,6 @@ struct task_group { #endif #ifdef CONFIG_FAIR_GROUP_SCHED - /* schedulable entities of this group on each CPU */ - struct sched_entity **se; /* runqueue "owned" by this group on each CPU */ struct cfs_rq **cfs_rq; unsigned long shares; @@ -934,7 +932,8 @@ struct dl_rq { }; #ifdef CONFIG_FAIR_GROUP_SCHED - +/* Check whether a task group is root tg */ +#define is_root_task_group(tg) ((tg) == &root_task_group) /* An entity is a task if it doesn't "own" a runqueue */ #define entity_is_task(se) (!se->my_q) @@ -2304,6 +2303,28 @@ struct cfs_tg_state { struct sched_entity se; struct sched_statistics stats; } __no_randomize_layout; + +static inline struct sched_entity *tg_se(struct task_group *tg, int cpu) +{ + struct cfs_tg_state *state; + + if (is_root_task_group(tg)) + return NULL; + + state = container_of(tg->cfs_rq[cpu], struct cfs_tg_state, cfs_rq); + return &state->se; +} + +static inline struct sched_entity *cfs_rq_se(struct cfs_rq *cfs_rq) +{ + struct cfs_tg_state *state; + + if (is_root_task_group(cfs_rq->tg)) + return NULL; + + state = container_of(cfs_rq, struct cfs_tg_state, cfs_rq); + return &state->se; +} #endif /* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */ @@ -2316,8 +2337,8 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu) #ifdef CONFIG_FAIR_GROUP_SCHED set_task_rq_fair(&p->se, p->se.cfs_rq, tg->cfs_rq[cpu]); p->se.cfs_rq = tg->cfs_rq[cpu]; - p->se.parent = tg->se[cpu]; - p->se.depth = tg->se[cpu] ? tg->se[cpu]->depth + 1 : 0; + p->se.parent = tg_se(tg, cpu); + p->se.depth = p->se.parent ? p->se.parent->depth + 1 : 0; #endif #ifdef CONFIG_RT_GROUP_SCHED -- cgit v1.2.3 From b8fea7af0e40feb6d9cbbd60b66ff0ec265e868f Mon Sep 17 00:00:00 2001 From: Zecheng Li Date: Fri, 22 May 2026 10:15:50 -0400 Subject: sched/fair: Allocate cfs_tg_state with percpu allocator To remove the cfs_rq pointer array in task_group, allocate the combined cfs_rq and sched_entity using the per-cpu allocator. This patch implements the following: - Changes task_group->cfs_rq from 'struct cfs_rq **' to 'struct cfs_rq __percpu *'. - Updates memory allocation in alloc_fair_sched_group() and free_fair_sched_group() to use alloc_percpu() and free_percpu() respectively. - Uses the inline accessor tg_cfs_rq(tg, cpu) with per_cpu_ptr() to retrieve the pointer to cfs_rq for the given task group and CPU. - Replaces direct accesses tg->cfs_rq[cpu] with calls to the new tg_cfs_rq(tg, cpu) helper. - Handles the root_task_group: since struct rq is already a per-cpu variable (runqueues), its embedded cfs_rq (rq->cfs) is also per-cpu. Therefore, we assign root_task_group.cfs_rq = &runqueues.cfs. - Cleanup the code in initializing the root task group. This change places each CPU's cfs_rq and sched_entity in its local per-cpu memory area to remove the per-task_group pointer arrays. Signed-off-by: Zecheng Li Signed-off-by: Zecheng Li Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: K Prateek Nayak Reviewed-by: Josh Don Link: https://patch.msgid.link/20260522141623.600235-4-zli94@ncsu.edu --- kernel/sched/core.c | 35 +++++++++++++--------------------- kernel/sched/fair.c | 54 +++++++++++++++++++++------------------------------- kernel/sched/sched.h | 14 ++++++++++---- 3 files changed, 45 insertions(+), 58 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 39cea012c230..dd031410ab1a 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -8907,7 +8907,7 @@ static struct kmem_cache *task_group_cache __ro_after_init; void __init sched_init(void) { - unsigned long ptr = 0; + unsigned long __maybe_unused ptr = 0; int i; /* Make sure the linker didn't screw up */ @@ -8923,33 +8923,24 @@ void __init sched_init(void) wait_bit_init(); #ifdef CONFIG_FAIR_GROUP_SCHED - ptr += nr_cpu_ids * sizeof(void **); -#endif -#ifdef CONFIG_RT_GROUP_SCHED - ptr += 2 * nr_cpu_ids * sizeof(void **); -#endif - if (ptr) { - ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT); + root_task_group.cfs_rq = &runqueues.cfs; -#ifdef CONFIG_FAIR_GROUP_SCHED - root_task_group.cfs_rq = (struct cfs_rq **)ptr; - ptr += nr_cpu_ids * sizeof(void **); - - root_task_group.shares = ROOT_TASK_GROUP_LOAD; - init_cfs_bandwidth(&root_task_group.cfs_bandwidth, NULL); + root_task_group.shares = ROOT_TASK_GROUP_LOAD; + init_cfs_bandwidth(&root_task_group.cfs_bandwidth, NULL); #endif /* CONFIG_FAIR_GROUP_SCHED */ #ifdef CONFIG_EXT_GROUP_SCHED - scx_tg_init(&root_task_group); + scx_tg_init(&root_task_group); #endif /* CONFIG_EXT_GROUP_SCHED */ #ifdef CONFIG_RT_GROUP_SCHED - root_task_group.rt_se = (struct sched_rt_entity **)ptr; - ptr += nr_cpu_ids * sizeof(void **); + ptr += 2 * nr_cpu_ids * sizeof(void **); + ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT); + root_task_group.rt_se = (struct sched_rt_entity **)ptr; + ptr += nr_cpu_ids * sizeof(void **); - root_task_group.rt_rq = (struct rt_rq **)ptr; - ptr += nr_cpu_ids * sizeof(void **); + root_task_group.rt_rq = (struct rt_rq **)ptr; + ptr += nr_cpu_ids * sizeof(void **); #endif /* CONFIG_RT_GROUP_SCHED */ - } init_defrootdomain(); @@ -9864,7 +9855,7 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, } for_each_online_cpu(i) { - struct cfs_rq *cfs_rq = tg->cfs_rq[i]; + struct cfs_rq *cfs_rq = tg_cfs_rq(tg, i); struct rq *rq = cfs_rq->rq; guard(rq_lock_irq)(rq); @@ -10032,7 +10023,7 @@ static u64 throttled_time_self(struct task_group *tg) u64 total = 0; for_each_possible_cpu(i) { - total += READ_ONCE(tg->cfs_rq[i]->throttled_clock_self_time); + total += READ_ONCE(tg_cfs_rq(tg, i)->throttled_clock_self_time); } return total; diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 447b0ac426d1..1d4ed883e630 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -334,7 +334,7 @@ static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) * to a tree or when we reach the top of the tree */ if (cfs_rq->tg->parent && - cfs_rq->tg->parent->cfs_rq[cpu]->on_list) { + tg_cfs_rq(cfs_rq->tg->parent, cpu)->on_list) { /* * If parent is already on the list, we add the child * just before. Thanks to circular linked property of @@ -342,7 +342,7 @@ static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) * of the list that starts by parent. */ list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, - &(cfs_rq->tg->parent->cfs_rq[cpu]->leaf_cfs_rq_list)); + &(tg_cfs_rq(cfs_rq->tg->parent, cpu)->leaf_cfs_rq_list)); /* * The branch is now connected to its tree so we can * reset tmp_alone_branch to the beginning of the @@ -5037,7 +5037,7 @@ static void __maybe_unused clear_tg_offline_cfs_rqs(struct rq *rq) rcu_read_lock(); list_for_each_entry_rcu(tg, &task_groups, list) { - struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; + struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq)); clear_tg_load_avg(cfs_rq); } @@ -6594,7 +6594,7 @@ static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) static inline int lb_throttled_hierarchy(struct task_struct *p, int dst_cpu) { - return throttled_hierarchy(task_group(p)->cfs_rq[dst_cpu]); + return throttled_hierarchy(tg_cfs_rq(task_group(p), dst_cpu)); } static inline bool task_is_throttled(struct task_struct *p) @@ -6740,7 +6740,7 @@ static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags); static int tg_unthrottle_up(struct task_group *tg, void *data) { struct rq *rq = data; - struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; + struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq)); struct task_struct *p, *tmp; if (--cfs_rq->throttle_count) @@ -6811,7 +6811,7 @@ static void record_throttle_clock(struct cfs_rq *cfs_rq) static int tg_throttle_down(struct task_group *tg, void *data) { struct rq *rq = data; - struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; + struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq)); if (cfs_rq->throttle_count++) return 0; @@ -7285,8 +7285,8 @@ static void sync_throttle(struct task_group *tg, int cpu) if (!tg->parent) return; - cfs_rq = tg->cfs_rq[cpu]; - pcfs_rq = tg->parent->cfs_rq[cpu]; + cfs_rq = tg_cfs_rq(tg, cpu); + pcfs_rq = tg_cfs_rq(tg->parent, cpu); cfs_rq->throttle_count = pcfs_rq->throttle_count; cfs_rq->throttled_clock_pelt = rq_clock_pelt(cpu_rq(cpu)); @@ -7478,7 +7478,7 @@ static void __maybe_unused update_runtime_enabled(struct rq *rq) rcu_read_lock(); list_for_each_entry_rcu(tg, &task_groups, list) { struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; - struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; + struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq)); raw_spin_lock(&cfs_b->lock); cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF; @@ -7507,7 +7507,7 @@ static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq) rcu_read_lock(); list_for_each_entry_rcu(tg, &task_groups, list) { - struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; + struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq)); if (!cfs_rq->runtime_enabled) continue; @@ -10382,7 +10382,7 @@ static inline int task_is_ineligible_on_dst_cpu(struct task_struct *p, int dest_ struct cfs_rq *dst_cfs_rq; #ifdef CONFIG_FAIR_GROUP_SCHED - dst_cfs_rq = task_group(p)->cfs_rq[dest_cpu]; + dst_cfs_rq = tg_cfs_rq(task_group(p), dest_cpu); #else dst_cfs_rq = &cpu_rq(dest_cpu)->cfs; #endif @@ -14812,7 +14812,7 @@ static int task_is_throttled_fair(struct task_struct *p, int cpu) struct cfs_rq *cfs_rq; #ifdef CONFIG_FAIR_GROUP_SCHED - cfs_rq = task_group(p)->cfs_rq[cpu]; + cfs_rq = tg_cfs_rq(task_group(p), cpu); #else cfs_rq = &cpu_rq(cpu)->cfs; #endif @@ -15076,39 +15076,31 @@ static void task_change_group_fair(struct task_struct *p) void free_fair_sched_group(struct task_group *tg) { - int i; - - for_each_possible_cpu(i) { - if (tg->cfs_rq) - kfree(tg->cfs_rq[i]); - } - - kfree(tg->cfs_rq); + free_percpu(tg->cfs_rq); } int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) { - struct cfs_tg_state *state; + struct cfs_tg_state __percpu *state; struct sched_entity *se; struct cfs_rq *cfs_rq; int i; - tg->cfs_rq = kzalloc_objs(cfs_rq, nr_cpu_ids); - if (!tg->cfs_rq) + state = alloc_percpu_gfp(struct cfs_tg_state, GFP_KERNEL); + if (!state) goto err; + tg->cfs_rq = &state->cfs_rq; tg->shares = NICE_0_LOAD; init_cfs_bandwidth(tg_cfs_bandwidth(tg), tg_cfs_bandwidth(parent)); for_each_possible_cpu(i) { - state = kzalloc_node(sizeof(*state), - GFP_KERNEL, cpu_to_node(i)); - if (!state) + cfs_rq = tg_cfs_rq(tg, i); + if (!cfs_rq) goto err; - cfs_rq = &state->cfs_rq; - se = &state->se; + se = tg_se(tg, i); init_cfs_rq(cfs_rq); init_tg_cfs_entry(tg, cfs_rq, se, i, tg_se(parent, i)); init_entity_runnable_average(se); @@ -15145,7 +15137,7 @@ void unregister_fair_sched_group(struct task_group *tg) destroy_cfs_bandwidth(tg_cfs_bandwidth(tg)); for_each_possible_cpu(cpu) { - struct cfs_rq *cfs_rq = tg->cfs_rq[cpu]; + struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu); struct sched_entity *se = tg_se(tg, cpu); struct rq *rq = cpu_rq(cpu); @@ -15182,8 +15174,6 @@ void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq, cfs_rq->rq = rq; init_cfs_rq_runtime(cfs_rq); - tg->cfs_rq[cpu] = cfs_rq; - /* se could be NULL for root_task_group */ if (!se) return; @@ -15276,7 +15266,7 @@ int sched_group_set_idle(struct task_group *tg, long idle) for_each_possible_cpu(i) { struct rq *rq = cpu_rq(i); struct sched_entity *se = tg_se(tg, i); - struct cfs_rq *grp_cfs_rq = tg->cfs_rq[i]; + struct cfs_rq *grp_cfs_rq = tg_cfs_rq(tg, i); bool was_idle = cfs_rq_is_idle(grp_cfs_rq); long idle_task_delta; struct rq_flags rf; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 823ba40cf098..c7c2dea65edd 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -485,7 +485,7 @@ struct task_group { #ifdef CONFIG_FAIR_GROUP_SCHED /* runqueue "owned" by this group on each CPU */ - struct cfs_rq **cfs_rq; + struct cfs_rq __percpu *cfs_rq; unsigned long shares; /* * load_avg can be heavily contended at clock tick time, so put @@ -2304,6 +2304,12 @@ struct cfs_tg_state { struct sched_statistics stats; } __no_randomize_layout; +/* Access a specific CPU's cfs_rq from a task group */ +static inline struct cfs_rq *tg_cfs_rq(struct task_group *tg, int cpu) +{ + return per_cpu_ptr(tg->cfs_rq, cpu); +} + static inline struct sched_entity *tg_se(struct task_group *tg, int cpu) { struct cfs_tg_state *state; @@ -2311,7 +2317,7 @@ static inline struct sched_entity *tg_se(struct task_group *tg, int cpu) if (is_root_task_group(tg)) return NULL; - state = container_of(tg->cfs_rq[cpu], struct cfs_tg_state, cfs_rq); + state = container_of(tg_cfs_rq(tg, cpu), struct cfs_tg_state, cfs_rq); return &state->se; } @@ -2335,8 +2341,8 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu) #endif #ifdef CONFIG_FAIR_GROUP_SCHED - set_task_rq_fair(&p->se, p->se.cfs_rq, tg->cfs_rq[cpu]); - p->se.cfs_rq = tg->cfs_rq[cpu]; + set_task_rq_fair(&p->se, p->se.cfs_rq, tg_cfs_rq(tg, cpu)); + p->se.cfs_rq = tg_cfs_rq(tg, cpu); p->se.parent = tg_se(tg, cpu); p->se.depth = p->se.parent ? p->se.parent->depth + 1 : 0; #endif -- cgit v1.2.3 From 1abbecd1d2d2fdd96e52f541f07ee2b163631bee Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Tue, 2 Jun 2026 05:00:01 +0000 Subject: sched/fair: Convert cfs bandwidth throttling to use guards Routine conversion of rcu_read_lock(), spin_lock*, and rq_lock usage within the cfs bandwidth controller to use class guards. Only notable changes are: - Checking for "cfs_rq->runtime_remaining <= 0" instead of the inverse to spot a throttle and break early. This also saves the need for extra indentation in the unthrottle case. - Reordering of list_del_rcu() against throttled_clock indicator update in unthrottle_cfs_rq(). Both are done with "cfs_b->lock" held after the "cfs_rq->throttled" is cleared which make the reordering safe against concurrent list modifications. No functional changes intended. Signed-off-by: K Prateek Nayak Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Ben Segall Tested-by: Aaron Lu Link: https://patch.msgid.link/20260602050005.11160-2-kprateek.nayak@amd.com --- kernel/sched/fair.c | 193 ++++++++++++++++++++++++---------------------------- 1 file changed, 90 insertions(+), 103 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 1d4ed883e630..261e5cedc717 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -5035,13 +5035,13 @@ static void __maybe_unused clear_tg_offline_cfs_rqs(struct rq *rq) */ rq_clock_start_loop_update(rq); - rcu_read_lock(); + guard(rcu)(); + list_for_each_entry_rcu(tg, &task_groups, list) { struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq)); clear_tg_load_avg(cfs_rq); } - rcu_read_unlock(); rq_clock_stop_loop_update(rq); } @@ -6540,13 +6540,10 @@ static int __assign_cfs_rq_runtime(struct cfs_bandwidth *cfs_b, static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq) { struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); - int ret; - raw_spin_lock(&cfs_b->lock); - ret = __assign_cfs_rq_runtime(cfs_b, cfs_rq, sched_cfs_bandwidth_slice()); - raw_spin_unlock(&cfs_b->lock); + guard(raw_spinlock)(&cfs_b->lock); - return ret; + return __assign_cfs_rq_runtime(cfs_b, cfs_rq, sched_cfs_bandwidth_slice()); } static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) @@ -6835,33 +6832,32 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) { struct rq *rq = rq_of(cfs_rq); struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); - int dequeue = 1; - raw_spin_lock(&cfs_b->lock); - /* This will start the period timer if necessary */ - if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, 1)) { + scoped_guard(raw_spinlock, &cfs_b->lock) { /* - * We have raced with bandwidth becoming available, and if we - * actually throttled the timer might not unthrottle us for an - * entire period. We additionally needed to make sure that any - * subsequent check_cfs_rq_runtime calls agree not to throttle - * us, as we may commit to do cfs put_prev+pick_next, so we ask - * for 1ns of runtime rather than just check cfs_b. + * Check if We have raced with bandwidth becoming available. If + * we actually throttled the timer might not unthrottle us for + * an entire period. We additionally needed to make sure that + * any subsequent check_cfs_rq_runtime calls agree not to + * throttle us, as we may commit to do cfs put_prev+pick_next, + * so we ask for 1ns of runtime rather than just check cfs_b. + * + * This will start the period timer if necessary. + */ + if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, 1)) + return false; + + /* + * No bandwidth available; Add ourselves on the list to be + * unthrottled later. */ - dequeue = 0; - } else { list_add_tail_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq); } - raw_spin_unlock(&cfs_b->lock); - - if (!dequeue) - return false; /* Throttle no longer required. */ /* freeze hierarchy runnable averages while throttled */ - rcu_read_lock(); - walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq); - rcu_read_unlock(); + scoped_guard(rcu) + walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq); /* * Note: distribution will already see us throttled via the @@ -6894,13 +6890,15 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) update_rq_clock(rq); - raw_spin_lock(&cfs_b->lock); - if (cfs_rq->throttled_clock) { + scoped_guard(raw_spinlock, &cfs_b->lock) { + list_del_rcu(&cfs_rq->throttled_list); + + if (!cfs_rq->throttled_clock) + break; + cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock; cfs_rq->throttled_clock = 0; } - list_del_rcu(&cfs_rq->throttled_list); - raw_spin_unlock(&cfs_b->lock); /* update hierarchical throttle state */ walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq); @@ -6929,9 +6927,8 @@ static void __cfsb_csd_unthrottle(void *arg) { struct cfs_rq *cursor, *tmp; struct rq *rq = arg; - struct rq_flags rf; - rq_lock(rq, &rf); + guard(rq_lock)(rq); /* * Iterating over the list can trigger several call to @@ -6948,7 +6945,7 @@ static void __cfsb_csd_unthrottle(void *arg) * race with group being freed in the window between removing it * from the list and advancing to the next entry in the list. */ - rcu_read_lock(); + guard(rcu)(); list_for_each_entry_safe(cursor, tmp, &rq->cfsb_csd_list, throttled_csd_list) { @@ -6958,10 +6955,7 @@ static void __cfsb_csd_unthrottle(void *arg) unthrottle_cfs_rq(cursor); } - rcu_read_unlock(); - rq_clock_stop_loop_update(rq); - rq_unlock(rq, &rf); } static inline void __unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) @@ -7001,11 +6995,11 @@ static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b) u64 runtime, remaining = 1; bool throttled = false; struct cfs_rq *cfs_rq, *tmp; - struct rq_flags rf; struct rq *rq; LIST_HEAD(local_unthrottle); - rcu_read_lock(); + guard(rcu)(); + list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq, throttled_list) { rq = rq_of(cfs_rq); @@ -7015,65 +7009,63 @@ static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b) break; } - rq_lock_irqsave(rq, &rf); + guard(rq_lock_irqsave)(rq); + if (!cfs_rq_throttled(cfs_rq)) - goto next; + continue; /* Already queued for async unthrottle */ if (!list_empty(&cfs_rq->throttled_csd_list)) - goto next; + continue; /* By the above checks, this should never be true */ WARN_ON_ONCE(cfs_rq->runtime_remaining > 0); - raw_spin_lock(&cfs_b->lock); - runtime = -cfs_rq->runtime_remaining + 1; - if (runtime > cfs_b->runtime) - runtime = cfs_b->runtime; - cfs_b->runtime -= runtime; - remaining = cfs_b->runtime; - raw_spin_unlock(&cfs_b->lock); + scoped_guard(raw_spinlock, &cfs_b->lock) { + runtime = -cfs_rq->runtime_remaining + 1; + if (runtime > cfs_b->runtime) + runtime = cfs_b->runtime; + cfs_b->runtime -= runtime; + remaining = cfs_b->runtime; + } cfs_rq->runtime_remaining += runtime; - /* we check whether we're throttled above */ - if (cfs_rq->runtime_remaining > 0) { - if (cpu_of(rq) != this_cpu) { - unthrottle_cfs_rq_async(cfs_rq); - } else { - /* - * We currently only expect to be unthrottling - * a single cfs_rq locally. - */ - WARN_ON_ONCE(!list_empty(&local_unthrottle)); - list_add_tail(&cfs_rq->throttled_csd_list, - &local_unthrottle); - } - } else { + /* + * Ran out of bandwidth during distribution! + * Indicate throttled entities and break early. + */ + if (cfs_rq->runtime_remaining <= 0) { throttled = true; + break; } -next: - rq_unlock_irqrestore(rq, &rf); + /* we check whether we're throttled above */ + if (cpu_of(rq) != this_cpu) { + unthrottle_cfs_rq_async(cfs_rq); + continue; + } + + /* + * We currently only expect to be unthrottling + * a single cfs_rq locally. + */ + WARN_ON_ONCE(!list_empty(&local_unthrottle)); + list_add_tail(&cfs_rq->throttled_csd_list, &local_unthrottle); } list_for_each_entry_safe(cfs_rq, tmp, &local_unthrottle, throttled_csd_list) { struct rq *rq = rq_of(cfs_rq); - rq_lock_irqsave(rq, &rf); + guard(rq_lock_irqsave)(rq); list_del_init(&cfs_rq->throttled_csd_list); - if (cfs_rq_throttled(cfs_rq)) unthrottle_cfs_rq(cfs_rq); - - rq_unlock_irqrestore(rq, &rf); } WARN_ON_ONCE(!list_empty(&local_unthrottle)); - rcu_read_unlock(); - return throttled; } @@ -7196,7 +7188,8 @@ static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq) if (slack_runtime <= 0) return; - raw_spin_lock(&cfs_b->lock); + guard(raw_spinlock)(&cfs_b->lock); + if (cfs_b->quota != RUNTIME_INF) { cfs_b->runtime += slack_runtime; @@ -7205,7 +7198,6 @@ static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq) !list_empty(&cfs_b->throttled_cfs_rq)) start_cfs_slack_bandwidth(cfs_b); } - raw_spin_unlock(&cfs_b->lock); /* even if it's not valid for return we don't want to try again */ cfs_rq->runtime_remaining -= slack_runtime; @@ -7228,25 +7220,21 @@ static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) */ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) { - u64 runtime = 0, slice = sched_cfs_bandwidth_slice(); - unsigned long flags; - /* confirm we're still not at a refresh boundary */ - raw_spin_lock_irqsave(&cfs_b->lock, flags); - cfs_b->slack_started = false; + scoped_guard(raw_spinlock_irqsave, &cfs_b->lock) { + u64 runtime = 0, slice = sched_cfs_bandwidth_slice(); - if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) { - raw_spin_unlock_irqrestore(&cfs_b->lock, flags); - return; - } + cfs_b->slack_started = false; - if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice) - runtime = cfs_b->runtime; + if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) + return; - raw_spin_unlock_irqrestore(&cfs_b->lock, flags); + if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice) + runtime = cfs_b->runtime; - if (!runtime) - return; + if (!runtime) + return; + } distribute_cfs_runtime(cfs_b); } @@ -7335,18 +7323,18 @@ static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer) { struct cfs_bandwidth *cfs_b = container_of(timer, struct cfs_bandwidth, period_timer); - unsigned long flags; int overrun; int idle = 0; int count = 0; - raw_spin_lock_irqsave(&cfs_b->lock, flags); + CLASS(raw_spinlock_irqsave, cfsb_guard)(&cfs_b->lock); + for (;;) { overrun = hrtimer_forward_now(timer, cfs_b->period); if (!overrun) break; - idle = do_sched_cfs_period_timer(cfs_b, overrun, flags); + idle = do_sched_cfs_period_timer(cfs_b, overrun, cfsb_guard.flags); if (++count > 3) { u64 new, old = ktime_to_ns(cfs_b->period); @@ -7379,11 +7367,13 @@ static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer) count = 0; } } - if (idle) + + if (idle) { cfs_b->period_active = 0; - raw_spin_unlock_irqrestore(&cfs_b->lock, flags); + return HRTIMER_NORESTART; + } - return idle ? HRTIMER_NORESTART : HRTIMER_RESTART; + return HRTIMER_RESTART; } void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b, struct cfs_bandwidth *parent) @@ -7450,14 +7440,12 @@ static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) */ for_each_possible_cpu(i) { struct rq *rq = cpu_rq(i); - unsigned long flags; if (list_empty(&rq->cfsb_csd_list)) continue; - local_irq_save(flags); - __cfsb_csd_unthrottle(rq); - local_irq_restore(flags); + scoped_guard(irqsave) + __cfsb_csd_unthrottle(rq); } } @@ -7475,16 +7463,15 @@ static void __maybe_unused update_runtime_enabled(struct rq *rq) lockdep_assert_rq_held(rq); - rcu_read_lock(); + guard(rcu)(); + list_for_each_entry_rcu(tg, &task_groups, list) { struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq)); - raw_spin_lock(&cfs_b->lock); - cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF; - raw_spin_unlock(&cfs_b->lock); + scoped_guard(raw_spinlock, &cfs_b->lock) + cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF; } - rcu_read_unlock(); } /* cpu offline callback */ @@ -7505,7 +7492,8 @@ static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq) */ rq_clock_start_loop_update(rq); - rcu_read_lock(); + guard(rcu)(); + list_for_each_entry_rcu(tg, &task_groups, list) { struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq)); @@ -7528,7 +7516,6 @@ static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq) cfs_rq->runtime_remaining = 1; unthrottle_cfs_rq(cfs_rq); } - rcu_read_unlock(); rq_clock_stop_loop_update(rq); } -- cgit v1.2.3 From 253edcf5436c916f2fbf7b880443c7f1ed76101d Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Tue, 2 Jun 2026 05:00:02 +0000 Subject: sched/fair: Use throttled_csd_list for local unthrottle When distribute_cfs_runtime() encounters a local cfs_rq, it adds it to a local list and unthrottles it at the end, when it is done unthrottling other cfs_rq(s) on cfs_b->throttled_cfs_rq until the bandwidth runs out. Instead of using a local list, reuse the local CPU's rq->throttled_csd_list and the __cfsb_csd_unthrottle() path for unthrottle. If this is the first cfs_rq to be queued on the "throttled_csd_list", it prevents the need for a remote CPUs to interrupt this local CPU if they themselves are performing async unthrottle. If this is not the first cfs_rq on the list, there is an async unthrottle operation pending on this local CPU and the unthrottle can be batched together. No functional changes intended. Signed-off-by: K Prateek Nayak Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Benjamin Segall Tested-by: Aaron Lu Link: https://patch.msgid.link/20260602050005.11160-3-kprateek.nayak@amd.com --- kernel/sched/fair.c | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 261e5cedc717..26a8bbb9e1e2 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6991,12 +6991,11 @@ static void unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b) { + bool throttled = false, unthrottle_local = false; int this_cpu = smp_processor_id(); u64 runtime, remaining = 1; - bool throttled = false; - struct cfs_rq *cfs_rq, *tmp; + struct cfs_rq *cfs_rq; struct rq *rq; - LIST_HEAD(local_unthrottle); guard(rcu)(); @@ -7047,24 +7046,23 @@ static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b) } /* - * We currently only expect to be unthrottling - * a single cfs_rq locally. + * Allow a parallel async unthrottle to unthrottle + * this cfs_rq too via __cfsb_csd_unthrottle(). + * If we are first, do it ourselves at the end and + * save on an IPI from remote CPUs. */ - WARN_ON_ONCE(!list_empty(&local_unthrottle)); - list_add_tail(&cfs_rq->throttled_csd_list, &local_unthrottle); + unthrottle_local = list_empty(&rq->cfsb_csd_list); + list_add_tail(&cfs_rq->throttled_csd_list, &rq->cfsb_csd_list); } - list_for_each_entry_safe(cfs_rq, tmp, &local_unthrottle, - throttled_csd_list) { - struct rq *rq = rq_of(cfs_rq); - - guard(rq_lock_irqsave)(rq); - - list_del_init(&cfs_rq->throttled_csd_list); - if (cfs_rq_throttled(cfs_rq)) - unthrottle_cfs_rq(cfs_rq); + if (unthrottle_local) { + /* + * Protect against an IPI that is also trying to flush + * the unthrottled cfs_rq(s) from this CPU's csd_list. + */ + scoped_guard(irqsave) + __cfsb_csd_unthrottle(cpu_rq(this_cpu)); } - WARN_ON_ONCE(!list_empty(&local_unthrottle)); return throttled; } -- cgit v1.2.3 From 28ad5427682bccf06074366f347a6083d6730c1e Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Tue, 2 Jun 2026 05:25:29 +0000 Subject: sched/fair: Call update_curr() before unthrottling the hierarchy Subsequent commits will allow update_curr() to throttle the hierarchy when the runtime accounting exceeds allocated quota. Call update_curr() before the unthrottle event, and in tg_unthrottle_up() to catch up on any remaining runtime and stabilize the "runtime_remaining" and "throttle_count" for that cfs_rq. Doing an update_curr() early ensures the cfs_rq is not throttled right back up again when the unthrottle is in progress. Since all callers of unthrottle_cfs_rq(), except two, already update the rq_clock and call rq_clock_start_loop_update(), move the update_rq_clock() from unthrottle_cfs_rq() to the callers that don't update the rq_clock. Signed-off-by: K Prateek Nayak Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Benjamin Segall Tested-by: Aaron Lu Link: https://patch.msgid.link/20260602052531.11450-1-kprateek.nayak@amd.com --- kernel/sched/core.c | 5 ++++- kernel/sched/fair.c | 21 +++++++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index dd031410ab1a..e745c58671ed 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -9859,11 +9859,14 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, struct rq *rq = cfs_rq->rq; guard(rq_lock_irq)(rq); + cfs_rq->runtime_enabled = runtime_enabled; cfs_rq->runtime_remaining = 1; - if (cfs_rq->throttled) + if (cfs_rq->throttled) { + update_rq_clock(rq); unthrottle_cfs_rq(cfs_rq); + } } if (runtime_was_enabled && !runtime_enabled) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 26a8bbb9e1e2..f91d85cd121b 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6740,6 +6740,15 @@ static int tg_unthrottle_up(struct task_group *tg, void *data) struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq)); struct task_struct *p, *tmp; + /* + * If cfs_rq->curr is set, the cfs_rq might not have caught up + * since the last clock update. Do it now before we begin + * queueing task onto it to save the need for unnecessarily + * unthrottle the hierarchy for this cfs_rq to be throttled + * right back again. + */ + update_curr(cfs_rq); + if (--cfs_rq->throttle_count) return 0; @@ -6882,14 +6891,16 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) * We can't unthrottle this cfs_rq without any runtime remaining because * any enqueue in tg_unthrottle_up() will immediately trigger a throttle, * which is not supposed to happen on unthrottle path. + * + * Catch up on the remaining runtime since last clock update before + * checking runtime remaining. */ + update_curr(cfs_rq); if (cfs_rq->runtime_enabled && cfs_rq->runtime_remaining <= 0) return; cfs_rq->throttled = 0; - update_rq_clock(rq); - scoped_guard(raw_spinlock, &cfs_b->lock) { list_del_rcu(&cfs_rq->throttled_list); @@ -6964,6 +6975,7 @@ static inline void __unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) bool first; if (rq == this_rq()) { + update_rq_clock(rq); unthrottle_cfs_rq(cfs_rq); return; } @@ -7017,6 +7029,11 @@ static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b) if (!list_empty(&cfs_rq->throttled_csd_list)) continue; + if (cfs_rq->curr) { + update_rq_clock(rq); + update_curr(cfs_rq); + } + /* By the above checks, this should never be true */ WARN_ON_ONCE(cfs_rq->runtime_remaining > 0); -- cgit v1.2.3 From 102a28344a60e637934ffca62d50ff8319b11165 Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Tue, 2 Jun 2026 05:25:30 +0000 Subject: sched/fair: Move the throttled tasks to a local list in tg_unthrottle_up() An update_curr() during the enqueue of throttled task will start throttling the hierarchy from subsequent commit. This can lead to tg_throttle_down() seeing non-empty throttled_limbo_list for the cfs_rq attaching the task from throttled_limbo_list one by one. For example: R | A / \ *B C | rq->curr *B is throttled with tasks on hte limbo list. When the tasks are unthrottled via tg_unthrottle_up() and entity of group B is placed onto A, update_curr() is called to catch up the vruntime and it may throttle group A causing the subsequent tg_throttle_down() to see the pending task's on B's limbo list. tg_unthrottle_up() /* --cfs_rq->throttle_count == 0 */ list_for_each_entry_safe(p, cfs_rq->throttled_limbo_list) enqueue_task_fair() enqueue_entity(se /* B->se */) update_curr(cfs_rq /* A->gcfs_rq */) account_cfs_rq_runtime(cfs_rq) throttle_cfs_rq(cfs_rq /* A->gcfs_rq */ ) tg_throttle_down() /* Reaches B->cfs_rq with throttle_count == 0 */ !!! !list_empty(&cfs_rq->throttled_limbo_list)) !!! Move the tasks from throttled_limbo_list onto a local list before starting the unthrottle to prevent the splat described above. If the hierarchy is throttled again in middle of an unthrottle, put the pending tasks back onto the limbo list to prevent running them unnecessarily. Signed-off-by: K Prateek Nayak Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Benjamin Segall Tested-by: Aaron Lu Link: https://patch.msgid.link/20260602052531.11450-2-kprateek.nayak@amd.com --- kernel/sched/fair.c | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index f91d85cd121b..3f3f09a021db 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6739,6 +6739,7 @@ static int tg_unthrottle_up(struct task_group *tg, void *data) struct rq *rq = data; struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq)); struct task_struct *p, *tmp; + LIST_HEAD(throttled_tasks); /* * If cfs_rq->curr is set, the cfs_rq might not have caught up @@ -6769,13 +6770,31 @@ static int tg_unthrottle_up(struct task_group *tg, void *data) cfs_rq->throttled_clock_self_time += delta; } + /* + * Move the tasks to a local list since an update_curr() during + * enqueue_task_fair() can throttle a higher cfs_rq, and it can + * see the "throttled_limbo_list" being non-empty in + * tg_throttle_down() if throttle_count turned 0 above. + */ + list_splice_init(&cfs_rq->throttled_limbo_list, &throttled_tasks); + /* Re-enqueue the tasks that have been throttled at this level. */ - list_for_each_entry_safe(p, tmp, &cfs_rq->throttled_limbo_list, throttle_node) { + list_for_each_entry_safe(p, tmp, &throttled_tasks, throttle_node) { + /* + * Back to being throttled! Break out and put the remaining + * tasks back onto the limbo_list to prevent running them + * unnecessarily. + */ + if (cfs_rq->throttle_count) + break; + list_del_init(&p->throttle_node); p->throttled = false; - enqueue_task_fair(rq_of(cfs_rq), p, ENQUEUE_WAKEUP); + enqueue_task_fair(rq, p, ENQUEUE_WAKEUP); } + list_splice(&throttled_tasks, &cfs_rq->throttled_limbo_list); + /* Add cfs_rq with load or one or more already running entities to the list */ if (!cfs_rq_is_decayed(cfs_rq)) list_add_leaf_cfs_rq(cfs_rq); -- cgit v1.2.3 From f666241e6bd5d9a494beca982e1953208dce531c Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 2 Jun 2026 07:10:05 +0000 Subject: sched/fair: Unify cfs_rq throttling via account_cfs_rq_runtime() assign_cfs_rq_runtime() during update_curr() sets the resched indicator and relies on check_cfs_rq_runtime() during pick_next_task() / put_prev_entity() to throttle the hierarchy once current task is preempted / blocks. Per-task throttle, on the other hand, uses throttle_cfs_rq() to simply propagate the throttle signals, and then relies on task work to individually throttle the runnable tasks on their way out to the userspace. Remove check_cfs_rq_runtime() and unify throttling into account_cfs_rq_runtime() which only sets the cfs_rq->throttled, cfs_rq->throttle_count indicators via throttle_cfs_rq() and optionally adds the task work to the current task (donor) it is on the throttled hierarchy. throttle_cfs_rq() requests for sched_cfs_bandwidth_slice() worth of bandwidth for the current hierarchy that enable it to continue running uninterrupted when selected. For the rest, it requests a bare minimum of "1" to ensure some bandwidth is available and pass the "runtime_remaining > 0" checks once selected. For SCHED_PROXY_EXEC, a mutex holder cannot exit to userspace without dropping it first and the mutex_unlock() ensures proxy is stopped before the mutex handoff which preserves the current semantics for running a throttled task until it exits to the userspace even if it acts as a donor. [ prateek: rebased on tip, comments, commit message. ] Reviewed-By: Benjamin Segall Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: K Prateek Nayak Signed-off-by: Peter Zijlstra (Intel) Tested-by: Aaron Lu Link: https://patch.msgid.link/20260602071005.11942-1-kprateek.nayak@amd.com --- kernel/sched/fair.c | 101 ++++++++++++++++++++++++---------------------------- 1 file changed, 46 insertions(+), 55 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 3f3f09a021db..f4ed841f766f 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -525,7 +525,7 @@ static int se_is_idle(struct sched_entity *se) #endif /* !CONFIG_FAIR_GROUP_SCHED */ static __always_inline -void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec); +bool account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec); /************************************************************** * Scheduling class tree data structure manipulation methods: @@ -6388,8 +6388,6 @@ pick_next_entity(struct rq *rq, struct cfs_rq *cfs_rq, bool protect) return se; } -static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq); - static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) { /* @@ -6399,9 +6397,6 @@ static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) if (prev->on_rq) update_curr(cfs_rq); - /* throttle cfs_rqs exceeding runtime */ - check_cfs_rq_runtime(cfs_rq); - if (prev->on_rq) { update_stats_wait_start_fair(cfs_rq, prev); /* Put 'current' back into the tree. */ @@ -6536,41 +6531,32 @@ static int __assign_cfs_rq_runtime(struct cfs_bandwidth *cfs_b, return cfs_rq->runtime_remaining > 0; } -/* returns 0 on failure to allocate runtime */ -static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq) -{ - struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); - - guard(raw_spinlock)(&cfs_b->lock); +static bool throttle_cfs_rq(struct cfs_rq *cfs_rq); - return __assign_cfs_rq_runtime(cfs_b, cfs_rq, sched_cfs_bandwidth_slice()); -} - -static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) +static bool __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) { /* dock delta_exec before expiring quota (as it could span periods) */ cfs_rq->runtime_remaining -= delta_exec; if (likely(cfs_rq->runtime_remaining > 0)) - return; + return false; if (cfs_rq->throttled) - return; + return true; /* - * if we're unable to extend our runtime we resched so that the active - * hierarchy can be throttled + * throttle_cfs_rq() will try to extend the runtime first + * before throttling the hierarchy. */ - if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr)) - resched_curr(rq_of(cfs_rq)); + return throttle_cfs_rq(cfs_rq); } static __always_inline -void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) +bool account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) { if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled) - return; + return false; - __account_cfs_rq_runtime(cfs_rq, delta_exec); + return __account_cfs_rq_runtime(cfs_rq, delta_exec); } static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) @@ -6858,10 +6844,24 @@ static int tg_throttle_down(struct task_group *tg, void *data) static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) { - struct rq *rq = rq_of(cfs_rq); struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); + struct sched_entity *curr = cfs_rq->curr; + struct rq *rq = rq_of(cfs_rq); scoped_guard(raw_spinlock, &cfs_b->lock) { + u64 target_runtime = 1; + + /* + * If cfs_rq->curr is still runnable, we are here from an + * update_curr(). Request sysctl_sched_cfs_bandwidth_slice + * worth of bandwidth to continue running. + * + * If the curr is not runnable, just request enough bandwidth + * to be runnable next time the pick selects this cfs_rq. + */ + if (curr && curr->on_rq) + target_runtime = sched_cfs_bandwidth_slice(); + /* * Check if We have raced with bandwidth becoming available. If * we actually throttled the timer might not unthrottle us for @@ -6872,7 +6872,7 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) * * This will start the period timer if necessary. */ - if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, 1)) + if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, target_runtime)) return false; /* @@ -6893,6 +6893,17 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) */ cfs_rq->throttled = 1; WARN_ON_ONCE(cfs_rq->throttled_clock); + + /* + * If current hierarchy was throttled, add throttle work to the + * current donor. In case of proxy-execution, the execution + * context cannot exit to the userspace while holding a mutex + * and the rule of throttle deferral to only throttle the + * throttled context at exit to userspace is still preserved. + */ + if (curr && curr->on_rq) + task_throttle_setup_work(rq->donor); + return true; } @@ -7283,7 +7294,7 @@ static void check_enqueue_throttle(struct cfs_rq *cfs_rq) if (!cfs_bandwidth_used()) return; - /* an active group must be handled by the update_curr()->put() path */ + /* an active group must be handled by the update_curr() path */ if (!cfs_rq->runtime_enabled || cfs_rq->curr) return; @@ -7293,8 +7304,6 @@ static void check_enqueue_throttle(struct cfs_rq *cfs_rq) /* update runtime allocation */ account_cfs_rq_runtime(cfs_rq, 0); - if (cfs_rq->runtime_remaining <= 0) - throttle_cfs_rq(cfs_rq); } static void sync_throttle(struct task_group *tg, int cpu) @@ -7324,25 +7333,6 @@ static void sync_throttle(struct task_group *tg, int cpu) cfs_rq->pelt_clock_throttled = 1; } -/* conditionally throttle active cfs_rq's from put_prev_entity() */ -static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) -{ - if (!cfs_bandwidth_used()) - return false; - - if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0)) - return false; - - /* - * it's possible for a throttled entity to be forced into a running - * state (e.g. set_curr_task), in this case we're finished. - */ - if (cfs_rq_throttled(cfs_rq)) - return true; - - return throttle_cfs_rq(cfs_rq); -} - static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer) { struct cfs_bandwidth *cfs_b = @@ -7596,8 +7586,7 @@ static void sched_fair_update_stop_tick(struct rq *rq, struct task_struct *p) #else /* !CONFIG_CFS_BANDWIDTH: */ -static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) {} -static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) { return false; } +static bool account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) { return false; } static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {} static inline void sync_throttle(struct task_group *tg, int cpu) {} static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} @@ -9934,8 +9923,6 @@ again: if (cfs_rq->curr && cfs_rq->curr->on_rq) update_curr(cfs_rq); - throttled |= check_cfs_rq_runtime(cfs_rq); - se = pick_next_entity(rq, cfs_rq, true); if (!se) goto again; @@ -14853,8 +14840,8 @@ static inline void task_tick_core(struct rq *rq, struct task_struct *curr) {} */ static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) { - struct cfs_rq *cfs_rq; struct sched_entity *se = &curr->se; + struct cfs_rq *cfs_rq; for_each_sched_entity(se) { cfs_rq = cfs_rq_of(se); @@ -15036,6 +15023,7 @@ static void switched_to_fair(struct rq *rq, struct task_struct *p) static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) { struct sched_entity *se = &p->se; + bool throttled = false; for_each_sched_entity(se) { struct cfs_rq *cfs_rq = cfs_rq_of(se); @@ -15046,9 +15034,12 @@ static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) set_next_entity(cfs_rq, se, first); /* ensure bandwidth has been allocated on our new cfs_rq */ - account_cfs_rq_runtime(cfs_rq, 0); + throttled |= account_cfs_rq_runtime(cfs_rq, 0); } + if (throttled) + task_throttle_setup_work(p); + se = &p->se; if (task_on_rq_queued(p)) { -- cgit v1.2.3 From 6a579050f8300adb794f09a5d1f4ff435d43ced5 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Chaudhary Date: Mon, 1 Jun 2026 09:26:26 +0530 Subject: printk: fix typos in comments Fix spelling/grammatical errors in printk.c and nbcon.c: - "precation" -> "precautionary" - "othrewise" -> "otherwise" - "An usable" -> "A usable" - "made a progress" -> "made progress" - "preemtible" -> "preemptible" - "mechasism" -> "mechanism" - "ownerhip" -> "ownership" Signed-off-by: Naveen Kumar Chaudhary Link: https://patch.msgid.link/pakfewagyzb7da3yuxnaxdaoma5w4j2c7i3xebmcld3xy4mqs5@zxsx2idpxrdq Reviewed-by: Petr Mladek Signed-off-by: Petr Mladek --- kernel/printk/nbcon.c | 6 +++--- kernel/printk/printk.c | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index d7044a7a214b..4b03b019cd5e 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -1487,7 +1487,7 @@ bool nbcon_allow_unsafe_takeover(void) * or write_thread(). * * When false, the write_thread() callback is used and would be - * called in a preemtible context unless disabled by the + * called in a preemptible context unless disabled by the * device_lock. The legacy handover is not allowed in this mode. * * Context: Any context except NMI. @@ -1868,7 +1868,7 @@ void nbcon_free(struct console *con) * Return: True if the console was acquired. False otherwise. * * Console drivers will usually use their own internal synchronization - * mechasism to synchronize between console printing and non-printing + * mechanism to synchronize between console printing and non-printing * activities (such as setting baud rates). However, nbcon console drivers * supporting atomic consoles may also want to mark unsafe sections when * performing non-printing activities in order to synchronize against their @@ -1954,7 +1954,7 @@ EXPORT_SYMBOL_GPL(nbcon_device_release); * * kdb emits messages on consoles registered for printk() without * storing them into the ring buffer. It has to acquire the console - * ownerhip so that it could call con->write_atomic() callback a safe way. + * ownership so that it could call con->write_atomic() callback a safe way. * * This function acquires the nbcon console using priority NBCON_PRIO_EMERGENCY * and marks it unsafe for handover/takeover. diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 0323149548f6..2fe9a963c823 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -188,7 +188,7 @@ static int __init control_devkmsg(char *str) /* * Sysctl cannot change it anymore. The kernel command line setting of * this parameter is to force the setting to be permanent throughout the - * runtime of the system. This is a precation measure against userspace + * runtime of the system. This is a precautionary measure against userspace * trying to be a smarta** and attempting to change it up on us. */ devkmsg_log |= DEVKMSG_LOG_MASK_LOCK; @@ -1975,7 +1975,7 @@ int console_lock_spinning_disable_and_check(int cookie) * the current owner is running and cannot reschedule until it * is ready to lose the lock. * - * Return: 1 if we got the lock, 0 othrewise + * Return: 1 if we got the lock, 0 otherwise */ static int console_trylock_spinning(void) { @@ -3285,7 +3285,7 @@ static bool console_flush_one_record(bool do_cond_resched, u64 *next_seq, bool * continue; /* - * An usable console made a progress. There might still be + * A usable console made progress. There might still be * pending messages. */ *try_again = true; -- cgit v1.2.3 From 69efd863a78584b9416ed6be0e1e7349124b4a00 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Mon, 1 Jun 2026 13:07:46 -0400 Subject: tracing/eprobes: Allow use of BTF names to dereference pointers Add syntax to the parsing of eprobes to be able to typecast a trace event field that is a pointer to a structure. Currently, a dereference must be a number, where the user has to figure out manually the offset of a member of a structure that they want to dereference. But for event probes that records a field that happens to be a pointer to a structure, it cannot dereference these values with BTF naming, but must use numerical offsets. For example, to find out what device a sk_buff is pointing to in the net_dev_xmit trace event, one must first use gdb to find the offsets of the members of the structures: (gdb) p &((struct sk_buff *)0)->dev $1 = (struct net_device **) 0x10 (gdb) p &((struct net_device *)0)->name $2 = (char (*)[16]) 0x118 And then use the raw numbers to dereference: # echo 'e:xmit net.net_dev_xmit +0x118(+0x10($skbaddr)):string' >> dynamic_events If BTF is in the kernel, then instead, the skbaddr can be typecast to sk_buff and use the normal dereference logic. # echo 'e:xmit net.net_dev_xmit (sk_buff)skbaddr->dev->name:string' >> dynamic_events # echo 1 > events/eprobes/xmit/enable # cat trace [..] sshd-session-1022 [000] b..2. 860.249343: xmit: (net.net_dev_xmit) arg1="enp7s0" sshd-session-1022 [000] b..2. 860.250061: xmit: (net.net_dev_xmit) arg1="enp7s0" sshd-session-1022 [000] b..2. 860.250142: xmit: (net.net_dev_xmit) arg1="enp7s0" sshd-session-1022 [000] b..2. 860.263553: xmit: (net.net_dev_xmit) arg1="enp7s0" sshd-session-1022 [000] b..2. 860.283820: xmit: (net.net_dev_xmit) arg1="enp7s0" sshd-session-1022 [000] b..2. 860.302716: xmit: (net.net_dev_xmit) arg1="enp7s0" sshd-session-1022 [000] b..2. 860.322905: xmit: (net.net_dev_xmit) arg1="enp7s0" sshd-session-1022 [000] b..2. 860.342828: xmit: (net.net_dev_xmit) arg1="enp7s0" sshd-session-1022 [000] b..2. 860.362268: xmit: (net.net_dev_xmit) arg1="enp7s0" sshd-session-1022 [000] b..2. 860.382335: xmit: (net.net_dev_xmit) arg1="enp7s0" sshd-session-1022 [000] b..2. 860.400856: xmit: (net.net_dev_xmit) arg1="enp7s0" sshd-session-1022 [000] b..2. 860.419893: xmit: (net.net_dev_xmit) arg1="enp7s0" The syntax is simply: (STRUCT)(FIELD)->MEMBER[->MEMBER..] Also add comments around the #else and #endif of #ifdef CONFIG_PROBE_EVENTS_BTF_ARGS to know what they are for. Link: https://lore.kernel.org/all/20260601130746.2139d926@gandalf.local.home/ Signed-off-by: Steven Rostedt (Google) Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/trace_probe.c | 173 ++++++++++++++++++++++++++++++++++++++------- kernel/trace/trace_probe.h | 5 +- 2 files changed, 150 insertions(+), 28 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index 695310571b08..fd1caa1f9723 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -332,6 +332,23 @@ static int parse_trace_event_arg(char *arg, struct fetch_insn *code, return -ENOENT; } +static int parse_trace_event(char *arg, struct fetch_insn *code, + struct traceprobe_parse_context *ctx) +{ + int ret; + + if (code->data) + return -EFAULT; + ret = parse_trace_event_arg(arg, code, ctx); + if (!ret) + return 0; + if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) { + code->op = FETCH_OP_COMM; + return 0; + } + return -EINVAL; +} + #ifdef CONFIG_PROBE_EVENTS_BTF_ARGS static u32 btf_type_int(const struct btf_type *t) @@ -376,11 +393,16 @@ static bool btf_type_is_char_array(struct btf *btf, const struct btf_type *type) && BTF_INT_BITS(intdata) == 8; } +static struct btf *ctx_btf(struct traceprobe_parse_context *ctx) +{ + return ctx->struct_btf ? : ctx->btf; +} + static int check_prepare_btf_string_fetch(char *typename, struct fetch_insn **pcode, struct traceprobe_parse_context *ctx) { - struct btf *btf = ctx->btf; + struct btf *btf = ctx_btf(ctx); if (!btf || !ctx->last_type) return 0; @@ -506,6 +528,15 @@ static int query_btf_context(struct traceprobe_parse_context *ctx) return 0; } +static void clear_struct_btf(struct traceprobe_parse_context *ctx) +{ + if (ctx->struct_btf) { + btf_put(ctx->struct_btf); + ctx->struct_btf = NULL; + ctx->last_struct = NULL; + } +} + static void clear_btf_context(struct traceprobe_parse_context *ctx) { if (ctx->btf) { @@ -554,22 +585,29 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type, struct fetch_insn *code = *pcode; const struct btf_member *field; u32 bitoffs, anon_offs; + bool is_struct = ctx->struct_btf != NULL; + struct btf *btf = ctx_btf(ctx); char *next; int is_ptr; s32 tid; do { - /* Outer loop for solving arrow operator ('->') */ - if (BTF_INFO_KIND(type->info) != BTF_KIND_PTR) { - trace_probe_log_err(ctx->offset, NO_PTR_STRCT); - return -EINVAL; - } - /* Convert a struct pointer type to a struct type */ - type = btf_type_skip_modifiers(ctx->btf, type->type, &tid); - if (!type) { - trace_probe_log_err(ctx->offset, BAD_BTF_TID); - return -EINVAL; + if (!is_struct) { + /* Outer loop for solving arrow operator ('->') */ + if (BTF_INFO_KIND(type->info) != BTF_KIND_PTR) { + trace_probe_log_err(ctx->offset, NO_PTR_STRCT); + return -EINVAL; + } + + /* Convert a struct pointer type to a struct type */ + type = btf_type_skip_modifiers(btf, type->type, &tid); + if (!type) { + trace_probe_log_err(ctx->offset, BAD_BTF_TID); + return -EINVAL; + } } + /* Only the first type can skip being a pointer */ + is_struct = false; bitoffs = 0; do { @@ -580,7 +618,7 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type, return is_ptr; anon_offs = 0; - field = btf_find_struct_member(ctx->btf, type, fieldname, + field = btf_find_struct_member(btf, type, fieldname, &anon_offs); if (IS_ERR(field)) { trace_probe_log_err(ctx->offset, BAD_BTF_TID); @@ -602,7 +640,7 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type, ctx->last_bitsize = 0; } - type = btf_type_skip_modifiers(ctx->btf, field->type, &tid); + type = btf_type_skip_modifiers(btf, field->type, &tid); if (!type) { trace_probe_log_err(ctx->offset, BAD_BTF_TID); return -EINVAL; @@ -640,7 +678,7 @@ static int parse_btf_arg(char *varname, int i, is_ptr, ret; u32 tid; - if (WARN_ON_ONCE(!ctx->funcname)) + if (WARN_ON_ONCE(!ctx->funcname && !(ctx->flags & TPARG_FL_TEVENT))) return -EINVAL; is_ptr = split_next_field(varname, &field, ctx); @@ -653,6 +691,19 @@ static int parse_btf_arg(char *varname, return -EOPNOTSUPP; } + if (ctx->flags & TPARG_FL_TEVENT) { + ret = parse_trace_event(varname, code, ctx); + if (ret < 0) { + trace_probe_log_err(ctx->offset, BAD_ATTACH_ARG); + return ret; + } + /* TEVENT is only here via a typecast */ + if (WARN_ON_ONCE(ctx->struct_btf == NULL)) + return -EINVAL; + type = ctx->last_struct; + goto found_type; + } + if (ctx->flags & TPARG_FL_RETURN && !strcmp(varname, "$retval")) { code->op = FETCH_OP_RETVAL; /* Check whether the function return type is not void */ @@ -709,6 +760,7 @@ static int parse_btf_arg(char *varname, found: type = btf_type_skip_modifiers(ctx->btf, tid, &tid); +found_type: if (!type) { trace_probe_log_err(ctx->offset, BAD_BTF_TID); return -EINVAL; @@ -727,7 +779,7 @@ found: static const struct fetch_type *find_fetch_type_from_btf_type( struct traceprobe_parse_context *ctx) { - struct btf *btf = ctx->btf; + struct btf *btf = ctx_btf(ctx); const char *typestr = NULL; if (btf && ctx->last_type) @@ -758,7 +810,67 @@ static int parse_btf_bitfield(struct fetch_insn **pcode, return 0; } -#else +static int query_btf_struct(const char *sname, struct traceprobe_parse_context *ctx) +{ + struct btf *btf = NULL; + int id; + + /* A struct_btf should only be used by a single argument */ + if (WARN_ON_ONCE(ctx->struct_btf)) { + btf_put(ctx->struct_btf); + ctx->struct_btf = NULL; + } + + id = bpf_find_btf_id(sname, BTF_KIND_STRUCT, &btf); + if (id < 0) + return id; + ctx->struct_btf = btf; + ctx->last_struct = btf_type_by_id(ctx->struct_btf, id); + return 0; +} + +static int handle_typecast(char *arg, struct fetch_insn **pcode, + struct fetch_insn *end, + struct traceprobe_parse_context *ctx) +{ + char *tmp; + int ret; + + /* Currently this only works for eprobes */ + if (!(ctx->flags & TPARG_FL_TEVENT)) { + trace_probe_log_err(ctx->offset, TYPECAST_NOT_EVENT); + return -EINVAL; + } + + tmp = strchr(arg, ')'); + if (!tmp) { + trace_probe_log_err(ctx->offset + strlen(arg), + DEREF_OPEN_BRACE); + return -EINVAL; + } + *tmp = '\0'; + ret = query_btf_struct(arg + 1, ctx); + *tmp = ')'; + + if (ret < 0) { + trace_probe_log_err(ctx->offset + 1, NO_PTR_STRCT); + return -EINVAL; + } + + tmp++; + + ctx->offset += tmp - arg; + ret = parse_btf_arg(tmp, pcode, end, ctx); + return ret; +} + +#else /* !CONFIG_PROBE_EVENTS_BTF_ARGS */ + +static void clear_struct_btf(struct traceprobe_parse_context *ctx) +{ + ctx->struct_btf = NULL; +} + static void clear_btf_context(struct traceprobe_parse_context *ctx) { ctx->btf = NULL; @@ -794,7 +906,15 @@ static int check_prepare_btf_string_fetch(char *typename, return 0; } -#endif +static int handle_typecast(char *arg, struct fetch_insn **pcode, + struct fetch_insn *end, + struct traceprobe_parse_context *ctx) +{ + trace_probe_log_err(ctx->offset, NOSUP_BTFARG); + return -EOPNOTSUPP; +} + +#endif /* CONFIG_PROBE_EVENTS_BTF_ARGS */ #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API @@ -948,16 +1068,9 @@ static int parse_probe_vars(char *orig_arg, const struct fetch_type *t, int len; if (ctx->flags & TPARG_FL_TEVENT) { - if (code->data) - return -EFAULT; - ret = parse_trace_event_arg(arg, code, ctx); - if (!ret) - return 0; - if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) { - code->op = FETCH_OP_COMM; - return 0; - } - goto inval; + if (parse_trace_event(arg, code, ctx) < 0) + goto inval; + return 0; } if (str_has_prefix(arg, "retval")) { @@ -1224,6 +1337,9 @@ parse_probe_arg(char *arg, const struct fetch_type *type, code->op = FETCH_OP_IMM; } break; + case '(': + ret = handle_typecast(arg, pcode, end, ctx); + break; default: if (isalpha(arg[0]) || arg[0] == '_') { /* BTF variable */ if (!tparg_is_function_entry(ctx->flags) && @@ -1556,6 +1672,9 @@ fail: } kfree(tmp); + /* struct_btf should not be passed to other arguments */ + clear_struct_btf(ctx); + return ret; } diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h index 1076f1df347b..15758cc11fc6 100644 --- a/kernel/trace/trace_probe.h +++ b/kernel/trace/trace_probe.h @@ -422,7 +422,9 @@ struct traceprobe_parse_context { const struct btf_param *params; /* Parameter of the function */ s32 nr_params; /* The number of the parameters */ struct btf *btf; /* The BTF to be used */ + struct btf *struct_btf; /* The BTF to be used for structs */ const struct btf_type *last_type; /* Saved type */ + const struct btf_type *last_struct; /* Saved structure */ u32 last_bitoffs; /* Saved bitoffs */ u32 last_bitsize; /* Saved bitsize */ struct trace_probe *tp; @@ -563,7 +565,8 @@ extern int traceprobe_define_arg_fields(struct trace_event_call *event_call, C(NEED_STRING_TYPE, "$comm and immediate-string only accepts string type"),\ C(TOO_MANY_ARGS, "Too many arguments are specified"), \ C(TOO_MANY_EARGS, "Too many entry arguments specified"), \ - C(EVENT_TOO_BIG, "Event too big (too many fields?)"), + C(EVENT_TOO_BIG, "Event too big (too many fields?)"), \ + C(TYPECAST_NOT_EVENT, "Typecasts are only for eprobe fields"), #undef C #define C(a, b) TP_ERR_##a -- cgit v1.2.3 From 3c56ee343f9412d81918635c3e25e22a5dd6d87e Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 2 Jun 2026 15:30:49 +0200 Subject: bpf: Reject exclusive maps for bpf_map_elem iterators Exclusive maps (aka excl_prog_hash) are meant to be reachable only from the single program whose hash matches. This is enforced by check_map_prog_compatibility() when the map is referenced from a program such as signed BPF loaders. A bpf_map_elem iterator, however, binds its target map at attach time in bpf_iter_attach_map() instead of referencing it from the program, so the exclusivity check is never reached. On top of that, the iterator exposes the map value as a writable buffer. Fixes: baefdbdf6812 ("bpf: Implement exclusive map creation") Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260602133052.423725-2-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- kernel/bpf/map_iter.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/map_iter.c b/kernel/bpf/map_iter.c index 261a03ea73d3..ae0741a09c6d 100644 --- a/kernel/bpf/map_iter.c +++ b/kernel/bpf/map_iter.c @@ -112,6 +112,10 @@ static int bpf_iter_attach_map(struct bpf_prog *prog, map = bpf_map_get_with_uref(linfo->map.map_fd); if (IS_ERR(map)) return PTR_ERR(map); + if (map->excl_prog_sha) { + err = -EPERM; + goto put_map; + } if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || -- cgit v1.2.3 From 002668809b068c528838c1ab1ff46c87bdbb095d Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 19 May 2026 21:01:28 +0200 Subject: rcu/nocb: reduce stack usage in nocb_gp_wait() When CONFIG_UBSAN_ALIGNMENT is enabled, the stack usage of nocb_gp_wait() grows above typical warning limits: In file included from kernel/rcu/tree.c:4930: kernel/rcu/tree_nocb.h: In function 'rcu_nocb_gp_kthread': kernel/rcu/tree_nocb.h:866:1: error: the frame size of 1968 bytes is larger than 1280 bytes [-Werror=frame-larger-than=] Apparently, the problem is passing rcu_data from a 'void *' pointer, which gcc assumes may be misaligned. When the function is not inlined into rcu_nocb_gp_kthread(), that is no longer visible to gcc. Add a 'noinline_for_stack' annotation that leads to skipping a lot of the alignment sanitizer checks and keeps the stack usage 60% lower here. Reviewed-by: Kunwu Chan Reviewed-by: Frederic Weisbecker Reviewed-by: Paul E. McKenney Signed-off-by: Arnd Bergmann Signed-off-by: Uladzislau Rezki (Sony) --- kernel/rcu/tree_nocb.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index 1047b30cd46b..373b877cf171 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -655,7 +655,7 @@ static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu) * No-CBs GP kthreads come here to wait for additional callbacks to show up * or for grace periods to end. */ -static void nocb_gp_wait(struct rcu_data *my_rdp) +static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) { bool bypass = false; int __maybe_unused cpu = my_rdp->cpu; -- cgit v1.2.3 From a99ce697ea5e27b867c9ba4ee55fa5ba3b8d1188 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 1 Jun 2026 08:56:04 -1000 Subject: cgroup: Migrate tasks to the root css when a controller is rebound cgroup_apply_control_disable() defers kill_css_finish() while a css is still populated, relying on css_update_populated() to fire the deferred kill once the populated count reaches zero. This deadlocks when a controller is rebound out of a hierarchy. Mounting an implicit_on_dfl controller such as perf_event as a v1 hierarchy steals it off the default hierarchy, and rebind_subsystems() kills its per-cgroup csses while they are still populated. The migration run in the same step keeps the old css for a controller no longer in the hierarchy's mask, so no task is migrated off the dying csses. Their populated count never reaches zero, the deferred kill_css_finish() never fires, and the next cgroup_lock_and_drain_offline() hangs forever under cgroup_mutex. That migration is already a no-op pass over the rebound subtree. Add cgroup_rebind_ss_mask so find_existing_css_set() resolves the leaving controllers to the root css. Their tasks are migrated there, the per-cgroup csses depopulate, and cgroup_apply_control_disable() kills them synchronously. The deferral stays correct for the rmdir and controller-disable paths it was meant for. Fixes: 1dffd95575eb ("cgroup: Defer kill_css_finish() in cgroup_apply_control_disable()") Reported-by: Mark Brown Closes: https://lore.kernel.org/all/41cd159c-54e5-45e0-81df-eaf36a6c028e@sirena.org.uk/ Reported-by: Bert Karwatzki Closes: https://lore.kernel.org/all/4e986b4ed7e16547805d54b6e67d09120bc4d2f2.camel@web.de/ Tested-by: Mark Brown Tested-by: Bert Karwatzki Signed-off-by: Tejun Heo --- kernel/cgroup/cgroup.c | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 6e92791d279e..4d442c224bf5 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -197,6 +197,14 @@ static u32 cgrp_dfl_implicit_ss_mask; /* some controllers can be threaded on the default hierarchy */ static u32 cgrp_dfl_threaded_ss_mask; +/* + * Set across rebind_subsystems() to the controllers leaving a hierarchy. + * Guarded by cgroup_mutex. Makes find_existing_css_set() resolve them to the + * root css so the affected tasks are migrated there before + * cgroup_apply_control_disable() kills the per-cgroup csses. + */ +static u32 cgroup_rebind_ss_mask; + /* The list of hierarchy roots */ LIST_HEAD(cgroup_roots); static int cgroup_root_count; @@ -1083,7 +1091,15 @@ static struct css_set *find_existing_css_set(struct css_set *old_cset, * won't change, so no need for locking. */ for_each_subsys(ss, i) { - if (root->subsys_mask & (1UL << i)) { + if (unlikely(cgroup_rebind_ss_mask & (1UL << i))) { + /* + * @ss is leaving this hierarchy and its per-cgroup + * csses are about to be killed. Resolve to the + * surviving root css so the tasks are migrated there. + */ + template[i] = cgroup_css(&root->cgrp, ss); + WARN_ON_ONCE(!template[i]); + } else if (root->subsys_mask & (1UL << i)) { /* * @ss is in this hierarchy, so we want the * effective css from @cgrp. @@ -1853,11 +1869,17 @@ int rebind_subsystems(struct cgroup_root *dst_root, u32 ss_mask) struct cgroup *scgrp = &cgrp_dfl_root.cgrp; /* - * Controllers from default hierarchy that need to be rebound - * are all disabled together in one go. + * Controllers leaving the default hierarchy are disabled + * together. cgroup_rebind_ss_mask makes cgroup_apply_control() + * migrate their tasks to the root css, so the per-cgroup csses + * are unpopulated when cgroup_finalize_control() kills them. + * Clear it before cgroup_finalize_control(), which does no + * css_set lookup. */ cgrp_dfl_root.subsys_mask &= ~dfl_disable_ss_mask; + cgroup_rebind_ss_mask = dfl_disable_ss_mask; WARN_ON(cgroup_apply_control(scgrp)); + cgroup_rebind_ss_mask = 0; cgroup_finalize_control(scgrp, 0); } @@ -1871,9 +1893,14 @@ int rebind_subsystems(struct cgroup_root *dst_root, u32 ss_mask) WARN_ON(!css || cgroup_css(dcgrp, ss)); if (src_root != &cgrp_dfl_root) { - /* disable from the source */ + /* + * Disable from the source, migrating its tasks to the + * root css first (see cgroup_rebind_ss_mask). + */ src_root->subsys_mask &= ~(1 << ssid); + cgroup_rebind_ss_mask = 1 << ssid; WARN_ON(cgroup_apply_control(scgrp)); + cgroup_rebind_ss_mask = 0; cgroup_finalize_control(scgrp, 0); } -- cgit v1.2.3 From 766e828b011ca5f971554001611b4acab7c244c1 Mon Sep 17 00:00:00 2001 From: Maoyi Xie Date: Thu, 28 May 2026 14:33:10 +0800 Subject: time/namespace: Export init_time_ns and do_timens_ktime_to_host() timens_ktime_to_host() in compares the current time namespace against init_time_ns for the fast path. It calls do_timens_ktime_to_host() for the offset case. Both symbols are needed at link time by any caller of the inline. All current callers are builtin, but ntsync can be built as module, which prevents it from using it. Export both with EXPORT_SYMBOL_GPL. Signed-off-by: Maoyi Xie Signed-off-by: Thomas Gleixner Link: https://patch.msgid.link/20260528063311.3300393-2-maoyixie.tju@gmail.com --- kernel/time/namespace.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel') diff --git a/kernel/time/namespace.c b/kernel/time/namespace.c index 4bca3f78c8ea..5fa0af66cf3f 100644 --- a/kernel/time/namespace.c +++ b/kernel/time/namespace.c @@ -57,6 +57,7 @@ ktime_t do_timens_ktime_to_host(clockid_t clockid, ktime_t tim, return tim; } +EXPORT_SYMBOL_GPL(do_timens_ktime_to_host); static struct ucounts *inc_time_namespaces(struct user_namespace *ns) { @@ -351,6 +352,7 @@ struct time_namespace init_time_ns = { .user_ns = &init_user_ns, .frozen_offsets = true, }; +EXPORT_SYMBOL_GPL(init_time_ns); void __init time_ns_init(void) { -- cgit v1.2.3 From c1ca14ca227e92101c7ae597213275b60f4212c6 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Chaudhary Date: Mon, 1 Jun 2026 09:27:46 +0530 Subject: clockevents: Fix duplicate type specifier in stub function parameter The stub for arch_inlined_clockevent_set_next_coupled() has 'u64 u64 cycles' in its parameter list. Since u64 is a typedef, the compiler parses the second 'u64' as the parameter name, making 'cycles' an unused token. Remove the duplicate so the parameter is correctly named. Fixes: 89f951a1e8ad ("clockevents: Provide support for clocksource coupled comparators") Signed-off-by: Naveen Kumar Chaudhary Signed-off-by: Thomas Gleixner Link: https://patch.msgid.link/7tostpvxzdn6tobmyow63a5rweatls5kux3scqp2vzhe7mv6uq@ecr746b4hyhf --- kernel/time/clockevents.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c index 5e22697b098d..0014d163f989 100644 --- a/kernel/time/clockevents.c +++ b/kernel/time/clockevents.c @@ -301,7 +301,7 @@ static int clockevents_program_min_delta(struct clock_event_device *dev) #include #else static __always_inline void -arch_inlined_clockevent_set_next_coupled(u64 u64 cycles, struct clock_event_device *dev) { } +arch_inlined_clockevent_set_next_coupled(u64 cycles, struct clock_event_device *dev) { } #endif static inline bool clockevent_set_next_coupled(struct clock_event_device *dev, ktime_t expires) -- cgit v1.2.3 From ce4abda5e12622f33450159e76c8f56d28d7f03d Mon Sep 17 00:00:00 2001 From: Naveen Kumar Chaudhary Date: Tue, 2 Jun 2026 23:37:37 +0530 Subject: time: Fix off-by-one in settimeofday() usec validation The validation check uses '>' instead of '>=' when comparing tv_usec against USEC_PER_SEC, allowing the value 1000000 through. After conversion to nanoseconds (*= 1000), this produces tv_nsec == NSEC_PER_SEC, violating the timespec invariant that tv_nsec must be less than NSEC_PER_SEC. Use '>=' to reject tv_usec values that are not in the valid range of 0 to 999999. Fixes: 5e0fb1b57bea ("y2038: time: avoid timespec usage in settimeofday()") Signed-off-by: Naveen Kumar Chaudhary Signed-off-by: Thomas Gleixner Acked-by: John Stultz Link: https://patch.msgid.link/4rikk44zew3s6577dugmx4jyblz7o5c57niuap6ct3td5yfm6w@gh7pcumg7qor --- kernel/time/time.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/time/time.c b/kernel/time/time.c index 0d832317d576..771cef87ad3b 100644 --- a/kernel/time/time.c +++ b/kernel/time/time.c @@ -207,7 +207,7 @@ SYSCALL_DEFINE2(settimeofday, struct __kernel_old_timeval __user *, tv, get_user(new_ts.tv_nsec, &tv->tv_usec)) return -EFAULT; - if (new_ts.tv_nsec > USEC_PER_SEC || new_ts.tv_nsec < 0) + if (new_ts.tv_nsec >= USEC_PER_SEC || new_ts.tv_nsec < 0) return -EINVAL; new_ts.tv_nsec *= NSEC_PER_USEC; -- cgit v1.2.3 From 86db4084b4b5d1a074bcc66c108a4c9d266812d4 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 8 May 2026 15:16:33 +0200 Subject: tick/sched: Fix TOCTOU in nohz idle time fetch When the nohz idle time is fetched, the current clock timestamp is taken outside the seqcount, which can result in a race as reported by Sashiko: get_cpu_sleep_time_us() tick_nohz_start_idle() ----------------------- --------------------- now = ktime_get() write_seqcount_begin(idle_sleeptime_seq); idle_entrytime = ktime_get() tick_sched_flag_set(ts, TS_FLAG_IDLE_ACTIVE); write_seqcount_end(&ts->idle_sleeptime_seq); read_seqcount_begin(idle_sleeptime_seq) delta = now - idle_entrytime); //!! But now < idle_entrytime idle = *sleeptime + delta; read_seqcount_retry(&ts->idle_sleeptime_seq, seq) Here the read side fetches the timestamp before the write side and its update. As a result the time delta computed on the read side is negative (ktime_t is signed) and breaks the cputime monotonicity guarantee. This could possibly be fixed with reading the current clock timestamp inside the seqcount but the reader overhead might then increase. Also simply checking that the current timestamp is above the idle entry time is enough to prevent any issue of the like. Fixes: 620a30fa0bd1 ("timers/nohz: Protect idle/iowait sleep time under seqcount") Reported-by: Sashiko Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Link: https://patch.msgid.link/20260508131647.43868-2-frederic@kernel.org --- kernel/time/tick-sched.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index cbbb87a0c6e7..171393367b5c 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -797,15 +797,16 @@ static u64 get_cpu_sleep_time_us(struct tick_sched *ts, ktime_t *sleeptime, *last_update_time = ktime_to_us(now); do { + ktime_t delta = 0; + seq = read_seqcount_begin(&ts->idle_sleeptime_seq); if (tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE) && compute_delta) { - ktime_t delta = ktime_sub(now, ts->idle_entrytime); - - idle = ktime_add(*sleeptime, delta); - } else { - idle = *sleeptime; + if (now > ts->idle_entrytime) + delta = ktime_sub(now, ts->idle_entrytime); } + + idle = ktime_add(*sleeptime, delta); } while (read_seqcount_retry(&ts->idle_sleeptime_seq, seq)); return ktime_to_us(idle); -- cgit v1.2.3 From 0236aaf07b406100c8c3a6b78dba211f32449f49 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 8 May 2026 15:16:34 +0200 Subject: sched/idle: Handle offlining first in idle loop Offline handling happens from within the inner idle loop, after the beginning of dyntick cputime accounting, nohz idle load balancing and TIF_NEED_RESCHED polling. This is not necessary and even buggy because: * There is no dyntick handling to do. And calling tick_nohz_idle_enter() messes up with the struct tick_sched reset that was performed on tick_sched_timer_dying(). * There is no nohz idle balancing to do. * Polling on TIF_RESCHED is irrelevant at this stage, there are no more tasks allowed to run. * No need to check if need_resched() before offline handling since stop_machine is done and all per-cpu kthread should be done with their job. Therefore move the offline handling at the beginning of the idle loop. This will also ease the idle cputime unification later by not elapsing idle time while offline through the call to: tick_nohz_idle_enter() -> tick_nohz_start_idle() Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Tested-by: Shrikanth Hegde Reviewed-by: Rafael J. Wysocki (Intel) Reviewed-by: Shrikanth Hegde Link: https://patch.msgid.link/20260508131647.43868-3-frederic@kernel.org --- kernel/sched/idle.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index a83be0c834dd..aa7e3dc59856 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -280,6 +280,14 @@ static void do_idle(void) int cpu = smp_processor_id(); bool got_tick = false; + if (cpu_is_offline(cpu)) { + local_irq_disable(); + /* All per-CPU kernel threads should be done by now. */ + WARN_ON_ONCE(need_resched()); + cpuhp_report_idle_dead(); + arch_cpu_idle_dead(); + } + /* * Check if we need to update blocked load */ @@ -331,11 +339,6 @@ static void do_idle(void) */ local_irq_disable(); - if (cpu_is_offline(cpu)) { - cpuhp_report_idle_dead(); - arch_cpu_idle_dead(); - } - arch_cpu_idle_enter(); rcu_nocb_flush_deferred_wakeup(); -- cgit v1.2.3 From 080b5c6d95034e46f5ed1abe98c06218a1386aef Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 8 May 2026 15:16:35 +0200 Subject: sched/cputime: Remove superfluous and error prone kcpustat_field() parameter The first parameter to kcpustat_field() is a pointer to the cpu kcpustat to be fetched from. This parameter is error prone because a copy to a kcpustat could be passed by accident instead of the original one. Also the kcpustat structure can already be retrieved with the help of the mandatory CPU argument. Remove the needless parameter. Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Tested-by: Shrikanth Hegde Reviewed-by: Shrikanth Hegde Link: https://patch.msgid.link/20260508131647.43868-4-frederic@kernel.org --- kernel/rcu/tree.c | 9 +++------ kernel/rcu/tree_stall.h | 7 +++---- kernel/sched/cputime.c | 5 ++--- 3 files changed, 8 insertions(+), 13 deletions(-) (limited to 'kernel') diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 55df6d37145e..3cbf79bee976 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -969,14 +969,11 @@ static int rcu_watching_snap_recheck(struct rcu_data *rdp) if (rcu_cpu_stall_cputime && rdp->snap_record.gp_seq != rdp->gp_seq) { int cpu = rdp->cpu; struct rcu_snap_record *rsrp; - struct kernel_cpustat *kcsp; - - kcsp = &kcpustat_cpu(cpu); rsrp = &rdp->snap_record; - rsrp->cputime_irq = kcpustat_field(kcsp, CPUTIME_IRQ, cpu); - rsrp->cputime_softirq = kcpustat_field(kcsp, CPUTIME_SOFTIRQ, cpu); - rsrp->cputime_system = kcpustat_field(kcsp, CPUTIME_SYSTEM, cpu); + rsrp->cputime_irq = kcpustat_field(CPUTIME_IRQ, cpu); + rsrp->cputime_softirq = kcpustat_field(CPUTIME_SOFTIRQ, cpu); + rsrp->cputime_system = kcpustat_field(CPUTIME_SYSTEM, cpu); rsrp->nr_hardirqs = kstat_cpu_irqs_sum(cpu) + arch_irq_stat_cpu(cpu); rsrp->nr_softirqs = kstat_cpu_softirqs_sum(cpu); rsrp->nr_csw = nr_context_switches_cpu(cpu); diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index b67532cb8770..cf7ae51cba40 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -479,7 +479,6 @@ static void print_cpu_stat_info(int cpu) { struct rcu_snap_record rsr, *rsrp; struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); - struct kernel_cpustat *kcsp = &kcpustat_cpu(cpu); if (!rcu_cpu_stall_cputime) return; @@ -488,9 +487,9 @@ static void print_cpu_stat_info(int cpu) if (rsrp->gp_seq != rdp->gp_seq) return; - rsr.cputime_irq = kcpustat_field(kcsp, CPUTIME_IRQ, cpu); - rsr.cputime_softirq = kcpustat_field(kcsp, CPUTIME_SOFTIRQ, cpu); - rsr.cputime_system = kcpustat_field(kcsp, CPUTIME_SYSTEM, cpu); + rsr.cputime_irq = kcpustat_field(CPUTIME_IRQ, cpu); + rsr.cputime_softirq = kcpustat_field(CPUTIME_SOFTIRQ, cpu); + rsr.cputime_system = kcpustat_field(CPUTIME_SYSTEM, cpu); pr_err("\t hardirqs softirqs csw/system\n"); pr_err("\t number: %8lld %10d %12lld\n", diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index fbf31db0d2f3..caaaf0a04ced 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -975,10 +975,9 @@ static int kcpustat_field_vtime(u64 *cpustat, return 0; } -u64 kcpustat_field(struct kernel_cpustat *kcpustat, - enum cpu_usage_stat usage, int cpu) +u64 kcpustat_field(enum cpu_usage_stat usage, int cpu) { - u64 *cpustat = kcpustat->cpustat; + u64 *cpustat = kcpustat_cpu(cpu).cpustat; u64 val = cpustat[usage]; struct rq *rq; int err; -- cgit v1.2.3 From 650a59805a9baeff76379ea9309df1395eb15a46 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 8 May 2026 15:16:36 +0200 Subject: sched/cputime: Correctly support generic vtime idle time Currently whether generic vtime is running or not, the idle cputime is fetched from the nohz accounting. However generic vtime already does its own idle cputime accounting. Only the kernel stat accessors are not plugged to support it. Read the idle generic vtime cputime when it's running, this will allow to later more clearly split nohz and vtime cputime accounting. Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Tested-by: Shrikanth Hegde Link: https://patch.msgid.link/20260508131647.43868-5-frederic@kernel.org --- kernel/sched/cputime.c | 38 +++++++++++++++++++++++++++++--------- kernel/time/tick-sched.c | 12 +++++++++--- 2 files changed, 38 insertions(+), 12 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index caaaf0a04ced..815d0f772cae 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -773,9 +773,9 @@ void vtime_guest_exit(struct task_struct *tsk) } EXPORT_SYMBOL_GPL(vtime_guest_exit); -void vtime_account_idle(struct task_struct *tsk) +static void __vtime_account_idle(struct vtime *vtime) { - account_idle_time(get_vtime_delta(&tsk->vtime)); + account_idle_time(get_vtime_delta(vtime)); } void vtime_task_switch_generic(struct task_struct *prev) @@ -784,7 +784,7 @@ void vtime_task_switch_generic(struct task_struct *prev) write_seqcount_begin(&vtime->seqcount); if (vtime->state == VTIME_IDLE) - vtime_account_idle(prev); + __vtime_account_idle(vtime); else __vtime_account_kernel(prev, vtime); vtime->state = VTIME_INACTIVE; @@ -926,6 +926,7 @@ static int kcpustat_field_vtime(u64 *cpustat, int cpu, u64 *val) { struct vtime *vtime = &tsk->vtime; + struct rq *rq = cpu_rq(cpu); unsigned int seq; do { @@ -967,6 +968,14 @@ static int kcpustat_field_vtime(u64 *cpustat, if (state == VTIME_GUEST && task_nice(tsk) > 0) *val += vtime->gtime + vtime_delta(vtime); break; + case CPUTIME_IDLE: + if (state == VTIME_IDLE && !atomic_read(&rq->nr_iowait)) + *val += vtime_delta(vtime); + break; + case CPUTIME_IOWAIT: + if (state == VTIME_IDLE && atomic_read(&rq->nr_iowait) > 0) + *val += vtime_delta(vtime); + break; default: break; } @@ -1029,8 +1038,8 @@ static int kcpustat_cpu_fetch_vtime(struct kernel_cpustat *dst, *dst = *src; cpustat = dst->cpustat; - /* Task is sleeping, dead or idle, nothing to add */ - if (state < VTIME_SYS) + /* Task is sleeping or dead, nothing to add */ + if (state < VTIME_IDLE) continue; delta = vtime_delta(vtime); @@ -1039,15 +1048,17 @@ static int kcpustat_cpu_fetch_vtime(struct kernel_cpustat *dst, * Task runs either in user (including guest) or kernel space, * add pending nohz time to the right place. */ - if (state == VTIME_SYS) { + switch (state) { + case VTIME_SYS: cpustat[CPUTIME_SYSTEM] += vtime->stime + delta; - } else if (state == VTIME_USER) { + break; + case VTIME_USER: if (task_nice(tsk) > 0) cpustat[CPUTIME_NICE] += vtime->utime + delta; else cpustat[CPUTIME_USER] += vtime->utime + delta; - } else { - WARN_ON_ONCE(state != VTIME_GUEST); + break; + case VTIME_GUEST: if (task_nice(tsk) > 0) { cpustat[CPUTIME_GUEST_NICE] += vtime->gtime + delta; cpustat[CPUTIME_NICE] += vtime->gtime + delta; @@ -1055,6 +1066,15 @@ static int kcpustat_cpu_fetch_vtime(struct kernel_cpustat *dst, cpustat[CPUTIME_GUEST] += vtime->gtime + delta; cpustat[CPUTIME_USER] += vtime->gtime + delta; } + break; + case VTIME_IDLE: + if (atomic_read(&cpu_rq(cpu)->nr_iowait) > 0) + cpustat[CPUTIME_IOWAIT] += delta; + else + cpustat[CPUTIME_IDLE] += delta; + break; + default: + WARN_ON_ONCE(1); } } while (read_seqcount_retry(&vtime->seqcount, seq)); diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 171393367b5c..597c3a0682e7 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -783,9 +783,10 @@ static void tick_nohz_start_idle(struct tick_sched *ts) sched_clock_idle_sleep_event(); } -static u64 get_cpu_sleep_time_us(struct tick_sched *ts, ktime_t *sleeptime, +static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, ktime_t *sleeptime, bool compute_delta, u64 *last_update_time) { + struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); ktime_t now, idle; unsigned int seq; @@ -796,6 +797,11 @@ static u64 get_cpu_sleep_time_us(struct tick_sched *ts, ktime_t *sleeptime, if (last_update_time) *last_update_time = ktime_to_us(now); + if (vtime_generic_enabled_cpu(cpu)) { + idle = kcpustat_field(idx, cpu); + return ktime_to_us(idle); + } + do { ktime_t delta = 0; @@ -834,7 +840,7 @@ u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time) { struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); - return get_cpu_sleep_time_us(ts, &ts->idle_sleeptime, + return get_cpu_sleep_time_us(cpu, CPUTIME_IDLE, &ts->idle_sleeptime, !nr_iowait_cpu(cpu), last_update_time); } EXPORT_SYMBOL_GPL(get_cpu_idle_time_us); @@ -860,7 +866,7 @@ u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time) { struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); - return get_cpu_sleep_time_us(ts, &ts->iowait_sleeptime, + return get_cpu_sleep_time_us(cpu, CPUTIME_IOWAIT, &ts->iowait_sleeptime, nr_iowait_cpu(cpu), last_update_time); } EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us); -- cgit v1.2.3 From cf6444c3e1bb7dd5974441bbd74840e9821d36f9 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 8 May 2026 15:16:39 +0200 Subject: tick/sched: Unify idle cputime accounting The non-vtime dynticks-idle cputime accounting is a big mess that accumulates within two concurrent statistics, each having their own shortcomings: * The accounting for online CPUs which is based on the delta between tick_nohz_start_idle() and tick_nohz_stop_idle(). Pros: - Works when the tick is off - Has nsecs granularity Cons: - Account idle steal time but doesn't substract it from idle cputime. - Assumes CONFIG_IRQ_TIME_ACCOUNTING by not accounting IRQs but the IRQ time is simply ignored when CONFIG_IRQ_TIME_ACCOUNTING=n - The windows between 1) idle task scheduling and the first call to tick_nohz_start_idle() and 2) idle task between the last tick_nohz_stop_idle() and the rest of the idle time are blindspots wrt. cputime accounting (though mostly insignificant amount) - Relies on private fields outside of kernel stats, with specific accessors. * The accounting for offline CPUs which is based on ticks and the jiffies delta during which the tick was stopped. Pros: - Handles steal time correctly - Handle CONFIG_IRQ_TIME_ACCOUNTING=y and CONFIG_IRQ_TIME_ACCOUNTING=n correctly. - Handles the whole idle task - Accounts directly to kernel stats, without midlayer accumulator. Cons: - Doesn't elapse when the tick is off, which doesn't make it suitable for online CPUs. - Has TICK_NSEC granularity (jiffies) - Needs to track the dyntick-idle ticks that were accounted and substract them from the total jiffies time spent while the tick was stopped. This is an ugly workaround. Having two different accounting for a single context is not the only problem: since those accountings are of different natures, it is possible to observe the global idle time going backward after a CPU goes offline. Clean up the situation with introducing a hybrid approach that stays coherent and works for both online and offline CPUs: * Tick based or native vtime accounting operate before the idle loop is entered and resume once the idle loop prepares to exit. * When the idle loop starts, switch to dynticks-idle accounting as is done currently, except that the statistics accumulate directly to the relevant kernel stat fields. * Private dyntick cputime accounting fields are removed. * Works on both online and offline case. Further improvement will include: * Only switch to dynticks-idle cputime accounting when the tick actually goes in dynticks mode. * Handle CONFIG_IRQ_TIME_ACCOUNTING=n correctly such that the dynticks-idle accounting still elapses while on IRQs. * Correctly substract idle steal cputime from idle time Reported-by: Xin Zhao Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Tested-by: Shrikanth Hegde Link: https://patch.msgid.link/20260508131647.43868-8-frederic@kernel.org --- kernel/sched/cputime.c | 62 +++++++++++++++++++++---------------------- kernel/time/tick-sched.c | 69 +++++++++++++----------------------------------- 2 files changed, 49 insertions(+), 82 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 815d0f772cae..a5733789e0bd 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -414,16 +414,30 @@ static void irqtime_account_process_tick(struct task_struct *p, int user_tick, } } -static void irqtime_account_idle_ticks(int ticks) -{ - irqtime_account_process_tick(current, 0, ticks); -} #else /* !CONFIG_IRQ_TIME_ACCOUNTING: */ -static inline void irqtime_account_idle_ticks(int ticks) { } static inline void irqtime_account_process_tick(struct task_struct *p, int user_tick, int nr_ticks) { } #endif /* !CONFIG_IRQ_TIME_ACCOUNTING */ +#ifdef CONFIG_NO_HZ_COMMON +void kcpustat_dyntick_start(void) +{ + if (!vtime_generic_enabled_this_cpu()) { + vtime_dyntick_start(); + __this_cpu_write(kernel_cpustat.idle_dyntick, 1); + } +} + +void kcpustat_dyntick_stop(void) +{ + if (!vtime_generic_enabled_this_cpu()) { + __this_cpu_write(kernel_cpustat.idle_dyntick, 0); + vtime_dyntick_stop(); + steal_account_process_time(ULONG_MAX); + } +} +#endif /* CONFIG_NO_HZ_COMMON */ + /* * Use precise platform statistics if available: */ @@ -437,11 +451,15 @@ void vtime_account_irq(struct task_struct *tsk, unsigned int offset) vtime_account_hardirq(tsk); } else if (pc & SOFTIRQ_OFFSET) { vtime_account_softirq(tsk); - } else if (!IS_ENABLED(CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE) && - is_idle_task(tsk)) { - vtime_account_idle(tsk); + } else if (!kcpustat_idle_dyntick()) { + if (!IS_ENABLED(CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE) && + is_idle_task(tsk)) { + vtime_account_idle(tsk); + } else { + vtime_account_kernel(tsk); + } } else { - vtime_account_kernel(tsk); + vtime_reset(); } } @@ -483,6 +501,9 @@ void account_process_tick(struct task_struct *p, int user_tick) if (vtime_accounting_enabled_this_cpu()) return; + if (kcpustat_idle_dyntick()) + return; + if (irqtime_enabled()) { irqtime_account_process_tick(p, user_tick, 1); return; @@ -504,29 +525,6 @@ void account_process_tick(struct task_struct *p, int user_tick) account_idle_time(cputime); } -/* - * Account multiple ticks of idle time. - * @ticks: number of stolen ticks - */ -void account_idle_ticks(unsigned long ticks) -{ - u64 cputime, steal; - - if (irqtime_enabled()) { - irqtime_account_idle_ticks(ticks); - return; - } - - cputime = ticks * TICK_NSEC; - steal = steal_account_process_time(ULONG_MAX); - - if (steal >= cputime) - return; - - cputime -= steal; - account_idle_time(cputime); -} - /* * Adjust tick based cputime random precision against scheduler runtime * accounting. diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 597c3a0682e7..c3efd3583cf9 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -285,8 +285,6 @@ static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs) if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && tick_sched_flag_test(ts, TS_FLAG_STOPPED)) { touch_softlockup_watchdog_sched(); - if (is_idle_task(current)) - ts->idle_jiffies++; /* * In case the current tick fired too early past its expected * expiration, make sure we don't bypass the next clock reprogramming @@ -753,8 +751,12 @@ static void tick_nohz_update_jiffies(ktime_t now) static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now) { + u64 *cpustat = kcpustat_this_cpu->cpustat; ktime_t delta; + if (vtime_generic_enabled_this_cpu()) + return; + if (WARN_ON_ONCE(!tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE))) return; @@ -762,9 +764,9 @@ static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now) write_seqcount_begin(&ts->idle_sleeptime_seq); if (nr_iowait_cpu(smp_processor_id()) > 0) - ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta); + cpustat[CPUTIME_IOWAIT] = ktime_add(cpustat[CPUTIME_IOWAIT], delta); else - ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta); + cpustat[CPUTIME_IDLE] = ktime_add(cpustat[CPUTIME_IDLE], delta); ts->idle_entrytime = now; tick_sched_flag_clear(ts, TS_FLAG_IDLE_ACTIVE); @@ -775,18 +777,21 @@ static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now) static void tick_nohz_start_idle(struct tick_sched *ts) { + if (vtime_generic_enabled_this_cpu()) + return; + write_seqcount_begin(&ts->idle_sleeptime_seq); ts->idle_entrytime = ktime_get(); tick_sched_flag_set(ts, TS_FLAG_IDLE_ACTIVE); write_seqcount_end(&ts->idle_sleeptime_seq); - sched_clock_idle_sleep_event(); } -static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, ktime_t *sleeptime, +static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, bool compute_delta, u64 *last_update_time) { struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); + u64 *cpustat = kcpustat_cpu(cpu).cpustat; ktime_t now, idle; unsigned int seq; @@ -812,7 +817,7 @@ static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, ktime_t *slee delta = ktime_sub(now, ts->idle_entrytime); } - idle = ktime_add(*sleeptime, delta); + idle = ktime_add(cpustat[idx], delta); } while (read_seqcount_retry(&ts->idle_sleeptime_seq, seq)); return ktime_to_us(idle); @@ -838,9 +843,7 @@ static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, ktime_t *slee */ u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time) { - struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); - - return get_cpu_sleep_time_us(cpu, CPUTIME_IDLE, &ts->idle_sleeptime, + return get_cpu_sleep_time_us(cpu, CPUTIME_IDLE, !nr_iowait_cpu(cpu), last_update_time); } EXPORT_SYMBOL_GPL(get_cpu_idle_time_us); @@ -864,9 +867,7 @@ EXPORT_SYMBOL_GPL(get_cpu_idle_time_us); */ u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time) { - struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); - - return get_cpu_sleep_time_us(cpu, CPUTIME_IOWAIT, &ts->iowait_sleeptime, + return get_cpu_sleep_time_us(cpu, CPUTIME_IOWAIT, nr_iowait_cpu(cpu), last_update_time); } EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us); @@ -1279,10 +1280,8 @@ void tick_nohz_idle_stop_tick(void) ts->idle_sleeps++; ts->idle_expires = expires; - if (!was_stopped && tick_sched_flag_test(ts, TS_FLAG_STOPPED)) { - ts->idle_jiffies = ts->last_jiffies; + if (!was_stopped && tick_sched_flag_test(ts, TS_FLAG_STOPPED)) nohz_balance_enter_idle(cpu); - } } else { tick_nohz_retain_tick(ts); } @@ -1311,6 +1310,7 @@ void tick_nohz_idle_enter(void) WARN_ON_ONCE(ts->timer_expires_base); tick_sched_flag_set(ts, TS_FLAG_INIDLE); + kcpustat_dyntick_start(); tick_nohz_start_idle(ts); local_irq_enable(); @@ -1436,37 +1436,12 @@ unsigned long tick_nohz_get_idle_calls_cpu(int cpu) return ts->idle_calls; } -static void tick_nohz_account_idle_time(struct tick_sched *ts, - ktime_t now) -{ - unsigned long ticks; - - ts->idle_exittime = now; - - if (vtime_accounting_enabled_this_cpu()) - return; - /* - * We stopped the tick in idle. update_process_times() would miss the - * time we slept, as it does only a 1 tick accounting. - * Enforce that this is accounted to idle ! - */ - ticks = jiffies - ts->idle_jiffies; - /* - * We might be one off. Do not randomly account a huge number of ticks! - */ - if (ticks && ticks < LONG_MAX) - account_idle_ticks(ticks); -} - void tick_nohz_idle_restart_tick(void) { struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); - if (tick_sched_flag_test(ts, TS_FLAG_STOPPED)) { - ktime_t now = ktime_get(); - tick_nohz_restart_sched_tick(ts, now); - tick_nohz_account_idle_time(ts, now); - } + if (tick_sched_flag_test(ts, TS_FLAG_STOPPED)) + tick_nohz_restart_sched_tick(ts, ktime_get()); } static void tick_nohz_idle_update_tick(struct tick_sched *ts, ktime_t now) @@ -1475,8 +1450,6 @@ static void tick_nohz_idle_update_tick(struct tick_sched *ts, ktime_t now) __tick_nohz_full_update_tick(ts, now); else tick_nohz_restart_sched_tick(ts, now); - - tick_nohz_account_idle_time(ts, now); } /** @@ -1518,6 +1491,7 @@ void tick_nohz_idle_exit(void) if (tick_stopped) tick_nohz_idle_update_tick(ts, now); + kcpustat_dyntick_stop(); local_irq_enable(); } @@ -1655,20 +1629,15 @@ void tick_setup_sched_timer(bool hrtimer) void tick_sched_timer_dying(int cpu) { struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); - ktime_t idle_sleeptime, iowait_sleeptime; unsigned long idle_calls, idle_sleeps; /* This must happen before hrtimers are migrated! */ if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES)) hrtimer_cancel(&ts->sched_timer); - idle_sleeptime = ts->idle_sleeptime; - iowait_sleeptime = ts->iowait_sleeptime; idle_calls = ts->idle_calls; idle_sleeps = ts->idle_sleeps; memset(ts, 0, sizeof(*ts)); - ts->idle_sleeptime = idle_sleeptime; - ts->iowait_sleeptime = iowait_sleeptime; ts->idle_calls = idle_calls; ts->idle_sleeps = idle_sleeps; } -- cgit v1.2.3 From bd0c77cd46c63d02bc33dacdba56133ec1fe44a0 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 8 May 2026 15:16:40 +0200 Subject: tick/sched: Remove nohz disabled special case in cputime fetch Even when nohz is not runtime enabled, the dynticks idle cputime accounting can run and the common idle cputime accessors are still relevant. Remove the nohz disabled special case accordingly. Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Tested-by: Shrikanth Hegde Link: https://patch.msgid.link/20260508131647.43868-9-frederic@kernel.org --- kernel/time/tick-sched.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index c3efd3583cf9..cb235ec7d2d6 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -795,9 +795,6 @@ static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, ktime_t now, idle; unsigned int seq; - if (!tick_nohz_active) - return -1; - now = ktime_get(); if (last_update_time) *last_update_time = ktime_to_us(now); @@ -839,7 +836,7 @@ static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, * This time is measured via accounting rather than sampling, * and is as accurate as ktime_get() is. * - * Return: -1 if NOHZ is not enabled, else total idle time of the @cpu + * Return: -1 if generic vtime is enabled, else total idle time of the @cpu */ u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time) { @@ -863,7 +860,7 @@ EXPORT_SYMBOL_GPL(get_cpu_idle_time_us); * This time is measured via accounting rather than sampling, * and is as accurate as ktime_get() is. * - * Return: -1 if NOHZ is not enabled, else total iowait time of @cpu + * Return: -1 if generic vtime is enabled, else total iowait time of @cpu */ u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time) { -- cgit v1.2.3 From a5fe724e206ec7ff3ceb15b285d94316c7fe6c41 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 8 May 2026 15:16:41 +0200 Subject: tick/sched: Move dyntick-idle cputime accounting to cputime code Although the dynticks-idle cputime accounting is necessarily tied to the tick subsystem, the actual related accounting code has no business residing there and should be part of the scheduler cputime code. Move away the relevant pieces and state machine to where they belong. Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Tested-by: Shrikanth Hegde Link: https://patch.msgid.link/20260508131647.43868-10-frederic@kernel.org --- kernel/sched/core.c | 6 +- kernel/sched/cputime.c | 148 ++++++++++++++++++++++++++++++++++++++++-- kernel/time/tick-sched.c | 163 ++++++++++------------------------------------- 3 files changed, 178 insertions(+), 139 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index b8871449d3c6..d797d6696c58 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5518,7 +5518,11 @@ void sched_exec(void) } DEFINE_PER_CPU(struct kernel_stat, kstat); -DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat); +DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat) = { +#ifdef CONFIG_NO_HZ_COMMON + .idle_sleeptime_seq = SEQCNT_ZERO(kernel_cpustat.idle_sleeptime_seq) +#endif +}; EXPORT_PER_CPU_SYMBOL(kstat); EXPORT_PER_CPU_SYMBOL(kernel_cpustat); diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index a5733789e0bd..4c00163b74b9 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -2,6 +2,7 @@ /* * Simple CPU accounting cgroup controller */ +#include #include #include #include "sched.h" @@ -420,22 +421,155 @@ static inline void irqtime_account_process_tick(struct task_struct *p, int user_ #endif /* !CONFIG_IRQ_TIME_ACCOUNTING */ #ifdef CONFIG_NO_HZ_COMMON -void kcpustat_dyntick_start(void) +static void kcpustat_idle_stop(struct kernel_cpustat *kc, u64 now) { - if (!vtime_generic_enabled_this_cpu()) { - vtime_dyntick_start(); - __this_cpu_write(kernel_cpustat.idle_dyntick, 1); - } + u64 *cpustat = kc->cpustat; + u64 delta; + + if (!kc->idle_elapse) + return; + + delta = now - kc->idle_entrytime; + + write_seqcount_begin(&kc->idle_sleeptime_seq); + if (nr_iowait_cpu(smp_processor_id()) > 0) + cpustat[CPUTIME_IOWAIT] += delta; + else + cpustat[CPUTIME_IDLE] += delta; + + kc->idle_entrytime = now; + kc->idle_elapse = false; + write_seqcount_end(&kc->idle_sleeptime_seq); } -void kcpustat_dyntick_stop(void) +static void kcpustat_idle_start(struct kernel_cpustat *kc, u64 now) { + write_seqcount_begin(&kc->idle_sleeptime_seq); + kc->idle_entrytime = now; + kc->idle_elapse = true; + write_seqcount_end(&kc->idle_sleeptime_seq); +} + +void kcpustat_dyntick_stop(u64 now) +{ + struct kernel_cpustat *kc = kcpustat_this_cpu; + if (!vtime_generic_enabled_this_cpu()) { - __this_cpu_write(kernel_cpustat.idle_dyntick, 0); + WARN_ON_ONCE(!kc->idle_dyntick); + kcpustat_idle_stop(kc, now); + kc->idle_dyntick = false; vtime_dyntick_stop(); steal_account_process_time(ULONG_MAX); } } + +void kcpustat_dyntick_start(u64 now) +{ + struct kernel_cpustat *kc = kcpustat_this_cpu; + + if (!vtime_generic_enabled_this_cpu()) { + vtime_dyntick_start(); + kc->idle_dyntick = true; + kcpustat_idle_start(kc, now); + } +} + +void kcpustat_irq_enter(u64 now) +{ + struct kernel_cpustat *kc = kcpustat_this_cpu; + + if (!vtime_generic_enabled_this_cpu()) + kcpustat_idle_stop(kc, now); +} + +void kcpustat_irq_exit(u64 now) +{ + struct kernel_cpustat *kc = kcpustat_this_cpu; + + if (!vtime_generic_enabled_this_cpu()) + kcpustat_idle_start(kc, now); +} + +static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, + bool compute_delta, u64 *last_update_time) +{ + struct kernel_cpustat *kc = &kcpustat_cpu(cpu); + u64 *cpustat = kc->cpustat; + unsigned int seq; + ktime_t now; + u64 idle; + + now = ktime_get(); + if (last_update_time) + *last_update_time = ktime_to_us(now); + + if (vtime_generic_enabled_cpu(cpu)) { + idle = kcpustat_field(idx, cpu); + goto to_us; + } + + do { + seq = read_seqcount_begin(&kc->idle_sleeptime_seq); + + idle = cpustat[idx]; + if (kc->idle_elapse && compute_delta && now > kc->idle_entrytime) + idle += (now - kc->idle_entrytime); + } while (read_seqcount_retry(&kc->idle_sleeptime_seq, seq)); + +to_us: + do_div(idle, NSEC_PER_USEC); + + return idle; +} + +/** + * get_cpu_idle_time_us - get the total idle time of a CPU + * @cpu: CPU number to query + * @last_update_time: variable to store update time in. Do not update + * counters if NULL. + * + * Return the cumulative idle time (since boot) for a given + * CPU, in microseconds. Note that this is partially broken due to + * the counter of iowait tasks that can be remotely updated without + * any synchronization. Therefore it is possible to observe backward + * values within two consecutive reads. + * + * This time is measured via accounting rather than sampling, + * and is as accurate as ktime_get() is. + * + * Return: -1 if generic vtime is enabled, else total idle time of the @cpu + */ +u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time) +{ + return get_cpu_sleep_time_us(cpu, CPUTIME_IDLE, + !nr_iowait_cpu(cpu), last_update_time); +} +EXPORT_SYMBOL_GPL(get_cpu_idle_time_us); + +/** + * get_cpu_iowait_time_us - get the total iowait time of a CPU + * @cpu: CPU number to query + * @last_update_time: variable to store update time in. Do not update + * counters if NULL. + * + * Return the cumulative iowait time (since boot) for a given + * CPU, in microseconds. Note this is partially broken due to + * the counter of iowait tasks that can be remotely updated without + * any synchronization. Therefore it is possible to observe backward + * values within two consecutive reads. + * + * This time is measured via accounting rather than sampling, + * and is as accurate as ktime_get() is. + * + * Return: -1 if generic vtime is enabled, else total iowait time of @cpu + */ +u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time) +{ + return get_cpu_sleep_time_us(cpu, CPUTIME_IOWAIT, + nr_iowait_cpu(cpu), last_update_time); +} +EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us); + #endif /* CONFIG_NO_HZ_COMMON */ /* diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index cb235ec7d2d6..fa03cf7b3cec 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -749,126 +749,6 @@ static void tick_nohz_update_jiffies(ktime_t now) touch_softlockup_watchdog_sched(); } -static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now) -{ - u64 *cpustat = kcpustat_this_cpu->cpustat; - ktime_t delta; - - if (vtime_generic_enabled_this_cpu()) - return; - - if (WARN_ON_ONCE(!tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE))) - return; - - delta = ktime_sub(now, ts->idle_entrytime); - - write_seqcount_begin(&ts->idle_sleeptime_seq); - if (nr_iowait_cpu(smp_processor_id()) > 0) - cpustat[CPUTIME_IOWAIT] = ktime_add(cpustat[CPUTIME_IOWAIT], delta); - else - cpustat[CPUTIME_IDLE] = ktime_add(cpustat[CPUTIME_IDLE], delta); - - ts->idle_entrytime = now; - tick_sched_flag_clear(ts, TS_FLAG_IDLE_ACTIVE); - write_seqcount_end(&ts->idle_sleeptime_seq); - - sched_clock_idle_wakeup_event(); -} - -static void tick_nohz_start_idle(struct tick_sched *ts) -{ - if (vtime_generic_enabled_this_cpu()) - return; - - write_seqcount_begin(&ts->idle_sleeptime_seq); - ts->idle_entrytime = ktime_get(); - tick_sched_flag_set(ts, TS_FLAG_IDLE_ACTIVE); - write_seqcount_end(&ts->idle_sleeptime_seq); - sched_clock_idle_sleep_event(); -} - -static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, - bool compute_delta, u64 *last_update_time) -{ - struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); - u64 *cpustat = kcpustat_cpu(cpu).cpustat; - ktime_t now, idle; - unsigned int seq; - - now = ktime_get(); - if (last_update_time) - *last_update_time = ktime_to_us(now); - - if (vtime_generic_enabled_cpu(cpu)) { - idle = kcpustat_field(idx, cpu); - return ktime_to_us(idle); - } - - do { - ktime_t delta = 0; - - seq = read_seqcount_begin(&ts->idle_sleeptime_seq); - - if (tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE) && compute_delta) { - if (now > ts->idle_entrytime) - delta = ktime_sub(now, ts->idle_entrytime); - } - - idle = ktime_add(cpustat[idx], delta); - } while (read_seqcount_retry(&ts->idle_sleeptime_seq, seq)); - - return ktime_to_us(idle); - -} - -/** - * get_cpu_idle_time_us - get the total idle time of a CPU - * @cpu: CPU number to query - * @last_update_time: variable to store update time in. Do not update - * counters if NULL. - * - * Return the cumulative idle time (since boot) for a given - * CPU, in microseconds. Note that this is partially broken due to - * the counter of iowait tasks that can be remotely updated without - * any synchronization. Therefore it is possible to observe backward - * values within two consecutive reads. - * - * This time is measured via accounting rather than sampling, - * and is as accurate as ktime_get() is. - * - * Return: -1 if generic vtime is enabled, else total idle time of the @cpu - */ -u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time) -{ - return get_cpu_sleep_time_us(cpu, CPUTIME_IDLE, - !nr_iowait_cpu(cpu), last_update_time); -} -EXPORT_SYMBOL_GPL(get_cpu_idle_time_us); - -/** - * get_cpu_iowait_time_us - get the total iowait time of a CPU - * @cpu: CPU number to query - * @last_update_time: variable to store update time in. Do not update - * counters if NULL. - * - * Return the cumulative iowait time (since boot) for a given - * CPU, in microseconds. Note this is partially broken due to - * the counter of iowait tasks that can be remotely updated without - * any synchronization. Therefore it is possible to observe backward - * values within two consecutive reads. - * - * This time is measured via accounting rather than sampling, - * and is as accurate as ktime_get() is. - * - * Return: -1 if generic vtime is enabled, else total iowait time of @cpu - */ -u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time) -{ - return get_cpu_sleep_time_us(cpu, CPUTIME_IOWAIT, - nr_iowait_cpu(cpu), last_update_time); -} -EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us); - /* Simplified variant of hrtimer_forward_now() */ static ktime_t tick_forward_now(ktime_t expires, ktime_t now) { @@ -1289,6 +1169,20 @@ void tick_nohz_idle_retain_tick(void) tick_nohz_retain_tick(this_cpu_ptr(&tick_cpu_sched)); } +static void tick_nohz_clock_sleep(struct tick_sched *ts) +{ + tick_sched_flag_set(ts, TS_FLAG_IDLE_ACTIVE); + sched_clock_idle_sleep_event(); +} + +static void tick_nohz_clock_wakeup(struct tick_sched *ts) +{ + if (tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE)) { + tick_sched_flag_clear(ts, TS_FLAG_IDLE_ACTIVE); + sched_clock_idle_wakeup_event(); + } +} + /** * tick_nohz_idle_enter - prepare for entering idle on the current CPU * @@ -1303,12 +1197,11 @@ void tick_nohz_idle_enter(void) local_irq_disable(); ts = this_cpu_ptr(&tick_cpu_sched); - WARN_ON_ONCE(ts->timer_expires_base); - tick_sched_flag_set(ts, TS_FLAG_INIDLE); - kcpustat_dyntick_start(); - tick_nohz_start_idle(ts); + ts->idle_entrytime = ktime_get(); + kcpustat_dyntick_start(ts->idle_entrytime); + tick_nohz_clock_sleep(ts); local_irq_enable(); } @@ -1336,10 +1229,13 @@ void tick_nohz_irq_exit(void) { struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); - if (tick_sched_flag_test(ts, TS_FLAG_INIDLE)) - tick_nohz_start_idle(ts); - else + if (tick_sched_flag_test(ts, TS_FLAG_INIDLE)) { + ts->idle_entrytime = ktime_get(); + kcpustat_irq_exit(ts->idle_entrytime); + tick_nohz_clock_sleep(ts); + } else { tick_nohz_full_update_tick(ts); + } } /** @@ -1484,11 +1380,11 @@ void tick_nohz_idle_exit(void) now = ktime_get(); if (idle_active) - tick_nohz_stop_idle(ts, now); + tick_nohz_clock_wakeup(ts); if (tick_stopped) tick_nohz_idle_update_tick(ts, now); - kcpustat_dyntick_stop(); + kcpustat_dyntick_stop(now); local_irq_enable(); } @@ -1545,9 +1441,14 @@ static inline void tick_nohz_irq_enter(void) if (!tick_sched_flag_test(ts, TS_FLAG_STOPPED | TS_FLAG_IDLE_ACTIVE)) return; + now = ktime_get(); - if (tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE)) - tick_nohz_stop_idle(ts, now); + + if (tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE)) { + tick_nohz_clock_wakeup(ts); + kcpustat_irq_enter(now); + } + /* * If all CPUs are idle we may need to update a stale jiffies value. * Note nohz_full is a special case: a timekeeper is guaranteed to stay -- cgit v1.2.3 From 29807c524d66e27762cdc8992c2cac89b4c3fda9 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 8 May 2026 15:16:42 +0200 Subject: tick/sched: Remove unused fields Remove fields after the dyntick-idle cputime migration to scheduler code. Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Tested-by: Shrikanth Hegde Link: https://patch.msgid.link/20260508131647.43868-11-frederic@kernel.org --- kernel/time/tick-sched.h | 12 ------------ kernel/time/timer_list.c | 6 +----- 2 files changed, 1 insertion(+), 17 deletions(-) (limited to 'kernel') diff --git a/kernel/time/tick-sched.h b/kernel/time/tick-sched.h index b4a7822f495d..79b9252047b1 100644 --- a/kernel/time/tick-sched.h +++ b/kernel/time/tick-sched.h @@ -44,9 +44,7 @@ struct tick_device { * to resume the tick timer operation in the timeline * when the CPU returns from nohz sleep. * @next_tick: Next tick to be fired when in dynticks mode. - * @idle_jiffies: jiffies at the entry to idle for idle time accounting * @idle_waketime: Time when the idle was interrupted - * @idle_sleeptime_seq: sequence counter for data consistency * @idle_entrytime: Time when the idle call was entered * @last_jiffies: Base jiffies snapshot when next event was last computed * @timer_expires_base: Base time clock monotonic for @timer_expires @@ -55,9 +53,6 @@ struct tick_device { * @idle_expires: Next tick in idle, for debugging purpose only * @idle_calls: Total number of idle calls * @idle_sleeps: Number of idle calls, where the sched tick was stopped - * @idle_exittime: Time when the idle state was left - * @idle_sleeptime: Sum of the time slept in idle with sched tick stopped - * @iowait_sleeptime: Sum of the time slept in idle with sched tick stopped, with IO outstanding * @tick_dep_mask: Tick dependency mask - is set, if someone needs the tick * @check_clocks: Notification mechanism about clocksource changes */ @@ -73,12 +68,10 @@ struct tick_sched { struct hrtimer sched_timer; ktime_t last_tick; ktime_t next_tick; - unsigned long idle_jiffies; ktime_t idle_waketime; unsigned int got_idle_tick; /* Idle entry */ - seqcount_t idle_sleeptime_seq; ktime_t idle_entrytime; /* Tick stop */ @@ -90,11 +83,6 @@ struct tick_sched { unsigned long idle_calls; unsigned long idle_sleeps; - /* Idle exit */ - ktime_t idle_exittime; - ktime_t idle_sleeptime; - ktime_t iowait_sleeptime; - /* Full dynticks handling */ atomic_t tick_dep_mask; diff --git a/kernel/time/timer_list.c b/kernel/time/timer_list.c index 427d7ddea3af..514802def1e0 100644 --- a/kernel/time/timer_list.c +++ b/kernel/time/timer_list.c @@ -152,14 +152,10 @@ static void print_cpu(struct seq_file *m, int cpu, u64 now) P_flag(highres, TS_FLAG_HIGHRES); P_ns(last_tick); P_flag(tick_stopped, TS_FLAG_STOPPED); - P(idle_jiffies); P(idle_calls); P(idle_sleeps); P_ns(idle_entrytime); P_ns(idle_waketime); - P_ns(idle_exittime); - P_ns(idle_sleeptime); - P_ns(iowait_sleeptime); P(last_jiffies); P(next_timer); P_ns(idle_expires); @@ -256,7 +252,7 @@ static void timer_list_show_tickdevices_header(struct seq_file *m) static inline void timer_list_header(struct seq_file *m, u64 now) { - SEQ_printf(m, "Timer List Version: v0.10\n"); + SEQ_printf(m, "Timer List Version: v0.11\n"); SEQ_printf(m, "HRTIMER_MAX_CLOCK_BASES: %d\n", HRTIMER_MAX_CLOCK_BASES); SEQ_printf(m, "now at %Ld nsecs\n", (unsigned long long)now); SEQ_printf(m, "\n"); -- cgit v1.2.3 From 6a1f6a9dd0736257f5e5af32dd955d186cdc075d Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 8 May 2026 15:16:43 +0200 Subject: tick/sched: Account tickless idle cputime only when tick is stopped There is no real point in switching to dyntick-idle cputime accounting mode if the tick is not actually stopped. This just adds overhead, notably fetching the GTOD, on each idle exit and each idle IRQ entry for no reason during short idle trips. Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Tested-by: Shrikanth Hegde Link: https://patch.msgid.link/20260508131647.43868-12-frederic@kernel.org --- kernel/time/tick-sched.c | 50 +++++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 24 deletions(-) (limited to 'kernel') diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index fa03cf7b3cec..c1ee0b256445 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -1157,8 +1157,10 @@ void tick_nohz_idle_stop_tick(void) ts->idle_sleeps++; ts->idle_expires = expires; - if (!was_stopped && tick_sched_flag_test(ts, TS_FLAG_STOPPED)) + if (!was_stopped && tick_sched_flag_test(ts, TS_FLAG_STOPPED)) { + kcpustat_dyntick_start(ts->idle_entrytime); nohz_balance_enter_idle(cpu); + } } else { tick_nohz_retain_tick(ts); } @@ -1200,7 +1202,6 @@ void tick_nohz_idle_enter(void) WARN_ON_ONCE(ts->timer_expires_base); tick_sched_flag_set(ts, TS_FLAG_INIDLE); ts->idle_entrytime = ktime_get(); - kcpustat_dyntick_start(ts->idle_entrytime); tick_nohz_clock_sleep(ts); local_irq_enable(); @@ -1230,9 +1231,10 @@ void tick_nohz_irq_exit(void) struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); if (tick_sched_flag_test(ts, TS_FLAG_INIDLE)) { - ts->idle_entrytime = ktime_get(); - kcpustat_irq_exit(ts->idle_entrytime); tick_nohz_clock_sleep(ts); + ts->idle_entrytime = ktime_get(); + if (tick_sched_flag_test(ts, TS_FLAG_STOPPED)) + kcpustat_irq_exit(ts->idle_entrytime); } else { tick_nohz_full_update_tick(ts); } @@ -1333,8 +1335,17 @@ void tick_nohz_idle_restart_tick(void) { struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); - if (tick_sched_flag_test(ts, TS_FLAG_STOPPED)) - tick_nohz_restart_sched_tick(ts, ktime_get()); + if (tick_sched_flag_test(ts, TS_FLAG_STOPPED)) { + /* + * Update entrytime here in case the tick restart is due to temporary + * polling on forced broadcast. The tick may be stopped again later within + * the same idle trip. The idle_entrytime was updated recently but make sure + * no tiny amount of idle time is accounted twice. + */ + ts->idle_entrytime = ktime_get(); + kcpustat_dyntick_stop(ts->idle_entrytime); + tick_nohz_restart_sched_tick(ts, ts->idle_entrytime); + } } static void tick_nohz_idle_update_tick(struct tick_sched *ts, ktime_t now) @@ -1364,7 +1375,6 @@ static void tick_nohz_idle_update_tick(struct tick_sched *ts, ktime_t now) void tick_nohz_idle_exit(void) { struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); - bool idle_active, tick_stopped; ktime_t now; local_irq_disable(); @@ -1373,18 +1383,13 @@ void tick_nohz_idle_exit(void) WARN_ON_ONCE(ts->timer_expires_base); tick_sched_flag_clear(ts, TS_FLAG_INIDLE); - idle_active = tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE); - tick_stopped = tick_sched_flag_test(ts, TS_FLAG_STOPPED); + tick_nohz_clock_wakeup(ts); - if (idle_active || tick_stopped) + if (tick_sched_flag_test(ts, TS_FLAG_STOPPED)) { now = ktime_get(); - - if (idle_active) - tick_nohz_clock_wakeup(ts); - - if (tick_stopped) + kcpustat_dyntick_stop(now); tick_nohz_idle_update_tick(ts, now); - kcpustat_dyntick_stop(now); + } local_irq_enable(); } @@ -1439,15 +1444,13 @@ static inline void tick_nohz_irq_enter(void) struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); ktime_t now; - if (!tick_sched_flag_test(ts, TS_FLAG_STOPPED | TS_FLAG_IDLE_ACTIVE)) + tick_nohz_clock_wakeup(ts); + + if (!tick_sched_flag_test(ts, TS_FLAG_STOPPED)) return; now = ktime_get(); - - if (tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE)) { - tick_nohz_clock_wakeup(ts); - kcpustat_irq_enter(now); - } + kcpustat_irq_enter(now); /* * If all CPUs are idle we may need to update a stale jiffies value. @@ -1456,8 +1459,7 @@ static inline void tick_nohz_irq_enter(void) * rare case (typically stop machine). So we must make sure we have a * last resort. */ - if (tick_sched_flag_test(ts, TS_FLAG_STOPPED)) - tick_nohz_update_jiffies(now); + tick_nohz_update_jiffies(now); } #else -- cgit v1.2.3 From 127b2eb44f36d5d7059f1af425b5800cb27440f9 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 8 May 2026 15:16:44 +0200 Subject: tick/sched: Consolidate idle time fetching APIs Fetching the idle cputime is available through a variety of accessors all over the place depending on the different accounting flavours and needs: - idle vtime generic accounting can be accessed by kcpustat_field(), kcpustat_cpu_fetch(), get_idle/iowait_time() and get_cpu_idle/iowait_time_us() - dynticks-idle accounting can only be accessed by get_idle/iowait_time() or get_cpu_idle/iowait_time_us() - CONFIG_NO_HZ_COMMON=n idle accounting can be accessed by kcpustat_field() kcpustat_cpu_fetch(), or get_idle/iowait_time() but not by get_cpu_idle/iowait_time_us() Moreover get_idle/iowait_time() relies on get_cpu_idle/iowait_time_us() with a non-sensical conversion to microseconds and back to nanoseconds on the way. Start consolidating the APIs with removing get_idle/iowait_time() and make kcpustat_field() and kcpustat_cpu_fetch() work for all cases. Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Tested-by: Shrikanth Hegde Link: https://patch.msgid.link/20260508131647.43868-13-frederic@kernel.org --- kernel/sched/cputime.c | 61 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 21 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 4c00163b74b9..c91fd67f93ea 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -490,24 +490,14 @@ void kcpustat_irq_exit(u64 now) kcpustat_idle_start(kc, now); } -static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, - bool compute_delta, u64 *last_update_time) +static u64 kcpustat_field_dyntick(int cpu, enum cpu_usage_stat idx, + bool compute_delta, u64 now) { struct kernel_cpustat *kc = &kcpustat_cpu(cpu); u64 *cpustat = kc->cpustat; unsigned int seq; - ktime_t now; u64 idle; - now = ktime_get(); - if (last_update_time) - *last_update_time = ktime_to_us(now); - - if (vtime_generic_enabled_cpu(cpu)) { - idle = kcpustat_field(idx, cpu); - goto to_us; - } - do { seq = read_seqcount_begin(&kc->idle_sleeptime_seq); @@ -516,12 +506,42 @@ static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, idle += (now - kc->idle_entrytime); } while (read_seqcount_retry(&kc->idle_sleeptime_seq, seq)); -to_us: - do_div(idle, NSEC_PER_USEC); - return idle; } +u64 kcpustat_field_idle(int cpu) +{ + return kcpustat_field_dyntick(cpu, CPUTIME_IDLE, + !nr_iowait_cpu(cpu), ktime_get()); +} +EXPORT_SYMBOL_GPL(kcpustat_field_idle); + +u64 kcpustat_field_iowait(int cpu) +{ + return kcpustat_field_dyntick(cpu, CPUTIME_IOWAIT, + nr_iowait_cpu(cpu), ktime_get()); +} +EXPORT_SYMBOL_GPL(kcpustat_field_iowait); + +static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, + bool compute_delta, u64 *last_update_time) +{ + ktime_t now = ktime_get(); + u64 res; + + if (vtime_generic_enabled_cpu(cpu)) + res = kcpustat_field(idx, cpu); + else + res = kcpustat_field_dyntick(cpu, idx, compute_delta, now); + + do_div(res, NSEC_PER_USEC); + + if (last_update_time) + *last_update_time = ktime_to_us(now); + + return res; +} + /** * get_cpu_idle_time_us - get the total idle time of a CPU * @cpu: CPU number to query @@ -569,7 +589,6 @@ u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time) nr_iowait_cpu(cpu), last_update_time); } EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us); - #endif /* CONFIG_NO_HZ_COMMON */ /* @@ -1123,8 +1142,8 @@ u64 kcpustat_field(enum cpu_usage_stat usage, int cpu) struct rq *rq; int err; - if (!vtime_accounting_enabled_cpu(cpu)) - return val; + if (!vtime_generic_enabled_cpu(cpu)) + return kcpustat_field_default(usage, cpu); rq = cpu_rq(cpu); @@ -1219,8 +1238,8 @@ void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu) struct rq *rq; int err; - if (!vtime_accounting_enabled_cpu(cpu)) { - *dst = *src; + if (!vtime_generic_enabled_cpu(cpu)) { + kcpustat_cpu_fetch_default(dst, cpu); return; } @@ -1233,7 +1252,7 @@ void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu) curr = rcu_dereference(rq->curr); if (WARN_ON_ONCE(!curr)) { rcu_read_unlock(); - *dst = *src; + kcpustat_cpu_fetch_default(dst, cpu); return; } -- cgit v1.2.3 From 3b45b4f188f3a0ebd16ab71efd2ffcc7a16ad861 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 8 May 2026 15:16:45 +0200 Subject: sched/cputime: Provide get_cpu_[idle|iowait]_time_us() off-case The last reason why get_cpu_idle/iowait_time_us() may return -1 now is if the config doesn't support nohz. The ad-hoc replacement solution by cpufreq is to compute jiffies minus the whole busy cputime. Although the intention should provide a coherent low resolution estimation of the idle and iowait time, the implementation is buggy because jiffies don't start at 0. Just provide instead a real get_cpu_[idle|iowait]_time_us() offcase. Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Tested-by: Shrikanth Hegde Link: https://patch.msgid.link/20260508131647.43868-14-frederic@kernel.org --- kernel/sched/cputime.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index c91fd67f93ea..335d2c127763 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -522,6 +522,13 @@ u64 kcpustat_field_iowait(int cpu) nr_iowait_cpu(cpu), ktime_get()); } EXPORT_SYMBOL_GPL(kcpustat_field_iowait); +#else +static u64 kcpustat_field_dyntick(int cpu, enum cpu_usage_stat idx, + bool compute_delta, ktime_t now) +{ + return kcpustat_cpu(cpu).cpustat[idx]; +} +#endif /* CONFIG_NO_HZ_COMMON */ static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, bool compute_delta, u64 *last_update_time) @@ -557,7 +564,7 @@ static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, * This time is measured via accounting rather than sampling, * and is as accurate as ktime_get() is. * - * Return: -1 if generic vtime is enabled, else total idle time of the @cpu + * Return: total idle time of the @cpu */ u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time) { @@ -581,7 +588,7 @@ EXPORT_SYMBOL_GPL(get_cpu_idle_time_us); * This time is measured via accounting rather than sampling, * and is as accurate as ktime_get() is. * - * Return: -1 if generic vtime is enabled, else total iowait time of @cpu + * Return: total iowait time of @cpu */ u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time) { @@ -589,7 +596,6 @@ u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time) nr_iowait_cpu(cpu), last_update_time); } EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us); -#endif /* CONFIG_NO_HZ_COMMON */ /* * Use precise platform statistics if available: -- cgit v1.2.3 From 7198e3927a14535475a24cce559f41f97e6c0b66 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 8 May 2026 15:16:46 +0200 Subject: sched/cputime: Handle idle irqtime gracefully The dyntick-idle cputime accounting always assumes that interrupt time accounting is enabled and consequently stops elapsing the idle time during dyntick-idle interrupts. This doesn't mix up well with disabled interrupt time accounting because then idle interrupts become a cputime blind-spot. Also this feature is disabled on most configurations and the overhead of pausing dyntick-idle accounting while in idle interrupts could then be avoided. Fix the situation with conditionally pausing dyntick-idle accounting during idle interrupts only iff either native vtime (which does interrupt time accounting) or generic interrupt time accounting are enabled. Also make sure that the accumulated interrupt time is not accidentally substracted from later accounting. Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Tested-by: Shrikanth Hegde Link: https://patch.msgid.link/20260508131647.43868-15-frederic@kernel.org --- kernel/sched/cputime.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 335d2c127763..94be22aa5cb6 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -47,7 +47,8 @@ static void irqtime_account_delta(struct irqtime *irqtime, u64 delta, u64_stats_update_begin(&irqtime->sync); cpustat[idx] += delta; irqtime->total += delta; - irqtime->tick_delta += delta; + if (!kcpustat_idle_dyntick()) + irqtime->tick_delta += delta; u64_stats_update_end(&irqtime->sync); } @@ -444,6 +445,10 @@ static void kcpustat_idle_stop(struct kernel_cpustat *kc, u64 now) static void kcpustat_idle_start(struct kernel_cpustat *kc, u64 now) { + /* Irqtime accounting might have been enabled in the middle of the IRQ */ + if (kc->idle_elapse) + return; + write_seqcount_begin(&kc->idle_sleeptime_seq); kc->idle_entrytime = now; kc->idle_elapse = true; @@ -478,7 +483,8 @@ void kcpustat_irq_enter(u64 now) { struct kernel_cpustat *kc = kcpustat_this_cpu; - if (!vtime_generic_enabled_this_cpu()) + if (!vtime_generic_enabled_this_cpu() && + (irqtime_enabled() || vtime_accounting_enabled_this_cpu())) kcpustat_idle_stop(kc, now); } @@ -486,7 +492,15 @@ void kcpustat_irq_exit(u64 now) { struct kernel_cpustat *kc = kcpustat_this_cpu; - if (!vtime_generic_enabled_this_cpu()) + /* + * Generic vtime already does its own idle accounting. + * But irqtime accounting or arch vtime which also accounts IRQs + * need to pause nohz accounting. Resume nohz accounting as long + * as the irqtime config is enabled to handle case where irqtime + * accounting got runtime disabled in the middle of an IRQ. + */ + if (!vtime_generic_enabled_this_cpu() && + (IS_ENABLED(CONFIG_IRQ_TIME_ACCOUNTING) || vtime_accounting_enabled_this_cpu())) kcpustat_idle_start(kc, now); } -- cgit v1.2.3 From 6199f9999a9b62b2b84a1bf5b52a9fd0bb8de5af Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 8 May 2026 15:16:47 +0200 Subject: sched/cputime: Handle dyntick-idle steal time correctly The dyntick-idle steal time is currently accounted when the tick restarts but the stolen idle time is not subtracted from the idle time that was already accounted. This is to avoid observing the idle time going backward as the dyntick-idle cputime accessors can't reliably know in advance the stolen idle time. In order to maintain a forward progressing idle cputime while subtracting idle steal time from it, keep track of the previously accounted idle stolen time and substract it from _later_ idle cputime accounting. Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Tested-by: Shrikanth Hegde Link: https://patch.msgid.link/20260508131647.43868-16-frederic@kernel.org --- kernel/sched/cputime.c | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 94be22aa5cb6..244b57417240 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -425,19 +425,32 @@ static inline void irqtime_account_process_tick(struct task_struct *p, int user_ static void kcpustat_idle_stop(struct kernel_cpustat *kc, u64 now) { u64 *cpustat = kc->cpustat; - u64 delta; + u64 delta, steal, steal_delta; + int iowait; if (!kc->idle_elapse) return; + iowait = nr_iowait_cpu(smp_processor_id()) > 0; delta = now - kc->idle_entrytime; + steal = steal_account_process_time(delta); + /* + * Record the idle time after substracting the steal time from + * previous update sequence. Don't substract the steal time from + * the current update sequence to avoid readers moving backward. + */ write_seqcount_begin(&kc->idle_sleeptime_seq); - if (nr_iowait_cpu(smp_processor_id()) > 0) + steal_delta = min_t(u64, kc->idle_stealtime[iowait], delta); + delta -= steal_delta; + kc->idle_stealtime[iowait] -= steal_delta; + + if (iowait) cpustat[CPUTIME_IOWAIT] += delta; else cpustat[CPUTIME_IDLE] += delta; + kc->idle_stealtime[iowait] += steal; kc->idle_entrytime = now; kc->idle_elapse = false; write_seqcount_end(&kc->idle_sleeptime_seq); @@ -464,7 +477,6 @@ void kcpustat_dyntick_stop(u64 now) kcpustat_idle_stop(kc, now); kc->idle_dyntick = false; vtime_dyntick_stop(); - steal_account_process_time(ULONG_MAX); } } @@ -508,6 +520,7 @@ static u64 kcpustat_field_dyntick(int cpu, enum cpu_usage_stat idx, bool compute_delta, u64 now) { struct kernel_cpustat *kc = &kcpustat_cpu(cpu); + int iowait = idx == CPUTIME_IOWAIT; u64 *cpustat = kc->cpustat; unsigned int seq; u64 idle; @@ -516,8 +529,13 @@ static u64 kcpustat_field_dyntick(int cpu, enum cpu_usage_stat idx, seq = read_seqcount_begin(&kc->idle_sleeptime_seq); idle = cpustat[idx]; - if (kc->idle_elapse && compute_delta && now > kc->idle_entrytime) - idle += (now - kc->idle_entrytime); + + if (kc->idle_elapse && compute_delta && now > kc->idle_entrytime) { + u64 delta = now - kc->idle_entrytime; + + delta -= min_t(u64, kc->idle_stealtime[iowait], delta); + idle += delta; + } } while (read_seqcount_retry(&kc->idle_sleeptime_seq, seq)); return idle; -- cgit v1.2.3 From e4a70f5fbd43f55b474028a2cee3d78e4b443dd7 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 20 May 2026 00:09:25 +0200 Subject: timers/migration: Fix hotplug migrator selection target on asymetric capacity machines When a top-level migrator is deactivated, either at CPU down hotplug time or when a CPU is domain isolated, a new migrator is elected among the available CPUs and woken up to take over the migration duty. However that election must happen at the scope of a given hierarchy and not globally, which the introduction of per-capacity hierarchies failed to handle. As a result a given hierarchy may end up without migrator to handle global timers. Fix it by making sure that the new migrator belongs to the same hierarchy as the outgoing CPU. Fixes: 098cbaad8e57 ("timers/migration: Split per-capacity hierarchies") Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Link: https://patch.msgid.link/20260519220926.63437-2-frederic@kernel.org --- kernel/time/timer_migration.c | 42 ++++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timer_migration.c b/kernel/time/timer_migration.c index 25e3c563eb74..8032b0044f44 100644 --- a/kernel/time/timer_migration.c +++ b/kernel/time/timer_migration.c @@ -1464,6 +1464,18 @@ static long tmigr_trigger_active(void *unused) return 0; } +static struct tmigr_hierarchy *__tmigr_get_hierarchy(unsigned int capacity) +{ + struct tmigr_hierarchy *iter; + + list_for_each_entry(iter, &tmigr_hierarchy_list, node) { + if (iter->capacity == capacity) + return iter; + } + + return NULL; +} + static int tmigr_clear_cpu_available(unsigned int cpu) { struct tmigr_cpu *tmc = this_cpu_ptr(&tmigr_cpu); @@ -1488,8 +1500,21 @@ static int tmigr_clear_cpu_available(unsigned int cpu) } if (firstexp != KTIME_MAX) { - migrator = cpumask_any(tmigr_available_cpumask); - work_on_cpu(migrator, tmigr_trigger_active, NULL); + struct tmigr_hierarchy *hier = __tmigr_get_hierarchy(arch_scale_cpu_capacity(cpu)); + + if (WARN_ON_ONCE(!hier)) + return -EINVAL; + + migrator = cpumask_any_and(tmigr_available_cpumask, hier->cpumask); + if (migrator < nr_cpu_ids) { + work_on_cpu(migrator, tmigr_trigger_active, NULL); + } else { + /* + * If deactivation returned an expiration, it belongs to an available + * nohz CPU in the hierarchy. + */ + WARN_ONCE(1, "Expected available CPU in the hierarchy\n"); + } } return 0; @@ -1915,12 +1940,9 @@ out: static struct tmigr_hierarchy *tmigr_get_hierarchy(unsigned int capacity) { - struct tmigr_hierarchy *hier = NULL, *iter; + struct tmigr_hierarchy *hier; - list_for_each_entry(iter, &tmigr_hierarchy_list, node) { - if (iter->capacity == capacity) - hier = iter; - } + hier = __tmigr_get_hierarchy(capacity); if (hier) return hier; @@ -1978,9 +2000,9 @@ static long connect_old_root_work(void *arg) struct tmigr_hierarchy *hier; int cpu = smp_processor_id(); - hier = tmigr_get_hierarchy(arch_scale_cpu_capacity(cpu)); - if (IS_ERR(hier)) - return PTR_ERR(hier); + hier = __tmigr_get_hierarchy(arch_scale_cpu_capacity(cpu)); + if (WARN_ON_ONCE(!hier)) + return -EINVAL; return tmigr_connect_old_root(hier, cpu, old_root, true); } -- cgit v1.2.3 From d4f198c13611257f7f29d3c614721d0ac5d362f5 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 20 May 2026 00:09:26 +0200 Subject: timers/migration: Deactivate per-capacity hierarchies under nohz_full NOHZ_FULL CPUs global timers are guaranteed to be handled by the timekeeper CPU, which never stops its tick and therefore remains active in the hierarchy. But since the introduction of per-capacity hierarchies, this guarantee is broken because the timekeeper may not belong to the same hierarchy as all the NOHZ_FULL CPUs. Fix it with simply turning off capacity awareness when NOHZ_FULL is running and force a single hierarchy. NOHZ_FULL is not exactly optimized powerwise anyway. Fixes: 098cbaad8e57 ("timers/migration: Split per-capacity hierarchies") Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Link: https://patch.msgid.link/20260519220926.63437-3-frederic@kernel.org --- kernel/time/timer_migration.c | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timer_migration.c b/kernel/time/timer_migration.c index 8032b0044f44..8ba53ad49173 100644 --- a/kernel/time/timer_migration.c +++ b/kernel/time/timer_migration.c @@ -1464,8 +1464,24 @@ static long tmigr_trigger_active(void *unused) return 0; } -static struct tmigr_hierarchy *__tmigr_get_hierarchy(unsigned int capacity) +static unsigned int tmigr_get_capacity(int cpu) { + /* + * nohz_full CPUs need to make sure there is always an available (online) + * and never idle migrator to handle all their global timers. That duty + * is served by the timekeeper which then never stops its tick. But the + * timekeeper must then belong to the same hierarchy as all the nohz_full + * CPUs. Simply turn off capacity awareness when nohz_full is running. + */ + if (tick_nohz_full_enabled()) + return SCHED_CAPACITY_SCALE; + else + return arch_scale_cpu_capacity(cpu); +} + +static struct tmigr_hierarchy *__tmigr_get_hierarchy(int cpu) +{ + unsigned int capacity = tmigr_get_capacity(cpu); struct tmigr_hierarchy *iter; list_for_each_entry(iter, &tmigr_hierarchy_list, node) { @@ -1500,7 +1516,7 @@ static int tmigr_clear_cpu_available(unsigned int cpu) } if (firstexp != KTIME_MAX) { - struct tmigr_hierarchy *hier = __tmigr_get_hierarchy(arch_scale_cpu_capacity(cpu)); + struct tmigr_hierarchy *hier = __tmigr_get_hierarchy(cpu); if (WARN_ON_ONCE(!hier)) return -EINVAL; @@ -1938,11 +1954,11 @@ out: return err; } -static struct tmigr_hierarchy *tmigr_get_hierarchy(unsigned int capacity) +static struct tmigr_hierarchy *tmigr_get_hierarchy(int cpu) { struct tmigr_hierarchy *hier; - hier = __tmigr_get_hierarchy(capacity); + hier = __tmigr_get_hierarchy(cpu); if (hier) return hier; @@ -1962,7 +1978,7 @@ static struct tmigr_hierarchy *tmigr_get_hierarchy(unsigned int capacity) for (int i = 0; i < tmigr_hierarchy_levels; i++) INIT_LIST_HEAD(&hier->level_list[i]); - hier->capacity = capacity; + hier->capacity = tmigr_get_capacity(cpu); list_add_tail(&hier->node, &tmigr_hierarchy_list); return hier; @@ -2000,7 +2016,7 @@ static long connect_old_root_work(void *arg) struct tmigr_hierarchy *hier; int cpu = smp_processor_id(); - hier = __tmigr_get_hierarchy(arch_scale_cpu_capacity(cpu)); + hier = __tmigr_get_hierarchy(cpu); if (WARN_ON_ONCE(!hier)) return -EINVAL; @@ -2016,7 +2032,7 @@ static int tmigr_add_cpu(unsigned int cpu) guard(mutex)(&tmigr_mutex); - hier = tmigr_get_hierarchy(arch_scale_cpu_capacity(cpu)); + hier = tmigr_get_hierarchy(cpu); if (IS_ERR(hier)) return PTR_ERR(hier); -- cgit v1.2.3 From 45b49d7e3ab6490a9b957a4075344093c43d1f7e Mon Sep 17 00:00:00 2001 From: Rosen Penev Date: Fri, 22 May 2026 16:16:18 -0700 Subject: timers/migration: Turn tmigr_hierarchy level_list into a flexible array The level_list array is allocated separately right after the parent struct. The size of the array is already known. Move level_list to the struct tail as a flexible array member and fold the two allocations into a single kzalloc_flex(). Signed-off-by: Rosen Penev Signed-off-by: Thomas Gleixner Assisted-by: Claude:Opus-4.7 Link: https://patch.msgid.link/20260522231618.41622-1-rosenp@gmail.com --- kernel/time/timer_migration.c | 16 +++++----------- kernel/time/timer_migration.h | 5 ++--- 2 files changed, 7 insertions(+), 14 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timer_migration.c b/kernel/time/timer_migration.c index 8ba53ad49173..548d84955f4c 100644 --- a/kernel/time/timer_migration.c +++ b/kernel/time/timer_migration.c @@ -1963,17 +1963,15 @@ static struct tmigr_hierarchy *tmigr_get_hierarchy(int cpu) if (hier) return hier; - hier = kzalloc(sizeof(*hier), GFP_KERNEL); + hier = kzalloc_flex(*hier, level_list, tmigr_hierarchy_levels); if (!hier) return ERR_PTR(-ENOMEM); hier->cpumask = kzalloc(cpumask_size(), GFP_KERNEL); - if (!hier->cpumask) - goto err; - - hier->level_list = kzalloc_objs(struct list_head, tmigr_hierarchy_levels); - if (!hier->level_list) - goto err; + if (!hier->cpumask) { + kfree(hier); + return ERR_PTR(-ENOMEM); + } for (int i = 0; i < tmigr_hierarchy_levels; i++) INIT_LIST_HEAD(&hier->level_list[i]); @@ -1982,10 +1980,6 @@ static struct tmigr_hierarchy *tmigr_get_hierarchy(int cpu) list_add_tail(&hier->node, &tmigr_hierarchy_list); return hier; -err: - kfree(hier->cpumask); - kfree(hier); - return ERR_PTR(-ENOMEM); } static int tmigr_connect_old_root(struct tmigr_hierarchy *hier, int cpu, diff --git a/kernel/time/timer_migration.h b/kernel/time/timer_migration.h index ea8db95fc63e..31735dd52327 100644 --- a/kernel/time/timer_migration.h +++ b/kernel/time/timer_migration.h @@ -9,19 +9,18 @@ * struct tmigr_hierarchy - a hierarchy associated to a given CPU capacity. * Homogeneous systems have only one hierarchy. * Heterogenous have one hierarchy per CPU capacity. - * @level_list: Per level lists of tmigr groups * @cpumask: CPUs belonging to this hierarchy * @root: The current root of the hierarchy * @capacity: CPU capacity associated to this hierarchy * @node: Node in the global hierarchy list + * @level_list: Per level lists of tmigr groups */ struct tmigr_hierarchy { - struct list_head *level_list; struct cpumask *cpumask; struct tmigr_group *root; unsigned long capacity; struct list_head node; - + struct list_head level_list[]; }; /** -- cgit v1.2.3 From 96942092d5e67c71af246fa3bc1422cdf80a5dc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Tue, 19 May 2026 08:26:17 +0200 Subject: vdso/treewide: Drop GENERIC_TIME_VSYSCALL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This Kconfig symbol is not used anymore, remove it. Signed-off-by: Thomas Weißschuh Signed-off-by: Thomas Gleixner Link: https://patch.msgid.link/20260519-vdso-generic_time_vsyscal-v1-3-5c2a5905d5f5@linutronix.de --- kernel/time/Kconfig | 4 ---- 1 file changed, 4 deletions(-) (limited to 'kernel') diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig index 02aac7c5aa76..d098ac39bde4 100644 --- a/kernel/time/Kconfig +++ b/kernel/time/Kconfig @@ -16,10 +16,6 @@ config ARCH_CLOCKSOURCE_INIT config ARCH_WANTS_CLOCKSOURCE_READ_INLINE bool -# Timekeeping vsyscall support -config GENERIC_TIME_VSYSCALL - bool - # The generic clock events infrastructure config GENERIC_CLOCKEVENTS def_bool !LEGACY_TIMER_TICK -- cgit v1.2.3 From 74e144274af39935b0f410c0ee4d2b91c3730414 Mon Sep 17 00:00:00 2001 From: Ji'an Zhou Date: Tue, 2 Jun 2026 09:12:04 +0000 Subject: futex/requeue: Prevent NULL pointer dereference in remove_waiter() on self-deadlock When FUTEX_CMP_REQUEUE_PI requeues a non-top waiter that already owns the target PI futex, task_blocks_on_rt_mutex() returns -EDEADLK before setting waiter->task. The subsequent remove_waiter() in rt_mutex_start_proxy_lock() dereferences the NULL waiter->task, causing a kernel crash. Add a self-deadlock check for non-top waiters before calling rt_mutex_start_proxy_lock(), analogous to the top-waiter check in futex_lock_pi_atomic(). Fixes: 3bfdc63936dd4773109b7b8c280c0f3b5ae7d349 ("rtmutex: Use waiter::task instead of current in remove_waiter()") Signed-off-by: Ji'an Zhou Signed-off-by: Thomas Gleixner Cc: stable@vger.kernel.org --- kernel/futex/requeue.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'kernel') diff --git a/kernel/futex/requeue.c b/kernel/futex/requeue.c index b597cb3d17fc..1d99a84dc9ad 100644 --- a/kernel/futex/requeue.c +++ b/kernel/futex/requeue.c @@ -643,6 +643,12 @@ retry_private: continue; } + /* Self-deadlock: non-top waiter already owns the PI futex. */ + if (rt_mutex_owner(&pi_state->pi_mutex) == this->task) { + ret = -EDEADLK; + break; + } + ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex, this->rt_waiter, this->task); -- cgit v1.2.3 From 02e545c4297a26dbbc41df81b831e7f605bcd306 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 1 Jun 2026 09:22:37 -1000 Subject: sched_ext: Don't warn on NULL cgrp_moving_from in scx_cgroup_move_task() A WARN fires when systemd's user manager writes "+cpu +memory +pids" to its own subtree_control while a sched_ext scheduler is loaded: WARNING: at kernel/sched/ext.c:3227 scx_cgroup_move_task+0xa8/0xb0 scx_cgroup_move_task+0xa8/0xb0 sched_move_task+0x134/0x290 cpu_cgroup_attach+0x39/0x70 cgroup_migrate_execute+0x37d/0x450 cgroup_update_dfl_csses+0x1e3/0x270 cgroup_subtree_control_write+0x3e7/0x440 scx_cgroup_can_attach() arms cgrp_moving_from only when a task's cpu cgroup changes. It can still be NULL when scx_cgroup_move_task() runs, through this sequence: Step Result --------------------------------- ---------------------------------- 1. cpu enabled on cgroup G cpu css = A 2. cpu toggled off then on for G A killed, B created (same cgroup) 3. an exiting task keeps A alive migration skips it, A now stale 4. +memory migrates G stale A vs current B pulls cpu in 5. cpu attach runs for all tasks hits a live, cpu-unchanged task 6. scx_cgroup_move_task() on it cgrp_moving_from NULL -> WARN The mismatch is that scx_cgroup_can_attach() keys on cgroup identity while migration drives the move on css identity, so a NULL cgrp_moving_from here is a legitimate css-only migration, not a missing prep. The call is already gated on cgrp_moving_from, so just drop the warning. ops.cgroup_prep_move() and ops.cgroup_move() stay paired. Fixes: 819513666966 ("sched_ext: Add cgroup support") Cc: stable@vger.kernel.org # v6.12+ Reported-by: Matt Fleming Closes: https://lore.kernel.org/all/20260601124156.2205704-1-mfleming@cloudflare.com/ Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index c1762420cc35..8e88a25bc602 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -4402,11 +4402,13 @@ void scx_cgroup_move_task(struct task_struct *p) return; /* - * @p must have ops.cgroup_prep_move() called on it and thus - * cgrp_moving_from set. + * scx_cgroup_can_attach() sets cgrp_moving_from only when the task's + * cgroup changes. Migration keys off css rather than cgroup identity, + * so it can hand an unchanged-cgroup task here with cgrp_moving_from + * NULL. Nothing to report to the BPF scheduler then, so skip it and + * keep prep_move and move paired. */ - if (SCX_HAS_OP(sch, cgroup_move) && - !WARN_ON_ONCE(!p->scx.cgrp_moving_from)) + if (SCX_HAS_OP(sch, cgroup_move) && p->scx.cgrp_moving_from) SCX_CALL_OP_TASK(sch, cgroup_move, task_rq(p), p, p->scx.cgrp_moving_from, tg_cgrp(task_group(p))); -- cgit v1.2.3 From 9c860d1d5d69f9cb19eb7c36573ee14065a9c85a Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Wed, 13 May 2026 12:35:13 +0000 Subject: mm: introduce for_each_free_list() Patch series "mm: misc cleanups from __GFP_UNMAPPED series". In v2 of the __GFP_UNMAPPED series [0], we realised that some of the patches could potentially be merged as independent cleanups. These are all independent of one another, if you think some are useful cleanups and others are pointless churn, it should be fine to just pick whatever subset you prefer. No functional change intended. This patch (of 4): There are a couple of places that iterate over the freelists with awareness of the data structures' layout. It seems ideally, code outside of mm should not be aware of the page allocator's freelists at all. But, this patch just doesn't hide them completely, it's just a meek incremental step in that direction: provide a macro to iterate over it without needing to be aware of the actual struct fields. Link: https://lore.kernel.org/20260513-page_alloc-unmapped-prep-v1-0-dacdf5402be8@google.com Link: https://lore.kernel.org/20260513-page_alloc-unmapped-prep-v1-1-dacdf5402be8@google.com Link: https://lore.kernel.org/all/20260320-page_alloc-unmapped-v2-0-28bf1bd54f41@google.com/ [0] Signed-off-by: Brendan Jackman Acked-by: Mike Rapoport (Microsoft) Reviewed-by: Vlastimil Babka (SUSE) Cc: Axel Rasmussen Cc: Barry Song Cc: David Hildenbrand Cc: Johannes Weiner Cc: Kairui Song Cc: Len Brown Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Michal Hocko Cc: "Rafael J. Wysocki" Cc: Shakeel Butt Cc: Suren Baghdasaryan Cc: Wei Xu Cc: Yuanchu Xie Cc: Zi Yan Signed-off-by: Andrew Morton --- kernel/power/snapshot.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c index a564650734dc..d933b5b2c05d 100644 --- a/kernel/power/snapshot.c +++ b/kernel/power/snapshot.c @@ -1244,8 +1244,9 @@ unsigned int snapshot_additional_pages(struct zone *zone) static void mark_free_pages(struct zone *zone) { unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT; + struct list_head *free_list; unsigned long flags; - unsigned int order, t; + unsigned int order; struct page *page; if (zone_is_empty(zone)) @@ -1269,9 +1270,8 @@ static void mark_free_pages(struct zone *zone) swsusp_unset_page_free(page); } - for_each_migratetype_order(order, t) { - list_for_each_entry(page, - &zone->free_area[order].free_list[t], buddy_list) { + for_each_free_list(free_list, zone, order) { + list_for_each_entry(page, free_list, buddy_list) { unsigned long i; pfn = page_to_pfn(page); -- cgit v1.2.3 From 7318301e060e3f1207a34f2fca6cd86770dd9160 Mon Sep 17 00:00:00 2001 From: Rosen Penev Date: Tue, 2 Jun 2026 20:17:58 -0700 Subject: dma: map_benchmark: turn dma_sg_map_param buf into a flexible array The buf pointer was kmalloc_array()'d immediately after the parent struct allocation, with the count (granule, validated to 1..1024 by the ioctl) trivially available beforehand. Move buf to the struct tail as a flexible array member and fold the two allocations into a single kzalloc_flex(), dropping the kfree(params->buf) in both the prepare error path and unprepare. Add __counted_by for extra runtime analysis. Assisted-by: Claude:Opus-4.7 Signed-off-by: Rosen Penev Reviewed-by: Qinxin Xia Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/20260603031758.290538-1-rosenp@gmail.com --- kernel/dma/map_benchmark.c | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) (limited to 'kernel') diff --git a/kernel/dma/map_benchmark.c b/kernel/dma/map_benchmark.c index 29eeb5fdf199..fdc070f419f6 100644 --- a/kernel/dma/map_benchmark.c +++ b/kernel/dma/map_benchmark.c @@ -121,35 +121,35 @@ static struct map_benchmark_ops dma_single_map_benchmark_ops = { struct dma_sg_map_param { struct sg_table sgt; struct device *dev; - void **buf; u32 npages; u32 dma_dir; + void *buf[] __counted_by(npages); }; static void *dma_sg_map_benchmark_prepare(struct map_benchmark_data *map) { + struct dma_sg_map_param *params; struct scatterlist *sg; + u32 npages; int i; - struct dma_sg_map_param *params = kzalloc(sizeof(*params), GFP_KERNEL); - - if (!params) - return NULL; /* * Set the number of scatterlist entries based on the granule. * In SG mode, 'granule' represents the number of scatterlist entries. * Each scatterlist entry corresponds to a single page. */ - params->npages = map->bparam.granule; + npages = map->bparam.granule; + + params = kzalloc_flex(*params, buf, npages); + if (!params) + return NULL; + + params->npages = npages; params->dma_dir = map->bparam.dma_dir; params->dev = map->dev; - params->buf = kmalloc_array(params->npages, sizeof(*params->buf), - GFP_KERNEL); - if (!params->buf) - goto out; if (sg_alloc_table(¶ms->sgt, params->npages, GFP_KERNEL)) - goto free_buf; + goto free_params; for_each_sgtable_sg(¶ms->sgt, sg, i) { params->buf[i] = (void *)__get_free_page(GFP_KERNEL); @@ -166,9 +166,7 @@ free_page: free_page((unsigned long)params->buf[i]); sg_free_table(¶ms->sgt); -free_buf: - kfree(params->buf); -out: +free_params: kfree(params); return NULL; } @@ -183,7 +181,6 @@ static void dma_sg_map_benchmark_unprepare(void *mparam) sg_free_table(¶ms->sgt); - kfree(params->buf); kfree(params); } -- cgit v1.2.3 From 560000d619ef162568746ce287f0c725e24ea967 Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Wed, 3 Jun 2026 09:37:23 +0800 Subject: dma-mapping: direct: fix missing mapping for THRU_HOST_BRIDGE segments In dma_direct_map_sg(), the case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE incorrectly used 'break' instead of falling through to MAP_NONE. As a result, segments traversing the host bridge skipped the required dma_direct_map_phys() call entirely, leaving sg->dma_address uninitialized and leading to DMA failures. Fix this by using 'fallthrough;'. Fixes: a25e7962db0d79 ("PCI/P2PDMA: Refactor the p2pdma mapping helpers") Reviewed-by: Logan Gunthorpe Signed-off-by: Li RongQing Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/20260603013723.2439-1-lirongqing@baidu.com --- kernel/dma/direct.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 583c5922bca2..4391b797d4db 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -476,7 +476,7 @@ int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents, * must be mapped with CPU physical address and not PCI * bus addresses. */ - break; + fallthrough; case PCI_P2PDMA_MAP_NONE: need_sync = true; sg->dma_address = dma_direct_map_phys(dev, sg_phys(sg), -- cgit v1.2.3 From 08d4a7837f008ea6031c1292aa839ad881d7b3f1 Mon Sep 17 00:00:00 2001 From: Dmitry Ilvokhin Date: Tue, 2 Jun 2026 07:12:51 +0000 Subject: genirq: Move NULL check into irqdesc_lock guard unlock expression irqdesc_lock uses __DEFINE_UNLOCK_GUARD() directly with a custom constructor that can set .lock to NULL. In preparation for removing the NULL check from __DEFINE_UNLOCK_GUARD(), move the NULL check into the irqdesc_lock unlock expression, making the NULL handling explicit at the call site. No functional change. Signed-off-by: Dmitry Ilvokhin Signed-off-by: Peter Zijlstra (Intel) Acked-by: Thomas Gleixner Link: https://patch.msgid.link/ab457810653e4356e29b2d74ba616478bd9328ad.1780064327.git.d@ilvokhin.com --- kernel/irq/internals.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h index 9412e57056f5..347cb333b9fe 100644 --- a/kernel/irq/internals.h +++ b/kernel/irq/internals.h @@ -171,7 +171,7 @@ void __irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags, bool bus) __DEFINE_CLASS_IS_CONDITIONAL(irqdesc_lock, true); __DEFINE_UNLOCK_GUARD(irqdesc_lock, struct irq_desc, - __irq_put_desc_unlock(_T->lock, _T->flags, _T->bus), + if (_T->lock) __irq_put_desc_unlock(_T->lock, _T->flags, _T->bus), unsigned long flags; bool bus); static inline class_irqdesc_lock_t class_irqdesc_lock_constructor(unsigned int irq, bool bus, -- cgit v1.2.3 From c1ffc9c6e4f8a13dd68e97920c9a24d095c6e41a Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:09:25 +0200 Subject: futex: Move futex task related data into a struct MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Having all these members in task_struct along with the required #ifdeffery is annoying, does not allow efficient initializing of the data with memset() and makes extending it tedious. Move it into a data structure and fix up all usage sites. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Mathieu Desnoyers Reviewed-by: André Almeida Link: https://patch.msgid.link/20260602090535.308220888@kernel.org --- kernel/exit.c | 4 ++-- kernel/futex/core.c | 59 +++++++++++++++++++++++++------------------------ kernel/futex/pi.c | 26 +++++++++++----------- kernel/futex/syscalls.c | 23 +++++++------------ 4 files changed, 53 insertions(+), 59 deletions(-) (limited to 'kernel') diff --git a/kernel/exit.c b/kernel/exit.c index 25e9cb6de7e7..1b4e55b60bc2 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -988,8 +988,8 @@ void __noreturn do_exit(long code) proc_exit_connector(tsk); mpol_put_task_policy(tsk); #ifdef CONFIG_FUTEX - if (unlikely(current->pi_state_cache)) - kfree(current->pi_state_cache); + if (unlikely(current->futex.pi_state_cache)) + kfree(current->futex.pi_state_cache); #endif /* * Make sure we are holding no locks: diff --git a/kernel/futex/core.c b/kernel/futex/core.c index ff2a4fb2993f..e7d33d2771ec 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -32,18 +32,19 @@ * "But they come in a choice of three flavours!" */ #include -#include -#include #include -#include +#include #include -#include +#include #include -#include -#include -#include #include #include +#include +#include +#include +#include +#include +#include #include "futex.h" #include "../locking/rtmutex_common.h" @@ -829,7 +830,7 @@ void wait_for_owner_exiting(int ret, struct task_struct *exiting) if (WARN_ON_ONCE(ret == -EBUSY && !exiting)) return; - mutex_lock(&exiting->futex_exit_mutex); + mutex_lock(&exiting->futex.exit_mutex); /* * No point in doing state checking here. If the waiter got here * while the task was in exec()->exec_futex_release() then it can @@ -838,7 +839,7 @@ void wait_for_owner_exiting(int ret, struct task_struct *exiting) * already. Highly unlikely and not a problem. Just one more round * through the futex maze. */ - mutex_unlock(&exiting->futex_exit_mutex); + mutex_unlock(&exiting->futex.exit_mutex); put_task_struct(exiting); } @@ -1047,7 +1048,7 @@ retry: * * In both cases the following conditions are met: * - * 1) task->robust_list->list_op_pending != NULL + * 1) task->futex.robust_list->list_op_pending != NULL * @pending_op == true * 2) The owner part of user space futex value == 0 * 3) Regular futex: @pi == false @@ -1152,7 +1153,7 @@ static inline int fetch_robust_entry(struct robust_list __user **entry, */ static void exit_robust_list(struct task_struct *curr) { - struct robust_list_head __user *head = curr->robust_list; + struct robust_list_head __user *head = curr->futex.robust_list; struct robust_list __user *entry, *next_entry, *pending; unsigned int limit = ROBUST_LIST_LIMIT, pi, pip; unsigned int next_pi; @@ -1246,7 +1247,7 @@ compat_fetch_robust_entry(compat_uptr_t *uentry, struct robust_list __user **ent */ static void compat_exit_robust_list(struct task_struct *curr) { - struct compat_robust_list_head __user *head = curr->compat_robust_list; + struct compat_robust_list_head __user *head = curr->futex.compat_robust_list; struct robust_list __user *entry, *next_entry, *pending; unsigned int limit = ROBUST_LIST_LIMIT, pi, pip; unsigned int next_pi; @@ -1322,7 +1323,7 @@ static void compat_exit_robust_list(struct task_struct *curr) */ static void exit_pi_state_list(struct task_struct *curr) { - struct list_head *next, *head = &curr->pi_state_list; + struct list_head *next, *head = &curr->futex.pi_state_list; struct futex_pi_state *pi_state; union futex_key key = FUTEX_KEY_INIT; @@ -1406,19 +1407,19 @@ static inline void exit_pi_state_list(struct task_struct *curr) { } static void futex_cleanup(struct task_struct *tsk) { - if (unlikely(tsk->robust_list)) { + if (unlikely(tsk->futex.robust_list)) { exit_robust_list(tsk); - tsk->robust_list = NULL; + tsk->futex.robust_list = NULL; } #ifdef CONFIG_COMPAT - if (unlikely(tsk->compat_robust_list)) { + if (unlikely(tsk->futex.compat_robust_list)) { compat_exit_robust_list(tsk); - tsk->compat_robust_list = NULL; + tsk->futex.compat_robust_list = NULL; } #endif - if (unlikely(!list_empty(&tsk->pi_state_list))) + if (unlikely(!list_empty(&tsk->futex.pi_state_list))) exit_pi_state_list(tsk); } @@ -1442,23 +1443,23 @@ static void futex_cleanup(struct task_struct *tsk) void futex_exit_recursive(struct task_struct *tsk) { /* If the state is FUTEX_STATE_EXITING then futex_exit_mutex is held */ - if (tsk->futex_state == FUTEX_STATE_EXITING) { - __assume_ctx_lock(&tsk->futex_exit_mutex); - mutex_unlock(&tsk->futex_exit_mutex); + if (tsk->futex.state == FUTEX_STATE_EXITING) { + __assume_ctx_lock(&tsk->futex.exit_mutex); + mutex_unlock(&tsk->futex.exit_mutex); } - tsk->futex_state = FUTEX_STATE_DEAD; + tsk->futex.state = FUTEX_STATE_DEAD; } static void futex_cleanup_begin(struct task_struct *tsk) - __acquires(&tsk->futex_exit_mutex) + __acquires(&tsk->futex.exit_mutex) { /* * Prevent various race issues against a concurrent incoming waiter * including live locks by forcing the waiter to block on - * tsk->futex_exit_mutex when it observes FUTEX_STATE_EXITING in + * tsk->futex.exit_mutex when it observes FUTEX_STATE_EXITING in * attach_to_pi_owner(). */ - mutex_lock(&tsk->futex_exit_mutex); + mutex_lock(&tsk->futex.exit_mutex); /* * Switch the state to FUTEX_STATE_EXITING under tsk->pi_lock. @@ -1472,23 +1473,23 @@ static void futex_cleanup_begin(struct task_struct *tsk) * be observed in exit_pi_state_list(). */ raw_spin_lock_irq(&tsk->pi_lock); - tsk->futex_state = FUTEX_STATE_EXITING; + tsk->futex.state = FUTEX_STATE_EXITING; raw_spin_unlock_irq(&tsk->pi_lock); } static void futex_cleanup_end(struct task_struct *tsk, int state) - __releases(&tsk->futex_exit_mutex) + __releases(&tsk->futex.exit_mutex) { /* * Lockless store. The only side effect is that an observer might * take another loop until it becomes visible. */ - tsk->futex_state = state; + tsk->futex.state = state; /* * Drop the exit protection. This unblocks waiters which observed * FUTEX_STATE_EXITING to reevaluate the state. */ - mutex_unlock(&tsk->futex_exit_mutex); + mutex_unlock(&tsk->futex.exit_mutex); } void futex_exec_release(struct task_struct *tsk) diff --git a/kernel/futex/pi.c b/kernel/futex/pi.c index 643199fdbe62..e037a97fe277 100644 --- a/kernel/futex/pi.c +++ b/kernel/futex/pi.c @@ -14,7 +14,7 @@ int refill_pi_state_cache(void) { struct futex_pi_state *pi_state; - if (likely(current->pi_state_cache)) + if (likely(current->futex.pi_state_cache)) return 0; pi_state = kzalloc_obj(*pi_state); @@ -28,17 +28,17 @@ int refill_pi_state_cache(void) refcount_set(&pi_state->refcount, 1); pi_state->key = FUTEX_KEY_INIT; - current->pi_state_cache = pi_state; + current->futex.pi_state_cache = pi_state; return 0; } static struct futex_pi_state *alloc_pi_state(void) { - struct futex_pi_state *pi_state = current->pi_state_cache; + struct futex_pi_state *pi_state = current->futex.pi_state_cache; WARN_ON(!pi_state); - current->pi_state_cache = NULL; + current->futex.pi_state_cache = NULL; return pi_state; } @@ -60,7 +60,7 @@ static void pi_state_update_owner(struct futex_pi_state *pi_state, if (new_owner) { raw_spin_lock(&new_owner->pi_lock); WARN_ON(!list_empty(&pi_state->list)); - list_add(&pi_state->list, &new_owner->pi_state_list); + list_add(&pi_state->list, &new_owner->futex.pi_state_list); pi_state->owner = new_owner; raw_spin_unlock(&new_owner->pi_lock); } @@ -96,7 +96,7 @@ void put_pi_state(struct futex_pi_state *pi_state) raw_spin_unlock_irqrestore(&pi_state->pi_mutex.wait_lock, flags); } - if (current->pi_state_cache) { + if (current->futex.pi_state_cache) { kfree(pi_state); } else { /* @@ -106,7 +106,7 @@ void put_pi_state(struct futex_pi_state *pi_state) */ pi_state->owner = NULL; refcount_set(&pi_state->refcount, 1); - current->pi_state_cache = pi_state; + current->futex.pi_state_cache = pi_state; } } @@ -179,7 +179,7 @@ void put_pi_state(struct futex_pi_state *pi_state) * * p->pi_lock: * - * p->pi_state_list -> pi_state->list, relation + * p->futex.pi_state_list -> pi_state->list, relation * pi_mutex->owner -> pi_state->owner, relation * * pi_state->refcount: @@ -327,7 +327,7 @@ static int handle_exit_race(u32 __user *uaddr, u32 uval, * If the futex exit state is not yet FUTEX_STATE_DEAD, tell the * caller that the alleged owner is busy. */ - if (tsk && tsk->futex_state != FUTEX_STATE_DEAD) + if (tsk && tsk->futex.state != FUTEX_STATE_DEAD) return -EBUSY; /* @@ -346,8 +346,8 @@ static int handle_exit_race(u32 __user *uaddr, u32 uval, * *uaddr = 0xC0000000; tsk = get_task(PID); * } if (!tsk->flags & PF_EXITING) { * ... attach(); - * tsk->futex_state = } else { - * FUTEX_STATE_DEAD; if (tsk->futex_state != + * tsk->futex.state = } else { + * FUTEX_STATE_DEAD; if (tsk->futex.state != * FUTEX_STATE_DEAD) * return -EAGAIN; * return -ESRCH; <--- FAIL @@ -396,7 +396,7 @@ static void __attach_to_pi_owner(struct task_struct *p, union futex_key *key, pi_state->key = *key; WARN_ON(!list_empty(&pi_state->list)); - list_add(&pi_state->list, &p->pi_state_list); + list_add(&pi_state->list, &p->futex.pi_state_list); /* * Assignment without holding pi_state->pi_mutex.wait_lock is safe * because there is no concurrency as the object is not published yet. @@ -440,7 +440,7 @@ static int attach_to_pi_owner(u32 __user *uaddr, u32 uval, union futex_key *key, * in futex_exit_release(), we do this protected by p->pi_lock: */ raw_spin_lock_irq(&p->pi_lock); - if (unlikely(p->futex_state != FUTEX_STATE_OK)) { + if (unlikely(p->futex.state != FUTEX_STATE_OK)) { /* * The task is on the way out. When the futex state is * FUTEX_STATE_DEAD, we know that the task has finished diff --git a/kernel/futex/syscalls.c b/kernel/futex/syscalls.c index 77ad9691f6a6..8944ff4930d7 100644 --- a/kernel/futex/syscalls.c +++ b/kernel/futex/syscalls.c @@ -25,17 +25,13 @@ * @head: pointer to the list-head * @len: length of the list-head, as userspace expects */ -SYSCALL_DEFINE2(set_robust_list, struct robust_list_head __user *, head, - size_t, len) +SYSCALL_DEFINE2(set_robust_list, struct robust_list_head __user *, head, size_t, len) { - /* - * The kernel knows only one size for now: - */ + /* The kernel knows only one size for now. */ if (unlikely(len != sizeof(*head))) return -EINVAL; - current->robust_list = head; - + current->futex.robust_list = head; return 0; } @@ -43,9 +39,9 @@ static inline void __user *futex_task_robust_list(struct task_struct *p, bool co { #ifdef CONFIG_COMPAT if (compat) - return p->compat_robust_list; + return p->futex.compat_robust_list; #endif - return p->robust_list; + return p->futex.robust_list; } static void __user *futex_get_robust_list_common(int pid, bool compat) @@ -475,15 +471,13 @@ SYSCALL_DEFINE4(futex_requeue, } #ifdef CONFIG_COMPAT -COMPAT_SYSCALL_DEFINE2(set_robust_list, - struct compat_robust_list_head __user *, head, - compat_size_t, len) +COMPAT_SYSCALL_DEFINE2(set_robust_list, struct compat_robust_list_head __user *, head, + compat_size_t, len) { if (unlikely(len != sizeof(*head))) return -EINVAL; - current->compat_robust_list = head; - + current->futex.compat_robust_list = head; return 0; } @@ -523,4 +517,3 @@ SYSCALL_DEFINE6(futex_time32, u32 __user *, uaddr, int, op, u32, val, return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3); } #endif /* CONFIG_COMPAT_32BIT_TIME */ - -- cgit v1.2.3 From d7b3f52c861f54ba2fff15696d3798277fb4c19f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:09:29 +0200 Subject: futex: Make futex_mm_init() void Nothing fails there. Mop up the leftovers of the early version of this, which did an allocation. While at it clean up the stubs and the #ifdef comments to make the header file readable. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260602090535.356789395@kernel.org --- kernel/fork.c | 8 ++------ kernel/futex/core.c | 3 +-- 2 files changed, 3 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/fork.c b/kernel/fork.c index 5f3fdfdb14c7..bb490d97c222 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1101,6 +1101,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, #endif mm_init_uprobes_state(mm); hugetlb_count_init(mm); + futex_mm_init(mm); mm_flags_clear_all(mm); if (current->mm) { @@ -1113,11 +1114,8 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, mm->def_flags = 0; } - if (futex_mm_init(mm)) - goto fail_mm_init; - if (mm_alloc_pgd(mm)) - goto fail_nopgd; + goto fail_mm_init; if (mm_alloc_id(mm)) goto fail_noid; @@ -1144,8 +1142,6 @@ fail_nocontext: mm_free_id(mm); fail_noid: mm_free_pgd(mm); -fail_nopgd: - futex_hash_free(mm); fail_mm_init: free_mm(mm); return NULL; diff --git a/kernel/futex/core.c b/kernel/futex/core.c index e7d33d2771ec..ec23de4912b3 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -1720,7 +1720,7 @@ static bool futex_ref_is_dead(struct futex_private_hash *fph) return atomic_long_read(&mm->futex_atomic) == 0; } -int futex_mm_init(struct mm_struct *mm) +void futex_mm_init(struct mm_struct *mm) { mutex_init(&mm->futex_hash_lock); RCU_INIT_POINTER(mm->futex_phash, NULL); @@ -1729,7 +1729,6 @@ int futex_mm_init(struct mm_struct *mm) mm->futex_ref = NULL; atomic_long_set(&mm->futex_atomic, 0); mm->futex_batches = get_state_synchronize_rcu(); - return 0; } void futex_hash_free(struct mm_struct *mm) -- cgit v1.2.3 From 1f7f4816b9b05e5110bc1c8a05c3c478e2dae11b Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:09:34 +0200 Subject: futex: Move futex related mm_struct data into a struct Having all these members in mm_struct along with the required #ifdeffery is annoying, does not allow efficient initializing of the data with memset() and makes extending it tedious. Move it into a data structure and fix up all usage sites. The extra struct for the private hash is intentional to make integration of other conditional mechanisms easier in terms of initialization and separation. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260602090535.407756793@kernel.org --- kernel/futex/core.c | 133 ++++++++++++++++++++++++---------------------------- 1 file changed, 61 insertions(+), 72 deletions(-) (limited to 'kernel') diff --git a/kernel/futex/core.c b/kernel/futex/core.c index ec23de4912b3..79456b0ac1c6 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -188,13 +188,13 @@ __futex_hash_private(union futex_key *key, struct futex_private_hash *fph) return NULL; if (!fph) - fph = rcu_dereference(key->private.mm->futex_phash); + fph = rcu_dereference(key->private.mm->futex.phash.hash); if (!fph || !fph->hash_mask) return NULL; - hash = jhash2((void *)&key->private.address, - sizeof(key->private.address) / 4, + hash = jhash2((void *)&key->private.address, sizeof(key->private.address) / 4, key->both.offset); + return &fph->queues[hash & fph->hash_mask]; } @@ -233,18 +233,17 @@ static void futex_rehash_private(struct futex_private_hash *old, } } -static bool __futex_pivot_hash(struct mm_struct *mm, - struct futex_private_hash *new) +static bool __futex_pivot_hash(struct mm_struct *mm, struct futex_private_hash *new) { + struct futex_mm_phash *mmph = &mm->futex.phash; struct futex_private_hash *fph; - WARN_ON_ONCE(mm->futex_phash_new); + WARN_ON_ONCE(mmph->hash_new); - fph = rcu_dereference_protected(mm->futex_phash, - lockdep_is_held(&mm->futex_hash_lock)); + fph = rcu_dereference_protected(mmph->hash, lockdep_is_held(&mmph->lock)); if (fph) { if (!futex_ref_is_dead(fph)) { - mm->futex_phash_new = new; + mmph->hash_new = new; return false; } @@ -252,8 +251,8 @@ static bool __futex_pivot_hash(struct mm_struct *mm, } new->state = FR_PERCPU; scoped_guard(rcu) { - mm->futex_batches = get_state_synchronize_rcu(); - rcu_assign_pointer(mm->futex_phash, new); + mmph->batches = get_state_synchronize_rcu(); + rcu_assign_pointer(mmph->hash, new); } kvfree_rcu(fph, rcu); return true; @@ -261,12 +260,12 @@ static bool __futex_pivot_hash(struct mm_struct *mm, static void futex_pivot_hash(struct mm_struct *mm) { - scoped_guard(mutex, &mm->futex_hash_lock) { + scoped_guard(mutex, &mm->futex.phash.lock) { struct futex_private_hash *fph; - fph = mm->futex_phash_new; + fph = mm->futex.phash.hash_new; if (fph) { - mm->futex_phash_new = NULL; + mm->futex.phash.hash_new = NULL; __futex_pivot_hash(mm, fph); } } @@ -289,7 +288,7 @@ again: scoped_guard(rcu) { struct futex_private_hash *fph; - fph = rcu_dereference(mm->futex_phash); + fph = rcu_dereference(mm->futex.phash.hash); if (!fph) return NULL; @@ -412,8 +411,7 @@ static int futex_mpol(struct mm_struct *mm, unsigned long addr) * private hash) is returned if existing. Otherwise a hash bucket from the * global hash is returned. */ -static struct futex_hash_bucket * -__futex_hash(union futex_key *key, struct futex_private_hash *fph) +static struct futex_hash_bucket *__futex_hash(union futex_key *key, struct futex_private_hash *fph) { int node = key->both.node; u32 hash; @@ -426,8 +424,7 @@ __futex_hash(union futex_key *key, struct futex_private_hash *fph) return hb; } - hash = jhash2((u32 *)key, - offsetof(typeof(*key), both.offset) / sizeof(u32), + hash = jhash2((u32 *)key, offsetof(typeof(*key), both.offset) / sizeof(u32), key->both.offset); if (node == FUTEX_NO_NODE) { @@ -442,8 +439,7 @@ __futex_hash(union futex_key *key, struct futex_private_hash *fph) */ node = (hash >> futex_hashshift) % nr_node_ids; if (!node_possible(node)) { - node = find_next_bit_wrap(node_possible_map.bits, - nr_node_ids, node); + node = find_next_bit_wrap(node_possible_map.bits, nr_node_ids, node); } } @@ -460,9 +456,8 @@ __futex_hash(union futex_key *key, struct futex_private_hash *fph) * Return: Initialized hrtimer_sleeper structure or NULL if no timeout * value given */ -struct hrtimer_sleeper * -futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout, - int flags, u64 range_ns) +struct hrtimer_sleeper *futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout, + int flags, u64 range_ns) { if (!time) return NULL; @@ -1554,17 +1549,17 @@ static void __futex_ref_atomic_begin(struct futex_private_hash *fph) * otherwise it would be impossible for it to have reported success * from futex_ref_is_dead(). */ - WARN_ON_ONCE(atomic_long_read(&mm->futex_atomic) != 0); + WARN_ON_ONCE(atomic_long_read(&mm->futex.phash.atomic) != 0); /* * Set the atomic to the bias value such that futex_ref_{get,put}() * will never observe 0. Will be fixed up in __futex_ref_atomic_end() * when folding in the percpu count. */ - atomic_long_set(&mm->futex_atomic, LONG_MAX); + atomic_long_set(&mm->futex.phash.atomic, LONG_MAX); smp_store_release(&fph->state, FR_ATOMIC); - call_rcu_hurry(&mm->futex_rcu, futex_ref_rcu); + call_rcu_hurry(&mm->futex.phash.rcu, futex_ref_rcu); } static void __futex_ref_atomic_end(struct futex_private_hash *fph) @@ -1585,7 +1580,7 @@ static void __futex_ref_atomic_end(struct futex_private_hash *fph) * Therefore the per-cpu counter is now stable, sum and reset. */ for_each_possible_cpu(cpu) { - unsigned int *ptr = per_cpu_ptr(mm->futex_ref, cpu); + unsigned int *ptr = per_cpu_ptr(mm->futex.phash.ref, cpu); count += *ptr; *ptr = 0; } @@ -1593,7 +1588,7 @@ static void __futex_ref_atomic_end(struct futex_private_hash *fph) /* * Re-init for the next cycle. */ - this_cpu_inc(*mm->futex_ref); /* 0 -> 1 */ + this_cpu_inc(*mm->futex.phash.ref); /* 0 -> 1 */ /* * Add actual count, subtract bias and initial refcount. @@ -1601,7 +1596,7 @@ static void __futex_ref_atomic_end(struct futex_private_hash *fph) * The moment this atomic operation happens, futex_ref_is_dead() can * become true. */ - ret = atomic_long_add_return(count - LONG_MAX - 1, &mm->futex_atomic); + ret = atomic_long_add_return(count - LONG_MAX - 1, &mm->futex.phash.atomic); if (!ret) wake_up_var(mm); @@ -1611,8 +1606,8 @@ static void __futex_ref_atomic_end(struct futex_private_hash *fph) static void futex_ref_rcu(struct rcu_head *head) { - struct mm_struct *mm = container_of(head, struct mm_struct, futex_rcu); - struct futex_private_hash *fph = rcu_dereference_raw(mm->futex_phash); + struct mm_struct *mm = container_of(head, struct mm_struct, futex.phash.rcu); + struct futex_private_hash *fph = rcu_dereference_raw(mm->futex.phash.hash); if (fph->state == FR_PERCPU) { /* @@ -1641,7 +1636,7 @@ static void futex_ref_drop(struct futex_private_hash *fph) /* * Can only transition the current fph; */ - WARN_ON_ONCE(rcu_dereference_raw(mm->futex_phash) != fph); + WARN_ON_ONCE(rcu_dereference_raw(mm->futex.phash.hash) != fph); /* * We enqueue at least one RCU callback. Ensure mm stays if the task * exits before the transition is completed. @@ -1652,9 +1647,9 @@ static void futex_ref_drop(struct futex_private_hash *fph) * In order to avoid the following scenario: * * futex_hash() __futex_pivot_hash() - * guard(rcu); guard(mm->futex_hash_lock); - * fph = mm->futex_phash; - * rcu_assign_pointer(&mm->futex_phash, new); + * guard(rcu); guard(mm->futex.phash.lock); + * fph = mm->futex.phash.hash; + * rcu_assign_pointer(&mm->futex.phash.hash, new); * futex_hash_allocate() * futex_ref_drop() * fph->state = FR_ATOMIC; @@ -1669,7 +1664,7 @@ static void futex_ref_drop(struct futex_private_hash *fph) * There must be at least one full grace-period between publishing a * new fph and trying to replace it. */ - if (poll_state_synchronize_rcu(mm->futex_batches)) { + if (poll_state_synchronize_rcu(mm->futex.phash.batches)) { /* * There was a grace-period, we can begin now. */ @@ -1677,7 +1672,7 @@ static void futex_ref_drop(struct futex_private_hash *fph) return; } - call_rcu_hurry(&mm->futex_rcu, futex_ref_rcu); + call_rcu_hurry(&mm->futex.phash.rcu, futex_ref_rcu); } static bool futex_ref_get(struct futex_private_hash *fph) @@ -1687,11 +1682,11 @@ static bool futex_ref_get(struct futex_private_hash *fph) guard(preempt)(); if (READ_ONCE(fph->state) == FR_PERCPU) { - __this_cpu_inc(*mm->futex_ref); + __this_cpu_inc(*mm->futex.phash.ref); return true; } - return atomic_long_inc_not_zero(&mm->futex_atomic); + return atomic_long_inc_not_zero(&mm->futex.phash.atomic); } static bool futex_ref_put(struct futex_private_hash *fph) @@ -1701,11 +1696,11 @@ static bool futex_ref_put(struct futex_private_hash *fph) guard(preempt)(); if (READ_ONCE(fph->state) == FR_PERCPU) { - __this_cpu_dec(*mm->futex_ref); + __this_cpu_dec(*mm->futex.phash.ref); return false; } - return atomic_long_dec_and_test(&mm->futex_atomic); + return atomic_long_dec_and_test(&mm->futex.phash.atomic); } static bool futex_ref_is_dead(struct futex_private_hash *fph) @@ -1717,27 +1712,23 @@ static bool futex_ref_is_dead(struct futex_private_hash *fph) if (smp_load_acquire(&fph->state) == FR_PERCPU) return false; - return atomic_long_read(&mm->futex_atomic) == 0; + return atomic_long_read(&mm->futex.phash.atomic) == 0; } void futex_mm_init(struct mm_struct *mm) { - mutex_init(&mm->futex_hash_lock); - RCU_INIT_POINTER(mm->futex_phash, NULL); - mm->futex_phash_new = NULL; - /* futex-ref */ - mm->futex_ref = NULL; - atomic_long_set(&mm->futex_atomic, 0); - mm->futex_batches = get_state_synchronize_rcu(); + memset(&mm->futex, 0, sizeof(mm->futex)); + mutex_init(&mm->futex.phash.lock); + mm->futex.phash.batches = get_state_synchronize_rcu(); } void futex_hash_free(struct mm_struct *mm) { struct futex_private_hash *fph; - free_percpu(mm->futex_ref); - kvfree(mm->futex_phash_new); - fph = rcu_dereference_raw(mm->futex_phash); + free_percpu(mm->futex.phash.ref); + kvfree(mm->futex.phash.hash_new); + fph = rcu_dereference_raw(mm->futex.phash.hash); if (fph) kvfree(fph); } @@ -1748,10 +1739,10 @@ static bool futex_pivot_pending(struct mm_struct *mm) guard(rcu)(); - if (!mm->futex_phash_new) + if (!mm->futex.phash.hash_new) return true; - fph = rcu_dereference(mm->futex_phash); + fph = rcu_dereference(mm->futex.phash.hash); return futex_ref_is_dead(fph); } @@ -1793,7 +1784,7 @@ static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags) * Once we've disabled the global hash there is no way back. */ scoped_guard(rcu) { - fph = rcu_dereference(mm->futex_phash); + fph = rcu_dereference(mm->futex.phash.hash); if (fph && !fph->hash_mask) { if (custom) return -EBUSY; @@ -1801,15 +1792,15 @@ static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags) } } - if (!mm->futex_ref) { + if (!mm->futex.phash.ref) { /* * This will always be allocated by the first thread and * therefore requires no locking. */ - mm->futex_ref = alloc_percpu(unsigned int); - if (!mm->futex_ref) + mm->futex.phash.ref = alloc_percpu(unsigned int); + if (!mm->futex.phash.ref) return -ENOMEM; - this_cpu_inc(*mm->futex_ref); /* 0 -> 1 */ + this_cpu_inc(*mm->futex.phash.ref); /* 0 -> 1 */ } fph = kvzalloc(struct_size(fph, queues, hash_slots), @@ -1832,14 +1823,14 @@ again: wait_var_event(mm, futex_pivot_pending(mm)); } - scoped_guard(mutex, &mm->futex_hash_lock) { + scoped_guard(mutex, &mm->futex.phash.lock) { struct futex_private_hash *free __free(kvfree) = NULL; struct futex_private_hash *cur, *new; - cur = rcu_dereference_protected(mm->futex_phash, - lockdep_is_held(&mm->futex_hash_lock)); - new = mm->futex_phash_new; - mm->futex_phash_new = NULL; + cur = rcu_dereference_protected(mm->futex.phash.hash, + lockdep_is_held(&mm->futex.phash.lock)); + new = mm->futex.phash.hash_new; + mm->futex.phash.hash_new = NULL; if (fph) { if (cur && !cur->hash_mask) { @@ -1849,7 +1840,7 @@ again: * the second one returns here. */ free = fph; - mm->futex_phash_new = new; + mm->futex.phash.hash_new = new; return -EBUSY; } if (cur && !new) { @@ -1879,7 +1870,7 @@ again: if (new) { /* - * Will set mm->futex_phash_new on failure; + * Will set mm->futex.phash.new_hash on failure; * futex_private_hash_get() will try again. */ if (!__futex_pivot_hash(mm, new) && custom) @@ -1898,11 +1889,9 @@ int futex_hash_allocate_default(void) return 0; scoped_guard(rcu) { - threads = min_t(unsigned int, - get_nr_threads(current), - num_online_cpus()); + threads = min_t(unsigned int, get_nr_threads(current), num_online_cpus()); - fph = rcu_dereference(current->mm->futex_phash); + fph = rcu_dereference(current->mm->futex.phash.hash); if (fph) { if (fph->custom) return 0; @@ -1929,7 +1918,7 @@ static int futex_hash_get_slots(void) struct futex_private_hash *fph; guard(rcu)(); - fph = rcu_dereference(current->mm->futex_phash); + fph = rcu_dereference(current->mm->futex.phash.hash); if (fph && fph->hash_mask) return fph->hash_mask + 1; return 0; -- cgit v1.2.3 From 2cb5251d3d64d57c172185b9b608f704b3015f26 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:09:38 +0200 Subject: futex: Provide UABI defines for robust list entry modifiers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The marker for PI futexes in the robust list is a hardcoded 0x1 which lacks any sensible form of documentation. Provide proper defines for the bit and the mask and fix up the usage sites. Thereby convert the boolean pi argument into a modifier argument, which allows new modifier bits to be trivially added and conveyed. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Mathieu Desnoyers Reviewed-by: André Almeida Link: https://patch.msgid.link/20260602090535.458758556@kernel.org --- kernel/futex/core.c | 53 +++++++++++++++++++++++++---------------------------- 1 file changed, 25 insertions(+), 28 deletions(-) (limited to 'kernel') diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 79456b0ac1c6..61f4f559afa1 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -1008,8 +1008,9 @@ void futex_unqueue_pi(struct futex_q *q) * dying task, and do notification if so: */ static int handle_futex_death(u32 __user *uaddr, struct task_struct *curr, - bool pi, bool pending_op) + unsigned int mod, bool pending_op) { + bool pi = !!(mod & FUTEX_ROBUST_MOD_PI); u32 uval, nval, mval; pid_t owner; int err; @@ -1127,21 +1128,21 @@ retry: */ static inline int fetch_robust_entry(struct robust_list __user **entry, struct robust_list __user * __user *head, - unsigned int *pi) + unsigned int *mod) { unsigned long uentry; if (get_user(uentry, (unsigned long __user *)head)) return -EFAULT; - *entry = (void __user *)(uentry & ~1UL); - *pi = uentry & 1; + *entry = (void __user *)(uentry & ~FUTEX_ROBUST_MOD_MASK); + *mod = uentry & FUTEX_ROBUST_MOD_MASK; return 0; } /* - * Walk curr->robust_list (very carefully, it's a userspace list!) + * Walk curr->futex.robust_list (very carefully, it's a userspace list!) * and mark any locks found there dead, and notify any waiters. * * We silently return on any sign of list-walking problem. @@ -1149,9 +1150,8 @@ static inline int fetch_robust_entry(struct robust_list __user **entry, static void exit_robust_list(struct task_struct *curr) { struct robust_list_head __user *head = curr->futex.robust_list; + unsigned int limit = ROBUST_LIST_LIMIT, cur_mod, next_mod, pend_mod; struct robust_list __user *entry, *next_entry, *pending; - unsigned int limit = ROBUST_LIST_LIMIT, pi, pip; - unsigned int next_pi; unsigned long futex_offset; int rc; @@ -1159,7 +1159,7 @@ static void exit_robust_list(struct task_struct *curr) * Fetch the list head (which was registered earlier, via * sys_set_robust_list()): */ - if (fetch_robust_entry(&entry, &head->list.next, &pi)) + if (fetch_robust_entry(&entry, &head->list.next, &cur_mod)) return; /* * Fetch the relative futex offset: @@ -1170,7 +1170,7 @@ static void exit_robust_list(struct task_struct *curr) * Fetch any possibly pending lock-add first, and handle it * if it exists: */ - if (fetch_robust_entry(&pending, &head->list_op_pending, &pip)) + if (fetch_robust_entry(&pending, &head->list_op_pending, &pend_mod)) return; next_entry = NULL; /* avoid warning with gcc */ @@ -1179,20 +1179,20 @@ static void exit_robust_list(struct task_struct *curr) * Fetch the next entry in the list before calling * handle_futex_death: */ - rc = fetch_robust_entry(&next_entry, &entry->next, &next_pi); + rc = fetch_robust_entry(&next_entry, &entry->next, &next_mod); /* * A pending lock might already be on the list, so * don't process it twice: */ if (entry != pending) { if (handle_futex_death((void __user *)entry + futex_offset, - curr, pi, HANDLE_DEATH_LIST)) + curr, cur_mod, HANDLE_DEATH_LIST)) return; } if (rc) return; entry = next_entry; - pi = next_pi; + cur_mod = next_mod; /* * Avoid excessively long or circular lists: */ @@ -1204,7 +1204,7 @@ static void exit_robust_list(struct task_struct *curr) if (pending) { handle_futex_death((void __user *)pending + futex_offset, - curr, pip, HANDLE_DEATH_PENDING); + curr, pend_mod, HANDLE_DEATH_PENDING); } } @@ -1223,29 +1223,28 @@ static void __user *futex_uaddr(struct robust_list __user *entry, */ static inline int compat_fetch_robust_entry(compat_uptr_t *uentry, struct robust_list __user **entry, - compat_uptr_t __user *head, unsigned int *pi) + compat_uptr_t __user *head, unsigned int *pflags) { if (get_user(*uentry, head)) return -EFAULT; - *entry = compat_ptr((*uentry) & ~1); - *pi = (unsigned int)(*uentry) & 1; + *entry = compat_ptr((*uentry) & ~FUTEX_ROBUST_MOD_MASK); + *pflags = (unsigned int)(*uentry) & FUTEX_ROBUST_MOD_MASK; return 0; } /* - * Walk curr->robust_list (very carefully, it's a userspace list!) + * Walk curr->futex.robust_list (very carefully, it's a userspace list!) * and mark any locks found there dead, and notify any waiters. * * We silently return on any sign of list-walking problem. */ static void compat_exit_robust_list(struct task_struct *curr) { - struct compat_robust_list_head __user *head = curr->futex.compat_robust_list; + struct compat_robust_list_head __user *head = current->futex.compat_robust_list; + unsigned int limit = ROBUST_LIST_LIMIT, cur_mod, next_mod, pend_mod; struct robust_list __user *entry, *next_entry, *pending; - unsigned int limit = ROBUST_LIST_LIMIT, pi, pip; - unsigned int next_pi; compat_uptr_t uentry, next_uentry, upending; compat_long_t futex_offset; int rc; @@ -1254,7 +1253,7 @@ static void compat_exit_robust_list(struct task_struct *curr) * Fetch the list head (which was registered earlier, via * sys_set_robust_list()): */ - if (compat_fetch_robust_entry(&uentry, &entry, &head->list.next, &pi)) + if (compat_fetch_robust_entry(&uentry, &entry, &head->list.next, &cur_mod)) return; /* * Fetch the relative futex offset: @@ -1265,8 +1264,7 @@ static void compat_exit_robust_list(struct task_struct *curr) * Fetch any possibly pending lock-add first, and handle it * if it exists: */ - if (compat_fetch_robust_entry(&upending, &pending, - &head->list_op_pending, &pip)) + if (compat_fetch_robust_entry(&upending, &pending, &head->list_op_pending, &pend_mod)) return; next_entry = NULL; /* avoid warning with gcc */ @@ -1276,7 +1274,7 @@ static void compat_exit_robust_list(struct task_struct *curr) * handle_futex_death: */ rc = compat_fetch_robust_entry(&next_uentry, &next_entry, - (compat_uptr_t __user *)&entry->next, &next_pi); + (compat_uptr_t __user *)&entry->next, &next_mod); /* * A pending lock might already be on the list, so * dont process it twice: @@ -1284,15 +1282,14 @@ static void compat_exit_robust_list(struct task_struct *curr) if (entry != pending) { void __user *uaddr = futex_uaddr(entry, futex_offset); - if (handle_futex_death(uaddr, curr, pi, - HANDLE_DEATH_LIST)) + if (handle_futex_death(uaddr, curr, cur_mod, HANDLE_DEATH_LIST)) return; } if (rc) return; uentry = next_uentry; entry = next_entry; - pi = next_pi; + cur_mod = next_mod; /* * Avoid excessively long or circular lists: */ @@ -1304,7 +1301,7 @@ static void compat_exit_robust_list(struct task_struct *curr) if (pending) { void __user *uaddr = futex_uaddr(pending, futex_offset); - handle_futex_death(uaddr, curr, pip, HANDLE_DEATH_PENDING); + handle_futex_death(uaddr, curr, pend_mod, HANDLE_DEATH_PENDING); } } #endif -- cgit v1.2.3 From 3ca9595d9fb6cce6633a5b03d98c2aecb5499838 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:09:55 +0200 Subject: futex: Add support for unlocking robust futexes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unlocking robust non-PI futexes happens in user space with the following sequence: 1) robust_list_set_op_pending(mutex); 2) robust_list_remove(mutex); lval = 0; 3) lval = atomic_xchg(lock, lval); 4) if (lval & WAITERS) 5) sys_futex(WAKE,....); 6) robust_list_clear_op_pending(); That opens a window between #3 and #6 where the mutex could be acquired by some other task which observes that it is the last user and: A) unmaps the mutex memory B) maps a different file, which ends up covering the same address When the original task exits before reaching #6 then the kernel robust list handling observes the pending op entry and tries to fix up user space. In case that the newly mapped data contains the TID of the exiting thread at the address of the mutex/futex the kernel will set the owner died bit in that memory and therefore corrupting unrelated data. PI futexes have a similar problem both for the non-contented user space unlock and the in kernel unlock: 1) robust_list_set_op_pending(mutex); 2) robust_list_remove(mutex); lval = gettid(); 3) if (!atomic_try_cmpxchg(lock, lval, 0)) 4) sys_futex(UNLOCK_PI,....); 5) robust_list_clear_op_pending(); Address the first part of the problem where the futexes have waiters and need to enter the kernel anyway. Add a new FUTEX_ROBUST_UNLOCK flag, which is valid for the sys_futex() FUTEX_UNLOCK_PI, FUTEX_WAKE, FUTEX_WAKE_BITSET operations. This deliberately omits FUTEX_WAKE_OP from this treatment as it's unclear whether this is needed and there is no usage of it in glibc either to investigate. For the futex2 syscall family this needs to be implemented with a new syscall. The sys_futex() case [ab]uses the @uaddr2 argument to hand the pointer to robust_list_head::list_pending_op into the kernel. This argument is only evaluated when the FUTEX_ROBUST_UNLOCK bit is set and is therefore backward compatible. This is an explicit argument to avoid the lookup of the robust list pointer and retrieving the pending op pointer from there. User space has the pointer already available so it can just put it into the @uaddr2 argument. Aside of that this allows the usage of multiple robust lists in the future without any changes to the internal functions as they just operate on the provided pointer. This requires a second flag FUTEX_ROBUST_LIST32 which indicates that the robust list pointer points to an u32 and not to an u64. This is required for two reasons: 1) sys_futex() has no compat variant 2) The gaming emulators use both both 64-bit and compat 32-bit robust lists in the same 64-bit application As a consequence 32-bit applications have to set this flag unconditionally so they can run on a 64-bit kernel in compat mode unmodified. 32-bit kernels return an error code when the flag is not set. 64-bit kernels will happily clear the full 64 bits if user space fails to set it. In case of FUTEX_UNLOCK_PI this clears the robust list pending op when the unlock succeeded. In case of errors, the user space value is still locked by the caller and therefore the above cannot happen. In case of FUTEX_WAKE* this does the unlock of the futex in the kernel and clears the robust list pending op when the unlock was successful. If not, the user space value is still locked and user space has to deal with the returned error. That means that the unlocking of non-PI robust futexes has to use the same try_cmpxchg() unlock scheme as PI futexes. If the clearing of the pending list op fails (fault) then the kernel clears the registered robust list pointer if it matches to prevent that exit() will try to handle invalid data. That's a valid paranoid decision because the robust list head sits usually in the TLS and if the TLS is not longer accessible then the chance for fixing up the resulting mess is very close to zero. The problem of non-contended unlocks still exists and will be addressed separately. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: André Almeida Link: https://patch.msgid.link/20260602090535.670514505@kernel.org --- kernel/futex/core.c | 53 +++++++++++++++++++++++++++++++++++++++++++++++-- kernel/futex/futex.h | 15 ++++++++++++-- kernel/futex/pi.c | 15 ++++++++++++-- kernel/futex/syscalls.c | 13 +++++++++--- kernel/futex/waitwake.c | 30 ++++++++++++++++++++++++++-- 5 files changed, 115 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 61f4f559afa1..77ccb7787c34 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -1062,7 +1062,7 @@ retry: owner = uval & FUTEX_TID_MASK; if (pending_op && !pi && !owner) { - futex_wake(uaddr, FLAGS_SIZE_32 | FLAGS_SHARED, 1, + futex_wake(uaddr, FLAGS_SIZE_32 | FLAGS_SHARED, NULL, 1, FUTEX_BITSET_MATCH_ANY); return 0; } @@ -1116,7 +1116,7 @@ retry: * PI futexes happens in exit_pi_state(): */ if (!pi && (uval & FUTEX_WAITERS)) { - futex_wake(uaddr, FLAGS_SIZE_32 | FLAGS_SHARED, 1, + futex_wake(uaddr, FLAGS_SIZE_32 | FLAGS_SHARED, NULL, 1, FUTEX_BITSET_MATCH_ANY); } @@ -1208,6 +1208,27 @@ static void exit_robust_list(struct task_struct *curr) } } +static bool robust_list_clear_pending(unsigned long __user *pop) +{ + struct robust_list_head __user *head = current->futex.robust_list; + + if (!put_user(0UL, pop)) + return true; + + /* + * Just give up. The robust list head is usually part of TLS, so the + * chance that this gets resolved is close to zero. + * + * If @pop_addr is the robust_list_head::list_op_pending pointer then + * clear the robust list head pointer to prevent further damage when the + * task exits. Better a few stale futexes than corrupted memory. But + * that's mostly an academic exercise. + */ + if (pop == (unsigned long __user *)&head->list_op_pending) + current->futex.robust_list = NULL; + return false; +} + #ifdef CONFIG_COMPAT static void __user *futex_uaddr(struct robust_list __user *entry, compat_long_t futex_offset) @@ -1304,6 +1325,21 @@ static void compat_exit_robust_list(struct task_struct *curr) handle_futex_death(uaddr, curr, pend_mod, HANDLE_DEATH_PENDING); } } + +static bool compat_robust_list_clear_pending(u32 __user *pop) +{ + struct compat_robust_list_head __user *head = current->futex.compat_robust_list; + + if (!put_user(0U, pop)) + return true; + + /* See comment in robust_list_clear_pending(). */ + if (pop == &head->list_op_pending) + current->futex.compat_robust_list = NULL; + return false; +} +#else +static bool compat_robust_list_clear_pending(u32 __user *pop_addr) { return false; } #endif #ifdef CONFIG_FUTEX_PI @@ -1397,6 +1433,19 @@ static void exit_pi_state_list(struct task_struct *curr) static inline void exit_pi_state_list(struct task_struct *curr) { } #endif +bool futex_robust_list_clear_pending(void __user *pop, unsigned int flags) +{ + bool size32bit = !!(flags & FLAGS_ROBUST_LIST32); + + if (!IS_ENABLED(CONFIG_64BIT) && !size32bit) + return false; + + if (IS_ENABLED(CONFIG_64BIT) && size32bit) + return compat_robust_list_clear_pending(pop); + + return robust_list_clear_pending(pop); +} + static void futex_cleanup(struct task_struct *tsk) { if (unlikely(tsk->futex.robust_list)) { diff --git a/kernel/futex/futex.h b/kernel/futex/futex.h index 9f6bf6f585fc..79ef2c709c81 100644 --- a/kernel/futex/futex.h +++ b/kernel/futex/futex.h @@ -40,6 +40,8 @@ #define FLAGS_NUMA 0x0080 #define FLAGS_STRICT 0x0100 #define FLAGS_MPOL 0x0200 +#define FLAGS_ROBUST_UNLOCK 0x0400 +#define FLAGS_ROBUST_LIST32 0x0800 /* FUTEX_ to FLAGS_ */ static inline unsigned int futex_to_flags(unsigned int op) @@ -52,6 +54,12 @@ static inline unsigned int futex_to_flags(unsigned int op) if (op & FUTEX_CLOCK_REALTIME) flags |= FLAGS_CLOCKRT; + if (op & FUTEX_ROBUST_UNLOCK) + flags |= FLAGS_ROBUST_UNLOCK; + + if (op & FUTEX_ROBUST_LIST32) + flags |= FLAGS_ROBUST_LIST32; + return flags; } @@ -449,13 +457,16 @@ extern int futex_unqueue_multiple(struct futex_vector *v, int count); extern int futex_wait_multiple(struct futex_vector *vs, unsigned int count, struct hrtimer_sleeper *to); -extern int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset); +extern int futex_wake(u32 __user *uaddr, unsigned int flags, void __user *pop, + int nr_wake, u32 bitset); extern int futex_wake_op(u32 __user *uaddr1, unsigned int flags, u32 __user *uaddr2, int nr_wake, int nr_wake2, int op); -extern int futex_unlock_pi(u32 __user *uaddr, unsigned int flags); +extern int futex_unlock_pi(u32 __user *uaddr, unsigned int flags, void __user *pop); extern int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int trylock); +bool futex_robust_list_clear_pending(void __user *pop, unsigned int flags); + #endif /* _FUTEX_H */ diff --git a/kernel/futex/pi.c b/kernel/futex/pi.c index e037a97fe277..9dd5c0b1ac6a 100644 --- a/kernel/futex/pi.c +++ b/kernel/futex/pi.c @@ -1139,7 +1139,7 @@ out: * This is the in-kernel slowpath: we look up the PI state (if any), * and do the rt-mutex unlock. */ -int futex_unlock_pi(u32 __user *uaddr, unsigned int flags) +static int __futex_unlock_pi(u32 __user *uaddr, unsigned int flags) { u32 curval, uval, vpid = task_pid_vnr(current); union futex_key key = FUTEX_KEY_INIT; @@ -1148,7 +1148,6 @@ int futex_unlock_pi(u32 __user *uaddr, unsigned int flags) if (!IS_ENABLED(CONFIG_FUTEX_PI)) return -ENOSYS; - retry: if (get_user(uval, uaddr)) return -EFAULT; @@ -1302,3 +1301,15 @@ pi_faulted: return ret; } +int futex_unlock_pi(u32 __user *uaddr, unsigned int flags, void __user *pop) +{ + int ret = __futex_unlock_pi(uaddr, flags); + + if (ret || !(flags & FLAGS_ROBUST_UNLOCK)) + return ret; + + if (!futex_robust_list_clear_pending(pop, flags)) + return -EFAULT; + + return 0; +} diff --git a/kernel/futex/syscalls.c b/kernel/futex/syscalls.c index 8944ff4930d7..2fa19d9d008d 100644 --- a/kernel/futex/syscalls.c +++ b/kernel/futex/syscalls.c @@ -118,6 +118,13 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, return -ENOSYS; } + if (flags & FLAGS_ROBUST_UNLOCK) { + if (cmd != FUTEX_WAKE && + cmd != FUTEX_WAKE_BITSET && + cmd != FUTEX_UNLOCK_PI) + return -ENOSYS; + } + switch (cmd) { case FUTEX_WAIT: val3 = FUTEX_BITSET_MATCH_ANY; @@ -128,7 +135,7 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, val3 = FUTEX_BITSET_MATCH_ANY; fallthrough; case FUTEX_WAKE_BITSET: - return futex_wake(uaddr, flags, val, val3); + return futex_wake(uaddr, flags, uaddr2, val, val3); case FUTEX_REQUEUE: return futex_requeue(uaddr, flags, uaddr2, flags, val, val2, NULL, 0); case FUTEX_CMP_REQUEUE: @@ -141,7 +148,7 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, case FUTEX_LOCK_PI2: return futex_lock_pi(uaddr, flags, timeout, 0); case FUTEX_UNLOCK_PI: - return futex_unlock_pi(uaddr, flags); + return futex_unlock_pi(uaddr, flags, uaddr2); case FUTEX_TRYLOCK_PI: return futex_lock_pi(uaddr, flags, NULL, 1); case FUTEX_WAIT_REQUEUE_PI: @@ -375,7 +382,7 @@ SYSCALL_DEFINE4(futex_wake, if (!futex_validate_input(flags, mask)) return -EINVAL; - return futex_wake(uaddr, FLAGS_STRICT | flags, nr, mask); + return futex_wake(uaddr, FLAGS_STRICT | flags, NULL, nr, mask); } /* diff --git a/kernel/futex/waitwake.c b/kernel/futex/waitwake.c index ceed9d879059..8f5e5d302356 100644 --- a/kernel/futex/waitwake.c +++ b/kernel/futex/waitwake.c @@ -149,13 +149,36 @@ void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q) wake_q_add_safe(wake_q, p); } +/* + * If requested, clear the robust list pending op and unlock the futex + */ +static bool futex_robust_unlock(u32 __user *uaddr, unsigned int flags, void __user *pop) +{ + if (!(flags & FLAGS_ROBUST_UNLOCK)) + return true; + + /* First unlock the futex, which requires release semantics. */ + scoped_user_write_access(uaddr, efault) + unsafe_atomic_store_release_user(0, uaddr, efault); + + /* + * Clear the pending list op now. If that fails, then the task is in + * deeper trouble as the robust list head is usually part of the TLS. + * The chance of survival is close to zero. + */ + return futex_robust_list_clear_pending(pop, flags); + +efault: + return false; +} + /* * Wake up waiters matching bitset queued on this futex (uaddr). */ -int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset) +int futex_wake(u32 __user *uaddr, unsigned int flags, void __user *pop, int nr_wake, u32 bitset) { - struct futex_q *this, *next; union futex_key key = FUTEX_KEY_INIT; + struct futex_q *this, *next; DEFINE_WAKE_Q(wake_q); int ret; @@ -166,6 +189,9 @@ int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset) if (unlikely(ret != 0)) return ret; + if (!futex_robust_unlock(uaddr, flags, pop)) + return -EFAULT; + if ((flags & FLAGS_STRICT) && !nr_wake) return 0; -- cgit v1.2.3 From 042df0c1d48609a85580dcbaff498c95ced20a5f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:09:59 +0200 Subject: futex: Add robust futex unlock IP range MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There will be a VDSO function to unlock robust futexes in user space. The unlock sequence is racy vs. clearing the list_pending_op pointer in the tasks robust list head. To plug this race the kernel needs to know the instruction window. As the VDSO is per MM the addresses are stored in mm_struct::futex. Architectures which implement support for this have to update these addresses when the VDSO is (re)mapped and indicate the pending op pointer size which is matching the IP. Arguably this could be resolved by chasing mm->context->vdso->image, but that's architecture specific and requires to touch quite some cache lines. Having it in mm::futex reduces the cache line impact and avoids having yet another set of architecture specific functionality. To support multi size robust list applications (gaming) this provides two ranges when COMPAT is enabled. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: André Almeida Link: https://patch.msgid.link/20260602090535.718926819@kernel.org --- kernel/futex/core.c | 46 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 77ccb7787c34..aad6e501fabd 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -1761,11 +1761,11 @@ static bool futex_ref_is_dead(struct futex_private_hash *fph) return atomic_long_read(&mm->futex.phash.atomic) == 0; } -void futex_mm_init(struct mm_struct *mm) +static void futex_hash_init_mm(struct futex_mm_data *fd) { - memset(&mm->futex, 0, sizeof(mm->futex)); - mutex_init(&mm->futex.phash.lock); - mm->futex.phash.batches = get_state_synchronize_rcu(); + memset(&fd->phash, 0, sizeof(fd->phash)); + mutex_init(&fd->phash.lock); + fd->phash.batches = get_state_synchronize_rcu(); } void futex_hash_free(struct mm_struct *mm) @@ -1969,19 +1969,47 @@ static int futex_hash_get_slots(void) return fph->hash_mask + 1; return 0; } +#else /* CONFIG_FUTEX_PRIVATE_HASH */ +static inline int futex_hash_allocate(unsigned int hslots, unsigned int flags) { return -EINVAL; } +static inline int futex_hash_get_slots(void) { return 0; } +static inline void futex_hash_init_mm(struct futex_mm_data *fd) { } +#endif /* !CONFIG_FUTEX_PRIVATE_HASH */ -#else +#ifdef CONFIG_FUTEX_ROBUST_UNLOCK +static void futex_invalidate_cs_ranges(struct futex_mm_data *fd) +{ + /* + * Invalidate start_ip so that the quick check fails for ip >= start_ip + * if VDSO is not mapped or the second slot is not available for compat + * tasks as they use VDSO32 which does not provide the 64-bit pointer + * variant. + */ + for (int i = 0; i < FUTEX_ROBUST_MAX_CS_RANGES; i++) + fd->unlock.cs_ranges[i].start_ip = ~0UL; +} -static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags) +void futex_reset_cs_ranges(struct futex_mm_data *fd) { - return -EINVAL; + memset(fd->unlock.cs_ranges, 0, sizeof(fd->unlock.cs_ranges)); + futex_invalidate_cs_ranges(fd); } -static int futex_hash_get_slots(void) +static void futex_robust_unlock_init_mm(struct futex_mm_data *fd) { - return 0; + /* mm_dup() preserves the range, mm_alloc() clears it */ + if (!fd->unlock.cs_ranges[0].start_ip) + futex_invalidate_cs_ranges(fd); } +#else /* CONFIG_FUTEX_ROBUST_UNLOCK */ +static inline void futex_robust_unlock_init_mm(struct futex_mm_data *fd) { } +#endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */ +#if defined(CONFIG_FUTEX_PRIVATE_HASH) || defined(CONFIG_FUTEX_ROBUST_UNLOCK) +void futex_mm_init(struct mm_struct *mm) +{ + futex_hash_init_mm(&mm->futex); + futex_robust_unlock_init_mm(&mm->futex); +} #endif int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4) -- cgit v1.2.3 From 7010c39d8fc5063af69ee63f905e592e046f8e5d Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:10:04 +0200 Subject: futex: Provide infrastructure to plug the non contended robust futex unlock race MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the FUTEX_ROBUST_UNLOCK mechanism is used for unlocking (PI-)futexes, then the unlock sequence in user space looks like this: 1) robust_list_set_op_pending(mutex); 2) robust_list_remove(mutex); lval = gettid(); 3) if (atomic_try_cmpxchg(&mutex->lock, lval, 0)) 4) robust_list_clear_op_pending(); else 5) sys_futex(OP | FUTEX_ROBUST_UNLOCK, ....); That still leaves a minimal race window between #3 and #4 where the mutex could be acquired by some other task, which observes that it is the last user and: 1) unmaps the mutex memory 2) maps a different file, which ends up covering the same address When then the original task exits before reaching #5 then the kernel robust list handling observes the pending op entry and tries to fix up user space. In case that the newly mapped data contains the TID of the exiting thread at the address of the mutex/futex the kernel will set the owner died bit in that memory and therefore corrupt unrelated data. On X86 this boils down to this simplified assembly sequence: mov %esi,%eax // Load TID into EAX xor %ecx,%ecx // Set ECX to 0 #3 lock cmpxchg %ecx,(%rdi) // Try the TID -> 0 transition .Lstart: jnz .Lend #4 movq %rcx,(%rdx) // Clear list_op_pending .Lend: If the cmpxchg() succeeds and the task is interrupted before it can clear list_op_pending in the robust list head (#4) and the task crashes in a signal handler or gets killed then it ends up in do_exit() and subsequently in the robust list handling, which then might run into the unmap/map issue described above. This is only relevant when user space was interrupted and a signal is pending. The fix-up has to be done before signal delivery is attempted because: 1) The signal might be fatal so get_signal() ends up in do_exit() 2) The signal handler might crash or the task is killed before returning from the handler. At that point the instruction pointer in pt_regs is not longer the instruction pointer of the initially interrupted unlock sequence. The right place to handle this is in __exit_to_user_mode_loop() before invoking arch_do_signal_or_restart() as this covers obviously both scenarios. As this is only relevant when the task was interrupted in user space, this is tied to RSEQ and the generic entry code as RSEQ keeps track of user space interrupts unconditionally even if the task does not have a RSEQ region installed. That makes the decision very lightweight: if (current->rseq.user_irq && within(regs, csr->unlock_ip_range)) futex_fixup_robust_unlock(regs, csr); futex_fixup_robust_unlock() then invokes a architecture specific function to return the pending op pointer or NULL. The function evaluates the register content to decide whether the pending ops pointer in the robust list head needs to be cleared. Assuming the above unlock sequence, then on x86 this decision is the trivial evaluation of the zero flag: return regs->eflags & X86_EFLAGS_ZF ? regs->dx : NULL; Other architectures might need to do more complex evaluations due to LLSC, but the approach is valid in general. The size of the pointer is determined from the matching range struct, which covers both 32-bit and 64-bit builds including COMPAT. The unlock sequence is going to be placed in the VDSO so that the kernel can keep everything synchronized, especially the register usage. The resulting code sequence for user space is: if (__vdso_futex_robust_list$SZ_try_unlock(lock, tid, &pending_op) != tid) err = sys_futex($OP | FUTEX_ROBUST_UNLOCK,....); Both the VDSO unlock and the kernel side unlock ensure that the pending_op pointer is always cleared when the lock becomes unlocked. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: André Almeida Link: https://patch.msgid.link/20260602090535.773669210@kernel.org --- kernel/entry/common.c | 9 ++++++--- kernel/futex/core.c | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/entry/common.c b/kernel/entry/common.c index 19d2244a9fef..e3d381fd3d25 100644 --- a/kernel/entry/common.c +++ b/kernel/entry/common.c @@ -1,11 +1,12 @@ // SPDX-License-Identifier: GPL-2.0 -#include -#include +#include #include +#include #include #include #include +#include #include /* Workaround to allow gradual conversion of architecture code */ @@ -60,8 +61,10 @@ static __always_inline unsigned long __exit_to_user_mode_loop(struct pt_regs *re if (ti_work & _TIF_PATCH_PENDING) klp_update_patch_state(current); - if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) + if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) { + futex_fixup_robust_unlock(regs); arch_do_signal_or_restart(regs); + } if (ti_work & _TIF_NOTIFY_RESUME) resume_user_mode_work(regs); diff --git a/kernel/futex/core.c b/kernel/futex/core.c index aad6e501fabd..6ea4a97796a1 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -46,6 +46,8 @@ #include #include +#include + #include "futex.h" #include "../locking/rtmutex_common.h" @@ -1446,6 +1448,22 @@ bool futex_robust_list_clear_pending(void __user *pop, unsigned int flags) return robust_list_clear_pending(pop); } +#ifdef CONFIG_FUTEX_ROBUST_UNLOCK +void __futex_fixup_robust_unlock(struct pt_regs *regs, struct futex_unlock_cs_range *csr) +{ + /* + * arch_futex_robust_unlock_get_pop() returns the list pending op pointer from + * @regs if the try_cmpxchg() succeeded. + */ + void __user *pop = arch_futex_robust_unlock_get_pop(regs); + + if (!pop) + return; + + futex_robust_list_clear_pending(pop, csr->pop_size32 ? FLAGS_ROBUST_LIST32 : 0); +} +#endif /* CONFIG_FUTEX_ROBUST_UNLOCK */ + static void futex_cleanup(struct task_struct *tsk) { if (unlikely(tsk->futex.robust_list)) { -- cgit v1.2.3 From 4793e8a6e20e4c17ed4e755ea40e5912cc3539af Mon Sep 17 00:00:00 2001 From: Gabriele Monaco Date: Mon, 1 Jun 2026 17:38:28 +0200 Subject: rv: Fix __user specifier usage in extract_params() The attributes variables extracted from syscalls in the helper are both defined with the __user specifier although only the actual pointer to user data should be marked. Remove the __user specifier from attr. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202604150820.Ny143u6X-lkp@intel.com Fixes: b133207deb72 ("rv: Add nomiss deadline monitor") Reviewed-by: Wen Yang Reviewed-by: Nam Cao Link: https://lore.kernel.org/r/20260601153840.124372-2-gmonaco@redhat.com Signed-off-by: Gabriele Monaco --- kernel/trace/rv/monitors/deadline/deadline.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/trace/rv/monitors/deadline/deadline.h b/kernel/trace/rv/monitors/deadline/deadline.h index 0bbfd2543329..78fca873d61e 100644 --- a/kernel/trace/rv/monitors/deadline/deadline.h +++ b/kernel/trace/rv/monitors/deadline/deadline.h @@ -95,7 +95,8 @@ static inline u8 get_server_type(struct task_struct *tsk) static inline int extract_params(struct pt_regs *regs, long id, pid_t *pid_out) { size_t size = offsetofend(struct sched_attr, sched_flags); - struct sched_attr __user *uattr, attr; + struct sched_attr __user *uattr; + struct sched_attr attr; int new_policy = -1, ret; unsigned long args[6]; -- cgit v1.2.3 From 700782ec8f6589c5792b323efd6e004dd183328b Mon Sep 17 00:00:00 2001 From: Gabriele Monaco Date: Mon, 1 Jun 2026 17:38:34 +0200 Subject: rv: Add automatic cleanup handlers for per-task HA monitors Hybrid automata monitors may start timers, depending on the model, these may remain active on an exiting task and cause false positives or even access freed memory. Add an enable/disable hook in the HA code, currently only populated by the per-task handler for registration and deregistration. This hooks to the sched_process_exit event and ensures the timer is stopped for every exiting task. The handler is enabled automatically but may be disabled, for instance if the monitor uses the event for another purpose (but should still manually ensure timers are stopped). Fixes: f5587d1b6ec9 ("rv: Add Hybrid Automata monitor type") Reviewed-by: Nam Cao Link: https://lore.kernel.org/r/20260601153840.124372-8-gmonaco@redhat.com Signed-off-by: Gabriele Monaco --- kernel/trace/rv/monitors/nomiss/nomiss.c | 4 ++-- kernel/trace/rv/monitors/opid/opid.c | 4 ++-- kernel/trace/rv/monitors/stall/stall.c | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/rv/monitors/nomiss/nomiss.c b/kernel/trace/rv/monitors/nomiss/nomiss.c index 31f90f3638d8..8ead8783c29f 100644 --- a/kernel/trace/rv/monitors/nomiss/nomiss.c +++ b/kernel/trace/rv/monitors/nomiss/nomiss.c @@ -227,7 +227,7 @@ static int enable_nomiss(void) { int retval; - retval = da_monitor_init(); + retval = ha_monitor_init(); if (retval) return retval; @@ -263,7 +263,7 @@ static void disable_nomiss(void) rv_detach_trace_probe("nomiss", sched_switch, handle_sched_switch); rv_detach_trace_probe("nomiss", sched_wakeup, handle_sched_wakeup); - da_monitor_destroy(); + ha_monitor_destroy(); } static struct rv_monitor rv_this = { diff --git a/kernel/trace/rv/monitors/opid/opid.c b/kernel/trace/rv/monitors/opid/opid.c index 4594c7c46601..2922318c6112 100644 --- a/kernel/trace/rv/monitors/opid/opid.c +++ b/kernel/trace/rv/monitors/opid/opid.c @@ -73,7 +73,7 @@ static int enable_opid(void) { int retval; - retval = da_monitor_init(); + retval = ha_monitor_init(); if (retval) return retval; @@ -90,7 +90,7 @@ static void disable_opid(void) rv_detach_trace_probe("opid", sched_set_need_resched_tp, handle_sched_need_resched); rv_detach_trace_probe("opid", sched_waking, handle_sched_waking); - da_monitor_destroy(); + ha_monitor_destroy(); } /* diff --git a/kernel/trace/rv/monitors/stall/stall.c b/kernel/trace/rv/monitors/stall/stall.c index 9ccfda6b0e73..3c38fb1a0159 100644 --- a/kernel/trace/rv/monitors/stall/stall.c +++ b/kernel/trace/rv/monitors/stall/stall.c @@ -103,7 +103,7 @@ static int enable_stall(void) { int retval; - retval = da_monitor_init(); + retval = ha_monitor_init(); if (retval) return retval; @@ -120,7 +120,7 @@ static void disable_stall(void) rv_detach_trace_probe("stall", sched_switch, handle_sched_switch); rv_detach_trace_probe("stall", sched_wakeup, handle_sched_wakeup); - da_monitor_destroy(); + ha_monitor_destroy(); } static struct rv_monitor rv_this = { -- cgit v1.2.3 From d9022172c1abea9e220d13ee453813dafec5b9e4 Mon Sep 17 00:00:00 2001 From: Gabriele Monaco Date: Mon, 1 Jun 2026 17:38:37 +0200 Subject: rv: Use 0 to check preemption enabled in opid Tracepoint handlers no longer run with preemption disabled by default since a46023d5616 ("tracing: Guard __DECLARE_TRACE() use of __DO_TRACE_CALL() with SRCU-fast"), the opid monitor should now count 1 in the preemption count as preemption disabled. Change the rule for preempt_off to preempt > 0. Reviewed-by: Nam Cao Link: https://lore.kernel.org/r/20260601153840.124372-11-gmonaco@redhat.com Signed-off-by: Gabriele Monaco --- kernel/trace/rv/monitors/opid/opid.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/rv/monitors/opid/opid.c b/kernel/trace/rv/monitors/opid/opid.c index 2922318c6112..3b6a85e815b8 100644 --- a/kernel/trace/rv/monitors/opid/opid.c +++ b/kernel/trace/rv/monitors/opid/opid.c @@ -22,14 +22,8 @@ static u64 ha_get_env(struct ha_monitor *ha_mon, enum envs_opid env, u64 time_ns if (env == irq_off_opid) return irqs_disabled(); else if (env == preempt_off_opid) { - /* - * If CONFIG_PREEMPTION is enabled, then the tracepoint itself disables - * preemption (adding one to the preempt_count). Since we are - * interested in the preempt_count at the time the tracepoint was - * hit, we consider 1 as still enabled. - */ if (IS_ENABLED(CONFIG_PREEMPTION)) - return (preempt_count() & PREEMPT_MASK) > 1; + return (preempt_count() & PREEMPT_MASK) > 0; return true; } return ENV_INVALID_VALUE; -- cgit v1.2.3 From 9bfaa86b405381326c971984fd6da184c289713f Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Wed, 3 Jun 2026 20:37:08 +0800 Subject: dma-debug: fix physical address retrieval in debug_dma_sync_sg_for_device In debug_dma_sync_sg_for_device(), when iterating over a scatterlist, the debug entry population mistakenly uses the head of the scatterlist 'sg' to fetch the physical address via sg_phys(), instead of using the current iterator variable 's'. This causes dma-debug to track the physical address of the very first scatterlist entry for all subsequent entries in the list. Fix this by passing the correct loop iterator 's' to sg_phys() Fixes: 9d4f645a1fd49ee ("dma-debug: store a phys_addr_t in struct dma_debug_entry") Signed-off-by: Li RongQing Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/20260603123708.1665-1-lirongqing@baidu.com --- kernel/dma/debug.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c index 3248f8b4d096..2c0e2cd89b5e 100644 --- a/kernel/dma/debug.c +++ b/kernel/dma/debug.c @@ -1556,7 +1556,7 @@ void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, struct dma_debug_entry ref = { .type = dma_debug_sg, .dev = dev, - .paddr = sg_phys(sg), + .paddr = sg_phys(s), .dev_addr = sg_dma_address(s), .size = sg_dma_len(s), .direction = direction, -- cgit v1.2.3 From 81fbb909ec07868415f6b2269922c8d1cc6a215a Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Wed, 3 Jun 2026 15:43:50 +0000 Subject: liveupdate: change file_set->count type to u64 for type safety This improves type safety and aligns the in-memory file_set->count with the serialized count type. It avoids potential truncation or sign conversion mismatch issues. Reviewed-by: Pratyush Yadav (Google) Acked-by: Mike Rapoport (Microsoft) Signed-off-by: Pasha Tatashin Link: https://patch.msgid.link/20260603154402.468928-2-pasha.tatashin@soleen.com Signed-off-by: Mike Rapoport (Microsoft) --- kernel/liveupdate/luo_internal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/liveupdate/luo_internal.h b/kernel/liveupdate/luo_internal.h index dd53d4a7277e..ae58206f14ac 100644 --- a/kernel/liveupdate/luo_internal.h +++ b/kernel/liveupdate/luo_internal.h @@ -52,7 +52,7 @@ static inline int luo_ucmd_respond(struct luo_ucmd *ucmd, struct luo_file_set { struct list_head files_list; struct luo_file_ser *files; - long count; + u64 count; }; /** -- cgit v1.2.3 From 6af06e11bd48bdefaf9381f6ff0bd65b1e5d98ab Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Wed, 3 Jun 2026 15:43:51 +0000 Subject: liveupdate: avoid mixing cleanup guards with goto in luo_session_retrieve_fd Refactoring luo_session_retrieve_fd() to avoid mixing automated cleanup-style guards with goto-based resource release, which is not recommended under the Linux kernel coding style. Reviewed-by: Pratyush Yadav (Google) Acked-by: Mike Rapoport (Microsoft) Signed-off-by: Pasha Tatashin Link: https://patch.msgid.link/20260603154402.468928-3-pasha.tatashin@soleen.com Signed-off-by: Mike Rapoport (Microsoft) --- kernel/liveupdate/luo_session.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/liveupdate/luo_session.c b/kernel/liveupdate/luo_session.c index 5c6cebc6e326..47566db64598 100644 --- a/kernel/liveupdate/luo_session.c +++ b/kernel/liveupdate/luo_session.c @@ -291,10 +291,11 @@ static int luo_session_retrieve_fd(struct luo_session *session, if (argp->fd < 0) return argp->fd; - guard(mutex)(&session->mutex); + mutex_lock(&session->mutex); err = luo_retrieve_file(&session->file_set, argp->token, &file); + mutex_unlock(&session->mutex); if (err < 0) - goto err_put_fd; + goto err_put_fd; err = luo_ucmd_respond(ucmd, sizeof(*argp)); if (err) -- cgit v1.2.3 From d376e4b55c9a0adb3e701c7eaff21d9ba655a1c6 Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Wed, 3 Jun 2026 15:43:52 +0000 Subject: liveupdate: centralize state management into struct luo_ser Transition the LUO to ABI v2, which centralizes state management into a single struct luo_ser header. Previously, LUO state was spread across multiple FDT properties and subnodes. ABI v2 simplifies this by placing all core state, including the liveupdate number and physical addresses for sessions and FLB headers into a centralized struct luo_ser. Note that this change introduces a semantic difference: the sessions and FLB serialization formats are no longer completely independent of the core LUO. Their metadata (such as physical addresses for sessions and FLB headers) is now coupled to and managed via the centralized struct luo_ser. Reviewed-by: Pratyush Yadav (Google) Acked-by: Mike Rapoport (Microsoft) Signed-off-by: Pasha Tatashin Link: https://patch.msgid.link/20260603154402.468928-4-pasha.tatashin@soleen.com Signed-off-by: Mike Rapoport (Microsoft) --- kernel/liveupdate/luo_core.c | 64 ++++++++++++++++++++++++++++------------ kernel/liveupdate/luo_flb.c | 60 ++++--------------------------------- kernel/liveupdate/luo_internal.h | 8 ++--- kernel/liveupdate/luo_session.c | 64 +++++++--------------------------------- 4 files changed, 65 insertions(+), 131 deletions(-) (limited to 'kernel') diff --git a/kernel/liveupdate/luo_core.c b/kernel/liveupdate/luo_core.c index 5d5827ced73c..085c0dfc1ef1 100644 --- a/kernel/liveupdate/luo_core.c +++ b/kernel/liveupdate/luo_core.c @@ -61,7 +61,6 @@ #include #include #include -#include #include "kexec_handover_internal.h" #include "luo_internal.h" @@ -86,9 +85,11 @@ early_param("liveupdate", early_liveupdate_param); static int __init luo_early_startup(void) { + struct luo_ser *luo_ser; + int err, header_size; phys_addr_t fdt_phys; - int err, ln_size; const void *ptr; + u64 luo_ser_pa; if (!kho_is_enabled()) { if (liveupdate_enabled()) @@ -119,26 +120,32 @@ static int __init luo_early_startup(void) return -EINVAL; } - ln_size = 0; - ptr = fdt_getprop(luo_global.fdt_in, 0, LUO_FDT_LIVEUPDATE_NUM, - &ln_size); - if (!ptr || ln_size != sizeof(luo_global.liveupdate_num)) { - pr_err("Unable to get live update number '%s' [%d]\n", - LUO_FDT_LIVEUPDATE_NUM, ln_size); + header_size = 0; + ptr = fdt_getprop(luo_global.fdt_in, 0, LUO_FDT_ABI_HEADER, &header_size); + if (!ptr || header_size != sizeof(u64)) { + pr_err("Unable to get ABI header '%s' [%d]\n", + LUO_FDT_ABI_HEADER, header_size); return -EINVAL; } - luo_global.liveupdate_num = get_unaligned((u64 *)ptr); + luo_ser_pa = get_unaligned((u64 *)ptr); + luo_ser = phys_to_virt(luo_ser_pa); + + luo_global.liveupdate_num = luo_ser->liveupdate_num; pr_info("Retrieved live update data, liveupdate number: %lld\n", luo_global.liveupdate_num); - err = luo_session_setup_incoming(luo_global.fdt_in); + err = luo_session_setup_incoming(luo_ser->sessions_pa); if (err) - return err; + goto out_free_ser; + + luo_flb_setup_incoming(luo_ser->flbs_pa); - err = luo_flb_setup_incoming(luo_global.fdt_in); + err = 0; +out_free_ser: + kho_restore_free(luo_ser); return err; } @@ -160,7 +167,8 @@ early_initcall(liveupdate_early_init); /* Called during boot to create outgoing LUO fdt tree */ static int __init luo_fdt_setup(void) { - const u64 ln = luo_global.liveupdate_num + 1; + struct luo_ser *luo_ser; + u64 luo_ser_pa; void *fdt_out; int err; @@ -170,27 +178,45 @@ static int __init luo_fdt_setup(void) return PTR_ERR(fdt_out); } + luo_ser = kho_alloc_preserve(sizeof(*luo_ser)); + if (IS_ERR(luo_ser)) { + err = PTR_ERR(luo_ser); + goto exit_free_fdt; + } + luo_ser_pa = virt_to_phys(luo_ser); + err = fdt_create(fdt_out, LUO_FDT_SIZE); err |= fdt_finish_reservemap(fdt_out); err |= fdt_begin_node(fdt_out, ""); err |= fdt_property_string(fdt_out, "compatible", LUO_FDT_COMPATIBLE); - err |= fdt_property(fdt_out, LUO_FDT_LIVEUPDATE_NUM, &ln, sizeof(ln)); - err |= luo_session_setup_outgoing(fdt_out); - err |= luo_flb_setup_outgoing(fdt_out); + err |= fdt_property(fdt_out, LUO_FDT_ABI_HEADER, &luo_ser_pa, + sizeof(luo_ser_pa)); err |= fdt_end_node(fdt_out); err |= fdt_finish(fdt_out); if (err) - goto exit_free; + goto exit_free_luo_ser; + + err = luo_session_setup_outgoing(&luo_ser->sessions_pa); + if (err) + goto exit_free_luo_ser; + + err = luo_flb_setup_outgoing(&luo_ser->flbs_pa); + if (err) + goto exit_free_luo_ser; + + luo_ser->liveupdate_num = luo_global.liveupdate_num + 1; err = kho_add_subtree(LUO_FDT_KHO_ENTRY_NAME, fdt_out, fdt_totalsize(fdt_out)); if (err) - goto exit_free; + goto exit_free_luo_ser; luo_global.fdt_out = fdt_out; return 0; -exit_free: +exit_free_luo_ser: + kho_unpreserve_free(luo_ser); +exit_free_fdt: kho_unpreserve_free(fdt_out); pr_err("failed to prepare LUO FDT: %d\n", err); diff --git a/kernel/liveupdate/luo_flb.c b/kernel/liveupdate/luo_flb.c index 8f5c5dd01cd0..5c27134ce7ba 100644 --- a/kernel/liveupdate/luo_flb.c +++ b/kernel/liveupdate/luo_flb.c @@ -44,13 +44,11 @@ #include #include #include -#include #include #include #include #include #include -#include #include "luo_internal.h" #define LUO_FLB_PGCNT 1ul @@ -551,27 +549,15 @@ int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, void **objp) return 0; } -int __init luo_flb_setup_outgoing(void *fdt_out) +int __init luo_flb_setup_outgoing(u64 *flbs_pa) { struct luo_flb_header_ser *header_ser; - u64 header_ser_pa; - int err; header_ser = kho_alloc_preserve(LUO_FLB_PGCNT << PAGE_SHIFT); if (IS_ERR(header_ser)) return PTR_ERR(header_ser); - header_ser_pa = virt_to_phys(header_ser); - - err = fdt_begin_node(fdt_out, LUO_FDT_FLB_NODE_NAME); - err |= fdt_property_string(fdt_out, "compatible", - LUO_FDT_FLB_COMPATIBLE); - err |= fdt_property(fdt_out, LUO_FDT_FLB_HEADER, &header_ser_pa, - sizeof(header_ser_pa)); - err |= fdt_end_node(fdt_out); - - if (err) - goto err_unpreserve; + *flbs_pa = virt_to_phys(header_ser); header_ser->pgcnt = LUO_FLB_PGCNT; luo_flb_global.outgoing.header_ser = header_ser; @@ -579,53 +565,19 @@ int __init luo_flb_setup_outgoing(void *fdt_out) luo_flb_global.outgoing.active = true; return 0; - -err_unpreserve: - kho_unpreserve_free(header_ser); - - return err; } -int __init luo_flb_setup_incoming(void *fdt_in) +void __init luo_flb_setup_incoming(u64 flbs_pa) { struct luo_flb_header_ser *header_ser; - int err, header_size, offset; - const void *ptr; - u64 header_ser_pa; - - offset = fdt_subnode_offset(fdt_in, 0, LUO_FDT_FLB_NODE_NAME); - if (offset < 0) { - pr_err("Unable to get FLB node [%s]\n", LUO_FDT_FLB_NODE_NAME); - - return -ENOENT; - } - - err = fdt_node_check_compatible(fdt_in, offset, - LUO_FDT_FLB_COMPATIBLE); - if (err) { - pr_err("FLB node is incompatible with '%s' [%d]\n", - LUO_FDT_FLB_COMPATIBLE, err); - - return -EINVAL; - } - - header_size = 0; - ptr = fdt_getprop(fdt_in, offset, LUO_FDT_FLB_HEADER, &header_size); - if (!ptr || header_size != sizeof(u64)) { - pr_err("Unable to get FLB header property '%s' [%d]\n", - LUO_FDT_FLB_HEADER, header_size); - return -EINVAL; - } - - header_ser_pa = get_unaligned((u64 *)ptr); - header_ser = phys_to_virt(header_ser_pa); + if (!flbs_pa) + return; + header_ser = phys_to_virt(flbs_pa); luo_flb_global.incoming.header_ser = header_ser; luo_flb_global.incoming.ser = (void *)(header_ser + 1); luo_flb_global.incoming.active = true; - - return 0; } /** diff --git a/kernel/liveupdate/luo_internal.h b/kernel/liveupdate/luo_internal.h index ae58206f14ac..fe22086bfbeb 100644 --- a/kernel/liveupdate/luo_internal.h +++ b/kernel/liveupdate/luo_internal.h @@ -79,8 +79,8 @@ extern struct rw_semaphore luo_register_rwlock; int luo_session_create(const char *name, struct file **filep); int luo_session_retrieve(const char *name, struct file **filep); -int __init luo_session_setup_outgoing(void *fdt); -int __init luo_session_setup_incoming(void *fdt); +int __init luo_session_setup_outgoing(u64 *sessions_pa); +int __init luo_session_setup_incoming(u64 sessions_pa); int luo_session_serialize(void); int luo_session_deserialize(void); @@ -102,8 +102,8 @@ int luo_flb_file_preserve(struct liveupdate_file_handler *fh); void luo_flb_file_unpreserve(struct liveupdate_file_handler *fh); void luo_flb_file_finish(struct liveupdate_file_handler *fh); void luo_flb_unregister_all(struct liveupdate_file_handler *fh); -int __init luo_flb_setup_outgoing(void *fdt); -int __init luo_flb_setup_incoming(void *fdt); +int __init luo_flb_setup_outgoing(u64 *flbs_pa); +void __init luo_flb_setup_incoming(u64 flbs_pa); void luo_flb_serialize(void); #ifdef CONFIG_LIVEUPDATE_TEST diff --git a/kernel/liveupdate/luo_session.c b/kernel/liveupdate/luo_session.c index 47566db64598..85782c6f3d6c 100644 --- a/kernel/liveupdate/luo_session.c +++ b/kernel/liveupdate/luo_session.c @@ -25,9 +25,8 @@ * * - Serialization: Session metadata is preserved using the KHO framework. When * a live update is triggered via kexec, an array of `struct luo_session_ser` - * is populated and placed in a preserved memory region. An FDT node is also - * created, containing the count of sessions and the physical address of this - * array. + * is populated and placed in a preserved memory region. The physical address + * of this array is stored in the centralized `struct luo_ser` structure. * * Session Lifecycle: * @@ -91,13 +90,11 @@ #include #include #include -#include #include #include #include #include #include -#include #include #include "luo_internal.h" @@ -527,75 +524,34 @@ int luo_session_retrieve(const char *name, struct file **filep) return err; } -int __init luo_session_setup_outgoing(void *fdt_out) +int __init luo_session_setup_outgoing(u64 *sessions_pa) { struct luo_session_header_ser *header_ser; - u64 header_ser_pa; - int err; header_ser = kho_alloc_preserve(LUO_SESSION_PGCNT << PAGE_SHIFT); if (IS_ERR(header_ser)) return PTR_ERR(header_ser); - header_ser_pa = virt_to_phys(header_ser); - - err = fdt_begin_node(fdt_out, LUO_FDT_SESSION_NODE_NAME); - err |= fdt_property_string(fdt_out, "compatible", - LUO_FDT_SESSION_COMPATIBLE); - err |= fdt_property(fdt_out, LUO_FDT_SESSION_HEADER, &header_ser_pa, - sizeof(header_ser_pa)); - err |= fdt_end_node(fdt_out); - if (err) - goto err_unpreserve; + *sessions_pa = virt_to_phys(header_ser); luo_session_global.outgoing.header_ser = header_ser; luo_session_global.outgoing.ser = (void *)(header_ser + 1); luo_session_global.outgoing.active = true; return 0; - -err_unpreserve: - kho_unpreserve_free(header_ser); - return err; } -int __init luo_session_setup_incoming(void *fdt_in) +int __init luo_session_setup_incoming(u64 sessions_pa) { struct luo_session_header_ser *header_ser; - int err, header_size, offset; - u64 header_ser_pa; - const void *ptr; - - offset = fdt_subnode_offset(fdt_in, 0, LUO_FDT_SESSION_NODE_NAME); - if (offset < 0) { - pr_err("Unable to get session node: [%s]\n", - LUO_FDT_SESSION_NODE_NAME); - return -EINVAL; - } - err = fdt_node_check_compatible(fdt_in, offset, - LUO_FDT_SESSION_COMPATIBLE); - if (err) { - pr_err("Session node incompatible [%s]\n", - LUO_FDT_SESSION_COMPATIBLE); - return -EINVAL; - } - - header_size = 0; - ptr = fdt_getprop(fdt_in, offset, LUO_FDT_SESSION_HEADER, &header_size); - if (!ptr || header_size != sizeof(u64)) { - pr_err("Unable to get session header '%s' [%d]\n", - LUO_FDT_SESSION_HEADER, header_size); - return -EINVAL; + if (sessions_pa) { + header_ser = phys_to_virt(sessions_pa); + luo_session_global.incoming.header_ser = header_ser; + luo_session_global.incoming.ser = (void *)(header_ser + 1); + luo_session_global.incoming.active = true; } - header_ser_pa = get_unaligned((u64 *)ptr); - header_ser = phys_to_virt(header_ser_pa); - - luo_session_global.incoming.header_ser = header_ser; - luo_session_global.incoming.ser = (void *)(header_ser + 1); - luo_session_global.incoming.active = true; - return 0; } -- cgit v1.2.3 From cf071b3536df76a2a75b83ca1fe8c043824352c3 Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Wed, 3 Jun 2026 15:43:53 +0000 Subject: liveupdate: register luo_ser as KHO subtree Entirely remove the LUO FDT wrapper since the FDT only carries the compatible string and the pointer to the centralized struct luo_ser. Instead, register the struct luo_ser via the KHO raw subtree API, placing the compatibility string inside the structure itself. Acked-by: Mike Rapoport (Microsoft) Reviewed-by: Pratyush Yadav (Google) Signed-off-by: Pasha Tatashin Link: https://patch.msgid.link/20260603154402.468928-5-pasha.tatashin@soleen.com Signed-off-by: Mike Rapoport (Microsoft) --- kernel/liveupdate/luo_core.c | 85 +++++++++++++------------------------------- 1 file changed, 25 insertions(+), 60 deletions(-) (limited to 'kernel') diff --git a/kernel/liveupdate/luo_core.c b/kernel/liveupdate/luo_core.c index 085c0dfc1ef1..69b00e7d0f8f 100644 --- a/kernel/liveupdate/luo_core.c +++ b/kernel/liveupdate/luo_core.c @@ -54,7 +54,6 @@ #include #include #include -#include #include #include #include @@ -67,8 +66,7 @@ static struct { bool enabled; - void *fdt_out; - void *fdt_in; + struct luo_ser *luo_ser_out; u64 liveupdate_num; } luo_global; @@ -85,11 +83,10 @@ early_param("liveupdate", early_liveupdate_param); static int __init luo_early_startup(void) { + phys_addr_t luo_ser_phys; struct luo_ser *luo_ser; - int err, header_size; - phys_addr_t fdt_phys; - const void *ptr; - u64 luo_ser_pa; + size_t len; + int err; if (!kho_is_enabled()) { if (liveupdate_enabled()) @@ -98,40 +95,29 @@ static int __init luo_early_startup(void) return 0; } - /* Retrieve LUO subtree, and verify its format. */ - err = kho_retrieve_subtree(LUO_FDT_KHO_ENTRY_NAME, &fdt_phys, NULL); + /* Retrieve LUO state from KHO. */ + err = kho_retrieve_subtree(LUO_KHO_ENTRY_NAME, &luo_ser_phys, &len); if (err) { if (err != -ENOENT) { - pr_err("failed to retrieve FDT '%s' from KHO: %pe\n", - LUO_FDT_KHO_ENTRY_NAME, ERR_PTR(err)); + pr_err("failed to retrieve LUO state '%s' from KHO: %pe\n", + LUO_KHO_ENTRY_NAME, ERR_PTR(err)); return err; } return 0; } - luo_global.fdt_in = phys_to_virt(fdt_phys); - err = fdt_node_check_compatible(luo_global.fdt_in, 0, - LUO_FDT_COMPATIBLE); - if (err) { - pr_err("FDT '%s' is incompatible with '%s' [%d]\n", - LUO_FDT_KHO_ENTRY_NAME, LUO_FDT_COMPATIBLE, err); - + if (len < sizeof(*luo_ser)) { + pr_err("LUO state is too small (%zu < %zu)\n", len, sizeof(*luo_ser)); return -EINVAL; } - header_size = 0; - ptr = fdt_getprop(luo_global.fdt_in, 0, LUO_FDT_ABI_HEADER, &header_size); - if (!ptr || header_size != sizeof(u64)) { - pr_err("Unable to get ABI header '%s' [%d]\n", - LUO_FDT_ABI_HEADER, header_size); - + luo_ser = phys_to_virt(luo_ser_phys); + if (strncmp(luo_ser->compatible, LUO_ABI_COMPATIBLE, LUO_ABI_COMPAT_LEN)) { + pr_err("LUO state is incompatible with '%s'\n", LUO_ABI_COMPATIBLE); return -EINVAL; } - luo_ser_pa = get_unaligned((u64 *)ptr); - luo_ser = phys_to_virt(luo_ser_pa); - luo_global.liveupdate_num = luo_ser->liveupdate_num; pr_info("Retrieved live update data, liveupdate number: %lld\n", luo_global.liveupdate_num); @@ -164,37 +150,20 @@ static int __init liveupdate_early_init(void) } early_initcall(liveupdate_early_init); -/* Called during boot to create outgoing LUO fdt tree */ -static int __init luo_fdt_setup(void) +/* Called during boot to create outgoing LUO state */ +static int __init luo_state_setup(void) { struct luo_ser *luo_ser; - u64 luo_ser_pa; - void *fdt_out; int err; - fdt_out = kho_alloc_preserve(LUO_FDT_SIZE); - if (IS_ERR(fdt_out)) { - pr_err("failed to allocate/preserve FDT memory\n"); - return PTR_ERR(fdt_out); - } - luo_ser = kho_alloc_preserve(sizeof(*luo_ser)); if (IS_ERR(luo_ser)) { - err = PTR_ERR(luo_ser); - goto exit_free_fdt; + pr_err("failed to allocate/preserve LUO state memory\n"); + return PTR_ERR(luo_ser); } - luo_ser_pa = virt_to_phys(luo_ser); - - err = fdt_create(fdt_out, LUO_FDT_SIZE); - err |= fdt_finish_reservemap(fdt_out); - err |= fdt_begin_node(fdt_out, ""); - err |= fdt_property_string(fdt_out, "compatible", LUO_FDT_COMPATIBLE); - err |= fdt_property(fdt_out, LUO_FDT_ABI_HEADER, &luo_ser_pa, - sizeof(luo_ser_pa)); - err |= fdt_end_node(fdt_out); - err |= fdt_finish(fdt_out); - if (err) - goto exit_free_luo_ser; + + strscpy(luo_ser->compatible, LUO_ABI_COMPATIBLE, sizeof(luo_ser->compatible)); + luo_ser->liveupdate_num = luo_global.liveupdate_num + 1; err = luo_session_setup_outgoing(&luo_ser->sessions_pa); if (err) @@ -204,21 +173,17 @@ static int __init luo_fdt_setup(void) if (err) goto exit_free_luo_ser; - luo_ser->liveupdate_num = luo_global.liveupdate_num + 1; - - err = kho_add_subtree(LUO_FDT_KHO_ENTRY_NAME, fdt_out, - fdt_totalsize(fdt_out)); + err = kho_add_subtree(LUO_KHO_ENTRY_NAME, luo_ser, sizeof(*luo_ser)); if (err) goto exit_free_luo_ser; - luo_global.fdt_out = fdt_out; + + luo_global.luo_ser_out = luo_ser; return 0; exit_free_luo_ser: kho_unpreserve_free(luo_ser); -exit_free_fdt: - kho_unpreserve_free(fdt_out); - pr_err("failed to prepare LUO FDT: %d\n", err); + pr_err("failed to prepare LUO state: %d\n", err); return err; } @@ -234,7 +199,7 @@ static int __init luo_late_startup(void) if (!liveupdate_enabled()) return 0; - err = luo_fdt_setup(); + err = luo_state_setup(); if (err) luo_global.enabled = false; -- cgit v1.2.3 From 51b71af922a7145e63fdc0cab075d681ecd89e4a Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Wed, 3 Jun 2026 15:43:54 +0000 Subject: liveupdate: Extract luo_file_deserialize_one helper Extract the logic for deserializing single entries for files into separate helper functions. In preparation to a linked-block serialization for files. This is a pure code movement, no other changes intended. Acked-by: Mike Rapoport (Microsoft) Reviewed-by: Pratyush Yadav (Google) Signed-off-by: Pasha Tatashin Link: https://patch.msgid.link/20260603154402.468928-6-pasha.tatashin@soleen.com Signed-off-by: Mike Rapoport (Microsoft) --- kernel/liveupdate/luo_file.c | 77 +++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 33 deletions(-) (limited to 'kernel') diff --git a/kernel/liveupdate/luo_file.c b/kernel/liveupdate/luo_file.c index 208987502f73..9eec07a9e9fc 100644 --- a/kernel/liveupdate/luo_file.c +++ b/kernel/liveupdate/luo_file.c @@ -753,6 +753,46 @@ int luo_file_finish(struct luo_file_set *file_set) return 0; } +static int luo_file_deserialize_one(struct luo_file_set *file_set, + struct luo_file_ser *ser) +{ + struct liveupdate_file_handler *fh; + bool handler_found = false; + struct luo_file *luo_file; + + down_read(&luo_register_rwlock); + list_private_for_each_entry(fh, &luo_file_handler_list, list) { + if (!strcmp(fh->compatible, ser->compatible)) { + if (try_module_get(fh->ops->owner)) + handler_found = true; + break; + } + } + up_read(&luo_register_rwlock); + + if (!handler_found) { + pr_warn("No registered handler for compatible '%.*s'\n", + (int)sizeof(ser->compatible), + ser->compatible); + return -ENOENT; + } + + luo_file = kzalloc_obj(*luo_file); + if (!luo_file) { + module_put(fh->ops->owner); + return -ENOMEM; + } + + luo_file->fh = fh; + luo_file->file = NULL; + luo_file->serialized_data = ser->data; + luo_file->token = ser->token; + mutex_init(&luo_file->mutex); + list_add_tail(&luo_file->list, &file_set->files_list); + + return 0; +} + /** * luo_file_deserialize - Reconstructs the list of preserved files in the new kernel. * @file_set: The incoming file_set to fill with deserialized data. @@ -782,6 +822,7 @@ int luo_file_deserialize(struct luo_file_set *file_set, struct luo_file_set_ser *file_set_ser) { struct luo_file_ser *file_ser; + int err; u64 i; if (!file_set_ser->files) { @@ -809,39 +850,9 @@ int luo_file_deserialize(struct luo_file_set *file_set, */ file_ser = file_set->files; for (i = 0; i < file_set->count; i++) { - struct liveupdate_file_handler *fh; - bool handler_found = false; - struct luo_file *luo_file; - - down_read(&luo_register_rwlock); - list_private_for_each_entry(fh, &luo_file_handler_list, list) { - if (!strcmp(fh->compatible, file_ser[i].compatible)) { - if (try_module_get(fh->ops->owner)) - handler_found = true; - break; - } - } - up_read(&luo_register_rwlock); - - if (!handler_found) { - pr_warn("No registered handler for compatible '%.*s'\n", - (int)sizeof(file_ser[i].compatible), - file_ser[i].compatible); - return -ENOENT; - } - - luo_file = kzalloc_obj(*luo_file); - if (!luo_file) { - module_put(fh->ops->owner); - return -ENOMEM; - } - - luo_file->fh = fh; - luo_file->file = NULL; - luo_file->serialized_data = file_ser[i].data; - luo_file->token = file_ser[i].token; - mutex_init(&luo_file->mutex); - list_add_tail(&luo_file->list, &file_set->files_list); + err = luo_file_deserialize_one(file_set, &file_ser[i]); + if (err) + return err; } return 0; -- cgit v1.2.3 From be9d10d167652e11283cd07c7daf187222808db1 Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Wed, 3 Jun 2026 15:43:55 +0000 Subject: liveupdate: Extract luo_session_deserialize_one helper Extract the logic for deserializing single entries for sessions into separate helper functions. In preparation to a linked-block serialization for sessions. This is a pure code movement, no other changes intended. Acked-by: Mike Rapoport (Microsoft) Reviewed-by: Pratyush Yadav (Google) Signed-off-by: Pasha Tatashin Link: https://patch.msgid.link/20260603154402.468928-7-pasha.tatashin@soleen.com Signed-off-by: Mike Rapoport (Microsoft) --- kernel/liveupdate/luo_session.c | 63 +++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 27 deletions(-) (limited to 'kernel') diff --git a/kernel/liveupdate/luo_session.c b/kernel/liveupdate/luo_session.c index 85782c6f3d6c..1cd315e0f6de 100644 --- a/kernel/liveupdate/luo_session.c +++ b/kernel/liveupdate/luo_session.c @@ -555,6 +555,40 @@ int __init luo_session_setup_incoming(u64 sessions_pa) return 0; } +static int luo_session_deserialize_one(struct luo_session_header *sh, + struct luo_session_ser *ser) +{ + struct luo_session *session; + int err; + + session = luo_session_alloc(ser->name); + if (IS_ERR(session)) { + pr_warn("Failed to allocate session [%.*s] during deserialization %pe\n", + (int)sizeof(ser->name), ser->name, session); + return PTR_ERR(session); + } + + err = luo_session_insert(sh, session); + if (err) { + pr_warn("Failed to insert session [%s] %pe\n", + session->name, ERR_PTR(err)); + luo_session_free(session); + return err; + } + + scoped_guard(mutex, &session->mutex) { + err = luo_file_deserialize(&session->file_set, + &ser->file_set_ser); + } + if (err) { + pr_warn("Failed to deserialize files for session [%s] %pe\n", + session->name, ERR_PTR(err)); + return err; + } + + return 0; +} + int luo_session_deserialize(void) { struct luo_session_header *sh = &luo_session_global.incoming; @@ -586,34 +620,9 @@ int luo_session_deserialize(void) * reliably reset devices and reclaim memory. */ for (int i = 0; i < sh->header_ser->count; i++) { - struct luo_session *session; - - session = luo_session_alloc(sh->ser[i].name); - if (IS_ERR(session)) { - pr_warn("Failed to allocate session [%.*s] during deserialization %pe\n", - (int)sizeof(sh->ser[i].name), - sh->ser[i].name, session); - err = PTR_ERR(session); - goto save_err; - } - - err = luo_session_insert(sh, session); - if (err) { - pr_warn("Failed to insert session [%s] %pe\n", - session->name, ERR_PTR(err)); - luo_session_free(session); - goto save_err; - } - - scoped_guard(mutex, &session->mutex) { - err = luo_file_deserialize(&session->file_set, - &sh->ser[i].file_set_ser); - } - if (err) { - pr_warn("Failed to deserialize files for session [%s] %pe\n", - session->name, ERR_PTR(err)); + err = luo_session_deserialize_one(sh, &sh->ser[i]); + if (err) goto save_err; - } } kho_restore_free(sh->header_ser); -- cgit v1.2.3 From 0349ff2887059112ce06831ab29aec47a2a7285a Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Wed, 3 Jun 2026 15:43:56 +0000 Subject: kho: add support for linked-block serialization Introduce a linked-block serialization mechanism for state handover. Previously, LUO used contiguous memory blocks for serializing sessions and files, which imposed limits on the total number of items that could be preserved across a live update. This commit adds the infrastructure for a more flexible, block-based approach where serialized data is stored in a chain of linked blocks. This is a generic KHO serialization block infrastructure that can be used by multiple subsystems. Signed-off-by: Pasha Tatashin Link: https://patch.msgid.link/20260603154402.468928-8-pasha.tatashin@soleen.com Signed-off-by: Mike Rapoport (Microsoft) --- kernel/liveupdate/Makefile | 1 + kernel/liveupdate/kho_block.c | 416 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 417 insertions(+) create mode 100644 kernel/liveupdate/kho_block.c (limited to 'kernel') diff --git a/kernel/liveupdate/Makefile b/kernel/liveupdate/Makefile index d2f779cbe279..eec9d3ae07eb 100644 --- a/kernel/liveupdate/Makefile +++ b/kernel/liveupdate/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 luo-y := \ + kho_block.o \ luo_core.o \ luo_file.o \ luo_flb.o \ diff --git a/kernel/liveupdate/kho_block.c b/kernel/liveupdate/kho_block.c new file mode 100644 index 000000000000..0d2a342ef422 --- /dev/null +++ b/kernel/liveupdate/kho_block.c @@ -0,0 +1,416 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (c) 2026, Google LLC. + * Pasha Tatashin + */ + +/** + * DOC: KHO Serialization Blocks + * + * KHO provides a mechanism to preserve stateful data across a kexec handover + * by serializing it into memory blocks, and provides the common + * infrastructure for managing these blocks. + * + * Each block consists of a header (struct kho_block_header_ser) followed by an + * array of serialized entries. Multiple blocks are linked together via a + * physical pointer in the header, forming a linked list that can be easily + * traversed in both the current and the next kernel. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include + +/* + * Safeguard limit for the number of serialization blocks. This is used to + * prevent infinite loops and excessive memory allocation in case of memory + * corruption in the preserved state. + * + * With a 4KB page size, 10k blocks is about 40MB. For 32-byte entries + * (e.g. 4 u64s), each block holds up to 127 entries (accounting for the + * 16-byte header), allowing the block set to hold up to 1.27M entries. + */ +#define KHO_MAX_BLOCKS 10000 + +/** + * kho_block_set_init - Initialize a block set. + * @bs: The block set to initialize. + * @entry_size: The size of each entry in the blocks. + */ +void kho_block_set_init(struct kho_block_set *bs, size_t entry_size) +{ + *bs = (struct kho_block_set)KHO_BLOCK_SET_INIT(*bs, entry_size); + WARN_ON_ONCE(!bs->count_per_block); +} + +/* Serialized entries start immediately after the block header */ +static void *kho_block_entries(struct kho_block *block) +{ + return (void *)(block->ser + 1); +} + +/* Get the address of the serialized entry at the specified index */ +static void *kho_block_entry(struct kho_block_set_it *it, u64 index) +{ + return kho_block_entries(it->block) + (index * it->bs->entry_size); +} + +/* Free serialized data */ +static void kho_block_free_ser(struct kho_block_set *bs, + struct kho_block_header_ser *ser) +{ + if (bs->incoming) + kho_restore_free(ser); + else + kho_unpreserve_free(ser); +} + +static struct kho_block_header_ser *kho_block_alloc_ser(struct kho_block_set *bs) +{ + WARN_ON_ONCE(bs->incoming); + return kho_alloc_preserve(KHO_BLOCK_SIZE); +} + +static int kho_block_add(struct kho_block_set *bs, + struct kho_block_header_ser *ser) +{ + struct kho_block *block, *last; + + if (bs->nblocks >= KHO_MAX_BLOCKS) + return -ENOSPC; + + block = kzalloc_obj(*block); + if (!block) + return -ENOMEM; + + block->ser = ser; + last = list_last_entry_or_null(&bs->blocks, struct kho_block, list); + list_add_tail(&block->list, &bs->blocks); + bs->nblocks++; + + if (last) + last->ser->next = virt_to_phys(ser); + else + bs->head_pa = virt_to_phys(ser); + + return 0; +} + +static int kho_block_set_grow_one(struct kho_block_set *bs) +{ + struct kho_block_header_ser *ser; + int err; + + ser = kho_block_alloc_ser(bs); + if (IS_ERR(ser)) + return PTR_ERR(ser); + + err = kho_block_add(bs, ser); + if (err) { + kho_block_free_ser(bs, ser); + return err; + } + + return 0; +} + +static void kho_block_set_shrink_one(struct kho_block_set *bs) +{ + struct kho_block *last, *new_last; + + if (list_empty(&bs->blocks)) + return; + + last = list_last_entry(&bs->blocks, struct kho_block, list); + list_del(&last->list); + bs->nblocks--; + kho_block_free_ser(bs, last->ser); + kfree(last); + + new_last = list_last_entry_or_null(&bs->blocks, struct kho_block, list); + if (new_last) + new_last->ser->next = 0; + else + bs->head_pa = 0; +} + +/** + * kho_block_set_grow - Expand the block set to accommodate the target count. + * @bs: The block set. + * @count: The target number of valid entries to accommodate. + * + * Dynamically preallocates and links preserved memory blocks if the target + * entry count exceeds the current total capacity of the set, ensuring they + * are available during serialization/deserialization. + * + * Context: Caller must hold a lock protecting the block set. + * Return: 0 on success, or a negative errno on failure. + */ +int kho_block_set_grow(struct kho_block_set *bs, u64 count) +{ + long orig_nblocks = bs->nblocks; + int err; + + if (WARN_ON_ONCE(bs->incoming)) + return -EINVAL; + + while (count > bs->nblocks * bs->count_per_block) { + err = kho_block_set_grow_one(bs); + if (err) + goto err_shrink; + } + + return 0; + +err_shrink: + while (bs->nblocks > orig_nblocks) + kho_block_set_shrink_one(bs); + return err; +} + +/** + * kho_block_set_shrink - Shrink the block set to accommodate the target count. + * @bs: The block set. + * @count: The target number of valid entries to accommodate. + * + * Releases and unallocates redundant preserved memory blocks. Checks if the + * last block in the set can be removed because the remaining entry count is + * fully accommodated by the preceding blocks. + * + * Note: It is the caller's responsibility to ensure that entries are removed + * in the reverse order of their insertion. Because shrinking destroys the last + * block in the set, removing entries in any other order would corrupt active + * data. + * + * Context: Caller must hold a lock protecting the block set. + */ +void kho_block_set_shrink(struct kho_block_set *bs, u64 count) +{ + while (bs->nblocks > 0 && count <= (bs->nblocks - 1) * bs->count_per_block) + kho_block_set_shrink_one(bs); +} + +/* + * kho_block_set_is_cyclic - Check for cycles in a linked list of blocks. + * Uses Floyd's cycle-finding algorithm to ensure sanity of the incoming list. + * + * Return: true if a cycle or corruption is detected, false otherwise. + */ +static bool kho_block_set_is_cyclic(struct kho_block_set *bs) +{ + struct kho_block_header_ser *fast; + struct kho_block_header_ser *slow; + int count = 0; + + fast = phys_to_virt(bs->head_pa); + slow = fast; + + while (fast) { + if (count++ >= KHO_MAX_BLOCKS) { + pr_err("Block set is corrupted\n"); + return true; + } + + if (!fast->next) + break; + + fast = phys_to_virt(fast->next); + if (!fast->next) + break; + + fast = phys_to_virt(fast->next); + slow = phys_to_virt(slow->next); + + if (slow == fast) { + pr_err("Block set is corrupted\n"); + return true; + } + } + + return false; +} + +/** + * kho_block_set_restore - Restore a block set from a physical address. + * @bs: The block set to restore. + * @head_pa: Physical address of the first block header. + * + * Restores a serialized block set from a given physical address. The caller is + * responsible for ensuring that the block set @bs has been allocated and + * initialized prior to calling this function. + * + * Return: 0 on success, or a negative errno on failure. + */ +int kho_block_set_restore(struct kho_block_set *bs, u64 head_pa) +{ + struct kho_block_header_ser *ser; + u64 next_pa = head_pa; + int err; + + /* Restored block sets use size from the previous kernel */ + bs->incoming = true; + if (!head_pa) + return 0; + + bs->head_pa = head_pa; + if (kho_block_set_is_cyclic(bs)) { + bs->head_pa = 0; + return -EINVAL; + } + + while (next_pa) { + ser = phys_to_virt(next_pa); + if (!ser->count || ser->count > bs->count_per_block) { + pr_warn("Block contains invalid entry count: %llu\n", + ser->count); + err = -EINVAL; + goto err_destroy; + } + err = kho_block_add(bs, ser); + if (err) + goto err_destroy; + next_pa = ser->next; + } + + return 0; + +err_destroy: + kho_block_set_destroy(bs); + + /* Free the remaining un-restored blocks in the physical chain */ + while (next_pa) { + struct kho_block_header_ser *next_ser = phys_to_virt(next_pa); + + next_pa = next_ser->next; + kho_block_free_ser(bs, next_ser); + } + return err; +} + +/** + * kho_block_set_destroy - Destroy all blocks in a block set. + * @bs: The block set. + */ +void kho_block_set_destroy(struct kho_block_set *bs) +{ + struct kho_block *block, *tmp; + + list_for_each_entry_safe(block, tmp, &bs->blocks, list) { + list_del(&block->list); + kho_block_free_ser(bs, block->ser); + kfree(block); + } + bs->nblocks = 0; + bs->head_pa = 0; +} + +/** + * kho_block_set_clear - Clear all serialized data in a block set. + * @bs: The block set to clear. + */ +void kho_block_set_clear(struct kho_block_set *bs) +{ + struct kho_block *block; + + list_for_each_entry(block, &bs->blocks, list) { + block->ser->count = 0; + memset(block->ser + 1, 0, KHO_BLOCK_SIZE - sizeof(*block->ser)); + } +} + +/** + * kho_block_set_it_init - Initialize a block set iterator. + * @it: The iterator to initialize. + * @bs: The block set to iterate over. + */ +void kho_block_set_it_init(struct kho_block_set_it *it, struct kho_block_set *bs) +{ + it->bs = bs; + it->block = list_first_entry_or_null(&bs->blocks, struct kho_block, list); + it->i = 0; +} + +/** + * kho_block_set_it_reserve_entry - Reserve and return the next available slot for writing. + * @it: The block iterator. + * + * Reserves a slot in the current block during state serialization to add a new + * entry, advancing the internal index. If the current block is full, it + * automatically moves to the next block in the set. + * + * Return: A pointer to the reserved entry slot, or NULL if the block set's + * capacity is fully exhausted. + */ +void *kho_block_set_it_reserve_entry(struct kho_block_set_it *it) +{ + void *entry; + + if (!it->block) + return NULL; + + if (it->i == it->bs->count_per_block) { + if (list_is_last(&it->block->list, &it->bs->blocks)) + return NULL; + it->block = list_next_entry(it->block, list); + it->i = 0; + } + + entry = kho_block_entry(it, it->i++); + it->block->ser->count = it->i; + return entry; +} + +/** + * kho_block_set_it_read_entry - Read the next serialized entry from the block set. + * @it: The block iterator. + * + * Iterates through previously written entries during state deserialization, + * respecting the actual count stored in each block's header. + * + * Return: A pointer to the next serialized entry, or NULL if all serialized + * entries have been read. + */ +void *kho_block_set_it_read_entry(struct kho_block_set_it *it) +{ + if (!it->block) + return NULL; + + if (it->i == it->block->ser->count) { + if (list_is_last(&it->block->list, &it->bs->blocks)) + return NULL; + it->block = list_next_entry(it->block, list); + it->i = 0; + } + + return kho_block_entry(it, it->i++); +} + +/** + * kho_block_set_it_prev - Return the previous entry slot in the block set. + * @it: The block iterator. + * + * If the current index is at the start of a block, it automatically moves to + * the end of the previous block. + * + * Return: A pointer to the previous entry slot, or NULL if at the very + * beginning of the block set. + */ +void *kho_block_set_it_prev(struct kho_block_set_it *it) +{ + if (!it->block) + return NULL; + + if (it->i == 0) { + if (list_is_first(&it->block->list, &it->bs->blocks)) + return NULL; + it->block = list_prev_entry(it->block, list); + it->i = it->bs->count_per_block; + } + + return kho_block_entry(it, --it->i); +} -- cgit v1.2.3 From b5a58a922e6f2f9f40faddd8e0e1fe3ce0ea9c56 Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Wed, 3 Jun 2026 15:43:57 +0000 Subject: liveupdate: defer session block allocation and physical address setting Currently, luo_session_setup_outgoing() allocates the session block and sets its physical address in the header immediately. With upcoming dynamic block-based session management, this makes the first block different from the rest. Move the allocation to where it is first needed. Acked-by: Mike Rapoport (Microsoft) Reviewed-by: Pratyush Yadav (Google) Signed-off-by: Pasha Tatashin Link: https://patch.msgid.link/20260603154402.468928-9-pasha.tatashin@soleen.com Signed-off-by: Mike Rapoport (Microsoft) --- kernel/liveupdate/luo_core.c | 4 +-- kernel/liveupdate/luo_internal.h | 2 +- kernel/liveupdate/luo_session.c | 68 +++++++++++++++++++++++++--------------- 3 files changed, 45 insertions(+), 29 deletions(-) (limited to 'kernel') diff --git a/kernel/liveupdate/luo_core.c b/kernel/liveupdate/luo_core.c index 69b00e7d0f8f..1b2bda22902d 100644 --- a/kernel/liveupdate/luo_core.c +++ b/kernel/liveupdate/luo_core.c @@ -165,9 +165,7 @@ static int __init luo_state_setup(void) strscpy(luo_ser->compatible, LUO_ABI_COMPATIBLE, sizeof(luo_ser->compatible)); luo_ser->liveupdate_num = luo_global.liveupdate_num + 1; - err = luo_session_setup_outgoing(&luo_ser->sessions_pa); - if (err) - goto exit_free_luo_ser; + luo_session_setup_outgoing(&luo_ser->sessions_pa); err = luo_flb_setup_outgoing(&luo_ser->flbs_pa); if (err) diff --git a/kernel/liveupdate/luo_internal.h b/kernel/liveupdate/luo_internal.h index fe22086bfbeb..ee18f9a11b91 100644 --- a/kernel/liveupdate/luo_internal.h +++ b/kernel/liveupdate/luo_internal.h @@ -79,7 +79,7 @@ extern struct rw_semaphore luo_register_rwlock; int luo_session_create(const char *name, struct file **filep); int luo_session_retrieve(const char *name, struct file **filep); -int __init luo_session_setup_outgoing(u64 *sessions_pa); +void __init luo_session_setup_outgoing(u64 *sessions_pa); int __init luo_session_setup_incoming(u64 sessions_pa); int luo_session_serialize(void); int luo_session_deserialize(void); diff --git a/kernel/liveupdate/luo_session.c b/kernel/liveupdate/luo_session.c index 1cd315e0f6de..2411849a34e3 100644 --- a/kernel/liveupdate/luo_session.c +++ b/kernel/liveupdate/luo_session.c @@ -108,15 +108,16 @@ static DECLARE_RWSEM(luo_session_serialize_rwsem); /** * struct luo_session_header - Header struct for managing LUO sessions. - * @count: The number of sessions currently tracked in the @list. - * @list: The head of the linked list of `struct luo_session` instances. - * @rwsem: A read-write semaphore providing synchronized access to the - * session list and other fields in this structure. - * @header_ser: The header data of serialization array. - * @ser: The serialized session data (an array of - * `struct luo_session_ser`). - * @active: Set to true when first initialized. If previous kernel did not - * send session data, active stays false for incoming. + * @count: The number of sessions currently tracked in the @list. + * @list: The head of the linked list of `struct luo_session` instances. + * @rwsem: A read-write semaphore providing synchronized access to the + * session list and other fields in this structure. + * @header_ser: The header data of serialization array. + * @ser: The serialized session data (an array of + * `struct luo_session_ser`). + * @sessions_pa: Points to the location of sessions_pa within struct luo_ser. + * @active: Set to true when first initialized. If previous kernel did not + * send session data, active stays false for incoming. */ struct luo_session_header { long count; @@ -124,6 +125,7 @@ struct luo_session_header { struct rw_semaphore rwsem; struct luo_session_header_ser *header_ser; struct luo_session_ser *ser; + u64 *sessions_pa; bool active; }; @@ -171,10 +173,30 @@ static void luo_session_free(struct luo_session *session) kfree(session); } +static int luo_session_grow_ser(struct luo_session_header *sh) +{ + struct luo_session_header_ser *header_ser; + + if (sh->count == LUO_SESSION_MAX) + return -ENOMEM; + + if (sh->header_ser) + return 0; + + header_ser = kho_alloc_preserve(LUO_SESSION_PGCNT << PAGE_SHIFT); + if (IS_ERR(header_ser)) + return PTR_ERR(header_ser); + + sh->header_ser = header_ser; + sh->ser = (void *)(header_ser + 1); + return 0; +} + static int luo_session_insert(struct luo_session_header *sh, struct luo_session *session) { struct luo_session *it; + int err; guard(rwsem_write)(&sh->rwsem); @@ -183,8 +205,9 @@ static int luo_session_insert(struct luo_session_header *sh, * for new session. */ if (sh == &luo_session_global.outgoing) { - if (sh->count == LUO_SESSION_MAX) - return -ENOMEM; + err = luo_session_grow_ser(sh); + if (err) + return err; } /* @@ -524,21 +547,10 @@ int luo_session_retrieve(const char *name, struct file **filep) return err; } -int __init luo_session_setup_outgoing(u64 *sessions_pa) +void __init luo_session_setup_outgoing(u64 *sessions_pa) { - struct luo_session_header_ser *header_ser; - - header_ser = kho_alloc_preserve(LUO_SESSION_PGCNT << PAGE_SHIFT); - if (IS_ERR(header_ser)) - return PTR_ERR(header_ser); - - *sessions_pa = virt_to_phys(header_ser); - - luo_session_global.outgoing.header_ser = header_ser; - luo_session_global.outgoing.ser = (void *)(header_ser + 1); + luo_session_global.outgoing.sessions_pa = sessions_pa; luo_session_global.outgoing.active = true; - - return 0; } int __init luo_session_setup_incoming(u64 sessions_pa) @@ -644,6 +656,8 @@ int luo_session_serialize(void) down_write(&luo_session_serialize_rwsem); down_write(&sh->rwsem); + *sh->sessions_pa = 0; + list_for_each_entry(session, &sh->list, list) { err = luo_session_freeze_one(session, &sh->ser[i]); if (err) @@ -653,7 +667,11 @@ int luo_session_serialize(void) sizeof(sh->ser[i].name)); i++; } - sh->header_ser->count = sh->count; + + if (sh->header_ser && sh->count > 0) { + sh->header_ser->count = sh->count; + *sh->sessions_pa = virt_to_phys(sh->header_ser); + } up_write(&sh->rwsem); return 0; -- cgit v1.2.3 From 2a441a14c2c03b39d1c89438dd28cef9d8fa57d5 Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Wed, 3 Jun 2026 15:43:58 +0000 Subject: liveupdate: Remove limit on the number of sessions Currently, the number of LUO sessions is limited by a fixed number of pre-allocated pages for serialization (16 pages, allowing for ~819 sessions). This limitation is problematic if LUO is used to support things such as systemd file descriptor store, and would be used not just as VM memory but to save other states on the machine. Remove this limit by transitioning to a linked-block approach for session metadata serialization. Instead of a single contiguous block, session metadata is now stored in a chain of 16-page blocks. Each block starts with a header containing the physical address of the next block and the number of session entries in the current block. Acked-by: Mike Rapoport (Microsoft) Reviewed-by: Pratyush Yadav (Google) Signed-off-by: Pasha Tatashin Link: https://patch.msgid.link/20260603154402.468928-10-pasha.tatashin@soleen.com Signed-off-by: Mike Rapoport (Microsoft) --- kernel/liveupdate/luo_session.c | 113 +++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 60 deletions(-) (limited to 'kernel') diff --git a/kernel/liveupdate/luo_session.c b/kernel/liveupdate/luo_session.c index 2411849a34e3..b79b2a488974 100644 --- a/kernel/liveupdate/luo_session.c +++ b/kernel/liveupdate/luo_session.c @@ -24,9 +24,10 @@ * ioctls on /dev/liveupdate. * * - Serialization: Session metadata is preserved using the KHO framework. When - * a live update is triggered via kexec, an array of `struct luo_session_ser` - * is populated and placed in a preserved memory region. The physical address - * of this array is stored in the centralized `struct luo_ser` structure. + * a live update is triggered via kexec, session metadata is serialized into + * a chain of linked-blocks and placed in a preserved memory region. The + * physical address of the first block header is stored in the centralized + * `struct luo_ser` structure. * * Session Lifecycle: * @@ -89,6 +90,7 @@ #include #include #include +#include #include #include #include @@ -98,23 +100,14 @@ #include #include "luo_internal.h" -/* 16 4K pages, give space for 744 sessions */ -#define LUO_SESSION_PGCNT 16ul -#define LUO_SESSION_MAX (((LUO_SESSION_PGCNT << PAGE_SHIFT) - \ - sizeof(struct luo_session_header_ser)) / \ - sizeof(struct luo_session_ser)) - static DECLARE_RWSEM(luo_session_serialize_rwsem); - /** * struct luo_session_header - Header struct for managing LUO sessions. * @count: The number of sessions currently tracked in the @list. * @list: The head of the linked list of `struct luo_session` instances. * @rwsem: A read-write semaphore providing synchronized access to the * session list and other fields in this structure. - * @header_ser: The header data of serialization array. - * @ser: The serialized session data (an array of - * `struct luo_session_ser`). + * @block_set: The set of serialization blocks. * @sessions_pa: Points to the location of sessions_pa within struct luo_ser. * @active: Set to true when first initialized. If previous kernel did not * send session data, active stays false for incoming. @@ -123,8 +116,7 @@ struct luo_session_header { long count; struct list_head list; struct rw_semaphore rwsem; - struct luo_session_header_ser *header_ser; - struct luo_session_ser *ser; + struct kho_block_set block_set; u64 *sessions_pa; bool active; }; @@ -143,10 +135,14 @@ static struct luo_session_global luo_session_global = { .incoming = { .list = LIST_HEAD_INIT(luo_session_global.incoming.list), .rwsem = __RWSEM_INITIALIZER(luo_session_global.incoming.rwsem), + .block_set = KHO_BLOCK_SET_INIT(luo_session_global.incoming.block_set, + sizeof(struct luo_session_ser)), }, .outgoing = { .list = LIST_HEAD_INIT(luo_session_global.outgoing.list), .rwsem = __RWSEM_INITIALIZER(luo_session_global.outgoing.rwsem), + .block_set = KHO_BLOCK_SET_INIT(luo_session_global.outgoing.block_set, + sizeof(struct luo_session_ser)), }, }; @@ -173,25 +169,6 @@ static void luo_session_free(struct luo_session *session) kfree(session); } -static int luo_session_grow_ser(struct luo_session_header *sh) -{ - struct luo_session_header_ser *header_ser; - - if (sh->count == LUO_SESSION_MAX) - return -ENOMEM; - - if (sh->header_ser) - return 0; - - header_ser = kho_alloc_preserve(LUO_SESSION_PGCNT << PAGE_SHIFT); - if (IS_ERR(header_ser)) - return PTR_ERR(header_ser); - - sh->header_ser = header_ser; - sh->ser = (void *)(header_ser + 1); - return 0; -} - static int luo_session_insert(struct luo_session_header *sh, struct luo_session *session) { @@ -205,7 +182,7 @@ static int luo_session_insert(struct luo_session_header *sh, * for new session. */ if (sh == &luo_session_global.outgoing) { - err = luo_session_grow_ser(sh); + err = kho_block_set_grow(&sh->block_set, sh->count + 1); if (err) return err; } @@ -232,6 +209,8 @@ static void luo_session_remove(struct luo_session_header *sh, guard(rwsem_write)(&sh->rwsem); list_del(&session->list); sh->count--; + if (sh == &luo_session_global.outgoing) + kho_block_set_shrink(&sh->block_set, sh->count); } static int luo_session_finish_one(struct luo_session *session) @@ -555,15 +534,17 @@ void __init luo_session_setup_outgoing(u64 *sessions_pa) int __init luo_session_setup_incoming(u64 sessions_pa) { - struct luo_session_header_ser *header_ser; + struct luo_session_header *sh = &luo_session_global.incoming; + int err; - if (sessions_pa) { - header_ser = phys_to_virt(sessions_pa); - luo_session_global.incoming.header_ser = header_ser; - luo_session_global.incoming.ser = (void *)(header_ser + 1); - luo_session_global.incoming.active = true; - } + if (!sessions_pa) + return 0; + err = kho_block_set_restore(&sh->block_set, sessions_pa); + if (err) + return err; + + sh->active = true; return 0; } @@ -605,6 +586,8 @@ int luo_session_deserialize(void) { struct luo_session_header *sh = &luo_session_global.incoming; static bool is_deserialized; + struct luo_session_ser *ser; + struct kho_block_set_it it; static int saved_err; int err; @@ -631,18 +614,19 @@ int luo_session_deserialize(void) * userspace to detect the failure and trigger a reboot, which will * reliably reset devices and reclaim memory. */ - for (int i = 0; i < sh->header_ser->count; i++) { - err = luo_session_deserialize_one(sh, &sh->ser[i]); + kho_block_set_it_init(&it, &sh->block_set); + while ((ser = kho_block_set_it_read_entry(&it))) { + err = luo_session_deserialize_one(sh, ser); if (err) goto save_err; } - kho_restore_free(sh->header_ser); - sh->header_ser = NULL; - sh->ser = NULL; + kho_block_set_destroy(&sh->block_set); return 0; + save_err: + kho_block_set_destroy(&sh->block_set); saved_err = err; return err; } @@ -651,36 +635,45 @@ int luo_session_serialize(void) { struct luo_session_header *sh = &luo_session_global.outgoing; struct luo_session *session; - int i = 0; + struct kho_block_set_it it; int err; down_write(&luo_session_serialize_rwsem); down_write(&sh->rwsem); *sh->sessions_pa = 0; + kho_block_set_it_init(&it, &sh->block_set); + list_for_each_entry(session, &sh->list, list) { - err = luo_session_freeze_one(session, &sh->ser[i]); - if (err) + struct luo_session_ser *ser = kho_block_set_it_reserve_entry(&it); + + /* This should not fail normally as blocks were pre-allocated */ + if (WARN_ON_ONCE(!ser)) { + err = -ENOSPC; goto err_undo; + } - strscpy(sh->ser[i].name, session->name, - sizeof(sh->ser[i].name)); - i++; - } + err = luo_session_freeze_one(session, ser); + if (err) { + kho_block_set_it_prev(&it); + goto err_undo; + } - if (sh->header_ser && sh->count > 0) { - sh->header_ser->count = sh->count; - *sh->sessions_pa = virt_to_phys(sh->header_ser); + strscpy(ser->name, session->name, sizeof(ser->name)); } + + if (sh->count > 0) + *sh->sessions_pa = kho_block_set_head_pa(&sh->block_set); up_write(&sh->rwsem); return 0; err_undo: list_for_each_entry_continue_reverse(session, &sh->list, list) { - i--; - luo_session_unfreeze_one(session, &sh->ser[i]); - memset(sh->ser[i].name, 0, sizeof(sh->ser[i].name)); + struct luo_session_ser *ser = kho_block_set_it_prev(&it); + + luo_session_unfreeze_one(session, ser); + memset(ser->name, 0, sizeof(ser->name)); } up_write(&sh->rwsem); up_write(&luo_session_serialize_rwsem); -- cgit v1.2.3 From 1d1153097f4dd417e2ea00404edec9fbd1d88f28 Mon Sep 17 00:00:00 2001 From: Pasha Tatashin Date: Wed, 3 Jun 2026 15:43:59 +0000 Subject: liveupdate: Remove limit on the number of files per session To remove the fixed limit on the number of preserved files per session, transition the file metadata serialization from a single contiguous memory block to a chain of linked blocks. Acked-by: Mike Rapoport (Microsoft) Reviewed-by: Pratyush Yadav (Google) Signed-off-by: Pasha Tatashin Link: https://patch.msgid.link/20260603154402.468928-11-pasha.tatashin@soleen.com Signed-off-by: Mike Rapoport (Microsoft) --- kernel/liveupdate/luo_file.c | 138 ++++++++++++++++++--------------------- kernel/liveupdate/luo_internal.h | 6 +- 2 files changed, 67 insertions(+), 77 deletions(-) (limited to 'kernel') diff --git a/kernel/liveupdate/luo_file.c b/kernel/liveupdate/luo_file.c index 9eec07a9e9fc..c39f96961a85 100644 --- a/kernel/liveupdate/luo_file.c +++ b/kernel/liveupdate/luo_file.c @@ -118,11 +118,6 @@ static LIST_HEAD(luo_file_handler_list); /* Keep track of files being preserved by LUO */ static DEFINE_XARRAY(luo_preserved_files); -/* 2 4K pages, give space for 128 files per file_set */ -#define LUO_FILE_PGCNT 2ul -#define LUO_FILE_MAX \ - ((LUO_FILE_PGCNT << PAGE_SHIFT) / sizeof(struct luo_file_ser)) - /** * struct luo_file - Represents a single preserved file instance. * @fh: Pointer to the &struct liveupdate_file_handler that manages @@ -174,39 +169,6 @@ struct luo_file { u64 token; }; -static int luo_alloc_files_mem(struct luo_file_set *file_set) -{ - size_t size; - void *mem; - - if (file_set->files) - return 0; - - WARN_ON_ONCE(file_set->count); - - size = LUO_FILE_PGCNT << PAGE_SHIFT; - mem = kho_alloc_preserve(size); - if (IS_ERR(mem)) - return PTR_ERR(mem); - - file_set->files = mem; - - return 0; -} - -static void luo_free_files_mem(struct luo_file_set *file_set) -{ - /* If file_set has files, no need to free preservation memory */ - if (file_set->count) - return; - - if (!file_set->files) - return; - - kho_unpreserve_free(file_set->files); - file_set->files = NULL; -} - static unsigned long luo_get_id(struct liveupdate_file_handler *fh, struct file *file) { @@ -276,16 +238,15 @@ int luo_preserve_file(struct luo_file_set *file_set, u64 token, int fd) if (luo_token_is_used(file_set, token)) return -EEXIST; - if (file_set->count == LUO_FILE_MAX) - return -ENOSPC; + err = kho_block_set_grow(&file_set->block_set, file_set->count + 1); + if (err) + return err; file = fget(fd); - if (!file) - return -EBADF; - - err = luo_alloc_files_mem(file_set); - if (err) - goto err_fput; + if (!file) { + err = -EBADF; + goto err_shrink; + } err = -ENOENT; down_read(&luo_register_rwlock); @@ -300,7 +261,7 @@ int luo_preserve_file(struct luo_file_set *file_set, u64 token, int fd) /* err is still -ENOENT if no handler was found */ if (err) - goto err_free_files_mem; + goto err_fput; err = xa_insert(&luo_preserved_files, luo_get_id(fh, file), file, GFP_KERNEL); @@ -343,10 +304,10 @@ err_erase_xa: xa_erase(&luo_preserved_files, luo_get_id(fh, file)); err_module_put: module_put(fh->ops->owner); -err_free_files_mem: - luo_free_files_mem(file_set); err_fput: fput(file); +err_shrink: + kho_block_set_shrink(&file_set->block_set, file_set->count); return err; } @@ -392,13 +353,14 @@ void luo_file_unpreserve_files(struct luo_file_set *file_set) list_del(&luo_file->list); file_set->count--; + kho_block_set_shrink(&file_set->block_set, file_set->count); fput(luo_file->file); mutex_destroy(&luo_file->mutex); kfree(luo_file); } - luo_free_files_mem(file_set); + kho_block_set_destroy(&file_set->block_set); } static int luo_file_freeze_one(struct luo_file_set *file_set, @@ -454,7 +416,7 @@ static void __luo_file_unfreeze(struct luo_file_set *file_set, luo_file_unfreeze_one(file_set, luo_file); } - memset(file_set->files, 0, LUO_FILE_PGCNT << PAGE_SHIFT); + kho_block_set_clear(&file_set->block_set); } /** @@ -493,19 +455,24 @@ static void __luo_file_unfreeze(struct luo_file_set *file_set, int luo_file_freeze(struct luo_file_set *file_set, struct luo_file_set_ser *file_set_ser) { - struct luo_file_ser *file_ser = file_set->files; struct luo_file *luo_file; + struct kho_block_set_it it; int err; - int i; if (!file_set->count) return 0; - if (WARN_ON(!file_ser)) - return -EINVAL; + kho_block_set_it_init(&it, &file_set->block_set); - i = 0; list_for_each_entry(luo_file, &file_set->files_list, list) { + struct luo_file_ser *file_ser = kho_block_set_it_reserve_entry(&it); + + /* This should not fail normally as blocks were pre-allocated */ + if (WARN_ON_ONCE(!file_ser)) { + err = -ENOSPC; + goto err_unfreeze; + } + err = luo_file_freeze_one(file_set, luo_file); if (err < 0) { pr_warn("Freeze failed for token[%#0llx] handler[%s] err[%pe]\n", @@ -514,16 +481,14 @@ int luo_file_freeze(struct luo_file_set *file_set, goto err_unfreeze; } - strscpy(file_ser[i].compatible, luo_file->fh->compatible, - sizeof(file_ser[i].compatible)); - file_ser[i].data = luo_file->serialized_data; - file_ser[i].token = luo_file->token; - i++; + strscpy(file_ser->compatible, luo_file->fh->compatible, + sizeof(file_ser->compatible)); + file_ser->data = luo_file->serialized_data; + file_ser->token = luo_file->token; } file_set_ser->count = file_set->count; - if (file_set->files) - file_set_ser->files = virt_to_phys(file_set->files); + file_set_ser->files = kho_block_set_head_pa(&file_set->block_set); return 0; @@ -741,14 +706,12 @@ int luo_file_finish(struct luo_file_set *file_set) module_put(luo_file->fh->ops->owner); list_del(&luo_file->list); file_set->count--; + kho_block_set_shrink(&file_set->block_set, file_set->count); mutex_destroy(&luo_file->mutex); kfree(luo_file); } - if (file_set->files) { - kho_restore_free(file_set->files); - file_set->files = NULL; - } + kho_block_set_destroy(&file_set->block_set); return 0; } @@ -822,16 +785,18 @@ int luo_file_deserialize(struct luo_file_set *file_set, struct luo_file_set_ser *file_set_ser) { struct luo_file_ser *file_ser; + struct kho_block_set_it it; int err; - u64 i; if (!file_set_ser->files) { WARN_ON(file_set_ser->count); return 0; } - file_set->count = file_set_ser->count; - file_set->files = phys_to_virt(file_set_ser->files); + file_set->count = 0; + err = kho_block_set_restore(&file_set->block_set, file_set_ser->files); + if (err) + return err; /* * Note on error handling: @@ -848,25 +813,50 @@ int luo_file_deserialize(struct luo_file_set *file_set, * userspace to detect the failure and trigger a reboot, which will * reliably reset devices and reclaim memory. */ - file_ser = file_set->files; - for (i = 0; i < file_set->count; i++) { - err = luo_file_deserialize_one(file_set, &file_ser[i]); + kho_block_set_it_init(&it, &file_set->block_set); + while ((file_ser = kho_block_set_it_read_entry(&it))) { + err = luo_file_deserialize_one(file_set, file_ser); if (err) - return err; + goto err_destroy_blocks; + file_set->count++; + } + + if (file_set->count != file_set_ser->count) { + pr_warn("File count mismatch: expected %llu, found %llu\n", + file_set_ser->count, file_set->count); + err = -EINVAL; + goto err_destroy_blocks; } return 0; + +err_destroy_blocks: + while (!list_empty(&file_set->files_list)) { + struct luo_file *luo_file; + + luo_file = list_first_entry(&file_set->files_list, + struct luo_file, list); + list_del(&luo_file->list); + module_put(luo_file->fh->ops->owner); + mutex_destroy(&luo_file->mutex); + kfree(luo_file); + } + file_set->count = 0; + kho_block_set_destroy(&file_set->block_set); + return err; } void luo_file_set_init(struct luo_file_set *file_set) { INIT_LIST_HEAD(&file_set->files_list); + kho_block_set_init(&file_set->block_set, sizeof(struct luo_file_ser)); } void luo_file_set_destroy(struct luo_file_set *file_set) { WARN_ON(file_set->count); WARN_ON(!list_empty(&file_set->files_list)); + WARN_ON(!kho_block_set_is_empty(&file_set->block_set)); } /** diff --git a/kernel/liveupdate/luo_internal.h b/kernel/liveupdate/luo_internal.h index ee18f9a11b91..64879ffe7378 100644 --- a/kernel/liveupdate/luo_internal.h +++ b/kernel/liveupdate/luo_internal.h @@ -10,6 +10,7 @@ #include #include +#include struct luo_ucmd { void __user *ubuffer; @@ -44,14 +45,13 @@ static inline int luo_ucmd_respond(struct luo_ucmd *ucmd, * struct luo_file_set - A set of files that belong to the same sessions. * @files_list: An ordered list of files associated with this session, it is * ordered by preservation time. - * @files: The physically contiguous memory block that holds the serialized - * state of files. + * @block_set: The set of serialization blocks. * @count: A counter tracking the number of files currently stored in the * @files_list for this session. */ struct luo_file_set { struct list_head files_list; - struct luo_file_ser *files; + struct kho_block_set block_set; u64 count; }; -- cgit v1.2.3 From 40a25d59e85b3c8709ac2424d44f65610467871e Mon Sep 17 00:00:00 2001 From: Davidlohr Bueso Date: Thu, 7 May 2026 04:29:13 -0700 Subject: locking/rtmutex: Skip remove_waiter() when waiter is not enqueued syzbot triggered the following splat in remove_waiter() via FUTEX_CMP_REQUEUE_PI: KASAN: null-ptr-deref in range [0x0000000000000a88-0x0000000000000a8f] class_raw_spinlock_constructor remove_waiter+0x159/0x1200 kernel/locking/rtmutex.c:1561 rt_mutex_start_proxy_lock+0x103/0x120 futex_requeue+0x10e4/0x20d0 __x64_sys_futex+0x34f/0x4d0 task_blocks_on_rt_mutex() does not arm the waiter upon deadlock detection, leaving waiter->task nil, where 3bfdc63936dd ("rtmutex: Use waiter::task instead of current in remove_waiter()") made this fatal. Furthermore, rt_mutex_start_proxy_lock() should not be calling into remove_waiter() upon a successfully grabbing the rtmutex. 1a1fb985f2e2 ("futex: Handle early deadlock return correctly"), moved the remove_waiter() out of __rt_mutex_start_proxy_lock() (where 'ret' was only ever 0 or < 0) into the wrapper. Tighten this check to account for try_to_take_rt_mutex(). Fixes: 3bfdc63936dd ("rtmutex: Use waiter::task instead of current in remove_waiter()") Reported-by: syzbot+78147abe6c524f183ee9@syzkaller.appspotmail.com Signed-off-by: Davidlohr Bueso Signed-off-by: Thomas Gleixner Cc: stable@vger.kernel.org Closes: https://lore.kernel.org/all/69f114ac.050a0220.ac8b.0003.GAE@google.com/ Link: https://patch.msgid.link/20260507112913.1019537-1-dave@stgolabs.net --- kernel/locking/rtmutex.c | 3 +++ kernel/locking/rtmutex_api.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c index 4f386ea6c792..daeeeef973e2 100644 --- a/kernel/locking/rtmutex.c +++ b/kernel/locking/rtmutex.c @@ -1558,6 +1558,9 @@ static void __sched remove_waiter(struct rt_mutex_base *lock, lockdep_assert_held(&lock->wait_lock); + if (!waiter_task) /* never enqueued */ + return; + scoped_guard(raw_spinlock, &waiter_task->pi_lock) { rt_mutex_dequeue(lock, waiter); waiter_task->pi_blocked_on = NULL; diff --git a/kernel/locking/rtmutex_api.c b/kernel/locking/rtmutex_api.c index 124219aea46e..514fce7a4e0a 100644 --- a/kernel/locking/rtmutex_api.c +++ b/kernel/locking/rtmutex_api.c @@ -365,7 +365,7 @@ int __sched rt_mutex_start_proxy_lock(struct rt_mutex_base *lock, raw_spin_lock_irq(&lock->wait_lock); ret = __rt_mutex_start_proxy_lock(lock, waiter, task, &wake_q); - if (unlikely(ret)) + if (unlikely(ret < 0)) remove_waiter(lock, waiter); preempt_disable(); raw_spin_unlock_irq(&lock->wait_lock); -- cgit v1.2.3 From 41de4c70b0036646ae224b05c47c7756c5c2178d Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 3 Jun 2026 15:46:56 -1000 Subject: sched_ext: Order single-cid cmask helpers as (cid, mask) __scx_cmask_set(), __scx_cmask_contains() and __scx_cmask_word() take the cmask first and the cid second. The kernel's bit and cpumask predicates put the index first: test_bit(nr, addr), cpumask_test_cpu(cpu, mask). Reorder the cmask helpers to (cid, mask) for consistency, ahead of new single-cid ops added next. Mask-level ops (and/or/andnot/copy/subset/intersects) keep (dst, src). Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext_cid.c | 2 +- kernel/sched/ext_cid.h | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext_cid.c b/kernel/sched/ext_cid.c index 808c6390da5a..66944a7ef79d 100644 --- a/kernel/sched/ext_cid.c +++ b/kernel/sched/ext_cid.c @@ -267,7 +267,7 @@ void scx_cpumask_to_cmask(const struct cpumask *src, struct scx_cmask *dst) s32 cid = __scx_cpu_to_cid(cpu); if (cid >= 0) - __scx_cmask_set(dst, cid); + __scx_cmask_set(cid, dst); } } diff --git a/kernel/sched/ext_cid.h b/kernel/sched/ext_cid.h index abea22ba2cc2..46fd8eda0443 100644 --- a/kernel/sched/ext_cid.h +++ b/kernel/sched/ext_cid.h @@ -147,13 +147,13 @@ static inline bool scx_is_cid_type(void) return static_branch_unlikely(&__scx_is_cid_type); } -static inline bool __scx_cmask_contains(const struct scx_cmask *m, u32 cid) +static inline bool __scx_cmask_contains(u32 cid, const struct scx_cmask *m) { return likely(cid >= m->base && cid < m->base + m->nr_cids); } /* Word in bits[] covering @cid. @cid must satisfy __scx_cmask_contains(). */ -static inline u64 *__scx_cmask_word(const struct scx_cmask *m, u32 cid) +static inline u64 *__scx_cmask_word(u32 cid, const struct scx_cmask *m) { return (u64 *)&m->bits[cid / 64 - m->base / 64]; } @@ -218,11 +218,11 @@ static inline void scx_cmask_reframe(struct scx_cmask *m, u32 base, u32 nr_cids) m->nr_cids = nr_cids; } -static inline void __scx_cmask_set(struct scx_cmask *m, u32 cid) +static inline void __scx_cmask_set(u32 cid, struct scx_cmask *m) { - if (!__scx_cmask_contains(m, cid)) + if (!__scx_cmask_contains(cid, m)) return; - *__scx_cmask_word(m, cid) |= BIT_U64(cid & 63); + *__scx_cmask_word(cid, m) |= BIT_U64(cid & 63); } #endif /* _KERNEL_SCHED_EXT_CID_H */ -- cgit v1.2.3 From 5839c07e89bbbc0542fcaae9491ae7970cdeebb8 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 3 Jun 2026 15:46:56 -1000 Subject: sched_ext: Add scx_cmask_test() and scx_cmask_for_each_cid() Add single-bit test and iterator over set cids in an scx_cmask. v2: Bound scx_cmask_for_each_cid() to the active span. (sashiko AI) Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext_cid.h | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/ext_cid.h b/kernel/sched/ext_cid.h index 46fd8eda0443..5745e5785e89 100644 --- a/kernel/sched/ext_cid.h +++ b/kernel/sched/ext_cid.h @@ -225,4 +225,47 @@ static inline void __scx_cmask_set(u32 cid, struct scx_cmask *m) *__scx_cmask_word(cid, m) |= BIT_U64(cid & 63); } +/** + * scx_cmask_test - test whether @cid is set in @m + * @cid: cid to test + * @m: cmask to test + * + * Return %false if @cid is outside @m's active range. Otherwise return the + * bit's value. Read via READ_ONCE so callers can race set/clear writers. + */ +static inline bool scx_cmask_test(u32 cid, const struct scx_cmask *m) +{ + if (!__scx_cmask_contains(cid, m)) + return false; + return READ_ONCE(*__scx_cmask_word(cid, m)) & BIT_U64(cid & 63); +} + +/* + * Words of bits[] the active range spans, 0 if empty. Tighter than the storage + * SCX_CMASK_NR_WORDS() sizes for the worst-case base alignment. + */ +static inline u32 scx_cmask_nr_used_words(const struct scx_cmask *m) +{ + if (!m->nr_cids) + return 0; + return ((m->base & 63) + m->nr_cids - 1) / 64 + 1; +} + +/** + * scx_cmask_for_each_cid - iterate set cids in @m + * @cid: s32 loop var that receives each set cid in turn + * @m: cmask to iterate + * + * Visits set bits within @m's active range in ascending order. Scans only the + * words the active range spans, where head and tail padding is kept zero, so + * no per-cid range check is needed. + */ +#define scx_cmask_for_each_cid(cid, m) \ + for (u64 __bs = (m)->base & ~63u, __wi = 0, \ + __nw = scx_cmask_nr_used_words(m); \ + __wi < __nw; __wi++) \ + for (u64 __w = READ_ONCE((m)->bits[__wi]); \ + __w && ((cid) = __bs + __wi * 64 + __ffs64(__w), true); \ + __w &= __w - 1) + #endif /* _KERNEL_SCHED_EXT_CID_H */ -- cgit v1.2.3 From 70390da50c30cb22a8b19054f15df1b1bb38904c Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 3 Jun 2026 15:46:56 -1000 Subject: sched_ext: Make scx_bpf_kick_cid() return s32 Switch scx_bpf_kick_cid() from void to s32 so future cap enforcement can surface failures. cid interface is introduced in this cycle and has no external users, so the ABI change is safe. Subsequent patches will add -EPERM returns when the calling sub-sched lacks the required cap on the target cid. v2: Return scx_cid_to_cpu()'s errno instead of -EINVAL. (Andrea) Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 32ebbc351564..62769abb553a 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -9405,9 +9405,10 @@ __bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux * @flags: %SCX_KICK_* flags * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * - * cid-addressed equivalent of scx_bpf_kick_cpu(). + * cid-addressed equivalent of scx_bpf_kick_cpu(). Return 0 on success, + * -errno otherwise. */ -__bpf_kfunc void scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux) +__bpf_kfunc s32 scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux) { struct scx_sched *sch; s32 cpu; @@ -9415,10 +9416,12 @@ __bpf_kfunc void scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux guard(rcu)(); sch = scx_prog_sched(aux); if (unlikely(!sch)) - return; + return -ENODEV; cpu = scx_cid_to_cpu(sch, cid); - if (cpu >= 0) - scx_kick_cpu(sch, cpu, flags); + if (cpu < 0) + return cpu; + scx_kick_cpu(sch, cpu, flags); + return 0; } /** -- cgit v1.2.3 From 705e1068071f82b6c66b9e28124fbb7123b04c1d Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 29 May 2026 22:00:20 +0200 Subject: timekeeping: Remove system_time_snapshot::real/boot/raw MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All users are converted over to ktime_get_snapshot_id() and system_time_snapshot::systime and ::monoraw. Remove the leftovers. Signed-off-by: Thomas Gleixner Tested-by: David Woodhouse Tested-by: Arthur Kiyanovski Reviewed-by: David Woodhouse Reviewed-by: Thomas Weißschuh Reviewed-by: Jacob Keller Link: https://patch.msgid.link/20260529195557.330029635@kernel.org --- kernel/time/timekeeping.c | 8 -------- 1 file changed, 8 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index ccd04addb021..a134b1bad923 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -1196,8 +1196,6 @@ void ktime_get_snapshot_id(clockid_t clock_id, struct system_time_snapshot *syst struct timekeeper *tk; struct tk_data *tkd; unsigned int seq; - ktime_t base_real; - ktime_t base_boot; /* Invalidate the snapshot for all failure cases */ systime_snapshot->valid = false; @@ -1239,18 +1237,12 @@ void ktime_get_snapshot_id(clockid_t clock_id, struct system_time_snapshot *syst offs_sys = *offs; base_raw = tk->tkr_raw.base; - /* Kept around until the callers are fixed up */ - base_real = ktime_add(base_sys, tk_core.timekeeper.offs_real); - base_boot = ktime_add(base_sys, tk_core.timekeeper.offs_boot); - nsec_sys = timekeeping_cycles_to_ns(&tk->tkr_mono, now); nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, now); } while (read_seqcount_retry(&tkd->seq, seq)); systime_snapshot->cycles = now; systime_snapshot->systime = ktime_add_ns(base_sys, offs_sys + nsec_sys); - systime_snapshot->real = ktime_add_ns(base_real, nsec_sys); - systime_snapshot->boot = ktime_add_ns(base_boot, nsec_sys); systime_snapshot->monoraw = ktime_add_ns(base_raw, nsec_raw); /* -- cgit v1.2.3 From 23c1bfa9f8dbefe0be1fa32aaf235c08cb24bd78 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 29 May 2026 22:00:24 +0200 Subject: timekeeping: Add CLOCK_AUX support for ktime_get_snapshot_id() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that all users are converted it's possible to enable snapshotting of CLOCK_AUX time. The underlying clocksource is the same as for all other CLOCK variants. Signed-off-by: Thomas Gleixner Tested-by: Arthur Kiyanovski Reviewed-by: David Woodhouse Reviewed-by: Thomas Weißschuh Reviewed-by: Jacob Keller Link: https://patch.msgid.link/20260529195557.380601005@kernel.org --- kernel/time/timekeeping.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'kernel') diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index a134b1bad923..e43aa11dbfd6 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -67,6 +67,7 @@ static inline bool tk_is_aux(const struct timekeeper *tk) { return tk->id >= TIMEKEEPER_AUX_FIRST && tk->id <= TIMEKEEPER_AUX_LAST; } +static inline struct tk_data *aux_get_tk_data(clockid_t id); #else static inline bool tk_get_aux_ts64(unsigned int tkid, struct timespec64 *ts) { @@ -77,6 +78,10 @@ static inline bool tk_is_aux(const struct timekeeper *tk) { return false; } +static inline struct tk_data *aux_get_tk_data(clockid_t id) +{ + return NULL; +} #endif static inline void tk_update_aux_offs(struct timekeeper *tk, ktime_t offs) @@ -1218,6 +1223,12 @@ void ktime_get_snapshot_id(clockid_t clock_id, struct system_time_snapshot *syst tkd = &tk_core; offs = &tk_core.timekeeper.offs_boot; break; + case CLOCK_AUX ... CLOCK_AUX_LAST: + tkd = aux_get_tk_data(clock_id); + if (!tkd) + return; + offs = &tkd->timekeeper.offs_aux; + break; default: WARN_ON_ONCE(1); return; @@ -1228,6 +1239,10 @@ void ktime_get_snapshot_id(clockid_t clock_id, struct system_time_snapshot *syst do { seq = read_seqcount_begin(&tkd->seq); + /* Aux clocks can be invalid */ + if (!tk->clock_valid) + return; + now = tk_clock_read(&tk->tkr_mono); systime_snapshot->cs_id = tk->tkr_mono.clock->id; systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq; -- cgit v1.2.3 From 6c14771816435a7ecf52d981a341e2de21463268 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 29 May 2026 22:00:28 +0200 Subject: timekeeping: Add system_counterval_t to struct system_device_crosststamp An upcoming extension to the PTP IOCTL requires to return the system counter value and the clocksource ID to user space. get_device_system_crosststamp() has this information already. Extend struct system_device_crosststamp with a system_counterval_t member and fill in the data. Signed-off-by: Thomas Gleixner Tested-by: David Woodhouse Tested-by: Arthur Kiyanovski Reviewed-by: David Woodhouse Reviewed-by: Jacob Keller Link: https://patch.msgid.link/20260529195557.429406675@kernel.org --- kernel/time/timekeeping.c | 46 +++++++++++++++++++++------------------------- 1 file changed, 21 insertions(+), 25 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index e43aa11dbfd6..372179ff27a3 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -1418,6 +1418,8 @@ static bool convert_base_to_cs(struct system_counterval_t *scv) return false; scv->cycles += base->offset; + /* Set the clocksource ID as scv::cycles is now clocksource based */ + scv->cs_id = cs->id; return true; } @@ -1485,11 +1487,11 @@ EXPORT_SYMBOL_GPL(ktime_real_to_base_clock); /** * get_device_system_crosststamp - Synchronously capture system/device timestamp - * @get_time_fn: Callback to get simultaneous device time and - * system counter from the device driver + * @get_time_fn: Callback to get simultaneous device time and system counter + * from the device driver * @ctx: Context passed to get_time_fn() - * @history_begin: Historical reference point used to interpolate system - * time when counter provided by the driver is before the current interval + * @history_begin: Historical reference point used to interpolate system time when + * the counter value provided by the driver is before the current interval * @xtstamp: Receives simultaneously captured system and device time * * Reads a timestamp from a device and correlates it to system time @@ -1502,14 +1504,12 @@ int get_device_system_crosststamp(int (*get_time_fn) struct system_time_snapshot *history_begin, struct system_device_crosststamp *xtstamp) { - struct system_counterval_t system_counterval = {}; + u64 syscnt_cycles, cycles, now, interval_start; struct timekeeper *tk = &tk_core.timekeeper; - u64 cycles, now, interval_start; - unsigned int clock_was_set_seq = 0; + unsigned int seq, clock_was_set_seq = 0; ktime_t base_real, base_raw; u64 nsec_real, nsec_raw; u8 cs_was_changed_seq; - unsigned int seq; bool do_interp; int ret; @@ -1519,19 +1519,20 @@ int get_device_system_crosststamp(int (*get_time_fn) * Try to synchronously capture device time and a system * counter value calling back into the device driver */ - ret = get_time_fn(&xtstamp->device, &system_counterval, ctx); + ret = get_time_fn(&xtstamp->device, &xtstamp->sys_counter, ctx); if (ret) return ret; /* * Verify that the clocksource ID associated with the captured * system counter value is the same as for the currently - * installed timekeeper clocksource + * installed timekeeper clocksource and convert to it. */ - if (system_counterval.cs_id == CSID_GENERIC || - !convert_base_to_cs(&system_counterval)) + if (xtstamp->sys_counter.cs_id == CSID_GENERIC || + !convert_base_to_cs(&xtstamp->sys_counter)) return -ENODEV; - cycles = system_counterval.cycles; + + cycles = syscnt_cycles = xtstamp->sys_counter.cycles; /* * Check whether the system counter value provided by the @@ -1573,24 +1574,19 @@ int get_device_system_crosststamp(int (*get_time_fn) * clocksource change */ if (!history_begin || - !timestamp_in_interval(history_begin->cycles, - cycles, system_counterval.cycles) || + !timestamp_in_interval(history_begin->cycles, cycles, syscnt_cycles) || history_begin->cs_was_changed_seq != cs_was_changed_seq) return -EINVAL; - partial_history_cycles = cycles - system_counterval.cycles; + + partial_history_cycles = cycles - syscnt_cycles; total_history_cycles = cycles - history_begin->cycles; - discontinuity = - history_begin->clock_was_set_seq != clock_was_set_seq; + discontinuity = history_begin->clock_was_set_seq != clock_was_set_seq; - ret = adjust_historical_crosststamp(history_begin, - partial_history_cycles, - total_history_cycles, - discontinuity, xtstamp); - if (ret) - return ret; + ret = adjust_historical_crosststamp(history_begin, partial_history_cycles, + total_history_cycles, discontinuity, xtstamp); } - return 0; + return ret; } EXPORT_SYMBOL_GPL(get_device_system_crosststamp); -- cgit v1.2.3 From 09fb74f77e02489d1d8c555d44bab99d4905127c Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 29 May 2026 22:01:00 +0200 Subject: timekeeping: Prepare for cross timestamps on arbitrary clock IDs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PTP device system crosstime stamps support only CLOCK_REALTIME, which is meaningless for AUX clocks. The PTP core hands in the clock ID already, so prepare the core code to honor it. - Add a new sys_systime field to struct system_device_crosststamp which aliases the sys_realtime field. Once all users are converted sys_realtime can be removed. - Prepare get_device_system_crosststamp() and the related code for it by switching to sys_systime and providing the initial changes to utilize different time keepers. No functional change intended. Signed-off-by: Thomas Gleixner Tested-by: David Woodhouse Tested-by: Arthur Kiyanovski Reviewed-by: David Woodhouse Reviewed-by: Thomas Weißschuh Reviewed-by: Jacob Keller Link: https://patch.msgid.link/20260529195557.846634842@kernel.org --- kernel/time/timekeeping.c | 60 +++++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 25 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 372179ff27a3..dae61c9e31e6 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -1312,7 +1312,7 @@ static int adjust_historical_crosststamp(struct system_time_snapshot *history, struct system_device_crosststamp *ts) { struct timekeeper *tk = &tk_core.timekeeper; - u64 corr_raw, corr_real; + u64 corr_raw, corr_sys; bool interp_forward; int ret; @@ -1329,8 +1329,7 @@ static int adjust_historical_crosststamp(struct system_time_snapshot *history, * Scale the monotonic raw time delta by: * partial_history_cycles / total_history_cycles */ - corr_raw = (u64)ktime_to_ns( - ktime_sub(ts->sys_monoraw, history->monoraw)); + corr_raw = (u64)ktime_to_ns(ktime_sub(ts->sys_monoraw, history->monoraw)); ret = scale64_check_overflow(partial_history_cycles, total_history_cycles, &corr_raw); if (ret) @@ -1338,30 +1337,29 @@ static int adjust_historical_crosststamp(struct system_time_snapshot *history, /* * If there is a discontinuity in the history, scale monotonic raw - * correction by: - * mult(real)/mult(raw) yielding the realtime correction - * Otherwise, calculate the realtime correction similar to monotonic - * raw calculation + * correction by: + * mult(sys)/mult(raw) yielding the system time correction + * + * Otherwise, calculate the system time correction similar to monotonic + * raw calculation */ if (discontinuity) { - corr_real = mul_u64_u32_div - (corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult); + corr_sys = mul_u64_u32_div(corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult); } else { - corr_real = (u64)ktime_to_ns( - ktime_sub(ts->sys_realtime, history->systime)); - ret = scale64_check_overflow(partial_history_cycles, - total_history_cycles, &corr_real); + corr_sys = (u64)ktime_to_ns(ktime_sub(ts->sys_systime, history->systime)); + ret = scale64_check_overflow(partial_history_cycles, total_history_cycles, + &corr_sys); if (ret) return ret; } - /* Fixup monotonic raw and real time time values */ + /* Fixup monotonic raw and system time time values */ if (interp_forward) { ts->sys_monoraw = ktime_add_ns(history->monoraw, corr_raw); - ts->sys_realtime = ktime_add_ns(history->systime, corr_real); + ts->sys_systime = ktime_add_ns(history->systime, corr_sys); } else { ts->sys_monoraw = ktime_sub_ns(ts->sys_monoraw, corr_raw); - ts->sys_realtime = ktime_sub_ns(ts->sys_realtime, corr_real); + ts->sys_systime = ktime_sub_ns(ts->sys_systime, corr_sys); } return 0; @@ -1505,16 +1503,29 @@ int get_device_system_crosststamp(int (*get_time_fn) struct system_device_crosststamp *xtstamp) { u64 syscnt_cycles, cycles, now, interval_start; - struct timekeeper *tk = &tk_core.timekeeper; unsigned int seq, clock_was_set_seq = 0; - ktime_t base_real, base_raw; - u64 nsec_real, nsec_raw; + ktime_t base_sys, base_raw, *offs; + u64 nsec_sys, nsec_raw; u8 cs_was_changed_seq; bool do_interp; + struct timekeeper *tk; + struct tk_data *tkd; int ret; + switch (xtstamp->clock_id) { + case CLOCK_REALTIME: + tkd = &tk_core; + offs = &tk_core.timekeeper.offs_real; + break; + default: + WARN_ON_ONCE(1); + return -ENODEV; + } + + tk = &tkd->timekeeper; + do { - seq = read_seqcount_begin(&tk_core.seq); + seq = read_seqcount_begin(&tkd->seq); /* * Try to synchronously capture device time and a system * counter value calling back into the device driver @@ -1549,15 +1560,14 @@ int get_device_system_crosststamp(int (*get_time_fn) do_interp = false; } - base_real = ktime_add(tk->tkr_mono.base, - tk_core.timekeeper.offs_real); + base_sys = ktime_add(tk->tkr_mono.base, *offs); base_raw = tk->tkr_raw.base; - nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, cycles); + nsec_sys = timekeeping_cycles_to_ns(&tk->tkr_mono, cycles); nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, cycles); - } while (read_seqcount_retry(&tk_core.seq, seq)); + } while (read_seqcount_retry(&tkd->seq, seq)); - xtstamp->sys_realtime = ktime_add_ns(base_real, nsec_real); + xtstamp->sys_systime = ktime_add_ns(base_sys, nsec_sys); xtstamp->sys_monoraw = ktime_add_ns(base_raw, nsec_raw); /* -- cgit v1.2.3 From 9aebde8af6fe247d92816c197badf7e236c8f037 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 29 May 2026 22:01:21 +0200 Subject: timekeeping: Add support for AUX clock cross timestamping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that all prerequisites are in place add the final support for AUX clocks in get_device_system_crosststamp(), which enables the PTP layer to support hardware cross timestamps with a new IOTCL. Signed-off-by: Thomas Gleixner Tested-by: Arthur Kiyanovski Reviewed-by: David Woodhouse Reviewed-by: Thomas Weißschuh Reviewed-by: Jacob Keller Link: https://patch.msgid.link/20260529195558.097464513@kernel.org --- kernel/time/timekeeping.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'kernel') diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index dae61c9e31e6..65d870c2d40d 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -1517,6 +1517,12 @@ int get_device_system_crosststamp(int (*get_time_fn) tkd = &tk_core; offs = &tk_core.timekeeper.offs_real; break; + case CLOCK_AUX ... CLOCK_AUX_LAST: + tkd = aux_get_tk_data(xtstamp->clock_id); + if (!tkd) + return -ENODEV; + offs = &tkd->timekeeper.offs_aux; + break; default: WARN_ON_ONCE(1); return -ENODEV; -- cgit v1.2.3 From ca1ec8bfac8c95d0fed9e3611ea21400d1f37262 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Fri, 29 May 2026 22:01:29 +0200 Subject: timekeeping: Add clocksource read_snapshot() method and hw_cycles to snapshot Add a read_snapshot() callback to struct clocksource which returns the derived clocksource value while also providing the underlying hardware counter reading and the related clocksource ID. This allows ktime_get_snapshot_id() to populate new hw_cycles and hw_csid fields in struct system_time_snapshot. For clocksources that are derived from an underlying counter (e.g., Hyper-V TSC page scales TSC to 10MHz, kvmclock scales TSC to 1GHz), this provides atomic access to both the derived value needed for timekeeping calculations, and the raw hardware counter needed by consumers like KVM's master clock and the vmclock PTP driver. [ tglx: Reworked it slightly ] Signed-off-by: David Woodhouse Signed-off-by: Thomas Gleixner Reviewed-by: Jacob Keller Assisted-by: Kiro:claude-opus-4.6-1m Link: https://patch.msgid.link/20260526230635.136914-1-dwmw2@infradead.org Link: https://patch.msgid.link/20260529195558.202568489@kernel.org --- kernel/time/timekeeping.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 65d870c2d40d..0d5b67f609bb 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -320,6 +320,7 @@ static __always_inline u64 tk_clock_read(const struct tk_read_base *tkr) return clock->read(clock); } + static inline void clocksource_disable_inline_read(void) { } static inline void clocksource_enable_inline_read(void) { } #endif @@ -1187,6 +1188,18 @@ noinstr time64_t __ktime_get_real_seconds(void) return tk->xtime_sec; } +static inline u64 tk_clock_read_snapshot(const struct tk_read_base *tkr, + struct clocksource_hw_snapshot *chs) +{ + struct clocksource *clock = READ_ONCE(tkr->clock); + + if (unlikely(clock->read_snapshot)) + return clock->read_snapshot(clock, chs); + + return clock->read(clock); +} + + /** * ktime_get_snapshot_id - Simultaneously snapshot a given clock ID with * CLOCK_MONOTONIC_RAW and the underlying @@ -1237,14 +1250,20 @@ void ktime_get_snapshot_id(clockid_t clock_id, struct system_time_snapshot *syst tk = &tkd->timekeeper; do { + struct clocksource_hw_snapshot chs = { }; + seq = read_seqcount_begin(&tkd->seq); /* Aux clocks can be invalid */ if (!tk->clock_valid) return; - now = tk_clock_read(&tk->tkr_mono); + now = tk_clock_read_snapshot(&tk->tkr_mono, &chs); systime_snapshot->cs_id = tk->tkr_mono.clock->id; + + systime_snapshot->hw_cycles = chs.hw_cycles; + systime_snapshot->hw_csid = chs.hw_csid; + systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq; systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq; -- cgit v1.2.3 From 2369ce0f16b89e3d85c9683c06eb104545999378 Mon Sep 17 00:00:00 2001 From: Chun-Tse Shao Date: Tue, 2 Jun 2026 11:13:48 -0700 Subject: perf: Reveal PMU type in fdinfo It gives useful info on knowing which PMUs are reserved by this process. Also add config which would be useful. Testing cycles: $ ./perf stat -e cycles & $ cat /proc/`pidof perf`/fdinfo/3 pos: 0 flags: 02000002 mnt_id: 16 ino: 3081 perf_event_attr.type: 0 perf_event_attr.config: 0x0 perf_event_attr.config1: 0x0 perf_event_attr.config2: 0x0 perf_event_attr.config3: 0x0 perf_event_attr.config4: 0x0 Testing L1-dcache-load-misses: $ ./perf stat -e L1-dcache-load-misses & $ cat /proc/`pidof perf`/fdinfo/3 pos: 0 flags: 02000002 mnt_id: 16 ino: 1072 perf_event_attr.type: 3 perf_event_attr.config: 0x10000 perf_event_attr.config1: 0x0 perf_event_attr.config2: 0x0 perf_event_attr.config3: 0x0 perf_event_attr.config4: 0x0 Signed-off-by: Chun-Tse Shao Assisted-by: Gemini:gemini-3.1-pro-preview Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Ian Rogers Link: https://patch.msgid.link/20260602181349.3969429-1-ctshao@google.com --- kernel/events/core.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) (limited to 'kernel') diff --git a/kernel/events/core.c b/kernel/events/core.c index 7935d5663944..95d806bba654 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -58,6 +58,7 @@ #include #include #include +#include #include "internal.h" @@ -7546,6 +7547,33 @@ static int perf_fasync(int fd, struct file *filp, int on) return 0; } +static void perf_show_fdinfo(struct seq_file *m, struct file *f) +{ + struct perf_event *event = f->private_data; + struct perf_event_context *ctx; + struct mutex *child_mutex; + + ctx = perf_event_ctx_lock(event); + child_mutex = event->parent ? &event->parent->child_mutex : &event->child_mutex; + mutex_lock(child_mutex); + + seq_printf(m, "perf_event_attr.type:\t%u\n", event->orig_type); + if (event->pmu) + seq_printf(m, "pmu_type:\t%u\n", event->pmu->type); + seq_printf(m, "perf_event_attr.config:\t0x%llx\n", (unsigned long long)event->attr.config); + seq_printf(m, "perf_event_attr.config1:\t0x%llx\n", + (unsigned long long)event->attr.config1); + seq_printf(m, "perf_event_attr.config2:\t0x%llx\n", + (unsigned long long)event->attr.config2); + seq_printf(m, "perf_event_attr.config3:\t0x%llx\n", + (unsigned long long)event->attr.config3); + seq_printf(m, "perf_event_attr.config4:\t0x%llx\n", + (unsigned long long)event->attr.config4); + + mutex_unlock(child_mutex); + perf_event_ctx_unlock(event, ctx); +} + static const struct file_operations perf_fops = { .release = perf_release, .read = perf_read, @@ -7554,6 +7582,7 @@ static const struct file_operations perf_fops = { .compat_ioctl = perf_compat_ioctl, .mmap = perf_mmap, .fasync = perf_fasync, + .show_fdinfo = perf_show_fdinfo, }; /* -- cgit v1.2.3 From 2e05f2fd0dd72aa8aa56cf355e1e39a3f565b4ca Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 3 Jun 2026 23:57:03 -1000 Subject: sched_ext: Add scx_arena_to_kaddr() / scx_kaddr_to_arena() Translating between a BPF-arena pointer and its kernel-side address is just an add or subtract of the arena's kern_vm start. More such translations are coming, so cache that start on scx_sched as @arena_kern_base at arena attach and wrap both directions. Convert the existing open-coded subtraction in scx_call_op_set_cpumask(). Signed-off-by: Tejun Heo --- kernel/sched/ext.c | 19 +++++++++---------- kernel/sched/ext_internal.h | 36 +++++++++++++++++++++++++++++++----- 2 files changed, 40 insertions(+), 15 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 62769abb553a..6567f626b3f0 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -622,19 +622,13 @@ static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, if (scx_is_cid_type()) { struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch); - unsigned long uaddr = (unsigned long)kern_va; - - /* arena.o, which defines these, is built only on MMU && 64BIT */ -#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) - uaddr -= bpf_arena_map_kern_vm_start(sch->arena_map); -#endif /* - * Build the per-CPU arena cmask and hand BPF the uaddr. Caller - * holds the rq lock with IRQs disabled, which makes us the sole - * user of the scratch area. + * Build the per-CPU arena cmask and hand BPF its arena address. + * Caller holds the rq lock with IRQs disabled, which makes us + * the sole user of the scratch area. */ scx_cpumask_to_cmask(cpumask, kern_va); - sch->ops_cid.set_cmask(task, (struct scx_cmask *)uaddr); + sch->ops_cid.set_cmask(task, scx_kaddr_to_arena(sch, kern_va)); } else { sch->ops.set_cpumask(task, cpumask); } @@ -6977,6 +6971,11 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, * runs through scx_sched_free_rcu_work() which puts it. */ sch->arena_map = cmd->arena_map; + /* BPF arena is only available on MMU && 64BIT */ +#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) + if (sch->arena_map) + sch->arena_kern_base = bpf_arena_map_kern_vm_start(sch->arena_map); +#endif cmd->arena_map = NULL; return sch; diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext_internal.h index 9bb65367f510..b04701190b23 100644 --- a/kernel/sched/ext_internal.h +++ b/kernel/sched/ext_internal.h @@ -1118,17 +1118,18 @@ struct scx_sched { * progs. NULL on cpu-form. * * @arena_pool sub-allocates @arena_map. Each gen_pool chunk is added - * at the kernel-side mapping address. Grows on demand and pages are - * not released until sched destroy. + * at the kernel-side mapping address. @arena_kern_base is the start + * of the arena's kern_vm range. See scx_arena_to_kaddr() and + * scx_kaddr_to_arena(). */ struct bpf_map *arena_map; struct gen_pool *arena_pool; + uintptr_t arena_kern_base; /* * Per-CPU arena cmask used by scx_call_op_set_cpumask() to hand a cmask - * to ops_cid.set_cmask(). The kernel writes through the stored kern_va; - * the BPF-arena uaddr handed to BPF is recovered by subtracting the - * arena's kern_vm_start. + * to ops_cid.set_cmask(). The kernel writes through the stored kern_va + * and hands BPF its arena pointer via scx_kaddr_to_arena(). */ struct scx_cmask * __percpu *set_cmask_scratch; @@ -1205,6 +1206,31 @@ struct scx_sched { struct scx_sched *ancestors[]; }; +/** + * scx_arena_to_kaddr - Translate a BPF-arena pointer to its kernel address + * @sch: scheduler whose arena hosts @bpf_ptr + * @bpf_ptr: BPF-arena pointer, only the low 32 bits are used + * + * The (u32) cast normalizes any input into the arena's 4 GiB kern_vm range, + * which combined with scratch-page fault recovery makes the returned pointer + * safe to dereference up to GUARD_SZ / 2 past the intended object. Accesses + * larger than GUARD_SZ / 2 must be explicitly bounds-checked. + */ +static inline void *scx_arena_to_kaddr(struct scx_sched *sch, const void *bpf_ptr) +{ + return (void *)(sch->arena_kern_base + (u32)(uintptr_t)bpf_ptr); +} + +/** + * scx_kaddr_to_arena - Translate a kernel arena address to its BPF form + * @sch: scheduler whose arena hosts @kaddr + * @kaddr: kernel-side arena address, supplied by trusted kernel code + */ +static inline void *scx_kaddr_to_arena(struct scx_sched *sch, const void *kaddr) +{ + return (void *)((uintptr_t)kaddr - sch->arena_kern_base); +} + enum scx_wake_flags { /* expose select WF_* flags as enums */ SCX_WAKE_FORK = WF_FORK, -- cgit v1.2.3 From d486b4934a8e504376b85cdb3766f306d57aff5b Mon Sep 17 00:00:00 2001 From: Amit Matityahu Date: Wed, 3 Jun 2026 17:01:39 +0000 Subject: timers/migration: Fix livelock in tmigr_handle_remote_up() tmigr_handle_remote_cpu() skips timer_expire_remote() when cpu == smp_processor_id(), assuming the local softirq path already handled this CPU's timers. This assumption is wrong because jiffies can advance after the handling of the CPU's global timers in run_timer_base(BASE_GLOBAL) and before tmigr_handle_remote() evaluates the expiry times. As a consequence a timer which expires after the CPU local timer wheel advanced and becomes expired in the remote handling is ignored and the callback is never invoked and removed from the timer wheel. What's worse is that fetch_next_timer_interrupt_remote() keeps reporting it as expired, and the event is re-queued with expires == now on each iteration. The goto-again loop spins indefinitely. Fix this by calling timer_expire_remote() unconditionally. That's minimal overhead for the common case as __run_timer_base() returns immediately if there is nothing to expire in the local wheel. [ tglx: Amend change log and add a comment ] Fixes: 7ee988770326 ("timers: Implement the hierarchical pull model") Reported-by: Alon Kariv Signed-off-by: Amit Matityahu Signed-off-by: Thomas Gleixner Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260603170139.33628-1-amitmat@amazon.com --- kernel/time/timer_migration.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timer_migration.c b/kernel/time/timer_migration.c index 1d0d3a4058d5..52c15affdbff 100644 --- a/kernel/time/timer_migration.c +++ b/kernel/time/timer_migration.c @@ -978,8 +978,12 @@ static void tmigr_handle_remote_cpu(unsigned int cpu, u64 now, /* Drop the lock to allow the remote CPU to exit idle */ raw_spin_unlock_irq(&tmc->lock); - if (cpu != smp_processor_id()) - timer_expire_remote(cpu); + /* + * This can't exclude the local CPU because jiffies might have advanced + * after the timer softirq invoked run_timer_base(BASE_GLOBAL) and the + * point where the jiffies snapshot @jif was taken in tmigr_handle_remote(). + */ + timer_expire_remote(cpu); /* * Lock ordering needs to be preserved - timer_base locks before tmigr -- cgit v1.2.3 From 786d2d84416a9a1c1a47b71a68d679d886284be2 Mon Sep 17 00:00:00 2001 From: Andrii Kuchmenko Date: Mon, 18 May 2026 17:32:33 +0300 Subject: module: decompress: check return value of module_extend_max_pages() module_extend_max_pages() calls kvrealloc() internally and returns -ENOMEM on allocation failure. The return value is never checked. If the initial allocation fails, info->pages remains NULL and info->max_pages remains 0. Subsequent calls to module_get_next_page() will attempt to dynamically grow the array by calling module_extend_max_pages(info, 0) since info->used_pages is 0. This results in kvrealloc(NULL, 0) returning ZERO_SIZE_PTR, which is treated as a success, leading to a dereference of ZERO_SIZE_PTR and a kernel oops. Fix: add the missing error check after module_extend_max_pages() and return immediately on failure. This matches the pattern used by every other kvrealloc() caller in the module loading path. Fixes: b1ae6dc41eaa ("module: add in-kernel support for decompressing") Cc: Dmitry Torokhov Cc: Luis Chamberlain Cc: stable@vger.kernel.org Signed-off-by: Andrii Kuchmenko Reviewed-by: Christophe Leroy (CS GROUP) [Sami: Corrected the analysis in the commit message.] Signed-off-by: Sami Tolvanen --- kernel/module/decompress.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel') diff --git a/kernel/module/decompress.c b/kernel/module/decompress.c index 36f52a232a12..cce098671be9 100644 --- a/kernel/module/decompress.c +++ b/kernel/module/decompress.c @@ -307,6 +307,8 @@ int module_decompress(struct load_info *info, const void *buf, size_t size) */ n_pages = DIV_ROUND_UP(size, PAGE_SIZE) * 2; error = module_extend_max_pages(info, n_pages); + if (error) + return error; data_size = MODULE_DECOMPRESS_FN(info, buf, size); if (data_size < 0) { -- cgit v1.2.3 From fbd6dc50d9aedc594ec3196211a190170a275ab6 Mon Sep 17 00:00:00 2001 From: Matt Bobrowski Date: Wed, 3 Jun 2026 20:18:22 +0000 Subject: bpf: clean up btf_scan_decl_tags() Refactor the newly introduced btf_scan_decl_tags() to improve readability and maintainability. The current implementation uses a manual if-else chain and a magic number offset to strip the "arg:" prefix from declaration tags. Replace the if-else logic with a table-driven approach using a static const array. This separates the tag data from the scanning logic, making the helper more extensible for future tags. Additionally, replace the magic number '4' with a sizeof-based calculation on the prefix string to ensure the offset remains synchronized with the search key. Finally, optimize the loop by moving the is_global check to the top of the block. This allows the verifier to fail-fast on static subprograms without performing unnecessary BTF string and type lookups. Signed-off-by: Matt Bobrowski Reviewed-by: Emil Tsalapatis Link: https://lore.kernel.org/r/20260603201822.770596-1-mattbobrowski@google.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/btf.c | 46 ++++++++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 16 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 68921d9172b5..55aa3ba1b1e0 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7808,14 +7808,28 @@ static int btf_scan_decl_tags(struct bpf_verifier_env *env, u32 arg_idx, bool is_global, u32 *tags) { int id = btf_named_start_id(btf, false) - 1; + const char tag_key[] = "arg:"; + static const struct { + const char *tag_value; + enum btf_arg_tag arg_tag; + } tag_values[] = { + { "ctx", ARG_TAG_CTX }, + { "trusted", ARG_TAG_TRUSTED }, + { "untrusted", ARG_TAG_UNTRUSTED }, + { "nonnull", ARG_TAG_NONNULL }, + { "nullable", ARG_TAG_NULLABLE }, + { "arena", ARG_TAG_ARENA }, + }; /* * The 'arg:' decl_tag takes precedence over the derivation * of the register type from the BTF type itself. */ - while ((id = btf_find_next_decl_tag(btf, fn_t, arg_idx, "arg:", id)) > 0) { - const struct btf_type *tag_t = btf_type_by_id(btf, id); - const char *tag = __btf_name_by_offset(btf, tag_t->name_off) + 4; + while ((id = btf_find_next_decl_tag(btf, fn_t, arg_idx, tag_key, id)) > 0) { + const struct btf_type *tag_t; + const char *tag; + int i; + bool found; /* disallow arg tags in static subprogs */ if (!is_global) { @@ -7825,19 +7839,19 @@ static int btf_scan_decl_tags(struct bpf_verifier_env *env, return -EOPNOTSUPP; } - if (strcmp(tag, "ctx") == 0) { - *tags |= ARG_TAG_CTX; - } else if (strcmp(tag, "trusted") == 0) { - *tags |= ARG_TAG_TRUSTED; - } else if (strcmp(tag, "untrusted") == 0) { - *tags |= ARG_TAG_UNTRUSTED; - } else if (strcmp(tag, "nonnull") == 0) { - *tags |= ARG_TAG_NONNULL; - } else if (strcmp(tag, "nullable") == 0) { - *tags |= ARG_TAG_NULLABLE; - } else if (strcmp(tag, "arena") == 0) { - *tags |= ARG_TAG_ARENA; - } else { + tag_t = btf_type_by_id(btf, id); + tag = __btf_name_by_offset(btf, tag_t->name_off) + (sizeof(tag_key) - 1); + + found = false; + for (i = 0; i < ARRAY_SIZE(tag_values); ++i) { + if (!strcmp(tag, tag_values[i].tag_value)) { + *tags |= tag_values[i].arg_tag; + found = true; + break; + } + } + + if (!found) { bpf_log(&env->log, "arg#%d has unsupported set of tags\n", arg_idx); return -EOPNOTSUPP; } -- cgit v1.2.3 From 80b89d0226a05e8b67969de99c31b51fcd54f76a Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 28 May 2026 15:20:14 -0700 Subject: bpf: Take mmap_lock in zap_pages() zap_vma_range() requires the owning mm's mmap_lock to be held. Taking mmap_read_lock under arena->lock would AB-BA against arena_vm_close() and arena_map_mmap(), both of which run with mmap_write_lock held and then acquire arena->lock. Instead drop arena->lock, mmget_not_zero() the vma's mm, take mmap_read_lock, and re-resolve the vma via find_vma() since it may have been unmapped or replaced while waiting. Track processed vmls with a per-call generation in vml->zap_gen and serialize zap_pages() callers with a new arena->zap_mutex so concurrent callers on different uaddr ranges do not mark each other's vmls processed before the zap is done. Reported-by: David Hildenbrand Fixes: 317460317a02 ("bpf: Introduce bpf_arena.") Signed-off-by: Alexei Starovoitov Reviewed-by: Emil Tsalapatis Link: https://lore.kernel.org/r/20260528222014.38980-1-alexei.starovoitov@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/arena.c | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 57 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 1727503b25d8..9b2dea229b38 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -60,6 +60,8 @@ struct bpf_arena { struct list_head vma_list; /* protects vma_list */ struct mutex lock; + u64 zap_gen; + struct mutex zap_mutex; struct irq_work free_irq; struct work_struct free_work; struct llist_head free_spans; @@ -289,6 +291,7 @@ static struct bpf_map *arena_map_alloc(union bpf_attr *attr) if (err) goto err_free_scratch; mutex_init(&arena->lock); + mutex_init(&arena->zap_mutex); raw_res_spin_lock_init(&arena->spinlock); err = populate_pgtable_except_pte(arena); if (err) @@ -391,6 +394,7 @@ struct vma_list { struct vm_area_struct *vma; struct list_head head; refcount_t mmap_count; + u64 zap_gen; }; static int remember_vma(struct bpf_arena *arena, struct vm_area_struct *vma) @@ -403,6 +407,7 @@ static int remember_vma(struct bpf_arena *arena, struct vm_area_struct *vma) refcount_set(&vml->mmap_count, 1); vma->vm_private_data = vml; vml->vma = vma; + vml->zap_gen = 0; list_add(&vml->head, &arena->vma_list); return 0; } @@ -746,12 +751,60 @@ out_free_pages: */ static void zap_pages(struct bpf_arena *arena, long uaddr, long page_cnt) { + unsigned long size = (unsigned long)page_cnt << PAGE_SHIFT; + struct vm_area_struct *vma; + struct mm_struct *mm; struct vma_list *vml; + unsigned long vm_start; + u64 my_gen; - guard(mutex)(&arena->lock); - /* iterate link list under lock */ - list_for_each_entry(vml, &arena->vma_list, head) - zap_vma_range(vml->vma, uaddr, PAGE_SIZE * page_cnt); + /* + * Taking mmap_read_lock() under arena->lock would deadlock against + * arena_vm_close(), which runs with mmap_write_lock held and then + * acquires arena->lock. Drop arena->lock for mmap_read_lock(). + * + * Use per-call my_gen, recorded in vml->zap_gen, to remember which + * vmls this invocation has already processed across the lock drop. + * Hold zap_mutex around the whole walk so concurrent zap_pages() + * callers cannot overwrite each other's marks on shared vmls -- + * otherwise call B's mark would make call A skip a vml that A has + * not yet zapped for A's uaddr range. + */ + mutex_lock(&arena->zap_mutex); + mutex_lock(&arena->lock); + my_gen = ++arena->zap_gen; + for (;;) { + mm = NULL; + list_for_each_entry(vml, &arena->vma_list, head) { + if (vml->zap_gen >= my_gen) + continue; + vml->zap_gen = my_gen; + if (!mmget_not_zero(vml->vma->vm_mm)) + continue; + mm = vml->vma->vm_mm; + vm_start = vml->vma->vm_start; + break; + } + if (!mm) + break; + mutex_unlock(&arena->lock); + + mmap_read_lock(mm); + /* + * Re-resolve: while we waited the VMA could have been unmapped + * and a different mapping installed at the same address. + */ + vma = find_vma(mm, vm_start); + if (vma && vma->vm_start == vm_start && + vma->vm_file && vma->vm_file->private_data == &arena->map) + zap_vma_range(vma, uaddr, size); + mmap_read_unlock(mm); + mmput(mm); + + mutex_lock(&arena->lock); + } + mutex_unlock(&arena->lock); + mutex_unlock(&arena->zap_mutex); } static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, bool sleepable) -- cgit v1.2.3 From 9a79524d1420e6b79a6868208c264f4518d1318e Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Tue, 26 May 2026 13:47:15 +0200 Subject: kcov: use WRITE_ONCE() for selftest mode stores The KCOV selftest enables coverage by setting current->kcov_mode to KCOV_MODE_TRACE_PC without installing a coverage area. If an interrupt records coverage in that window, the access should fault and expose the bug. When building for QEMU raspi0 (Raspberry Pi Zero, ARMv6, CONFIG_CPU_V6K=y, CONFIG_CURRENT_POINTER_IN_TPIDRURO=y) with GCC 13.3.0, the store that enables the mode is removed. The generated kcov_init() code only stores zero after the wait loop: mrc 15, 0, r3, cr13, cr0, {3} str r4, [r3, #2028] where r4 is zero. There is no store of KCOV_MODE_TRACE_PC before the loop, so the selftest reports success without exercising coverage. Use WRITE_ONCE() for the temporary mode stores. With the same compiler and config, kcov_init() contains the intended mode store: mov r3, #2 mrc 15, 0, r2, cr13, cr0, {3} str r3, [r2, #2028] Now that the KCOV selftest is actually executed, it may expose KCOV instrumentation issues depending on the kernel config. That is expected for a selftest that was intended to catch coverage from interrupt paths. Link: https://lore.kernel.org/20260526114715.38280-1-kmehltretter@gmail.com Fixes: 6cd0dd934b03 ("kcov: Add interrupt handling self test") Assisted-by: Codex:gpt-5 Signed-off-by: Karl Mehltretter Reviewed-by: Alexander Potapenko Cc: Andrey Konovalov Cc: Dmitry Vyukov Cc: Kees Cook Cc: Marco Elver Cc: Peter Zijlstra Cc: Signed-off-by: Andrew Morton --- kernel/kcov.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/kcov.c b/kernel/kcov.c index fd2503030729..1df373fb562b 100644 --- a/kernel/kcov.c +++ b/kernel/kcov.c @@ -1119,10 +1119,10 @@ static void __init selftest(void) * potentially traced functions in this region. */ start = jiffies; - current->kcov_mode = KCOV_MODE_TRACE_PC; + WRITE_ONCE(current->kcov_mode, KCOV_MODE_TRACE_PC); while ((jiffies - start) * MSEC_PER_SEC / HZ < 300) ; - current->kcov_mode = 0; + WRITE_ONCE(current->kcov_mode, 0); pr_err("done running self test\n"); } #endif -- cgit v1.2.3 From 16b4d3e2fb24aac3e68a8d86e3bc5e302e1b5cb7 Mon Sep 17 00:00:00 2001 From: Mykyta Yatsenko Date: Fri, 5 Jun 2026 04:41:21 -0700 Subject: bpf: Implement resizable hashmap basic functions Use rhashtable_lookup_likely() for lookups, rhashtable_remove_fast() for deletes, and rhashtable_lookup_get_insert_fast() for inserts. Updates modify values in place under RCU rather than allocating a new element and swapping the pointer (as regular htab does). This trades read consistency for performance: concurrent readers may see partial updates. BPF_F_LOCK support and special-field handling (timers, kptrs, etc.) follow in a later commit. Initialize rhashtable with bpf_mem_alloc element cache. Require BPF_F_NO_PREALLOC. Limit max_entries to 2^31. Free elements via rhashtable_free_and_destroy(). Signed-off-by: Mykyta Yatsenko Link: https://lore.kernel.org/r/20260605-rhash-v7-4-5b8e05f8630d@meta.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/hashtab.c | 311 ++++++++++++++++++++++++++++++++++++++++++++++++++ kernel/bpf/syscall.c | 3 + kernel/bpf/verifier.c | 1 + 3 files changed, 315 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 3dd9b4924ae4..10f3a058747b 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -2739,3 +2740,313 @@ const struct bpf_map_ops htab_of_maps_map_ops = { BATCH_OPS(htab), .map_btf_id = &htab_map_btf_ids[0], }; + +struct rhtab_elem { + struct rhash_head node; + /* key bytes, then value bytes follow */ + u8 data[] __aligned(8); +}; + +struct bpf_rhtab { + struct bpf_map map; + struct rhashtable ht; + struct bpf_mem_alloc ma; + u32 elem_size; +}; + +static const struct rhashtable_params rhtab_params = { + .head_offset = offsetof(struct rhtab_elem, node), + .key_offset = offsetof(struct rhtab_elem, data), +}; + +static inline void *rhtab_elem_value(struct rhtab_elem *l, u32 key_size) +{ + return l->data + round_up(key_size, 8); +} + +static struct bpf_map *rhtab_map_alloc(union bpf_attr *attr) +{ + struct rhashtable_params params; + struct bpf_rhtab *rhtab; + int err = 0; + + rhtab = bpf_map_area_alloc(sizeof(*rhtab), NUMA_NO_NODE); + if (!rhtab) + return ERR_PTR(-ENOMEM); + + bpf_map_init_from_attr(&rhtab->map, attr); + + if (rhtab->map.max_entries > 1UL << 31) { + err = -E2BIG; + goto free_rhtab; + } + + rhtab->elem_size = sizeof(struct rhtab_elem) + round_up(rhtab->map.key_size, 8) + + round_up(rhtab->map.value_size, 8); + + params = rhtab_params; + params.key_len = rhtab->map.key_size; + params.nelem_hint = (u32)attr->map_extra; + params.automatic_shrinking = true; + + err = rhashtable_init(&rhtab->ht, ¶ms); + if (err) + goto free_rhtab; + + /* Set max_elems after rhashtable_init() since init zeroes the struct */ + rhtab->ht.max_elems = rhtab->map.max_entries; + + err = bpf_mem_alloc_init(&rhtab->ma, rhtab->elem_size, false); + if (err) + goto destroy_rhtab; + + return &rhtab->map; + +destroy_rhtab: + rhashtable_destroy(&rhtab->ht); +free_rhtab: + bpf_map_area_free(rhtab); + return ERR_PTR(err); +} + +static int rhtab_map_alloc_check(union bpf_attr *attr) +{ + if (!(attr->map_flags & BPF_F_NO_PREALLOC)) + return -EINVAL; + + if (attr->map_flags & BPF_F_ZERO_SEED) + return -EINVAL; + + if (attr->key_size > U16_MAX) + return -E2BIG; + + if (attr->map_extra >> 32) + return -EINVAL; + + if ((u32)attr->map_extra > U16_MAX) + return -E2BIG; + + if ((u32)attr->map_extra > attr->max_entries) + return -EINVAL; + + return htab_map_alloc_check(attr); +} + +static void rhtab_free_elem(void *ptr, void *arg) +{ + struct bpf_rhtab *rhtab = arg; + struct rhtab_elem *elem = ptr; + + bpf_mem_cache_free_rcu(&rhtab->ma, elem); +} + +static void rhtab_map_free(struct bpf_map *map) +{ + struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map); + + rhashtable_free_and_destroy(&rhtab->ht, rhtab_free_elem, rhtab); + bpf_mem_alloc_destroy(&rhtab->ma); + bpf_map_area_free(rhtab); +} + +static void *rhtab_lookup_elem(struct bpf_map *map, void *key) +{ + struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map); + + /* Hold RCU lock in case sleepable program calls via gen_lookup */ + guard(rcu)(); + + return rhashtable_lookup_likely(&rhtab->ht, key, rhtab_params); +} + +static void *rhtab_map_lookup_elem(struct bpf_map *map, void *key) __must_hold(RCU) +{ + struct rhtab_elem *l; + + l = rhtab_lookup_elem(map, key); + return l ? rhtab_elem_value(l, map->key_size) : NULL; +} + +static void rhtab_read_elem_value(struct bpf_map *map, void *dst, struct rhtab_elem *elem, + u64 flags) +{ + void *src = rhtab_elem_value(elem, map->key_size); + + if (flags & BPF_F_LOCK) + copy_map_value_locked(map, dst, src, true); + else + copy_map_value(map, dst, src); +} + +static int rhtab_delete_elem(struct bpf_rhtab *rhtab, struct rhtab_elem *elem, void *copy, + u64 flags) +{ + int err; + + /* + * disable_instrumentation() mitigates the deadlock for programs running in NMI context. + * rhashtable locks bucket with local_irq_save(). Only NMI programs may reenter + * rhashtable code, bpf_disable_instrumentation() disables programs running in NMI, except + * raw tracepoints, which we don't have in rhashtable. + */ + bpf_disable_instrumentation(); + err = rhashtable_remove_fast(&rhtab->ht, &elem->node, rhtab_params); + bpf_enable_instrumentation(); + + if (err) + return err; + + if (copy) { + rhtab_read_elem_value(&rhtab->map, copy, elem, flags); + check_and_init_map_value(&rhtab->map, copy); + } + + bpf_mem_cache_free_rcu(&rhtab->ma, elem); + return 0; +} + + +static long rhtab_map_delete_elem(struct bpf_map *map, void *key) +{ + struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map); + struct rhtab_elem *elem; + + guard(rcu)(); + + elem = rhtab_lookup_elem(map, key); + if (!elem) + return -ENOENT; + + return rhtab_delete_elem(rhtab, elem, NULL, 0); +} + +static int rhtab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, void *value, u64 flags) +{ + struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map); + struct rhtab_elem *elem; + int err; + + err = bpf_map_check_op_flags(map, flags, BPF_F_LOCK); + if (err) + return err; + + guard(rcu)(); + + elem = rhtab_lookup_elem(map, key); + if (!elem) + return -ENOENT; + + return rhtab_delete_elem(rhtab, elem, value, flags); +} + +static long rhtab_map_update_existing(struct bpf_map *map, struct rhtab_elem *elem, void *value, + u64 map_flags) +{ + void *old_val = rhtab_elem_value(elem, map->key_size); + + if (map_flags & BPF_NOEXIST) + return -EEXIST; + + if (map_flags & BPF_F_LOCK) + copy_map_value_locked(map, old_val, value, false); + else + copy_map_value(map, old_val, value); + return 0; +} + +static long rhtab_map_update_elem(struct bpf_map *map, void *key, void *value, u64 map_flags) +{ + struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map); + struct rhtab_elem *elem, *tmp; + + if (unlikely((map_flags & ~BPF_F_LOCK) > BPF_EXIST)) + return -EINVAL; + + if ((map_flags & BPF_F_LOCK) && !btf_record_has_field(map->record, BPF_SPIN_LOCK)) + return -EINVAL; + + guard(rcu)(); + elem = rhtab_lookup_elem(map, key); + if (elem) + return rhtab_map_update_existing(map, elem, value, map_flags); + + if (map_flags & BPF_EXIST) + return -ENOENT; + + /* Check max_entries limit before inserting new element */ + if (atomic_read(&rhtab->ht.nelems) >= map->max_entries) + return -E2BIG; + + elem = bpf_mem_cache_alloc(&rhtab->ma); + if (!elem) + return -ENOMEM; + + memcpy(elem->data, key, map->key_size); + copy_map_value(map, rhtab_elem_value(elem, map->key_size), value); + + /* Prevent deadlock for NMI programs attempting to take bucket lock */ + bpf_disable_instrumentation(); + tmp = rhashtable_lookup_get_insert_fast(&rhtab->ht, &elem->node, rhtab_params); + bpf_enable_instrumentation(); + + if (tmp) { + bpf_mem_cache_free(&rhtab->ma, elem); + if (IS_ERR(tmp)) + return PTR_ERR(tmp); + + return rhtab_map_update_existing(map, tmp, value, map_flags); + } + + return 0; +} + +static int rhtab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) +{ + struct bpf_insn *insn = insn_buf; + const int ret = BPF_REG_0; + + BUILD_BUG_ON(!__same_type(&rhtab_lookup_elem, + (void *(*)(struct bpf_map *map, void *key)) NULL)); + *insn++ = BPF_EMIT_CALL(rhtab_lookup_elem); + *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1); + *insn++ = BPF_ALU64_IMM(BPF_ADD, ret, + offsetof(struct rhtab_elem, data) + round_up(map->key_size, 8)); + + return insn - insn_buf; +} + +static void rhtab_map_free_internal_structs(struct bpf_map *map) +{ +} + +static int rhtab_map_get_next_key(struct bpf_map *map, void *key, void *next_key) +{ + return -EOPNOTSUPP; +} + +static u64 rhtab_map_mem_usage(const struct bpf_map *map) +{ + struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map); + u64 num_entries; + + /* Excludes rhashtable bucket overhead (~ nelems * sizeof(void *) at 75% load). */ + num_entries = atomic_read(&rhtab->ht.nelems); + return sizeof(struct bpf_rhtab) + rhtab->elem_size * num_entries; +} + +BTF_ID_LIST_SINGLE(rhtab_map_btf_ids, struct, bpf_rhtab) +const struct bpf_map_ops rhtab_map_ops = { + .map_meta_equal = bpf_map_meta_equal, + .map_alloc_check = rhtab_map_alloc_check, + .map_alloc = rhtab_map_alloc, + .map_free = rhtab_map_free, + .map_get_next_key = rhtab_map_get_next_key, + .map_release_uref = rhtab_map_free_internal_structs, + .map_lookup_elem = rhtab_map_lookup_elem, + .map_lookup_and_delete_elem = rhtab_map_lookup_and_delete_elem, + .map_update_elem = rhtab_map_update_elem, + .map_delete_elem = rhtab_map_delete_elem, + .map_gen_lookup = rhtab_map_gen_lookup, + .map_mem_usage = rhtab_map_mem_usage, + .map_btf_id = &rhtab_map_btf_ids[0], +}; diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 625a4366fe6d..1faae184de48 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1398,6 +1398,7 @@ static int map_create_alloc(union bpf_attr *attr, bpfptr_t uattr, struct bpf_ver if (attr->map_type != BPF_MAP_TYPE_BLOOM_FILTER && attr->map_type != BPF_MAP_TYPE_ARENA && + attr->map_type != BPF_MAP_TYPE_RHASH && attr->map_extra != 0) { bpf_log(log, "Invalid map_extra.\n"); return -EINVAL; @@ -1469,6 +1470,7 @@ static int map_create_alloc(union bpf_attr *attr, bpfptr_t uattr, struct bpf_ver case BPF_MAP_TYPE_CGROUP_ARRAY: case BPF_MAP_TYPE_ARRAY_OF_MAPS: case BPF_MAP_TYPE_HASH: + case BPF_MAP_TYPE_RHASH: case BPF_MAP_TYPE_PERCPU_HASH: case BPF_MAP_TYPE_HASH_OF_MAPS: case BPF_MAP_TYPE_RINGBUF: @@ -2259,6 +2261,7 @@ static int map_lookup_and_delete_elem(union bpf_attr *attr) map->map_type == BPF_MAP_TYPE_PERCPU_HASH || map->map_type == BPF_MAP_TYPE_LRU_HASH || map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || + map->map_type == BPF_MAP_TYPE_RHASH || map->map_type == BPF_MAP_TYPE_STACK_TRACE) { if (!bpf_map_is_offloaded(map)) { bpf_disable_instrumentation(); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 8ed484cb1a8a..7d27ba396d32 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -17657,6 +17657,7 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, if (prog->sleepable) switch (map->map_type) { case BPF_MAP_TYPE_HASH: + case BPF_MAP_TYPE_RHASH: case BPF_MAP_TYPE_LRU_HASH: case BPF_MAP_TYPE_ARRAY: case BPF_MAP_TYPE_PERCPU_HASH: -- cgit v1.2.3 From 818e0084822742fc00eacbf5df3476a5e72c7d0e Mon Sep 17 00:00:00 2001 From: Mykyta Yatsenko Date: Fri, 5 Jun 2026 04:41:22 -0700 Subject: bpf: Implement iteration ops for resizable hashtab MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement get_next_key, batch lookup/lookup-and-delete, for_each_map_elem callback, and the seq_file BPF iterator for BPF_MAP_TYPE_RHASH. get_next_key() and batch use rhashtable_next_key() — stateless, matches the syscall UAPI shape (no kernel-side iterator state). get_next_key falls back to the first key when prev_key was concurrently deleted (matches htab semantics). Batch reports cursor loss as -EAGAIN so userspace can distinguish it from end-of-iteration (-ENOENT) and restart from NULL. The seq_file BPF iterator uses rhashtable_walk_* instead. It runs only from read() syscall context, so the walker's spin_lock is safe, and seq_file's per-fd state lets the walker handle rehash correctly (retry on -EAGAIN) for stronger coverage than the stateless API can provide. Signed-off-by: Mykyta Yatsenko Link: https://lore.kernel.org/r/20260605-rhash-v7-5-5b8e05f8630d@meta.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/hashtab.c | 347 +++++++++++++++++++++++++++++++++++++++++++++++++- kernel/bpf/map_iter.c | 3 +- 2 files changed, 348 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 10f3a058747b..a149713d0953 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -3020,8 +3020,79 @@ static void rhtab_map_free_internal_structs(struct bpf_map *map) } static int rhtab_map_get_next_key(struct bpf_map *map, void *key, void *next_key) + __must_hold_shared(RCU) { - return -EOPNOTSUPP; + struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map); + struct rhtab_elem *elem; + + elem = rhashtable_next_key(&rhtab->ht, key); + + /* if not found, return the first key */ + if (PTR_ERR(elem) == -ENOENT) + elem = rhashtable_next_key(&rhtab->ht, NULL); + + if (IS_ERR(elem)) + return PTR_ERR(elem); + if (!elem) + return -ENOENT; + + memcpy(next_key, elem->data, map->key_size); + return 0; +} + +static void rhtab_map_seq_show_elem(struct bpf_map *map, void *key, struct seq_file *m) +{ + void *value; + + /* Guarantee that hashtab value is not freed */ + guard(rcu)(); + + value = rhtab_map_lookup_elem(map, key); + if (!value) + return; + + btf_type_seq_show(map->btf, map->btf_key_type_id, key, m); + seq_puts(m, ": "); + btf_type_seq_show(map->btf, map->btf_value_type_id, value, m); + seq_putc(m, '\n'); +} + +static long bpf_each_rhash_elem(struct bpf_map *map, bpf_callback_t callback_fn, + void *callback_ctx, u64 flags) +{ + struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map); + void *prev_key = NULL; + struct rhtab_elem *elem; + int num_elems = 0; + u64 ret = 0; + + cant_migrate(); + + if (flags != 0) + return -EINVAL; + + rcu_read_lock(); + /* + * Best-effort iteration: if rhashtable is concurrently resized or + * elements are deleted/inserted, there may be missed or duplicate + * elements visited. + */ + while ((elem = rhashtable_next_key(&rhtab->ht, prev_key))) { + if (IS_ERR(elem)) + break; + num_elems++; + ret = callback_fn((u64)(long)map, + (u64)(long)elem->data, + (u64)(long)rhtab_elem_value(elem, map->key_size), + (u64)(long)callback_ctx, 0); + if (ret) + break; + + prev_key = elem->data; /* valid while RCU held */ + } + rcu_read_unlock(); + + return num_elems; } static u64 rhtab_map_mem_usage(const struct bpf_map *map) @@ -3034,6 +3105,275 @@ static u64 rhtab_map_mem_usage(const struct bpf_map *map) return sizeof(struct bpf_rhtab) + rhtab->elem_size * num_entries; } +static int __rhtab_map_lookup_and_delete_batch(struct bpf_map *map, + const union bpf_attr *attr, + union bpf_attr __user *uattr, + bool do_delete) +{ + struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map); + void __user *uvalues = u64_to_user_ptr(attr->batch.values); + void __user *ukeys = u64_to_user_ptr(attr->batch.keys); + void __user *ubatch = u64_to_user_ptr(attr->batch.in_batch); + void *cursor = NULL, *keys = NULL, *values = NULL, *dst_key, *dst_val; + struct rhtab_elem **del_elems = NULL; + u32 max_count, total, key_size, value_size, i; + bool has_next_cursor = false; + struct rhtab_elem *elem; + u64 elem_map_flags, map_flags; + int ret = 0; + + elem_map_flags = attr->batch.elem_flags; + ret = bpf_map_check_op_flags(map, elem_map_flags, BPF_F_LOCK); + if (ret) + return ret; + + map_flags = attr->batch.flags; + if (map_flags) + return -EINVAL; + + max_count = attr->batch.count; + if (!max_count) + return 0; + + if (put_user(0, &uattr->batch.count)) + return -EFAULT; + + key_size = map->key_size; + value_size = map->value_size; + + keys = kvmalloc_array(max_count, key_size, GFP_USER | __GFP_NOWARN); + values = kvmalloc_array(max_count, value_size, GFP_USER | __GFP_NOWARN); + if (do_delete) + del_elems = kvmalloc_array(max_count, sizeof(void *), + GFP_USER | __GFP_NOWARN); + cursor = kmalloc(key_size, GFP_USER | __GFP_NOWARN); + + if (!keys || !values || !cursor || (do_delete && !del_elems)) { + ret = -ENOMEM; + goto free; + } + + if (ubatch && copy_from_user(cursor, ubatch, key_size)) { + ret = -EFAULT; + goto free; + } + + dst_key = keys; + dst_val = values; + total = 0; + + rcu_read_lock(); + + /* + * Cursor stores the key of the next-to-process element (stashed by + * the previous batch). Look it up directly so the element is included + * here rather than skipped by next_key(). If the cursor was deleted + * concurrently (or by the previous do_delete batch), return -EAGAIN + * so userspace can distinguish a lost cursor from end-of-iteration + * (-ENOENT) and restart from a NULL cursor. + */ + if (ubatch) { + elem = rhtab_lookup_elem(map, cursor); + if (!elem) { + rcu_read_unlock(); + ret = -EAGAIN; + goto free; + } + } else { + elem = rhashtable_next_key(&rhtab->ht, NULL); + } + + while (elem && !IS_ERR(elem) && total < max_count) { + memcpy(dst_key, elem->data, key_size); + rhtab_read_elem_value(map, dst_val, elem, elem_map_flags); + check_and_init_map_value(map, dst_val); + + if (do_delete) + del_elems[total] = elem; + + elem = rhashtable_next_key(&rhtab->ht, dst_key); + dst_key += key_size; + dst_val += value_size; + total++; + + /* Bail to userspace to avoid stalls. */ + if (need_resched()) + break; + } + + if (elem && !IS_ERR(elem)) { + /* Stash next-to-process key as cursor for the next batch. */ + memcpy(cursor, elem->data, key_size); + has_next_cursor = true; + } + + if (do_delete) { + for (i = 0; i < total; i++) + rhtab_delete_elem(rhtab, del_elems[i], NULL, 0); + } + + rcu_read_unlock(); + + if (total == 0) { + ret = -ENOENT; + goto free; + } + + /* No more elements after this batch. */ + if (!has_next_cursor) + ret = -ENOENT; + + if (copy_to_user(ukeys, keys, (size_t)total * key_size) || + copy_to_user(uvalues, values, (size_t)total * value_size) || + put_user(total, &uattr->batch.count) || + (has_next_cursor && + copy_to_user(u64_to_user_ptr(attr->batch.out_batch), + cursor, key_size))) { + ret = -EFAULT; + goto free; + } + +free: + kfree(cursor); + kvfree(keys); + kvfree(values); + kvfree(del_elems); + return ret; +} + +static int rhtab_map_lookup_batch(struct bpf_map *map, const union bpf_attr *attr, + union bpf_attr __user *uattr) +{ + return __rhtab_map_lookup_and_delete_batch(map, attr, uattr, false); +} + +static int rhtab_map_lookup_and_delete_batch(struct bpf_map *map, const union bpf_attr *attr, + union bpf_attr __user *uattr) +{ + return __rhtab_map_lookup_and_delete_batch(map, attr, uattr, true); +} + +struct bpf_iter_seq_rhash_map_info { + struct bpf_map *map; + struct bpf_rhtab *rhtab; + struct rhashtable_iter iter; +}; + +static void *bpf_rhash_map_seq_start(struct seq_file *seq, loff_t *pos) + __acquires(RCU) +{ + struct bpf_iter_seq_rhash_map_info *info = seq->private; + struct rhtab_elem *elem; + + rhashtable_walk_start(&info->iter); + /* + * Re-deliver the element returned by walk_next() at the end of the + * previous read() — bpf_seq_read may have stopped before show() + * consumed it. Rehash rewinds the walker; retry on -EAGAIN. + */ + do { + elem = rhashtable_walk_peek(&info->iter); + } while (PTR_ERR(elem) == -EAGAIN); + + if (IS_ERR(elem)) + return NULL; + + if (elem && *pos == 0) + ++*pos; + return elem; +} + +static void *bpf_rhash_map_seq_next(struct seq_file *seq, void *v, loff_t *pos) +{ + struct bpf_iter_seq_rhash_map_info *info = seq->private; + struct rhtab_elem *elem; + + ++*pos; + + /* Rehash rewinds the walker; retry until it stops returning -EAGAIN. */ + do { + elem = rhashtable_walk_next(&info->iter); + } while (PTR_ERR(elem) == -EAGAIN); + + if (IS_ERR(elem)) + return NULL; + return elem; +} + +static int __bpf_rhash_map_seq_show(struct seq_file *seq, + struct rhtab_elem *elem) +{ + struct bpf_iter_seq_rhash_map_info *info = seq->private; + struct bpf_iter__bpf_map_elem ctx = {}; + struct bpf_iter_meta meta; + struct bpf_prog *prog; + int ret = 0; + + meta.seq = seq; + prog = bpf_iter_get_info(&meta, elem == NULL); + if (prog) { + ctx.meta = &meta; + ctx.map = info->map; + if (elem) { + ctx.key = elem->data; + ctx.value = rhtab_elem_value(elem, info->map->key_size); + } + ret = bpf_iter_run_prog(prog, &ctx); + } + + return ret; +} + +static int bpf_rhash_map_seq_show(struct seq_file *seq, void *v) +{ + return __bpf_rhash_map_seq_show(seq, v); +} + +static void bpf_rhash_map_seq_stop(struct seq_file *seq, void *v) + __releases(RCU) +{ + struct bpf_iter_seq_rhash_map_info *info = seq->private; + + if (!v) + (void)__bpf_rhash_map_seq_show(seq, NULL); + + rhashtable_walk_stop(&info->iter); +} + +static int bpf_iter_init_rhash_map(void *priv_data, struct bpf_iter_aux_info *aux) +{ + struct bpf_iter_seq_rhash_map_info *info = priv_data; + struct bpf_map *map = aux->map; + + bpf_map_inc_with_uref(map); + info->map = map; + info->rhtab = container_of(map, struct bpf_rhtab, map); + rhashtable_walk_enter(&info->rhtab->ht, &info->iter); + return 0; +} + +static void bpf_iter_fini_rhash_map(void *priv_data) +{ + struct bpf_iter_seq_rhash_map_info *info = priv_data; + + rhashtable_walk_exit(&info->iter); + bpf_map_put_with_uref(info->map); +} + +static const struct seq_operations bpf_rhash_map_seq_ops = { + .start = bpf_rhash_map_seq_start, + .next = bpf_rhash_map_seq_next, + .stop = bpf_rhash_map_seq_stop, + .show = bpf_rhash_map_seq_show, +}; + +static const struct bpf_iter_seq_info rhash_iter_seq_info = { + .seq_ops = &bpf_rhash_map_seq_ops, + .init_seq_private = bpf_iter_init_rhash_map, + .fini_seq_private = bpf_iter_fini_rhash_map, + .seq_priv_size = sizeof(struct bpf_iter_seq_rhash_map_info), +}; + BTF_ID_LIST_SINGLE(rhtab_map_btf_ids, struct, bpf_rhtab) const struct bpf_map_ops rhtab_map_ops = { .map_meta_equal = bpf_map_meta_equal, @@ -3047,6 +3387,11 @@ const struct bpf_map_ops rhtab_map_ops = { .map_update_elem = rhtab_map_update_elem, .map_delete_elem = rhtab_map_delete_elem, .map_gen_lookup = rhtab_map_gen_lookup, + .map_seq_show_elem = rhtab_map_seq_show_elem, + .map_set_for_each_callback_args = map_set_for_each_callback_args, + .map_for_each_callback = bpf_each_rhash_elem, .map_mem_usage = rhtab_map_mem_usage, + BATCH_OPS(rhtab), .map_btf_id = &rhtab_map_btf_ids[0], + .iter_seq_info = &rhash_iter_seq_info, }; diff --git a/kernel/bpf/map_iter.c b/kernel/bpf/map_iter.c index ae0741a09c6d..c19b360bad9e 100644 --- a/kernel/bpf/map_iter.c +++ b/kernel/bpf/map_iter.c @@ -123,7 +123,8 @@ static int bpf_iter_attach_map(struct bpf_prog *prog, is_percpu = true; else if (map->map_type != BPF_MAP_TYPE_HASH && map->map_type != BPF_MAP_TYPE_LRU_HASH && - map->map_type != BPF_MAP_TYPE_ARRAY) + map->map_type != BPF_MAP_TYPE_ARRAY && + map->map_type != BPF_MAP_TYPE_RHASH) goto put_map; key_acc_size = prog->aux->max_rdonly_access; -- cgit v1.2.3 From 6905f8601298ecd2d1932a4b4849bf265201118e Mon Sep 17 00:00:00 2001 From: Mykyta Yatsenko Date: Fri, 5 Jun 2026 04:41:23 -0700 Subject: bpf: Allow special fields in resizable hashtab Add support for timers, workqueues, task work, spin locks and kptrs. Without this, users needing deferred callbacks, BPF_F_LOCK, or refcounted kernel pointers in a dynamically-sized map have no option - fixed-size htab is the only map supporting these field types. Resizable hashtab should offer the same capability. kptr semantics under in-place updates are identical to array map. Properly clean up BTF record fields on element delete and map teardown by wiring up bpf_obj_free_fields through a memory allocator destructor, matching the pattern used by htab for non-prealloc maps. Signed-off-by: Mykyta Yatsenko Link: https://lore.kernel.org/r/20260605-rhash-v7-6-5b8e05f8630d@meta.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/hashtab.c | 111 ++++++++++++++++++++++++++++++++++++++++++++++----- kernel/bpf/syscall.c | 3 ++ 2 files changed, 104 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index a149713d0953..7b9408b8320c 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -497,28 +497,26 @@ static void htab_dtor_ctx_free(void *ctx) kfree(ctx); } -static int htab_set_dtor(struct bpf_htab *htab, void (*dtor)(void *, void *)) +static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma, + void (*dtor)(void *, void *)) { - u32 key_size = htab->map.key_size; - struct bpf_mem_alloc *ma; struct htab_btf_record *hrec; int err; /* No need for dtors. */ - if (IS_ERR_OR_NULL(htab->map.record)) + if (IS_ERR_OR_NULL(map->record)) return 0; hrec = kzalloc(sizeof(*hrec), GFP_KERNEL); if (!hrec) return -ENOMEM; - hrec->key_size = key_size; - hrec->record = btf_record_dup(htab->map.record); + hrec->key_size = map->key_size; + hrec->record = btf_record_dup(map->record); if (IS_ERR(hrec->record)) { err = PTR_ERR(hrec->record); kfree(hrec); return err; } - ma = htab_is_percpu(htab) ? &htab->pcpu_ma : &htab->ma; bpf_mem_alloc_set_dtor(ma, dtor, htab_dtor_ctx_free, hrec); return 0; } @@ -535,9 +533,9 @@ static int htab_map_check_btf(struct bpf_map *map, const struct btf *btf, * populated in htab_map_alloc(), so it will always appear as NULL. */ if (htab_is_percpu(htab)) - return htab_set_dtor(htab, htab_pcpu_mem_dtor); + return bpf_ma_set_dtor(map, &htab->pcpu_ma, htab_pcpu_mem_dtor); else - return htab_set_dtor(htab, htab_mem_dtor); + return bpf_ma_set_dtor(map, &htab->ma, htab_mem_dtor); } static struct bpf_map *htab_map_alloc(union bpf_attr *attr) @@ -2752,6 +2750,7 @@ struct bpf_rhtab { struct rhashtable ht; struct bpf_mem_alloc ma; u32 elem_size; + bool freeing_internal; }; static const struct rhashtable_params rhtab_params = { @@ -2832,11 +2831,34 @@ static int rhtab_map_alloc_check(union bpf_attr *attr) return htab_map_alloc_check(attr); } +static void rhtab_check_and_free_fields(struct bpf_rhtab *rhtab, + struct rhtab_elem *elem) +{ + if (IS_ERR_OR_NULL(rhtab->map.record)) + return; + + bpf_obj_free_fields(rhtab->map.record, + rhtab_elem_value(elem, rhtab->map.key_size)); +} + +static void rhtab_mem_dtor(void *obj, void *ctx) +{ + struct htab_btf_record *hrec = ctx; + struct rhtab_elem *elem = obj; + + if (IS_ERR_OR_NULL(hrec->record)) + return; + + bpf_obj_free_fields(hrec->record, + rhtab_elem_value(elem, hrec->key_size)); +} + static void rhtab_free_elem(void *ptr, void *arg) { struct bpf_rhtab *rhtab = arg; struct rhtab_elem *elem = ptr; + bpf_map_free_internal_structs(&rhtab->map, rhtab_elem_value(elem, rhtab->map.key_size)); bpf_mem_cache_free_rcu(&rhtab->ma, elem); } @@ -2900,7 +2922,8 @@ static int rhtab_delete_elem(struct bpf_rhtab *rhtab, struct rhtab_elem *elem, v rhtab_read_elem_value(&rhtab->map, copy, elem, flags); check_and_init_map_value(&rhtab->map, copy); } - + /* Release internal structs: kptr, bpf_timer, task_work, wq */ + rhtab_check_and_free_fields(rhtab, elem); bpf_mem_cache_free_rcu(&rhtab->ma, elem); return 0; } @@ -2942,6 +2965,7 @@ static int rhtab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, void static long rhtab_map_update_existing(struct bpf_map *map, struct rhtab_elem *elem, void *value, u64 map_flags) { + struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map); void *old_val = rhtab_elem_value(elem, map->key_size); if (map_flags & BPF_NOEXIST) @@ -2951,6 +2975,17 @@ static long rhtab_map_update_existing(struct bpf_map *map, struct rhtab_elem *el copy_map_value_locked(map, old_val, value, false); else copy_map_value(map, old_val, value); + + /* + * Torn reads: a concurrent reader without BPF_F_LOCK may observe + * the value mid-copy. Callers requiring consistent reads must use + * BPF_F_LOCK, matching arraymap semantics. + * + * copy_map_value() skips special-field offsets, so old timers/ + * kptrs/etc. still sit in the slot. Cancel them after the copy + * to match arraymap's update semantics. + */ + rhtab_check_and_free_fields(rhtab, elem); return 0; } @@ -2973,6 +3008,14 @@ static long rhtab_map_update_elem(struct bpf_map *map, void *key, void *value, u if (map_flags & BPF_EXIST) return -ENOENT; + /* + * Reject new insertions while map_release_uref cleanup walks the + * table. Without this, new elements could keep triggering rehash + * and prevent the walk from terminating. + */ + if (READ_ONCE(rhtab->freeing_internal)) + return -EBUSY; + /* Check max_entries limit before inserting new element */ if (atomic_read(&rhtab->ht.nelems) >= map->max_entries) return -E2BIG; @@ -2983,6 +3026,7 @@ static long rhtab_map_update_elem(struct bpf_map *map, void *key, void *value, u memcpy(elem->data, key, map->key_size); copy_map_value(map, rhtab_elem_value(elem, map->key_size), value); + check_and_init_map_value(map, rhtab_elem_value(elem, map->key_size)); /* Prevent deadlock for NMI programs attempting to take bucket lock */ bpf_disable_instrumentation(); @@ -3015,8 +3059,54 @@ static int rhtab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) return insn - insn_buf; } +static int rhtab_map_check_btf(struct bpf_map *map, const struct btf *btf, + const struct btf_type *key_type, + const struct btf_type *value_type) +{ + struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map); + + return bpf_ma_set_dtor(map, &rhtab->ma, rhtab_mem_dtor); +} + static void rhtab_map_free_internal_structs(struct bpf_map *map) { + struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map); + struct rhashtable_iter iter; + struct rhtab_elem *elem; + + if (!bpf_map_has_internal_structs(map)) + return; + + /* + * Block new insertions. Once observed, no new growth is triggered, + * so any in-flight rehash will drain and the walker is guaranteed + * to stop returning -EAGAIN. Treat -EAGAIN as "rehash in progress, + * retry"; do not wait for the worker. + */ + WRITE_ONCE(rhtab->freeing_internal, true); + + rhashtable_walk_enter(&rhtab->ht, &iter); + rhashtable_walk_start(&iter); + + while ((elem = rhashtable_walk_next(&iter))) { + if (IS_ERR(elem)) { + if (PTR_ERR(elem) == -EAGAIN) + continue; + break; + } + + bpf_map_free_internal_structs(map, rhtab_elem_value(elem, map->key_size)); + + if (need_resched()) { /* Avoid stalls on large maps */ + rhashtable_walk_stop(&iter); + cond_resched(); + rhashtable_walk_start(&iter); + } + } + + rhashtable_walk_stop(&iter); + rhashtable_walk_exit(&iter); + WRITE_ONCE(rhtab->freeing_internal, false); } static int rhtab_map_get_next_key(struct bpf_map *map, void *key, void *next_key) @@ -3382,6 +3472,7 @@ const struct bpf_map_ops rhtab_map_ops = { .map_free = rhtab_map_free, .map_get_next_key = rhtab_map_get_next_key, .map_release_uref = rhtab_map_free_internal_structs, + .map_check_btf = rhtab_map_check_btf, .map_lookup_elem = rhtab_map_lookup_elem, .map_lookup_and_delete_elem = rhtab_map_lookup_and_delete_elem, .map_update_elem = rhtab_map_update_elem, diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 1faae184de48..31a3b70a0b5d 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1280,6 +1280,7 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token, case BPF_SPIN_LOCK: case BPF_RES_SPIN_LOCK: if (map->map_type != BPF_MAP_TYPE_HASH && + map->map_type != BPF_MAP_TYPE_RHASH && map->map_type != BPF_MAP_TYPE_ARRAY && map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && map->map_type != BPF_MAP_TYPE_SK_STORAGE && @@ -1294,6 +1295,7 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token, case BPF_WORKQUEUE: case BPF_TASK_WORK: if (map->map_type != BPF_MAP_TYPE_HASH && + map->map_type != BPF_MAP_TYPE_RHASH && map->map_type != BPF_MAP_TYPE_LRU_HASH && map->map_type != BPF_MAP_TYPE_ARRAY) { ret = -EOPNOTSUPP; @@ -1305,6 +1307,7 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token, case BPF_KPTR_PERCPU: case BPF_REFCOUNT: if (map->map_type != BPF_MAP_TYPE_HASH && + map->map_type != BPF_MAP_TYPE_RHASH && map->map_type != BPF_MAP_TYPE_PERCPU_HASH && map->map_type != BPF_MAP_TYPE_LRU_HASH && map->map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH && -- cgit v1.2.3 From 9dcbb5045fe5a00f04c99aede5a10726f8bb937b Mon Sep 17 00:00:00 2001 From: Mykyta Yatsenko Date: Fri, 5 Jun 2026 04:41:24 -0700 Subject: bpf: Optimize word-sized keys for resizable hashtable Specialize the lookup/update/delete paths for keys whose size matches sizeof(long) (4 bytes on 32-bit, 8 bytes on 64-bit). A static-const rhashtable_params lets the compiler inline a custom XOR-fold hashfn and a single-word equality cmpfn, eliminating the indirect jhash dispatch. The same hashfn and cmpfn are installed into rhashtable's stored params at rhashtable_init time, so the rehash worker, slow-path inserts, and rhashtable_next_key() all agree with the inlined fast paths. The seq_file BPF iterator uses rhashtable_walk_* and is unaffected. Signed-off-by: Mykyta Yatsenko Link: https://lore.kernel.org/r/20260605-rhash-v7-7-5b8e05f8630d@meta.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/hashtab.c | 47 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 7b9408b8320c..b4366cad3cfa 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -2763,6 +2763,31 @@ static inline void *rhtab_elem_value(struct rhtab_elem *l, u32 key_size) return l->data + round_up(key_size, 8); } +/* Specialize hash function and objcmp for long sized key */ +static __always_inline int rhtab_key_cmp_long(struct rhashtable_compare_arg *arg, + const void *ptr) +{ + const unsigned long key1 = *(const unsigned long *)arg->key; + const struct rhtab_elem *key2 = ptr; + + return key1 != *(const unsigned long *)key2->data; +} + +static __always_inline u32 rhtab_hashfn_long(const void *data, u32 len, u32 seed) +{ + u64 k = *(const unsigned long *)data; + + return (u32)(k ^ (k >> 32)) ^ seed; +} + +static const struct rhashtable_params rhtab_params_long = { + .head_offset = offsetof(struct rhtab_elem, node), + .key_offset = offsetof(struct rhtab_elem, data), + .key_len = sizeof(long), + .hashfn = rhtab_hashfn_long, + .obj_cmpfn = rhtab_key_cmp_long, +}; + static struct bpf_map *rhtab_map_alloc(union bpf_attr *attr) { struct rhashtable_params params; @@ -2788,6 +2813,11 @@ static struct bpf_map *rhtab_map_alloc(union bpf_attr *attr) params.nelem_hint = (u32)attr->map_extra; params.automatic_shrinking = true; + if (rhtab->map.key_size == sizeof(long)) { + params.hashfn = rhtab_hashfn_long; + params.obj_cmpfn = rhtab_key_cmp_long; + } + err = rhashtable_init(&rhtab->ht, ¶ms); if (err) goto free_rhtab; @@ -2878,6 +2908,9 @@ static void *rhtab_lookup_elem(struct bpf_map *map, void *key) /* Hold RCU lock in case sleepable program calls via gen_lookup */ guard(rcu)(); + if (map->key_size == sizeof(long)) + return rhashtable_lookup_likely(&rhtab->ht, key, rhtab_params_long); + return rhashtable_lookup_likely(&rhtab->ht, key, rhtab_params); } @@ -2912,7 +2945,12 @@ static int rhtab_delete_elem(struct bpf_rhtab *rhtab, struct rhtab_elem *elem, v * raw tracepoints, which we don't have in rhashtable. */ bpf_disable_instrumentation(); - err = rhashtable_remove_fast(&rhtab->ht, &elem->node, rhtab_params); + + if (rhtab->map.key_size == sizeof(long)) + err = rhashtable_remove_fast(&rhtab->ht, &elem->node, rhtab_params_long); + else + err = rhashtable_remove_fast(&rhtab->ht, &elem->node, rhtab_params); + bpf_enable_instrumentation(); if (err) @@ -3030,7 +3068,12 @@ static long rhtab_map_update_elem(struct bpf_map *map, void *key, void *value, u /* Prevent deadlock for NMI programs attempting to take bucket lock */ bpf_disable_instrumentation(); - tmp = rhashtable_lookup_get_insert_fast(&rhtab->ht, &elem->node, rhtab_params); + + if (map->key_size == sizeof(long)) + tmp = rhashtable_lookup_get_insert_fast(&rhtab->ht, &elem->node, rhtab_params_long); + else + tmp = rhashtable_lookup_get_insert_fast(&rhtab->ht, &elem->node, rhtab_params); + bpf_enable_instrumentation(); if (tmp) { -- cgit v1.2.3 From 27ffbfd14d774adfc64ae1f8f76aa6195411087a Mon Sep 17 00:00:00 2001 From: Song Chen Date: Wed, 3 Jun 2026 17:19:10 +0800 Subject: bpf: Reject registration of duplicated kfunc Search for duplicated kfunc in btf_vmlinux and btf_modules before a kernel module attempts to register a kfunc. If kfunc would shadow existing kfunc then pr_err() and reject module loading. Reviewed-by: Yonghong Song Signed-off-by: Song Chen Link: https://lore.kernel.org/r/20260603091910.7212-1-chensong_2000@126.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/btf.c | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 55aa3ba1b1e0..ef4402274786 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -8771,6 +8771,39 @@ static int btf_check_iter_kfuncs(struct btf *btf, const char *func_name, return 0; } +static int btf_check_kfunc_name(struct btf *btf, const char *func_name, u32 kind) +{ +#ifdef CONFIG_DEBUG_INFO_BTF_MODULES + struct btf_module *btf_mod, *tmp; +#endif + s32 id; + + if (!btf_is_module(btf)) + return 0; + + id = btf_find_by_name_kind(bpf_get_btf_vmlinux(), func_name, kind); + if (id >= 0) { + pr_err("kfunc %s (id: %d) is already present in vmlinux.\n", + func_name, id); + return -EINVAL; + } + +#ifdef CONFIG_DEBUG_INFO_BTF_MODULES + guard(mutex)(&btf_module_mutex); + list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) { + if (btf_mod->btf == btf) + continue; + id = btf_find_by_name_kind(btf_mod->btf, func_name, kind); + if (id >= 0) { + pr_err("kfunc %s (id: %d) is already present in module %s.\n", + func_name, id, btf_mod->module->name); + return -EINVAL; + } + } +#endif + return 0; +} + static int btf_check_kfunc_protos(struct btf *btf, u32 func_id, u32 func_flags) { const struct btf_type *func; @@ -8784,7 +8817,8 @@ static int btf_check_kfunc_protos(struct btf *btf, u32 func_id, u32 func_flags) /* sanity check kfunc name */ func_name = btf_name_by_offset(btf, func->name_off); - if (!func_name || !func_name[0]) + if (!func_name || !func_name[0] || + btf_check_kfunc_name(btf, func_name, BTF_INFO_KIND(func->info))) return -EINVAL; func = btf_type_by_id(btf, func->type); -- cgit v1.2.3 From aa496720618f1a6054f1c870bf10b4f6c99bf656 Mon Sep 17 00:00:00 2001 From: Zhao Zhang Date: Tue, 2 Jun 2026 16:43:33 +0800 Subject: bpf: Reject fragmented frames in devmap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Devmap broadcast redirects clone the packet for all but the last destination. For native XDP, that clone path copies only the linear xdp_frame data, while fragmented frames keep skb_shared_info in tailroom outside the linear area. Cloning such a frame leaves XDP_FLAGS_HAS_FRAGS set but without valid frag metadata, and the later free path can interpret uninitialized tail data as skb_shared_info, leading to an out-of-bounds access during frame return. Reject fragmented native XDP frames in dev_map_enqueue_clone(). Add the same restriction to the generic XDP clone path in dev_map_redirect_clone(). Generic XDP represents fragmented packets as nonlinear skbs, and rejecting them here keeps clone-based broadcast support aligned between native and generic XDP. Fixes: e624d4ed4aa8 ("xdp: Extend xdp_redirect_map with broadcast support") Cc: stable@kernel.org Reported-by: Yuan Tan Reported-by: Zhengchuan Liang Reported-by: Xin Liu Assisted-by: Codex:GPT-5.4 Signed-off-by: Zhao Zhang Signed-off-by: Ren Wei Reviewed-by: Emil Tsalapatis Reviewed-by: Toke Høiland-Jørgensen Link: https://lore.kernel.org/r/21c2d153dd25603d359069a02bf06779b51f6423.1780385378.git.zzhan461@ucr.edu Signed-off-by: Alexei Starovoitov --- kernel/bpf/devmap.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c index cc0a43ebab6b..5b9eac5342a9 100644 --- a/kernel/bpf/devmap.c +++ b/kernel/bpf/devmap.c @@ -581,6 +581,10 @@ static int dev_map_enqueue_clone(struct bpf_dtab_netdev *obj, { struct xdp_frame *nxdpf; + /* Frags live outside the linear frame and cannot be cloned safely. */ + if (unlikely(xdp_frame_has_frags(xdpf))) + return -EOPNOTSUPP; + nxdpf = xdpf_clone(xdpf); if (!nxdpf) return -ENOMEM; @@ -726,6 +730,9 @@ static int dev_map_redirect_clone(struct bpf_dtab_netdev *dst, struct sk_buff *nskb; int err; + if (unlikely(skb_is_nonlinear(skb))) + return -EOPNOTSUPP; + nskb = skb_clone(skb, GFP_ATOMIC); if (!nskb) return -ENOMEM; -- cgit v1.2.3 From f64c723741c911544cca4c838d7a291b06b3ad1d Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 1 Jun 2026 08:37:28 -1000 Subject: bpf: Replace scratch PTE atomically when allocating arena pages apply_range_set_cb() maps the pages for a new arena allocation and returned -EBUSY when the target PTE was already populated. Kernel-fault recovery leaves the per-arena scratch page in unallocated arena PTEs, so a later bpf_arena_alloc_pages() over such a page hits that -EBUSY, and every subsequent allocation of it fails the same way. Allocation must install the real page over scratch instead. Overwriting the scratch PTE in place is a valid->valid change, which arm64 forbids without break-before-make. Route through an invalid entry instead: ptep_try_set() fills only a none slot, so the PTE goes scratch->none->page. On finding scratch, clear it and flush_tlb_before_set() before retrying. The new flush_tlb_before_set() is a no-op except on arches like arm64 that need the break-before-make TLB invalidate. The loop also copes with a concurrent fault re-scratching the slot. Arches without ptep_try_set() never install the scratch page, so keep the must-be-empty check and set_pte_at() for them. Fixes: dc11a4dba246 ("bpf: Recover arena kernel faults with scratch page") Signed-off-by: Tejun Heo Cc: Alexei Starovoitov Cc: David Hildenbrand Acked-by: Kumar Kartikeya Dwivedi Link: https://lore.kernel.org/r/20260601183728.1800490-1-tj@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/bpf/arena.c | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 9b2dea229b38..af49c154473d 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -144,6 +144,7 @@ static long compute_pgoff(struct bpf_arena *arena, long uaddr) struct apply_range_data { struct page **pages; + struct page *scratch_page; int i; }; @@ -156,19 +157,44 @@ static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data) { struct apply_range_data *d = data; struct page *page; + pte_t pteval; if (!data) return 0; - /* sanity check */ - if (unlikely(!pte_none(ptep_get(pte)))) - return -EBUSY; page = d->pages[d->i]; /* paranoia, similar to vmap_pages_pte_range() */ if (WARN_ON_ONCE(!pfn_valid(page_to_pfn(page)))) return -EINVAL; - set_pte_at(&init_mm, addr, pte, mk_pte(page, PAGE_KERNEL)); + pteval = mk_pte(page, PAGE_KERNEL); +#ifdef ptep_try_set + /* + * Kernel-fault recovery may have installed the scratch page here, and + * some architectures (arm64) prohibit valid->valid PTE transitions. + * Install atomically into a none slot. If scratch is present, clear it + * and flush_tlb_before_set() (break-before-make) before retrying. + */ + while (!ptep_try_set(pte, pteval)) { + pte_t old = ptep_get(pte); + + if (pte_none(old)) + continue; + if (WARN_ON_ONCE(pte_page(old) != d->scratch_page)) + return -EBUSY; + ptep_get_and_clear(&init_mm, addr, pte); + flush_tlb_before_set(addr); + } +#else + /* + * Without ptep_try_set() there is no atomic installer, but such arches + * also do not wire up bpf_arena_handle_page_fault(), so no scratch page + * is ever installed and the slot is always none here. + */ + if (unlikely(!pte_none(ptep_get(pte)))) + return -EBUSY; + set_pte_at(&init_mm, addr, pte, pteval); +#endif d->i++; return 0; } @@ -480,7 +506,8 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) if (ret) goto out_sigsegv_memcg; - struct apply_range_data data = { .pages = &page, .i = 0 }; + struct apply_range_data data = { .pages = &page, .i = 0, + .scratch_page = arena->scratch_page }; /* Account into memcg of the process that created bpf_arena */ ret = bpf_map_alloc_pages(map, NUMA_NO_NODE, 1, &page); if (ret) { @@ -670,6 +697,7 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt return 0; } data.pages = pages; + data.scratch_page = arena->scratch_page; if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) goto out_free_pages; -- cgit v1.2.3 From a3863aa4f55e5c17f32e1fd64a0a64adf2af16d9 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 5 Jun 2026 13:20:52 -0700 Subject: bpf: Fix dead error check on acquire_reference() in check_kfunc_call acquire_reference() returns a signed int that may be a negative errno but was converted to unsigned, which makes the subsequent error check deadcode. Fix it by declaring 'id' as int so the error path is taken correctly. Fixes: 308c7a0ae885 ("bpf: Refactor object relationship tracking and fix dynptr UAF bug") Acked-by: Eduard Zingerman Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260605202056.1780352-2-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 7d27ba396d32..a741bf447931 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -12817,9 +12817,10 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_kfunc_call_arg_meta meta; struct bpf_insn_aux_data *insn_aux; int err, insn_idx = *insn_idx_p; - u32 i, nargs, ptr_type_id, id; const struct btf_param *args; + u32 i, nargs, ptr_type_id; struct btf *desc_btf; + int id; /* skip for now, but return error when we find this in fixup_kfunc_call */ if (!insn->imm) -- cgit v1.2.3 From 73d475dc6c13177fce0d9d892bff33299c8ad56a Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 5 Jun 2026 13:20:53 -0700 Subject: bpf: Check acquire_reference() error for "__ref" struct_ops arguments When acquiring references for struct_ops program arguments tagged with "__ref", the return value of acquire_reference() was stored directly into u32 ctx_arg_info[i].ref_id without checking for failure. acquire_reference() returns -ENOMEM when acquire_reference_state() fails to allocate, so the error was silently stored as a ref_id instead of aborting verification. Fix it by checking the return. Fixes: a687df2008f6 ("bpf: Support getting referenced kptr from struct_ops argument") Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260605202056.1780352-3-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a741bf447931..3b874bbbaac0 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -18363,9 +18363,13 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) /* Acquire references for struct_ops program arguments tagged with "__ref" */ if (!subprog && env->prog->type == BPF_PROG_TYPE_STRUCT_OPS) { - for (i = 0; i < aux->ctx_arg_info_size; i++) - aux->ctx_arg_info[i].ref_id = aux->ctx_arg_info[i].refcounted ? - acquire_reference(env, 0, 0) : 0; + for (i = 0; i < aux->ctx_arg_info_size; i++) { + ret = aux->ctx_arg_info[i].refcounted ? acquire_reference(env, 0, 0) : 0; + if (ret < 0) + goto out; + + aux->ctx_arg_info[i].ref_id = ret; + } } ret = do_check(env); -- cgit v1.2.3 From 41025f441fe6addd93d2c333a3a184331e8ef6cf Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 5 Jun 2026 13:20:54 -0700 Subject: bpf: Compare parent_id in refsafe() for REF_TYPE_PTR refsafe() compared each reference's id and type but not its parent_id, so two states whose PTR references differ only in the parent object they were derived from could be wrongly treated as equivalent and pruned. Fix it by checking parent_id too. Fixes: 308c7a0ae885 ("bpf: Refactor object relationship tracking and fix dynptr UAF bug") Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260605202056.1780352-4-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/states.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 5945956a7573..06d9ae24f006 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -890,6 +890,9 @@ static bool refsafe(struct bpf_verifier_state *old, struct bpf_verifier_state *c return false; switch (old->refs[i].type) { case REF_TYPE_PTR: + if (!check_ids(old->refs[i].parent_id, cur->refs[i].parent_id, idmap)) + return false; + break; case REF_TYPE_IRQ: break; case REF_TYPE_LOCK: -- cgit v1.2.3 From ac7f6c9da6b6b46bba34a45c51603c81e7d42eb2 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Fri, 5 Jun 2026 13:20:55 -0700 Subject: bpf: Remove WARN_ON_ONCE in check_ids() check_ids() warned when it ran out of idmap slots, assuming this was impossible because the slots are bounded by the number of registers and stack slots. That assumption no longer holds: referenced dynptrs acquire an intermediate reference that lives in refs[] but is not backed by any register or stack slot [0], so a program can accumulate more reference ids than the idmap can hold and exhaust it. Exhaustion is fine for verification correctness. check_ids() already returns false, which makes the states compare as not equivalent and prevents unsound pruning. The only effect of the WARN_ON_ONCE() is log noise, or a panic under panic_on_warn. Drop the warning and keep returning false. [0] 308c7a0ae885 ("bpf: Refactor object relationship tracking and fix dynptr UAF bug") Signed-off-by: Amery Hung Link: https://lore.kernel.org/r/20260605202056.1780352-5-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/states.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 06d9ae24f006..32f346ce3ffc 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -343,8 +343,12 @@ static bool check_ids(u32 old_id, u32 cur_id, struct bpf_idmap *idmap) return true; } - /* We ran out of idmap slots, which should be impossible */ - WARN_ON_ONCE(1); + /* + * idmap slots are bounded by the number of registers and stack slots. + * Since referenced dynptrs acquire intermediate references that do + * not live in either, so the map can be exhausted. Since it is unlikely, + * fail the verification by treating the states as not equivalent. + */ return false; } -- cgit v1.2.3 From 4a7910ee060d8ce55612f5b3cc267f3a265a3cec Mon Sep 17 00:00:00 2001 From: Kaitao Cheng Date: Fri, 5 Jun 2026 17:41:43 +0800 Subject: bpf: Clear rb node linkage when freeing bpf_rb_root bpf_rb_root_free() detaches the root by copying the current rb_root_cached and then replacing the live root with RB_ROOT_CACHED. It then walks the copied root and drops each object contained in the tree. This leaves the rb node state intact while dropping the object. If the object is refcounted and survives the drop, its bpf_rb_node_kern still contains an owner pointer to the freed root and stale rb tree linkage. If a later bpf_rb_root allocation reuses the same address, bpf_rbtree_remove() can incorrectly pass the owner check and call rb_erase_cached() on a node whose rb pointers belong to the old tree. Mirror the list draining behavior by marking nodes as busy while the root is being detached, then clear the rb node and release the owner before dropping the containing object. This makes surviving nodes unowned and safe to reject from remove or accept for a later add. Fixes: 9c395c1b99bd ("bpf: Add basic bpf_rb_{root,node} support") Signed-off-by: Kaitao Cheng Acked-by: Yonghong Song Link: https://lore.kernel.org/r/20260605094143.5509-1-kaitao.cheng@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/helpers.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 03004e4451f5..8ba2b8965caf 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -2306,6 +2306,7 @@ void bpf_rb_root_free(const struct btf_field *field, void *rb_root, struct bpf_spin_lock *spin_lock) { struct rb_root_cached orig_root, *root = rb_root; + struct bpf_rb_node_kern *node; struct rb_node *pos, *n; void *obj; @@ -2314,14 +2315,20 @@ void bpf_rb_root_free(const struct btf_field *field, void *rb_root, __bpf_spin_lock_irqsave(spin_lock); orig_root = *root; + bpf_rbtree_postorder_for_each_entry_safe(pos, n, &orig_root.rb_root) { + node = rb_entry(pos, struct bpf_rb_node_kern, rb_node); + WRITE_ONCE(node->owner, BPF_PTR_POISON); + } *root = RB_ROOT_CACHED; __bpf_spin_unlock_irqrestore(spin_lock); bpf_rbtree_postorder_for_each_entry_safe(pos, n, &orig_root.rb_root) { obj = pos; obj -= field->graph_root.node_offset; - - + node = rb_entry(pos, struct bpf_rb_node_kern, rb_node); + RB_CLEAR_NODE(pos); + /* Ensure __bpf_rbtree_add() sees the node as unlinked. */ + smp_store_release(&node->owner, NULL); __bpf_obj_drop_impl(obj, field->graph_root.value_rec, false); } } -- cgit v1.2.3 From e2a49fdb1beed150125b4104c90eb2a96ec7f63a Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Fri, 5 Jun 2026 23:52:47 +0800 Subject: bpf: Check tail zero of bpf_map_info Since there're 4 bytes padding at the end of struct bpf_map_info, they won't be checked by bpf_check_uarg_tail_zero(). pahole -C bpf_map_info ./vmlinux struct bpf_map_info { ... __u64 hash __attribute__((__aligned__(8))); /* 88 8 */ __u32 hash_size; /* 96 4 */ /* size: 104, cachelines: 2, members: 18 */ /* padding: 4 */ /* forced alignments: 1 */ /* last cacheline: 40 bytes */ } __attribute__((__aligned__(8))); If a future kernel extension adds a new 4-byte field, older userspace programs allocating this structure on the stack might inadvertently pass uninitialized stack garbage into the new field, permanently breaking backward compatibility. -- sashiko [1] Fix it by changing sizeof(info) to offsetofend(struct bpf_map_info, hash_size). And, add "__u32 :32" to the tail of struct bpf_map_info. [1] https://lore.kernel.org/bpf/20260513224823.6494FC19425@smtp.kernel.org/ Fixes: ea2e6467ac36 ("bpf: Return hashes of maps in BPF_OBJ_GET_INFO_BY_FD") Acked-by: Mykyta Yatsenko Signed-off-by: Leon Hwang Link: https://lore.kernel.org/r/20260605155249.20772-2-leon.hwang@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/syscall.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 31a3b70a0b5d..89f020a44fc9 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -5406,10 +5406,11 @@ static int bpf_map_get_info_by_fd(struct file *file, { struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info); struct bpf_map_info info; - u32 info_len = attr->info.info_len; + u32 info_len = attr->info.info_len, len; int err; - err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len); + len = offsetofend(struct bpf_map_info, hash_size); + err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), len, info_len); if (err) return err; info_len = min_t(u32, sizeof(info), info_len); -- cgit v1.2.3 From 786be2b05980a5828e67fc564ad7517e2adbe9bd Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Fri, 5 Jun 2026 23:52:48 +0800 Subject: bpf: Check tail zero of bpf_prog_info Since there're 4 bytes padding at the end of struct bpf_prog_info, they won't be checked by bpf_check_uarg_tail_zero(). pahole -C bpf_prog_info ./vmlinux struct bpf_prog_info { ... __u32 attach_btf_obj_id; /* 220 4 */ __u32 attach_btf_id; /* 224 4 */ /* size: 232, cachelines: 4, members: 38 */ /* sum members: 224 */ /* sum bitfield members: 1 bits, bit holes: 1, sum bit holes: 31 bits */ /* padding: 4 */ /* forced alignments: 9 */ /* last cacheline: 40 bytes */ } __attribute__((__aligned__(8))); If a future kernel extension adds a new 4-byte field, older userspace programs allocating this structure on the stack might inadvertently pass uninitialized stack garbage into the new field, permanently breaking backward compatibility. -- sashiko [1] Fix it by changing sizeof(info) to offsetofend(struct bpf_prog_info, attach_btf_id). And, add "__u32 :32" to the tail of struct bpf_prog_info. [1] https://lore.kernel.org/bpf/20260513224823.6494FC19425@smtp.kernel.org/ Fixes: aba64c7da983 ("bpf: Add verified_insns to bpf_prog_info and fdinfo") Acked-by: Mykyta Yatsenko Signed-off-by: Leon Hwang Link: https://lore.kernel.org/r/20260605155249.20772-3-leon.hwang@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/syscall.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 89f020a44fc9..c5d4ae957e87 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -5121,10 +5121,11 @@ static int bpf_prog_get_info_by_fd(struct file *file, u32 info_len = attr->info.info_len; struct bpf_prog_kstats stats; char __user *uinsns; - u32 ulen; + u32 ulen, len; int err; - err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len); + len = offsetofend(struct bpf_prog_info, attach_btf_id); + err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), len, info_len); if (err) return err; info_len = min_t(u32, sizeof(info), info_len); -- cgit v1.2.3 From a66e3b5bacf38d6ab29fa05a9754f7a114485605 Mon Sep 17 00:00:00 2001 From: Dawei Feng Date: Wed, 3 Jun 2026 18:53:15 +0800 Subject: bpf: NUL-terminate replaced sysctl value When writing to sysctls, proc_sys_call_handler() guarantees that the buffer passed to proc handlers is NUL-terminated. If bpf_sysctl_set_new_value() replaces the pending sysctl value, it can hand a replacement buffer directly to proc handlers. However, the helper currently copies only buf_len bytes into that buffer without appending a NUL terminator, leaving downstream parsers vulnerable to out-of-bounds access. Fix this by appending a '\0' after the replaced value to restore the expected sysctl semantics. Since the helper already rejects buf_len greater than PAGE_SIZE - 1, there is always room for the extra byte. Reproduced in a QEMU x86_64 guest booted with KASAN while exercising the sysctl replacement path with a cgroup/sysctl BPF program. The reproducer targets `/proc/sys/net/core/flow_limit_cpu_bitmap`, fills the original user write buffer with non-zero bytes, and overrides the sysctl value so the replacement buffer lacks a terminating NUL. Under that setup, the pre-fix kernel reported: BUG: KASAN: slab-out-of-bounds in strnchrnul+0x72/0x90 Read of size 1 at addr ffff88800de57000 by task repro_patch3/66 CPU: 0 UID: 0 PID: 66 Comm: repro_patch3 Not tainted 7.1.0-rc3-00269-g8370ca1f87cc #6 PREEMPT(lazy) Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.15.0-1 04/01/2014 Call Trace: dump_stack_lvl+0x68/0xa0 print_report+0xcb/0x5e0 ? __virt_addr_valid+0x21d/0x3f0 ? strnchrnul+0x72/0x90 ? strnchrnul+0x72/0x90 kasan_report+0xca/0x100 ? strnchrnul+0x72/0x90 strnchrnul+0x72/0x90 bitmap_parse+0x37/0x2e0 flow_limit_cpu_sysctl+0xc6/0x840 ? __pfx_flow_limit_cpu_sysctl+0x10/0x10 ? __kvmalloc_node_noprof+0x5ba/0x870 proc_sys_call_handler+0x31d/0x480 ? __pfx_proc_sys_call_handler+0x10/0x10 ? selinux_file_permission+0x39f/0x500 ? lock_is_held_type+0x9e/0x120 vfs_write+0x98e/0x1000 ... The buggy address is located 0 bytes to the right of allocated 4096-byte region [ffff88800de56000, ffff88800de57000) With this fix applied, rerunning the same sysctl-targeted path yields no corresponding KASAN reports. Signed-off-by: Zilin Guan Signed-off-by: Dawei Feng Acked-by: Yonghong Song Link: https://lore.kernel.org/r/20260603105317.944304-2-dawei.feng@seu.edu.cn Signed-off-by: Alexei Starovoitov --- kernel/bpf/cgroup.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 2c2bdaa86aa7..3fe87b9d368e 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -2343,6 +2343,7 @@ BPF_CALL_3(bpf_sysctl_set_new_value, struct bpf_sysctl_kern *, ctx, return -E2BIG; memcpy(ctx->new_val, buf, buf_len); + ((char *)ctx->new_val)[buf_len] = '\0'; ctx->new_len = buf_len; ctx->new_updated = 1; -- cgit v1.2.3 From 4c21b5927d4364bfe7365f2700da5fea0ed0d004 Mon Sep 17 00:00:00 2001 From: Dawei Feng Date: Wed, 3 Jun 2026 18:53:16 +0800 Subject: bpf: use kvfree() for replaced sysctl write buffer proc_sys_call_handler() allocates its temporary sysctl buffer with kvzalloc() and passes it to __cgroup_bpf_run_filter_sysctl(). Since kvzalloc() may fall back to vmalloc() for large allocations, freeing that buffer with kfree() is wrong and can corrupt memory. Use kvfree() to safely handle both kmalloc and kvzalloc()/vmalloc allocations. The bug was first flagged by an experimental analysis tool we are developing for kernel memory-management bugs while analyzing v6.13-rc1. The tool is still under development and is not yet publicly available. Manual inspection confirms that the bug is still present in v7.1-rc5. Reproduced the bug based on v7.1-rc4 in a QEMU x86_64 guest booted with KASAN and CONFIG_FAILSLAB enabled. To exercise the replacement path, the test tree also included the accompanying fix for the stale ret == 1 check in __cgroup_bpf_run_filter_sysctl(). The reproducer confines failslab injections to the proc_sys_call_handler() range, uses stacktrace-depth=32, and injects fail-nth=1 while writing 8191 bytes to /proc/sys/kernel/domainname from a task in the target cgroup. Under that setup, fail-nth=1 triggered the fault: BUG: unable to handle page fault for address: ffffeb0200024d48 #PF: supervisor read access in kernel mode #PF: error_code(0x0000) - not-present page PGD 0 P4D 0 Oops: Oops: 0000 SMP KASAN NOPTI CPU: 2 UID: 0 PID: 209 Comm: repro_proc_sys_ Not tainted 7.1.0-rc4-00686-g97625979a5d4 PREEMPT(lazy) Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.15.0-1 04/01/2014 RIP: 0010:kfree+0x6e/0x510 ... Call Trace: ? __cgroup_bpf_run_filter_sysctl+0x626/0xc30 __cgroup_bpf_run_filter_sysctl+0x74d/0xc30 ? __pfx___cgroup_bpf_run_filter_sysctl+0x10/0x10 ? srso_return_thunk+0x5/0x5f ? __kvmalloc_node_noprof+0x345/0x870 ? proc_sys_call_handler+0x250/0x480 ? srso_return_thunk+0x5/0x5f proc_sys_call_handler+0x3a2/0x480 ? __pfx_proc_sys_call_handler+0x10/0x10 ? srso_return_thunk+0x5/0x5f ? selinux_file_permission+0x39f/0x500 ? srso_return_thunk+0x5/0x5f ? lock_is_held_type+0x9e/0x120 vfs_write+0x98e/0x1000 ... With this fix applied on top of the same test setup, rerunning the reproducer with fail-nth=1 yields no corresponding Oops reports. Fixes: 4508943794ef ("proc: use kvzalloc for our kernel buffer") Cc: stable@vger.kernel.org Reviewed-by: Emil Tsalapatis Reviewed-by: Jiayuan Chen Acked-by: Yonghong Song Signed-off-by: Zilin Guan Signed-off-by: Dawei Feng Link: https://lore.kernel.org/r/20260603105317.944304-3-dawei.feng@seu.edu.cn Signed-off-by: Alexei Starovoitov --- kernel/bpf/cgroup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 3fe87b9d368e..4bf0ec94e719 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -1937,7 +1937,7 @@ int __cgroup_bpf_run_filter_sysctl(struct ctl_table_header *head, kfree(ctx.cur_val); if (ret == 1 && ctx.new_updated) { - kfree(*buf); + kvfree(*buf); *buf = ctx.new_val; *pcount = ctx.new_len; } else { -- cgit v1.2.3 From 2566c3b24219c5b30e35205cba029ff34ff7c78b Mon Sep 17 00:00:00 2001 From: Dawei Feng Date: Wed, 3 Jun 2026 18:53:17 +0800 Subject: bpf: Restore sysctl new-value from 1 to 0 Commit 4e63acdff864 ("bpf: Introduce bpf_sysctl_{get,set}_new_value helpers") changed the success return value to 0, but failed to update the corresponding check in __cgroup_bpf_run_filter_sysctl(). Since bpf_prog_run_array_cg() now returns 0 on success, the legacy ret == 1 condition is never satisfied. As a result, the modified value is ignored, and bpf_sysctl_set_new_value() fails to replace the write buffer. Fix this by checking for a return value of 0 instead, so cgroup/sysctl programs can correctly replace the pending sysctl buffer. This bug was discovered during a manual code review. Tested via a cgroup/sysctl BPF reproducer overriding writes to a target sysctl. Pre-fix, bpf_sysctl_set_new_value("foo") was silently ignored: the write returned 8192 and the value remained "600". Post-fix, the BPF replacement buffer properly propagates: the write returns 3 and the value updates to "foo". Fixes: f10d05966196 ("bpf: Make BPF_PROG_RUN_ARRAY return -err instead of allow boolean") Cc: stable@vger.kernel.org Acked-by: Yonghong Song Signed-off-by: Zilin Guan Signed-off-by: Dawei Feng Reviewed-by: Jiayuan Chen Acked-by: Xu Kuohai Link: https://lore.kernel.org/r/20260603105317.944304-4-dawei.feng@seu.edu.cn Signed-off-by: Alexei Starovoitov --- kernel/bpf/cgroup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 4bf0ec94e719..35d1f1428ef3 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -1936,7 +1936,7 @@ int __cgroup_bpf_run_filter_sysctl(struct ctl_table_header *head, kfree(ctx.cur_val); - if (ret == 1 && ctx.new_updated) { + if (!ret && ctx.new_updated) { kvfree(*buf); *buf = ctx.new_val; *pcount = ctx.new_len; -- cgit v1.2.3 From b1f7f67b74c2e29e9c83f7952290a3c7f156bf9a Mon Sep 17 00:00:00 2001 From: Xu Kuohai Date: Fri, 5 Jun 2026 14:02:42 +0000 Subject: bpf: Add validation for bpf_set_retval argument The bpf_set_retval() helper is used by cgroup BPF programs to set the return value of the target hook. The argument type for this helper is ARG_ANYTHING. This allows setting a positive value, which no cgroup hook expects and can cause issues, such as: - BPF_LSM_CGROUP: a positive value from bpf_lsm_socket_create bypasses the err < 0 check in __sock_create(), leaving the socket object unallocated. The positive return value is then propagated to the syscall entry __sys_socket(), which also bypasses the IS_ERR() guard and ultimately causes a NULL pointer dereference. - BPF_CGROUP_DEVICE: a positive value can be returned through cgroup device bpf prog -> devcgroup_check_permission() -> bdev_permission() -> bdev_file_open_by_dev(), where ERR_PTR(positive) produces a pointer that IS_ERR() does not catch, leading to a wild pointer dereference. - BPF_CGROUP_SOCK: a positive value can be returned through cgroup sock bpf prog -> __cgroup_bpf_run_filter_sk() -> inet_create() -> __sock_create(), where inet_create() frees the newly allocated sk via sk_common_release() and sets sock->sk = NULL on the non-zero return, but __sock_create() only checks err < 0 for cleanup, so a positive retval bypasses cleanup and returns a socket with NULL sk to userspace, triggering a NULL pointer dereference on subsequent socket operations. - BPF_CGROUP_SYSCTL: a positive value can be returned through the cgroup bpf prog -> __cgroup_bpf_run_filter_sysctl() -> proc_sys_call_handler(), where a non-zero return bypasses the normal sysctl proc_handler and is returned directly to userspace as return value of read() or write() syscall. So add validation for the argument of the bpf_set_retval() helper. For BPF_LSM_CGROUP, enforce the LSM hook specific range returned by bpf_lsm_get_retval_range(). For all other cgroup program types, restrict the argument to [-MAX_ERRNO, 0], which matches the kernel convention of 0 for success and negative errno for error. BPF_CGROUP_GETSOCKOPT is an exception, since valid getsockopt implementations may return positive values, as allowed by commit c4dcfdd406aa ("bpf: Move getsockopt retval to struct bpf_cg_run_ctx"). Also refine the return value range of bpf_get_retval() so that values returned by bpf_get_retval() can be passed directly to bpf_set_retval() without extra manual bounds checking. Fixes: b44123b4a3dc ("bpf: Add cgroup helpers bpf_{get,set}_retval to get/set syscall return value") Fixes: 69fd337a975c ("bpf: per-cgroup lsm flavor") Reported-by: Quan Sun <2022090917019@std.uestc.edu.cn> Closes: https://lore.kernel.org/all/567d3206-74a5-44e5-99c6-779c425f399e@std.uestc.edu.cn Signed-off-by: Xu Kuohai Link: https://lore.kernel.org/r/20260605140243.664590-3-xukuohai@huaweicloud.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 3b874bbbaac0..935595138aa0 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -9770,7 +9770,9 @@ static int do_refine_retval_range(struct bpf_verifier_env *env, int func_id, struct bpf_call_arg_meta *meta) { + struct bpf_retval_range range; struct bpf_reg_state *ret_reg = ®s[BPF_REG_0]; + enum bpf_prog_type prog_type = resolve_prog_type(env->prog); if (ret_type != RET_INTEGER) return 0; @@ -9790,6 +9792,29 @@ static int do_refine_retval_range(struct bpf_verifier_env *env, reg_set_urange32(ret_reg, 0, nr_cpu_ids - 1); reg_bounds_sync(ret_reg); break; + case BPF_FUNC_get_retval: + /* + * bpf_get_retval may see arbitrary value passed by bpf_prog_run_array_cg for + * CGROUP_GETSOCKOPT type. + */ + if (prog_type == BPF_PROG_TYPE_CGROUP_SOCKOPT && + env->prog->expected_attach_type == BPF_CGROUP_GETSOCKOPT) + break; + + if (prog_type == BPF_PROG_TYPE_LSM && + env->prog->expected_attach_type == BPF_LSM_CGROUP) { + if (!env->prog->aux->attach_func_proto->type) + break; + bpf_lsm_get_retval_range(env->prog, &range); + } else { + range.minval = -MAX_ERRNO; + range.maxval = 0; + } + + reg_set_srange64(ret_reg, range.minval, range.maxval); + reg_set_srange32(ret_reg, range.minval, range.maxval); + reg_bounds_sync(ret_reg); + break; } return reg_bounds_sanity_check(env, ret_reg, "retval"); @@ -10259,6 +10284,24 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn } break; case BPF_FUNC_set_retval: + { + struct bpf_retval_range range = { + .minval = -MAX_ERRNO, + .maxval = 0, + .return_32bit = true + }; + struct bpf_reg_state *r1 = ®s[BPF_REG_1]; + + if (r1->type != SCALAR_VALUE) { + verbose(env, "R1 is not a scalar\n"); + return -EINVAL; + } + + /* CGROUP_GETSOCKOPT is allowed to return arbitrary value */ + if (prog_type == BPF_PROG_TYPE_CGROUP_SOCKOPT && + env->prog->expected_attach_type == BPF_CGROUP_GETSOCKOPT) + break; + if (prog_type == BPF_PROG_TYPE_LSM && env->prog->expected_attach_type == BPF_LSM_CGROUP) { if (!env->prog->aux->attach_func_proto->type) { @@ -10268,8 +10311,20 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn verbose(env, "BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); return -EINVAL; } + bpf_lsm_get_retval_range(env->prog, &range); } + + err = mark_chain_precision(env, BPF_REG_1); + if (err) + return err; + + if (!retval_range_within(range, r1)) { + verbose_invalid_scalar(env, r1, range, "At bpf_set_retval", "R1"); + return -EINVAL; + } + break; + } case BPF_FUNC_dynptr_write: { enum bpf_dynptr_type dynptr_type = meta.dynptr.type; -- cgit v1.2.3 From 63a673e8a4112af267106264f50584947786845a Mon Sep 17 00:00:00 2001 From: KP Singh Date: Fri, 5 Jun 2026 23:35:17 +0200 Subject: bpf: Expose signature verdict via bpf_prog_aux BPF_PROG_LOAD verifies the loader signature but does not record the outcome on the BPF program. [BPF] LSMs and audit can read attr->signature and attr->keyring_id to infer "was this signed, and if so, against which keyring". Add prog->aux->sig (verdict + keyring_{type,serial}), populated by bpf_prog_load before the LSM hook. keyring_type classifies the keyring the load referenced (builtin, secondary, platform or user), while keyring_serial records the serial of the keyring the signature was actually validated against. System keyrings carry a pseudo key pointer with no user-visible serial and are reported as 0, as are unsigned loads. Failed verifications reject the load before the hook runs, so it observes only either UNSIGNED or VERIFIED. Signed-off-by: KP Singh Co-developed-by: Daniel Borkmann Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260605213518.544262-1-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- kernel/bpf/syscall.c | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index c5d4ae957e87..5fcfc32c7cb4 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2871,8 +2871,22 @@ static bool is_perfmon_prog_type(enum bpf_prog_type prog_type) } } +static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id) +{ + switch (keyring_id) { + case 0: + return BPF_SIG_KEYRING_BUILTIN; + case (s32)(unsigned long)VERIFY_USE_SECONDARY_KEYRING: + return BPF_SIG_KEYRING_SECONDARY; + case (s32)(unsigned long)VERIFY_USE_PLATFORM_KEYRING: + return BPF_SIG_KEYRING_PLATFORM; + default: + return BPF_SIG_KEYRING_USER; + } +} + static int bpf_prog_verify_signature(struct bpf_prog *prog, union bpf_attr *attr, - bool is_kernel) + bool is_kernel, s32 *keyring_serial) { bpfptr_t usig = make_bpfptr(attr->signature, is_kernel); struct bpf_dynptr_kern sig_ptr, insns_ptr; @@ -2908,7 +2922,8 @@ static int bpf_prog_verify_signature(struct bpf_prog *prog, union bpf_attr *attr err = bpf_verify_pkcs7_signature((struct bpf_dynptr *)&insns_ptr, (struct bpf_dynptr *)&sig_ptr, key); - + if (!err) + *keyring_serial = bpf_key_serial(key); bpf_key_put(key); kvfree(sig); return err; @@ -3095,13 +3110,17 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at /* eBPF programs must be GPL compatible to use GPL-ed functions */ prog->gpl_compatible = license_is_gpl_compatible(license) ? 1 : 0; - if (attr->signature) { - err = bpf_prog_verify_signature(prog, attr, uattr.is_kernel); + err = bpf_prog_verify_signature(prog, attr, uattr.is_kernel, + &prog->aux->sig.keyring_serial); if (err) goto free_prog; + prog->aux->sig.keyring_type = bpf_classify_keyring(attr->keyring_id); + prog->aux->sig.verdict = BPF_SIG_VERIFIED; + } else { + prog->aux->sig.keyring_type = BPF_SIG_KEYRING_NONE; + prog->aux->sig.verdict = BPF_SIG_UNSIGNED; } - prog->orig_prog = NULL; prog->jited = 0; -- cgit v1.2.3 From 9722955b54307e9070994f2382ec06af3d7405e0 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 2 Jun 2026 09:40:12 +0200 Subject: bpf: Add simple xattr support to bpffs Add support for extended attributes on bpffs inodes so that user space and BPF LSM programs can attach metadata, for example, a content hash or a security label - to a pinned object or directory. BPF LSM or user space tooling can then uniformly look at this (e.g. security.bpf.*) in similar way to other fs'es. The store is in-memory and non-persistent: it lives only for the lifetime of the mount, like everything else in bpffs. The modelling is similar to tmpfs. bpffs serves the trusted.* and security.* namespaces; user.* is left unsupported. As bpffs is FS_USERNS_MOUNT, security.* is reachable by the unprivileged mounter in a user namespace, and thus we are using the simple_xattr_set_limited infra there (trusted.* needs global CAP_SYS_ADMIN). bpf_fill_super() is open-coded instead of using simple_fill_super(), because the root inode must now be allocated through bpf_fs_alloc_inode() i.e. carry the bpf_fs_inode wrapper and come from the right cache - which requires s_op (and s_xattr) to be installed before the first inode is created. While at it, also harden s_iflags with SB_I_NOEXEC and SB_I_NODEV. bpf_fs_listxattr() is only reachable through the filesystem via i_op->listxattr, so the BPF token inode is left untouched. Name-based fsetxattr()/fgetxattr() on a token fd still work since the get/set handlers are installed at the superblock. For security.* namespace, we use simple_xattr_set_limited() but there was no simple_xattr_add_limited() API yet which was needed in bpf_fs_initxattrs() to avoid underflows in the accounting. The symlink target is freed in bpf_free_inode() rather than in bpf_destroy_inode() so that it is released only after an RCU grace period, as an RCU path walk following the symlink may still dereference inode->i_link in security_inode_follow_link(). Lastly, the bpf_symlink() allocated the symlink target is switched to GFP_KERNEL_ACCOUNT, so the string is charged to the caller's memcg. Signed-off-by: Daniel Borkmann Link: https://patch.msgid.link/20260602074012.416289-1-daniel@iogearbox.net Cc: Christian Brauner Signed-off-by: Christian Brauner (Amutable) --- kernel/bpf/inode.c | 256 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 237 insertions(+), 19 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c index 25c06a011825..c3f79b5a2f8c 100644 --- a/kernel/bpf/inode.c +++ b/kernel/bpf/inode.c @@ -21,6 +21,9 @@ #include #include #include +#include +#include + #include "preload/bpf_preload.h" enum bpf_type { @@ -30,6 +33,23 @@ enum bpf_type { BPF_TYPE_LINK, }; +struct bpf_fs_inode { + struct list_head xattrs; + struct simple_xattr_limits xlimits; + struct inode vfs_inode; +}; + +static inline struct bpf_fs_inode *BPF_FS_I(struct inode *inode) +{ + return container_of(inode, struct bpf_fs_inode, vfs_inode); +} + +static struct kmem_cache *bpf_fs_inode_cachep __ro_after_init; + +static int bpf_fs_initxattrs(struct inode *inode, + const struct xattr *xattr_array, void *fs_info); +static ssize_t bpf_fs_listxattr(struct dentry *dentry, char *buf, size_t size); + static void *bpf_any_get(void *raw, enum bpf_type type) { switch (type) { @@ -94,10 +114,17 @@ static void *bpf_fd_probe_obj(u32 ufd, enum bpf_type *type) } static const struct inode_operations bpf_dir_iops; +static const struct inode_operations bpf_symlink_iops; -static const struct inode_operations bpf_prog_iops = { }; -static const struct inode_operations bpf_map_iops = { }; -static const struct inode_operations bpf_link_iops = { }; +static const struct inode_operations bpf_prog_iops = { + .listxattr = bpf_fs_listxattr, +}; +static const struct inode_operations bpf_map_iops = { + .listxattr = bpf_fs_listxattr, +}; +static const struct inode_operations bpf_link_iops = { + .listxattr = bpf_fs_listxattr, +}; struct inode *bpf_get_inode(struct super_block *sb, const struct inode *dir, @@ -153,11 +180,19 @@ static struct dentry *bpf_mkdir(struct mnt_idmap *idmap, struct inode *dir, struct dentry *dentry, umode_t mode) { struct inode *inode; + int ret; inode = bpf_get_inode(dir->i_sb, dir, mode | S_IFDIR); if (IS_ERR(inode)) return ERR_CAST(inode); + ret = security_inode_init_security(inode, dir, &dentry->d_name, + bpf_fs_initxattrs, NULL); + if (ret && ret != -EOPNOTSUPP) { + iput(inode); + return ERR_PTR(ret); + } + inode->i_op = &bpf_dir_iops; inode->i_fop = &simple_dir_operations; @@ -330,10 +365,20 @@ static int bpf_mkobj_ops(struct dentry *dentry, umode_t mode, void *raw, const struct file_operations *fops) { struct inode *dir = dentry->d_parent->d_inode; - struct inode *inode = bpf_get_inode(dir->i_sb, dir, mode); + struct inode *inode; + int ret; + + inode = bpf_get_inode(dir->i_sb, dir, mode); if (IS_ERR(inode)) return PTR_ERR(inode); + ret = security_inode_init_security(inode, dir, &dentry->d_name, + bpf_fs_initxattrs, NULL); + if (ret && ret != -EOPNOTSUPP) { + iput(inode); + return ret; + } + inode->i_op = iops; inode->i_fop = fops; inode->i_private = raw; @@ -382,9 +427,11 @@ bpf_lookup(struct inode *dir, struct dentry *dentry, unsigned flags) static int bpf_symlink(struct mnt_idmap *idmap, struct inode *dir, struct dentry *dentry, const char *target) { - char *link = kstrdup(target, GFP_USER | __GFP_NOWARN); struct inode *inode; + char *link; + int ret; + link = kstrdup(target, GFP_KERNEL_ACCOUNT | __GFP_NOWARN); if (!link) return -ENOMEM; @@ -394,13 +441,25 @@ static int bpf_symlink(struct mnt_idmap *idmap, struct inode *dir, return PTR_ERR(inode); } - inode->i_op = &simple_symlink_inode_operations; + inode->i_op = &bpf_symlink_iops; inode->i_link = link; + ret = security_inode_init_security(inode, dir, &dentry->d_name, + bpf_fs_initxattrs, NULL); + if (ret && ret != -EOPNOTSUPP) { + iput(inode); + return ret; + } + bpf_dentry_finalize(dentry, inode, dir); return 0; } +static const struct inode_operations bpf_symlink_iops = { + .get_link = simple_get_link, + .listxattr = bpf_fs_listxattr, +}; + static const struct inode_operations bpf_dir_iops = { .lookup = bpf_lookup, .mkdir = bpf_mkdir, @@ -409,6 +468,7 @@ static const struct inode_operations bpf_dir_iops = { .rename = simple_rename, .link = simple_link, .unlink = simple_unlink, + .listxattr = bpf_fs_listxattr, }; /* pin iterator link into bpffs */ @@ -762,22 +822,147 @@ static int bpf_show_options(struct seq_file *m, struct dentry *root) return 0; } +static struct inode *bpf_fs_alloc_inode(struct super_block *sb) +{ + struct bpf_fs_inode *bi; + + bi = alloc_inode_sb(sb, bpf_fs_inode_cachep, GFP_KERNEL); + if (!bi) + return NULL; + INIT_LIST_HEAD_RCU(&bi->xattrs); + simple_xattr_limits_init(&bi->xlimits); + return &bi->vfs_inode; +} + static void bpf_destroy_inode(struct inode *inode) { + struct bpf_mount_opts *opts = inode->i_sb->s_fs_info; + struct bpf_fs_inode *bi = BPF_FS_I(inode); enum bpf_type type; - if (S_ISLNK(inode->i_mode)) - kfree(inode->i_link); if (!bpf_inode_type(inode, &type)) bpf_any_put(inode->i_private, type); - free_inode_nonrcu(inode); + simple_xattrs_free(&opts->xa_cache, &bi->xattrs, NULL); +} + +static void bpf_free_inode(struct inode *inode) +{ + if (S_ISLNK(inode->i_mode)) + kfree(inode->i_link); + kmem_cache_free(bpf_fs_inode_cachep, BPF_FS_I(inode)); +} + +static int bpf_fs_xattr_get(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *value, size_t size) +{ + struct bpf_mount_opts *opts = inode->i_sb->s_fs_info; + struct bpf_fs_inode *bi = BPF_FS_I(inode); + + name = xattr_full_name(handler, name); + return simple_xattr_get(&opts->xa_cache, &bi->xattrs, name, value, size); +} + +enum { + BPF_FS_XATTR_UNSPEC, + BPF_FS_XATTR_SECURITY, + BPF_FS_XATTR_TRUSTED, +}; + +static int bpf_fs_xattr_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, struct dentry *unused, + struct inode *inode, const char *name, + const void *value, size_t size, int flags) +{ + struct bpf_mount_opts *opts = inode->i_sb->s_fs_info; + struct bpf_fs_inode *bi = BPF_FS_I(inode); + struct simple_xattr *old; + int err = -EINVAL; + + name = xattr_full_name(handler, name); + switch (handler->flags) { + case BPF_FS_XATTR_SECURITY: + err = simple_xattr_set_limited(&opts->xa_cache, &bi->xattrs, + &bi->xlimits, name, value, size, + flags); + break; + case BPF_FS_XATTR_TRUSTED: + old = simple_xattr_set(&opts->xa_cache, &bi->xattrs, name, + value, size, flags); + err = IS_ERR(old) ? PTR_ERR(old) : 0; + if (!err) + simple_xattr_free_rcu(old); + break; + } + if (err) + return err; + inode_set_ctime_current(inode); + return 0; +} + +static const struct xattr_handler bpf_fs_trusted_xattr_handler = { + .prefix = XATTR_TRUSTED_PREFIX, + .flags = BPF_FS_XATTR_TRUSTED, + .get = bpf_fs_xattr_get, + .set = bpf_fs_xattr_set, +}; + +static const struct xattr_handler bpf_fs_security_xattr_handler = { + .prefix = XATTR_SECURITY_PREFIX, + .flags = BPF_FS_XATTR_SECURITY, + .get = bpf_fs_xattr_get, + .set = bpf_fs_xattr_set, +}; + +static const struct xattr_handler * const bpf_fs_xattr_handlers[] = { + &bpf_fs_trusted_xattr_handler, + &bpf_fs_security_xattr_handler, + NULL, +}; + +static ssize_t bpf_fs_listxattr(struct dentry *dentry, char *buf, size_t size) +{ + struct inode *inode = d_inode(dentry); + + return simple_xattr_list(inode, &BPF_FS_I(inode)->xattrs, buf, size); +} + +static int bpf_fs_initxattrs(struct inode *inode, + const struct xattr *xattr_array, void *fs_info) +{ + struct bpf_mount_opts *opts = inode->i_sb->s_fs_info; + struct bpf_fs_inode *bi = BPF_FS_I(inode); + const struct xattr *xattr; + int err; + + for (xattr = xattr_array; xattr->name != NULL; xattr++) { + CLASS(simple_xattr, new_xattr)(xattr->value, xattr->value_len); + if (IS_ERR(new_xattr)) + return PTR_ERR(new_xattr); + + new_xattr->name = kasprintf(GFP_KERNEL_ACCOUNT, + XATTR_SECURITY_PREFIX "%s", + xattr->name); + if (!new_xattr->name) + return -ENOMEM; + + err = simple_xattr_add_limited(&opts->xa_cache, &bi->xattrs, + &bi->xlimits, new_xattr); + if (err) + return err; + + retain_and_null_ptr(new_xattr); + } + return 0; } const struct super_operations bpf_super_ops = { .statfs = simple_statfs, .drop_inode = inode_just_drop, .show_options = bpf_show_options, + .alloc_inode = bpf_fs_alloc_inode, .destroy_inode = bpf_destroy_inode, + .free_inode = bpf_free_inode, }; enum { @@ -996,25 +1181,38 @@ out: static int bpf_fill_super(struct super_block *sb, struct fs_context *fc) { - static const struct tree_descr bpf_rfiles[] = { { "" } }; struct bpf_mount_opts *opts = sb->s_fs_info; struct inode *inode; - int ret; /* Mounting an instance of BPF FS requires privileges */ if (fc->user_ns != &init_user_ns && !capable(CAP_SYS_ADMIN)) return -EPERM; - ret = simple_fill_super(sb, BPF_FS_MAGIC, bpf_rfiles); - if (ret) - return ret; - + sb->s_blocksize = PAGE_SIZE; + sb->s_blocksize_bits = PAGE_SHIFT; + sb->s_magic = BPF_FS_MAGIC; sb->s_op = &bpf_super_ops; + sb->s_xattr = bpf_fs_xattr_handlers; + sb->s_iflags |= SB_I_NOEXEC; + sb->s_iflags |= SB_I_NODEV; + sb->s_time_gran = 1; + + inode = bpf_get_inode(sb, NULL, S_IFDIR | 0777); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + inode->i_ino = 1; + inode->i_op = &bpf_dir_iops; + inode->i_fop = &simple_dir_operations; + set_nlink(inode, 2); + + sb->s_root = d_make_root(inode); + if (!sb->s_root) + return -ENOMEM; - inode = sb->s_root->d_inode; + inode = d_inode(sb->s_root); inode->i_uid = opts->uid; inode->i_gid = opts->gid; - inode->i_op = &bpf_dir_iops; inode->i_mode &= ~S_IALLUGO; populate_bpffs(sb->s_root); inode->i_mode |= S_ISVTX | opts->mode; @@ -1068,6 +1266,7 @@ static void bpf_kill_super(struct super_block *sb) struct bpf_mount_opts *opts = sb->s_fs_info; kill_anon_super(sb); + simple_xattr_cache_cleanup(&opts->xa_cache); kfree(opts); } @@ -1080,18 +1279,37 @@ static struct file_system_type bpf_fs_type = { .fs_flags = FS_USERNS_MOUNT, }; +static void bpf_fs_inode_init_once(void *foo) +{ + struct bpf_fs_inode *bi = foo; + + inode_init_once(&bi->vfs_inode); +} + static int __init bpf_init(void) { int ret; + bpf_fs_inode_cachep = kmem_cache_create("bpf_fs_inode_cache", + sizeof(struct bpf_fs_inode), + 0, SLAB_ACCOUNT, + bpf_fs_inode_init_once); + if (!bpf_fs_inode_cachep) + return -ENOMEM; + ret = sysfs_create_mount_point(fs_kobj, "bpf"); if (ret) - return ret; + goto out_cache; ret = register_filesystem(&bpf_fs_type); - if (ret) + if (ret) { sysfs_remove_mount_point(fs_kobj, "bpf"); + goto out_cache; + } + return 0; +out_cache: + kmem_cache_destroy(bpf_fs_inode_cachep); return ret; } fs_initcall(bpf_init); -- cgit v1.2.3 From 37363191cbe8f83586ad6a818460d010070ead00 Mon Sep 17 00:00:00 2001 From: Nuoqi Gui Date: Sat, 6 Jun 2026 18:50:37 +0800 Subject: bpf: Fold reg->var_off into PTR_TO_FLOW_KEYS bounds check Constant pointer arithmetic on a PTR_TO_FLOW_KEYS register lands the constant in reg->var_off (e.g. flow_keys(imm=4096)), but the PTR_TO_FLOW_KEYS path in check_mem_access() passes only insn->off to check_flow_keys_access() and never folds reg->var_off.value. The verifier therefore accepts an access that, at runtime, dereferences past struct bpf_flow_keys -- a verifier/runtime divergence that yields an out-of-bounds read and write of kernel stack memory. Commit 022ac0750883 ("bpf: use reg->var_off instead of reg->off for pointers") removed the generic "off += reg->off" that check_mem_access() applied before the per-type dispatch and replaced it with per-path folding of reg->var_off.value (for example the ctx path now folds the register offset via check_ctx_access()). The PTR_TO_FLOW_KEYS path was not given the equivalent fold, so a constant offset that used to be folded and rejected is now silently accepted: before 022ac0750883: the offset stays in reg->off and is folded generically, so the access is checked with off=4096 and rejected. after 022ac0750883: the offset lands in reg->var_off, the flow_keys path checks off=0 and accepts; at runtime the access dereferences base + 0x1000. For a BPF_PROG_TYPE_FLOW_DISSECTOR program the following is accepted: r2 = *(u64 *)(r1 + 144) ; R2=flow_keys (PTR_TO_FLOW_KEYS) r2 += 0x1000 ; R2=flow_keys(imm=4096), accepted r0 = *(u64 *)(r2 + 0) ; accepted, var_off.value=0x1000 ignored while the equivalent insn->off form r0 = *(u64 *)(r2 + 0x1000) has the same effective offset but is correctly rejected with "invalid access to flow keys off=4096 size=8", which isolates the defect to the missing var_off fold. Once attached as a flow dissector, the accepted program reads kernel stack past struct bpf_flow_keys (a kernel-stack / KASLR information leak) and can likewise write past it, corrupting kernel memory. Fix it by folding reg->var_off.value into the offset before the bounds check and rejecting non-constant offsets, mirroring the other pointer types (e.g. check_ctx_access()). Fixes: 022ac0750883 ("bpf: use reg->var_off instead of reg->off for pointers") Signed-off-by: Nuoqi Gui Acked-by: Eduard Zingerman Link: https://lore.kernel.org/r/20260606-c3-01-v3-v3-1-97c51f592f15@mails.tsinghua.edu.cn Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 935595138aa0..68ddd465584c 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -4728,9 +4728,21 @@ static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, struct b return err; } -static int check_flow_keys_access(struct bpf_verifier_env *env, int off, - int size) +static int check_flow_keys_access(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, argno_t argno, + int off, int size) { + /* Only a constant offset is allowed here; fold it into off. */ + if (!tnum_is_const(reg->var_off)) { + char tn_buf[48]; + + tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); + verbose(env, "%s invalid variable offset to flow keys: off=%d, var_off=%s\n", + reg_arg_name(env, argno), off, tn_buf); + return -EACCES; + } + off += reg->var_off.value; + if (size < 0 || off < 0 || (u64)off + size > sizeof(struct bpf_flow_keys)) { verbose(env, "invalid access to flow keys off=%d size=%d\n", @@ -6239,7 +6251,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b return -EACCES; } - err = check_flow_keys_access(env, off, size); + err = check_flow_keys_access(env, reg, argno, off, size); if (!err && t == BPF_READ && value_regno >= 0) mark_reg_unknown(env, regs, value_regno); } else if (type_is_sk_pointer(reg->type)) { -- cgit v1.2.3 From 5b038319be442c620f774e6fc9e9283deeca1c75 Mon Sep 17 00:00:00 2001 From: David Windsor Date: Fri, 5 Jun 2026 10:57:07 -0400 Subject: bpf: Reject sleepable BPF_LSM_CGROUP programs at load time The cgroup shim runs under rcu_read_lock_dont_migrate(), so we should not attach any sleepable BPF programs there. Add support to the verifier to explicitly reject attempts to load sleepable BPF programs destined for LSM cgroup attachment. Without this, we get the following splat from a BPF_LSM_CGROUP program marked BPF_F_SLEEPABLE attached to file_open when it calls bpf_get_dentry_xattr(): BUG: sleeping function called from invalid context at kernel/locking/rwsem.c:1567 in_atomic(): 0, irqs_disabled(): 0, non_block: 0, pid: 34317, name: load preempt_count: 0, expected: 0 RCU nest depth: 2, expected: 0 Call Trace: down_read+0x76/0x480 ext4_xattr_get+0x11f/0x700 __vfs_getxattr+0xf0/0x150 bpf_get_dentry_xattr+0xbb/0xf0 bpf_prog_e76a298dac9218c6_test_open+0x6a/0x85 __cgroup_bpf_run_lsm_current+0x326/0x840 bpf_trampoline_6442534646+0x62/0x14d security_file_open+0x34/0x60 do_dentry_open+0x340/0x1260 vfs_open+0x7a/0x440 path_openat+0x1bac/0x30a0 libbpf provides a .s named section variant for every sleepable program type except lsm_cgroup, reflecting that per-cgroup LSM programs are intended to only run in a non-sleepable context. The above splat was obtained by bypassing libbpf by using bpf(2) directly. Fixes: 69fd337a975c ("bpf: per-cgroup lsm flavor") Signed-off-by: David Windsor Acked-by: Yonghong Song Acked-by: Song Liu Link: https://lore.kernel.org/bpf/20260605145707.608579-1-dwindsor@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 68ddd465584c..926ff63a0b61 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -19172,8 +19172,10 @@ static bool can_be_sleepable(struct bpf_prog *prog) return false; } } - return prog->type == BPF_PROG_TYPE_LSM || - prog->type == BPF_PROG_TYPE_KPROBE /* only for uprobes */ || + if (prog->type == BPF_PROG_TYPE_LSM) + return prog->expected_attach_type != BPF_LSM_CGROUP; + + return prog->type == BPF_PROG_TYPE_KPROBE /* only for uprobes */ || prog->type == BPF_PROG_TYPE_STRUCT_OPS || prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT || prog->type == BPF_PROG_TYPE_TRACEPOINT; -- cgit v1.2.3 From e57f13eaab259ece7c9e8d81ba2c40c4f057ca2c Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Sat, 6 Jun 2026 14:39:26 +0200 Subject: ftrace: Add ftrace_hash_count function Adding external ftrace_hash_count function so we could get hash count outside of ftrace object. Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260606123955.345967-2-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/trace/ftrace.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index b2611de3f594..57ab01fd00bd 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -6288,11 +6288,16 @@ int modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr) } EXPORT_SYMBOL_GPL(modify_ftrace_direct); -static unsigned long hash_count(struct ftrace_hash *hash) +static inline unsigned long hash_count(struct ftrace_hash *hash) { return hash ? hash->count : 0; } +unsigned long ftrace_hash_count(struct ftrace_hash *hash) +{ + return hash_count(hash); +} + /** * hash_add - adds two struct ftrace_hash and returns the result * @a: struct ftrace_hash object -- cgit v1.2.3 From af7c32365090a1a8ff981f85d7c24b344a2eaa75 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Sat, 6 Jun 2026 14:39:27 +0200 Subject: ftrace: Add ftrace_hash_remove function Adding ftrace_hash_remove function that removes all entries from struct ftrace_hash object without freeing them. It will be used in following changes where entries are allocated as part of another structure and are free-ed separately. Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260606123955.345967-3-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/trace/ftrace.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'kernel') diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 57ab01fd00bd..45548b0200eb 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -1249,6 +1249,25 @@ remove_hash_entry(struct ftrace_hash *hash, hash->count--; } +void ftrace_hash_remove(struct ftrace_hash *hash) +{ + struct ftrace_func_entry *entry; + struct hlist_head *hhd; + struct hlist_node *tn; + int size; + int i; + + if (!hash || !hash->count) + return; + size = 1 << hash->size_bits; + for (i = 0; i < size; i++) { + hhd = &hash->buckets[i]; + hlist_for_each_entry_safe(entry, tn, hhd, hlist) + remove_hash_entry(hash, entry); + } + FTRACE_WARN_ON(hash->count); +} + static void ftrace_hash_clear(struct ftrace_hash *hash) { struct hlist_head *hhd; -- cgit v1.2.3 From 2cd298c106e00ba1d8799b022594f131703f32fa Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Sat, 6 Jun 2026 14:39:28 +0200 Subject: ftrace: Add add_ftrace_hash_entry function Renaming __add_hash_entry to add_ftrace_hash_entry and making it global, it will be used in following changes outside ftrace.c object. Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260606123955.345967-4-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/trace/ftrace.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 45548b0200eb..f93e34dd2328 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -1198,8 +1198,7 @@ ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip) return __ftrace_lookup_ip(hash, ip); } -static void __add_hash_entry(struct ftrace_hash *hash, - struct ftrace_func_entry *entry) +void add_ftrace_hash_entry(struct ftrace_hash *hash, struct ftrace_func_entry *entry) { struct hlist_head *hhd; unsigned long key; @@ -1221,7 +1220,7 @@ add_ftrace_hash_entry_direct(struct ftrace_hash *hash, unsigned long ip, unsigne entry->ip = ip; entry->direct = direct; - __add_hash_entry(hash, entry); + add_ftrace_hash_entry(hash, entry); return entry; } @@ -1477,7 +1476,7 @@ static struct ftrace_hash *__move_hash(struct ftrace_hash *src, int size) hhd = &src->buckets[i]; hlist_for_each_entry_safe(entry, tn, hhd, hlist) { remove_hash_entry(src, entry); - __add_hash_entry(new_hash, entry); + add_ftrace_hash_entry(new_hash, entry); } } return new_hash; @@ -5360,7 +5359,7 @@ int ftrace_func_mapper_add_ip(struct ftrace_func_mapper *mapper, map->entry.ip = ip; map->data = data; - __add_hash_entry(&mapper->hash, &map->entry); + add_ftrace_hash_entry(&mapper->hash, &map->entry); return 0; } -- cgit v1.2.3 From e6abd4cd157bf63cd89c74f8f10abae76e7b0359 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Sat, 6 Jun 2026 14:39:29 +0200 Subject: bpf: Use mutex lock pool for bpf trampolines Adding mutex lock pool that replaces bpf trampolines mutex. For tracing_multi link coming in following changes we need to lock all the involved trampolines during the attachment. This could mean thousands of mutex locks, which is not convenient. As suggested by Andrii we can replace bpf trampolines mutex with mutex pool, where each trampoline is hash-ed to one of the locks from the pool. It's better to lock all the pool mutexes (32 at the moment) than thousands of them. There is 48 (MAX_LOCK_DEPTH) lock limit allowed to be simultaneously held by task, so we need to keep 32 mutexes (5 bits) in the pool, so when we lock them all in following changes the lockdep won't scream. Removing the mutex_is_locked in bpf_trampoline_put, because we removed the mutex from bpf_trampoline. Suggested-by: Andrii Nakryiko Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260606123955.345967-5-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/bpf/trampoline.c | 77 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 53 insertions(+), 24 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index a4298a25d4ba..c0b4732627be 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -30,6 +30,35 @@ static struct hlist_head trampoline_ip_table[TRAMPOLINE_TABLE_SIZE]; /* serializes access to trampoline tables */ static DEFINE_MUTEX(trampoline_mutex); +/* + * Keep 32 trampoline locks (5 bits) in the pool so trampoline_lock_all() + * stays below MAX_LOCK_DEPTH. Each pool slot has a distinct lockdep + * class because trampoline_lock_all() takes all pool mutexes at once; + * otherwise lockdep would report recursive locking on same-class mutexes. + */ +#define TRAMPOLINE_LOCKS_BITS 5 +#define TRAMPOLINE_LOCKS_TABLE_SIZE (1 << TRAMPOLINE_LOCKS_BITS) + +static struct { + struct mutex mutex; + struct lock_class_key key; +} trampoline_locks[TRAMPOLINE_LOCKS_TABLE_SIZE]; + +static struct mutex *select_trampoline_lock(struct bpf_trampoline *tr) +{ + return &trampoline_locks[hash_ptr(tr, TRAMPOLINE_LOCKS_BITS)].mutex; +} + +static void trampoline_lock(struct bpf_trampoline *tr) +{ + mutex_lock(select_trampoline_lock(tr)); +} + +static void trampoline_unlock(struct bpf_trampoline *tr) +{ + mutex_unlock(select_trampoline_lock(tr)); +} + #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex); @@ -69,9 +98,9 @@ static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, unsigned long ip, if (cmd == FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF) { /* This is called inside register_ftrace_direct_multi(), so - * tr->mutex is already locked. + * trampoline's mutex is already locked. */ - lockdep_assert_held_once(&tr->mutex); + lockdep_assert_held_once(select_trampoline_lock(tr)); /* Instead of updating the trampoline here, we propagate * -EAGAIN to register_ftrace_direct(). Then we can @@ -91,7 +120,7 @@ static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, unsigned long ip, } /* The normal locking order is - * tr->mutex => direct_mutex (ftrace.c) => ftrace_lock (ftrace.c) + * select_trampoline_lock(tr) => direct_mutex (ftrace.c) => ftrace_lock (ftrace.c) * * The following two commands are called from * @@ -99,12 +128,12 @@ static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, unsigned long ip, * cleanup_direct_functions_after_ipmodify * * In both cases, direct_mutex is already locked. Use - * mutex_trylock(&tr->mutex) to avoid deadlock in race condition - * (something else is making changes to this same trampoline). + * mutex_trylock(select_trampoline_lock(tr)) to avoid deadlock in race condition + * (something else holds the same pool lock). */ - if (!mutex_trylock(&tr->mutex)) { - /* sleep 1 ms to make sure whatever holding tr->mutex makes - * some progress. + if (!mutex_trylock(select_trampoline_lock(tr))) { + /* sleep 1 ms to make sure whatever holding select_trampoline_lock(tr) + * makes some progress. */ msleep(1); return -EAGAIN; @@ -129,7 +158,7 @@ static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, unsigned long ip, break; } - mutex_unlock(&tr->mutex); + trampoline_unlock(tr); return ret; } #endif @@ -359,7 +388,6 @@ static struct bpf_trampoline *bpf_trampoline_lookup(u64 key, unsigned long ip) head = &trampoline_ip_table[hash_64(tr->ip, TRAMPOLINE_HASH_BITS)]; hlist_add_head(&tr->hlist_ip, head); refcount_set(&tr->refcnt, 1); - mutex_init(&tr->mutex); for (i = 0; i < BPF_TRAMP_MAX; i++) INIT_HLIST_HEAD(&tr->progs_hlist[i]); out: @@ -843,9 +871,9 @@ int bpf_trampoline_link_prog(struct bpf_tramp_link *link, { int err; - mutex_lock(&tr->mutex); + trampoline_lock(tr); err = __bpf_trampoline_link_prog(link, tr, tgt_prog); - mutex_unlock(&tr->mutex); + trampoline_unlock(tr); return err; } @@ -886,9 +914,9 @@ int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, { int err; - mutex_lock(&tr->mutex); + trampoline_lock(tr); err = __bpf_trampoline_unlink_prog(link, tr, tgt_prog); - mutex_unlock(&tr->mutex); + trampoline_unlock(tr); return err; } @@ -998,12 +1026,12 @@ int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, if (!tr) return -ENOMEM; - mutex_lock(&tr->mutex); + trampoline_lock(tr); shim_link = cgroup_shim_find(tr, bpf_func); if (shim_link && !IS_ERR(bpf_link_inc_not_zero(&shim_link->link.link))) { /* Reusing existing shim attached by the other program. */ - mutex_unlock(&tr->mutex); + trampoline_unlock(tr); bpf_trampoline_put(tr); /* bpf_trampoline_get above */ return 0; } @@ -1023,16 +1051,16 @@ int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, shim_link->trampoline = tr; /* note, we're still holding tr refcnt from above */ - mutex_unlock(&tr->mutex); + trampoline_unlock(tr); return 0; err: - mutex_unlock(&tr->mutex); + trampoline_unlock(tr); if (shim_link) bpf_link_put(&shim_link->link.link); - /* have to release tr while _not_ holding its mutex */ + /* have to release tr while _not_ holding pool mutex for trampoline */ bpf_trampoline_put(tr); /* bpf_trampoline_get above */ return err; @@ -1053,9 +1081,9 @@ void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog) if (WARN_ON_ONCE(!tr)) return; - mutex_lock(&tr->mutex); + trampoline_lock(tr); shim_link = cgroup_shim_find(tr, bpf_func); - mutex_unlock(&tr->mutex); + trampoline_unlock(tr); if (shim_link) bpf_link_put(&shim_link->link.link); @@ -1073,14 +1101,14 @@ struct bpf_trampoline *bpf_trampoline_get(u64 key, if (!tr) return NULL; - mutex_lock(&tr->mutex); + trampoline_lock(tr); if (tr->func.addr) goto out; memcpy(&tr->func.model, &tgt_info->fmodel, sizeof(tgt_info->fmodel)); tr->func.addr = (void *)tgt_info->tgt_addr; out: - mutex_unlock(&tr->mutex); + trampoline_unlock(tr); return tr; } @@ -1093,7 +1121,6 @@ void bpf_trampoline_put(struct bpf_trampoline *tr) mutex_lock(&trampoline_mutex); if (!refcount_dec_and_test(&tr->refcnt)) goto out; - WARN_ON_ONCE(mutex_is_locked(&tr->mutex)); for (i = 0; i < BPF_TRAMP_MAX; i++) if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[i]))) @@ -1379,6 +1406,8 @@ static int __init init_trampolines(void) INIT_HLIST_HEAD(&trampoline_key_table[i]); for (i = 0; i < TRAMPOLINE_TABLE_SIZE; i++) INIT_HLIST_HEAD(&trampoline_ip_table[i]); + for (i = 0; i < TRAMPOLINE_LOCKS_TABLE_SIZE; i++) + __mutex_init(&trampoline_locks[i].mutex, "trampoline_lock", &trampoline_locks[i].key); return 0; } late_initcall(init_trampolines); -- cgit v1.2.3 From 8a35e8db740f96ec17b85db5a0f83c028c707a3e Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Sat, 6 Jun 2026 14:39:30 +0200 Subject: bpf: Add struct bpf_trampoline_ops object In following changes we will need to override ftrace direct attachment behaviour. In order to do that we are adding struct bpf_trampoline_ops object that defines callbacks for ftrace direct attachment: register_fentry unregister_fentry modify_fentry The new struct bpf_trampoline_ops object is passed as an argument to __bpf_trampoline_link/unlink_prog functions. At the moment the default trampoline_ops is set to the current ftrace direct attachment functions, so there's no functional change for the current code. Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260606123955.345967-6-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/bpf/trampoline.c | 59 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 18 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index c0b4732627be..5c943832fb9d 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -59,8 +59,18 @@ static void trampoline_unlock(struct bpf_trampoline *tr) mutex_unlock(select_trampoline_lock(tr)); } +struct bpf_trampoline_ops { + int (*register_fentry)(struct bpf_trampoline *tr, void *new_addr, void *data); + int (*unregister_fentry)(struct bpf_trampoline *tr, u32 orig_flags, void *old_addr, + void *data); + int (*modify_fentry)(struct bpf_trampoline *tr, u32 orig_flags, void *old_addr, + void *new_addr, bool lock_direct_mutex, void *data); +}; + #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS -static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex); +static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex, + const struct bpf_trampoline_ops *ops, void *data); +static const struct bpf_trampoline_ops trampoline_ops; #ifdef CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS static struct bpf_trampoline *direct_ops_ip_lookup(struct ftrace_ops *ops, unsigned long ip) @@ -145,13 +155,15 @@ static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, unsigned long ip, if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) && !(tr->flags & BPF_TRAMP_F_ORIG_STACK)) - ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */); + ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */, + &trampoline_ops, NULL); break; case FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY_PEER: tr->flags &= ~BPF_TRAMP_F_SHARE_IPMODIFY; if (tr->flags & BPF_TRAMP_F_ORIG_STACK) - ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */); + ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */, + &trampoline_ops, NULL); break; default: ret = -EINVAL; @@ -415,7 +427,7 @@ static int bpf_trampoline_update_fentry(struct bpf_trampoline *tr, u32 orig_flag } static int unregister_fentry(struct bpf_trampoline *tr, u32 orig_flags, - void *old_addr) + void *old_addr, void *data __maybe_unused) { int ret; @@ -429,7 +441,7 @@ static int unregister_fentry(struct bpf_trampoline *tr, u32 orig_flags, static int modify_fentry(struct bpf_trampoline *tr, u32 orig_flags, void *old_addr, void *new_addr, - bool lock_direct_mutex) + bool lock_direct_mutex, void *data __maybe_unused) { int ret; @@ -443,7 +455,7 @@ static int modify_fentry(struct bpf_trampoline *tr, u32 orig_flags, } /* first time registering */ -static int register_fentry(struct bpf_trampoline *tr, void *new_addr) +static int register_fentry(struct bpf_trampoline *tr, void *new_addr, void *data __maybe_unused) { void *ip = tr->func.addr; unsigned long faddr; @@ -465,6 +477,12 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr) return ret; } +static const struct bpf_trampoline_ops trampoline_ops = { + .register_fentry = register_fentry, + .unregister_fentry = unregister_fentry, + .modify_fentry = modify_fentry, +}; + static struct bpf_tramp_links * bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_arg) { @@ -632,7 +650,8 @@ out: return ERR_PTR(err); } -static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex) +static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex, + const struct bpf_trampoline_ops *ops, void *data) { struct bpf_tramp_image *im; struct bpf_tramp_links *tlinks; @@ -645,7 +664,7 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut return PTR_ERR(tlinks); if (total == 0) { - err = unregister_fentry(tr, orig_flags, tr->cur_image->image); + err = ops->unregister_fentry(tr, orig_flags, tr->cur_image->image, data); bpf_tramp_image_put(tr->cur_image); tr->cur_image = NULL; goto out; @@ -715,11 +734,11 @@ again: if (tr->cur_image) /* progs already running at this address */ - err = modify_fentry(tr, orig_flags, tr->cur_image->image, - im->image, lock_direct_mutex); + err = ops->modify_fentry(tr, orig_flags, tr->cur_image->image, + im->image, lock_direct_mutex, data); else /* first time registering */ - err = register_fentry(tr, im->image); + err = ops->register_fentry(tr, im->image, data); #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS if (err == -EAGAIN) { @@ -793,7 +812,9 @@ static int bpf_freplace_check_tgt_prog(struct bpf_prog *tgt_prog) static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr, - struct bpf_prog *tgt_prog) + struct bpf_prog *tgt_prog, + const struct bpf_trampoline_ops *ops, + void *data) { struct bpf_fsession_link *fslink = NULL; enum bpf_tramp_prog_type kind; @@ -851,7 +872,7 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, } else { tr->progs_cnt[kind]++; } - err = bpf_trampoline_update(tr, true /* lock_direct_mutex */); + err = bpf_trampoline_update(tr, true /* lock_direct_mutex */, ops, data); if (err) { hlist_del_init(&link->tramp_hlist); if (kind == BPF_TRAMP_FSESSION) { @@ -872,14 +893,16 @@ int bpf_trampoline_link_prog(struct bpf_tramp_link *link, int err; trampoline_lock(tr); - err = __bpf_trampoline_link_prog(link, tr, tgt_prog); + err = __bpf_trampoline_link_prog(link, tr, tgt_prog, &trampoline_ops, NULL); trampoline_unlock(tr); return err; } static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr, - struct bpf_prog *tgt_prog) + struct bpf_prog *tgt_prog, + const struct bpf_trampoline_ops *ops, + void *data) { enum bpf_tramp_prog_type kind; int err; @@ -904,7 +927,7 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, } hlist_del_init(&link->tramp_hlist); tr->progs_cnt[kind]--; - return bpf_trampoline_update(tr, true /* lock_direct_mutex */); + return bpf_trampoline_update(tr, true /* lock_direct_mutex */, ops, data); } /* bpf_trampoline_unlink_prog() should never fail. */ @@ -915,7 +938,7 @@ int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, int err; trampoline_lock(tr); - err = __bpf_trampoline_unlink_prog(link, tr, tgt_prog); + err = __bpf_trampoline_unlink_prog(link, tr, tgt_prog, &trampoline_ops, NULL); trampoline_unlock(tr); return err; } @@ -1044,7 +1067,7 @@ int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, goto err; } - err = __bpf_trampoline_link_prog(&shim_link->link, tr, NULL); + err = __bpf_trampoline_link_prog(&shim_link->link, tr, NULL, &trampoline_ops, NULL); if (err) goto err; -- cgit v1.2.3 From bf4bc3e11c4195123055780b84dccfb8d2569535 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Sat, 6 Jun 2026 14:39:31 +0200 Subject: bpf: Move trampoline image setup into bpf_trampoline_ops callbacks Moving trampoline image setup into bpf_trampoline_ops callbacks, so we can have different image handling for multi attachment which is coming in following changes. Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260606123955.345967-7-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/bpf/trampoline.c | 66 ++++++++++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 28 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 5c943832fb9d..1006031ea021 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -60,11 +60,10 @@ static void trampoline_unlock(struct bpf_trampoline *tr) } struct bpf_trampoline_ops { - int (*register_fentry)(struct bpf_trampoline *tr, void *new_addr, void *data); - int (*unregister_fentry)(struct bpf_trampoline *tr, u32 orig_flags, void *old_addr, - void *data); - int (*modify_fentry)(struct bpf_trampoline *tr, u32 orig_flags, void *old_addr, - void *new_addr, bool lock_direct_mutex, void *data); + int (*register_fentry)(struct bpf_trampoline *tr, struct bpf_tramp_image *im, void *data); + int (*unregister_fentry)(struct bpf_trampoline *tr, u32 orig_flags, void *data); + int (*modify_fentry)(struct bpf_trampoline *tr, u32 orig_flags, struct bpf_tramp_image *im, + bool lock_direct_mutex, void *data); }; #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS @@ -426,9 +425,11 @@ static int bpf_trampoline_update_fentry(struct bpf_trampoline *tr, u32 orig_flag return bpf_arch_text_poke(ip, old_t, new_t, old_addr, new_addr); } -static int unregister_fentry(struct bpf_trampoline *tr, u32 orig_flags, - void *old_addr, void *data __maybe_unused) +static void bpf_tramp_image_put(struct bpf_tramp_image *im); + +static int unregister_fentry(struct bpf_trampoline *tr, u32 orig_flags, void *data __maybe_unused) { + void *old_addr = tr->cur_image->image; int ret; if (tr->func.ftrace_managed) @@ -436,13 +437,19 @@ static int unregister_fentry(struct bpf_trampoline *tr, u32 orig_flags, else ret = bpf_trampoline_update_fentry(tr, orig_flags, old_addr, NULL); - return ret; + if (ret) + return ret; + + bpf_tramp_image_put(tr->cur_image); + tr->cur_image = NULL; + return 0; } -static int modify_fentry(struct bpf_trampoline *tr, u32 orig_flags, - void *old_addr, void *new_addr, +static int modify_fentry(struct bpf_trampoline *tr, u32 orig_flags, struct bpf_tramp_image *im, bool lock_direct_mutex, void *data __maybe_unused) { + void *old_addr = tr->cur_image->image; + void *new_addr = im->image; int ret; if (tr->func.ftrace_managed) { @@ -451,12 +458,20 @@ static int modify_fentry(struct bpf_trampoline *tr, u32 orig_flags, ret = bpf_trampoline_update_fentry(tr, orig_flags, old_addr, new_addr); } - return ret; + + if (ret) + return ret; + + bpf_tramp_image_put(tr->cur_image); + tr->cur_image = im; + return 0; } /* first time registering */ -static int register_fentry(struct bpf_trampoline *tr, void *new_addr, void *data __maybe_unused) +static int register_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *im, + void *data __maybe_unused) { + void *new_addr = im->image; void *ip = tr->func.addr; unsigned long faddr; int ret; @@ -474,7 +489,11 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr, void *data ret = bpf_trampoline_update_fentry(tr, 0, NULL, new_addr); } - return ret; + if (ret) + return ret; + + tr->cur_image = im; + return 0; } static const struct bpf_trampoline_ops trampoline_ops = { @@ -664,9 +683,7 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut return PTR_ERR(tlinks); if (total == 0) { - err = ops->unregister_fentry(tr, orig_flags, tr->cur_image->image, data); - bpf_tramp_image_put(tr->cur_image); - tr->cur_image = NULL; + err = ops->unregister_fentry(tr, orig_flags, data); goto out; } @@ -734,11 +751,10 @@ again: if (tr->cur_image) /* progs already running at this address */ - err = ops->modify_fentry(tr, orig_flags, tr->cur_image->image, - im->image, lock_direct_mutex, data); + err = ops->modify_fentry(tr, orig_flags, im, lock_direct_mutex, data); else /* first time registering */ - err = ops->register_fentry(tr, im->image, data); + err = ops->register_fentry(tr, im, data); #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS if (err == -EAGAIN) { @@ -750,22 +766,16 @@ again: goto again; } #endif - if (err) - goto out_free; - if (tr->cur_image) - bpf_tramp_image_put(tr->cur_image); - tr->cur_image = im; +out_free: + if (err) + bpf_tramp_image_free(im); out: /* If any error happens, restore previous flags */ if (err) tr->flags = orig_flags; kfree(tlinks); return err; - -out_free: - bpf_tramp_image_free(im); - goto out; } static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog) -- cgit v1.2.3 From e6cc9ed677e622265bbd015892be58a1eece6238 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Sat, 6 Jun 2026 14:39:32 +0200 Subject: bpf: Add bpf_trampoline_add/remove_prog functions Separate bpf_trampoline_add/remove_prog functions from __bpf_trampoline_link/unlink functions to be able to add/remove trampoline programs without the image being updated in following changes. No functional change is intended. Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260606123955.345967-8-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/bpf/trampoline.c | 108 +++++++++++++++++++++++++++--------------------- 1 file changed, 61 insertions(+), 47 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 1006031ea021..701138ef424a 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -820,41 +820,16 @@ static int bpf_freplace_check_tgt_prog(struct bpf_prog *tgt_prog) return 0; } -static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, - struct bpf_trampoline *tr, - struct bpf_prog *tgt_prog, - const struct bpf_trampoline_ops *ops, - void *data) +static int bpf_trampoline_add_prog(struct bpf_trampoline *tr, + struct bpf_tramp_link *link, + int cnt) { struct bpf_fsession_link *fslink = NULL; enum bpf_tramp_prog_type kind; struct bpf_tramp_link *link_exiting; struct hlist_head *prog_list; - int err = 0; - int cnt = 0, i; kind = bpf_attach_type_to_tramp(link->link.prog); - if (tr->extension_prog) - /* cannot attach fentry/fexit if extension prog is attached. - * cannot overwrite extension prog either. - */ - return -EBUSY; - - for (i = 0; i < BPF_TRAMP_MAX; i++) - cnt += tr->progs_cnt[i]; - - if (kind == BPF_TRAMP_REPLACE) { - /* Cannot attach extension if fentry/fexit are in use. */ - if (cnt) - return -EBUSY; - err = bpf_freplace_check_tgt_prog(tgt_prog); - if (err) - return err; - tr->extension_prog = link->link.prog; - return bpf_arch_text_poke(tr->func.addr, BPF_MOD_NOP, - BPF_MOD_JUMP, NULL, - link->link.prog->bpf_func); - } if (kind == BPF_TRAMP_FSESSION) { prog_list = &tr->progs_hlist[BPF_TRAMP_FENTRY]; cnt++; @@ -882,17 +857,64 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, } else { tr->progs_cnt[kind]++; } - err = bpf_trampoline_update(tr, true /* lock_direct_mutex */, ops, data); - if (err) { - hlist_del_init(&link->tramp_hlist); - if (kind == BPF_TRAMP_FSESSION) { - tr->progs_cnt[BPF_TRAMP_FENTRY]--; - hlist_del_init(&fslink->fexit.tramp_hlist); - tr->progs_cnt[BPF_TRAMP_FEXIT]--; - } else { - tr->progs_cnt[kind]--; - } + return 0; +} + +static void bpf_trampoline_remove_prog(struct bpf_trampoline *tr, + struct bpf_tramp_link *link) +{ + struct bpf_fsession_link *fslink; + enum bpf_tramp_prog_type kind; + + kind = bpf_attach_type_to_tramp(link->link.prog); + if (kind == BPF_TRAMP_FSESSION) { + fslink = container_of(link, struct bpf_fsession_link, link.link); + hlist_del_init(&fslink->fexit.tramp_hlist); + tr->progs_cnt[BPF_TRAMP_FEXIT]--; + kind = BPF_TRAMP_FENTRY; + } + hlist_del_init(&link->tramp_hlist); + tr->progs_cnt[kind]--; +} + +static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, + struct bpf_trampoline *tr, + struct bpf_prog *tgt_prog, + const struct bpf_trampoline_ops *ops, + void *data) +{ + enum bpf_tramp_prog_type kind; + int err = 0; + int cnt = 0, i; + + kind = bpf_attach_type_to_tramp(link->link.prog); + if (tr->extension_prog) + /* cannot attach fentry/fexit if extension prog is attached. + * cannot overwrite extension prog either. + */ + return -EBUSY; + + for (i = 0; i < BPF_TRAMP_MAX; i++) + cnt += tr->progs_cnt[i]; + + if (kind == BPF_TRAMP_REPLACE) { + /* Cannot attach extension if fentry/fexit are in use. */ + if (cnt) + return -EBUSY; + err = bpf_freplace_check_tgt_prog(tgt_prog); + if (err) + return err; + tr->extension_prog = link->link.prog; + return bpf_arch_text_poke(tr->func.addr, BPF_MOD_NOP, + BPF_MOD_JUMP, NULL, + link->link.prog->bpf_func); } + err = bpf_trampoline_add_prog(tr, link, cnt); + if (err) + return err; + err = bpf_trampoline_update(tr, true /* lock_direct_mutex */, ops, data); + if (err) + bpf_trampoline_remove_prog(tr, link); return err; } @@ -927,16 +949,8 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, guard(mutex)(&tgt_prog->aux->ext_mutex); tgt_prog->aux->is_extended = false; return err; - } else if (kind == BPF_TRAMP_FSESSION) { - struct bpf_fsession_link *fslink = - container_of(link, struct bpf_fsession_link, link.link); - - hlist_del_init(&fslink->fexit.tramp_hlist); - tr->progs_cnt[BPF_TRAMP_FEXIT]--; - kind = BPF_TRAMP_FENTRY; } - hlist_del_init(&link->tramp_hlist); - tr->progs_cnt[kind]--; + bpf_trampoline_remove_prog(tr, link); return bpf_trampoline_update(tr, true /* lock_direct_mutex */, ops, data); } -- cgit v1.2.3 From 65499074efaf574fef6365ac63b785a3ec98913d Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Sat, 6 Jun 2026 14:39:33 +0200 Subject: bpf: Add struct bpf_tramp_node object Adding struct bpf_tramp_node to decouple the link out of the trampoline attachment info. At the moment the object for attaching bpf program to the trampoline is 'struct bpf_tramp_link': struct bpf_tramp_link { struct bpf_link link; struct hlist_node tramp_hlist; u64 cookie; } The link holds the bpf_prog pointer and forces one link - one program binding logic. In following changes we want to attach program to multiple trampolines but we want to keep just one bpf_link object. Splitting struct bpf_tramp_link into: struct bpf_tramp_link { struct bpf_link link; struct bpf_tramp_node node; }; struct bpf_tramp_node { struct bpf_link *link; struct hlist_node tramp_hlist; u64 cookie; }; The 'struct bpf_tramp_link' defines standard single trampoline link and 'struct bpf_tramp_node' is the attachment trampoline object with pointer to the bpf_link object. This will allow us to define link for multiple trampolines, like: struct bpf_tracing_multi_link { struct bpf_link link; ... int nodes_cnt; struct bpf_tracing_multi_node nodes[] __counted_by(nodes_cnt); }; Cc: Hengqi Chen Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260606123955.345967-9-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/bpf/bpf_struct_ops.c | 27 ++++++----- kernel/bpf/syscall.c | 39 ++++++++------- kernel/bpf/trampoline.c | 115 ++++++++++++++++++++++---------------------- 3 files changed, 93 insertions(+), 88 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 5e51c1211673..51b16e5f5534 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -594,8 +594,8 @@ const struct bpf_link_ops bpf_struct_ops_link_lops = { .dealloc = bpf_struct_ops_link_dealloc, }; -int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, - struct bpf_tramp_link *link, +int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_nodes *tnodes, + struct bpf_tramp_node *node, const struct btf_func_model *model, void *stub_func, void **_image, u32 *_image_off, @@ -605,13 +605,13 @@ int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, void *image = *_image; int size; - tlinks[BPF_TRAMP_FENTRY].links[0] = link; - tlinks[BPF_TRAMP_FENTRY].nr_links = 1; + tnodes[BPF_TRAMP_FENTRY].nodes[0] = node; + tnodes[BPF_TRAMP_FENTRY].nr_nodes = 1; if (model->ret_size > 0) flags |= BPF_TRAMP_F_RET_FENTRY_RET; - size = arch_bpf_trampoline_size(model, flags, tlinks, stub_func); + size = arch_bpf_trampoline_size(model, flags, tnodes, stub_func); if (size <= 0) return size ? : -EFAULT; @@ -628,7 +628,7 @@ int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, size = arch_prepare_bpf_trampoline(NULL, image + image_off, image + image_off + size, - model, flags, tlinks, stub_func); + model, flags, tnodes, stub_func); if (size <= 0) { if (image != *_image) bpf_struct_ops_image_free(image); @@ -693,7 +693,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, const struct btf_type *module_type; const struct btf_member *member; const struct btf_type *t = st_ops_desc->type; - struct bpf_tramp_links *tlinks; + struct bpf_tramp_nodes *tnodes; void *udata, *kdata; int prog_fd, err; u32 i, trampoline_start, image_off = 0; @@ -720,8 +720,8 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, if (uvalue->common.state || refcount_read(&uvalue->common.refcnt)) return -EINVAL; - tlinks = kzalloc_objs(*tlinks, BPF_TRAMP_MAX); - if (!tlinks) + tnodes = kzalloc_objs(*tnodes, BPF_TRAMP_MAX); + if (!tnodes) return -ENOMEM; uvalue = (struct bpf_struct_ops_value *)st_map->uvalue; @@ -817,8 +817,9 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, err = -ENOMEM; goto reset_unlock; } - bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS, - &bpf_struct_ops_link_lops, prog, prog->expected_attach_type); + bpf_tramp_link_init(link, BPF_LINK_TYPE_STRUCT_OPS, + &bpf_struct_ops_link_lops, prog, prog->expected_attach_type, 0); + *plink++ = &link->link; /* Poison pointer on error instead of return for backward compatibility */ @@ -832,7 +833,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, *pksym++ = ksym; trampoline_start = image_off; - err = bpf_struct_ops_prepare_trampoline(tlinks, link, + err = bpf_struct_ops_prepare_trampoline(tnodes, &link->node, &st_ops->func_models[i], *(void **)(st_ops->cfi_stubs + moff), &image, &image_off, @@ -911,7 +912,7 @@ reset_unlock: memset(uvalue, 0, map->value_size); memset(kvalue, 0, map->value_size); unlock: - kfree(tlinks); + kfree(tnodes); mutex_unlock(&st_map->lock); if (!err) bpf_struct_ops_map_add_ksyms(st_map); diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 5fcfc32c7cb4..fd69fdb9290b 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -3288,6 +3288,15 @@ void bpf_link_init(struct bpf_link *link, enum bpf_link_type type, bpf_link_init_sleepable(link, type, ops, prog, attach_type, false); } +void bpf_tramp_link_init(struct bpf_tramp_link *link, enum bpf_link_type type, + const struct bpf_link_ops *ops, struct bpf_prog *prog, + enum bpf_attach_type attach_type, u64 cookie) +{ + bpf_link_init(&link->link, type, ops, prog, attach_type); + link->node.link = &link->link; + link->node.cookie = cookie; +} + static void bpf_link_free_id(int id) { if (!id) @@ -3595,7 +3604,7 @@ static void bpf_tracing_link_release(struct bpf_link *link) struct bpf_tracing_link *tr_link = container_of(link, struct bpf_tracing_link, link.link); - WARN_ON_ONCE(bpf_trampoline_unlink_prog(&tr_link->link, + WARN_ON_ONCE(bpf_trampoline_unlink_prog(&tr_link->link.node, tr_link->trampoline, tr_link->tgt_prog)); @@ -3608,8 +3617,7 @@ static void bpf_tracing_link_release(struct bpf_link *link) static void bpf_tracing_link_dealloc(struct bpf_link *link) { - struct bpf_tracing_link *tr_link = - container_of(link, struct bpf_tracing_link, link.link); + struct bpf_tracing_link *tr_link = container_of(link, struct bpf_tracing_link, link.link); kfree(tr_link); } @@ -3617,8 +3625,8 @@ static void bpf_tracing_link_dealloc(struct bpf_link *link) static void bpf_tracing_link_show_fdinfo(const struct bpf_link *link, struct seq_file *seq) { - struct bpf_tracing_link *tr_link = - container_of(link, struct bpf_tracing_link, link.link); + struct bpf_tracing_link *tr_link = container_of(link, struct bpf_tracing_link, link.link); + u32 target_btf_id, target_obj_id; bpf_trampoline_unpack_key(tr_link->trampoline->key, @@ -3631,17 +3639,16 @@ static void bpf_tracing_link_show_fdinfo(const struct bpf_link *link, link->attach_type, target_obj_id, target_btf_id, - tr_link->link.cookie); + tr_link->link.node.cookie); } static int bpf_tracing_link_fill_link_info(const struct bpf_link *link, struct bpf_link_info *info) { - struct bpf_tracing_link *tr_link = - container_of(link, struct bpf_tracing_link, link.link); + struct bpf_tracing_link *tr_link = container_of(link, struct bpf_tracing_link, link.link); info->tracing.attach_type = link->attach_type; - info->tracing.cookie = tr_link->link.cookie; + info->tracing.cookie = tr_link->link.node.cookie; bpf_trampoline_unpack_key(tr_link->trampoline->key, &info->tracing.target_obj_id, &info->tracing.target_btf_id); @@ -3728,9 +3735,9 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, fslink = kzalloc_obj(*fslink, GFP_USER); if (fslink) { - bpf_link_init(&fslink->fexit.link, BPF_LINK_TYPE_TRACING, - &bpf_tracing_link_lops, prog, attach_type); - fslink->fexit.cookie = bpf_cookie; + bpf_tramp_link_init(&fslink->fexit, BPF_LINK_TYPE_TRACING, + &bpf_tracing_link_lops, prog, attach_type, + bpf_cookie); link = &fslink->link; } else { link = NULL; @@ -3742,10 +3749,8 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, err = -ENOMEM; goto out_put_prog; } - bpf_link_init(&link->link.link, BPF_LINK_TYPE_TRACING, - &bpf_tracing_link_lops, prog, attach_type); - - link->link.cookie = bpf_cookie; + bpf_tramp_link_init(&link->link, BPF_LINK_TYPE_TRACING, + &bpf_tracing_link_lops, prog, attach_type, bpf_cookie); mutex_lock(&prog->aux->dst_mutex); @@ -3848,7 +3853,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, if (err) goto out_unlock; - err = bpf_trampoline_link_prog(&link->link, tr, tgt_prog); + err = bpf_trampoline_link_prog(&link->link.node, tr, tgt_prog); if (err) { bpf_link_cleanup(&link_primer); link = NULL; diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 701138ef424a..6a45c09fc0d8 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -502,30 +502,29 @@ static const struct bpf_trampoline_ops trampoline_ops = { .modify_fentry = modify_fentry, }; -static struct bpf_tramp_links * +static struct bpf_tramp_nodes * bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_arg) { - struct bpf_tramp_link *link; - struct bpf_tramp_links *tlinks; - struct bpf_tramp_link **links; + struct bpf_tramp_node *node, **nodes; + struct bpf_tramp_nodes *tnodes; int kind; *total = 0; - tlinks = kzalloc_objs(*tlinks, BPF_TRAMP_MAX); - if (!tlinks) + tnodes = kzalloc_objs(*tnodes, BPF_TRAMP_MAX); + if (!tnodes) return ERR_PTR(-ENOMEM); for (kind = 0; kind < BPF_TRAMP_MAX; kind++) { - tlinks[kind].nr_links = tr->progs_cnt[kind]; + tnodes[kind].nr_nodes = tr->progs_cnt[kind]; *total += tr->progs_cnt[kind]; - links = tlinks[kind].links; + nodes = tnodes[kind].nodes; - hlist_for_each_entry(link, &tr->progs_hlist[kind], tramp_hlist) { - *ip_arg |= link->link.prog->call_get_func_ip; - *links++ = link; + hlist_for_each_entry(node, &tr->progs_hlist[kind], tramp_hlist) { + *ip_arg |= node->link->prog->call_get_func_ip; + *nodes++ = node; } } - return tlinks; + return tnodes; } static void bpf_tramp_image_free(struct bpf_tramp_image *im) @@ -673,14 +672,14 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut const struct bpf_trampoline_ops *ops, void *data) { struct bpf_tramp_image *im; - struct bpf_tramp_links *tlinks; + struct bpf_tramp_nodes *tnodes; u32 orig_flags = tr->flags; bool ip_arg = false; int err, total, size; - tlinks = bpf_trampoline_get_progs(tr, &total, &ip_arg); - if (IS_ERR(tlinks)) - return PTR_ERR(tlinks); + tnodes = bpf_trampoline_get_progs(tr, &total, &ip_arg); + if (IS_ERR(tnodes)) + return PTR_ERR(tnodes); if (total == 0) { err = ops->unregister_fentry(tr, orig_flags, data); @@ -690,8 +689,8 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut /* clear all bits except SHARE_IPMODIFY and TAIL_CALL_CTX */ tr->flags &= (BPF_TRAMP_F_SHARE_IPMODIFY | BPF_TRAMP_F_TAIL_CALL_CTX); - if (tlinks[BPF_TRAMP_FEXIT].nr_links || - tlinks[BPF_TRAMP_MODIFY_RETURN].nr_links) { + if (tnodes[BPF_TRAMP_FEXIT].nr_nodes || + tnodes[BPF_TRAMP_MODIFY_RETURN].nr_nodes) { /* NOTE: BPF_TRAMP_F_RESTORE_REGS and BPF_TRAMP_F_SKIP_FRAME * should not be set together. */ @@ -722,7 +721,7 @@ again: #endif size = arch_bpf_trampoline_size(&tr->func.model, tr->flags, - tlinks, tr->func.addr); + tnodes, tr->func.addr); if (size < 0) { err = size; goto out; @@ -740,7 +739,7 @@ again: } err = arch_prepare_bpf_trampoline(im, im->image, im->image + size, - &tr->func.model, tr->flags, tlinks, + &tr->func.model, tr->flags, tnodes, tr->func.addr); if (err < 0) goto out_free; @@ -774,7 +773,7 @@ out: /* If any error happens, restore previous flags */ if (err) tr->flags = orig_flags; - kfree(tlinks); + kfree(tnodes); return err; } @@ -821,15 +820,15 @@ static int bpf_freplace_check_tgt_prog(struct bpf_prog *tgt_prog) } static int bpf_trampoline_add_prog(struct bpf_trampoline *tr, - struct bpf_tramp_link *link, + struct bpf_tramp_node *node, int cnt) { struct bpf_fsession_link *fslink = NULL; enum bpf_tramp_prog_type kind; - struct bpf_tramp_link *link_exiting; + struct bpf_tramp_node *node_existing; struct hlist_head *prog_list; - kind = bpf_attach_type_to_tramp(link->link.prog); + kind = bpf_attach_type_to_tramp(node->link->prog); if (kind == BPF_TRAMP_FSESSION) { prog_list = &tr->progs_hlist[BPF_TRAMP_FENTRY]; cnt++; @@ -838,21 +837,21 @@ static int bpf_trampoline_add_prog(struct bpf_trampoline *tr, } if (cnt >= BPF_MAX_TRAMP_LINKS) return -E2BIG; - if (!hlist_unhashed(&link->tramp_hlist)) + if (!hlist_unhashed(&node->tramp_hlist)) /* prog already linked */ return -EBUSY; - hlist_for_each_entry(link_exiting, prog_list, tramp_hlist) { - if (link_exiting->link.prog != link->link.prog) + hlist_for_each_entry(node_existing, prog_list, tramp_hlist) { + if (node_existing->link->prog != node->link->prog) continue; /* prog already linked */ return -EBUSY; } - hlist_add_head(&link->tramp_hlist, prog_list); + hlist_add_head(&node->tramp_hlist, prog_list); if (kind == BPF_TRAMP_FSESSION) { tr->progs_cnt[BPF_TRAMP_FENTRY]++; - fslink = container_of(link, struct bpf_fsession_link, link.link); - hlist_add_head(&fslink->fexit.tramp_hlist, &tr->progs_hlist[BPF_TRAMP_FEXIT]); + fslink = container_of(node, struct bpf_fsession_link, link.link.node); + hlist_add_head(&fslink->fexit.node.tramp_hlist, &tr->progs_hlist[BPF_TRAMP_FEXIT]); tr->progs_cnt[BPF_TRAMP_FEXIT]++; } else { tr->progs_cnt[kind]++; @@ -861,23 +860,23 @@ static int bpf_trampoline_add_prog(struct bpf_trampoline *tr, } static void bpf_trampoline_remove_prog(struct bpf_trampoline *tr, - struct bpf_tramp_link *link) + struct bpf_tramp_node *node) { struct bpf_fsession_link *fslink; enum bpf_tramp_prog_type kind; - kind = bpf_attach_type_to_tramp(link->link.prog); + kind = bpf_attach_type_to_tramp(node->link->prog); if (kind == BPF_TRAMP_FSESSION) { - fslink = container_of(link, struct bpf_fsession_link, link.link); - hlist_del_init(&fslink->fexit.tramp_hlist); + fslink = container_of(node, struct bpf_fsession_link, link.link.node); + hlist_del_init(&fslink->fexit.node.tramp_hlist); tr->progs_cnt[BPF_TRAMP_FEXIT]--; kind = BPF_TRAMP_FENTRY; } - hlist_del_init(&link->tramp_hlist); + hlist_del_init(&node->tramp_hlist); tr->progs_cnt[kind]--; } -static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, +static int __bpf_trampoline_link_prog(struct bpf_tramp_node *node, struct bpf_trampoline *tr, struct bpf_prog *tgt_prog, const struct bpf_trampoline_ops *ops, @@ -887,7 +886,7 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, int err = 0; int cnt = 0, i; - kind = bpf_attach_type_to_tramp(link->link.prog); + kind = bpf_attach_type_to_tramp(node->link->prog); if (tr->extension_prog) /* cannot attach fentry/fexit if extension prog is attached. * cannot overwrite extension prog either. @@ -904,33 +903,33 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, err = bpf_freplace_check_tgt_prog(tgt_prog); if (err) return err; - tr->extension_prog = link->link.prog; + tr->extension_prog = node->link->prog; return bpf_arch_text_poke(tr->func.addr, BPF_MOD_NOP, BPF_MOD_JUMP, NULL, - link->link.prog->bpf_func); + node->link->prog->bpf_func); } - err = bpf_trampoline_add_prog(tr, link, cnt); + err = bpf_trampoline_add_prog(tr, node, cnt); if (err) return err; err = bpf_trampoline_update(tr, true /* lock_direct_mutex */, ops, data); if (err) - bpf_trampoline_remove_prog(tr, link); + bpf_trampoline_remove_prog(tr, node); return err; } -int bpf_trampoline_link_prog(struct bpf_tramp_link *link, +int bpf_trampoline_link_prog(struct bpf_tramp_node *node, struct bpf_trampoline *tr, struct bpf_prog *tgt_prog) { int err; trampoline_lock(tr); - err = __bpf_trampoline_link_prog(link, tr, tgt_prog, &trampoline_ops, NULL); + err = __bpf_trampoline_link_prog(node, tr, tgt_prog, &trampoline_ops, NULL); trampoline_unlock(tr); return err; } -static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, +static int __bpf_trampoline_unlink_prog(struct bpf_tramp_node *node, struct bpf_trampoline *tr, struct bpf_prog *tgt_prog, const struct bpf_trampoline_ops *ops, @@ -939,7 +938,7 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, enum bpf_tramp_prog_type kind; int err; - kind = bpf_attach_type_to_tramp(link->link.prog); + kind = bpf_attach_type_to_tramp(node->link->prog); if (kind == BPF_TRAMP_REPLACE) { WARN_ON_ONCE(!tr->extension_prog); err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP, @@ -950,19 +949,19 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, tgt_prog->aux->is_extended = false; return err; } - bpf_trampoline_remove_prog(tr, link); + bpf_trampoline_remove_prog(tr, node); return bpf_trampoline_update(tr, true /* lock_direct_mutex */, ops, data); } /* bpf_trampoline_unlink_prog() should never fail. */ -int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, +int bpf_trampoline_unlink_prog(struct bpf_tramp_node *node, struct bpf_trampoline *tr, struct bpf_prog *tgt_prog) { int err; trampoline_lock(tr); - err = __bpf_trampoline_unlink_prog(link, tr, tgt_prog, &trampoline_ops, NULL); + err = __bpf_trampoline_unlink_prog(node, tr, tgt_prog, &trampoline_ops, NULL); trampoline_unlock(tr); return err; } @@ -977,7 +976,7 @@ static void bpf_shim_tramp_link_release(struct bpf_link *link) if (!shim_link->trampoline) return; - WARN_ON_ONCE(bpf_trampoline_unlink_prog(&shim_link->link, shim_link->trampoline, NULL)); + WARN_ON_ONCE(bpf_trampoline_unlink_prog(&shim_link->link.node, shim_link->trampoline, NULL)); bpf_trampoline_put(shim_link->trampoline); } @@ -1023,8 +1022,8 @@ static struct bpf_shim_tramp_link *cgroup_shim_alloc(const struct bpf_prog *prog p->type = BPF_PROG_TYPE_LSM; p->expected_attach_type = BPF_LSM_MAC; bpf_prog_inc(p); - bpf_link_init(&shim_link->link.link, BPF_LINK_TYPE_UNSPEC, - &bpf_shim_tramp_link_lops, p, attach_type); + bpf_tramp_link_init(&shim_link->link, BPF_LINK_TYPE_UNSPEC, + &bpf_shim_tramp_link_lops, p, attach_type, 0); bpf_cgroup_atype_get(p->aux->attach_btf_id, cgroup_atype); return shim_link; @@ -1033,15 +1032,15 @@ static struct bpf_shim_tramp_link *cgroup_shim_alloc(const struct bpf_prog *prog static struct bpf_shim_tramp_link *cgroup_shim_find(struct bpf_trampoline *tr, bpf_func_t bpf_func) { - struct bpf_tramp_link *link; + struct bpf_tramp_node *node; int kind; for (kind = 0; kind < BPF_TRAMP_MAX; kind++) { - hlist_for_each_entry(link, &tr->progs_hlist[kind], tramp_hlist) { - struct bpf_prog *p = link->link.prog; + hlist_for_each_entry(node, &tr->progs_hlist[kind], tramp_hlist) { + struct bpf_prog *p = node->link->prog; if (p->bpf_func == bpf_func) - return container_of(link, struct bpf_shim_tramp_link, link); + return container_of(node, struct bpf_shim_tramp_link, link.node); } } @@ -1091,7 +1090,7 @@ int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, goto err; } - err = __bpf_trampoline_link_prog(&shim_link->link, tr, NULL, &trampoline_ops, NULL); + err = __bpf_trampoline_link_prog(&shim_link->link.node, tr, NULL, &trampoline_ops, NULL); if (err) goto err; @@ -1406,7 +1405,7 @@ bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog) int __weak arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, const struct btf_func_model *m, u32 flags, - struct bpf_tramp_links *tlinks, + struct bpf_tramp_nodes *tnodes, void *func_addr) { return -ENOTSUPP; @@ -1440,7 +1439,7 @@ int __weak arch_protect_bpf_trampoline(void *image, unsigned int size) } int __weak arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, - struct bpf_tramp_links *tlinks, void *func_addr) + struct bpf_tramp_nodes *tnodes, void *func_addr) { return -ENOTSUPP; } -- cgit v1.2.3 From 880db5d4abb29e931d82b9feefb4382f76fcf9e5 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Sat, 6 Jun 2026 14:39:34 +0200 Subject: bpf: Factor fsession link to use struct bpf_tramp_node Now that we split trampoline attachment object (bpf_tramp_node) from the link object (bpf_tramp_link) we can use bpf_tramp_node as fsession's fexit attachment object and get rid of the bpf_fsession_link object. Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260606123955.345967-10-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/bpf/syscall.c | 21 ++++++--------------- kernel/bpf/trampoline.c | 12 ++++++------ 2 files changed, 12 insertions(+), 21 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index fd69fdb9290b..0cfc8bcb3dc9 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -3730,21 +3730,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, key = bpf_trampoline_compute_key(tgt_prog, NULL, btf_id); } - if (prog->expected_attach_type == BPF_TRACE_FSESSION) { - struct bpf_fsession_link *fslink; - - fslink = kzalloc_obj(*fslink, GFP_USER); - if (fslink) { - bpf_tramp_link_init(&fslink->fexit, BPF_LINK_TYPE_TRACING, - &bpf_tracing_link_lops, prog, attach_type, - bpf_cookie); - link = &fslink->link; - } else { - link = NULL; - } - } else { - link = kzalloc_obj(*link, GFP_USER); - } + link = kzalloc_obj(*link, GFP_USER); if (!link) { err = -ENOMEM; goto out_put_prog; @@ -3752,6 +3738,11 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, bpf_tramp_link_init(&link->link, BPF_LINK_TYPE_TRACING, &bpf_tracing_link_lops, prog, attach_type, bpf_cookie); + if (prog->expected_attach_type == BPF_TRACE_FSESSION) { + link->fexit.link = &link->link.link; + link->fexit.cookie = bpf_cookie; + } + mutex_lock(&prog->aux->dst_mutex); /* There are a few possible cases here: diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 6a45c09fc0d8..5776d2b8e36e 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -823,7 +823,7 @@ static int bpf_trampoline_add_prog(struct bpf_trampoline *tr, struct bpf_tramp_node *node, int cnt) { - struct bpf_fsession_link *fslink = NULL; + struct bpf_tracing_link *tr_link = NULL; enum bpf_tramp_prog_type kind; struct bpf_tramp_node *node_existing; struct hlist_head *prog_list; @@ -850,8 +850,8 @@ static int bpf_trampoline_add_prog(struct bpf_trampoline *tr, hlist_add_head(&node->tramp_hlist, prog_list); if (kind == BPF_TRAMP_FSESSION) { tr->progs_cnt[BPF_TRAMP_FENTRY]++; - fslink = container_of(node, struct bpf_fsession_link, link.link.node); - hlist_add_head(&fslink->fexit.node.tramp_hlist, &tr->progs_hlist[BPF_TRAMP_FEXIT]); + tr_link = container_of(node, struct bpf_tracing_link, link.node); + hlist_add_head(&tr_link->fexit.tramp_hlist, &tr->progs_hlist[BPF_TRAMP_FEXIT]); tr->progs_cnt[BPF_TRAMP_FEXIT]++; } else { tr->progs_cnt[kind]++; @@ -862,13 +862,13 @@ static int bpf_trampoline_add_prog(struct bpf_trampoline *tr, static void bpf_trampoline_remove_prog(struct bpf_trampoline *tr, struct bpf_tramp_node *node) { - struct bpf_fsession_link *fslink; + struct bpf_tracing_link *tr_link; enum bpf_tramp_prog_type kind; kind = bpf_attach_type_to_tramp(node->link->prog); if (kind == BPF_TRAMP_FSESSION) { - fslink = container_of(node, struct bpf_fsession_link, link.link.node); - hlist_del_init(&fslink->fexit.node.tramp_hlist); + tr_link = container_of(node, struct bpf_tracing_link, link.node); + hlist_del_init(&tr_link->fexit.tramp_hlist); tr->progs_cnt[BPF_TRAMP_FEXIT]--; kind = BPF_TRAMP_FENTRY; } -- cgit v1.2.3 From d14e6b4346bf397eca7cb5f4b7b0b8054be632d8 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Sat, 6 Jun 2026 14:39:35 +0200 Subject: bpf: Add multi tracing attach types Adding new program attach types multi tracing attachment: BPF_TRACE_FENTRY_MULTI BPF_TRACE_FEXIT_MULTI and their base support in verifier code. Programs with such attach type will use specific link attachment interface coming in following changes. This was suggested by Andrii some (long) time ago and turned out to be easier than having special program flag for that. Bpf programs with such types have 'bpf_multi_func' function set as their attach_btf_id and keep module reference when it's specified by attach_prog_fd. They are also accepted as sleepable programs during verification, and the real validation for specific BTF_IDs/functions will happen during the multi link attachment in following changes. Suggested-by: Andrii Nakryiko Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260606123955.345967-11-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/bpf/fixups.c | 1 + kernel/bpf/syscall.c | 28 ++++++++++++++++++++++++---- kernel/bpf/trampoline.c | 5 ++++- kernel/bpf/verifier.c | 40 +++++++++++++++++++++++++++++++++++++++- 4 files changed, 68 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 5aa3f7d99ac9..0cf9735929f5 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -2186,6 +2186,7 @@ patch_map_ops_generic: insn->imm == BPF_FUNC_get_func_ret) { if (eatype == BPF_TRACE_FEXIT || eatype == BPF_TRACE_FSESSION || + eatype == BPF_TRACE_FEXIT_MULTI || eatype == BPF_MODIFY_RETURN) { /* Load nr_args from ctx - 8 */ insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 0cfc8bcb3dc9..efdd6639a598 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include @@ -2719,7 +2720,8 @@ static int bpf_prog_load_check_attach(enum bpf_prog_type prog_type, enum bpf_attach_type expected_attach_type, struct btf *attach_btf, u32 btf_id, - struct bpf_prog *dst_prog) + struct bpf_prog *dst_prog, + bool multi_func) { if (btf_id) { if (btf_id > BTF_MAX_TYPE) @@ -2739,6 +2741,14 @@ bpf_prog_load_check_attach(enum bpf_prog_type prog_type, } } + if (multi_func) { + if (prog_type != BPF_PROG_TYPE_TRACING) + return -EINVAL; + if (!attach_btf || btf_id) + return -EINVAL; + return 0; + } + if (attach_btf && (!btf_id || dst_prog)) return -EINVAL; @@ -2946,6 +2956,11 @@ static int bpf_prog_mark_insn_arrays_ready(struct bpf_prog *prog) return 0; } +extern int bpf_multi_func(void); +int __init __used bpf_multi_func(void) { return 0; } + +BTF_ID_LIST_GLOBAL_SINGLE(bpf_multi_func_btf_id, func, bpf_multi_func) + /* last field in 'union bpf_attr' used by this command */ #define BPF_PROG_LOAD_LAST_FIELD keyring_id @@ -2958,6 +2973,7 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at bool bpf_cap; int err; char license[128]; + bool multi_func; if (CHECK_ATTR(BPF_PROG_LOAD)) return -EINVAL; @@ -3024,6 +3040,8 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at if (is_perfmon_prog_type(type) && !bpf_token_capable(token, CAP_PERFMON)) goto put_token; + multi_func = is_tracing_multi(attr->expected_attach_type); + /* attach_prog_fd/attach_btf_obj_fd can specify fd of either bpf_prog * or btf, we need to check which one it is */ @@ -3045,7 +3063,7 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at goto put_token; } } - } else if (attr->attach_btf_id) { + } else if (attr->attach_btf_id || multi_func) { /* fall back to vmlinux BTF, if BTF type ID is specified */ attach_btf = bpf_get_btf_vmlinux(); if (IS_ERR(attach_btf)) { @@ -3061,7 +3079,7 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at if (bpf_prog_load_check_attach(type, attr->expected_attach_type, attach_btf, attr->attach_btf_id, - dst_prog)) { + dst_prog, multi_func)) { if (dst_prog) bpf_prog_put(dst_prog); if (attach_btf) @@ -3084,7 +3102,7 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at prog->expected_attach_type = attr->expected_attach_type; prog->sleepable = !!(attr->prog_flags & BPF_F_SLEEPABLE); prog->aux->attach_btf = attach_btf; - prog->aux->attach_btf_id = attr->attach_btf_id; + prog->aux->attach_btf_id = multi_func ? bpf_multi_func_btf_id[0] : attr->attach_btf_id; prog->aux->dst_prog = dst_prog; prog->aux->dev_bound = !!attr->prog_ifindex; prog->aux->xdp_has_frags = attr->prog_flags & BPF_F_XDP_HAS_FRAGS; @@ -4480,6 +4498,8 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type) case BPF_TRACE_FENTRY: case BPF_TRACE_FEXIT: case BPF_TRACE_FSESSION: + case BPF_TRACE_FENTRY_MULTI: + case BPF_TRACE_FEXIT_MULTI: case BPF_MODIFY_RETURN: return BPF_PROG_TYPE_TRACING; case BPF_LSM_MAC: diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 5776d2b8e36e..ae7e4fdfe2a3 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -182,7 +182,8 @@ bool bpf_prog_has_trampoline(const struct bpf_prog *prog) switch (ptype) { case BPF_PROG_TYPE_TRACING: if (eatype == BPF_TRACE_FENTRY || eatype == BPF_TRACE_FEXIT || - eatype == BPF_MODIFY_RETURN || eatype == BPF_TRACE_FSESSION) + eatype == BPF_MODIFY_RETURN || eatype == BPF_TRACE_FSESSION || + eatype == BPF_TRACE_FENTRY_MULTI || eatype == BPF_TRACE_FEXIT_MULTI) return true; return false; case BPF_PROG_TYPE_LSM: @@ -781,10 +782,12 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog) { switch (prog->expected_attach_type) { case BPF_TRACE_FENTRY: + case BPF_TRACE_FENTRY_MULTI: return BPF_TRAMP_FENTRY; case BPF_MODIFY_RETURN: return BPF_TRAMP_MODIFY_RETURN; case BPF_TRACE_FEXIT: + case BPF_TRACE_FEXIT_MULTI: return BPF_TRAMP_FEXIT; case BPF_TRACE_FSESSION: return BPF_TRAMP_FSESSION; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 926ff63a0b61..0e593f3335e9 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -16382,6 +16382,8 @@ static bool return_retval_range(struct bpf_verifier_env *env, struct bpf_retval_ case BPF_TRACE_FENTRY: case BPF_TRACE_FEXIT: case BPF_TRACE_FSESSION: + case BPF_TRACE_FENTRY_MULTI: + case BPF_TRACE_FEXIT_MULTI: *range = retval_range(0, 0); break; case BPF_TRACE_RAW_TP: @@ -18772,6 +18774,11 @@ static int check_attach_modify_return(unsigned long addr, const char *func_name) #endif /* CONFIG_FUNCTION_ERROR_INJECTION */ +static bool is_tracing_multi_id(const struct bpf_prog *prog, u32 btf_id) +{ + return is_tracing_multi(prog->expected_attach_type) && bpf_multi_func_btf_id[0] == btf_id; +} + int bpf_check_attach_target(struct bpf_verifier_log *log, const struct bpf_prog *prog, const struct bpf_prog *tgt_prog, @@ -18894,6 +18901,8 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, prog_extension && (tgt_prog->expected_attach_type == BPF_TRACE_FENTRY || tgt_prog->expected_attach_type == BPF_TRACE_FEXIT || + tgt_prog->expected_attach_type == BPF_TRACE_FENTRY_MULTI || + tgt_prog->expected_attach_type == BPF_TRACE_FEXIT_MULTI || tgt_prog->expected_attach_type == BPF_TRACE_FSESSION)) { /* Program extensions can extend all program types * except fentry/fexit. The reason is the following. @@ -19000,6 +19009,8 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, case BPF_TRACE_FENTRY: case BPF_TRACE_FEXIT: case BPF_TRACE_FSESSION: + case BPF_TRACE_FENTRY_MULTI: + case BPF_TRACE_FEXIT_MULTI: if (prog->expected_attach_type == BPF_TRACE_FSESSION && !bpf_jit_supports_fsession()) { bpf_log(log, "JIT does not support fsession\n"); @@ -19029,7 +19040,18 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, if (ret < 0) return ret; - if (tgt_prog) { + /* + * *.multi programs don't need an address during program + * verification, we just take the module ref if needed. + */ + if (is_tracing_multi_id(prog, btf_id)) { + if (btf_is_module(btf)) { + mod = btf_try_get_module(btf); + if (!mod) + return -ENOENT; + } + addr = 0; + } else if (tgt_prog) { if (subprog == 0) addr = (long) tgt_prog->bpf_func; else @@ -19057,6 +19079,12 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, ret = -EINVAL; switch (prog->type) { case BPF_PROG_TYPE_TRACING: + /* *.multi sleepable programs will pass initial sleepable check, + * the actual attached btf ids are checked later during the link + * attachment. + */ + if (is_tracing_multi_id(prog, btf_id)) + ret = 0; if (!check_attach_sleepable(btf_id, addr, tname)) ret = 0; /* fentry/fexit/fmod_ret progs can also be sleepable if they are @@ -19167,6 +19195,8 @@ static bool can_be_sleepable(struct bpf_prog *prog) case BPF_TRACE_ITER: case BPF_TRACE_FSESSION: case BPF_TRACE_RAW_TP: + case BPF_TRACE_FENTRY_MULTI: + case BPF_TRACE_FEXIT_MULTI: return true; default: return false; @@ -19260,6 +19290,14 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) return -EINVAL; } + /* + * We don't get trampoline for tracing_multi programs at this point, + * it's done when tracing_multi link is created. + */ + if (prog->type == BPF_PROG_TYPE_TRACING && + is_tracing_multi(prog->expected_attach_type)) + return 0; + key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id); tr = bpf_trampoline_get(key, &tgt_info); if (!tr) -- cgit v1.2.3 From bd06659d3b8abe7a79ae473209ee89bf3a23af36 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Sat, 6 Jun 2026 14:39:36 +0200 Subject: bpf: Move sleepable verification code to btf_id_allow_sleepable Move sleepable verification code to btf_id_allow_sleepable function. It will be used in following changes. Adding code to retrieve type's name instead of passing it from bpf_check_attach_target function, because this function will be called from another place in following changes and it's easier to retrieve the name directly in here. Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260606123955.345967-12-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 82 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 50 insertions(+), 32 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 0e593f3335e9..df21592fc560 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -18779,6 +18779,55 @@ static bool is_tracing_multi_id(const struct bpf_prog *prog, u32 btf_id) return is_tracing_multi(prog->expected_attach_type) && bpf_multi_func_btf_id[0] == btf_id; } +static int btf_id_allow_sleepable(u32 btf_id, unsigned long addr, const struct bpf_prog *prog, + const struct btf *btf) +{ + const struct btf_type *t; + const char *tname; + + switch (prog->type) { + case BPF_PROG_TYPE_TRACING: + t = btf_type_by_id(btf, btf_id); + if (!t) + return -EINVAL; + tname = btf_name_by_offset(btf, t->name_off); + if (!tname) + return -EINVAL; + + /* + * *.multi sleepable programs will pass initial sleepable check, + * the actual attached btf ids are checked later during the link + * attachment. + */ + if (is_tracing_multi_id(prog, btf_id)) + return 0; + if (!check_attach_sleepable(btf_id, addr, tname)) + return 0; + /* + * fentry/fexit/fmod_ret progs can also be sleepable if they are + * in the fmodret id set with the KF_SLEEPABLE flag. + */ + else { + u32 *flags = btf_kfunc_is_modify_return(btf, btf_id, prog); + + if (flags && (*flags & KF_SLEEPABLE)) + return 0; + } + break; + case BPF_PROG_TYPE_LSM: + /* + * LSM progs check that they are attached to bpf_lsm_*() funcs. + * Only some of them are sleepable. + */ + if (bpf_lsm_is_sleepable_hook(btf_id)) + return 0; + break; + default: + break; + } + return -EINVAL; +} + int bpf_check_attach_target(struct bpf_verifier_log *log, const struct bpf_prog *prog, const struct bpf_prog *tgt_prog, @@ -19076,38 +19125,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, } if (prog->sleepable) { - ret = -EINVAL; - switch (prog->type) { - case BPF_PROG_TYPE_TRACING: - /* *.multi sleepable programs will pass initial sleepable check, - * the actual attached btf ids are checked later during the link - * attachment. - */ - if (is_tracing_multi_id(prog, btf_id)) - ret = 0; - if (!check_attach_sleepable(btf_id, addr, tname)) - ret = 0; - /* fentry/fexit/fmod_ret progs can also be sleepable if they are - * in the fmodret id set with the KF_SLEEPABLE flag. - */ - else { - u32 *flags = btf_kfunc_is_modify_return(btf, btf_id, - prog); - - if (flags && (*flags & KF_SLEEPABLE)) - ret = 0; - } - break; - case BPF_PROG_TYPE_LSM: - /* LSM progs check that they are attached to bpf_lsm_*() funcs. - * Only some of them are sleepable. - */ - if (bpf_lsm_is_sleepable_hook(btf_id)) - ret = 0; - break; - default: - break; - } + ret = btf_id_allow_sleepable(btf_id, addr, prog, btf); if (ret) { module_put(mod); bpf_log(log, "%s is not sleepable\n", tname); -- cgit v1.2.3 From aef4dfa790b22d8052cfb78044eadbe03c876c39 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Sat, 6 Jun 2026 14:39:37 +0200 Subject: bpf: Add bpf_trampoline_multi_attach/detach functions Adding bpf_trampoline_multi_attach/detach functions that allows to attach/detach tracing program to multiple functions/trampolines. The attachment is defined with bpf_program and array of BTF ids of functions to attach the bpf program to. Adding bpf_tracing_multi_link object that holds all the attached trampolines and is initialized in attach and used in detach. The attachment allocates or uses currently existing trampoline for each function to attach and links it with the bpf program. The attach works as follows: - we get all the needed trampolines - lock them and add the bpf program to each (__bpf_trampoline_link_prog) - the trampoline_multi_ops passed in __bpf_trampoline_link_prog gathers ftrace_hash (ip -> trampoline) objects - we call update_ftrace_direct_add/mod to update needed locations - we unlock all the trampolines The detach works as follows: - we lock all the needed trampolines - remove the program from each (__bpf_trampoline_unlink_prog) - the trampoline_multi_ops passed in __bpf_trampoline_unlink_prog gathers ftrace_hash (ip -> trampoline) objects - we call update_ftrace_direct_del/mod to update needed locations - we unlock and put all the trampolines We store the old image/flags in the trampoline before the update and use it in case we need to rollback the attachment. We keep the ftrace_hash objects allocated during attach in the link so they can be used for detach as well. Adding trampoline_(un)lock_all functions to (un)lock all trampolines to gate the tracing_multi attachment. Note this is supported only for archs (x86_64) with ftrace direct and have single ops support. CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS && CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS It also needs CONFIG_BPF_SYSCALL enabled. Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260606123955.345967-13-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/bpf/trampoline.c | 271 ++++++++++++++++++++++++++++++++++++++++++++++++ kernel/bpf/verifier.c | 55 ++++++++++ 2 files changed, 326 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index ae7e4fdfe2a3..957e5d7f9554 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -1447,6 +1447,277 @@ int __weak arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, return -ENOTSUPP; } +#if defined(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS) && \ + defined(CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS) && \ + defined(CONFIG_BPF_SYSCALL) + +static void trampoline_lock_all(void) +{ + int i; + + for (i = 0; i < TRAMPOLINE_LOCKS_TABLE_SIZE; i++) + mutex_lock(&trampoline_locks[i].mutex); +} + +static void trampoline_unlock_all(void) +{ + int i; + + for (i = 0; i < TRAMPOLINE_LOCKS_TABLE_SIZE; i++) + mutex_unlock(&trampoline_locks[i].mutex); +} + +static void remove_tracing_multi_data(struct bpf_tracing_multi_data *data) +{ + ftrace_hash_remove(data->reg); + ftrace_hash_remove(data->unreg); + ftrace_hash_remove(data->modify); +} + +static void clear_tracing_multi_data(struct bpf_tracing_multi_data *data) +{ + remove_tracing_multi_data(data); + + free_ftrace_hash(data->reg); + free_ftrace_hash(data->unreg); + free_ftrace_hash(data->modify); +} + +static int init_tracing_multi_data(struct bpf_tracing_multi_data *data) +{ + data->reg = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS); + data->unreg = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS); + data->modify = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS); + + if (!data->reg || !data->unreg || !data->modify) { + clear_tracing_multi_data(data); + return -ENOMEM; + } + return 0; +} + +static void ftrace_hash_add(struct ftrace_hash *hash, struct ftrace_func_entry *entry, + unsigned long ip, unsigned long direct) +{ + entry->ip = ip; + entry->direct = direct; + add_ftrace_hash_entry(hash, entry); +} + +static int register_fentry_multi(struct bpf_trampoline *tr, struct bpf_tramp_image *im, void *ptr) +{ + unsigned long addr = (unsigned long) im->image; + unsigned long ip = ftrace_location(tr->ip); + struct bpf_tracing_multi_data *data = ptr; + + if (bpf_trampoline_use_jmp(tr->flags)) + addr = ftrace_jmp_set(addr); + + ftrace_hash_add(data->reg, data->entry, ip, addr); + tr->cur_image = im; + return 0; +} + +static int unregister_fentry_multi(struct bpf_trampoline *tr, u32 orig_flags, void *ptr) +{ + unsigned long addr = (unsigned long) tr->cur_image->image; + unsigned long ip = ftrace_location(tr->ip); + struct bpf_tracing_multi_data *data = ptr; + + if (bpf_trampoline_use_jmp(tr->flags)) + addr = ftrace_jmp_set(addr); + + ftrace_hash_add(data->unreg, data->entry, ip, addr); + tr->cur_image = NULL; + return 0; +} + +static int modify_fentry_multi(struct bpf_trampoline *tr, u32 orig_flags, struct bpf_tramp_image *im, + bool lock_direct_mutex, void *ptr) +{ + unsigned long addr = (unsigned long) im->image; + unsigned long ip = ftrace_location(tr->ip); + struct bpf_tracing_multi_data *data = ptr; + + if (bpf_trampoline_use_jmp(tr->flags)) + addr = ftrace_jmp_set(addr); + + ftrace_hash_add(data->modify, data->entry, ip, addr); + tr->cur_image = im; + return 0; +} + +static const struct bpf_trampoline_ops trampoline_multi_ops = { + .register_fentry = register_fentry_multi, + .unregister_fentry = unregister_fentry_multi, + .modify_fentry = modify_fentry_multi, +}; + +static void bpf_trampoline_multi_attach_init(struct bpf_trampoline *tr) +{ + tr->multi_attach.old_image = tr->cur_image; + tr->multi_attach.old_flags = tr->flags; +} + +static void bpf_trampoline_multi_attach_free(struct bpf_trampoline *tr) +{ + if (tr->multi_attach.old_image) + bpf_tramp_image_put(tr->multi_attach.old_image); + + tr->multi_attach.old_image = NULL; + tr->multi_attach.old_flags = 0; +} + +static void bpf_trampoline_multi_attach_rollback(struct bpf_trampoline *tr) +{ + if (tr->cur_image) + bpf_tramp_image_put(tr->cur_image); + tr->cur_image = tr->multi_attach.old_image; + tr->flags = tr->multi_attach.old_flags; + + tr->multi_attach.old_image = NULL; + tr->multi_attach.old_flags = 0; +} + +#define for_each_mnode_cnt(mnode, link, cnt) \ + for (i = 0, mnode = &link->nodes[i]; i < cnt; i++, mnode = &link->nodes[i]) + +#define for_each_mnode(mnode, link) \ + for_each_mnode_cnt(mnode, link, link->nodes_cnt) + +int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids, + struct bpf_tracing_multi_link *link) +{ + struct bpf_tracing_multi_data *data = &link->data; + struct bpf_attach_target_info tgt_info = {}; + struct btf *btf = prog->aux->attach_btf; + struct bpf_tracing_multi_node *mnode; + struct bpf_trampoline *tr; + int i, err, rollback_cnt; + u64 key; + + for_each_mnode(mnode, link) { + rollback_cnt = i; + + err = bpf_check_attach_btf_id_multi(btf, prog, ids[i], &tgt_info); + if (err) + goto rollback_put; + + key = bpf_trampoline_compute_key(NULL, btf, ids[i]); + + tr = bpf_trampoline_get(key, &tgt_info); + if (!tr) { + err = -ENOMEM; + goto rollback_put; + } + + mnode->trampoline = tr; + mnode->node.link = &link->link; + + cond_resched(); + } + + err = init_tracing_multi_data(data); + if (err) { + rollback_cnt = link->nodes_cnt; + goto rollback_put; + } + + trampoline_lock_all(); + + for_each_mnode(mnode, link) { + bpf_trampoline_multi_attach_init(mnode->trampoline); + + data->entry = &mnode->entry; + err = __bpf_trampoline_link_prog(&mnode->node, mnode->trampoline, NULL, + &trampoline_multi_ops, data); + if (err) { + rollback_cnt = i; + goto rollback_unlink; + } + } + + rollback_cnt = link->nodes_cnt; + if (ftrace_hash_count(data->reg)) { + err = update_ftrace_direct_add(&direct_ops, data->reg); + if (err) + goto rollback_unlink; + } + + if (ftrace_hash_count(data->modify)) { + err = update_ftrace_direct_mod(&direct_ops, data->modify, true); + if (err) { + if (ftrace_hash_count(data->reg)) + WARN_ON_ONCE(update_ftrace_direct_del(&direct_ops, data->reg)); + goto rollback_unlink; + } + } + + for_each_mnode(mnode, link) + bpf_trampoline_multi_attach_free(mnode->trampoline); + + trampoline_unlock_all(); + + remove_tracing_multi_data(data); + return 0; + +rollback_unlink: + for_each_mnode_cnt(mnode, link, rollback_cnt) { + bpf_trampoline_remove_prog(mnode->trampoline, &mnode->node); + bpf_trampoline_multi_attach_rollback(mnode->trampoline); + } + + trampoline_unlock_all(); + + clear_tracing_multi_data(data); + rollback_cnt = link->nodes_cnt; + +rollback_put: + for_each_mnode_cnt(mnode, link, rollback_cnt) + bpf_trampoline_put(mnode->trampoline); + + return err; +} + +int bpf_trampoline_multi_detach(struct bpf_prog *prog, struct bpf_tracing_multi_link *link) +{ + struct bpf_tracing_multi_data *data = &link->data; + struct bpf_tracing_multi_node *mnode; + int i; + + trampoline_lock_all(); + + for_each_mnode(mnode, link) { + data->entry = &mnode->entry; + bpf_trampoline_multi_attach_init(mnode->trampoline); + WARN_ON_ONCE(__bpf_trampoline_unlink_prog(&mnode->node, mnode->trampoline, + NULL, &trampoline_multi_ops, data)); + } + + if (ftrace_hash_count(data->unreg)) + WARN_ON_ONCE(update_ftrace_direct_del(&direct_ops, data->unreg)); + if (ftrace_hash_count(data->modify)) + WARN_ON_ONCE(update_ftrace_direct_mod(&direct_ops, data->modify, true)); + + for_each_mnode(mnode, link) + bpf_trampoline_multi_attach_free(mnode->trampoline); + + trampoline_unlock_all(); + + for_each_mnode(mnode, link) + bpf_trampoline_put(mnode->trampoline); + + clear_tracing_multi_data(data); + return 0; +} + +#undef for_each_mnode_cnt +#undef for_each_mnode + +#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS && + CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS && + CONFIG_BPF_SYSCALL */ + static int __init init_trampolines(void) { int i; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index df21592fc560..5c594047ff0a 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -19328,6 +19328,61 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) return 0; } +int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 btf_id, + struct bpf_attach_target_info *tgt_info) +{ + const struct btf_type *t; + unsigned long addr; + const char *tname; + int err; + + if (!btf_id || !btf) + return -EINVAL; + + /* Check noreturn attachment. */ + if (prog->expected_attach_type == BPF_TRACE_FEXIT_MULTI && + btf_id_set_contains(&noreturn_deny, btf_id)) + return -EINVAL; + /* Check denied attachment. */ + if (btf_id_set_contains(&btf_id_deny, btf_id)) + return -EINVAL; + + /* Check and get function target data. */ + t = btf_type_by_id(btf, btf_id); + if (!t) + return -EINVAL; + tname = btf_name_by_offset(btf, t->name_off); + if (!tname) + return -EINVAL; + if (!btf_type_is_func(t)) + return -EINVAL; + t = btf_type_by_id(btf, t->type); + if (!btf_type_is_func_proto(t)) + return -EINVAL; + err = btf_distill_func_proto(NULL, btf, t, tname, &tgt_info->fmodel); + if (err < 0) + return err; + if (btf_is_module(btf)) { + /* The bpf program already holds reference to module. */ + if (WARN_ON_ONCE(!prog->aux->mod)) + return -EINVAL; + addr = find_kallsyms_symbol_value(prog->aux->mod, tname); + } else { + addr = kallsyms_lookup_name(tname); + } + if (!addr || !ftrace_location(addr)) + return -ENOENT; + + /* Check sleepable program attachment. */ + if (prog->sleepable) { + err = btf_id_allow_sleepable(btf_id, addr, prog, btf); + if (err) + return err; + } + tgt_info->tgt_addr = addr; + return 0; +} + struct btf *bpf_get_btf_vmlinux(void) { if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { -- cgit v1.2.3 From c1d32dea5d4694c1a6c14d1d1c3192d0e18ffc7b Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Sat, 6 Jun 2026 14:39:38 +0200 Subject: bpf: Add support for tracing multi link Adding new link to allow to attach program to multiple function BTF IDs. The link is represented by struct bpf_tracing_multi_link. To configure the link, new fields are added to bpf_attr::link_create to pass array of BTF IDs; struct { __aligned_u64 ids; __u32 cnt; } tracing_multi; Each BTF ID represents function (BTF_KIND_FUNC) that the link will attach bpf program to. We use previously added bpf_trampoline_multi_attach/detach functions to attach/detach the link. The linkinfo/fdinfo callbacks will be implemented in following changes. Note this is supported only for archs (x86_64) with ftrace direct and have single ops support. CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS && CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS Note using sort_r (instead of plain sort) in check_dup_ids, because we will use the swap callback in following changes. Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260606123955.345967-14-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/bpf/syscall.c | 2 + kernel/trace/bpf_trace.c | 130 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index efdd6639a598..d551b9da0cfb 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -5885,6 +5885,8 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr) ret = bpf_iter_link_attach(attr, uattr, prog); else if (prog->expected_attach_type == BPF_LSM_CGROUP) ret = cgroup_bpf_link_attach(attr, prog); + else if (is_tracing_multi(prog->expected_attach_type)) + ret = bpf_tracing_multi_attach(prog, attr); else ret = bpf_tracing_prog_attach(prog, attr->link_create.target_fd, diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index d853f97bd154..9e3cb547651e 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -42,6 +42,7 @@ #define MAX_UPROBE_MULTI_CNT (1U << 20) #define MAX_KPROBE_MULTI_CNT (1U << 20) +#define MAX_TRACING_MULTI_CNT (1U << 20) #ifdef CONFIG_MODULES struct bpf_trace_module { @@ -3641,3 +3642,132 @@ __bpf_kfunc int bpf_copy_from_user_task_str_dynptr(const struct bpf_dynptr *dptr } __bpf_kfunc_end_defs(); + +#if defined(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS) && \ + defined(CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS) + +static void bpf_tracing_multi_link_release(struct bpf_link *link) +{ + struct bpf_tracing_multi_link *tr_link = + container_of(link, struct bpf_tracing_multi_link, link); + + WARN_ON_ONCE(bpf_trampoline_multi_detach(link->prog, tr_link)); +} + +static void bpf_tracing_multi_link_dealloc(struct bpf_link *link) +{ + struct bpf_tracing_multi_link *tr_link = + container_of(link, struct bpf_tracing_multi_link, link); + + kvfree(tr_link); +} + +static const struct bpf_link_ops bpf_tracing_multi_link_lops = { + .release = bpf_tracing_multi_link_release, + .dealloc_deferred = bpf_tracing_multi_link_dealloc, +}; + +static int ids_cmp_r(const void *pa, const void *pb, const void *priv __maybe_unused) +{ + u32 a = *(u32 *) pa; + u32 b = *(u32 *) pb; + + return (a > b) - (a < b); +} + +static void ids_swap_r(void *a, void *b, int size __maybe_unused, + const void *priv __maybe_unused) +{ + u32 *id_a = a, *id_b = b; + + swap(*id_a, *id_b); +} + +static int check_dup_ids(u32 *ids, u32 cnt) +{ + int err = 0; + + /* + * Sort ids array (together with cookies array if defined) + * and check it for duplicates. The ids and cookies arrays + * are left sorted. + */ + sort_r_nonatomic(ids, cnt, sizeof(ids[0]), ids_cmp_r, ids_swap_r, NULL); + + for (int i = 1; i < cnt; i++) { + if (ids[i] == ids[i - 1]) { + err = -EINVAL; + break; + } + } + return err; +} + +int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) +{ + struct bpf_tracing_multi_link *link = NULL; + struct bpf_link_primer link_primer; + u32 cnt, *ids = NULL; + u32 __user *uids; + int err; + + uids = u64_to_user_ptr(attr->link_create.tracing_multi.ids); + cnt = attr->link_create.tracing_multi.cnt; + + if (!cnt || !uids) + return -EINVAL; + if (cnt > MAX_TRACING_MULTI_CNT) + return -E2BIG; + if (attr->link_create.flags || attr->link_create.target_fd) + return -EINVAL; + + ids = kvmalloc_objs(*ids, cnt); + if (!ids) + return -ENOMEM; + + if (copy_from_user(ids, uids, cnt * sizeof(*ids))) { + err = -EFAULT; + goto error; + } + + err = check_dup_ids(ids, cnt); + if (err) + goto error; + + link = kvzalloc_flex(*link, nodes, cnt); + if (!link) { + err = -ENOMEM; + goto error; + } + + bpf_link_init(&link->link, BPF_LINK_TYPE_TRACING_MULTI, + &bpf_tracing_multi_link_lops, prog, prog->expected_attach_type); + + err = bpf_link_prime(&link->link, &link_primer); + if (err) + goto error; + + link->nodes_cnt = cnt; + + err = bpf_trampoline_multi_attach(prog, ids, link); + kvfree(ids); + if (err) { + bpf_link_cleanup(&link_primer); + return err; + } + return bpf_link_settle(&link_primer); + +error: + kvfree(ids); + kvfree(link); + return err; +} + +#else + +int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) +{ + return -EOPNOTSUPP; +} + +#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS && CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS */ -- cgit v1.2.3 From 46b42af27d40021a97c147d23de8cb29eb5020df Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Sat, 6 Jun 2026 14:39:39 +0200 Subject: bpf: Add support for tracing_multi link cookies Add support to specify cookies for tracing_multi link. Cookies are provided in array where each value is paired with provided BTF ID value with the same array index. Such cookie can be retrieved by bpf program with bpf_get_attach_cookie helper call. We need to sort cookies array together with ids array in check_dup_ids, to keep the id->cookie relation. Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260606123955.345967-15-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/bpf/trampoline.c | 1 + kernel/trace/bpf_trace.c | 37 +++++++++++++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 957e5d7f9554..a3537fda50cf 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -1613,6 +1613,7 @@ int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids, mnode->trampoline = tr; mnode->node.link = &link->link; + mnode->node.cookie = link->cookies ? link->cookies[i] : 0; cond_resched(); } diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 9e3cb547651e..e33492739ed1 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -3659,6 +3659,7 @@ static void bpf_tracing_multi_link_dealloc(struct bpf_link *link) struct bpf_tracing_multi_link *tr_link = container_of(link, struct bpf_tracing_multi_link, link); + kvfree(tr_link->cookies); kvfree(tr_link); } @@ -3678,13 +3679,24 @@ static int ids_cmp_r(const void *pa, const void *pb, const void *priv __maybe_un static void ids_swap_r(void *a, void *b, int size __maybe_unused, const void *priv __maybe_unused) { - u32 *id_a = a, *id_b = b; + u64 *cookie_a, *cookie_b, *cookies; + u32 *id_a = a, *id_b = b, *ids; + void **data = (void **) priv; + ids = data[0]; + cookies = data[1]; + + if (cookies) { + cookie_a = cookies + (id_a - ids); + cookie_b = cookies + (id_b - ids); + swap(*cookie_a, *cookie_b); + } swap(*id_a, *id_b); } -static int check_dup_ids(u32 *ids, u32 cnt) +static int check_dup_ids(u32 *ids, u64 *cookies, u32 cnt) { + void *data[2] = { ids, cookies }; int err = 0; /* @@ -3692,7 +3704,7 @@ static int check_dup_ids(u32 *ids, u32 cnt) * and check it for duplicates. The ids and cookies arrays * are left sorted. */ - sort_r_nonatomic(ids, cnt, sizeof(ids[0]), ids_cmp_r, ids_swap_r, NULL); + sort_r_nonatomic(ids, cnt, sizeof(ids[0]), ids_cmp_r, ids_swap_r, data); for (int i = 1; i < cnt; i++) { if (ids[i] == ids[i - 1]) { @@ -3708,6 +3720,8 @@ int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) struct bpf_tracing_multi_link *link = NULL; struct bpf_link_primer link_primer; u32 cnt, *ids = NULL; + u64 __user *ucookies; + u64 *cookies = NULL; u32 __user *uids; int err; @@ -3730,7 +3744,20 @@ int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) goto error; } - err = check_dup_ids(ids, cnt); + ucookies = u64_to_user_ptr(attr->link_create.tracing_multi.cookies); + if (ucookies) { + cookies = kvmalloc_objs(*cookies, cnt); + if (!cookies) { + err = -ENOMEM; + goto error; + } + if (copy_from_user(cookies, ucookies, cnt * sizeof(*cookies))) { + err = -EFAULT; + goto error; + } + } + + err = check_dup_ids(ids, cookies, cnt); if (err) goto error; @@ -3748,6 +3775,7 @@ int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) goto error; link->nodes_cnt = cnt; + link->cookies = cookies; err = bpf_trampoline_multi_attach(prog, ids, link); kvfree(ids); @@ -3758,6 +3786,7 @@ int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) return bpf_link_settle(&link_primer); error: + kvfree(cookies); kvfree(ids); kvfree(link); return err; -- cgit v1.2.3 From ba042ed6446fc524c1d804227765b45616f9cba3 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Sat, 6 Jun 2026 14:39:40 +0200 Subject: bpf: Add support for tracing_multi link session Adding support to use session attachment with tracing_multi link. Adding new BPF_TRACE_FSESSION_MULTI program attach type, that follows the BPF_TRACE_FSESSION behaviour but on the tracing_multi link. Such program is called on entry and exit of the attached function and allows to pass cookie value from entry to exit execution. Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260606123955.345967-16-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/bpf/fixups.c | 1 + kernel/bpf/syscall.c | 1 + kernel/bpf/trampoline.c | 44 ++++++++++++++++++++++++++++++++++++-------- kernel/bpf/verifier.c | 20 +++++++++++++++----- kernel/trace/bpf_trace.c | 15 ++++++++++++++- 5 files changed, 67 insertions(+), 14 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 0cf9735929f5..3cf2cc6e3ab6 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -2187,6 +2187,7 @@ patch_map_ops_generic: if (eatype == BPF_TRACE_FEXIT || eatype == BPF_TRACE_FSESSION || eatype == BPF_TRACE_FEXIT_MULTI || + eatype == BPF_TRACE_FSESSION_MULTI || eatype == BPF_MODIFY_RETURN) { /* Load nr_args from ctx - 8 */ insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index d551b9da0cfb..d4188a992bd8 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -4498,6 +4498,7 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type) case BPF_TRACE_FENTRY: case BPF_TRACE_FEXIT: case BPF_TRACE_FSESSION: + case BPF_TRACE_FSESSION_MULTI: case BPF_TRACE_FENTRY_MULTI: case BPF_TRACE_FEXIT_MULTI: case BPF_MODIFY_RETURN: diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index a3537fda50cf..1a721fc4bef5 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -183,7 +183,8 @@ bool bpf_prog_has_trampoline(const struct bpf_prog *prog) case BPF_PROG_TYPE_TRACING: if (eatype == BPF_TRACE_FENTRY || eatype == BPF_TRACE_FEXIT || eatype == BPF_MODIFY_RETURN || eatype == BPF_TRACE_FSESSION || - eatype == BPF_TRACE_FENTRY_MULTI || eatype == BPF_TRACE_FEXIT_MULTI) + eatype == BPF_TRACE_FENTRY_MULTI || eatype == BPF_TRACE_FEXIT_MULTI || + eatype == BPF_TRACE_FSESSION_MULTI) return true; return false; case BPF_PROG_TYPE_LSM: @@ -790,6 +791,7 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog) case BPF_TRACE_FEXIT_MULTI: return BPF_TRAMP_FEXIT; case BPF_TRACE_FSESSION: + case BPF_TRACE_FSESSION_MULTI: return BPF_TRAMP_FSESSION; case BPF_LSM_MAC: if (!prog->aux->attach_func_proto->type) @@ -822,13 +824,30 @@ static int bpf_freplace_check_tgt_prog(struct bpf_prog *tgt_prog) return 0; } +static struct bpf_tramp_node *fsession_exit(struct bpf_tramp_node *node) +{ + if (node->link->type == BPF_LINK_TYPE_TRACING) { + struct bpf_tracing_link *link; + + link = container_of(node->link, struct bpf_tracing_link, link.link); + return &link->fexit; + } else if (node->link->type == BPF_LINK_TYPE_TRACING_MULTI) { + struct bpf_tracing_multi_link *link; + struct bpf_tracing_multi_node *mnode; + + link = container_of(node->link, struct bpf_tracing_multi_link, link); + mnode = container_of(node, struct bpf_tracing_multi_node, node); + return &link->fexits[mnode - link->nodes]; + } + return NULL; +} + static int bpf_trampoline_add_prog(struct bpf_trampoline *tr, struct bpf_tramp_node *node, int cnt) { - struct bpf_tracing_link *tr_link = NULL; enum bpf_tramp_prog_type kind; - struct bpf_tramp_node *node_existing; + struct bpf_tramp_node *node_existing, *fexit; struct hlist_head *prog_list; kind = bpf_attach_type_to_tramp(node->link->prog); @@ -853,8 +872,10 @@ static int bpf_trampoline_add_prog(struct bpf_trampoline *tr, hlist_add_head(&node->tramp_hlist, prog_list); if (kind == BPF_TRAMP_FSESSION) { tr->progs_cnt[BPF_TRAMP_FENTRY]++; - tr_link = container_of(node, struct bpf_tracing_link, link.node); - hlist_add_head(&tr_link->fexit.tramp_hlist, &tr->progs_hlist[BPF_TRAMP_FEXIT]); + fexit = fsession_exit(node); + if (WARN_ON_ONCE(!fexit)) + return -EINVAL; + hlist_add_head(&fexit->tramp_hlist, &tr->progs_hlist[BPF_TRAMP_FEXIT]); tr->progs_cnt[BPF_TRAMP_FEXIT]++; } else { tr->progs_cnt[kind]++; @@ -865,13 +886,15 @@ static int bpf_trampoline_add_prog(struct bpf_trampoline *tr, static void bpf_trampoline_remove_prog(struct bpf_trampoline *tr, struct bpf_tramp_node *node) { - struct bpf_tracing_link *tr_link; enum bpf_tramp_prog_type kind; + struct bpf_tramp_node *fexit; kind = bpf_attach_type_to_tramp(node->link->prog); if (kind == BPF_TRAMP_FSESSION) { - tr_link = container_of(node, struct bpf_tracing_link, link.node); - hlist_del_init(&tr_link->fexit.tramp_hlist); + fexit = fsession_exit(node); + if (WARN_ON_ONCE(!fexit)) + return; + hlist_del_init(&fexit->tramp_hlist); tr->progs_cnt[BPF_TRAMP_FEXIT]--; kind = BPF_TRAMP_FENTRY; } @@ -1615,6 +1638,11 @@ int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids, mnode->node.link = &link->link; mnode->node.cookie = link->cookies ? link->cookies[i] : 0; + if (prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI) { + link->fexits[i].link = &link->link; + link->fexits[i].cookie = link->cookies ? link->cookies[i] : 0; + } + cond_resched(); } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5c594047ff0a..0c1cf506c219 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -16384,6 +16384,7 @@ static bool return_retval_range(struct bpf_verifier_env *env, struct bpf_retval_ case BPF_TRACE_FSESSION: case BPF_TRACE_FENTRY_MULTI: case BPF_TRACE_FEXIT_MULTI: + case BPF_TRACE_FSESSION_MULTI: *range = retval_range(0, 0); break; case BPF_TRACE_RAW_TP: @@ -18952,7 +18953,8 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, tgt_prog->expected_attach_type == BPF_TRACE_FEXIT || tgt_prog->expected_attach_type == BPF_TRACE_FENTRY_MULTI || tgt_prog->expected_attach_type == BPF_TRACE_FEXIT_MULTI || - tgt_prog->expected_attach_type == BPF_TRACE_FSESSION)) { + tgt_prog->expected_attach_type == BPF_TRACE_FSESSION || + tgt_prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI)) { /* Program extensions can extend all program types * except fentry/fexit. The reason is the following. * The fentry/fexit programs are used for performance @@ -19058,9 +19060,11 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, case BPF_TRACE_FENTRY: case BPF_TRACE_FEXIT: case BPF_TRACE_FSESSION: + case BPF_TRACE_FSESSION_MULTI: case BPF_TRACE_FENTRY_MULTI: case BPF_TRACE_FEXIT_MULTI: - if (prog->expected_attach_type == BPF_TRACE_FSESSION && + if ((prog->expected_attach_type == BPF_TRACE_FSESSION || + prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI) && !bpf_jit_supports_fsession()) { bpf_log(log, "JIT does not support fsession\n"); return -EOPNOTSUPP; @@ -19215,6 +19219,7 @@ static bool can_be_sleepable(struct bpf_prog *prog) case BPF_TRACE_RAW_TP: case BPF_TRACE_FENTRY_MULTI: case BPF_TRACE_FEXIT_MULTI: + case BPF_TRACE_FSESSION_MULTI: return true; default: return false; @@ -19301,6 +19306,7 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) return -EINVAL; } else if ((prog->expected_attach_type == BPF_TRACE_FEXIT || prog->expected_attach_type == BPF_TRACE_FSESSION || + prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI || prog->expected_attach_type == BPF_MODIFY_RETURN) && btf_id_set_contains(&noreturn_deny, btf_id)) { verbose(env, "Attaching fexit/fsession/fmod_ret to __noreturn function '%s' is rejected.\n", @@ -19340,7 +19346,8 @@ int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 bt return -EINVAL; /* Check noreturn attachment. */ - if (prog->expected_attach_type == BPF_TRACE_FEXIT_MULTI && + if ((prog->expected_attach_type == BPF_TRACE_FEXIT_MULTI || + prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI) && btf_id_set_contains(&noreturn_deny, btf_id)) return -EINVAL; /* Check denied attachment. */ @@ -19623,7 +19630,9 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1); *cnt = 1; } else if (desc->func_id == special_kfunc_list[KF_bpf_session_is_return] && - env->prog->expected_attach_type == BPF_TRACE_FSESSION) { + (env->prog->expected_attach_type == BPF_TRACE_FSESSION || + env->prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI)) { + /* * inline the bpf_session_is_return() for fsession: * bool bpf_session_is_return(void *ctx) @@ -19636,7 +19645,8 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, insn_buf[2] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 1); *cnt = 3; } else if (desc->func_id == special_kfunc_list[KF_bpf_session_cookie] && - env->prog->expected_attach_type == BPF_TRACE_FSESSION) { + (env->prog->expected_attach_type == BPF_TRACE_FSESSION || + env->prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI)) { /* * inline bpf_session_cookie() for fsession: * __u64 *bpf_session_cookie(void *ctx) diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index e33492739ed1..a0d688fffc5a 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -1334,7 +1334,8 @@ static inline bool is_uprobe_session(const struct bpf_prog *prog) static inline bool is_trace_fsession(const struct bpf_prog *prog) { return prog->type == BPF_PROG_TYPE_TRACING && - prog->expected_attach_type == BPF_TRACE_FSESSION; + (prog->expected_attach_type == BPF_TRACE_FSESSION || + prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI); } static const struct bpf_func_proto * @@ -3659,6 +3660,7 @@ static void bpf_tracing_multi_link_dealloc(struct bpf_link *link) struct bpf_tracing_multi_link *tr_link = container_of(link, struct bpf_tracing_multi_link, link); + kvfree(tr_link->fexits); kvfree(tr_link->cookies); kvfree(tr_link); } @@ -3718,6 +3720,7 @@ static int check_dup_ids(u32 *ids, u64 *cookies, u32 cnt) int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) { struct bpf_tracing_multi_link *link = NULL; + struct bpf_tramp_node *fexits = NULL; struct bpf_link_primer link_primer; u32 cnt, *ids = NULL; u64 __user *ucookies; @@ -3761,6 +3764,14 @@ int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) if (err) goto error; + if (prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI) { + fexits = kvmalloc_objs(*fexits, cnt); + if (!fexits) { + err = -ENOMEM; + goto error; + } + } + link = kvzalloc_flex(*link, nodes, cnt); if (!link) { err = -ENOMEM; @@ -3776,6 +3787,7 @@ int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) link->nodes_cnt = cnt; link->cookies = cookies; + link->fexits = fexits; err = bpf_trampoline_multi_attach(prog, ids, link); kvfree(ids); @@ -3786,6 +3798,7 @@ int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) return bpf_link_settle(&link_primer); error: + kvfree(fexits); kvfree(cookies); kvfree(ids); kvfree(link); -- cgit v1.2.3 From 8abecdafd57553c053bb68db47ed32a54972d5f4 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Sat, 6 Jun 2026 14:39:41 +0200 Subject: bpf: Add support for tracing_multi link fdinfo Adding tracing_multi link fdinfo support with following output: pos: 0 flags: 02000000 mnt_id: 19 ino: 3087 link_type: tracing_multi link_id: 9 prog_tag: 599ba0e317244f86 prog_id: 94 attach_type: 59 cnt: 10 obj-id btf-id cookie func 1 91593 8 bpf_fentry_test1+0x4/0x10 1 91595 9 bpf_fentry_test2+0x4/0x10 1 91596 7 bpf_fentry_test3+0x4/0x20 1 91597 5 bpf_fentry_test4+0x4/0x20 1 91598 4 bpf_fentry_test5+0x4/0x20 1 91599 2 bpf_fentry_test6+0x4/0x20 1 91600 3 bpf_fentry_test7+0x4/0x10 1 91601 1 bpf_fentry_test8+0x4/0x10 1 91602 10 bpf_fentry_test9+0x4/0x10 1 91594 6 bpf_fentry_test10+0x4/0x10 Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260606123955.345967-17-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/trace/bpf_trace.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) (limited to 'kernel') diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index a0d688fffc5a..90432f0fc2a8 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -3665,9 +3665,39 @@ static void bpf_tracing_multi_link_dealloc(struct bpf_link *link) kvfree(tr_link); } +#ifdef CONFIG_PROC_FS +static void bpf_tracing_multi_show_fdinfo(const struct bpf_link *link, + struct seq_file *seq) +{ + struct bpf_tracing_multi_link *tr_link = + container_of(link, struct bpf_tracing_multi_link, link); + bool has_cookies = !!tr_link->cookies; + + seq_printf(seq, "attach_type:\t%u\n", tr_link->link.attach_type); + seq_printf(seq, "cnt:\t%u\n", tr_link->nodes_cnt); + + seq_printf(seq, "%s\t %s\t %s\t %s\n", "obj-id", "btf-id", "cookie", "func"); + for (int i = 0; i < tr_link->nodes_cnt; i++) { + struct bpf_tracing_multi_node *mnode = &tr_link->nodes[i]; + u32 btf_id, obj_id; + + bpf_trampoline_unpack_key(mnode->trampoline->key, &obj_id, &btf_id); + seq_printf(seq, "%u\t %u\t %llu\t %pS\n", + obj_id, btf_id, + has_cookies ? tr_link->cookies[i] : 0, + (void *) mnode->trampoline->ip); + + cond_resched(); + } +} +#endif + static const struct bpf_link_ops bpf_tracing_multi_link_lops = { .release = bpf_tracing_multi_link_release, .dealloc_deferred = bpf_tracing_multi_link_dealloc, +#ifdef CONFIG_PROC_FS + .show_fdinfo = bpf_tracing_multi_show_fdinfo, +#endif }; static int ids_cmp_r(const void *pa, const void *pb, const void *priv __maybe_unused) -- cgit v1.2.3 From 89edbdfc5d0308cef57b71359331de5c4ddbf763 Mon Sep 17 00:00:00 2001 From: Mykyta Yatsenko Date: Sun, 7 Jun 2026 13:30:41 -0700 Subject: bpf: Fix NMI/tracepoint re-entry deadlock on lru locks NMI and tracepoint BPF programs can re-enter the per-CPU or global LRU lock that bpf_lru_pop_free()/push_free() already hold on the same CPU, AA-deadlocking. Lockdep reports "inconsistent {INITIAL USE} -> {IN-NMI}" on &l->lock (syzbot c69a0a2c816716f1e0d5) and "possible recursive locking detected" on &loc_l->lock (syzbot 18b26edb69b2e19f3b33). Prior trylock and rqspinlock based fixes (see links) were nacked because compromised on reliability. This patch converts every LRU lock site to rqspinlock_t and adds a recovery path for some failure windows to avoid node leaks. Failure recovery: - *_pop_free top-level: return NULL; prealloc_lru_pop() already treats that as no-free-element (-ENOMEM). - Cross-CPU steal: skip the victim's locked loc_l, try next CPU. - Post-steal local lock fail: publish stolen node to lockless per-CPU free_llist; next pop on this CPU picks it up. - push_free fail: mark node pending_free=1. __local_list_flush(), __local_list_pop_pending() reclaim the node from pending_list. __bpf_lru_list_shrink_inactive() reclaims the node from inactive list. Nodes from active list are reclaimed by __bpf_lru_list_shrink() or after __bpf_lru_list_rotate_active() demotes it to the inactive. Fixes: 3a08c2fd7634 ("bpf: LRU List") Reported-by: syzbot+c69a0a2c816716f1e0d5@syzkaller.appspotmail.com Closes: https://syzkaller.appspot.com/bug?extid=c69a0a2c816716f1e0d5 Reported-by: syzbot+18b26edb69b2e19f3b33@syzkaller.appspotmail.com Closes: https://syzkaller.appspot.com/bug?extid=18b26edb69b2e19f3b33 Link: https://lore.kernel.org/bpf/CAPPBnEYO4R+m+SpVc2gNj_x31R6fo1uJvj2bK2YS1P09GWT6kQ@mail.gmail.com/ Link: https://lore.kernel.org/bpf/CAPPBnEZmFA3ab8Uc=PEm0bdojZy=7T_F5_+eyZSHyZR3MBG4Vw@mail.gmail.com/ Link: https://lore.kernel.org/bpf/20251030030010.95352-1-dongml2@chinatelecom.cn/ Link: https://lore.kernel.org/bpf/20260119142120.28170-1-leon.hwang@linux.dev/ Signed-off-by: Mykyta Yatsenko Link: https://lore.kernel.org/r/20260607-lru_map_spin-v3-1-bcd9332e911b@meta.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/bpf_lru_list.c | 165 ++++++++++++++++++++++++++++------------------ kernel/bpf/bpf_lru_list.h | 25 +++++-- 2 files changed, 119 insertions(+), 71 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/bpf_lru_list.c b/kernel/bpf/bpf_lru_list.c index e7a2fc60523f..5ed7cb4b98c0 100644 --- a/kernel/bpf/bpf_lru_list.c +++ b/kernel/bpf/bpf_lru_list.c @@ -13,23 +13,8 @@ #define PERCPU_FREE_TARGET (4) #define PERCPU_NR_SCANS PERCPU_FREE_TARGET -/* Helpers to get the local list index */ -#define LOCAL_LIST_IDX(t) ((t) - BPF_LOCAL_LIST_T_OFFSET) -#define LOCAL_FREE_LIST_IDX LOCAL_LIST_IDX(BPF_LRU_LOCAL_LIST_T_FREE) -#define LOCAL_PENDING_LIST_IDX LOCAL_LIST_IDX(BPF_LRU_LOCAL_LIST_T_PENDING) #define IS_LOCAL_LIST_TYPE(t) ((t) >= BPF_LOCAL_LIST_T_OFFSET) -/* Local list helpers */ -static struct list_head *local_free_list(struct bpf_lru_locallist *loc_l) -{ - return &loc_l->lists[LOCAL_FREE_LIST_IDX]; -} - -static struct list_head *local_pending_list(struct bpf_lru_locallist *loc_l) -{ - return &loc_l->lists[LOCAL_PENDING_LIST_IDX]; -} - /* bpf_lru_node helpers */ static bool bpf_lru_node_is_ref(const struct bpf_lru_node *node) { @@ -72,6 +57,7 @@ static void __bpf_lru_node_move_to_free(struct bpf_lru_list *l, bpf_lru_list_count_dec(l, node->type); node->type = tgt_free_type; + WRITE_ONCE(node->pending_free, 0); list_move(&node->list, free_list); } @@ -87,6 +73,9 @@ static void __bpf_lru_node_move_in(struct bpf_lru_list *l, bpf_lru_list_count_inc(l, tgt_type); node->type = tgt_type; bpf_lru_node_clear_ref(node); + /* Reset pending_free only when moving to the free list */ + if (tgt_type == BPF_LRU_LIST_T_FREE) + WRITE_ONCE(node->pending_free, 0); list_move(&node->list, &l->lists[tgt_type]); } @@ -212,9 +201,11 @@ __bpf_lru_list_shrink_inactive(struct bpf_lru *lru, unsigned int i = 0; list_for_each_entry_safe_reverse(node, tmp_node, inactive, list) { - if (bpf_lru_node_is_ref(node)) { + if (bpf_lru_node_is_ref(node) && + !READ_ONCE(node->pending_free)) { __bpf_lru_node_move(l, node, BPF_LRU_LIST_T_ACTIVE); - } else if (lru->del_from_htab(lru->del_arg, node)) { + } else if (READ_ONCE(node->pending_free) || + lru->del_from_htab(lru->del_arg, node)) { __bpf_lru_node_move_to_free(l, node, free_list, tgt_free_type); if (++nshrinked == tgt_nshrink) @@ -273,7 +264,8 @@ static unsigned int __bpf_lru_list_shrink(struct bpf_lru *lru, list_for_each_entry_safe_reverse(node, tmp_node, force_shrink_list, list) { - if (lru->del_from_htab(lru->del_arg, node)) { + if (READ_ONCE(node->pending_free) || + lru->del_from_htab(lru->del_arg, node)) { __bpf_lru_node_move_to_free(l, node, free_list, tgt_free_type); return 1; @@ -290,8 +282,10 @@ static void __local_list_flush(struct bpf_lru_list *l, struct bpf_lru_node *node, *tmp_node; list_for_each_entry_safe_reverse(node, tmp_node, - local_pending_list(loc_l), list) { - if (bpf_lru_node_is_ref(node)) + &loc_l->pending_list, list) { + if (READ_ONCE(node->pending_free)) + __bpf_lru_node_move_in(l, node, BPF_LRU_LIST_T_FREE); + else if (bpf_lru_node_is_ref(node)) __bpf_lru_node_move_in(l, node, BPF_LRU_LIST_T_ACTIVE); else __bpf_lru_node_move_in(l, node, @@ -307,9 +301,12 @@ static void bpf_lru_list_push_free(struct bpf_lru_list *l, if (WARN_ON_ONCE(IS_LOCAL_LIST_TYPE(node->type))) return; - raw_spin_lock_irqsave(&l->lock, flags); + if (raw_res_spin_lock_irqsave(&l->lock, flags)) { + WRITE_ONCE(node->pending_free, 1); + return; + } __bpf_lru_node_move(l, node, BPF_LRU_LIST_T_FREE); - raw_spin_unlock_irqrestore(&l->lock, flags); + raw_res_spin_unlock_irqrestore(&l->lock, flags); } static void bpf_lru_list_pop_free_to_local(struct bpf_lru *lru, @@ -318,8 +315,10 @@ static void bpf_lru_list_pop_free_to_local(struct bpf_lru *lru, struct bpf_lru_list *l = &lru->common_lru.lru_list; struct bpf_lru_node *node, *tmp_node; unsigned int nfree = 0; + LIST_HEAD(tmp_free); - raw_spin_lock(&l->lock); + if (raw_res_spin_lock(&l->lock)) + return; __local_list_flush(l, loc_l); @@ -327,7 +326,7 @@ static void bpf_lru_list_pop_free_to_local(struct bpf_lru *lru, list_for_each_entry_safe(node, tmp_node, &l->lists[BPF_LRU_LIST_T_FREE], list) { - __bpf_lru_node_move_to_free(l, node, local_free_list(loc_l), + __bpf_lru_node_move_to_free(l, node, &tmp_free, BPF_LRU_LOCAL_LIST_T_FREE); if (++nfree == lru->target_free) break; @@ -335,10 +334,19 @@ static void bpf_lru_list_pop_free_to_local(struct bpf_lru *lru, if (nfree < lru->target_free) __bpf_lru_list_shrink(lru, l, lru->target_free - nfree, - local_free_list(loc_l), + &tmp_free, BPF_LRU_LOCAL_LIST_T_FREE); - raw_spin_unlock(&l->lock); + raw_res_spin_unlock(&l->lock); + + /* + * Transfer the harvested nodes from the temporary list_head into + * the lockless per-CPU free llist. + */ + list_for_each_entry_safe(node, tmp_node, &tmp_free, list) { + list_del(&node->list); + llist_add(&node->llist, &loc_l->free_llist); + } } static void __local_list_add_pending(struct bpf_lru *lru, @@ -350,22 +358,21 @@ static void __local_list_add_pending(struct bpf_lru *lru, *(u32 *)((void *)node + lru->hash_offset) = hash; node->cpu = cpu; node->type = BPF_LRU_LOCAL_LIST_T_PENDING; + WRITE_ONCE(node->pending_free, 0); bpf_lru_node_clear_ref(node); - list_add(&node->list, local_pending_list(loc_l)); + list_add(&node->list, &loc_l->pending_list); } static struct bpf_lru_node * __local_list_pop_free(struct bpf_lru_locallist *loc_l) { - struct bpf_lru_node *node; + struct llist_node *llnode; - node = list_first_entry_or_null(local_free_list(loc_l), - struct bpf_lru_node, - list); - if (node) - list_del(&node->list); + llnode = llist_del_first(&loc_l->free_llist); + if (!llnode) + return NULL; - return node; + return container_of(llnode, struct bpf_lru_node, llist); } static struct bpf_lru_node * @@ -376,10 +383,10 @@ __local_list_pop_pending(struct bpf_lru *lru, struct bpf_lru_locallist *loc_l) ignore_ref: /* Get from the tail (i.e. older element) of the pending list. */ - list_for_each_entry_reverse(node, local_pending_list(loc_l), - list) { + list_for_each_entry_reverse(node, &loc_l->pending_list, list) { if ((!bpf_lru_node_is_ref(node) || force) && - lru->del_from_htab(lru->del_arg, node)) { + (READ_ONCE(node->pending_free) || + lru->del_from_htab(lru->del_arg, node))) { list_del(&node->list); return node; } @@ -404,7 +411,8 @@ static struct bpf_lru_node *bpf_percpu_lru_pop_free(struct bpf_lru *lru, l = per_cpu_ptr(lru->percpu_lru, cpu); - raw_spin_lock_irqsave(&l->lock, flags); + if (raw_res_spin_lock_irqsave(&l->lock, flags)) + return NULL; __bpf_lru_list_rotate(lru, l); @@ -420,7 +428,7 @@ static struct bpf_lru_node *bpf_percpu_lru_pop_free(struct bpf_lru *lru, __bpf_lru_node_move(l, node, BPF_LRU_LIST_T_INACTIVE); } - raw_spin_unlock_irqrestore(&l->lock, flags); + raw_res_spin_unlock_irqrestore(&l->lock, flags); return node; } @@ -437,7 +445,8 @@ static struct bpf_lru_node *bpf_common_lru_pop_free(struct bpf_lru *lru, loc_l = per_cpu_ptr(clru->local_list, cpu); - raw_spin_lock_irqsave(&loc_l->lock, flags); + if (raw_res_spin_lock_irqsave(&loc_l->lock, flags)) + return NULL; node = __local_list_pop_free(loc_l); if (!node) { @@ -448,17 +457,22 @@ static struct bpf_lru_node *bpf_common_lru_pop_free(struct bpf_lru *lru, if (node) __local_list_add_pending(lru, loc_l, cpu, node, hash); - raw_spin_unlock_irqrestore(&loc_l->lock, flags); + raw_res_spin_unlock_irqrestore(&loc_l->lock, flags); if (node) return node; - /* No free nodes found from the local free list and + /* + * No free nodes found from the local free list and * the global LRU list. * * Steal from the local free/pending list of the * current CPU and remote CPU in RR. It starts * with the loc_l->next_steal CPU. + * + * Acquire the victim's lock before touching either list. On + * acquisition failure (rqspinlock AA or timeout) skip the victim + * and try the next CPU. */ first_steal = loc_l->next_steal; @@ -466,24 +480,36 @@ static struct bpf_lru_node *bpf_common_lru_pop_free(struct bpf_lru *lru, do { steal_loc_l = per_cpu_ptr(clru->local_list, steal); - raw_spin_lock_irqsave(&steal_loc_l->lock, flags); - - node = __local_list_pop_free(steal_loc_l); - if (!node) - node = __local_list_pop_pending(lru, steal_loc_l); - - raw_spin_unlock_irqrestore(&steal_loc_l->lock, flags); + if (!raw_res_spin_lock_irqsave(&steal_loc_l->lock, flags)) { + node = __local_list_pop_free(steal_loc_l); + if (!node) + node = __local_list_pop_pending(lru, steal_loc_l); + raw_res_spin_unlock_irqrestore(&steal_loc_l->lock, flags); + } steal = cpumask_next_wrap(steal, cpu_possible_mask); } while (!node && steal != first_steal); loc_l->next_steal = steal; - if (node) { - raw_spin_lock_irqsave(&loc_l->lock, flags); - __local_list_add_pending(lru, loc_l, cpu, node, hash); - raw_spin_unlock_irqrestore(&loc_l->lock, flags); + if (!node) + return NULL; + + if (raw_res_spin_lock_irqsave(&loc_l->lock, flags)) { + /* + * The local pending lock can't be acquired (rqspinlock AA + * or timeout). Return the stolen node to the per-CPU + * free_llist instead of orphaning it; the next pop_free on + * this CPU will pick it up. + */ + node->type = BPF_LRU_LOCAL_LIST_T_FREE; + bpf_lru_node_clear_ref(node); + WRITE_ONCE(node->pending_free, 0); + llist_add(&node->llist, &loc_l->free_llist); + return NULL; } + __local_list_add_pending(lru, loc_l, cpu, node, hash); + raw_res_spin_unlock_irqrestore(&loc_l->lock, flags); return node; } @@ -511,18 +537,24 @@ static void bpf_common_lru_push_free(struct bpf_lru *lru, loc_l = per_cpu_ptr(lru->common_lru.local_list, node->cpu); - raw_spin_lock_irqsave(&loc_l->lock, flags); + if (raw_res_spin_lock_irqsave(&loc_l->lock, flags)) { + WRITE_ONCE(node->pending_free, 1); + return; + } if (unlikely(node->type != BPF_LRU_LOCAL_LIST_T_PENDING)) { - raw_spin_unlock_irqrestore(&loc_l->lock, flags); + raw_res_spin_unlock_irqrestore(&loc_l->lock, + flags); goto check_lru_list; } node->type = BPF_LRU_LOCAL_LIST_T_FREE; bpf_lru_node_clear_ref(node); - list_move(&node->list, local_free_list(loc_l)); + list_del(&node->list); + + raw_res_spin_unlock_irqrestore(&loc_l->lock, flags); - raw_spin_unlock_irqrestore(&loc_l->lock, flags); + llist_add(&node->llist, &loc_l->free_llist); return; } @@ -538,11 +570,14 @@ static void bpf_percpu_lru_push_free(struct bpf_lru *lru, l = per_cpu_ptr(lru->percpu_lru, node->cpu); - raw_spin_lock_irqsave(&l->lock, flags); + if (raw_res_spin_lock_irqsave(&l->lock, flags)) { + WRITE_ONCE(node->pending_free, 1); + return; + } __bpf_lru_node_move(l, node, BPF_LRU_LIST_T_FREE); - raw_spin_unlock_irqrestore(&l->lock, flags); + raw_res_spin_unlock_irqrestore(&l->lock, flags); } void bpf_lru_push_free(struct bpf_lru *lru, struct bpf_lru_node *node) @@ -565,6 +600,7 @@ static void bpf_common_lru_populate(struct bpf_lru *lru, void *buf, node = (struct bpf_lru_node *)(buf + node_offset); node->type = BPF_LRU_LIST_T_FREE; + node->pending_free = 0; bpf_lru_node_clear_ref(node); list_add(&node->list, &l->lists[BPF_LRU_LIST_T_FREE]); buf += elem_size; @@ -594,6 +630,7 @@ again: node = (struct bpf_lru_node *)(buf + node_offset); node->cpu = cpu; node->type = BPF_LRU_LIST_T_FREE; + node->pending_free = 0; bpf_lru_node_clear_ref(node); list_add(&node->list, &l->lists[BPF_LRU_LIST_T_FREE]); i++; @@ -618,14 +655,12 @@ void bpf_lru_populate(struct bpf_lru *lru, void *buf, u32 node_offset, static void bpf_lru_locallist_init(struct bpf_lru_locallist *loc_l, int cpu) { - int i; - - for (i = 0; i < NR_BPF_LRU_LOCAL_LIST_T; i++) - INIT_LIST_HEAD(&loc_l->lists[i]); + INIT_LIST_HEAD(&loc_l->pending_list); + init_llist_head(&loc_l->free_llist); loc_l->next_steal = cpu; - raw_spin_lock_init(&loc_l->lock); + raw_res_spin_lock_init(&loc_l->lock); } static void bpf_lru_list_init(struct bpf_lru_list *l) @@ -640,7 +675,7 @@ static void bpf_lru_list_init(struct bpf_lru_list *l) l->next_inactive_rotation = &l->lists[BPF_LRU_LIST_T_INACTIVE]; - raw_spin_lock_init(&l->lock); + raw_res_spin_lock_init(&l->lock); } int bpf_lru_init(struct bpf_lru *lru, bool percpu, u32 hash_offset, diff --git a/kernel/bpf/bpf_lru_list.h b/kernel/bpf/bpf_lru_list.h index fe2661a58ea9..8d0ee61622af 100644 --- a/kernel/bpf/bpf_lru_list.h +++ b/kernel/bpf/bpf_lru_list.h @@ -6,11 +6,11 @@ #include #include -#include +#include +#include #define NR_BPF_LRU_LIST_T (3) #define NR_BPF_LRU_LIST_COUNT (2) -#define NR_BPF_LRU_LOCAL_LIST_T (2) #define BPF_LOCAL_LIST_T_OFFSET NR_BPF_LRU_LIST_T enum bpf_lru_list_type { @@ -22,10 +22,22 @@ enum bpf_lru_list_type { }; struct bpf_lru_node { - struct list_head list; + /* + * A node is in at most one list at a time. The free path on the + * per-CPU locallist uses an llist, so share storage via a union. + */ + union { + struct list_head list; + struct llist_node llist; + }; u16 cpu; u8 type; u8 ref; + /* + * Marks nodes whose *_push_free() lock acquire failed; reclaimed + * by flush/shrink which honor the flag instead of del_from_htab(). + */ + u8 pending_free; }; struct bpf_lru_list { @@ -34,13 +46,14 @@ struct bpf_lru_list { /* The next inactive list rotation starts from here */ struct list_head *next_inactive_rotation; - raw_spinlock_t lock ____cacheline_aligned_in_smp; + rqspinlock_t lock ____cacheline_aligned_in_smp; }; struct bpf_lru_locallist { - struct list_head lists[NR_BPF_LRU_LOCAL_LIST_T]; + struct list_head pending_list; + struct llist_head free_llist; u16 next_steal; - raw_spinlock_t lock; + rqspinlock_t lock; }; struct bpf_common_lru { -- cgit v1.2.3 From 53040a81ae57cdca8af8ac36fe4e661730cf7c6b Mon Sep 17 00:00:00 2001 From: Nuoqi Gui Date: Sun, 7 Jun 2026 21:24:13 +0800 Subject: bpf: Keep dynamic inner array lookups nullable An ARRAY_OF_MAPS can use an array created with BPF_F_INNER_MAP as its inner map template. A concrete inner array with a different max_entries value can then replace the template. After a successful outer map lookup, the verifier represents the resulting map pointer using the inner map template. Const-key lookup nullness elision consequently uses the template max_entries even though the runtime helper uses the concrete inner map max_entries. Do not elide lookup result nullness for maps marked with BPF_F_INNER_MAP, because the template max_entries does not prove that the key is in bounds for the concrete runtime map. Fixes: d2102f2f5d75 ("bpf: verifier: Support eliding map lookup nullness") Signed-off-by: Nuoqi Gui Acked-by: Eduard Zingerman Acked-by: Jiri Olsa Cc: stable@vger.kernel.org Link: https://lore.kernel.org/bpf/20260607-f01-v2-v2-1-da48453146e8@mails.tsinghua.edu.cn Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 0c1cf506c219..ed7ba0e6a9ce 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8179,7 +8179,7 @@ static int get_constant_map_key(struct bpf_verifier_env *env, return 0; } -static bool can_elide_value_nullness(enum bpf_map_type type); +static bool can_elide_value_nullness(const struct bpf_map *map); static int check_func_arg(struct bpf_verifier_env *env, u32 arg, struct bpf_call_arg_meta *meta, @@ -8298,7 +8298,7 @@ skip_type_check: err = check_helper_mem_access(env, reg, argno_from_reg(regno), key_size, BPF_READ, false, NULL); if (err) return err; - if (can_elide_value_nullness(meta->map.ptr->map_type)) { + if (can_elide_value_nullness(meta->map.ptr)) { err = get_constant_map_key(env, reg, key_size, &meta->const_map_key); if (err < 0) { meta->const_map_key = -1; @@ -10068,13 +10068,16 @@ static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno state->callback_subprogno == subprogno); } -/* Returns whether or not the given map type can potentially elide +/* Returns whether or not the given map can potentially elide * lookup return value nullness check. This is possible if the key * is statically known. */ -static bool can_elide_value_nullness(enum bpf_map_type type) +static bool can_elide_value_nullness(const struct bpf_map *map) { - switch (type) { + if (map->map_flags & BPF_F_INNER_MAP) + return false; + + switch (map->map_type) { case BPF_MAP_TYPE_ARRAY: case BPF_MAP_TYPE_PERCPU_ARRAY: return true; @@ -10414,7 +10417,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn } if (func_id == BPF_FUNC_map_lookup_elem && - can_elide_value_nullness(meta.map.ptr->map_type) && + can_elide_value_nullness(meta.map.ptr) && meta.const_map_key >= 0 && meta.const_map_key < meta.map.ptr->max_entries) ret_flag &= ~PTR_MAYBE_NULL; -- cgit v1.2.3 From 50dff00615522f3ec03449680ca23beb4cfc549c Mon Sep 17 00:00:00 2001 From: Sechang Lim Date: Mon, 8 Jun 2026 05:00:00 +0000 Subject: bpf: Fix NULL pointer dereference in bpf_task_from_vpid() bpf_task_from_vpid() looks up a task in the pid namespace of the current task, via find_task_by_vpid(): find_task_by_vpid(vpid) find_task_by_pid_ns(vpid, task_active_pid_ns(current)) find_pid_ns(nr, ns) -> idr_find(&ns->idr, nr) cgroup_skb programs run in softirq, which may interrupt a task that is itself in do_exit(). Once that task has passed exit_notify() -> release_task() -> __unhash_process(), its thread_pid is cleared, so task_active_pid_ns(current) returns NULL and find_pid_ns() dereferences &NULL->idr: BUG: kernel NULL pointer dereference, address: 0000000000000050 RIP: 0010:idr_find+0x11/0x30 lib/idr.c:176 Call Trace: find_pid_ns kernel/pid.c:370 [inline] find_task_by_pid_ns+0x3b/0xe0 kernel/pid.c:485 bpf_task_from_vpid+0x5b/0x200 kernel/bpf/helpers.c:2916 bpf_prog_run_array_cg+0x17e/0x530 kernel/bpf/cgroup.c:81 __cgroup_bpf_run_filter_skb+0x12b/0x250 kernel/bpf/cgroup.c:1612 sk_filter_trim_cap+0x1dc/0x4c0 net/core/filter.c:148 tcp_v4_rcv+0x18d1/0x2200 net/ipv4/tcp_ipv4.c:2223 do_exit+0xa63/0x1270 kernel/exit.c:1010 get_signal+0x141c/0x1530 kernel/signal.c:3037 Bail out when current has no pid namespace. Fixes: 675c3596ff32 ("bpf: Add bpf_task_from_vpid() kfunc") Signed-off-by: Sechang Lim Acked-by: Leon Hwang Link: https://lore.kernel.org/bpf/20260608050001.2545245-1-rhkrqnwk98@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/helpers.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 8ba2b8965caf..8e196c9b7c50 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -3009,11 +3009,13 @@ __bpf_kfunc struct task_struct *bpf_task_from_vpid(s32 vpid) { struct task_struct *p; - rcu_read_lock(); + guard(rcu)(); + if (!task_active_pid_ns(current)) + return NULL; + p = find_task_by_vpid(vpid); if (p) p = bpf_task_acquire(p); - rcu_read_unlock(); return p; } -- cgit v1.2.3 From c82dfce478334dca4e9a42c4519a55ee2a2c43c7 Mon Sep 17 00:00:00 2001 From: Shashank Balaji Date: Mon, 1 Jun 2026 19:19:41 +0900 Subject: kernel: param: initialize module_kset in a pure_initcall Commit "driver core: platform: set mod_name in driver registration" will set struct device_driver's mod_name member for platform driver registration. For a driver to be registered with its mod_name set, module_kset needs to be initialized, which currently happens in a subsys_initcall in param_sysfs_init(). The tegra cbb drivers register themselves before module_kset init, in a core_initcall. This works currently because lookup_or_create_module_kobject(), which dereferences module_kset via kset_find_obj(), is not called if mod_name is not set, which is the case now. So in preparation for the commit "driver core: platform: set mod_name in driver registration", move module_kset init to pure_initcall level, ensuring it happens before tegra cbb driver registration. Suggested-by: Gary Guo Reviewed-by: Gary Guo Co-developed-by: Rahul Bukte Signed-off-by: Rahul Bukte Signed-off-by: Shashank Balaji Link: https://patch.msgid.link/20260601101942.4002661-1-shashank.mahadasyam@sony.com Signed-off-by: Danilo Krummrich --- kernel/params.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/params.c b/kernel/params.c index 74d620bc2521..a668863a4bb6 100644 --- a/kernel/params.c +++ b/kernel/params.c @@ -942,9 +942,9 @@ const struct kobj_type module_ktype = { /* * param_sysfs_init - create "module" kset * - * This must be done before the initramfs is unpacked and - * request_module() thus becomes possible, because otherwise the - * module load would fail in mod_sysfs_init. + * This must be done before any driver registration so that when a driver comes + * from a built-in module, the driver core can add the module under /sys/module + * and create the associated driver symlinks. */ static int __init param_sysfs_init(void) { @@ -957,7 +957,7 @@ static int __init param_sysfs_init(void) return 0; } -subsys_initcall(param_sysfs_init); +pure_initcall(param_sysfs_init); /* * param_sysfs_builtin_init - add sysfs version and parameter -- cgit v1.2.3 From c13a0316aef5f4b73e8b4bf6943737f836d65e1d Mon Sep 17 00:00:00 2001 From: Youngjun Park Date: Tue, 24 Mar 2026 01:08:21 +0900 Subject: mm/swap, PM: hibernate: fix swapoff race in uswsusp by pinning swap device Patch series "mm/swap, PM: hibernate: fix swapoff race in uswsusp by pinning swap device", v8. Currently, in the uswsusp path, only the swap type value is retrieved at lookup time without holding a reference. If swapoff races after the type is acquired, subsequent slot allocations operate on a stale swap device. Additionally, grabbing and releasing the swap device reference on every slot allocation is inefficient across the entire hibernation swap path. This patch series addresses these issues: - Patch 1: Fixes the swapoff race in uswsusp by pinning the swap device from the point it is looked up until the session completes. - Patch 2: Removes the overhead of per-slot reference counting in alloc/free paths and cleans up the redundant SWP_WRITEOK check. This patch (of 2): Hibernation via uswsusp (/dev/snapshot ioctls) has a race window: after selecting the resume swap area but before user space is frozen, swapoff may run and invalidate the selected swap device. Fix this by pinning the swap device with SWP_HIBERNATION while it is in use. The pin is exclusive, which is sufficient since hibernate_acquire() already prevents concurrent hibernation sessions. The kernel swsusp path (sysfs-based hibernate/resume) uses find_hibernation_swap_type() which is not affected by the pin. It freezes user space before touching swap, so swapoff cannot race. Introduce dedicated helpers: - pin_hibernation_swap_type(): Look up and pin the swap device. Used by the uswsusp path. - find_hibernation_swap_type(): Lookup without pinning. Used by the kernel swsusp path. - unpin_hibernation_swap_type(): Clear the hibernation pin. While a swap device is pinned, swapoff is prevented from proceeding. Link: https://lore.kernel.org/20260323160822.1409904-1-youngjun.park@lge.com Link: https://lore.kernel.org/20260323160822.1409904-2-youngjun.park@lge.com Signed-off-by: Youngjun Park Reviewed-by: Kairui Song Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: Kemeng Shi Cc: Nhat Pham Cc: "Rafael J . Wysocki" Signed-off-by: Andrew Morton --- kernel/power/swap.c | 2 +- kernel/power/user.c | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/power/swap.c b/kernel/power/swap.c index 2e64869bb5a0..cc4764149e8f 100644 --- a/kernel/power/swap.c +++ b/kernel/power/swap.c @@ -341,7 +341,7 @@ static int swsusp_swap_check(void) * This is called before saving the image. */ if (swsusp_resume_device) - res = swap_type_of(swsusp_resume_device, swsusp_resume_block); + res = find_hibernation_swap_type(swsusp_resume_device, swsusp_resume_block); else res = find_first_swap(&swsusp_resume_device); if (res < 0) diff --git a/kernel/power/user.c b/kernel/power/user.c index be77f3556bd7..d0fcfba7ac23 100644 --- a/kernel/power/user.c +++ b/kernel/power/user.c @@ -71,7 +71,7 @@ static int snapshot_open(struct inode *inode, struct file *filp) memset(&data->handle, 0, sizeof(struct snapshot_handle)); if ((filp->f_flags & O_ACCMODE) == O_RDONLY) { /* Hibernating. The image device should be accessible. */ - data->swap = swap_type_of(swsusp_resume_device, 0); + data->swap = pin_hibernation_swap_type(swsusp_resume_device, 0); data->mode = O_RDONLY; data->free_bitmaps = false; error = pm_notifier_call_chain_robust(PM_HIBERNATION_PREPARE, PM_POST_HIBERNATION); @@ -90,8 +90,10 @@ static int snapshot_open(struct inode *inode, struct file *filp) data->free_bitmaps = !error; } } - if (error) + if (error) { + unpin_hibernation_swap_type(data->swap); hibernate_release(); + } data->frozen = false; data->ready = false; @@ -115,6 +117,7 @@ static int snapshot_release(struct inode *inode, struct file *filp) data = filp->private_data; data->dev = 0; free_all_swap_pages(data->swap); + unpin_hibernation_swap_type(data->swap); if (data->frozen) { pm_restore_gfp_mask(); free_basic_memory_bitmaps(); @@ -235,11 +238,17 @@ static int snapshot_set_swap_area(struct snapshot_data *data, offset = swap_area.offset; } + /* + * Unpin the swap device if a swap area was already + * set by SNAPSHOT_SET_SWAP_AREA. + */ + unpin_hibernation_swap_type(data->swap); + /* * User space encodes device types as two-byte values, * so we need to recode them */ - data->swap = swap_type_of(swdev, offset); + data->swap = pin_hibernation_swap_type(swdev, offset); if (data->swap < 0) return swdev ? -ENODEV : -EINVAL; data->dev = swdev; -- cgit v1.2.3 From 29922fdfc2a4008d66418bedd0ebf5038fc54efa Mon Sep 17 00:00:00 2001 From: Hongyan Xia Date: Fri, 5 Jun 2026 09:43:39 +0000 Subject: sched/fair: Fix cpu_util runnable_avg arithmetic If we take runnable_avg in max(runnable_avg, util_avg) in cpu_util(), we should then add or subtract task runnable_avg, but the arithmetic below is still with task util_avg. This mixes runnable_avg with util_avg which is incorrect. Fix by always doing arithmetic with runnable_avg and only take max(runnable_avg, util_avg) at the last step. Fixes: 7d0583cf9ec7 ("sched/fair, cpufreq: Introduce 'runnable boosting'") Signed-off-by: Hongyan Xia Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Vincent Guittot Link: https://patch.msgid.link/20260605094318.37931-1-hongyan.xia@transsion.com --- kernel/sched/fair.c | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index f4ed841f766f..1b23e73f48b0 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -8968,25 +8968,32 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) static unsigned long cpu_util(int cpu, struct task_struct *p, int dst_cpu, int boost) { + bool add_task = p && task_cpu(p) != cpu && dst_cpu == cpu; + bool sub_task = p && task_cpu(p) == cpu && dst_cpu != cpu; struct cfs_rq *cfs_rq = &cpu_rq(cpu)->cfs; unsigned long util = READ_ONCE(cfs_rq->avg.util_avg); unsigned long runnable; - if (boost) { - runnable = READ_ONCE(cfs_rq->avg.runnable_avg); - util = max(util, runnable); - } - /* * If @dst_cpu is -1 or @p migrates from @cpu to @dst_cpu remove its * contribution. If @p migrates from another CPU to @cpu add its * contribution. In all the other cases @cpu is not impacted by the * migration so its util_avg is already correct. */ - if (p && task_cpu(p) == cpu && dst_cpu != cpu) - lsub_positive(&util, task_util(p)); - else if (p && task_cpu(p) != cpu && dst_cpu == cpu) + if (add_task) util += task_util(p); + else if (sub_task) + lsub_positive(&util, task_util(p)); + + if (boost) { + runnable = READ_ONCE(cfs_rq->avg.runnable_avg); + if (add_task) + runnable += READ_ONCE(p->se.avg.runnable_avg); + else if (sub_task) + lsub_positive(&runnable, + READ_ONCE(p->se.avg.runnable_avg)); + util = max(util, runnable); + } if (sched_feat(UTIL_EST)) { unsigned long util_est; -- cgit v1.2.3 From 76124a050ddbc8b252172205ab04f10a83c03a4d Mon Sep 17 00:00:00 2001 From: Liang Luo Date: Mon, 8 Jun 2026 15:18:42 +0800 Subject: sched/core: Combine separate 'else' and 'if' statements The kernel coding style recommends using 'else if' instead of placing 'if' on a separate line after 'else'. This change makes the code consistent with the rest of the kernel codebase. Signed-off-by: Liang Luo Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260608071842.325159-1-luoliang@kylinos.cn --- kernel/sched/core.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index e745c58671ed..2f4530eb543f 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3816,8 +3816,7 @@ ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags, en_flags |= ENQUEUE_RQ_SELECTED; if (wake_flags & WF_MIGRATED) en_flags |= ENQUEUE_MIGRATED; - else - if (p->in_iowait) { + else if (p->in_iowait) { delayacct_blkio_end(p); atomic_dec(&task_rq(p)->nr_iowait); } -- cgit v1.2.3 From 9ebe5c3c29f6217412ff256134516d4dff0e5624 Mon Sep 17 00:00:00 2001 From: Liang Luo Date: Mon, 8 Jun 2026 15:55:00 +0800 Subject: sched/deadline: Use task_on_rq_migrating() helper Replace the open-coded "p->on_rq == TASK_ON_RQ_MIGRATING" comparisons in enqueue_task_dl() and dequeue_task_dl() with the existing task_on_rq_migrating() helper, consistent with the rest of the scheduler code. No functional change. Signed-off-by: Liang Luo Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: K Prateek Nayak Link: https://patch.msgid.link/20260608075500.387271-1-luoliang@kylinos.cn --- kernel/sched/deadline.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 4754dbe4232d..5ccb06effea0 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -2530,7 +2530,7 @@ static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags) check_schedstat_required(); update_stats_wait_start_dl(dl_rq, dl_se); - if (p->on_rq == TASK_ON_RQ_MIGRATING) + if (task_on_rq_migrating(p)) flags |= ENQUEUE_MIGRATING; enqueue_dl_entity(dl_se, flags); @@ -2552,7 +2552,7 @@ static bool dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags) { update_curr_dl(rq); - if (p->on_rq == TASK_ON_RQ_MIGRATING) + if (task_on_rq_migrating(p)) flags |= DEQUEUE_MIGRATING; dequeue_dl_entity(&p->dl, flags); -- cgit v1.2.3 From b9452b594fd3aecbfd4aa0a6a1f741330a37dab7 Mon Sep 17 00:00:00 2001 From: Paul Moses Date: Fri, 5 Jun 2026 23:43:09 +0000 Subject: bpf: Validate BTF repeated field counts before expansion btf_parse_struct_metas() walks user-supplied BTF during BPF_BTF_LOAD, and btf_repeat_fields() expands repeatable fields from array elements into the fixed BTF_FIELDS_MAX scratch array used by btf_parse_fields(). The remaining-capacity check performs the expanded field count calculation in u32. A malformed BTF can wrap that calculation, causing the check to pass even when the expanded field count exceeds the scratch array capacity. The following memcpy() can then write past the end of the array. Use checked addition and multiplication before copying repeated fields and reject impossible counts. Fixes: 797d73ee232d ("bpf: Check the remaining info_cnt before repeating btf fields") Cc: stable@vger.kernel.org Signed-off-by: Paul Moses Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260605234301.1109063-1-p@1g4.org Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/btf.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index ef4402274786..15ae7c43f594 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -3667,7 +3667,7 @@ end: static int btf_repeat_fields(struct btf_field_info *info, int info_cnt, u32 field_cnt, u32 repeat_cnt, u32 elem_size) { - u32 i, j; + u32 i, j, total_cnt, total_repeats; u32 cur; /* Ensure not repeating fields that should not be repeated. */ @@ -3685,10 +3685,9 @@ static int btf_repeat_fields(struct btf_field_info *info, int info_cnt, } } - /* The type of struct size or variable size is u32, - * so the multiplication will not overflow. - */ - if (field_cnt * (repeat_cnt + 1) > info_cnt) + if (check_add_overflow(repeat_cnt, 1, &total_repeats) || + check_mul_overflow(field_cnt, total_repeats, &total_cnt) || + total_cnt > (u32)info_cnt) return -E2BIG; cur = field_cnt; -- cgit v1.2.3 From bb0c250e8e1132723795c1046442ceb01a5ed1b1 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Tue, 9 Jun 2026 14:33:56 +0200 Subject: timers/migration: Temporarily disable per capacity hierarchies Some workloads with different CPU capacities consume more power with timer migration than before. The recently introduced per capacity hierarchies were supposed to alleviate this problem. However it appears to also regress other types of workloads, especially when plenty of capacities live together in the same machine. Disable the feature until a reasonable solution is found. Fixes: 098cbaad8e57 ("timers/migration: Split per-capacity hierarchies") Reported-by: Christian Loehle Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Link: https://patch.msgid.link/20260609123356.28449-1-frederic@kernel.org Closes: https://lore.kernel.org/all/3b79338f-6cfc-4722-8062-9103db2c8ad1@arm.com --- kernel/time/timer_migration.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/time/timer_migration.c b/kernel/time/timer_migration.c index 548d84955f4c..e9d96d96e251 100644 --- a/kernel/time/timer_migration.c +++ b/kernel/time/timer_migration.c @@ -1473,7 +1473,7 @@ static unsigned int tmigr_get_capacity(int cpu) * timekeeper must then belong to the same hierarchy as all the nohz_full * CPUs. Simply turn off capacity awareness when nohz_full is running. */ - if (tick_nohz_full_enabled()) + if (tick_nohz_full_enabled() || !IS_ENABLED(CONFIG_BROKEN)) return SCHED_CAPACITY_SCALE; else return arch_scale_cpu_capacity(cpu); -- cgit v1.2.3 From fa75b7c85b0d2b6ab1c3ee0f06d35e2b98078c45 Mon Sep 17 00:00:00 2001 From: Nuoqi Gui Date: Tue, 9 Jun 2026 22:43:50 +0800 Subject: bpf: Enforce write checks for BTF pointer helper access check_mem_reg() verifies both read and write access for global subprogram memory arguments. When the caller register is PTR_TO_BTF_ID, check_helper_mem_access() currently forwards the access to check_ptr_to_btf_access() as BPF_READ regardless of the requested access type. This lets a BTF-backed kernel object field pointer pass the caller-side writable memory check for a global subprogram argument. The callee is then validated with a generic writable PTR_TO_MEM argument and can store through it, even though an equivalent direct BTF field store is rejected with "only read is supported". Forward the requested access type to check_ptr_to_btf_access(). This enforces existing BTF write restrictions for global subprogram memory arguments as well. Fixes: 3e30be4288b3 ("bpf: Allow helpers access trusted PTR_TO_BTF_ID.") Signed-off-by: Nuoqi Gui Link: https://lore.kernel.org/bpf/20260609-f01-04-btf-writable-arg-v1-1-f449cd970669@mails.tsinghua.edu.cn Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ed7ba0e6a9ce..cdff3e6eb96e 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6777,7 +6777,7 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ zero_size_allowed, access_type, meta); case PTR_TO_BTF_ID: return check_ptr_to_btf_access(env, regs, reg, argno, 0, - access_size, BPF_READ, -1); + access_size, access_type, -1); case PTR_TO_CTX: /* Only permit reading or writing syscall context using helper calls. */ if (is_var_ctx_off_allowed(env->prog)) { -- cgit v1.2.3 From 51a7c560cffdd3653ac2b930d01410569b23b23e Mon Sep 17 00:00:00 2001 From: Yuho Choi Date: Mon, 8 Jun 2026 13:07:48 -0400 Subject: PM: QoS: Fix misc device registration unwind cpu_latency_qos_init() registers cpu_dma_latency first and, when CONFIG_PM_QOS_CPU_SYSTEM_WAKEUP is enabled, registers cpu_wakeup_latency afterwards. The second registration overwrites the first return value. As a result, a failure to register cpu_dma_latency can be masked if the second registration succeeds. Conversely, if cpu_dma_latency succeeds and cpu_wakeup_latency fails, the function returns an error while leaving the first misc device registered. Return immediately on the first registration failure and deregister cpu_dma_latency if the second registration fails. Fixes: a4e6512a79d8 ("PM: QoS: Introduce a CPU system wakeup QoS limit") Signed-off-by: Yuho Choi Link: https://patch.msgid.link/20260608170748.82273-1-dbgh9129@gmail.com Signed-off-by: Rafael J. Wysocki --- kernel/power/qos.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/power/qos.c b/kernel/power/qos.c index 398b994b73aa..1944dbeb0d4c 100644 --- a/kernel/power/qos.c +++ b/kernel/power/qos.c @@ -519,18 +519,23 @@ static int __init cpu_latency_qos_init(void) int ret; ret = misc_register(&cpu_latency_qos_miscdev); - if (ret < 0) + if (ret < 0) { pr_err("%s: %s setup failed\n", __func__, cpu_latency_qos_miscdev.name); + return ret; + } #ifdef CONFIG_PM_QOS_CPU_SYSTEM_WAKEUP ret = misc_register(&cpu_wakeup_latency_qos_miscdev); - if (ret < 0) + if (ret < 0) { pr_err("%s: %s setup failed\n", __func__, cpu_wakeup_latency_qos_miscdev.name); + misc_deregister(&cpu_latency_qos_miscdev); + return ret; + } #endif - return ret; + return 0; } late_initcall(cpu_latency_qos_init); #endif /* CONFIG_CPU_IDLE */ -- cgit v1.2.3 From 2f884d371fafea137afea504d49ee4a7c8d7985b Mon Sep 17 00:00:00 2001 From: Vlad Poenaru Date: Tue, 9 Jun 2026 06:55:57 -0700 Subject: bpf: Allow LPM map access from sleepable BPF programs trie_lookup_elem() annotates its rcu_dereference_check() walks with only rcu_read_lock_bh_held(). Because rcu_dereference_check(p, c) resolves to "c || rcu_read_lock_held()", this passes for XDP/NAPI and classic RCU readers but fails for sleepable BPF programs, which enter via __bpf_prog_enter_sleepable() and hold only rcu_read_lock_trace(). trie_update_elem() and trie_delete_elem() have the same problem in a different form: they walk the trie with plain rcu_dereference(), which asserts rcu_read_lock_held() unconditionally. Both are reachable from sleepable BPF programs via the bpf_map_update_elem / bpf_map_delete_elem helpers, and from the syscall path under classic rcu_read_lock(). In the writer paths the trie is actually protected by trie->lock (an rqspinlock taken across the walk); we never relied on the RCU read-side lock to keep nodes alive there. A sleepable LSM hook that ends up touching an LPM trie therefore triggers lockdep on debug kernels: ============================= WARNING: suspicious RCU usage 7.1.0-... Tainted: G E ----------------------------- kernel/bpf/lpm_trie.c:249 suspicious rcu_dereference_check() usage! 1 lock held by net_tests/540: #0: (rcu_tasks_trace_srcu_struct){....}-{0:0}, at: __bpf_prog_enter_sleepable+0x26/0x280 Call Trace: dump_stack_lvl lockdep_rcu_suspicious trie_lookup_elem bpf_prog_..._enforce_security_socket_connect bpf_trampoline_... security_socket_connect __sys_connect do_syscall_64 This is lockdep-only -- no UAF, since Tasks Trace RCU does serialize against the trie's reclaim path -- but it spams the console once per distinct callsite on every debug kernel running a sleepable BPF LSM that touches an LPM trie, which is increasingly common. For the lookup path, switch the rcu_dereference_check() annotation from rcu_read_lock_bh_held() to bpf_rcu_lock_held(), which accepts all three contexts (classic, BH, Tasks Trace). Other map types already follow this convention. For trie_update_elem() and trie_delete_elem(), annotate the walks as rcu_dereference_protected(*p, 1) -- matching trie_free() in the same file -- since trie->lock is held across the walk. rqspinlock has no lockdep_map, so the predicate degenerates to '1' rather than lockdep_is_held(&trie->lock); the protection is real but not machine-verifiable. trie_get_next_key() also uses bare rcu_dereference() but is reachable only from the BPF syscall, which holds classic rcu_read_lock() before dispatching, so it is left untouched. Fixes: 694cea395fde ("bpf: Allow RCU-protected lookups to happen from bh context") Cc: stable@vger.kernel.org Signed-off-by: Vlad Poenaru Reviewed-by: Emil Tsalapatis Link: https://lore.kernel.org/r/20260609135558.193287-2-vlad.wing@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/lpm_trie.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/lpm_trie.c b/kernel/bpf/lpm_trie.c index 0f57608b385d..4d6f25db9ba1 100644 --- a/kernel/bpf/lpm_trie.c +++ b/kernel/bpf/lpm_trie.c @@ -246,7 +246,7 @@ static void *trie_lookup_elem(struct bpf_map *map, void *_key) /* Start walking the trie from the root node ... */ - for (node = rcu_dereference_check(trie->root, rcu_read_lock_bh_held()); + for (node = rcu_dereference_check(trie->root, bpf_rcu_lock_held()); node;) { unsigned int next_bit; size_t matchlen; @@ -280,7 +280,7 @@ static void *trie_lookup_elem(struct bpf_map *map, void *_key) */ next_bit = extract_bit(key->data, node->prefixlen); node = rcu_dereference_check(node->child[next_bit], - rcu_read_lock_bh_held()); + bpf_rcu_lock_held()); } if (!found) @@ -359,7 +359,7 @@ static long trie_update_elem(struct bpf_map *map, */ slot = &trie->root; - while ((node = rcu_dereference(*slot))) { + while ((node = rcu_dereference_protected(*slot, 1))) { matchlen = longest_prefix_match(trie, node, key); if (node->prefixlen != matchlen || @@ -482,7 +482,7 @@ static long trie_delete_elem(struct bpf_map *map, void *_key) trim = &trie->root; trim2 = trim; parent = NULL; - while ((node = rcu_dereference(*trim))) { + while ((node = rcu_dereference_protected(*trim, 1))) { matchlen = longest_prefix_match(trie, node, key); if (node->prefixlen != matchlen || -- cgit v1.2.3 From a3d76e27bbbf91d1025ce99eb55068ae0aa14322 Mon Sep 17 00:00:00 2001 From: Vlad Poenaru Date: Tue, 9 Jun 2026 06:55:58 -0700 Subject: bpf: Allow sleepable programs to use LPM trie maps directly The previous change relaxed the rcu_dereference annotations in lpm_trie.c so the trie walks no longer trip lockdep when reached from a sleepable BPF program holding only rcu_read_lock_trace(). By itself that only helps tries reached as the inner map of a map-of-maps, or from the classic-RCU syscall path: a sleepable program that references an LPM trie directly is still rejected at load time by check_map_prog_compatibility(), whose sleepable whitelist omits BPF_MAP_TYPE_LPM_TRIE: Sleepable programs can only use array, hash, ringbuf and local storage maps LPM trie nodes are allocated from a bpf_mem_alloc (trie->ma) and freed with bpf_mem_cache_free_rcu(), which chains a regular RCU grace period into a Tasks Trace grace period before the node -- and the value embedded in it that trie_lookup_elem() returns to the program -- is released. That is the same reclaim discipline BPF_MAP_TYPE_HASH relies on for sleepable access, so a value handed to a sleepable reader cannot be freed while the program is still running under rcu_read_lock_trace(). The writer paths take trie->lock across the walk and never relied on the RCU read-side lock to keep nodes alive. Add BPF_MAP_TYPE_LPM_TRIE to the sleepable map whitelist so these programs can use LPM tries directly. Signed-off-by: Vlad Poenaru Reviewed-by: Emil Tsalapatis Link: https://lore.kernel.org/r/20260609135558.193287-3-vlad.wing@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index cdff3e6eb96e..954b85609f32 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -17737,6 +17737,7 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, case BPF_MAP_TYPE_PERCPU_HASH: case BPF_MAP_TYPE_PERCPU_ARRAY: case BPF_MAP_TYPE_LRU_PERCPU_HASH: + case BPF_MAP_TYPE_LPM_TRIE: case BPF_MAP_TYPE_ARRAY_OF_MAPS: case BPF_MAP_TYPE_HASH_OF_MAPS: case BPF_MAP_TYPE_RINGBUF: -- cgit v1.2.3 From 94c8d1c21be40a845357854f98ec07e21bb14bc9 Mon Sep 17 00:00:00 2001 From: Justin Suess Date: Tue, 9 Jun 2026 22:25:43 +0200 Subject: bpf: Reject bpf_obj_drop() from tracing progs bpf_obj_drop() runs bpf_obj_free_fields() synchronously for program-allocated objects. When such an object contains NMI unsafe fields, tracing programs that can run from arbitrary instrumented context can reach that destruction from unsafe contexts, including NMI. NMI is likely one instance of this problem, and other instances would include possible unsafe reentrancy. Deferring bpf_obj_drop() is not appealing either: it would add delayed-free machinery to a release operation that otherwise has straightforward synchronous ownership semantics. Reject bpf_obj_drop() and bpf_percpu_obj_drop() from tracing programs that may run from unsafe contexts unless every field in the object's BTF record is explicitly NMI safe. Do not reject sleepable BPF_PROG_TYPE_TRACING programs, since they are not the arbitrary/NMI contexts that motivate the restriction. Note that while bpf_rb_root and bpf_list_head would be NMI safe on their own to free, the objects recursively held by them may not be; be conservative and just mark them as not NMI safe for now. Use a whitelist for the NMI-safe field set instead of listing only known NMI unsafe fields. Locks, async fields, unreferenced kptrs, and refcounts are known to be NMI safe because their destruction is either a no-op, simple state reset, or async cancellation. Referenced kptrs, percpu referenced kptrs, uptrs, graph roots, graph nodes, and any future field type are rejected until audited for arbitrary tracing and NMI contexts. This is less susceptible to future changes in fields that were previously safe by exclusion, and to new fields being added without updating this check. Convert the existing recursive local-object drop success case to a syscall program in the same commit, since this verifier change makes the old tracing program form invalid. The test still exercises bpf_obj_drop() releasing a referenced task kptr from a safe program type. Fixes: ac9f06050a35 ("bpf: Introduce bpf_obj_drop") Signed-off-by: Justin Suess Co-developed-by: Kumar Kartikeya Dwivedi Signed-off-by: Kumar Kartikeya Dwivedi Link: https://lore.kernel.org/r/20260609202548.3571690-2-memxor@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 954b85609f32..eb46a81a8c51 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -205,6 +205,7 @@ static int release_reference_nomark(struct bpf_verifier_state *state, int id); static int release_reference(struct bpf_verifier_env *env, int id); static void invalidate_non_owning_refs(struct bpf_verifier_env *env); static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env); +static bool is_tracing_prog_type(enum bpf_prog_type type); static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg); static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg); @@ -12881,6 +12882,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, int *insn_idx_p) { bool sleepable, rcu_lock, rcu_unlock, preempt_disable, preempt_enable; + enum bpf_prog_type prog_type = resolve_prog_type(env->prog); struct bpf_reg_state *regs = cur_regs(env); const char *func_name, *ptr_type_name; const struct btf_type *t, *ptr_type; @@ -12957,6 +12959,21 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err < 0) return err; + if ((is_bpf_obj_drop_kfunc(meta.func_id) || + is_bpf_percpu_obj_drop_kfunc(meta.func_id)) && (is_tracing_prog_type(prog_type) || + /* is_tracing_prog_type() for now doesn't cover non-iterator tracing progs. */ + (prog_type == BPF_PROG_TYPE_TRACING && env->prog->expected_attach_type != BPF_TRACE_ITER + && !env->prog->sleepable))) { + struct btf_struct_meta *struct_meta; + + struct_meta = btf_find_struct_meta(meta.arg_btf, meta.arg_btf_id); + if (struct_meta && btf_record_has_nmi_unsafe_fields(struct_meta->record)) { + verbose(env, "%s cannot be used in tracing programs on types with NMI unsafe fields\n", + func_name); + return -EINVAL; + } + } + if (is_bpf_rbtree_add_kfunc(meta.func_id)) { err = push_callback_call(env, insn, insn_idx, meta.subprogno, set_rbtree_add_callback_state); -- cgit v1.2.3 From a3a81d247651218e47153f2d2afd7aee236726fd Mon Sep 17 00:00:00 2001 From: Justin Suess Date: Tue, 9 Jun 2026 22:25:44 +0200 Subject: bpf: Cancel special fields on map value recycle Map update and delete paths currently call bpf_obj_free_fields() when a value is being replaced or recycled. That makes field destruction depend on the context of the update/delete operation. For tracing programs this can include NMI context, where referenced kptr destructors, uptr unpinning, and graph root destruction are not generally safe. Introduce bpf_obj_cancel_fields() for the reusable-value path. It only performs NMI-safe cleanup for timer, workqueue, and task_work fields. Fields that need full destruction are left attached to the recycled value and are destroyed by the final cleanup path instead. Switch array and hashtab update/delete/recycle paths to this cancel helper. Keep bpf_obj_free_fields() for final map destruction and for bpf_mem_alloc destructors. Preallocated hashtabs do not have allocator destructors, so teardown continues to walk the normal and extra elements and fully destroy their fields. This deliberately relaxes the eager-free semantics of map update/delete for special fields. Programs that relied on a recycled map slot becoming empty immediately after update/delete were relying on behavior that cannot be implemented safely from every BPF execution context without offloading arbitrary destructors. There is a chance this change breaks programs making assumptions regarding the eager freeing of fields. If so, we can relax semantics to cancellation only when irqs_disabled() is true in the future. However, theoretically, map values that get reused eagerly already have weaker guarantees as parallel users can recreate freed fields before the new element becomes visible again. Fixes: 14a324f6a67e ("bpf: Wire up freeing of referenced kptr") Signed-off-by: Justin Suess Co-developed-by: Kumar Kartikeya Dwivedi Signed-off-by: Kumar Kartikeya Dwivedi Link: https://lore.kernel.org/r/20260609202548.3571690-3-memxor@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/arraymap.c | 8 ++++---- kernel/bpf/hashtab.c | 32 ++++++++++++++++++-------------- kernel/bpf/syscall.c | 5 +++++ 3 files changed, 27 insertions(+), 18 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index e6271a2bf6d6..248b4818178c 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -384,7 +384,7 @@ static long array_map_update_elem(struct bpf_map *map, void *key, void *value, if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { val = this_cpu_ptr(array->pptrs[index & array->index_mask]); copy_map_value(map, val, value); - bpf_obj_free_fields(array->map.record, val); + bpf_obj_cancel_fields(map, val); } else { val = array->value + (u64)array->elem_size * (index & array->index_mask); @@ -392,7 +392,7 @@ static long array_map_update_elem(struct bpf_map *map, void *key, void *value, copy_map_value_locked(map, val, value, false); else copy_map_value(map, val, value); - bpf_obj_free_fields(array->map.record, val); + bpf_obj_cancel_fields(map, val); } return 0; } @@ -432,14 +432,14 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, cpu = map_flags >> 32; ptr = per_cpu_ptr(pptr, cpu); copy_map_value(map, ptr, value); - bpf_obj_free_fields(array->map.record, ptr); + bpf_obj_cancel_fields(map, ptr); goto unlock; } for_each_possible_cpu(cpu) { ptr = per_cpu_ptr(pptr, cpu); val = (map_flags & BPF_F_ALL_CPUS) ? value : value + size * cpu; copy_map_value(map, ptr, val); - bpf_obj_free_fields(array->map.record, ptr); + bpf_obj_cancel_fields(map, ptr); } unlock: rcu_read_unlock(); diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index b4366cad3cfa..9f394e1aa2e8 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -243,6 +243,10 @@ static void htab_free_prealloced_fields(struct bpf_htab *htab) if (IS_ERR_OR_NULL(htab->map.record)) return; + /* + * Preallocated maps do not have a bpf_mem_alloc destructor, so fully + * destroy every element, including the extra elements. + */ if (htab_has_extra_elems(htab)) num_entries += num_possible_cpus(); for (i = 0; i < num_entries; i++) { @@ -833,8 +837,8 @@ static int htab_lru_map_gen_lookup(struct bpf_map *map, return insn - insn_buf; } -static void check_and_free_fields(struct bpf_htab *htab, - struct htab_elem *elem) +static void check_and_cancel_fields(struct bpf_htab *htab, + struct htab_elem *elem) { if (IS_ERR_OR_NULL(htab->map.record)) return; @@ -844,11 +848,11 @@ static void check_and_free_fields(struct bpf_htab *htab, int cpu; for_each_possible_cpu(cpu) - bpf_obj_free_fields(htab->map.record, per_cpu_ptr(pptr, cpu)); + bpf_obj_cancel_fields(&htab->map, per_cpu_ptr(pptr, cpu)); } else { void *map_value = htab_elem_value(elem, htab->map.key_size); - bpf_obj_free_fields(htab->map.record, map_value); + bpf_obj_cancel_fields(&htab->map, map_value); } } @@ -883,7 +887,7 @@ static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node) htab_unlock_bucket(b, flags); if (l == tgt_l) - check_and_free_fields(htab, l); + check_and_cancel_fields(htab, l); return l == tgt_l; } @@ -948,7 +952,7 @@ find_first_elem: static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l) { - check_and_free_fields(htab, l); + check_and_cancel_fields(htab, l); if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH) bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr); @@ -1001,7 +1005,7 @@ static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l) if (htab_is_prealloc(htab)) { bpf_map_dec_elem_count(&htab->map); - check_and_free_fields(htab, l); + check_and_cancel_fields(htab, l); pcpu_freelist_push(&htab->freelist, &l->fnode); } else { dec_elem_count(htab); @@ -1018,7 +1022,7 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr, /* copy true value_size bytes */ ptr = this_cpu_ptr(pptr); copy_map_value(&htab->map, ptr, value); - bpf_obj_free_fields(htab->map.record, ptr); + bpf_obj_cancel_fields(&htab->map, ptr); } else { u32 size = round_up(htab->map.value_size, 8); void *val; @@ -1028,7 +1032,7 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr, cpu = map_flags >> 32; ptr = per_cpu_ptr(pptr, cpu); copy_map_value(&htab->map, ptr, value); - bpf_obj_free_fields(htab->map.record, ptr); + bpf_obj_cancel_fields(&htab->map, ptr); return; } @@ -1036,7 +1040,7 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr, ptr = per_cpu_ptr(pptr, cpu); val = (map_flags & BPF_F_ALL_CPUS) ? value : value + size * cpu; copy_map_value(&htab->map, ptr, val); - bpf_obj_free_fields(htab->map.record, ptr); + bpf_obj_cancel_fields(&htab->map, ptr); } } } @@ -1252,11 +1256,11 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, if (l_old) { hlist_nulls_del_rcu(&l_old->hash_node); - /* l_old has already been stashed in htab->extra_elems, free - * its special fields before it is available for reuse. + /* l_old has already been stashed in htab->extra_elems, cancel + * its reusable special fields before it is available for reuse. */ if (htab_is_prealloc(htab)) - check_and_free_fields(htab, l_old); + check_and_cancel_fields(htab, l_old); } htab_unlock_bucket(b, flags); if (l_old && !htab_is_prealloc(htab)) @@ -1269,7 +1273,7 @@ err: static void htab_lru_push_free(struct bpf_htab *htab, struct htab_elem *elem) { - check_and_free_fields(htab, elem); + check_and_cancel_fields(htab, elem); bpf_map_dec_elem_count(&htab->map); bpf_lru_push_free(&htab->lru, &elem->lru_node); } diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index d4188a992bd8..7ed949f70f82 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -808,6 +808,11 @@ void bpf_obj_free_task_work(const struct btf_record *rec, void *obj) bpf_task_work_cancel_and_free(obj + rec->task_work_off); } +void bpf_obj_cancel_fields(struct bpf_map *map, void *obj) +{ + bpf_map_free_internal_structs(map, obj); +} + void bpf_obj_free_fields(const struct btf_record *rec, void *obj) { const struct btf_field *fields; -- cgit v1.2.3 From 476a847da37ea090bb7e05c396ff54c116372c3e Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Sun, 22 Mar 2026 16:19:55 +0100 Subject: vhost_task_create: kill unnecessary .exit_signal initialization The only reason for this janitorial change is that this initialization adds unnecessary noise to "git grep exit_signal". args.exit_signal has no effect with CLONE_THREAD, not to mention it is zero-initialized by the compiler anyway. Signed-off-by: Oleg Nesterov Acked-by: Jason Wang Signed-off-by: Michael S. Tsirkin Message-ID: --- kernel/vhost_task.c | 1 - 1 file changed, 1 deletion(-) (limited to 'kernel') diff --git a/kernel/vhost_task.c b/kernel/vhost_task.c index 3f1ed7ef0582..3717885a5992 100644 --- a/kernel/vhost_task.c +++ b/kernel/vhost_task.c @@ -123,7 +123,6 @@ struct vhost_task *vhost_task_create(bool (*fn)(void *), struct kernel_clone_args args = { .flags = CLONE_FS | CLONE_UNTRACED | CLONE_VM | CLONE_THREAD | CLONE_SIGHAND, - .exit_signal = 0, .fn = vhost_task_fn, .name = name, .user_worker = 1, -- cgit v1.2.3 From 10627ddc0167aab5c1c390a10ef461e9937aba08 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Wed, 10 Jun 2026 12:55:38 +0200 Subject: bpf: Tighten cgroup storage cookie checks for prog arrays The fix in commit abad3d0bad72 ("bpf: Fix oob access in cgroup local storage") is still incomplete. The prog-array compatibility check treats a program with no cgroup storage as compatible with any stored storage cookie. This allows a storage-less program to bridge a tail call chain between an entry program and a storage-using callee even though cgroup local storage at runtime still follows the caller's context, that is, A -> B(no storage) -> C(storage) path. Requiring exact cookie equality would break the legitimate case of a storage-less leaf program being tail called from a storage-using one. Instead, only accept a zero storage cookie if the program cannot perform tail calls itself. This keeps A -> B(no storage) working while rejecting the A -> B(no storage) -> C(storage) bridge. Fixes: abad3d0bad72 ("bpf: Fix oob access in cgroup local storage") Reported-by: Lin Ma Signed-off-by: Daniel Borkmann Acked-by: Yonghong Song Link: https://lore.kernel.org/r/20260610105539.705887-1-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- kernel/bpf/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index a656a8572bdb..649cce41e13f 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -2481,7 +2481,7 @@ static bool __bpf_prog_map_compatible(struct bpf_map *map, cookie = aux->cgroup_storage[i] ? aux->cgroup_storage[i]->cookie : 0; ret = map->owner->storage_cookie[i] == cookie || - !cookie; + (!cookie && !aux->tail_call_reachable); } if (ret && map->owner->attach_func_proto != aux->attach_func_proto) { -- cgit v1.2.3 From c095741713d1bc317b53e2da2b222e7448b6021f Mon Sep 17 00:00:00 2001 From: Aaron Lu Date: Wed, 3 Jun 2026 17:51:08 +0800 Subject: sched/fair: Fix newidle vs core-sched While testing Prateek's throttle series, I noticed a panic issue when coresched is enabled and bisected to this patch. I fed the panic log and this patch to an agent and its analysis looks correct to me(cpu56 and cpu57 are siblings in a VM): cpu57 (holds core-wide lock) pick_next_task() [core scheduling] for_each_cpu_wrap(i, smt_mask, 57): i=57: pick_task(rq_57) pick_task_fair(rq_57) -> picks task A rq_57->core_pick = task A // task_rq(A) == rq_57 i=56: pick_task(rq_56) pick_task_fair(rq_56) cfs_rq->nr_queued == 0 goto idle sched_balance_newidle(rq_56) raw_spin_rq_unlock(rq_56) // core-wide lock released newidle_balance() pulls task A: rq_57 -> rq_56 // task_rq(A) == rq_56 now raw_spin_rq_lock(rq_56) // core-wide lock re-acquired return > 0 goto again pick_task_fair(rq_56) -> picks task A rq_56->core_pick = task A // first loop done // rq_57->core_pick is still task A (set before lock release) // but task_rq(A) == rq_56 now next = rq_57->core_pick // = task A put_prev_set_next_task(rq_57, prev, task A) __set_next_task_fair(rq_57, task A) hrtick_start_fair(rq_57, task A) WARN_ON_ONCE(task_rq(task A) != rq_57) // task_rq(A) == rq_56 IOW: by allowing pick_task_fair() to do newidle_balance and not returning RETRY_TASK, it can end up selecting the same task on two CPUs. Restore the previous state by never doing newidle when core scheduling is enabled. Tested-by: Sven Schnelle Signed-off-by: "Aaron Lu" Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260603095108.GA1684319@bytedance.com --- kernel/sched/fair.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 1b23e73f48b0..d78467ec6ee1 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -9942,6 +9942,9 @@ again: return p; idle: + if (sched_core_enabled(rq)) + return NULL; + new_tasks = sched_balance_newidle(rq, rf); if (new_tasks < 0) return RETRY_TASK; -- cgit v1.2.3 From a734d9fca84e1d4fa0cb442ef5f84c88f8212d32 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 10 Jun 2026 23:20:09 +0200 Subject: futex: Optimize futex hash bucket access patterns Breno reported significant c2c HITM in a futex hash heavy workload. It turns out that the hash bucket to private hash table reverse pointer (futex_hash_bucket::priv) was to blame. Notably when the hash buckets are heavily contended, the: 'fph = bh->priv;' load in futex_hash() will typically miss and consequently become quite expensive. Since this load in particular is quite superfluous, removing it is fairly straight forward. However, removing it does not in fact achieve anything much. The pain moves to the next user, notably: futex_hash_put(). Therefore rework the whole private hash refcounting to avoid needing this back pointer (and removing it). Instead of passing around 'struct futex_hash_bucket *hb', pass around a new structure that contains it and the related 'struct futex_private_hash *fph' pointer in tandem. Funnily this turns out to remove more code than it adds and significantly improves futex hash performance (as measured by 'perf bench futex hash'): SKL dual socket 112 threads: Baseline Patched shared (16k) 1571857 1641435 + 4.4% autosize (512) 646390 903371 +39.7% -b 256 464395 587014 +26.4% -b 512 715687 995943 +39.2% -b 1024 995085 1396328 +40.3% -b 2048 1293114 1668395 +29.0% -b 4096 2124438 2240228 + 5.5% Zen3 dual socket 256 threads: Baseline Patched shared (16k) 1275840 1381279 + 8.2% autosize (512) 1252745 1482179 +18.3% -b 256 856274 955455 +11.5% -b 512 1267490 1544010 +21.8% -b 1024 1424013 1625424 +14.1% -b 2048 1505181 1669342 +10.9% -b 4096 1465993 1688932 +15.2% AMD EPYC 9D64 (Zen4, single socket) 176 threads: Baseline Patched Delta shared (16k) 1,230,599 1,368,655 +11.2% autosize (1024) 1,285,440 1,556,946 +21.1% -b 256 1,341,471 1,520,303 +13.3% -b 512 1,438,330 1,599,319 +11.2% -b 1024 1,443,772 1,622,493 +12.4% -b 2048 1,472,108 1,643,975 +11.7% -b 4096 1,333,098 1,570,897 +17.8% Reported-by: Breno Leitao Signed-off-by: Peter Zijlstra (Intel) Tested-by: Breno Leitao Tested-by: Thomas Gleixner Link: https://patch.msgid.link/20260610135510.GB1430057@noisy.programming.kicks-ass.net --- kernel/futex/core.c | 111 ++++++++++++++---------------------------------- kernel/futex/futex.h | 37 ++++++++++------ kernel/futex/pi.c | 21 ++++----- kernel/futex/requeue.c | 20 ++++----- kernel/futex/waitwake.c | 17 +++++--- 5 files changed, 87 insertions(+), 119 deletions(-) (limited to 'kernel') diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 6ea4a97796a1..179b26e9c934 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -127,7 +127,7 @@ late_initcall(fail_futex_debugfs); #endif /* CONFIG_FAIL_FUTEX */ static struct futex_hash_bucket * -__futex_hash(union futex_key *key, struct futex_private_hash *fph); +__futex_hash(union futex_key *key, struct futex_private_hash *fph, struct futex_private_hash **fph_p); #ifdef CONFIG_FUTEX_PRIVATE_HASH static bool futex_ref_get(struct futex_private_hash *fph); @@ -136,15 +136,6 @@ static bool futex_ref_is_dead(struct futex_private_hash *fph); enum { FR_PERCPU = 0, FR_ATOMIC }; -static inline bool futex_key_is_private(union futex_key *key) -{ - /* - * Relies on get_futex_key() to set either bit for shared - * futexes -- see comment with union futex_key. - */ - return !(key->both.offset & (FUT_OFF_INODE | FUT_OFF_MMSHARED)); -} - static bool futex_private_hash_get(struct futex_private_hash *fph) { return futex_ref_get(fph); @@ -152,48 +143,15 @@ static bool futex_private_hash_get(struct futex_private_hash *fph) void futex_private_hash_put(struct futex_private_hash *fph) { - if (futex_ref_put(fph)) + if (fph && futex_ref_put(fph)) wake_up_var(fph->mm); } -/** - * futex_hash_get - Get an additional reference for the local hash. - * @hb: ptr to the private local hash. - * - * Obtain an additional reference for the already obtained hash bucket. The - * caller must already own an reference. - */ -void futex_hash_get(struct futex_hash_bucket *hb) -{ - struct futex_private_hash *fph = hb->priv; - - if (!fph) - return; - WARN_ON_ONCE(!futex_private_hash_get(fph)); -} - -void futex_hash_put(struct futex_hash_bucket *hb) -{ - struct futex_private_hash *fph = hb->priv; - - if (!fph) - return; - futex_private_hash_put(fph); -} - static struct futex_hash_bucket * __futex_hash_private(union futex_key *key, struct futex_private_hash *fph) { u32 hash; - if (!futex_key_is_private(key)) - return NULL; - - if (!fph) - fph = rcu_dereference(key->private.mm->futex.phash.hash); - if (!fph || !fph->hash_mask) - return NULL; - hash = jhash2((void *)&key->private.address, sizeof(key->private.address) / 4, key->both.offset); @@ -214,13 +172,12 @@ static void futex_rehash_private(struct futex_private_hash *old, spin_lock(&hb_old->lock); plist_for_each_entry_safe(this, tmp, &hb_old->chain, list) { - plist_del(&this->list, &hb_old->chain); futex_hb_waiters_dec(hb_old); WARN_ON_ONCE(this->lock_ptr != &hb_old->lock); - hb_new = __futex_hash(&this->key, new); + hb_new = __futex_hash(&this->key, new, NULL); futex_hb_waiters_inc(hb_new); /* * The new pointer isn't published yet but an already @@ -273,9 +230,8 @@ static void futex_pivot_hash(struct mm_struct *mm) } } -struct futex_private_hash *futex_private_hash(void) +struct futex_private_hash *futex_private_hash(struct mm_struct *mm) { - struct mm_struct *mm = current->mm; /* * Ideally we don't loop. If there is a replacement in progress * then a new private hash is already prepared and a reference can't be @@ -301,18 +257,17 @@ again: goto again; } -struct futex_hash_bucket *futex_hash(union futex_key *key) +struct futex_bucket_ref futex_hash(union futex_key *key) { - struct futex_private_hash *fph; - struct futex_hash_bucket *hb; - again: scoped_guard(rcu) { - hb = __futex_hash(key, NULL); - fph = hb->priv; + struct futex_private_hash *fph = NULL; + struct futex_hash_bucket *hb; + + hb = __futex_hash(key, NULL, &fph); if (!fph || futex_private_hash_get(fph)) - return hb; + return (struct futex_bucket_ref){ .hb = hb, .fph = fph }; } futex_pivot_hash(key->private.mm); goto again; @@ -320,15 +275,9 @@ again: #else /* !CONFIG_FUTEX_PRIVATE_HASH */ -static struct futex_hash_bucket * -__futex_hash_private(union futex_key *key, struct futex_private_hash *fph) -{ - return NULL; -} - -struct futex_hash_bucket *futex_hash(union futex_key *key) +struct futex_bucket_ref futex_hash(union futex_key *key) { - return __futex_hash(key, NULL); + return (struct futex_bucket_ref){ .hb = __futex_hash(key, NULL, NULL), .fph = NULL }; } #endif /* CONFIG_FUTEX_PRIVATE_HASH */ @@ -406,6 +355,8 @@ static int futex_mpol(struct mm_struct *mm, unsigned long addr) * __futex_hash - Return the hash bucket * @key: Pointer to the futex key for which the hash is calculated * @fph: Pointer to private hash if known + * @fph_p: Pointer to a private hash pointer; output for the private hash + * used when set. * * We hash on the keys returned from get_futex_key (see below) and return the * corresponding hash bucket. @@ -413,18 +364,23 @@ static int futex_mpol(struct mm_struct *mm, unsigned long addr) * private hash) is returned if existing. Otherwise a hash bucket from the * global hash is returned. */ -static struct futex_hash_bucket *__futex_hash(union futex_key *key, struct futex_private_hash *fph) +static struct futex_hash_bucket * +__futex_hash(union futex_key *key, struct futex_private_hash *fph, struct futex_private_hash **fph_p) { int node = key->both.node; u32 hash; - if (node == FUTEX_NO_NODE) { - struct futex_hash_bucket *hb; - - hb = __futex_hash_private(key, fph); - if (hb) - return hb; +#ifdef CONFIG_FUTEX_PRIVATE_HASH + if (node == FUTEX_NO_NODE && futex_key_is_private(key)) { + if (!fph) + fph = rcu_dereference(key->private.mm->futex.phash.hash); + if (fph && fph->hash_mask) { + if (fph_p) + *fph_p = fph; + return __futex_hash_private(key, fph); + } } +#endif hash = jhash2((u32 *)key, offsetof(typeof(*key), both.offset) / sizeof(u32), key->both.offset); @@ -1367,7 +1323,7 @@ static void exit_pi_state_list(struct task_struct *curr) * on the mutex. */ WARN_ON(curr != current); - guard(private_hash)(); + guard(private_hash)(current->mm); /* * We are a ZOMBIE and nobody can enqueue itself on * pi_state_list anymore, but we have to be careful @@ -1379,7 +1335,8 @@ static void exit_pi_state_list(struct task_struct *curr) pi_state = list_entry(next, struct futex_pi_state, list); key = pi_state->key; if (1) { - CLASS(hb, hb)(&key); + CLASS(hbr, hbr)(&key); + auto hb = hbr.hb; /* * We can race against put_pi_state() removing itself from the @@ -1576,12 +1533,8 @@ void futex_exit_release(struct task_struct *tsk) futex_cleanup_end(tsk, FUTEX_STATE_DEAD); } -static void futex_hash_bucket_init(struct futex_hash_bucket *fhb, - struct futex_private_hash *fph) +static void futex_hash_bucket_init(struct futex_hash_bucket *fhb) { -#ifdef CONFIG_FUTEX_PRIVATE_HASH - fhb->priv = fph; -#endif atomic_set(&fhb->waiters, 0); plist_head_init(&fhb->chain); spin_lock_init(&fhb->lock); @@ -1877,7 +1830,7 @@ static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags) fph->mm = mm; for (i = 0; i < hash_slots; i++) - futex_hash_bucket_init(&fph->queues[i], fph); + futex_hash_bucket_init(&fph->queues[i]); if (custom) { /* @@ -2082,7 +2035,7 @@ static int __init futex_init(void) BUG_ON(!table); for (i = 0; i < hashsize; i++) - futex_hash_bucket_init(&table[i], NULL); + futex_hash_bucket_init(&table[i]); futex_queues[n] = table; } diff --git a/kernel/futex/futex.h b/kernel/futex/futex.h index 79ef2c709c81..f00f0863ed44 100644 --- a/kernel/futex/futex.h +++ b/kernel/futex/futex.h @@ -134,6 +134,15 @@ static inline bool should_fail_futex(bool fshared) } #endif +static inline bool futex_key_is_private(union futex_key *key) +{ + /* + * Relies on get_futex_key() to set either bit for shared + * futexes -- see comment with union futex_key. + */ + return !(key->both.offset & (FUT_OFF_INODE | FUT_OFF_MMSHARED)); +} + /* * Hash buckets are shared by all the futex_keys that hash to the same * location. Each key may have multiple futex_q structures, one for each task @@ -143,7 +152,6 @@ struct futex_hash_bucket { atomic_t waiters; spinlock_t lock; struct plist_head chain; - struct futex_private_hash *priv; } ____cacheline_aligned_in_smp; /* @@ -183,7 +191,7 @@ typedef void (futex_wake_fn)(struct wake_q_head *wake_q, struct futex_q *q); * @requeue_pi_key: the requeue_pi target futex key * @bitset: bitset for the optional bitmasked wakeup * @requeue_state: State field for futex_requeue_pi() - * @drop_hb_ref: Waiter should drop the extra hash bucket reference if true + * @drop_fph: Waiter should drop the extra private hash reference when set * @requeue_wait: RCU wait for futex_requeue_pi() (RT only) * * We use this hashed waitqueue, instead of a normal wait_queue_entry_t, so @@ -210,7 +218,7 @@ struct futex_q { union futex_key *requeue_pi_key; u32 bitset; atomic_t requeue_state; - bool drop_hb_ref; + struct futex_private_hash *drop_fph; #ifdef CONFIG_PREEMPT_RT struct rcuwait requeue_wait; #endif @@ -230,28 +238,29 @@ extern struct hrtimer_sleeper * futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout, int flags, u64 range_ns); -extern struct futex_hash_bucket *futex_hash(union futex_key *key); -#ifdef CONFIG_FUTEX_PRIVATE_HASH -extern void futex_hash_get(struct futex_hash_bucket *hb); -extern void futex_hash_put(struct futex_hash_bucket *hb); +struct futex_bucket_ref { + struct futex_hash_bucket *hb; + struct futex_private_hash *fph; +}; -extern struct futex_private_hash *futex_private_hash(void); +#ifdef CONFIG_FUTEX_PRIVATE_HASH +extern struct futex_private_hash *futex_private_hash(struct mm_struct *mm); extern void futex_private_hash_put(struct futex_private_hash *fph); #else /* !CONFIG_FUTEX_PRIVATE_HASH */ -static inline void futex_hash_get(struct futex_hash_bucket *hb) { } -static inline void futex_hash_put(struct futex_hash_bucket *hb) { } -static inline struct futex_private_hash *futex_private_hash(void) { return NULL; } +static inline struct futex_private_hash *futex_private_hash(struct mm_struct *mm) { return NULL; } static inline void futex_private_hash_put(struct futex_private_hash *fph) { } #endif -DEFINE_CLASS(hb, struct futex_hash_bucket *, - if (_T) futex_hash_put(_T), +extern struct futex_bucket_ref futex_hash(union futex_key *key); + +DEFINE_CLASS(hbr, struct futex_bucket_ref, + if (_T.fph) futex_private_hash_put(_T.fph), futex_hash(key), union futex_key *key); DEFINE_CLASS(private_hash, struct futex_private_hash *, if (_T) futex_private_hash_put(_T), - futex_private_hash(), void); + futex_private_hash(mm), struct mm_struct *mm); /** * futex_match - Check whether two futex keys are equal diff --git a/kernel/futex/pi.c b/kernel/futex/pi.c index 9dd5c0b1ac6a..795011ea1202 100644 --- a/kernel/futex/pi.c +++ b/kernel/futex/pi.c @@ -945,7 +945,8 @@ retry: retry_private: if (1) { - CLASS(hb, hb)(&q.key); + CLASS(hbr, hbr)(&q.key); + auto hb = hbr.hb; futex_q_lock(&q, hb); @@ -1009,7 +1010,7 @@ retry_private: * the thread, performing resize, will block on hb->lock during * the requeue. */ - futex_hash_put(no_free_ptr(hb)); + futex_private_hash_put(no_free_ptr(hbr.fph)); /* * Must be done before we enqueue the waiter, here is unfortunately * under the hb lock, but that *should* work because it does nothing. @@ -1100,11 +1101,9 @@ no_block: __release(&hb->lock); futex_unqueue_pi(&q); spin_unlock(q.lock_ptr); - if (q.drop_hb_ref) { - CLASS(hb, hb)(&q.key); - /* Additional reference from futex_unlock_pi() */ - futex_hash_put(hb); - } + + /* Additional reference from futex_unlock_pi() */ + futex_private_hash_put(q.drop_fph); goto out; out_unlock_put_key: @@ -1161,7 +1160,8 @@ retry: if (ret) return ret; - CLASS(hb, hb)(&key); + CLASS(hbr, hbr)(&key); + auto hb = hbr.hb; spin_lock(&hb->lock); retry_hb: @@ -1218,8 +1218,9 @@ retry_hb: * Acquire a reference for the leaving waiter to ensure * valid futex_q::lock_ptr. */ - futex_hash_get(hb); - top_waiter->drop_hb_ref = true; + if (futex_key_is_private(&key)) + top_waiter->drop_fph = futex_private_hash(key.private.mm); + __futex_unqueue(top_waiter); raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); goto retry_hb; diff --git a/kernel/futex/requeue.c b/kernel/futex/requeue.c index b597cb3d17fc..79823ad13683 100644 --- a/kernel/futex/requeue.c +++ b/kernel/futex/requeue.c @@ -241,8 +241,8 @@ void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key, * Acquire a reference for the waiter to ensure valid * futex_q::lock_ptr. */ - futex_hash_get(hb); - q->drop_hb_ref = true; + if (futex_key_is_private(key)) + q->drop_fph = futex_private_hash(key->private.mm); q->lock_ptr = &hb->lock; task = READ_ONCE(q->task); @@ -459,8 +459,10 @@ retry: retry_private: if (1) { - CLASS(hb, hb1)(&key1); - CLASS(hb, hb2)(&key2); + CLASS(hbr, hbr1)(&key1); + CLASS(hbr, hbr2)(&key2); + auto hb1 = hbr1.hb; + auto hb2 = hbr2.hb; futex_hb_waiters_inc(hb2); double_lock_hb(hb1, hb2); @@ -832,7 +834,8 @@ int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, switch (futex_requeue_pi_wakeup_sync(&q)) { case Q_REQUEUE_PI_IGNORE: { - CLASS(hb, hb)(&q.key); + CLASS(hbr, hbr)(&q.key); + auto hb = hbr.hb; /* The waiter is still on uaddr1 */ spin_lock(&hb->lock); ret = handle_early_requeue_pi_wakeup(hb, &q, to); @@ -902,11 +905,8 @@ int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, default: BUG(); } - if (q.drop_hb_ref) { - CLASS(hb, hb)(&q.key); - /* Additional reference from requeue_pi_wake_futex() */ - futex_hash_put(hb); - } + /* Additional reference from requeue_pi_wake_futex() */ + futex_private_hash_put(q.drop_fph); out: if (to) { diff --git a/kernel/futex/waitwake.c b/kernel/futex/waitwake.c index 8f5e5d302356..d4483d15d30a 100644 --- a/kernel/futex/waitwake.c +++ b/kernel/futex/waitwake.c @@ -195,7 +195,8 @@ int futex_wake(u32 __user *uaddr, unsigned int flags, void __user *pop, int nr_w if ((flags & FLAGS_STRICT) && !nr_wake) return 0; - CLASS(hb, hb)(&key); + CLASS(hbr, hbr)(&key); + auto hb = hbr.hb; /* Make sure we really have tasks to wakeup */ if (!futex_hb_waiters_pending(hb)) @@ -292,8 +293,10 @@ retry: retry_private: if (1) { - CLASS(hb, hb1)(&key1); - CLASS(hb, hb2)(&key2); + CLASS(hbr, hbr1)(&key1); + CLASS(hbr, hbr2)(&key2); + auto hb1 = hbr1.hb; + auto hb2 = hbr2.hb; double_lock_hb(hb1, hb2); op_ret = futex_atomic_op_inuser(op, uaddr2); @@ -435,7 +438,7 @@ int futex_wait_multiple_setup(struct futex_vector *vs, int count, int *woken) * Make sure to have a reference on the private_hash such that we * don't block on rehash after changing the task state below. */ - guard(private_hash)(); + guard(private_hash)(current->mm); /* * Enqueuing multiple futexes is tricky, because we need to enqueue @@ -472,7 +475,8 @@ retry: u32 val = vs[i].w.val; if (1) { - CLASS(hb, hb)(&q->key); + CLASS(hbr, hbr)(&q->key); + auto hb = hbr.hb; futex_q_lock(q, hb); ret = futex_get_value_locked(&uval, uaddr); @@ -647,7 +651,8 @@ retry: retry_private: if (1) { - CLASS(hb, hb)(&q->key); + CLASS(hbr, hbr)(&q->key); + auto hb = hbr.hb; futex_q_lock(q, hb); -- cgit v1.2.3 From 7cfa62cf9432df67b1c95a59984a03a3bc023f98 Mon Sep 17 00:00:00 2001 From: Dmitry Ilvokhin Date: Thu, 4 Jun 2026 07:15:06 +0000 Subject: locking/percpu-rwsem: Extract __percpu_up_read() Move the percpu_up_read() slowpath out of the inline function into a new __percpu_up_read() to avoid binary size increase from adding a tracepoint to an inlined function. Signed-off-by: Dmitry Ilvokhin Signed-off-by: Peter Zijlstra (Intel) Acked-by: Usama Arif Link: https://patch.msgid.link/3dd2a1b9ab4f469e1892766cb63f41d6b0f53d29.1780506267.git.d@ilvokhin.com --- kernel/locking/percpu-rwsem.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'kernel') diff --git a/kernel/locking/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c index ef234469baac..f3ee7a0d6047 100644 --- a/kernel/locking/percpu-rwsem.c +++ b/kernel/locking/percpu-rwsem.c @@ -288,3 +288,21 @@ void percpu_up_write(struct percpu_rw_semaphore *sem) rcu_sync_exit(&sem->rss); } EXPORT_SYMBOL_GPL(percpu_up_write); + +void __percpu_up_read(struct percpu_rw_semaphore *sem) +{ + lockdep_assert_preemption_disabled(); + /* + * slowpath; reader will only ever wake a single blocked + * writer. + */ + smp_mb(); /* B matches C */ + /* + * In other words, if they see our decrement (presumably to + * aggregate zero, as that is the only time it matters) they + * will also see our critical section. + */ + this_cpu_dec(*sem->read_count); + rcuwait_wake_up(&sem->writer); +} +EXPORT_SYMBOL_GPL(__percpu_up_read); -- cgit v1.2.3 From 4f070ccb4dc4692e3b6757819fb80655f58b4f12 Mon Sep 17 00:00:00 2001 From: Dmitry Ilvokhin Date: Thu, 4 Jun 2026 07:15:07 +0000 Subject: locking: Add contended_release tracepoint to sleepable locks Add the contended_release trace event. This tracepoint fires on the holder side when a contended lock is released, complementing the existing contention_begin/contention_end tracepoints which fire on the waiter side. This enables correlating lock hold time under contention with waiter events by lock address. Add trace_contended_release()/trace_call__contended_release() calls to the slowpath unlock paths of sleepable locks: mutex, rtmutex, semaphore, rwsem, percpu-rwsem, and RT-specific rwbase locks. Where possible, trace_contended_release() fires before the lock is released and before the waiter is woken. For some lock types, the tracepoint fires after the release but before the wake. Making the placement consistent across all lock types is not worth the added complexity. For reader/writer locks, the tracepoint fires for every reader releasing while a writer is waiting, not only for the last reader. Signed-off-by: Dmitry Ilvokhin Signed-off-by: Peter Zijlstra (Intel) Acked-by: Paul E. McKenney Acked-by: Usama Arif Link: https://patch.msgid.link/02f4f6c5ce6761e7f6587cf0ff2289d962ecddd4.1780506267.git.d@ilvokhin.com --- kernel/locking/mutex.c | 4 ++++ kernel/locking/percpu-rwsem.c | 11 +++++++++++ kernel/locking/rtmutex.c | 1 + kernel/locking/rwbase_rt.c | 6 ++++++ kernel/locking/rwsem.c | 10 ++++++++-- kernel/locking/semaphore.c | 4 ++++ 6 files changed, 34 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c index 09534628dc01..43b7f7e281a0 100644 --- a/kernel/locking/mutex.c +++ b/kernel/locking/mutex.c @@ -1023,6 +1023,9 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne wake_q_add(&wake_q, next); } + if (trace_contended_release_enabled() && waiter) + trace_call__contended_release(lock); + if (owner & MUTEX_FLAG_HANDOFF) __mutex_handoff(lock, next); @@ -1220,6 +1223,7 @@ EXPORT_SYMBOL(ww_mutex_lock_interruptible); EXPORT_TRACEPOINT_SYMBOL_GPL(contention_begin); EXPORT_TRACEPOINT_SYMBOL_GPL(contention_end); +EXPORT_TRACEPOINT_SYMBOL_GPL(contended_release); /** * atomic_dec_and_mutex_lock - return holding mutex if we dec to 0 diff --git a/kernel/locking/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c index f3ee7a0d6047..f7e152c40d6d 100644 --- a/kernel/locking/percpu-rwsem.c +++ b/kernel/locking/percpu-rwsem.c @@ -263,6 +263,9 @@ void percpu_up_write(struct percpu_rw_semaphore *sem) { rwsem_release(&sem->dep_map, _RET_IP_); + if (trace_contended_release_enabled() && wq_has_sleeper(&sem->waiters)) + trace_call__contended_release(sem); + /* * Signal the writer is done, no fast path yet. * @@ -292,6 +295,14 @@ EXPORT_SYMBOL_GPL(percpu_up_write); void __percpu_up_read(struct percpu_rw_semaphore *sem) { lockdep_assert_preemption_disabled(); + /* + * After percpu_up_write() completes, rcu_sync_is_idle() can still + * return false during the grace period, forcing readers into this + * slowpath. Only trace when a writer is actually waiting for + * readers to drain. + */ + if (trace_contended_release_enabled() && rcuwait_active(&sem->writer)) + trace_call__contended_release(sem); /* * slowpath; reader will only ever wake a single blocked * writer. diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c index 9147d6a31b78..28beae7d21fe 100644 --- a/kernel/locking/rtmutex.c +++ b/kernel/locking/rtmutex.c @@ -1470,6 +1470,7 @@ static void __sched rt_mutex_slowunlock(struct rt_mutex_base *lock) raw_spin_lock_irqsave(&lock->wait_lock, flags); } + trace_contended_release(lock); /* * The wakeup next waiter path does not suffer from the above * race. See the comments there. diff --git a/kernel/locking/rwbase_rt.c b/kernel/locking/rwbase_rt.c index 82e078c0665a..2835c9ef9b3f 100644 --- a/kernel/locking/rwbase_rt.c +++ b/kernel/locking/rwbase_rt.c @@ -174,6 +174,8 @@ static void __sched __rwbase_read_unlock(struct rwbase_rt *rwb, static __always_inline void rwbase_read_unlock(struct rwbase_rt *rwb, unsigned int state) { + if (trace_contended_release_enabled() && rt_mutex_owner(&rwb->rtmutex)) + trace_call__contended_release(rwb); /* * rwb->readers can only hit 0 when a writer is waiting for the * active readers to leave the critical section. @@ -205,6 +207,8 @@ static inline void rwbase_write_unlock(struct rwbase_rt *rwb) unsigned long flags; raw_spin_lock_irqsave(&rtm->wait_lock, flags); + if (trace_contended_release_enabled() && rt_mutex_has_waiters(rtm)) + trace_call__contended_release(rwb); __rwbase_write_unlock(rwb, WRITER_BIAS, flags); } @@ -214,6 +218,8 @@ static inline void rwbase_write_downgrade(struct rwbase_rt *rwb) unsigned long flags; raw_spin_lock_irqsave(&rtm->wait_lock, flags); + if (trace_contended_release_enabled() && rt_mutex_has_waiters(rtm)) + trace_call__contended_release(rwb); /* Release it and account current as reader */ __rwbase_write_unlock(rwb, WRITER_BIAS - 1, flags); } diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c index bf647097369c..b9c180ac1eee 100644 --- a/kernel/locking/rwsem.c +++ b/kernel/locking/rwsem.c @@ -1387,6 +1387,8 @@ static inline void __up_read(struct rw_semaphore *sem) rwsem_clear_reader_owned(sem); tmp = atomic_long_add_return_release(-RWSEM_READER_BIAS, &sem->count); DEBUG_RWSEMS_WARN_ON(tmp < 0, sem); + if (trace_contended_release_enabled() && (tmp & RWSEM_FLAG_WAITERS)) + trace_call__contended_release(sem); if (unlikely((tmp & (RWSEM_LOCK_MASK|RWSEM_FLAG_WAITERS)) == RWSEM_FLAG_WAITERS)) { clear_nonspinnable(sem); @@ -1413,8 +1415,10 @@ static inline void __up_write(struct rw_semaphore *sem) preempt_disable(); rwsem_clear_owner(sem); tmp = atomic_long_fetch_add_release(-RWSEM_WRITER_LOCKED, &sem->count); - if (unlikely(tmp & RWSEM_FLAG_WAITERS)) + if (unlikely(tmp & RWSEM_FLAG_WAITERS)) { + trace_contended_release(sem); rwsem_wake(sem); + } preempt_enable(); } @@ -1437,8 +1441,10 @@ static inline void __downgrade_write(struct rw_semaphore *sem) tmp = atomic_long_fetch_add_release( -RWSEM_WRITER_LOCKED+RWSEM_READER_BIAS, &sem->count); rwsem_set_reader_owned(sem); - if (tmp & RWSEM_FLAG_WAITERS) + if (tmp & RWSEM_FLAG_WAITERS) { + trace_contended_release(sem); rwsem_downgrade_wake(sem); + } preempt_enable(); } diff --git a/kernel/locking/semaphore.c b/kernel/locking/semaphore.c index 74d41433ba13..233730c25933 100644 --- a/kernel/locking/semaphore.c +++ b/kernel/locking/semaphore.c @@ -230,6 +230,10 @@ void __sched up(struct semaphore *sem) sem->count++; else __up(sem, &wake_q); + + if (trace_contended_release_enabled() && !wake_q_empty(&wake_q)) + trace_call__contended_release(sem); + raw_spin_unlock_irqrestore(&sem->lock, flags); if (!wake_q_empty(&wake_q)) wake_up_q(&wake_q); -- cgit v1.2.3 From 6001896f00984d317fb75160ba05c4a885fbe2a0 Mon Sep 17 00:00:00 2001 From: Sun Jian Date: Fri, 12 Jun 2026 19:40:31 +0800 Subject: bpf: Run generic devmap egress prog on private skb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generic XDP devmap multi redirect uses skb_clone() for intermediate destinations and sends the last destination with the original skb. This can leave multiple destinations sharing the same packet data. This becomes visible after generic devmap egress-program support was added: a devmap egress program may mutate packet data, and another destination sharing the same data can observe that mutation. Native XDP broadcast redirect does not have this issue because xdpf_clone() copies the frame data for each destination. Generic XDP should provide the same per-destination isolation before running a devmap egress program. Fix this by making cloned skbs private before running the generic devmap egress program. Use skb_copy() instead of skb_unshare() so allocation failure does not consume the skb and the existing caller error paths keep their ownership semantics. Fixes: 2ea5eabaf04a ("bpf: devmap: Implement devmap prog execution for generic XDP") Suggested-by: Jiayuan Chen Suggested-by: Jakub Kicinski Reviewed-by: Toke Høiland-Jørgensen Signed-off-by: Sun Jian Link: https://lore.kernel.org/r/20260612114032.244616-2-sun.jian.kdev@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/devmap.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c index 5b9eac5342a9..dc7b859e8bbf 100644 --- a/kernel/bpf/devmap.c +++ b/kernel/bpf/devmap.c @@ -710,6 +710,18 @@ int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, if (unlikely(err)) return err; + if (dst->xdp_prog && skb_cloned(skb)) { + struct sk_buff *nskb; + + nskb = skb_copy(skb, GFP_ATOMIC); + if (!nskb) + return -ENOMEM; + + nskb->mac_len = skb->mac_len; + consume_skb(skb); + skb = nskb; + } + /* Redirect has already succeeded semantically at this point, so we just * return 0 even if packet is dropped. Helper below takes care of * freeing skb. -- cgit v1.2.3 From 4c71303c837449158815c521fcee4ec3b8721dbd Mon Sep 17 00:00:00 2001 From: Xu Kuohai Date: Wed, 10 Jun 2026 20:17:23 +0000 Subject: bpf: Fix setting retval to -EPERM for cgroup hooks not returning errno When a cgroup BPF program exits with 0, bpf_prog_run_array_cg() sets the hook return value to -EPERM if it is not a valid errno. This is correct for errno-based hooks, which return 0 on success and negative errno on failure, but wrong for boolean and void LSM hooks. Boolean LSM hooks should only return true or false, and void LSM hooks have no return value at all. Fix it by skipping setting -EPERM for hooks not returning errno. Fixes: 69fd337a975c ("bpf: per-cgroup lsm flavor") Signed-off-by: Xu Kuohai Link: https://lore.kernel.org/r/20260610201724.733943-2-xukuohai@huaweicloud.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/bpf_lsm.c | 20 ++++++++++++++++++++ kernel/bpf/cgroup.c | 47 ++++++++++++++++++++++++++++++++++------------- 2 files changed, 54 insertions(+), 13 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index c5c925f00202..564071a92d7d 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -427,6 +427,26 @@ BTF_ID(func, bpf_lsm_audit_rule_known) BTF_ID(func, bpf_lsm_inode_xattr_skipcap) BTF_SET_END(bool_lsm_hooks) +/* hooks returning void */ +#define LSM_HOOK_void(DEFAULT, NAME, ...) BTF_ID(func, bpf_lsm_##NAME) +#define LSM_HOOK_int(DEFAULT, NAME, ...) /* nothing */ +#define LSM_HOOK(RET, DEFAULT, NAME, ...) LSM_HOOK_##RET(DEFAULT, NAME, __VA_ARGS__) +BTF_SET_START(void_lsm_hooks) +#include +#undef LSM_HOOK +#undef LSM_HOOK_void +#undef LSM_HOOK_int +BTF_SET_END(void_lsm_hooks) + +bool bpf_lsm_hook_returns_errno(u32 btf_id) +{ + if (btf_id_set_contains(&bool_lsm_hooks, btf_id)) + return false; + if (btf_id_set_contains(&void_lsm_hooks, btf_id)) + return false; + return true; +} + int bpf_lsm_get_retval_range(const struct bpf_prog *prog, struct bpf_retval_range *retval_range) { diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 35d1f1428ef3..83ce66296ac1 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -55,6 +55,28 @@ void __init cgroup_bpf_lifetime_notifier_init(void) &cgroup_bpf_lifetime_nb)); } +#ifdef CONFIG_BPF_LSM +struct cgroup_lsm_atype { + u32 attach_btf_id; + int refcnt; + bool returns_errno; +}; + +static struct cgroup_lsm_atype cgroup_lsm_atype[CGROUP_LSM_NUM]; + +static bool cgroup_bpf_hook_returns_errno(enum cgroup_bpf_attach_type atype) +{ + if (atype >= CGROUP_LSM_START && atype <= CGROUP_LSM_END) + return READ_ONCE(cgroup_lsm_atype[atype - CGROUP_LSM_START].returns_errno); + return true; +} +#else +static bool cgroup_bpf_hook_returns_errno(enum cgroup_bpf_attach_type atype) +{ + return true; +} +#endif + /* __always_inline is necessary to prevent indirect call through run_prog * function pointer. */ @@ -83,7 +105,8 @@ bpf_prog_run_array_cg(const struct cgroup_bpf *cgrp, *(ret_flags) |= (func_ret >> 1); func_ret &= 1; } - if (!func_ret && !IS_ERR_VALUE((long)run_ctx.retval)) + if (!func_ret && cgroup_bpf_hook_returns_errno(atype) && + !IS_ERR_VALUE((long)run_ctx.retval)) run_ctx.retval = -EPERM; item++; } @@ -156,13 +179,6 @@ unsigned int __cgroup_bpf_run_lsm_current(const void *ctx, } #ifdef CONFIG_BPF_LSM -struct cgroup_lsm_atype { - u32 attach_btf_id; - int refcnt; -}; - -static struct cgroup_lsm_atype cgroup_lsm_atype[CGROUP_LSM_NUM]; - static enum cgroup_bpf_attach_type bpf_cgroup_atype_find(enum bpf_attach_type attach_type, u32 attach_btf_id) { @@ -191,10 +207,13 @@ void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype) lockdep_assert_held(&cgroup_mutex); - WARN_ON_ONCE(cgroup_lsm_atype[i].attach_btf_id && - cgroup_lsm_atype[i].attach_btf_id != attach_btf_id); - - cgroup_lsm_atype[i].attach_btf_id = attach_btf_id; + if (!cgroup_lsm_atype[i].attach_btf_id) { + cgroup_lsm_atype[i].attach_btf_id = attach_btf_id; + WRITE_ONCE(cgroup_lsm_atype[i].returns_errno, + bpf_lsm_hook_returns_errno(attach_btf_id)); + } else { + WARN_ON_ONCE(cgroup_lsm_atype[i].attach_btf_id != attach_btf_id); + } cgroup_lsm_atype[i].refcnt++; } @@ -203,8 +222,10 @@ void bpf_cgroup_atype_put(int cgroup_atype) int i = cgroup_atype - CGROUP_LSM_START; cgroup_lock(); - if (--cgroup_lsm_atype[i].refcnt <= 0) + if (--cgroup_lsm_atype[i].refcnt <= 0) { + WRITE_ONCE(cgroup_lsm_atype[i].returns_errno, true); cgroup_lsm_atype[i].attach_btf_id = 0; + } WARN_ON_ONCE(cgroup_lsm_atype[i].refcnt < 0); cgroup_unlock(); } -- cgit v1.2.3 From f24df84cbe05e4471c04ac4b921fc0340bbc7752 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 9 Jun 2026 17:14:45 +0200 Subject: time/jiffies: Register jiffies clocksource before usage Teddy reported that a XEN HVM has a long boot delay, which was bisected to the recent enhancements to the negative motion detection. It turned out that the jiffies clocksource is used in early boot before it is registered, which leaves the max_delta_raw field at zero. That causes the read out to be clamped to the max delta of 0, which means time is not making progress. Cure it by ensuring that it is initialized before its first usage in timekeeping_init(). Fixes: 76031d9536a0 ("clocksource: Make negative motion detection more robust") Reported-by: Teddy Astie Signed-off-by: Thomas Gleixner Tested-by: Teddy Astie Cc: stable@vger.kernel.org Link: https://patch.msgid.link/87y0gn3fve.ffs@fw13 Closes: https://lore.kernel.org/all/1780914594.8631fc262581453bbf619ec5b2062170.19ea6c8227b000701b@vates.tech --- kernel/time/jiffies.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c index 1c954f330dfe..d51428867a33 100644 --- a/kernel/time/jiffies.c +++ b/kernel/time/jiffies.c @@ -60,15 +60,14 @@ EXPORT_SYMBOL(get_jiffies_64); EXPORT_SYMBOL(jiffies); -static int __init init_jiffies_clocksource(void) -{ - return __clocksource_register(&clocksource_jiffies); -} - -core_initcall(init_jiffies_clocksource); +static bool cs_jiffies_registered __initdata; struct clocksource * __init __weak clocksource_default_clock(void) { + if (!cs_jiffies_registered) { + __clocksource_register(&clocksource_jiffies); + cs_jiffies_registered = true; + } return &clocksource_jiffies; } -- cgit v1.2.3 From 87bd2ad568e15b90d5f7d4bcd70342d05dad649c Mon Sep 17 00:00:00 2001 From: WenTao Liang Date: Fri, 12 Jun 2026 00:17:38 +0800 Subject: posix-cpu-timers: Fix pid refcount leak in do_cpu_nanosleep() error path In do_cpu_nanosleep(), posix_cpu_timer_create() takes a pid reference via get_pid() and stores it in timer.it.cpu.pid. If the subsequent posix_cpu_timer_set() call fails, the function returns immediately without calling posix_cpu_timer_del() to release the pid reference, causing a leak. Fix it by calling posix_cpu_timer_del() before the unlock-and-return on the error path, consistent with the other exit paths in the same function. Fixes: 1da177e4c3f4 ("Linux-2.6.12-rc2") Signed-off-by: WenTao Liang Signed-off-by: Thomas Gleixner Reviewed-by: Frederic Weisbecker Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260611161738.97043-1-vulab@iscas.ac.cn --- kernel/time/posix-cpu-timers.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 395e297093f8..74775b94d11b 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -1506,6 +1506,7 @@ static int do_cpu_nanosleep(const clockid_t which_clock, int flags, spin_lock_irq(&timer.it_lock); error = posix_cpu_timer_set(&timer, flags, &it, NULL); if (error) { + posix_cpu_timer_del(&timer); spin_unlock_irq(&timer.it_lock); return error; } -- cgit v1.2.3 From 2148794eeaf2a898adc791e9472eb80ea55984da Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Sat, 13 Jun 2026 11:07:55 -0700 Subject: bpf: Raise maximum call chain depth to 16 frames Bump MAX_CALL_FRAMES from 8 to 16 to allow deeper call chains that Rust-BPF requires and update selftests. Link: https://lore.kernel.org/r/20260613180755.29671-1-alexei.starovoitov@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index eb46a81a8c51..2abc79dbf281 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3144,7 +3144,7 @@ static void mark_indirect_target(struct bpf_verifier_env *env, int idx) env->insn_aux_data[idx].indirect_target = true; } -#define LR_FRAMENO_BITS 3 +#define LR_FRAMENO_BITS 4 #define LR_SPI_BITS 6 #define LR_ENTRY_BITS (LR_SPI_BITS + LR_FRAMENO_BITS + 1) #define LR_SIZE_BITS 4 @@ -3153,7 +3153,11 @@ static void mark_indirect_target(struct bpf_verifier_env *env, int idx) #define LR_SIZE_MASK ((1ull << LR_SIZE_BITS) - 1) #define LR_SPI_OFF LR_FRAMENO_BITS #define LR_IS_REG_OFF (LR_SPI_BITS + LR_FRAMENO_BITS) -#define LINKED_REGS_MAX 6 +#define LINKED_REGS_MAX 5 + +static_assert(MAX_CALL_FRAMES <= (1 << LR_FRAMENO_BITS)); +static_assert(LINKED_REGS_MAX < (1 << LR_SIZE_BITS)); +static_assert(LINKED_REGS_MAX * LR_ENTRY_BITS + LR_SIZE_BITS <= 64); struct linked_reg { u8 frameno; @@ -3177,10 +3181,11 @@ static struct linked_reg *linked_regs_push(struct linked_regs *s) return NULL; } -/* Use u64 as a vector of 6 10-bit values, use first 4-bits to track +/* + * Use u64 as a vector of 5 11-bit values, use first 4-bits to track * number of elements currently in stack. - * Pack one history entry for linked registers as 10 bits in the following format: - * - 3-bits frameno + * Pack one history entry for linked registers as 11 bits in the following format: + * - 4-bits frameno * - 6-bits spi_or_reg * - 1-bit is_reg */ -- cgit v1.2.3 From 4d87a251d45b4a95eb4c0abcfab809c9f231258a Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Thu, 11 Jun 2026 13:42:24 +0200 Subject: bpf: Guard __get_user acesss with access_ok for uprobe_multi data As reported by sashiko [1] we need to use access_ok to check the user space data bounds before we use __get-user to get it. [1] https://lore.kernel.org/bpf/20260610145235.CB1441F00893@smtp.kernel.org/ Fixes: 0b779b61f651 ("bpf: Add cookies support for uprobe_multi link") Fixes: 89ae89f53d20 ("bpf: Add multi uprobe link") Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260611114230.950379-2-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/trace/bpf_trace.c | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'kernel') diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 90432f0fc2a8..b5a12af2d3f8 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -3224,6 +3224,7 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr unsigned long __user *uoffsets; u64 __user *ucookies; void __user *upath; + unsigned long size; u32 flags, cnt, i; struct path path; char *name; @@ -3261,6 +3262,16 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr uref_ctr_offsets = u64_to_user_ptr(attr->link_create.uprobe_multi.ref_ctr_offsets); ucookies = u64_to_user_ptr(attr->link_create.uprobe_multi.cookies); + /* + * All uoffsets/uref_ctr_offsets/ucookies arrays have the same value + * size, we need to check their address range is safe for __get_user + * calls. + */ + size = sizeof(*uoffsets) * cnt; + if (!access_ok(uoffsets, size) || !access_ok(uref_ctr_offsets, size) || + !access_ok(ucookies, size)) + return -EFAULT; + name = strndup_user(upath, PATH_MAX); if (IS_ERR(name)) { err = PTR_ERR(name); -- cgit v1.2.3 From 65d81609e93140d8dd745fd41eb8a195f83ba7cd Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Thu, 11 Jun 2026 13:42:25 +0200 Subject: bpf: Use user_path_at for path resolution in uprobe_multi Resolve the uprobe_multi user path with user_path_at() instead of copying the string with strndup_user() and passing it to kern_path(). This removes the temporary allocation and keeps the lookup logic in one helper. Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260611114230.950379-3-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/trace/bpf_trace.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index b5a12af2d3f8..f8990bc6b64c 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -3227,7 +3227,6 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr unsigned long size; u32 flags, cnt, i; struct path path; - char *name; pid_t pid; int err; @@ -3272,14 +3271,7 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr !access_ok(ucookies, size)) return -EFAULT; - name = strndup_user(upath, PATH_MAX); - if (IS_ERR(name)) { - err = PTR_ERR(name); - return err; - } - - err = kern_path(name, LOOKUP_FOLLOW, &path); - kfree(name); + err = user_path_at(AT_FDCWD, upath, LOOKUP_FOLLOW, &path); if (err) return err; -- cgit v1.2.3 From 26330a9226417c9a3395db9fdb403f7d7371e6b7 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Thu, 11 Jun 2026 13:42:26 +0200 Subject: bpf: Add support to specify uprobe_multi target via file descriptor Allow uprobe_multi link to identify the target binary by an already opened file descriptor. Adding new BPF_F_UPROBE_MULTI_PATH_FD flag and the path_fd field for the attr.link_create.uprobe_multi struct. When the flag is set, we resolve the target from path_fd, without the flag, we keep the existing string path behavior. I don't see a use case for supporting O_PATH file descriptors, because we need to read the binary first to get probes offsets, so I'm using the CLASS(fd, f), which fails for O_PATH fds. Assisted-by: Codex:GPT-5.4 Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260611114230.950379-4-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/bpf/syscall.c | 4 ++-- kernel/trace/bpf_trace.c | 43 +++++++++++++++++++++++++++++++++++++------ 2 files changed, 39 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 7ed949f70f82..b44106c8ea75 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -3480,7 +3480,7 @@ static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp) seq_printf(m, "link_type:\t%s\n", link->flags == BPF_F_KPROBE_MULTI_RETURN ? "kretprobe_multi" : "kprobe_multi"); else if (link->type == BPF_LINK_TYPE_UPROBE_MULTI) - seq_printf(m, "link_type:\t%s\n", link->flags == BPF_F_UPROBE_MULTI_RETURN ? + seq_printf(m, "link_type:\t%s\n", link->flags & BPF_F_UPROBE_MULTI_RETURN ? "uretprobe_multi" : "uprobe_multi"); else seq_printf(m, "link_type:\t%s\n", bpf_link_type_strs[type]); @@ -5840,7 +5840,7 @@ err_put: return err; } -#define BPF_LINK_CREATE_LAST_FIELD link_create.uprobe_multi.pid +#define BPF_LINK_CREATE_LAST_FIELD link_create.uprobe_multi.path_fd static int link_create(union bpf_attr *attr, bpfptr_t uattr) { struct bpf_prog *prog; diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index f8990bc6b64c..82f8feea6931 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -23,6 +23,7 @@ #include #include #include +#include #include @@ -3214,6 +3215,38 @@ static u64 bpf_uprobe_multi_cookie(struct bpf_run_ctx *ctx) return run_ctx->uprobe->cookie; } +static int bpf_uprobe_multi_get_path(const union bpf_attr *attr, struct path *path) +{ + void __user *upath = u64_to_user_ptr(attr->link_create.uprobe_multi.path); + u32 path_fd = attr->link_create.uprobe_multi.path_fd; + u32 flags = attr->link_create.uprobe_multi.flags; + + if (flags & BPF_F_UPROBE_MULTI_PATH_FD) { + /* + * When BPF_F_UPROBE_MULTI_PATH_FD is set, the executable is + * identified by path_fd, upath must be NULL. + */ + if (upath) + return -EINVAL; + + CLASS(fd, f)(path_fd); + if (fd_empty(f)) + return -EBADF; + *path = fd_file(f)->f_path; + path_get(path); + return 0; + } + + /* + * When BPF_F_UPROBE_MULTI_PATH_FD is not set, the path is resolved + * relative to the cwd (AT_FDCWD) or absolute using the upath string. + */ + if (!upath || path_fd) + return -EINVAL; + + return user_path_at(AT_FDCWD, upath, LOOKUP_FOLLOW, path); +} + int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) { struct bpf_uprobe_multi_link *link = NULL; @@ -3223,7 +3256,6 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr struct task_struct *task = NULL; unsigned long __user *uoffsets; u64 __user *ucookies; - void __user *upath; unsigned long size; u32 flags, cnt, i; struct path path; @@ -3241,19 +3273,18 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr return -EINVAL; flags = attr->link_create.uprobe_multi.flags; - if (flags & ~BPF_F_UPROBE_MULTI_RETURN) + if (flags & ~(BPF_F_UPROBE_MULTI_RETURN | BPF_F_UPROBE_MULTI_PATH_FD)) return -EINVAL; /* - * path, offsets and cnt are mandatory, + * offsets and cnt are mandatory, * ref_ctr_offsets and cookies are optional */ - upath = u64_to_user_ptr(attr->link_create.uprobe_multi.path); uoffsets = u64_to_user_ptr(attr->link_create.uprobe_multi.offsets); cnt = attr->link_create.uprobe_multi.cnt; pid = attr->link_create.uprobe_multi.pid; - if (!upath || !uoffsets || !cnt || pid < 0) + if (!uoffsets || !cnt || pid < 0) return -EINVAL; if (cnt > MAX_UPROBE_MULTI_CNT) return -E2BIG; @@ -3271,7 +3302,7 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr !access_ok(ucookies, size)) return -EFAULT; - err = user_path_at(AT_FDCWD, upath, LOOKUP_FOLLOW, &path); + err = bpf_uprobe_multi_get_path(attr, &path); if (err) return err; -- cgit v1.2.3 From 670e057744e0cc8047bf96d15d18c46e16ae2e93 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 13 May 2026 16:01:28 +0200 Subject: s390/idle: Provide arch specific kcpustat_field_idle()/kcpustat_field_iowait() The former s390 specific arch_cpu_idle_time() implementation was removed, since its implementation was racy and reported idle time could go backwards [1]. However this removal was not necessary, since independently of the s390 architecture specific races there exists the iowait counter update race, which can also lead to reported idle time going backwards [2]. With Frederic Weisbecker's recent cpu idle time accounting refactoring kernel_cpustat got a sequence counter. Use this to implement s390 specific variants of kcpustat_field_idle() and kcpustat_field_iowait(). This is logically a revert of [1] and moves cpu idle time accounting back into s390 architecture code, which is also more precise than the dyntick idle time accounting by nohz/scheduler. For comparing cross cpu time stamps it is necessary to use the stcke instead of the stckf instruction in irq entry path. Furthermore this open-codes a sequence lock in assembler and C code, which is required to update the irq entry time stamp to the per cpu idle_data structure in a race free manner. [1] commit be76ea614460 ("s390/idle: remove arch_cpu_idle_time() and corresponding code") [2] commit ead70b752373 ("timers/nohz: Add a comment about broken iowait counter update race") Signed-off-by: Heiko Carstens Acked-by: Frederic Weisbecker Signed-off-by: Alexander Gordeev --- kernel/sched/cputime.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 244b57417240..ed49a1e23d17 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -421,7 +421,7 @@ static inline void irqtime_account_process_tick(struct task_struct *p, int user_ int nr_ticks) { } #endif /* !CONFIG_IRQ_TIME_ACCOUNTING */ -#ifdef CONFIG_NO_HZ_COMMON +#if defined(CONFIG_NO_HZ_COMMON) && !defined(CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE) static void kcpustat_idle_stop(struct kernel_cpustat *kc, u64 now) { u64 *cpustat = kc->cpustat; @@ -560,7 +560,7 @@ static u64 kcpustat_field_dyntick(int cpu, enum cpu_usage_stat idx, { return kcpustat_cpu(cpu).cpustat[idx]; } -#endif /* CONFIG_NO_HZ_COMMON */ +#endif /* CONFIG_NO_HZ_COMMON && !CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE */ static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, bool compute_delta, u64 *last_update_time) -- cgit v1.2.3 From fd38b75c4b43295b10d69772a46d1c74dbd6fc81 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Tue, 16 Jun 2026 07:15:17 -0700 Subject: kernel/fork: clear PF_BLOCK_TS in copy_process() PF_BLOCK_TS is only set in blk_time_get_ns() when current->plug is non-NULL, and blk_finish_plug() clears it via __blk_flush_plug() before NULLing the plug pointer. copy_process() breaks the invariant by inheriting PF_BLOCK_TS from the parent while resetting the child's plug to NULL. Clear PF_BLOCK_TS alongside that assignment so callers can rely on "PF_BLOCK_TS set implies current->plug != NULL" and dereference current->plug unguarded. Fixes: 06b23f92af87 ("block: update cached timestamp post schedule/preemption") Cc: stable@vger.kernel.org Signed-off-by: Usama Arif Link: https://patch.msgid.link/20260616141604.328820-2-usama.arif@linux.dev Signed-off-by: Jens Axboe --- kernel/fork.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/fork.c b/kernel/fork.c index addc555a1077..1fafcb9bb047 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -2337,6 +2337,7 @@ __latent_entropy struct task_struct *copy_process( #ifdef CONFIG_BLOCK p->plug = NULL; + p->flags &= ~PF_BLOCK_TS; #endif futex_init_task(p); -- cgit v1.2.3 From fad156c2af227f42ca796cbb20ddc354a6dd9932 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Tue, 16 Jun 2026 07:15:18 -0700 Subject: block: invalidate cached plug timestamp after task switch blk_time_get_ns() caches ktime_get_ns() in current->plug->cur_ktime and marks the task with PF_BLOCK_TS. That cache is only valid while the task keeps running; if the task is switched out, wall-clock time advances and the cached value must not be reused when the task runs again. The existing invalidation covers explicit plug flushes through __blk_flush_plug(), and the schedule() / rtmutex paths through sched_update_worker(). It does not cover in-kernel preemption paths such as preempt_schedule(), preempt_schedule_notrace(), and preempt_schedule_irq(), which enter __schedule(SM_PREEMPT) directly and return without calling sched_update_worker(). As a result, a task preempted while holding a plug with PF_BLOCK_TS set can reuse a stale plug->cur_ktime after it is scheduled back in. blk-iocost then consumes that stale timestamp through ioc_now(), producing stale vnow values for throttle decisions, and through ioc_rqos_done(), inflating on-queue time and feeding false missed-QoS samples into vrate adjustment. Move the schedule-side invalidation to finish_task_switch(), which runs for the scheduled-in task after every actual context switch regardless of which schedule entry point was used. Keep __blk_flush_plug() as the explicit flush/finish-plug invalidation path, and remove only the PF_BLOCK_TS handling from sched_update_worker(). Fixes: 06b23f92af87 ("block: update cached timestamp post schedule/preemption") Cc: stable@vger.kernel.org Signed-off-by: Usama Arif Link: https://patch.msgid.link/20260616141604.328820-3-usama.arif@linux.dev Signed-off-by: Jens Axboe --- kernel/sched/core.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 8b791e9e9f67..e97e98c33be5 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5368,6 +5368,12 @@ static struct rq *finish_task_switch(struct task_struct *prev) */ kmap_local_sched_in(); + /* + * Any cached block-layer timestamp (plug->cur_ktime) is stale now, + * invalidate it. + */ + blk_plug_invalidate_ts(); + fire_sched_in_preempt_notifiers(current); /* * When switching through a kernel thread, the loop in @@ -7290,12 +7296,10 @@ static inline void sched_submit_work(struct task_struct *tsk) static void sched_update_worker(struct task_struct *tsk) { - if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER | PF_BLOCK_TS)) { - if (tsk->flags & PF_BLOCK_TS) - blk_plug_invalidate_ts(tsk); + if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER)) { if (tsk->flags & PF_WQ_WORKER) wq_worker_running(tsk); - else if (tsk->flags & PF_IO_WORKER) + else io_wq_worker_running(tsk); } } -- cgit v1.2.3 From 8fa30821180a9a19e78e9f4df1c0ba710252801e Mon Sep 17 00:00:00 2001 From: Mikhail Gavrilov Date: Tue, 16 Jun 2026 12:09:14 +0500 Subject: timekeeping: Register default clocksource before taking tk_core.lock Commit f24df84cbe05 ("time/jiffies: Register jiffies clocksource before usage") moved the jiffies clocksource registration into clocksource_default_clock(), so that it is registered lazily on the first call. __clocksource_register() acquires clocksource_mutex, but the first caller is timekeeping_init(), which invokes clocksource_default_clock() while holding tk_core.lock, a raw spinlock. Acquiring a sleeping mutex while holding a raw spinlock is invalid. The default clocksource only has to be registered before tk_setup_internals() consumes its mult/shift/maxadj. Neither clocksource_default_clock(), the ->enable() callback, nor the registration itself need tk_core.lock, so fetch and enable the clock before acquiring the lock. This preserves the "register before usage" ordering while keeping clocksource_mutex out of the raw spinlock section. clocksource_default_clock() has a second caller, clocksource_done_booting(), which invokes it with clocksource_mutex already held. That path avoids a recursive lock because timekeeping_init() has already run and set cs_jiffies_registered, so the registration is skipped there. This change does not alter that; it only fixes the invalid wait context in timekeeping_init(). Fixes: f24df84cbe05 ("time/jiffies: Register jiffies clocksource before usage") Signed-off-by: Mikhail Gavrilov Signed-off-by: Thomas Gleixner Reported-by: Breno Leitao Reported-by: Oleg Nesterov Reviewed-by: Breno Leitao Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260616070914.65818-1-mikhail.v.gavrilov@gmail.com --- kernel/time/timekeeping.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 0d5b67f609bb..b1b5ec43c0f2 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -2061,13 +2061,14 @@ void __init timekeeping_init(void) */ wall_to_mono = timespec64_sub(boot_offset, wall_time); + clock = clocksource_default_clock(); + if (clock->enable) + clock->enable(clock); + guard(raw_spinlock_irqsave)(&tk_core.lock); ntp_init(); - clock = clocksource_default_clock(); - if (clock->enable) - clock->enable(clock); tk_setup_internals(tks, clock); tk_set_xtime(tks, &wall_time); -- cgit v1.2.3 From 26aff38fefb1d6cd87e22525f41cc8f1aa61b24f Mon Sep 17 00:00:00 2001 From: Zhan Xusheng Date: Tue, 16 Jun 2026 19:20:17 +0800 Subject: posix-cpu-timers: Use u64 multiplication in update_rlimit_cpu() update_rlimit_cpu() converts the RLIMIT_CPU value to nanoseconds with u64 nsecs = rlim_new * NSEC_PER_SEC; On 32-bit kernels both rlim_new (unsigned long) and NSEC_PER_SEC (1000000000L) are 32-bit, so the multiplication is performed in unsigned long and truncated for rlim_new > 4 seconds before being widened to u64. The same file already casts to u64 for the matching computation in check_process_timers(): u64 softns = (u64)soft * NSEC_PER_SEC; As a result, the truncated value is installed into the CPUCLOCK_PROF expiry cache (nextevt), causing the process CPU timer to be programmed to fire prematurely for any RLIMIT_CPU soft limit >= 5 seconds. The actual SIGXCPU/SIGKILL decision in check_process_timers() already casts to u64 and is therefore correct, so limit enforcement is not broken; only the expiry-cache programming is wrong. Apply the same cast here so both paths convert rlim_cur identically. 64-bit kernels are unaffected. Fixes: 858cf3a8c599 ("timers/itimer: Convert internal cputime_t units to nsec") Signed-off-by: Zhan Xusheng Signed-off-by: Thomas Gleixner Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260616112017.1681372-1-zhanxusheng@xiaomi.com --- kernel/time/posix-cpu-timers.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 74775b94d11b..5e633d8750d1 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -41,7 +41,7 @@ void posix_cputimers_group_init(struct posix_cputimers *pct, u64 cpu_limit) */ int update_rlimit_cpu(struct task_struct *task, unsigned long rlim_new) { - u64 nsecs = rlim_new * NSEC_PER_SEC; + u64 nsecs = (u64)rlim_new * NSEC_PER_SEC; unsigned long irq_fl; if (!lock_task_sighand(task, &irq_fl)) -- cgit v1.2.3 From f8aceb1adb05896d66a3abbc1b0f41b90c9179ae Mon Sep 17 00:00:00 2001 From: Ethan Nelson-Moore Date: Mon, 8 Jun 2026 20:43:13 -0700 Subject: hrtimer: Correct CONFIG_NO_HZ_COMMON macro name in comment A comment in kernel/time/hrtimer.c incorrectly refers to CONFIG_NOHZ_COMMON instead of CONFIG_NO_HZ_COMMON. Correct it. Discovered while searching for CONFIG_* symbols referenced in code but not defined in any Kconfig file. Signed-off-by: Ethan Nelson-Moore Signed-off-by: Thomas Gleixner Link: https://patch.msgid.link/20260609034314.25029-1-enelsonmoore@gmail.com --- kernel/time/hrtimer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 638ce623c342..313dcea127fe 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -799,7 +799,7 @@ static inline void hrtimer_switch_to_hres(void) { } * * This is only invoked when: * - CONFIG_HIGH_RES_TIMERS is enabled. - * - CONFIG_NOHZ_COMMON is enabled + * - CONFIG_NO_HZ_COMMON is enabled * * For the other cases this function is empty and because the call sites * are optimized out it vanishes as well, i.e. no need for lots of -- cgit v1.2.3 From 9ef450ca74e43dacf9a2a15db7a851052c78dcf0 Mon Sep 17 00:00:00 2001 From: Zhongqiu Han Date: Tue, 16 Jun 2026 23:47:33 +0800 Subject: cpufreq: schedutil: Fix uncleared need_freq_update on the .adjust_perf() path The need_freq_update flag makes sugov_should_update_freq() return true regardless of the rate_limit_us throttling, and is cleared in sugov_update_next_freq(). sugov_update_single_freq() and sugov_update_shared() go through that helper, so the flag does not persist there. However, sugov_update_single_perf(), used by drivers implementing the .adjust_perf() callback (e.g. intel_pstate or amd-pstate in passive mode) calls cpufreq_driver_adjust_perf() directly and never goes through sugov_update_next_freq(), so the need_freq_update flag is not cleared in that path. Before commit 75da043d8f88 ("cpufreq/sched: Set need_freq_update in ignore_dl_rate_limit()"), this was effectively harmless because sugov_should_update_freq() still honored the rate limit even when need_freq_update was set. After that change, the flag forces sugov_should_update_freq() to always return true, so once set, it stays effective indefinitely on the .adjust_perf() path. As a result, cpufreq_driver_adjust_perf() gets called on every scheduler utilization update (with the runqueue lock held) rather than being throttled by rate_limit_us, even if the driver itself may skip redundant hardware updates. Clear need_freq_update at the end of the adjust_perf path as well. Fixes: 75da043d8f88 ("cpufreq/sched: Set need_freq_update in ignore_dl_rate_limit()") Signed-off-by: Zhongqiu Han Reviewed-by: Hongyan Xia Reviewed-by: Christian Loehle Cc: All applicable [ rjw: Subject and changelog edits ] Link: https://patch.msgid.link/20260616154733.2405236-1-zhongqiu.han@oss.qualcomm.com Signed-off-by: Rafael J. Wysocki --- kernel/sched/cpufreq_schedutil.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index ae9fd211cec1..a4e689eefdfb 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -486,6 +486,7 @@ static void sugov_update_single_perf(struct update_util_data *hook, u64 time, cpufreq_driver_adjust_perf(sg_policy->policy, sg_cpu->bw_min, sg_cpu->util, max_cap); + sg_policy->need_freq_update = false; sg_policy->last_freq_update_time = time; } -- cgit v1.2.3 From 07669b0abe4ce76c716e8437e198e1337cf43d1f Mon Sep 17 00:00:00 2001 From: Shardul Deshpande Date: Fri, 12 Jun 2026 23:46:32 +0530 Subject: treewide: fix transposed "sign" typos and update spelling.txt Several comments transpose the letters in "assigned" and "unsigned", spelling them with "sing" instead of "sign". Correct all of them. Of these, the misspelling of "assigned" is not yet flagged by checkpatch, so also add it to scripts/spelling.txt. The remaining matches of `grep -ri singed` are RISINGEDGE register and enum names, not typos. Link: https://lore.kernel.org/20260612181633.734458-1-iamsharduld@gmail.com Signed-off-by: Shardul Deshpande Suggested-by: Andrew Morton Reviewed-by: SeongJae Park Cc: Joe Perches Signed-off-by: Andrew Morton --- kernel/bpf/helpers.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index b5314c9fed3c..6f56ba88fb61 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -3601,7 +3601,7 @@ __bpf_kfunc int bpf_copy_from_user_task_str(void *dst, u32 dst__sz, return ret + 1; } -/* Keep unsinged long in prototype so that kfunc is usable when emitted to +/* Keep unsigned long in prototype so that kfunc is usable when emitted to * vmlinux.h in BPF programs directly, but note that while in BPF prog, the * unsigned long always points to 8-byte region on stack, the kernel may only * read and write the 4-bytes on 32-bit. -- cgit v1.2.3 From e62d4192e593630f355094adc467058a05bdc935 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Thu, 18 Jun 2026 14:18:27 +0200 Subject: perf: Fix addr_filter_ranges lifetime Lee Jia Jie reported that since event::addr_filter_ranges is used under RCU, it should be RCU freed. Reported-by: Lee Jia Jie Signed-off-by: Peter Zijlstra (Intel) --- kernel/events/core.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/events/core.c b/kernel/events/core.c index 7935d5663944..c3a84c7bcaeb 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -5303,6 +5303,7 @@ static void free_event_rcu(struct rcu_head *head) if (event->ns) put_pid_ns(event->ns); perf_event_free_filter(event); + kfree(event->addr_filter_ranges); kmem_cache_free(perf_event_cache, event); } @@ -5750,8 +5751,6 @@ static void __free_event(struct perf_event *event) if (event->attach_state & PERF_ATTACH_CALLCHAIN) put_callchain_buffers(); - kfree(event->addr_filter_ranges); - if (event->attach_state & PERF_ATTACH_EXCLUSIVE) exclusive_event_destroy(event); -- cgit v1.2.3 From de3ab9bd3133899efb92e4cd05ba4203e58fc0a3 Mon Sep 17 00:00:00 2001 From: Rik van Riel Date: Tue, 16 Jun 2026 16:38:17 -0400 Subject: sched/mmcid: Fix OOB clear_bit when CID is MM_CID_UNSET in fixup path In mm_cid_fixup_cpus_to_tasks(), when rq->curr has the target mm and mm_cid.active is set, the CID is checked with cid_in_transit() before setting the transition bit. In per-CPU mode a newly forked or exec'd task can be running with mm_cid.cid == MM_CID_UNSET because CIDs are assigned lazily on schedule-in. With cid_in_transit() the guard passes for MM_CID_UNSET (no transit bit), converts it to MM_CID_UNSET | MM_CID_TRANSIT and stores it back; later mm_cid_schedout() feeds this to clear_bit() with MM_CID_UNSET as the bit number, triggering an out-of-bounds write. Symptoms: this is genuine memory corruption, but a bounded out-of-bounds write, not an arbitrary one. MM_CID_UNSET is the fixed sentinel BIT(31), so once the bad value reaches mm_cid_schedout() the cid_from_transit_cid() strip leaves MM_CID_UNSET, which fails the "cid < max_cids" convergence test and falls into mm_drop_cid() -> clear_bit(MM_CID_UNSET, mm_cidmask(mm)). The cid bitmap is embedded in the mm_struct slab object (after cpu_bitmap and mm_cpus_allowed) and is only num_possible_cpus() bits wide, so clearing bit 31 is a deterministic OOB bit-clear at a fixed offset of 2^31 / 8 == 256 MiB past the bitmap base. The address is not attacker-influenced (fixed sentinel -> fixed offset) and the op only clears a single bit; what sits 256 MiB further along the direct map is whatever kernel object happens to live there, so this corrupts one bit of unpredictable kernel memory -- it is not an arbitrary-address or arbitrary-value write. It triggers only in per-CPU CID mode, when a CPU is running an active task of the target mm whose cid is still MM_CID_UNSET -- the fork()/execve() window before that task's next schedule-in assigns it a real CID -- and a per-CPU -> per-task fixup walks over it (the mode fallback driven by a thread exit, sched_mm_cid_exit(), or by the deferred max_cids recompute in mm_cid_work_fn()). In practice syzkaller surfaced it as a KASAN use-after-free reported in __schedule -> mm_cid_switch_to, where the offending clear_bit() is inlined via mm_cid_schedout() -> mm_drop_cid(). Guard the transition-bit assignment against MM_CID_UNSET, in addition to the existing cid_in_transit() check, so the bit is only set on a genuine task-owned CID. A CPU-owned (MM_CID_ONCPU) CID of a running active task is handled by the cid_on_cpu(pcp->cid) branch above and never reaches this path, so excluding MM_CID_UNSET (and the already-transitioning case) is sufficient. Fixes: fbd0e71dc370 ("sched/mmcid: Provide CID ownership mode fixup functions") Signed-off-by: Rik van Riel Signed-off-by: Thomas Gleixner Assisted-by: Claude:claude-opus-4-8 syzkaller Reviewed-by: Mathieu Desnoyers Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260616203818.1516263-1-riel@surriel.com --- kernel/sched/core.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 8b791e9e9f67..3cc6fb1d2054 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -10909,8 +10909,19 @@ static void mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm) } else if (rq->curr->mm == mm && rq->curr->mm_cid.active) { unsigned int cid = rq->curr->mm_cid.cid; - /* Ensure it has the transition bit set */ - if (!cid_in_transit(cid)) { + /* + * Set the transition bit only on a genuine task-owned + * CID. A running active task can legitimately have + * MM_CID_UNSET here: in per-CPU mode CIDs are assigned + * lazily on schedule-in, so the fork()/execve() window + * leaves the task active with no owned CID. Setting the + * transition bit on MM_CID_UNSET would later feed + * clear_bit() an out-of-bounds bit number via + * mm_cid_schedout(), so exclude it. A CPU-owned + * (MM_CID_ONCPU) CID is handled by the cid_on_cpu() + * branch above and never reaches here. + */ + if (cid != MM_CID_UNSET && !cid_in_transit(cid)) { cid = cid_to_transit_cid(cid); rq->curr->mm_cid.cid = cid; pcp->cid = cid; -- cgit v1.2.3 From 89038cc87d80c77e7aa6f42a64b2573b74af339f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 19 Jun 2026 14:52:08 +0200 Subject: locking/rt: Fix the incorrect RCU protection in rt_spin_unlock() rt_spin_unlock() releases the RCU protection before unlocking the lock. That opens the door for the following UAF scenario: T1 T2 spin_lock(&p->lock); rcu_read_lock(); invalidate(p); p = rcu_dereference(ptr); rcu_assign_pointer(ptr, NULL); if (!p) return; spin_unlock(&p->lock); spin_lock(&p->lock) lock(&lock->lock); rcu_read_lock(); kfree_rcu(p); rcu_read_unlock(); .... spin_unlock(&p->lock) rcu_read_unlock(); // Ends grace period rcu_do_batch() kfree(p); UAF -> rt_mutex_cmpxchg_release(&lock->lock...) Regular spinlocks keep preemption disabled accross the unlock operation, which provides full RCU protection, but the RT substitution fails to resemble that. Same applies for the rwlock substitution. Move the rcu_read_unlock() invocation past the unlock operations to match the non-RT semantics. This makes it asymmetric vs. rt_xxx_lock(), but that's harmless as the caller needs to hold RCU read lock across the lock operation. The migrate_enable() call stays before the unlock operation because there is no per CPU operation in the unlock path which would require migration to be kept disabled. Fixes: 0f383b6dc96e ("locking/spinlock: Provide RT variant") Reported-by: syzbot+000c800a02097aaa10ed@syzkaller.appspotmail.com Decoded-by: Jann Horn Signed-off-by: Thomas Gleixner Reviewed-by: Sebastian Andrzej Siewior Acked-by: Al Viro Cc: stable@vger.kernel.org Link: https://patch.msgid.link/87jyrud75z.ffs@fw13 --- kernel/locking/spinlock_rt.c | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/locking/spinlock_rt.c b/kernel/locking/spinlock_rt.c index db1e11b45de6..1d5e1b3c60bf 100644 --- a/kernel/locking/spinlock_rt.c +++ b/kernel/locking/spinlock_rt.c @@ -79,10 +79,27 @@ void __sched rt_spin_unlock(spinlock_t *lock) __releases(RCU) { spin_release(&lock->dep_map, _RET_IP_); migrate_enable(); - rcu_read_unlock(); if (unlikely(!rt_mutex_cmpxchg_release(&lock->lock, current, NULL))) rt_mutex_slowunlock(&lock->lock); + + /* + * This must be last to prevent the following UAF: + * + * T1 T2 + * spin_lock(&p->lock); rcu_read_lock(); + * invalidate(p); p = rcu_dereference(ptr); + * rcu_assign_pointer(ptr, NULL); if (!p) return; + * spin_unlock(&p->lock); spin_lock(&p->lock); + * kfree_rcu(p); rcu_read_unlock(); + * .... + * spin_unlock(&p->lock) + * rcu_read_unlock(); // Ends grace period + * rcu_do_batch() + * kfree(p); + * UAF -> rt_mutex_cmpxchg_release(&p->lock.lock...) + */ + rcu_read_unlock(); } EXPORT_SYMBOL(rt_spin_unlock); @@ -262,17 +279,21 @@ void __sched rt_read_unlock(rwlock_t *rwlock) __releases(RCU) { rwlock_release(&rwlock->dep_map, _RET_IP_); migrate_enable(); - rcu_read_unlock(); rwbase_read_unlock(&rwlock->rwbase, TASK_RTLOCK_WAIT); + + /* This must be last. See comment in rt_spin_unlock() */ + rcu_read_unlock(); } EXPORT_SYMBOL(rt_read_unlock); void __sched rt_write_unlock(rwlock_t *rwlock) __releases(RCU) { rwlock_release(&rwlock->dep_map, _RET_IP_); - rcu_read_unlock(); migrate_enable(); rwbase_write_unlock(&rwlock->rwbase); + + /* This must be last. See comment in rt_spin_unlock() */ + rcu_read_unlock(); } EXPORT_SYMBOL(rt_write_unlock); -- cgit v1.2.3 From 673db10729fb121ea1b16fe57791a0cb9eac1eb5 Mon Sep 17 00:00:00 2001 From: Bradley Morgan Date: Fri, 19 Jun 2026 16:37:17 +0000 Subject: cpu: hotplug: Preserve per instance callback errors cpuhp_invoke_callback() unwinds earlier callbacks for the same hotplug state when one instance fails. The rollback path currently reuses ret, so a successful rollback can hide the original error and make the failed transition look successful. Keep the rollback result separate from the original error. Fixes: 724a86881d03 ("smp/hotplug: Callback vs state-machine consistency") Signed-off-by: Bradley Morgan Signed-off-by: Thomas Gleixner Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260619163719.12103-1-include@grrlz.net --- kernel/cpu.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/cpu.c b/kernel/cpu.c index 8df2d773fe3b..3ed24c711e3f 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -175,7 +175,7 @@ static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state, struct cpuhp_step *step = cpuhp_get_step(state); int (*cbm)(unsigned int cpu, struct hlist_node *node); int (*cb)(unsigned int cpu); - int ret, cnt; + int ret, cnt, rollback_ret; if (st->fail == state) { st->fail = CPUHP_INVALID; @@ -239,12 +239,12 @@ err: break; trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); - ret = cbm(cpu, node); - trace_cpuhp_exit(cpu, st->state, state, ret); + rollback_ret = cbm(cpu, node); + trace_cpuhp_exit(cpu, st->state, state, rollback_ret); /* * Rollback must not fail, */ - WARN_ON_ONCE(ret); + WARN_ON_ONCE(rollback_ret); } return ret; } -- cgit v1.2.3 From 86f436567f2516a0083b210bedc933544826a2c3 Mon Sep 17 00:00:00 2001 From: Bradley Morgan Date: Fri, 19 Jun 2026 16:37:18 +0000 Subject: cpu: hotplug: Bound hotplug states sysfs output states_show() adds CPU hotplug state names into a single sysfs buffer using sprintf(). With enough registered states, this can write past the end of the PAGE_SIZE buffer. Use sysfs_emit_at() so output is bounded. Fixes: 98f8cdce1db5 ("cpu/hotplug: Add sysfs state interface") Signed-off-by: Bradley Morgan Signed-off-by: Thomas Gleixner Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260619163719.12103-2-include@grrlz.net --- kernel/cpu.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/cpu.c b/kernel/cpu.c index 3ed24c711e3f..77fdb2013ab8 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -2865,21 +2865,17 @@ static const struct attribute_group cpuhp_cpu_attr_group = { .name = "hotplug", }; -static ssize_t states_show(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t states_show(struct device *dev, struct device_attribute *attr, char *buf) { - ssize_t cur, res = 0; + ssize_t res = 0; int i; mutex_lock(&cpuhp_state_mutex); for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) { struct cpuhp_step *sp = cpuhp_get_step(i); - if (sp->name) { - cur = sprintf(buf, "%3d: %s\n", i, sp->name); - buf += cur; - res += cur; - } + if (sp->name) + res += sysfs_emit_at(buf, res, "%3d: %s\n", i, sp->name); } mutex_unlock(&cpuhp_state_mutex); return res; -- cgit v1.2.3 From d1d53aa30ab3b5ae89161c9cc840b3f7489ad386 Mon Sep 17 00:00:00 2001 From: Nuoqi Gui Date: Thu, 18 Jun 2026 01:50:26 +0800 Subject: bpf: Fix stack slot index in nospec checks check_stack_write_fixed_off() computes the byte slot for a fixed-offset stack write as -off - 1, and records each written byte in slot_type[] with (slot - i) % BPF_REG_SIZE. The Spectre v4 sanitization pre-check uses slot_type[i] instead. For a 4-byte write at fp-8 after the lower half of fp-8 has been zeroed, the pre-check scans bytes 0..3 and sees STACK_ZERO while the actual write updates bytes 7..4. That can leave the second half-slot write without nospec_result even though the bytes being overwritten still require sanitization. Use the same slot index in the sanitization pre-check that the write path uses when updating slot_type[]. Fixes: 2039f26f3aca ("bpf: Fix leakage due to insufficient speculative store bypass mitigation") Acked-by: Luis Gerhorst Reviewed-by: Jiayuan Chen Reviewed-by: Emil Tsalapatis Signed-off-by: Nuoqi Gui Link: https://lore.kernel.org/r/20260618-f01-11-stack-nospec-slot-index-v3-1-780297041721@mails.tsinghua.edu.cn Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 2abc79dbf281..50e80dbbc178 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3479,7 +3479,8 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, bool sanitize = reg && is_spillable_regtype(reg->type); for (i = 0; i < size; i++) { - u8 type = state->stack[spi].slot_type[i]; + u8 type = state->stack[spi].slot_type[(slot - i) % + BPF_REG_SIZE]; if (type != STACK_MISC && type != STACK_ZERO) { sanitize = true; -- cgit v1.2.3 From 5e72b5b157299f703d0c08c543e68916d263b4a4 Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Mon, 15 Jun 2026 12:55:36 -0700 Subject: bpf: Fix build_id caching in stack_map_get_build_id_offset() This patch is a follow up to recent implementation of stack_map_get_build_id_offset_sleepable() [1]. stack_map_get_build_id_offset() and its sleepable variant each cached only the last successfully resolved VMA, with separate bookkeeping in each function. A run of IPs in a VMA with no usable build ID will repeat the lookup for every frame: find_vma() in the non-sleepable path, a VMA lock and a blocking build_id_parse_file() in the sleepable. Factor the per-call cache into a shared struct stack_map_build_id_cache with two independent slots [2][3], used by both functions: * resolved - last VMA that produced a build ID (file, build_id and range), reused to skip the lookup and the parse; * unresolved - last VMA with no usable build ID (range only), reused to emit a raw IP without another lookup or parse. Keeping the slots independent means a build-ID-less VMA no longer evicts the last resolved build ID, so a trace alternating between a binary and a region without one stops re-resolving the binary on every return. The shared lookup tests [vm_start, vm_end), matching the sleepable path; the non-sleepable path previously reused a build ID for ip == vm_end (range_in_vma() is inclusive) and now re-resolves it correctly. [1] https://lore.kernel.org/bpf/20260525223948.1920986-1-ihor.solodrai@linux.dev/ [2] https://lore.kernel.org/bpf/CAEf4Bza2fRDGhLQoPE-EzM7F34xaEJfi5Exmxb-iWVUN3F06=g@mail.gmail.com/ [3] https://lore.kernel.org/bpf/CAEf4BzZXJFr=1iiVx937ht=4PYQkQHg=eFk810zhMDzXQG3ihw@mail.gmail.com/ Suggested-by: Andrii Nakryiko Signed-off-by: Ihor Solodrai Link: https://lore.kernel.org/r/20260615195536.1065107-1-ihor.solodrai@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/stackmap.c | 183 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 130 insertions(+), 53 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 77ba03216c09..41fe87d7302f 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -175,6 +175,95 @@ static inline void stack_map_build_id_set_valid(struct bpf_stack_build_id *id, memcpy(id->build_id, build_id, BUILD_ID_SIZE_MAX); } +/* + * A cached VMA lookup result. The range [vm_start, vm_end) is always set. + * vm_pgoff, file, build_id are set only when the build ID was resolved. + * Zero vm_end marks the slot empty. build_id aliases the id_offs[] entry. + */ +struct stack_map_cached_vma { + unsigned long vm_start; + unsigned long vm_end; + unsigned long vm_pgoff; + struct file *file; /* pinned in the sleepable path; NULL otherwise */ + const unsigned char *build_id; +}; + +/* + * Per stack_map_get_build_id_offset() call cache of the last VMA with a build ID + * resolved and the last VMA with no usable build ID. Adjacent stack frames tend + * to land in the same VMA or the same backing file, so caching the last result + * of each kind lets us skip unnecessary VMA lookups and build ID parse calls. + * Keeping the two slots independent means a build-ID-less VMA doesn't evict the + * last resolved build ID. + */ +struct stack_map_build_id_cache { + struct stack_map_cached_vma resolved; + struct stack_map_cached_vma unresolved; +}; + +/* + * Fill @id from a cached range covering @ip. On a hit this writes @id (resolved + * range -> build ID + offset, unresolved range -> raw ip) and returns 0; on a + * miss it leaves @id untouched and returns -ENOENT. + */ +static int stack_map_build_id_set_from_cache(struct stack_map_build_id_cache *cache, + struct bpf_stack_build_id *id, u64 ip) +{ + unsigned long vm_start, vm_end, vm_pgoff; + u64 offset; + + vm_start = cache->resolved.vm_start; + vm_end = cache->resolved.vm_end; + if (vm_end && ip >= vm_start && ip < vm_end) { + vm_pgoff = cache->resolved.vm_pgoff; + offset = stack_map_build_id_offset(vm_pgoff, vm_start, ip); + stack_map_build_id_set_valid(id, offset, cache->resolved.build_id); + return 0; + } + + vm_start = cache->unresolved.vm_start; + vm_end = cache->unresolved.vm_end; + if (vm_end && ip >= vm_start && ip < vm_end) { + stack_map_build_id_set_ip(id); + return 0; + } + + return -ENOENT; +} + +/* + * Record @vma's build ID as the last resolved one. @file is the pinned backing + * file in the sleepable path (released when evicted), or NULL otherwise. + */ +static void stack_map_build_id_cache_set_resolved(struct stack_map_build_id_cache *cache, + struct file *file, + const unsigned char *build_id, + unsigned long vm_start, + unsigned long vm_end, + unsigned long vm_pgoff) +{ + if (cache->resolved.file) + fput(cache->resolved.file); + cache->resolved = (struct stack_map_cached_vma){ + .vm_start = vm_start, + .vm_end = vm_end, + .vm_pgoff = vm_pgoff, + .file = file, + .build_id = build_id, + }; +} + +/* Record [vm_start, vm_end) as a range with no usable build ID. */ +static void stack_map_build_id_cache_set_unresolved(struct stack_map_build_id_cache *cache, + unsigned long vm_start, + unsigned long vm_end) +{ + cache->unresolved = (struct stack_map_cached_vma){ + .vm_start = vm_start, + .vm_end = vm_end, + }; +} + struct stack_map_vma_lock { struct vm_area_struct *vma; struct mm_struct *mm; @@ -244,15 +333,9 @@ static void stack_map_unlock_vma(struct stack_map_vma_lock *lock) static void stack_map_get_build_id_offset_sleepable(struct bpf_stack_build_id *id_offs, u32 trace_nr) { - struct mm_struct *mm = current->mm; - struct stack_map_vma_lock lock = { .mm = mm }; - struct { - struct file *file; - const unsigned char *build_id; - unsigned long vm_start; - unsigned long vm_end; - unsigned long vm_pgoff; - } cache = {}; + struct stack_map_vma_lock lock = { .mm = current->mm }; + struct stack_map_build_id_cache cache = {}; + struct stack_map_cached_vma *res = &cache.resolved; unsigned long vm_pgoff, vm_start, vm_end; struct vm_area_struct *vma; struct file *file; @@ -262,44 +345,39 @@ static void stack_map_get_build_id_offset_sleepable(struct bpf_stack_build_id *i for (u32 i = 0; i < trace_nr; i++) { ip = READ_ONCE(id_offs[i].ip); - /* - * Range cache fast path: if ip falls within the previously - * resolved VMA range, reuse the cache build_id without - * re-acquiring the VMA lock. - */ - if (cache.build_id && ip >= cache.vm_start && ip < cache.vm_end) { - offset = stack_map_build_id_offset(cache.vm_pgoff, cache.vm_start, ip); - stack_map_build_id_set_valid(&id_offs[i], offset, cache.build_id); + if (!stack_map_build_id_set_from_cache(&cache, &id_offs[i], ip)) continue; - } vma = stack_map_lock_vma(&lock, ip); if (!vma) { stack_map_build_id_set_ip(&id_offs[i]); continue; } + + vm_pgoff = vma->vm_pgoff; + vm_start = vma->vm_start; + vm_end = vma->vm_end; + if (vma_is_anonymous(vma) || !vma->vm_file) { - stack_map_build_id_set_ip(&id_offs[i]); stack_map_unlock_vma(&lock); + stack_map_build_id_set_ip(&id_offs[i]); + stack_map_build_id_cache_set_unresolved(&cache, vm_start, vm_end); continue; } file = vma->vm_file; - vm_pgoff = vma->vm_pgoff; - vm_start = vma->vm_start; - vm_end = vma->vm_end; offset = stack_map_build_id_offset(vm_pgoff, vm_start, ip); /* - * Same backing file as previous (e.g. different VMAs - * of the same ELF binary). Reuse the cache build_id. + * Same backing file as the last resolved VMA (another mapping + * of the same ELF binary): reuse its build_id without re-parsing. */ - if (file == cache.file) { + if (file == res->file) { stack_map_unlock_vma(&lock); - stack_map_build_id_set_valid(&id_offs[i], offset, cache.build_id); - cache.vm_start = vm_start; - cache.vm_end = vm_end; - cache.vm_pgoff = vm_pgoff; + stack_map_build_id_set_valid(&id_offs[i], offset, res->build_id); + res->vm_start = vm_start; + res->vm_end = vm_end; + res->vm_pgoff = vm_pgoff; continue; } @@ -310,21 +388,17 @@ static void stack_map_get_build_id_offset_sleepable(struct bpf_stack_build_id *i if (build_id_parse_file(file, id_offs[i].build_id, NULL)) { stack_map_build_id_set_ip(&id_offs[i]); fput(file); + stack_map_build_id_cache_set_unresolved(&cache, vm_start, vm_end); continue; } stack_map_build_id_set_valid(&id_offs[i], offset, id_offs[i].build_id); - if (cache.file) - fput(cache.file); - cache.file = file; - cache.build_id = id_offs[i].build_id; - cache.vm_start = vm_start; - cache.vm_end = vm_end; - cache.vm_pgoff = vm_pgoff; + stack_map_build_id_cache_set_resolved(&cache, file, id_offs[i].build_id, + vm_start, vm_end, vm_pgoff); } - if (cache.file) - fput(cache.file); + if (res->file) + fput(res->file); } /* @@ -343,8 +417,8 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, struct mmap_unlock_irq_work *work = NULL; bool irq_work_busy = bpf_mmap_unlock_get_irq_work(&work); bool has_user_ctx = user && current && current->mm; - struct vm_area_struct *vma, *prev_vma = NULL; - const unsigned char *prev_build_id = NULL; + struct stack_map_build_id_cache cache = {}; + struct vm_area_struct *vma; int i; if (may_fault && has_user_ctx) { @@ -365,27 +439,30 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, for (i = 0; i < trace_nr; i++) { u64 ip = READ_ONCE(id_offs[i].ip); - u64 offset; - if (prev_build_id && range_in_vma(prev_vma, ip, ip)) { - vma = prev_vma; - offset = stack_map_build_id_offset(vma->vm_pgoff, vma->vm_start, ip); - stack_map_build_id_set_valid(&id_offs[i], offset, prev_build_id); + if (!stack_map_build_id_set_from_cache(&cache, &id_offs[i], ip)) continue; - } + vma = find_vma(current->mm, ip); if (!vma || vma_is_anonymous(vma) || fetch_build_id(vma, id_offs[i].build_id, may_fault)) { - /* per entry fall back to ips */ + /* per entry fall back to ips; cache build-ID-less range */ stack_map_build_id_set_ip(&id_offs[i]); - prev_vma = vma; - prev_build_id = NULL; + if (vma) + stack_map_build_id_cache_set_unresolved(&cache, + vma->vm_start, vma->vm_end); continue; } - offset = stack_map_build_id_offset(vma->vm_pgoff, vma->vm_start, ip); - stack_map_build_id_set_valid(&id_offs[i], offset, id_offs[i].build_id); - prev_vma = vma; - prev_build_id = id_offs[i].build_id; + /* + * mmap_lock is held for the whole loop, so the cached VMA + * fields stay valid; no file pinning is needed here. + */ + stack_map_build_id_set_valid(&id_offs[i], + stack_map_build_id_offset(vma->vm_pgoff, vma->vm_start, ip), + id_offs[i].build_id); + stack_map_build_id_cache_set_resolved(&cache, NULL, id_offs[i].build_id, + vma->vm_start, vma->vm_end, + vma->vm_pgoff); } bpf_mmap_unlock_mm(work, current->mm); } -- cgit v1.2.3 From a933bade82b9cd9197c6c9a390623cfb1f8c0da7 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Mon, 15 Jun 2026 16:21:46 -0700 Subject: bpf: Emit verbose message when prog-specific btf_struct_access rejects a write When BPF_WRITE goes through a PTR_TO_BTF_ID register, check_ptr_to_btf_access() delegates to env->ops->btf_struct_access(). Most implementations (bpf_scx_btf_struct_access, tc_cls_act_btf_struct_access, etc.) return -EACCES for disallowed fields without logging anything, so the verifier rejects the program with an empty message. For example a scx program doing 1: R1=trusted_ptr_task_struct() ... 4: (7b) *(u64 *)(r1 +0) = r2 verification time 83 usec the program is rejected leaves the user guessing which field is off-limits. Emit verbose message. Signed-off-by: Alexei Starovoitov Reviewed-by: Emil Tsalapatis Acked-by: Yonghong Song Link: https://lore.kernel.org/r/20260615232146.5491-1-alexei.starovoitov@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 50e80dbbc178..a2b348f98080 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5787,6 +5787,10 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, return -EFAULT; } ret = env->ops->btf_struct_access(&env->log, reg, off, size); + if (ret < 0) + verbose(env, + "%s cannot write into ptr_%s at off=%d size=%d\n", + reg_arg_name(env, argno), tname, off, size); } else { /* Writes are permitted with default btf_struct_access for * program allocated objects (which always have id > 0), -- cgit v1.2.3 From 39799c63578ec64488e14aced9ea07af6f958f35 Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Tue, 16 Jun 2026 02:14:54 -0400 Subject: bpf: Allow type tag BTF records to succeed other modifier records llvm commit [1] allowed attaching type tag records to modifier BTF records. This is useful for using typedefs that encompass a base type and a type tag, e.g.: typedef struct rbtree __arena rbtree_t; Modify btf_check_type_tags() so that it allows this sequence of records. The function now only checks for record loops in BTF modifier record chains. Rename to btf_check_modifier_chain_length to reflect this. Also expand the BTF modifier traversal code to take into account that type record can be interleaved with other modifier records. In effect this means traversing all modifiers to collect the type tags. Also modify existing selftests to now accept modifier records (const, typedef) that point to type tag records. [1] https://github.com/llvm/llvm-project/pull/203089 Signed-off-by: Emil Tsalapatis Link: https://lore.kernel.org/r/20260616061454.7869-1-emil@etsalapatis.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/btf.c | 209 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 123 insertions(+), 86 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 15ae7c43f594..64572f85edc8 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -28,6 +28,7 @@ #include #include #include +#include #include @@ -3472,12 +3473,69 @@ static int btf_find_struct(const struct btf *btf, const struct btf_type *t, return BTF_FIELD_FOUND; } +struct btf_type_tag_match { + const char *name; + u32 flag; +}; + +struct btf_type_tag_walk_ctx { + const struct btf_type *t; /* Input/Output */ + u32 id; /* Output */ + u32 res; /* Output */ +}; + +static int btf_type_tag_walk(const struct btf *btf, + struct btf_type_tag_walk_ctx *ctx, + const struct btf_type_tag_match *matches, + u32 match_cnt) +{ + const struct btf_type *t = ctx->t; + u32 res = 0; + const char *tag; + u32 id, i; + + do { + id = t->type; + t = btf_type_by_id(btf, id); + + if (!btf_type_is_modifier(t)) + break; + + if (!btf_type_is_type_tag(t) || btf_type_kflag(t)) + continue; + + tag = __btf_name_by_offset(btf, t->name_off); + for (i = 0; i < match_cnt; i++) { + if (strcmp(tag, matches[i].name)) + continue; + res |= matches[i].flag; + break; + } + } while (true); + + /* We only support a single tag. */ + if (hweight32(res) > 1) + return -EINVAL; + + ctx->t = t; + ctx->id = id; + ctx->res = res; + + return 0; +} + static int btf_find_kptr(const struct btf *btf, const struct btf_type *t, u32 off, int sz, struct btf_field_info *info, u32 field_mask) { - enum btf_field_type type; - const char *tag_value; - bool is_type_tag; + static const struct btf_type_tag_match kptr_type_tags[] = { + { "kptr_untrusted", BPF_KPTR_UNREF }, + { "kptr", BPF_KPTR_REF }, + { "percpu_kptr", BPF_KPTR_PERCPU }, + { "uptr", BPF_UPTR }, + }; + struct btf_type_tag_walk_ctx ctx; + enum btf_field_type type = 0; + int err; u32 res_id; /* Permit modifiers on the pointer itself */ @@ -3486,30 +3544,20 @@ static int btf_find_kptr(const struct btf *btf, const struct btf_type *t, /* For PTR, sz is always == 8 */ if (!btf_type_is_ptr(t)) return BTF_FIELD_IGNORE; - t = btf_type_by_id(btf, t->type); - is_type_tag = btf_type_is_type_tag(t) && !btf_type_kflag(t); - if (!is_type_tag) - return BTF_FIELD_IGNORE; - /* Reject extra tags */ - if (btf_type_is_type_tag(btf_type_by_id(btf, t->type))) - return -EINVAL; - tag_value = __btf_name_by_offset(btf, t->name_off); - if (!strcmp("kptr_untrusted", tag_value)) - type = BPF_KPTR_UNREF; - else if (!strcmp("kptr", tag_value)) - type = BPF_KPTR_REF; - else if (!strcmp("percpu_kptr", tag_value)) - type = BPF_KPTR_PERCPU; - else if (!strcmp("uptr", tag_value)) - type = BPF_UPTR; - else - return -EINVAL; + + ctx.t = t; + err = btf_type_tag_walk(btf, &ctx, kptr_type_tags, + ARRAY_SIZE(kptr_type_tags)); + if (err) + return err; + + t = ctx.t; + res_id = ctx.id; + type = ctx.res; if (!(type & field_mask)) return BTF_FIELD_IGNORE; - /* Get the base type */ - t = btf_type_skip_modifiers(btf, t->type, &res_id); /* Only pointer to struct is allowed */ if (!__btf_type_is_struct(t)) return -EINVAL; @@ -5859,11 +5907,10 @@ struct btf_struct_meta *btf_find_struct_meta(const struct btf *btf, u32 btf_id) return bsearch(&btf_id, tab->types, tab->cnt, sizeof(tab->types[0]), btf_id_cmp_func); } -static int btf_check_type_tags(struct btf_verifier_env *env, - struct btf *btf, int start_id) +static int btf_check_modifier_chain_length(struct btf_verifier_env *env, + struct btf *btf, int start_id) { int i, n, good_id = start_id - 1; - bool in_tags; n = btf_nr_types(btf); for (i = start_id; i < n; i++) { @@ -5879,20 +5926,12 @@ static int btf_check_type_tags(struct btf_verifier_env *env, cond_resched(); - in_tags = btf_type_is_type_tag(t); while (btf_type_is_modifier(t)) { if (!chain_limit--) { btf_verifier_log(env, "Max chain length or cycle detected"); return -ELOOP; } - if (btf_type_is_type_tag(t)) { - if (!in_tags) { - btf_verifier_log(env, "Type tags don't precede modifiers"); - return -EINVAL; - } - } else if (in_tags) { - in_tags = false; - } + if (cur_id <= good_id) break; /* Move to next type */ @@ -5970,7 +6009,7 @@ static struct btf *btf_parse(const union bpf_attr *attr, bpfptr_t uattr, if (err) goto errout; - err = btf_check_type_tags(env, btf, 1); + err = btf_check_modifier_chain_length(env, btf, 1); if (err) goto errout; @@ -6378,7 +6417,7 @@ static struct btf *btf_parse_base(struct btf_verifier_env *env, const char *name if (err) goto errout; - err = btf_check_type_tags(env, btf, 1); + err = btf_check_modifier_chain_length(env, btf, 1); if (err) goto errout; @@ -6504,7 +6543,7 @@ static struct btf *btf_parse_module(const char *module_name, const void *data, if (err) goto errout; - err = btf_check_type_tags(env, btf, btf_nr_types(base_btf)); + err = btf_check_modifier_chain_length(env, btf, btf_nr_types(base_btf)); if (err) goto errout; @@ -6810,14 +6849,18 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, const struct bpf_prog *prog, struct bpf_insn_access_aux *info) { + static const struct btf_type_tag_match ctx_type_tags[] = { + { "user", MEM_USER }, + { "percpu", MEM_PERCPU }, + }; const struct btf_type *t = prog->aux->attach_func_proto; struct bpf_prog *tgt_prog = prog->aux->dst_prog; struct btf *btf = bpf_prog_get_target_btf(prog); const char *tname = prog->aux->attach_func_name; struct bpf_verifier_log *log = info->log; + struct btf_type_tag_walk_ctx ctx; const struct btf_param *args; bool ptr_err_raw_tp = false; - const char *tag_value; u32 nr_args, arg; int i, ret; @@ -7020,22 +7063,18 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, } info->btf = btf; - info->btf_id = t->type; - t = btf_type_by_id(btf, t->type); - - if (btf_type_is_type_tag(t) && !btf_type_kflag(t)) { - tag_value = __btf_name_by_offset(btf, t->name_off); - if (strcmp(tag_value, "user") == 0) - info->reg_type |= MEM_USER; - if (strcmp(tag_value, "percpu") == 0) - info->reg_type |= MEM_PERCPU; + ctx.t = t; + ret = btf_type_tag_walk(btf, &ctx, ctx_type_tags, + ARRAY_SIZE(ctx_type_tags)); + if (ret) { + bpf_log(log, "func '%s' arg%d type %s has multiple type tags\n", + tname, arg, btf_type_str(t)); + return false; } + info->reg_type |= ctx.res; + info->btf_id = ctx.id; + t = ctx.t; - /* skip modifiers */ - while (btf_type_is_modifier(t)) { - info->btf_id = t->type; - t = btf_type_by_id(btf, t->type); - } if (!btf_type_is_struct(t)) { bpf_log(log, "func '%s' arg%d type %s is not a struct\n", @@ -7074,7 +7113,7 @@ static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, u32 i, moff, mtrue_end, msize = 0, total_nelems = 0; const struct btf_type *mtype, *elem_type = NULL; const struct btf_member *member; - const char *tname, *mname, *tag_value; + const char *tname, *mname; u32 vlen, elem_id, mid; again: @@ -7270,8 +7309,15 @@ error: } if (btf_type_is_ptr(mtype)) { - const struct btf_type *stype, *t; + static const struct btf_type_tag_match walk_type_tags[] = { + { "user", MEM_USER }, + { "percpu", MEM_PERCPU }, + { "rcu", MEM_RCU }, + }; enum bpf_type_flag tmp_flag = 0; + struct btf_type_tag_walk_ctx ctx = { .t = mtype }; + const struct btf_type *stype; + int err; u32 id; if (msize != size || off != moff) { @@ -7281,22 +7327,17 @@ error: return -EACCES; } - /* check type tag */ - t = btf_type_by_id(btf, mtype->type); - if (btf_type_is_type_tag(t) && !btf_type_kflag(t)) { - tag_value = __btf_name_by_offset(btf, t->name_off); - /* check __user tag */ - if (strcmp(tag_value, "user") == 0) - tmp_flag = MEM_USER; - /* check __percpu tag */ - if (strcmp(tag_value, "percpu") == 0) - tmp_flag = MEM_PERCPU; - /* check __rcu tag */ - if (strcmp(tag_value, "rcu") == 0) - tmp_flag = MEM_RCU; + err = btf_type_tag_walk(btf, &ctx, walk_type_tags, + ARRAY_SIZE(walk_type_tags)); + if (err) { + bpf_log(log, "type '%s' has multiple type tags\n", + btf_type_str(mtype)); + return err; } + tmp_flag = ctx.res; + id = ctx.id; + stype = ctx.t; - stype = btf_type_skip_modifiers(btf, mtype->type, &id); if (btf_type_is_struct(stype)) { *next_btf_id = id; *flag |= tmp_flag; @@ -7867,7 +7908,12 @@ static int btf_scan_type_tags(struct bpf_verifier_env *env, const struct btf *btf, u32 type_id, u32 *tags) { + static const struct btf_type_tag_match func_type_tags[] = { + { "arena", ARG_TAG_ARENA }, + }; + struct btf_type_tag_walk_ctx ctx; const struct btf_type *t; + int err; /* Find the first pointer type in the chain. */ t = btf_type_skip_modifiers(btf, type_id, NULL); @@ -7879,24 +7925,15 @@ static int btf_scan_type_tags(struct bpf_verifier_env *env, if (!t || !btf_type_is_ptr(t)) return 0; - /* We got a pointer, get all associated type tags. */ - for (t = btf_type_by_id(btf, t->type); t && btf_type_is_modifier(t); - t = btf_type_by_id(btf, t->type)) { - - /* Skip non-type tag modifiers. */ - if (!btf_type_is_type_tag(t)) - continue; - - const char *tag = __btf_name_by_offset(btf, t->name_off); - - if (strcmp(tag, "arena") == 0) { - *tags |= ARG_TAG_ARENA; - } else { - bpf_log(&env->log, "function signature member has unsupported type tag '%s'\n", - tag); - return -EOPNOTSUPP; - } + ctx.t = t; + err = btf_type_tag_walk(btf, &ctx, func_type_tags, + ARRAY_SIZE(func_type_tags)); + if (err) { + bpf_log(&env->log, + "function signature member has multiple type tags\n"); + return err; } + *tags |= ctx.res; return 0; } -- cgit v1.2.3 From d5dc200c3a3f217de072af269dd90adddf90e48d Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Tue, 16 Jun 2026 10:30:56 +0200 Subject: bpf: Add missing access_ok call to copy_user_syms As reported by sashiko we use __get_user without prior access_ok call on the user space pointer. Adding the missing call for the whole pointer array. Plus removing the err check in the error path, because it's not needed and also we can return -ENOMEM directly from the first kvmalloc_array fail path. Cc: stable@vger.kernel.org [1] https://lore.kernel.org/bpf/20260611115503.AC16D1F00893@smtp.kernel.org/ Fixes: 0236fec57a15 ("bpf: Resolve symbols with ftrace_lookup_symbols for kprobe multi link") Reported-by: Sashiko Closes: https://lore.kernel.org/bpf/20260611115503.AC16D1F00893@smtp.kernel.org/ Signed-off-by: Jiri Olsa Reviewed-by: Emil Tsalapatis Link: https://lore.kernel.org/r/20260616083056.405652-1-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/trace/bpf_trace.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 82f8feea6931..75495a5c3507 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -2376,9 +2376,12 @@ static int copy_user_syms(struct user_syms *us, unsigned long __user *usyms, u32 int err = -ENOMEM; unsigned int i; + if (!access_ok(usyms, cnt * sizeof(*usyms))) + return -EFAULT; + syms = kvmalloc_array(cnt, sizeof(*syms), GFP_KERNEL); if (!syms) - goto error; + return -ENOMEM; buf = kvmalloc_array(cnt, KSYM_NAME_LEN, GFP_KERNEL); if (!buf) @@ -2403,10 +2406,8 @@ static int copy_user_syms(struct user_syms *us, unsigned long __user *usyms, u32 return 0; error: - if (err) { - kvfree(syms); - kvfree(buf); - } + kvfree(syms); + kvfree(buf); return err; } -- cgit v1.2.3 From 8405c4626460503027461652f96d8bb10c2a9173 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thi=C3=A9baud=20Weksteen?= Date: Thu, 18 Jun 2026 14:09:33 +1000 Subject: bpf: Fix BPF_PROG_ASSOC_STRUCT_OPS last field check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When struct prog_assoc_struct_ops was added, BPF_PROG_ASSOC_STRUCT_OPS_LAST_FIELD referenced prog_fd instead of the actual last field, flags. Fixes: b5709f6d26d6 ("bpf: Support associating BPF program with struct_ops") Signed-off-by: Thiébaud Weksteen Reviewed-by: Jakub Sitnicki Acked-by: Yonghong Song Link: https://lore.kernel.org/r/20260618040934.4113938-1-tweek@google.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/syscall.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index b44106c8ea75..6db306d23b47 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -6308,7 +6308,7 @@ static int prog_stream_read(union bpf_attr *attr) return ret; } -#define BPF_PROG_ASSOC_STRUCT_OPS_LAST_FIELD prog_assoc_struct_ops.prog_fd +#define BPF_PROG_ASSOC_STRUCT_OPS_LAST_FIELD prog_assoc_struct_ops.flags static int prog_assoc_struct_ops(union bpf_attr *attr) { -- cgit v1.2.3 From f08aaee3152d0dfc578b3f2586932d82062701dd Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Thu, 18 Jun 2026 23:35:19 -0700 Subject: bpf: Fix effective prog array index with BPF_F_PREORDER replace_effective_prog() and purge_effective_progs() located the slot in the effective array by walking the program hlist and counting entries linearly. That count does not match the array layout: compute_effective_ progs() places BPF_F_PREORDER programs at the front (ancestor cgroup first, attach order within a cgroup) and the rest after them (descendant cgroup first). So when a preorder program is present, the linear hlist position no longer equals the program's index in the effective array. For replace_effective_prog() (bpf_link_update()) this overwrote the wrong slot, corrupting the effective order. For purge_effective_progs(), it could dummy out a slot belonging to a different program and leave the detached program in the array while bpf_prog_put() drops its reference, i.e. a use-after-free. Fix both by replaying compute_effective_progs()'s placement (including the per-cgroup preorder reversal) in a shared effective_prog_pos() helper. Identify the entry by its struct bpf_prog_list pointer rather than by (prog, link) value, so the lookup resolves to exactly the attachment the syscall selected even when the same bpf_prog is attached to several cgroups in the hierarchy. Fixes: 4b82b181a26c ("bpf: Allow pre-ordering for bpf cgroup progs") Signed-off-by: Amery Hung Acked-by: Yonghong Song Link: https://lore.kernel.org/r/20260619063520.2690547-2-ameryhung@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/cgroup.c | 108 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 62 insertions(+), 46 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 83ce66296ac1..4355ccb78a9c 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -939,19 +939,65 @@ static int cgroup_bpf_attach(struct cgroup *cgrp, return ret; } +static int effective_prog_pos(struct cgroup *cgrp, + enum cgroup_bpf_attach_type atype, + struct bpf_prog_list *target_pl) +{ + int cnt = 0, preorder_cnt = 0, fstart, bstart, init_bstart, pos = -1; + struct bpf_prog_list *pl; + struct cgroup *p = cgrp; + + /* count effective programs to find where the preorder region ends */ + do { + if (cnt == 0 || (p->bpf.flags[atype] & BPF_F_ALLOW_MULTI)) + cnt += prog_list_length(&p->bpf.progs[atype], &preorder_cnt); + p = cgroup_parent(p); + } while (p); + + /* replay compute_effective_progs() placement and record target's slot */ + cnt = 0; + p = cgrp; + fstart = preorder_cnt; + bstart = preorder_cnt - 1; + do { + if (cnt > 0 && !(p->bpf.flags[atype] & BPF_F_ALLOW_MULTI)) + continue; + + init_bstart = bstart; + hlist_for_each_entry(pl, &p->bpf.progs[atype], node) { + if (!prog_list_prog(pl)) + continue; + + if (pl->flags & BPF_F_PREORDER) { + if (pl == target_pl) + pos = bstart; + bstart--; + } else { + if (pl == target_pl) + pos = fstart; + fstart++; + } + cnt++; + } + + /* reverse pre-ordering progs at this cgroup level */ + if (pos >= bstart + 1 && pos <= init_bstart) + pos = bstart + 1 + init_bstart - pos; + } while ((p = cgroup_parent(p))); + + return pos; +} + /* Swap updated BPF program for given link in effective program arrays across * all descendant cgroups. This function is guaranteed to succeed. */ static void replace_effective_prog(struct cgroup *cgrp, enum cgroup_bpf_attach_type atype, - struct bpf_cgroup_link *link) + struct bpf_prog_list *pl) { struct bpf_prog_array_item *item; struct cgroup_subsys_state *css; struct bpf_prog_array *progs; - struct bpf_prog_list *pl; - struct hlist_head *head; - struct cgroup *cg; int pos; css_for_each_descendant_pre(css, &cgrp->self) { @@ -960,27 +1006,15 @@ static void replace_effective_prog(struct cgroup *cgrp, if (percpu_ref_is_zero(&desc->bpf.refcnt)) continue; - /* find position of link in effective progs array */ - for (pos = 0, cg = desc; cg; cg = cgroup_parent(cg)) { - if (pos && !(cg->bpf.flags[atype] & BPF_F_ALLOW_MULTI)) - continue; + pos = effective_prog_pos(desc, atype, pl); + if (WARN_ON_ONCE(pos < 0)) + continue; - head = &cg->bpf.progs[atype]; - hlist_for_each_entry(pl, head, node) { - if (!prog_list_prog(pl)) - continue; - if (pl->link == link) - goto found; - pos++; - } - } -found: - BUG_ON(!cg); progs = rcu_dereference_protected( desc->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); item = &progs->items[pos]; - WRITE_ONCE(item->prog, link->link.prog); + WRITE_ONCE(item->prog, pl->link->link.prog); } } @@ -1024,7 +1058,7 @@ static int __cgroup_bpf_replace(struct cgroup *cgrp, cgrp->bpf.revisions[atype] += 1; old_prog = xchg(&link->link.prog, new_prog); - replace_effective_prog(cgrp, atype, link); + replace_effective_prog(cgrp, atype, pl); bpf_prog_put(old_prog); return 0; } @@ -1091,19 +1125,14 @@ static struct bpf_prog_list *find_detach_entry(struct hlist_head *progs, * recomputing the array in place. * * @cgrp: The cgroup which descendants to travers - * @prog: A program to detach or NULL - * @link: A link to detach or NULL + * @pl: The prog_list entry being detached * @atype: Type of detach operation */ -static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog *prog, - struct bpf_cgroup_link *link, +static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog_list *pl, enum cgroup_bpf_attach_type atype) { struct cgroup_subsys_state *css; struct bpf_prog_array *progs; - struct bpf_prog_list *pl; - struct hlist_head *head; - struct cgroup *cg; int pos; /* recompute effective prog array in place */ @@ -1113,24 +1142,11 @@ static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog *prog, if (percpu_ref_is_zero(&desc->bpf.refcnt)) continue; - /* find position of link or prog in effective progs array */ - for (pos = 0, cg = desc; cg; cg = cgroup_parent(cg)) { - if (pos && !(cg->bpf.flags[atype] & BPF_F_ALLOW_MULTI)) - continue; - - head = &cg->bpf.progs[atype]; - hlist_for_each_entry(pl, head, node) { - if (!prog_list_prog(pl)) - continue; - if (pl->prog == prog && pl->link == link) - goto found; - pos++; - } - } - + pos = effective_prog_pos(desc, atype, pl); /* no link or prog match, skip the cgroup of this layer */ - continue; -found: + if (pos < 0) + continue; + progs = rcu_dereference_protected( desc->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); @@ -1196,7 +1212,7 @@ static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, /* if update effective array failed replace the prog with a dummy prog*/ pl->prog = old_prog; pl->link = link; - purge_effective_progs(cgrp, old_prog, link, atype); + purge_effective_progs(cgrp, pl, atype); } /* now can actually delete it from this cgroup list */ -- cgit v1.2.3 From 269f2b43fae692d1f3988c9f888a6301aa537b82 Mon Sep 17 00:00:00 2001 From: Wang Yan Date: Mon, 22 Jun 2026 18:33:48 +0800 Subject: time: Fix off-by-one in compat settimeofday() usec validation The compat version of settimeofday() uses '>' instead of '>=' when validating tv_usec against USEC_PER_SEC, allowing the value 1000000 to pass the check. After the subsequent conversion to nanoseconds (tv_nsec *= NSEC_PER_USEC), this results in tv_nsec == NSEC_PER_SEC, which violates the timespec invariant that tv_nsec must be strictly less than NSEC_PER_SEC. The native settimeofday() was already fixed in commit ce4abda5e126 ("time: Fix off-by-one in settimeofday() usec validation"), but the compat counterpart was missed. Fix it by using '>=' to reject tv_usec values outside the valid range [0, USEC_PER_SEC - 1]. Fixes: 5e0fb1b57bea ("y2038: time: avoid timespec usage in settimeofday()") Signed-off-by: Wang Yan Signed-off-by: Thomas Gleixner Acked-by: Arnd Bergmann Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260622103348.120255-1-wangyan01@kylinos.cn --- kernel/time/time.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/time/time.c b/kernel/time/time.c index 771cef87ad3b..0dd63a91e7c5 100644 --- a/kernel/time/time.c +++ b/kernel/time/time.c @@ -251,7 +251,7 @@ COMPAT_SYSCALL_DEFINE2(settimeofday, struct old_timeval32 __user *, tv, get_user(new_ts.tv_nsec, &tv->tv_usec)) return -EFAULT; - if (new_ts.tv_nsec > USEC_PER_SEC || new_ts.tv_nsec < 0) + if (new_ts.tv_nsec >= USEC_PER_SEC || new_ts.tv_nsec < 0) return -EINVAL; new_ts.tv_nsec *= NSEC_PER_USEC; -- cgit v1.2.3 From bba2c3615bd6cfee7456d1130f2e6b01b3f4e9ba Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 22 Jun 2026 05:32:54 -1000 Subject: sched_ext: Move sources under kernel/sched/ext/ The sched_ext sources had grown to ten ext* files directly under kernel/sched/. Move them into a new kernel/sched/ext/ subdirectory and drop the now-redundant ext_ prefix. ext.c/h keep their names. kernel/sched/ext.{c,h} -> kernel/sched/ext/ext.{c,h} kernel/sched/ext_internal.h -> kernel/sched/ext/internal.h kernel/sched/ext_types.h -> kernel/sched/ext/types.h kernel/sched/ext_idle.{c,h} -> kernel/sched/ext/idle.{c,h} kernel/sched/ext_cid.{c,h} -> kernel/sched/ext/cid.{c,h} kernel/sched/ext_arena.{c,h} -> kernel/sched/ext/arena.{c,h} The include paths in build_policy.c and sched.h, the MAINTAINERS glob, and a few documentation and comment references are updated to match. No code or symbol changes. Suggested-by: Linus Torvalds Reviewed-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/build_policy.c | 18 +- kernel/sched/ext.c | 10882 ------------------------------------------ kernel/sched/ext.h | 95 - kernel/sched/ext/arena.c | 131 + kernel/sched/ext/arena.h | 18 + kernel/sched/ext/cid.c | 707 +++ kernel/sched/ext/cid.h | 271 ++ kernel/sched/ext/ext.c | 10882 ++++++++++++++++++++++++++++++++++++++++++ kernel/sched/ext/ext.h | 95 + kernel/sched/ext/idle.c | 1511 ++++++ kernel/sched/ext/idle.h | 27 + kernel/sched/ext/internal.h | 1653 +++++++ kernel/sched/ext/types.h | 144 + kernel/sched/ext_arena.c | 131 - kernel/sched/ext_arena.h | 18 - kernel/sched/ext_cid.c | 707 --- kernel/sched/ext_cid.h | 271 -- kernel/sched/ext_idle.c | 1511 ------ kernel/sched/ext_idle.h | 27 - kernel/sched/ext_internal.h | 1653 ------- kernel/sched/ext_types.h | 144 - kernel/sched/sched.h | 2 +- 22 files changed, 15449 insertions(+), 15449 deletions(-) delete mode 100644 kernel/sched/ext.c delete mode 100644 kernel/sched/ext.h create mode 100644 kernel/sched/ext/arena.c create mode 100644 kernel/sched/ext/arena.h create mode 100644 kernel/sched/ext/cid.c create mode 100644 kernel/sched/ext/cid.h create mode 100644 kernel/sched/ext/ext.c create mode 100644 kernel/sched/ext/ext.h create mode 100644 kernel/sched/ext/idle.c create mode 100644 kernel/sched/ext/idle.h create mode 100644 kernel/sched/ext/internal.h create mode 100644 kernel/sched/ext/types.h delete mode 100644 kernel/sched/ext_arena.c delete mode 100644 kernel/sched/ext_arena.h delete mode 100644 kernel/sched/ext_cid.c delete mode 100644 kernel/sched/ext_cid.h delete mode 100644 kernel/sched/ext_idle.c delete mode 100644 kernel/sched/ext_idle.h delete mode 100644 kernel/sched/ext_internal.h delete mode 100644 kernel/sched/ext_types.h (limited to 'kernel') diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c index 067979a7b69e..d74b54f81992 100644 --- a/kernel/sched/build_policy.c +++ b/kernel/sched/build_policy.c @@ -61,15 +61,15 @@ # include # include # include -# include "ext_types.h" -# include "ext_internal.h" -# include "ext_cid.h" -# include "ext_arena.h" -# include "ext_idle.h" -# include "ext.c" -# include "ext_cid.c" -# include "ext_arena.c" -# include "ext_idle.c" +# include "ext/types.h" +# include "ext/internal.h" +# include "ext/cid.h" +# include "ext/arena.h" +# include "ext/idle.h" +# include "ext/ext.c" +# include "ext/cid.c" +# include "ext/arena.c" +# include "ext/idle.c" #endif #include "syscalls.c" diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c deleted file mode 100644 index 0db6fa2daea3..000000000000 --- a/kernel/sched/ext.c +++ /dev/null @@ -1,10882 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst - * - * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. - * Copyright (c) 2022 Tejun Heo - * Copyright (c) 2022 David Vernet - */ - -static DEFINE_RAW_SPINLOCK(scx_sched_lock); - -/* - * NOTE: sched_ext is in the process of growing multiple scheduler support and - * scx_root usage is in a transitional state. Naked dereferences are safe if the - * caller is one of the tasks attached to SCX and explicit RCU dereference is - * necessary otherwise. Naked scx_root dereferences trigger sparse warnings but - * are used as temporary markers to indicate that the dereferences need to be - * updated to point to the associated scheduler instances rather than scx_root. - */ -struct scx_sched __rcu *scx_root; - -/* - * All scheds, writers must hold both scx_enable_mutex and scx_sched_lock. - * Readers can hold either or rcu_read_lock(). - */ -static LIST_HEAD(scx_sched_all); - -#ifdef CONFIG_EXT_SUB_SCHED -static const struct rhashtable_params scx_sched_hash_params = { - .key_len = sizeof_field(struct scx_sched, ops.sub_cgroup_id), - .key_offset = offsetof(struct scx_sched, ops.sub_cgroup_id), - .head_offset = offsetof(struct scx_sched, hash_node), - .insecure_elasticity = true, /* inserted under scx_sched_lock */ -}; - -static struct rhashtable scx_sched_hash; -#endif - -/* see SCX_OPS_TID_TO_TASK */ -static const struct rhashtable_params scx_tid_hash_params = { - .key_len = sizeof_field(struct sched_ext_entity, tid), - .key_offset = offsetof(struct sched_ext_entity, tid), - .head_offset = offsetof(struct sched_ext_entity, tid_hash_node), - .insecure_elasticity = true, /* inserted/removed under scx_tasks_lock */ -}; -static struct rhashtable scx_tid_hash; - -/* - * During exit, a task may schedule after losing its PIDs. When disabling the - * BPF scheduler, we need to be able to iterate tasks in every state to - * guarantee system safety. Maintain a dedicated task list which contains every - * task between its fork and eventual free. - */ -static DEFINE_RAW_SPINLOCK(scx_tasks_lock); -static LIST_HEAD(scx_tasks); - -/* ops enable/disable */ -static DEFINE_MUTEX(scx_enable_mutex); -DEFINE_STATIC_KEY_FALSE(__scx_enabled); -DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); -static atomic_t scx_enable_state_var = ATOMIC_INIT(SCX_DISABLED); -static DEFINE_RAW_SPINLOCK(scx_bypass_lock); -static bool scx_init_task_enabled; -static bool scx_switching_all; -DEFINE_STATIC_KEY_FALSE(__scx_switched_all); -static DEFINE_STATIC_KEY_FALSE(__scx_tid_to_task_enabled); - -/* - * True once SCX_OPS_TID_TO_TASK has been negotiated with the root scheduler - * and the tid->task table is live. Wraps the static key so callers don't - * take the address, and hints "likely enabled" for the common case where - * the feature is in use. - */ -static inline bool scx_tid_to_task_enabled(void) -{ - return static_branch_likely(&__scx_tid_to_task_enabled); -} - -static atomic_long_t scx_nr_rejected = ATOMIC_LONG_INIT(0); -static atomic_long_t scx_hotplug_seq = ATOMIC_LONG_INIT(0); - -/* Global cursor for the per-CPU tid allocator. Starts at 1; tid 0 is reserved. */ -static atomic64_t scx_tid_cursor = ATOMIC64_INIT(1); - -#ifdef CONFIG_EXT_SUB_SCHED -/* - * The sub sched being enabled. Used by scx_disable_and_exit_task() to exit - * tasks for the sub-sched being enabled. Use a global variable instead of a - * per-task field as all enables are serialized. - */ -static struct scx_sched *scx_enabling_sub_sched; -#else -#define scx_enabling_sub_sched (struct scx_sched *)NULL -#endif /* CONFIG_EXT_SUB_SCHED */ - -/* - * A monotonically increasing sequence number that is incremented every time a - * scheduler is enabled. This can be used to check if any custom sched_ext - * scheduler has ever been used in the system. - */ -static atomic_long_t scx_enable_seq = ATOMIC_LONG_INIT(0); - -/* - * Watchdog interval. All scx_sched's share a single watchdog timer and the - * interval is half of the shortest sch->watchdog_timeout. - */ -static unsigned long scx_watchdog_interval; - -/* - * The last time the delayed work was run. This delayed work relies on - * ksoftirqd being able to run to service timer interrupts, so it's possible - * that this work itself could get wedged. To account for this, we check that - * it's not stalled in the timer tick, and trigger an error if it is. - */ -static unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; - -static struct delayed_work scx_watchdog_work; - -/* - * For %SCX_KICK_WAIT: Each CPU has a pointer to an array of kick_sync sequence - * numbers. The arrays are allocated with kvzalloc() as size can exceed percpu - * allocator limits on large machines. O(nr_cpu_ids^2) allocation, allocated - * lazily when enabling and freed when disabling to avoid waste when sched_ext - * isn't active. - */ -struct scx_kick_syncs { - struct rcu_head rcu; - unsigned long syncs[]; -}; - -static DEFINE_PER_CPU(struct scx_kick_syncs __rcu *, scx_kick_syncs); - -/* - * Per-CPU buffered allocator state for p->scx.tid. Each CPU pulls a chunk of - * SCX_TID_CHUNK ids from scx_tid_cursor and hands them out locally without - * further synchronization. See scx_alloc_tid(). - */ -struct scx_tid_alloc { - u64 next; - u64 end; -}; -static DEFINE_PER_CPU(struct scx_tid_alloc, scx_tid_alloc); - -/* - * Direct dispatch marker. - * - * Non-NULL values are used for direct dispatch from enqueue path. A valid - * pointer points to the task currently being enqueued. An ERR_PTR value is used - * to indicate that direct dispatch has already happened. - */ -static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); - -static const struct rhashtable_params dsq_hash_params = { - .key_len = sizeof_field(struct scx_dispatch_q, id), - .key_offset = offsetof(struct scx_dispatch_q, id), - .head_offset = offsetof(struct scx_dispatch_q, hash_node), -}; - -static LLIST_HEAD(dsqs_to_free); - -/* string formatting from BPF */ -struct scx_bstr_buf { - u64 data[MAX_BPRINTF_VARARGS]; - char line[SCX_EXIT_MSG_LEN]; -}; - -static DEFINE_RAW_SPINLOCK(scx_exit_bstr_buf_lock); -static struct scx_bstr_buf scx_exit_bstr_buf; - -/* ops debug dump */ -static DEFINE_RAW_SPINLOCK(scx_dump_lock); - -struct scx_dump_data { - s32 cpu; - bool first; - s32 cursor; - struct seq_buf *s; - const char *prefix; - struct scx_bstr_buf buf; -}; - -static struct scx_dump_data scx_dump_data = { - .cpu = -1, -}; - -/* /sys/kernel/sched_ext interface */ -static struct kset *scx_kset; - -/* - * Parameters that can be adjusted through /sys/module/sched_ext/parameters. - * There usually is no reason to modify these as normal scheduler operation - * shouldn't be affected by them. The knobs are primarily for debugging. - */ -static unsigned int scx_slice_bypass_us = SCX_SLICE_BYPASS / NSEC_PER_USEC; -static unsigned int scx_bypass_lb_intv_us = SCX_BYPASS_LB_DFL_INTV_US; - -static int set_slice_us(const char *val, const struct kernel_param *kp) -{ - return param_set_uint_minmax(val, kp, 100, 100 * USEC_PER_MSEC); -} - -static const struct kernel_param_ops slice_us_param_ops = { - .set = set_slice_us, - .get = param_get_uint, -}; - -static int set_bypass_lb_intv_us(const char *val, const struct kernel_param *kp) -{ - return param_set_uint_minmax(val, kp, 0, 10 * USEC_PER_SEC); -} - -static const struct kernel_param_ops bypass_lb_intv_us_param_ops = { - .set = set_bypass_lb_intv_us, - .get = param_get_uint, -}; - -#undef MODULE_PARAM_PREFIX -#define MODULE_PARAM_PREFIX "sched_ext." - -module_param_cb(slice_bypass_us, &slice_us_param_ops, &scx_slice_bypass_us, 0600); -MODULE_PARM_DESC(slice_bypass_us, "bypass slice in microseconds, applied on [un]load (100us to 100ms)"); -module_param_cb(bypass_lb_intv_us, &bypass_lb_intv_us_param_ops, &scx_bypass_lb_intv_us, 0600); -MODULE_PARM_DESC(bypass_lb_intv_us, "bypass load balance interval in microseconds (0 (disable) to 10s)"); - -#undef MODULE_PARAM_PREFIX - -#define CREATE_TRACE_POINTS -#include - -static void run_deferred(struct rq *rq); -static bool task_dead_and_done(struct task_struct *p); -static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); -static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind); - -__printf(5, 6) bool __scx_exit(struct scx_sched *sch, - enum scx_exit_kind kind, s64 exit_code, - s32 exit_cpu, const char *fmt, ...) -{ - va_list args; - bool ret; - - va_start(args, fmt); - ret = scx_vexit(sch, kind, exit_code, exit_cpu, fmt, args); - va_end(args); - - return ret; -} - -#define SCX_HAS_OP(sch, op) test_bit(SCX_OP_IDX(op), (sch)->has_op) - -static long jiffies_delta_msecs(unsigned long at, unsigned long now) -{ - if (time_after(at, now)) - return jiffies_to_msecs(at - now); - else - return -(long)jiffies_to_msecs(now - at); -} - -static bool u32_before(u32 a, u32 b) -{ - return (s32)(a - b) < 0; -} - -#ifdef CONFIG_EXT_SUB_SCHED -/** - * scx_parent - Find the parent sched - * @sch: sched to find the parent of - * - * Returns the parent scheduler or %NULL if @sch is root. - */ -static struct scx_sched *scx_parent(struct scx_sched *sch) -{ - if (sch->level) - return sch->ancestors[sch->level - 1]; - else - return NULL; -} - -/** - * scx_next_descendant_pre - find the next descendant for pre-order walk - * @pos: the current position (%NULL to initiate traversal) - * @root: sched whose descendants to walk - * - * To be used by scx_for_each_descendant_pre(). Find the next descendant to - * visit for pre-order traversal of @root's descendants. @root is included in - * the iteration and the first node to be visited. - */ -static struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, - struct scx_sched *root) -{ - struct scx_sched *next; - - lockdep_assert(lockdep_is_held(&scx_enable_mutex) || - lockdep_is_held(&scx_sched_lock)); - - /* if first iteration, visit @root */ - if (!pos) - return root; - - /* visit the first child if exists */ - next = list_first_entry_or_null(&pos->children, struct scx_sched, sibling); - if (next) - return next; - - /* no child, visit my or the closest ancestor's next sibling */ - while (pos != root) { - if (!list_is_last(&pos->sibling, &scx_parent(pos)->children)) - return list_next_entry(pos, sibling); - pos = scx_parent(pos); - } - - return NULL; -} - -static struct scx_sched *scx_find_sub_sched(u64 cgroup_id) -{ - return rhashtable_lookup(&scx_sched_hash, &cgroup_id, - scx_sched_hash_params); -} - -static void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) -{ - rcu_assign_pointer(p->scx.sched, sch); -} -#else /* CONFIG_EXT_SUB_SCHED */ -static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; } -static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; } -static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {} -#endif /* CONFIG_EXT_SUB_SCHED */ - -/** - * scx_is_descendant - Test whether sched is a descendant - * @sch: sched to test - * @ancestor: ancestor sched to test against - * - * Test whether @sch is a descendant of @ancestor. - */ -static bool scx_is_descendant(struct scx_sched *sch, struct scx_sched *ancestor) -{ - if (sch->level < ancestor->level) - return false; - return sch->ancestors[ancestor->level] == ancestor; -} - -/** - * scx_for_each_descendant_pre - pre-order walk of a sched's descendants - * @pos: iteration cursor - * @root: sched to walk the descendants of - * - * Walk @root's descendants. @root is included in the iteration and the first - * node to be visited. Must be called with either scx_enable_mutex or - * scx_sched_lock held. - */ -#define scx_for_each_descendant_pre(pos, root) \ - for ((pos) = scx_next_descendant_pre(NULL, (root)); (pos); \ - (pos) = scx_next_descendant_pre((pos), (root))) - -static struct scx_dispatch_q *find_global_dsq(struct scx_sched *sch, s32 cpu) -{ - return &sch->pnode[cpu_to_node(cpu)]->global_dsq; -} - -static struct scx_dispatch_q *find_user_dsq(struct scx_sched *sch, u64 dsq_id) -{ - return rhashtable_lookup(&sch->dsq_hash, &dsq_id, dsq_hash_params); -} - -static const struct sched_class *scx_setscheduler_class(struct task_struct *p) -{ - if (p->sched_class == &stop_sched_class) - return &stop_sched_class; - - return __setscheduler_class(p->policy, p->prio); -} - -static struct scx_dispatch_q *bypass_dsq(struct scx_sched *sch, s32 cpu) -{ - return &per_cpu_ptr(sch->pcpu, cpu)->bypass_dsq; -} - -static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 cpu) -{ -#ifdef CONFIG_EXT_SUB_SCHED - /* - * If @sch is a sub-sched which is bypassing, its tasks should go into - * the bypass DSQs of the nearest ancestor which is not bypassing. The - * not-bypassing ancestor is responsible for scheduling all tasks from - * bypassing sub-trees. If all ancestors including root are bypassing, - * all tasks should go to the root's bypass DSQs. - * - * Whenever a sched starts bypassing, all runnable tasks in its subtree - * are re-enqueued after scx_bypassing() is turned on, guaranteeing that - * all tasks are transferred to the right DSQs. - */ - while (scx_parent(sch) && scx_bypassing(sch, cpu)) - sch = scx_parent(sch); -#endif /* CONFIG_EXT_SUB_SCHED */ - - return bypass_dsq(sch, cpu); -} - -/** - * bypass_dsp_enabled - Check if bypass dispatch path is enabled - * @sch: scheduler to check - * - * When a descendant scheduler enters bypass mode, bypassed tasks are scheduled - * by the nearest non-bypassing ancestor, or the root scheduler if all ancestors - * are bypassing. In the former case, the ancestor is not itself bypassing but - * its bypass DSQs will be populated with bypassed tasks from descendants. Thus, - * the ancestor's bypass dispatch path must be active even though its own - * bypass_depth remains zero. - * - * This function checks bypass_dsp_enable_depth which is managed separately from - * bypass_depth to enable this decoupling. See enable_bypass_dsp() and - * disable_bypass_dsp(). - */ -static bool bypass_dsp_enabled(struct scx_sched *sch) -{ - return unlikely(atomic_read(&sch->bypass_dsp_enable_depth)); -} - -/** - * rq_is_open - Is the rq available for immediate execution of an SCX task? - * @rq: rq to test - * @enq_flags: optional %SCX_ENQ_* of the task being enqueued - * - * Returns %true if @rq is currently open for executing an SCX task. After a - * %false return, @rq is guaranteed to invoke SCX dispatch path at least once - * before going to idle and not inserting a task into @rq's local DSQ after a - * %false return doesn't cause @rq to stall. - */ -static bool rq_is_open(struct rq *rq, u64 enq_flags) -{ - lockdep_assert_rq_held(rq); - - /* - * A higher-priority class task is either running or in the process of - * waking up on @rq. - */ - if (sched_class_above(rq->next_class, &ext_sched_class)) - return false; - - /* - * @rq is either in transition to or in idle and there is no - * higher-priority class task waking up on it. - */ - if (sched_class_above(&ext_sched_class, rq->next_class)) - return true; - - /* - * @rq is either picking, in transition to, or running an SCX task. - */ - - /* - * If we're in the dispatch path holding rq lock, $curr may or may not - * be ready depending on whether the on-going dispatch decides to extend - * $curr's slice. We say yes here and resolve it at the end of dispatch. - * See balance_one(). - */ - if (rq->scx.flags & SCX_RQ_IN_BALANCE) - return true; - - /* - * %SCX_ENQ_PREEMPT clears $curr's slice if on SCX and kicks dispatch, - * so allow it to avoid spuriously triggering reenq on a combined - * PREEMPT|IMMED insertion. - */ - if (enq_flags & SCX_ENQ_PREEMPT) - return true; - - /* - * @rq is either in transition to or running an SCX task and can't go - * idle without another SCX dispatch cycle. - */ - return false; -} - -/* - * Track the rq currently locked. - * - * This allows kfuncs to safely operate on rq from any scx ops callback, - * knowing which rq is already locked. - */ -DEFINE_PER_CPU(struct rq *, scx_locked_rq_state); - -static inline void update_locked_rq(struct rq *rq) -{ - /* - * Check whether @rq is actually locked. This can help expose bugs - * or incorrect assumptions about the context in which a kfunc or - * callback is executed. - */ - if (rq) - lockdep_assert_rq_held(rq); - __this_cpu_write(scx_locked_rq_state, rq); -} - -/* - * SCX ops can recurse via scx_bpf_sub_dispatch() - the inner call must not - * clobber the outer's scx_locked_rq_state. Save it on entry, restore on exit. - */ -#define SCX_CALL_OP(sch, op, locked_rq, args...) \ -do { \ - struct rq *__prev_locked_rq; \ - \ - if (locked_rq) { \ - __prev_locked_rq = scx_locked_rq(); \ - update_locked_rq(locked_rq); \ - } \ - (sch)->ops.op(args); \ - if (locked_rq) \ - update_locked_rq(__prev_locked_rq); \ -} while (0) - -/* - * Flipped on enable per sch->is_cid_type. Declared in ext_internal.h so - * subsystem inlines can read it. - */ -DEFINE_STATIC_KEY_FALSE(__scx_is_cid_type); - -/* - * scx_cpu_arg() wraps a cpu arg being handed to an SCX op. For cid-form - * schedulers it resolves to the matching cid; for cpu-form it passes @cpu - * through. scx_cpu_ret() is the inverse for a cpu/cid returned from an op - * (currently only ops.select_cpu); it validates the BPF-supplied cid and - * triggers scx_error() on @sch if invalid. - */ -static s32 scx_cpu_arg(s32 cpu) -{ - if (scx_is_cid_type()) - return __scx_cpu_to_cid(cpu); - return cpu; -} - -static s32 scx_cpu_ret(struct scx_sched *sch, s32 cpu_or_cid) -{ - if (cpu_or_cid < 0 || !scx_is_cid_type()) - return cpu_or_cid; - return scx_cid_to_cpu(sch, cpu_or_cid); -} - -#define SCX_CALL_OP_RET(sch, op, locked_rq, args...) \ -({ \ - struct rq *__prev_locked_rq; \ - __typeof__((sch)->ops.op(args)) __ret; \ - \ - if (locked_rq) { \ - __prev_locked_rq = scx_locked_rq(); \ - update_locked_rq(locked_rq); \ - } \ - __ret = (sch)->ops.op(args); \ - if (locked_rq) \ - update_locked_rq(__prev_locked_rq); \ - __ret; \ -}) - -/* - * SCX_CALL_OP_TASK*() invokes an SCX op that takes one or two task arguments - * and records them in current->scx.kf_tasks[] for the duration of the call. A - * kfunc invoked from inside such an op can then use - * scx_kf_arg_task_ok() to verify that its task argument is one of - * those subject tasks. - * - * Every SCX_CALL_OP_TASK*() call site invokes its op with @p's rq lock held - - * either via the @locked_rq argument here, or (for ops.select_cpu()) via @p's - * pi_lock held by try_to_wake_up() with rq tracking via scx_rq.in_select_cpu. - * So if kf_tasks[] is set, @p's scheduler-protected fields are stable. - * - * kf_tasks[] can not stack, so task-based SCX ops must not nest. The - * WARN_ON_ONCE() in each macro catches a re-entry of any of the three variants - * while a previous one is still in progress. - */ -#define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...) \ -do { \ - WARN_ON_ONCE(current->scx.kf_tasks[0]); \ - current->scx.kf_tasks[0] = task; \ - SCX_CALL_OP((sch), op, locked_rq, task, ##args); \ - current->scx.kf_tasks[0] = NULL; \ -} while (0) - -#define SCX_CALL_OP_TASK_RET(sch, op, locked_rq, task, args...) \ -({ \ - __typeof__((sch)->ops.op(task, ##args)) __ret; \ - WARN_ON_ONCE(current->scx.kf_tasks[0]); \ - current->scx.kf_tasks[0] = task; \ - __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task, ##args); \ - current->scx.kf_tasks[0] = NULL; \ - __ret; \ -}) - -#define SCX_CALL_OP_2TASKS_RET(sch, op, locked_rq, task0, task1, args...) \ -({ \ - __typeof__((sch)->ops.op(task0, task1, ##args)) __ret; \ - WARN_ON_ONCE(current->scx.kf_tasks[0]); \ - current->scx.kf_tasks[0] = task0; \ - current->scx.kf_tasks[1] = task1; \ - __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task0, task1, ##args); \ - current->scx.kf_tasks[0] = NULL; \ - current->scx.kf_tasks[1] = NULL; \ - __ret; \ -}) - -/** - * scx_call_op_set_cpumask - invoke ops.set_cpumask / ops_cid.set_cmask for @task - * @sch: scx_sched being invoked - * @rq: rq to update as the currently-locked rq, or NULL - * @task: task whose affinity is changing - * @cpumask: new cpumask - * - * For cid-form schedulers, translate @cpumask to a cmask via the per-cpu - * scratch in ext_cid.c and dispatch through the ops_cid union view. Caller - * must hold @rq's rq lock so this_cpu_ptr is stable across the call. - */ -static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, - struct task_struct *task, - const struct cpumask *cpumask) -{ - WARN_ON_ONCE(current->scx.kf_tasks[0]); - current->scx.kf_tasks[0] = task; - if (rq) - update_locked_rq(rq); - - if (scx_is_cid_type()) { - struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch); - /* - * Build the per-CPU arena cmask and hand BPF its arena address. - * Caller holds the rq lock with IRQs disabled, which makes us - * the sole user of the scratch area. - */ - scx_cpumask_to_cmask(cpumask, kern_va); - sch->ops_cid.set_cmask(task, scx_kaddr_to_arena(sch, kern_va)); - } else { - sch->ops.set_cpumask(task, cpumask); - } - - if (rq) - update_locked_rq(NULL); - current->scx.kf_tasks[0] = NULL; -} - -/* see SCX_CALL_OP_TASK() */ -static __always_inline bool scx_kf_arg_task_ok(struct scx_sched *sch, - struct task_struct *p) -{ - if (unlikely((p != current->scx.kf_tasks[0] && - p != current->scx.kf_tasks[1]))) { - scx_error(sch, "called on a task not being operated on"); - return false; - } - - return true; -} - -enum scx_dsq_iter_flags { - /* iterate in the reverse dispatch order */ - SCX_DSQ_ITER_REV = 1U << 16, - - __SCX_DSQ_ITER_HAS_SLICE = 1U << 30, - __SCX_DSQ_ITER_HAS_VTIME = 1U << 31, - - __SCX_DSQ_ITER_USER_FLAGS = SCX_DSQ_ITER_REV, - __SCX_DSQ_ITER_ALL_FLAGS = __SCX_DSQ_ITER_USER_FLAGS | - __SCX_DSQ_ITER_HAS_SLICE | - __SCX_DSQ_ITER_HAS_VTIME, -}; - -/** - * nldsq_next_task - Iterate to the next task in a non-local DSQ - * @dsq: non-local dsq being iterated - * @cur: current position, %NULL to start iteration - * @rev: walk backwards - * - * Returns %NULL when iteration is finished. - */ -static struct task_struct *nldsq_next_task(struct scx_dispatch_q *dsq, - struct task_struct *cur, bool rev) -{ - struct list_head *list_node; - struct scx_dsq_list_node *dsq_lnode; - - lockdep_assert_held(&dsq->lock); - - if (cur) - list_node = &cur->scx.dsq_list.node; - else - list_node = &dsq->list; - - /* find the next task, need to skip BPF iteration cursors */ - do { - if (rev) - list_node = list_node->prev; - else - list_node = list_node->next; - - if (list_node == &dsq->list) - return NULL; - - dsq_lnode = container_of(list_node, struct scx_dsq_list_node, - node); - } while (dsq_lnode->flags & SCX_DSQ_LNODE_ITER_CURSOR); - - return container_of(dsq_lnode, struct task_struct, scx.dsq_list); -} - -#define nldsq_for_each_task(p, dsq) \ - for ((p) = nldsq_next_task((dsq), NULL, false); (p); \ - (p) = nldsq_next_task((dsq), (p), false)) - -/** - * nldsq_cursor_next_task - Iterate to the next task given a cursor in a non-local DSQ - * @cursor: scx_dsq_list_node initialized with INIT_DSQ_LIST_CURSOR() - * @dsq: non-local dsq being iterated - * - * Find the next task in a cursor based iteration. The caller must have - * initialized @cursor using INIT_DSQ_LIST_CURSOR() and can release the DSQ lock - * between the iteration steps. - * - * Only tasks which were queued before @cursor was initialized are visible. This - * bounds the iteration and guarantees that vtime never jumps in the other - * direction while iterating. - */ -static struct task_struct *nldsq_cursor_next_task(struct scx_dsq_list_node *cursor, - struct scx_dispatch_q *dsq) -{ - bool rev = cursor->flags & SCX_DSQ_ITER_REV; - struct task_struct *p; - - lockdep_assert_held(&dsq->lock); - BUG_ON(!(cursor->flags & SCX_DSQ_LNODE_ITER_CURSOR)); - - if (list_empty(&cursor->node)) - p = NULL; - else - p = container_of(cursor, struct task_struct, scx.dsq_list); - - /* skip cursors and tasks that were queued after @cursor init */ - do { - p = nldsq_next_task(dsq, p, rev); - } while (p && unlikely(u32_before(cursor->priv, p->scx.dsq_seq))); - - if (p) { - if (rev) - list_move_tail(&cursor->node, &p->scx.dsq_list.node); - else - list_move(&cursor->node, &p->scx.dsq_list.node); - } else { - list_del_init(&cursor->node); - } - - return p; -} - -/** - * nldsq_cursor_lost_task - Test whether someone else took the task since iteration - * @cursor: scx_dsq_list_node initialized with INIT_DSQ_LIST_CURSOR() - * @rq: rq @p was on - * @dsq: dsq @p was on - * @p: target task - * - * @p is a task returned by nldsq_cursor_next_task(). The locks may have been - * dropped and re-acquired inbetween. Verify that no one else took or is in the - * process of taking @p from @dsq. - * - * On %false return, the caller can assume full ownership of @p. - */ -static bool nldsq_cursor_lost_task(struct scx_dsq_list_node *cursor, - struct rq *rq, struct scx_dispatch_q *dsq, - struct task_struct *p) -{ - lockdep_assert_rq_held(rq); - lockdep_assert_held(&dsq->lock); - - /* - * @p could have already left $src_dsq, got re-enqueud, or be in the - * process of being consumed by someone else. - */ - if (unlikely(p->scx.dsq != dsq || - u32_before(cursor->priv, p->scx.dsq_seq) || - p->scx.holding_cpu >= 0)) - return true; - - /* if @p has stayed on @dsq, its rq couldn't have changed */ - if (WARN_ON_ONCE(rq != task_rq(p))) - return true; - - return false; -} - -/* - * BPF DSQ iterator. Tasks in a non-local DSQ can be iterated in [reverse] - * dispatch order. BPF-visible iterator is opaque and larger to allow future - * changes without breaking backward compatibility. Can be used with - * bpf_for_each(). See bpf_iter_scx_dsq_*(). - */ -struct bpf_iter_scx_dsq_kern { - struct scx_dsq_list_node cursor; - struct scx_dispatch_q *dsq; - u64 slice; - u64 vtime; -} __attribute__((aligned(8))); - -struct bpf_iter_scx_dsq { - u64 __opaque[6]; -} __attribute__((aligned(8))); - - -static u32 scx_get_task_state(const struct task_struct *p) -{ - return p->scx.flags & SCX_TASK_STATE_MASK; -} - -static void scx_set_task_state(struct task_struct *p, u32 state) -{ - u32 prev_state = scx_get_task_state(p); - bool warn = false; - - switch (state) { - case SCX_TASK_NONE: - warn = prev_state == SCX_TASK_DEAD; - break; - case SCX_TASK_INIT_BEGIN: - warn = prev_state != SCX_TASK_NONE; - break; - case SCX_TASK_INIT: - warn = prev_state != SCX_TASK_INIT_BEGIN; - p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; - break; - case SCX_TASK_READY: - warn = !(prev_state == SCX_TASK_INIT || - prev_state == SCX_TASK_ENABLED); - break; - case SCX_TASK_ENABLED: - warn = prev_state != SCX_TASK_READY; - break; - case SCX_TASK_DEAD: - warn = !(prev_state == SCX_TASK_NONE || - prev_state == SCX_TASK_INIT_BEGIN); - break; - default: - WARN_ONCE(1, "sched_ext: Invalid task state %d -> %d for %s[%d]", - prev_state, state, p->comm, p->pid); - return; - } - - WARN_ONCE(warn, "sched_ext: Invalid task state transition 0x%x -> 0x%x for %s[%d]", - prev_state, state, p->comm, p->pid); - - p->scx.flags &= ~SCX_TASK_STATE_MASK; - p->scx.flags |= state; -} - -/* - * SCX task iterator. - */ -struct scx_task_iter { - struct sched_ext_entity cursor; - struct task_struct *locked_task; - struct rq *rq; - struct rq_flags rf; - u32 cnt; - bool list_locked; -#ifdef CONFIG_EXT_SUB_SCHED - struct cgroup *cgrp; - struct cgroup_subsys_state *css_pos; - struct css_task_iter css_iter; -#endif -}; - -/** - * scx_task_iter_start - Lock scx_tasks_lock and start a task iteration - * @iter: iterator to init - * @cgrp: Optional root of cgroup subhierarchy to iterate - * - * Initialize @iter. Once initialized, @iter must eventually be stopped with - * scx_task_iter_stop(). - * - * If @cgrp is %NULL, scx_tasks is used for iteration and this function returns - * with scx_tasks_lock held and @iter->cursor inserted into scx_tasks. - * - * If @cgrp is not %NULL, @cgrp and its descendants' tasks are walked using - * @iter->css_iter. The caller must be holding cgroup_lock() to prevent cgroup - * task migrations. - * - * The two modes of iterations are largely independent and it's likely that - * scx_tasks can be removed in favor of always using cgroup iteration if - * CONFIG_SCHED_CLASS_EXT depends on CONFIG_CGROUPS. - * - * scx_tasks_lock and the rq lock may be released using scx_task_iter_unlock() - * between this and the first next() call or between any two next() calls. If - * the locks are released between two next() calls, the caller is responsible - * for ensuring that the task being iterated remains accessible either through - * RCU read lock or obtaining a reference count. - * - * All tasks which existed when the iteration started are guaranteed to be - * visited as long as they are not dead. - */ -static void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp) -{ - memset(iter, 0, sizeof(*iter)); - -#ifdef CONFIG_EXT_SUB_SCHED - if (cgrp) { - lockdep_assert_held(&cgroup_mutex); - iter->cgrp = cgrp; - iter->css_pos = css_next_descendant_pre(NULL, &iter->cgrp->self); - css_task_iter_start(iter->css_pos, CSS_TASK_ITER_WITH_DEAD, - &iter->css_iter); - return; - } -#endif - raw_spin_lock_irq(&scx_tasks_lock); - - iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; - list_add(&iter->cursor.tasks_node, &scx_tasks); - iter->list_locked = true; -} - -static void __scx_task_iter_rq_unlock(struct scx_task_iter *iter) -{ - if (iter->locked_task) { - __balance_callbacks(iter->rq, &iter->rf); - task_rq_unlock(iter->rq, iter->locked_task, &iter->rf); - iter->locked_task = NULL; - } -} - -/** - * scx_task_iter_unlock - Unlock rq and scx_tasks_lock held by a task iterator - * @iter: iterator to unlock - * - * If @iter is in the middle of a locked iteration, it may be locking the rq of - * the task currently being visited in addition to scx_tasks_lock. Unlock both. - * This function can be safely called anytime during an iteration. The next - * iterator operation will automatically restore the necessary locking. - */ -static void scx_task_iter_unlock(struct scx_task_iter *iter) -{ - __scx_task_iter_rq_unlock(iter); - if (iter->list_locked) { - iter->list_locked = false; - raw_spin_unlock_irq(&scx_tasks_lock); - } -} - -static void __scx_task_iter_maybe_relock(struct scx_task_iter *iter) -{ - if (!iter->list_locked) { - raw_spin_lock_irq(&scx_tasks_lock); - iter->list_locked = true; - } -} - -/** - * scx_task_iter_relock - Re-acquire scx_tasks_lock and, optionally, @p's rq - * @iter: iterator to relock - * @p: task whose rq to lock, or %NULL for scx_tasks_lock only - * - * Counterpart to scx_task_iter_unlock(). Locking @p's rq is optional. Once - * re-acquired, both locks are managed by the iterator from here on. - */ -static void scx_task_iter_relock(struct scx_task_iter *iter, - struct task_struct *p) -{ - __scx_task_iter_maybe_relock(iter); - if (p) { - iter->rq = task_rq_lock(p, &iter->rf); - iter->locked_task = p; - } -} - -/** - * scx_task_iter_stop - Stop a task iteration and unlock scx_tasks_lock - * @iter: iterator to exit - * - * Exit a previously initialized @iter. Must be called with scx_tasks_lock held - * which is released on return. If the iterator holds a task's rq lock, that rq - * lock is also released. See scx_task_iter_start() for details. - */ -static void scx_task_iter_stop(struct scx_task_iter *iter) -{ -#ifdef CONFIG_EXT_SUB_SCHED - if (iter->cgrp) { - if (iter->css_pos) - css_task_iter_end(&iter->css_iter); - __scx_task_iter_rq_unlock(iter); - return; - } -#endif - __scx_task_iter_maybe_relock(iter); - list_del_init(&iter->cursor.tasks_node); - scx_task_iter_unlock(iter); -} - -/** - * scx_task_iter_next - Next task - * @iter: iterator to walk - * - * Visit the next task. See scx_task_iter_start() for details. Locks are dropped - * and re-acquired every %SCX_TASK_ITER_BATCH iterations to avoid causing stalls - * by holding scx_tasks_lock for too long. - */ -static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) -{ - struct list_head *cursor = &iter->cursor.tasks_node; - struct sched_ext_entity *pos; - - if (!(++iter->cnt % SCX_TASK_ITER_BATCH)) { - scx_task_iter_unlock(iter); - cond_resched(); - } - -#ifdef CONFIG_EXT_SUB_SCHED - if (iter->cgrp) { - while (iter->css_pos) { - struct task_struct *p; - - p = css_task_iter_next(&iter->css_iter); - if (p) - return p; - - css_task_iter_end(&iter->css_iter); - iter->css_pos = css_next_descendant_pre(iter->css_pos, - &iter->cgrp->self); - if (iter->css_pos) - css_task_iter_start(iter->css_pos, CSS_TASK_ITER_WITH_DEAD, - &iter->css_iter); - } - return NULL; - } -#endif - __scx_task_iter_maybe_relock(iter); - - list_for_each_entry(pos, cursor, tasks_node) { - if (&pos->tasks_node == &scx_tasks) - return NULL; - if (!(pos->flags & SCX_TASK_CURSOR)) { - list_move(cursor, &pos->tasks_node); - return container_of(pos, struct task_struct, scx); - } - } - - /* can't happen, should always terminate at scx_tasks above */ - BUG(); -} - -/** - * scx_task_iter_next_locked - Next non-idle task with its rq locked - * @iter: iterator to walk - * - * Visit the non-idle task with its rq lock held. Allows callers to specify - * whether they would like to filter out dead tasks. See scx_task_iter_start() - * for details. - */ -static struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter) -{ - struct task_struct *p; - - __scx_task_iter_rq_unlock(iter); - - while ((p = scx_task_iter_next(iter))) { - /* - * scx_task_iter is used to prepare and move tasks into SCX - * while loading the BPF scheduler and vice-versa while - * unloading. The init_tasks ("swappers") should be excluded - * from the iteration because: - * - * - It's unsafe to use __setschduler_prio() on an init_task to - * determine the sched_class to use as it won't preserve its - * idle_sched_class. - * - * - ops.init/exit_task() can easily be confused if called with - * init_tasks as they, e.g., share PID 0. - * - * As init_tasks are never scheduled through SCX, they can be - * skipped safely. Note that is_idle_task() which tests %PF_IDLE - * doesn't work here: - * - * - %PF_IDLE may not be set for an init_task whose CPU hasn't - * yet been onlined. - * - * - %PF_IDLE can be set on tasks that are not init_tasks. See - * play_idle_precise() used by CONFIG_IDLE_INJECT. - * - * Test for idle_sched_class as only init_tasks are on it. - */ - if (p->sched_class == &idle_sched_class) - continue; - - iter->rq = task_rq_lock(p, &iter->rf); - iter->locked_task = p; - - /* - * cgroup_task_dead() removes the dead tasks from cset->tasks - * after sched_ext_dead() and cgroup iteration may see tasks - * which already finished sched_ext_dead(). %SCX_TASK_DEAD is - * set by sched_ext_dead() under @p's rq lock. Test it to - * avoid visiting tasks which are already dead from SCX POV. - */ - if (scx_get_task_state(p) == SCX_TASK_DEAD) { - __scx_task_iter_rq_unlock(iter); - continue; - } - - return p; - } - return NULL; -} - -/** - * scx_add_event - Increase an event counter for 'name' by 'cnt' - * @sch: scx_sched to account events for - * @name: an event name defined in struct scx_event_stats - * @cnt: the number of the event occurred - * - * This can be used when preemption is not disabled. - */ -#define scx_add_event(sch, name, cnt) do { \ - this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \ - trace_sched_ext_event(#name, (cnt)); \ -} while(0) - -/** - * __scx_add_event - Increase an event counter for 'name' by 'cnt' - * @sch: scx_sched to account events for - * @name: an event name defined in struct scx_event_stats - * @cnt: the number of the event occurred - * - * This should be used only when preemption is disabled. - */ -#define __scx_add_event(sch, name, cnt) do { \ - __this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \ - trace_sched_ext_event(#name, cnt); \ -} while(0) - -/** - * scx_agg_event - Aggregate an event counter 'kind' from 'src_e' to 'dst_e' - * @dst_e: destination event stats - * @src_e: source event stats - * @kind: a kind of event to be aggregated - */ -#define scx_agg_event(dst_e, src_e, kind) do { \ - (dst_e)->kind += READ_ONCE((src_e)->kind); \ -} while(0) - -/** - * scx_dump_event - Dump an event 'kind' in 'events' to 's' - * @s: output seq_buf - * @events: event stats - * @kind: a kind of event to dump - */ -#define scx_dump_event(s, events, kind) do { \ - dump_line(&(s), "%40s: %16lld", #kind, (events)->kind); \ -} while (0) - - -static void scx_read_events(struct scx_sched *sch, - struct scx_event_stats *events); - -static enum scx_enable_state scx_enable_state(void) -{ - return atomic_read(&scx_enable_state_var); -} - -static enum scx_enable_state scx_set_enable_state(enum scx_enable_state to) -{ - return atomic_xchg(&scx_enable_state_var, to); -} - -static bool scx_tryset_enable_state(enum scx_enable_state to, - enum scx_enable_state from) -{ - int from_v = from; - - return atomic_try_cmpxchg(&scx_enable_state_var, &from_v, to); -} - -/** - * wait_ops_state - Busy-wait the specified ops state to end - * @p: target task - * @opss: state to wait the end of - * - * Busy-wait for @p to transition out of @opss. This can only be used when the - * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also - * has load_acquire semantics to ensure that the caller can see the updates made - * in the enqueueing and dispatching paths. - */ -static void wait_ops_state(struct task_struct *p, unsigned long opss) -{ - do { - cpu_relax(); - } while (atomic_long_read_acquire(&p->scx.ops_state) == opss); -} - -static inline bool __cpu_valid(s32 cpu) -{ - return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu)); -} - -/** - * scx_cpu_valid - Verify a cpu number, to be used on ops input args - * @sch: scx_sched to abort on error - * @cpu: cpu number which came from a BPF ops - * @where: extra information reported on error - * - * @cpu is a cpu number which came from the BPF scheduler and can be any value. - * Verify that it is in range and one of the possible cpus. If invalid, trigger - * an ops error. - */ -bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where) -{ - if (__cpu_valid(cpu)) { - return true; - } else { - scx_error(sch, "invalid CPU %d%s%s", cpu, where ? " " : "", where ?: ""); - return false; - } -} - -/** - * ops_sanitize_err - Sanitize a -errno value - * @sch: scx_sched to error out on error - * @ops_name: operation to blame on failure - * @err: -errno value to sanitize - * - * Verify @err is a valid -errno. If not, trigger scx_error() and return - * -%EPROTO. This is necessary because returning a rogue -errno up the chain can - * cause misbehaviors. For an example, a large negative return from - * ops.init_task() triggers an oops when passed up the call chain because the - * value fails IS_ERR() test after being encoded with ERR_PTR() and then is - * handled as a pointer. - */ -static int ops_sanitize_err(struct scx_sched *sch, const char *ops_name, s32 err) -{ - if (err < 0 && err >= -MAX_ERRNO) - return err; - - scx_error(sch, "ops.%s() returned an invalid errno %d", ops_name, err); - return -EPROTO; -} - -static void deferred_bal_cb_workfn(struct rq *rq) -{ - run_deferred(rq); -} - -static void deferred_irq_workfn(struct irq_work *irq_work) -{ - struct rq *rq = container_of(irq_work, struct rq, scx.deferred_irq_work); - - raw_spin_rq_lock(rq); - run_deferred(rq); - raw_spin_rq_unlock(rq); -} - -/** - * schedule_deferred - Schedule execution of deferred actions on an rq - * @rq: target rq - * - * Schedule execution of deferred actions on @rq. Deferred actions are executed - * with @rq locked but unpinned, and thus can unlock @rq to e.g. migrate tasks - * to other rqs. - */ -static void schedule_deferred(struct rq *rq) -{ - /* - * This is the fallback when schedule_deferred_locked() can't use - * the cheaper balance callback or wakeup hook paths (the target - * CPU is not in balance or wakeup). Currently, this is primarily - * hit by reenqueue operations targeting a remote CPU. - * - * Queue on the target CPU. The deferred work can run from any CPU - * correctly - the _locked() path already processes remote rqs from - * the calling CPU - but targeting the owning CPU allows IPI delivery - * without waiting for the calling CPU to re-enable IRQs and is - * cheaper as the reenqueue runs locally. - */ - irq_work_queue_on(&rq->scx.deferred_irq_work, cpu_of(rq)); -} - -/** - * schedule_deferred_locked - Schedule execution of deferred actions on an rq - * @rq: target rq - * - * Schedule execution of deferred actions on @rq. Equivalent to - * schedule_deferred() but requires @rq to be locked and can be more efficient. - */ -static void schedule_deferred_locked(struct rq *rq) -{ - lockdep_assert_rq_held(rq); - - /* - * If in the middle of waking up a task, task_woken_scx() will be called - * afterwards which will then run the deferred actions, no need to - * schedule anything. - */ - if (rq->scx.flags & SCX_RQ_IN_WAKEUP) - return; - - /* Don't do anything if there already is a deferred operation. */ - if (rq->scx.flags & SCX_RQ_BAL_CB_PENDING) - return; - - /* - * If in balance, the balance callbacks will be called before rq lock is - * released. Schedule one. - * - * - * We can't directly insert the callback into the - * rq's list: The call can drop its lock and make the pending balance - * callback visible to unrelated code paths that call rq_pin_lock(). - * - * Just let balance_one() know that it must do it itself. - */ - if (rq->scx.flags & SCX_RQ_IN_BALANCE) { - rq->scx.flags |= SCX_RQ_BAL_CB_PENDING; - return; - } - - /* - * No scheduler hooks available. Use the generic irq_work path. The - * above WAKEUP and BALANCE paths should cover most of the cases and the - * time to IRQ re-enable shouldn't be long. - */ - schedule_deferred(rq); -} - -static void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, - u64 reenq_flags, struct rq *locked_rq) -{ - struct rq *rq; - - /* - * Allowing reenqueues doesn't make sense while bypassing. This also - * blocks from new reenqueues to be scheduled on dead scheds. - */ - if (unlikely(READ_ONCE(sch->bypass_depth))) - return; - - if (dsq->id == SCX_DSQ_LOCAL) { - rq = container_of(dsq, struct rq, scx.local_dsq); - - struct scx_sched_pcpu *sch_pcpu = per_cpu_ptr(sch->pcpu, cpu_of(rq)); - struct scx_deferred_reenq_local *drl = &sch_pcpu->deferred_reenq_local; - - /* - * Pairs with smp_mb() in process_deferred_reenq_locals() and - * guarantees that there is a reenq_local() afterwards. - */ - smp_mb(); - - if (list_empty(&drl->node) || - (READ_ONCE(drl->flags) & reenq_flags) != reenq_flags) { - - guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); - - if (list_empty(&drl->node)) - list_move_tail(&drl->node, &rq->scx.deferred_reenq_locals); - WRITE_ONCE(drl->flags, drl->flags | reenq_flags); - } - } else if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN)) { - rq = this_rq(); - - struct scx_dsq_pcpu *dsq_pcpu = per_cpu_ptr(dsq->pcpu, cpu_of(rq)); - struct scx_deferred_reenq_user *dru = &dsq_pcpu->deferred_reenq_user; - - /* - * Pairs with smp_mb() in process_deferred_reenq_users() and - * guarantees that there is a reenq_user() afterwards. - */ - smp_mb(); - - if (list_empty(&dru->node) || - (READ_ONCE(dru->flags) & reenq_flags) != reenq_flags) { - - guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); - - if (list_empty(&dru->node)) - list_move_tail(&dru->node, &rq->scx.deferred_reenq_users); - WRITE_ONCE(dru->flags, dru->flags | reenq_flags); - } - } else { - scx_error(sch, "DSQ 0x%llx not allowed for reenq", dsq->id); - return; - } - - if (rq == locked_rq) - schedule_deferred_locked(rq); - else - schedule_deferred(rq); -} - -static void schedule_reenq_local(struct rq *rq, u64 reenq_flags) -{ - struct scx_sched *root = rcu_dereference_sched(scx_root); - - if (WARN_ON_ONCE(!root)) - return; - - schedule_dsq_reenq(root, &rq->scx.local_dsq, reenq_flags, rq); -} - -/** - * touch_core_sched - Update timestamp used for core-sched task ordering - * @rq: rq to read clock from, must be locked - * @p: task to update the timestamp for - * - * Update @p->scx.core_sched_at timestamp. This is used by scx_prio_less() to - * implement global or local-DSQ FIFO ordering for core-sched. Should be called - * when a task becomes runnable and its turn on the CPU ends (e.g. slice - * exhaustion). - */ -static void touch_core_sched(struct rq *rq, struct task_struct *p) -{ - lockdep_assert_rq_held(rq); - -#ifdef CONFIG_SCHED_CORE - /* - * It's okay to update the timestamp spuriously. Use - * sched_core_disabled() which is cheaper than enabled(). - * - * As this is used to determine ordering between tasks of sibling CPUs, - * it may be better to use per-core dispatch sequence instead. - */ - if (!sched_core_disabled()) - p->scx.core_sched_at = sched_clock_cpu(cpu_of(rq)); -#endif -} - -/** - * touch_core_sched_dispatch - Update core-sched timestamp on dispatch - * @rq: rq to read clock from, must be locked - * @p: task being dispatched - * - * If the BPF scheduler implements custom core-sched ordering via - * ops.core_sched_before(), @p->scx.core_sched_at is used to implement FIFO - * ordering within each local DSQ. This function is called from dispatch paths - * and updates @p->scx.core_sched_at if custom core-sched ordering is in effect. - */ -static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) -{ - lockdep_assert_rq_held(rq); - -#ifdef CONFIG_SCHED_CORE - if (unlikely(SCX_HAS_OP(scx_root, core_sched_before))) - touch_core_sched(rq, p); -#endif -} - -static void update_curr_scx(struct rq *rq) -{ - struct task_struct *curr = rq->curr; - s64 delta_exec; - - delta_exec = update_curr_common(rq); - if (unlikely(delta_exec <= 0)) - return; - - if (curr->scx.slice != SCX_SLICE_INF) { - curr->scx.slice -= min_t(u64, curr->scx.slice, delta_exec); - if (!curr->scx.slice) - touch_core_sched(rq, curr); - } - - dl_server_update(&rq->ext_server, delta_exec); -} - -static bool scx_dsq_priq_less(struct rb_node *node_a, - const struct rb_node *node_b) -{ - const struct task_struct *a = - container_of(node_a, struct task_struct, scx.dsq_priq); - const struct task_struct *b = - container_of(node_b, struct task_struct, scx.dsq_priq); - - return time_before64(a->scx.dsq_vtime, b->scx.dsq_vtime); -} - -static void dsq_inc_nr(struct scx_dispatch_q *dsq, struct task_struct *p, u64 enq_flags) -{ - /* scx_bpf_dsq_nr_queued() reads ->nr without locking, use WRITE_ONCE() */ - WRITE_ONCE(dsq->nr, dsq->nr + 1); - - /* - * Once @p reaches a local DSQ, it can only leave it by being dispatched - * to the CPU or dequeued. In both cases, the only way @p can go back to - * the BPF sched is through enqueueing. If being inserted into a local - * DSQ with IMMED, persist the state until the next enqueueing event in - * do_enqueue_task() so that we can maintain IMMED protection through - * e.g. SAVE/RESTORE cycles and slice extensions. - */ - if (enq_flags & SCX_ENQ_IMMED) { - if (unlikely(dsq->id != SCX_DSQ_LOCAL)) { - WARN_ON_ONCE(!(enq_flags & SCX_ENQ_GDSQ_FALLBACK)); - return; - } - p->scx.flags |= SCX_TASK_IMMED; - } - - if (p->scx.flags & SCX_TASK_IMMED) { - struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); - - if (WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL)) - return; - - rq->scx.nr_immed++; - - /* - * If @rq already had other tasks or the current task is not - * done yet, @p can't go on the CPU immediately. Re-enqueue. - */ - if (unlikely(dsq->nr > 1 || !rq_is_open(rq, enq_flags))) - schedule_reenq_local(rq, 0); - } -} - -static void dsq_dec_nr(struct scx_dispatch_q *dsq, struct task_struct *p) -{ - /* see dsq_inc_nr() */ - WRITE_ONCE(dsq->nr, dsq->nr - 1); - - if (p->scx.flags & SCX_TASK_IMMED) { - struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); - - if (WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL) || - WARN_ON_ONCE(rq->scx.nr_immed <= 0)) - return; - - rq->scx.nr_immed--; - } -} - -static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p) -{ - p->scx.slice = READ_ONCE(sch->slice_dfl); - __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1); -} - -/* - * Return true if @p is moving due to an internal SCX migration, false - * otherwise. - */ -static inline bool task_scx_migrating(struct task_struct *p) -{ - /* - * We only need to check sticky_cpu: it is set to the destination - * CPU in move_remote_task_to_local_dsq() before deactivate_task() - * and cleared when the task is enqueued on the destination, so it - * is only non-negative during an internal SCX migration. - */ - return p->scx.sticky_cpu >= 0; -} - -/* - * Call ops.dequeue() if the task is in BPF custody and not migrating. - * Clears %SCX_TASK_IN_CUSTODY when the callback is invoked. - */ -static void call_task_dequeue(struct scx_sched *sch, struct rq *rq, - struct task_struct *p, u64 deq_flags) -{ - if (!(p->scx.flags & SCX_TASK_IN_CUSTODY) || task_scx_migrating(p)) - return; - - if (SCX_HAS_OP(sch, dequeue)) - SCX_CALL_OP_TASK(sch, dequeue, rq, p, deq_flags); - - p->scx.flags &= ~SCX_TASK_IN_CUSTODY; -} - -static void local_dsq_post_enq(struct scx_sched *sch, struct scx_dispatch_q *dsq, - struct task_struct *p, u64 enq_flags) -{ - struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); - - call_task_dequeue(sch, rq, p, 0); - - /* - * Note that @rq's lock may be dropped between this enqueue and @p - * actually getting on CPU. This gives higher-class tasks (e.g. RT) - * an opportunity to wake up on @rq and prevent @p from running. - * Here are some concrete examples: - * - * Example 1: - * - * We dispatch two tasks from a single ops.dispatch(): - * - First, a local task to this CPU's local DSQ; - * - Second, a local/remote task to a remote CPU's local DSQ. - * We must drop the local rq lock in order to finish the second - * dispatch. In that time, an RT task can wake up on the local rq. - * - * Example 2: - * - * We dispatch a local/remote task to a remote CPU's local DSQ. - * We must drop the remote rq lock before the dispatched task can run, - * which gives an RT task an opportunity to wake up on the remote rq. - * - * Both examples work the same if we replace dispatching with moving - * the tasks from a user-created DSQ. - * - * We must detect these wakeups so that we can re-enqueue IMMED tasks - * from @rq's local DSQ. scx_wakeup_preempt() serves exactly this - * purpose, but for it to be invoked, we must ensure that we bump - * @rq->next_class to &ext_sched_class if it's currently idle. - * - * wakeup_preempt() does the bumping, and since we only invoke it if - * @rq->next_class is below &ext_sched_class, it will also - * resched_curr(rq). - */ - if (sched_class_above(p->sched_class, rq->next_class)) - wakeup_preempt(rq, p, 0); - - /* - * If @rq is in balance, the CPU is already vacant and looking for the - * next task to run. No need to preempt or trigger resched after moving - * @p into its local DSQ. - * Note that the wakeup_preempt() above may have already triggered - * a resched if @rq->next_class was idle. It's harmless, since - * need_resched is cleared immediately after task pick. - */ - if (rq->scx.flags & SCX_RQ_IN_BALANCE) - return; - - if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && - rq->curr->sched_class == &ext_sched_class) { - rq->curr->scx.slice = 0; - resched_curr(rq); - } -} - -static void dispatch_enqueue(struct scx_sched *sch, struct rq *rq, - struct scx_dispatch_q *dsq, struct task_struct *p, - u64 enq_flags) -{ - bool is_local = dsq->id == SCX_DSQ_LOCAL; - - WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); - WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || - !RB_EMPTY_NODE(&p->scx.dsq_priq)); - - if (!is_local) { - raw_spin_lock_nested(&dsq->lock, - (enq_flags & SCX_ENQ_NESTED) ? SINGLE_DEPTH_NESTING : 0); - - if (unlikely(dsq->id == SCX_DSQ_INVALID)) { - scx_error(sch, "attempting to dispatch to a destroyed dsq"); - /* fall back to the global dsq */ - raw_spin_unlock(&dsq->lock); - dsq = find_global_dsq(sch, task_cpu(p)); - raw_spin_lock(&dsq->lock); - } - } - - if (unlikely((dsq->id & SCX_DSQ_FLAG_BUILTIN) && - (enq_flags & SCX_ENQ_DSQ_PRIQ))) { - /* - * SCX_DSQ_LOCAL and SCX_DSQ_GLOBAL DSQs always consume from - * their FIFO queues. To avoid confusion and accidentally - * starving vtime-dispatched tasks by FIFO-dispatched tasks, we - * disallow any internal DSQ from doing vtime ordering of - * tasks. - */ - scx_error(sch, "cannot use vtime ordering for built-in DSQs"); - enq_flags &= ~SCX_ENQ_DSQ_PRIQ; - } - - if (enq_flags & SCX_ENQ_DSQ_PRIQ) { - struct rb_node *rbp; - - /* - * A PRIQ DSQ shouldn't be using FIFO enqueueing. As tasks are - * linked to both the rbtree and list on PRIQs, this can only be - * tested easily when adding the first task. - */ - if (unlikely(RB_EMPTY_ROOT(&dsq->priq) && - nldsq_next_task(dsq, NULL, false))) - scx_error(sch, "DSQ ID 0x%016llx already had FIFO-enqueued tasks", - dsq->id); - - p->scx.dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; - rb_add(&p->scx.dsq_priq, &dsq->priq, scx_dsq_priq_less); - - /* - * Find the previous task and insert after it on the list so - * that @dsq->list is vtime ordered. - */ - rbp = rb_prev(&p->scx.dsq_priq); - if (rbp) { - struct task_struct *prev = - container_of(rbp, struct task_struct, - scx.dsq_priq); - list_add(&p->scx.dsq_list.node, &prev->scx.dsq_list.node); - /* first task unchanged - no update needed */ - } else { - list_add(&p->scx.dsq_list.node, &dsq->list); - /* not builtin and new task is at head - use fastpath */ - rcu_assign_pointer(dsq->first_task, p); - } - } else { - /* a FIFO DSQ shouldn't be using PRIQ enqueuing */ - if (unlikely(!RB_EMPTY_ROOT(&dsq->priq))) - scx_error(sch, "DSQ ID 0x%016llx already had PRIQ-enqueued tasks", - dsq->id); - - if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) { - list_add(&p->scx.dsq_list.node, &dsq->list); - /* new task inserted at head - use fastpath */ - if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN)) - rcu_assign_pointer(dsq->first_task, p); - } else { - /* - * dsq->list can contain parked BPF iterator cursors, so - * list_empty() here isn't a reliable proxy for "no real - * task in the DSQ". Test dsq->first_task directly. - */ - list_add_tail(&p->scx.dsq_list.node, &dsq->list); - if (!dsq->first_task && !(dsq->id & SCX_DSQ_FLAG_BUILTIN)) - rcu_assign_pointer(dsq->first_task, p); - } - } - - /* seq records the order tasks are queued, used by BPF DSQ iterator */ - WRITE_ONCE(dsq->seq, dsq->seq + 1); - p->scx.dsq_seq = dsq->seq; - - dsq_inc_nr(dsq, p, enq_flags); - p->scx.dsq = dsq; - - /* - * Update custody and call ops.dequeue() before clearing ops_state: - * once ops_state is cleared, waiters in ops_dequeue() can proceed - * and dequeue_task_scx() will RMW p->scx.flags. If we clear - * ops_state first, both sides would modify p->scx.flags - * concurrently in a non-atomic way. - */ - if (is_local) { - local_dsq_post_enq(sch, dsq, p, enq_flags); - } else { - /* - * Task on global/bypass DSQ: leave custody, task on - * non-terminal DSQ: enter custody. - */ - if (dsq->id == SCX_DSQ_GLOBAL || dsq->id == SCX_DSQ_BYPASS) - call_task_dequeue(sch, rq, p, 0); - else - p->scx.flags |= SCX_TASK_IN_CUSTODY; - - raw_spin_unlock(&dsq->lock); - } - - /* - * We're transitioning out of QUEUEING or DISPATCHING. store_release to - * match waiters' load_acquire. - */ - if (enq_flags & SCX_ENQ_CLEAR_OPSS) - atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); -} - -static void task_unlink_from_dsq(struct task_struct *p, - struct scx_dispatch_q *dsq) -{ - WARN_ON_ONCE(list_empty(&p->scx.dsq_list.node)); - - if (p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { - rb_erase(&p->scx.dsq_priq, &dsq->priq); - RB_CLEAR_NODE(&p->scx.dsq_priq); - p->scx.dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; - } - - list_del_init(&p->scx.dsq_list.node); - dsq_dec_nr(dsq, p); - - if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN) && dsq->first_task == p) { - struct task_struct *first_task; - - first_task = nldsq_next_task(dsq, NULL, false); - rcu_assign_pointer(dsq->first_task, first_task); - } -} - -static void dispatch_dequeue(struct rq *rq, struct task_struct *p) -{ - struct scx_dispatch_q *dsq = p->scx.dsq; - bool is_local = dsq == &rq->scx.local_dsq; - - lockdep_assert_rq_held(rq); - - if (!dsq) { - /* - * If !dsq && on-list, @p is on @rq's ddsp_deferred_locals. - * Unlinking is all that's needed to cancel. - */ - if (unlikely(!list_empty(&p->scx.dsq_list.node))) - list_del_init(&p->scx.dsq_list.node); - - /* - * When dispatching directly from the BPF scheduler to a local - * DSQ, the task isn't associated with any DSQ but - * @p->scx.holding_cpu may be set under the protection of - * %SCX_OPSS_DISPATCHING. - */ - if (p->scx.holding_cpu >= 0) - p->scx.holding_cpu = -1; - - return; - } - - if (!is_local) - raw_spin_lock(&dsq->lock); - - /* - * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx.dsq_* can't - * change underneath us. - */ - if (p->scx.holding_cpu < 0) { - /* @p must still be on @dsq, dequeue */ - task_unlink_from_dsq(p, dsq); - } else { - /* - * We're racing against dispatch_to_local_dsq() which already - * removed @p from @dsq and set @p->scx.holding_cpu. Clear the - * holding_cpu which tells dispatch_to_local_dsq() that it lost - * the race. - */ - WARN_ON_ONCE(!list_empty(&p->scx.dsq_list.node)); - p->scx.holding_cpu = -1; - } - p->scx.dsq = NULL; - - if (!is_local) - raw_spin_unlock(&dsq->lock); -} - -/* - * Abbreviated version of dispatch_dequeue() that can be used when both @p's rq - * and dsq are locked. - */ -static void dispatch_dequeue_locked(struct task_struct *p, - struct scx_dispatch_q *dsq) -{ - lockdep_assert_rq_held(task_rq(p)); - lockdep_assert_held(&dsq->lock); - - task_unlink_from_dsq(p, dsq); - p->scx.dsq = NULL; -} - -static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch, - struct rq *rq, u64 dsq_id, - s32 tcpu) -{ - struct scx_dispatch_q *dsq; - - if (dsq_id == SCX_DSQ_LOCAL) - return &rq->scx.local_dsq; - - if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { - s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK); - - if (!scx_cpu_valid(sch, cpu, "in SCX_DSQ_LOCAL_ON dispatch verdict")) - return find_global_dsq(sch, tcpu); - - return &cpu_rq(cpu)->scx.local_dsq; - } - - if (dsq_id == SCX_DSQ_GLOBAL) - dsq = find_global_dsq(sch, tcpu); - else - dsq = find_user_dsq(sch, dsq_id); - - if (unlikely(!dsq)) { - scx_error(sch, "non-existent DSQ 0x%llx", dsq_id); - return find_global_dsq(sch, tcpu); - } - - return dsq; -} - -static void mark_direct_dispatch(struct scx_sched *sch, - struct task_struct *ddsp_task, - struct task_struct *p, u64 dsq_id, - u64 enq_flags) -{ - /* - * Mark that dispatch already happened from ops.select_cpu() or - * ops.enqueue() by spoiling direct_dispatch_task with a non-NULL value - * which can never match a valid task pointer. - */ - __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH)); - - /* @p must match the task on the enqueue path */ - if (unlikely(p != ddsp_task)) { - if (IS_ERR(ddsp_task)) - scx_error(sch, "%s[%d] already direct-dispatched", - p->comm, p->pid); - else - scx_error(sch, "scheduling for %s[%d] but trying to direct-dispatch %s[%d]", - ddsp_task->comm, ddsp_task->pid, - p->comm, p->pid); - return; - } - - WARN_ON_ONCE(p->scx.ddsp_dsq_id != SCX_DSQ_INVALID); - WARN_ON_ONCE(p->scx.ddsp_enq_flags); - - p->scx.ddsp_dsq_id = dsq_id; - p->scx.ddsp_enq_flags = enq_flags; -} - -/* - * Clear @p direct dispatch state when leaving the scheduler. - * - * Direct dispatch state must be cleared in the following cases: - * - direct_dispatch(): cleared on the synchronous enqueue path, deferred - * dispatch keeps the state until consumed - * - process_ddsp_deferred_locals(): cleared after consuming deferred state, - * - do_enqueue_task(): cleared on enqueue fallbacks where the dispatch - * verdict is ignored (local/global/bypass) - * - dequeue_task_scx(): cleared after dispatch_dequeue(), covering deferred - * cancellation and holding_cpu races - * - scx_disable_task(): cleared for queued wakeup tasks, which are excluded by - * the scx_bypass() loop, so that stale state is not reused by a subsequent - * scheduler instance - */ -static inline void clear_direct_dispatch(struct task_struct *p) -{ - p->scx.ddsp_dsq_id = SCX_DSQ_INVALID; - p->scx.ddsp_enq_flags = 0; -} - -static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, - u64 enq_flags) -{ - struct rq *rq = task_rq(p); - struct scx_dispatch_q *dsq = - find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, task_cpu(p)); - u64 ddsp_enq_flags; - - touch_core_sched_dispatch(rq, p); - - p->scx.ddsp_enq_flags |= enq_flags; - - /* - * We are in the enqueue path with @rq locked and pinned, and thus can't - * double lock a remote rq and enqueue to its local DSQ. For - * DSQ_LOCAL_ON verdicts targeting the local DSQ of a remote CPU, defer - * the enqueue so that it's executed when @rq can be unlocked. - */ - if (dsq->id == SCX_DSQ_LOCAL && dsq != &rq->scx.local_dsq) { - unsigned long opss; - - opss = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_STATE_MASK; - - switch (opss & SCX_OPSS_STATE_MASK) { - case SCX_OPSS_NONE: - break; - case SCX_OPSS_QUEUEING: - /* - * As @p was never passed to the BPF side, _release is - * not strictly necessary. Still do it for consistency. - */ - atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); - break; - default: - WARN_ONCE(true, "sched_ext: %s[%d] has invalid ops state 0x%lx in direct_dispatch()", - p->comm, p->pid, opss); - atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); - break; - } - - WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); - list_add_tail(&p->scx.dsq_list.node, - &rq->scx.ddsp_deferred_locals); - schedule_deferred_locked(rq); - return; - } - - ddsp_enq_flags = p->scx.ddsp_enq_flags; - clear_direct_dispatch(p); - - dispatch_enqueue(sch, rq, dsq, p, ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS); -} - -static bool scx_rq_online(struct rq *rq) -{ - /* - * Test both cpu_active() and %SCX_RQ_ONLINE. %SCX_RQ_ONLINE indicates - * the online state as seen from the BPF scheduler. cpu_active() test - * guarantees that, if this function returns %true, %SCX_RQ_ONLINE will - * stay set until the current scheduling operation is complete even if - * we aren't locking @rq. - */ - return likely((rq->scx.flags & SCX_RQ_ONLINE) && cpu_active(cpu_of(rq))); -} - -static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, - int sticky_cpu) -{ - struct scx_sched *sch = scx_task_sched(p); - struct task_struct **ddsp_taskp; - struct scx_dispatch_q *dsq; - unsigned long qseq; - - WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_QUEUED)); - - /* internal movements - rq migration / RESTORE */ - if (sticky_cpu == cpu_of(rq)) - goto local_norefill; - - /* - * Clear persistent TASK_IMMED for fresh enqueues, see dsq_inc_nr(). - * Note that exiting and migration-disabled tasks that skip - * ops.enqueue() below will lose IMMED protection unless - * %SCX_OPS_ENQ_EXITING / %SCX_OPS_ENQ_MIGRATION_DISABLED are set. - */ - p->scx.flags &= ~SCX_TASK_IMMED; - - /* - * If !scx_rq_online(), we already told the BPF scheduler that the CPU - * is offline and are just running the hotplug path. Don't bother the - * BPF scheduler. - */ - if (!scx_rq_online(rq)) - goto local; - - if (scx_bypassing(sch, cpu_of(rq))) { - __scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1); - goto bypass; - } - - if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) - goto direct; - - /* see %SCX_OPS_ENQ_EXITING */ - if (!(sch->ops.flags & SCX_OPS_ENQ_EXITING) && - unlikely(p->flags & PF_EXITING)) { - __scx_add_event(sch, SCX_EV_ENQ_SKIP_EXITING, 1); - goto local; - } - - /* see %SCX_OPS_ENQ_MIGRATION_DISABLED */ - if (!(sch->ops.flags & SCX_OPS_ENQ_MIGRATION_DISABLED) && - is_migration_disabled(p)) { - __scx_add_event(sch, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED, 1); - goto local; - } - - if (unlikely(!SCX_HAS_OP(sch, enqueue))) - goto global; - - /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ - qseq = rq->scx.ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; - - WARN_ON_ONCE(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE); - atomic_long_set(&p->scx.ops_state, SCX_OPSS_QUEUEING | qseq); - - ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); - WARN_ON_ONCE(*ddsp_taskp); - *ddsp_taskp = p; - - SCX_CALL_OP_TASK(sch, enqueue, rq, p, enq_flags); - - *ddsp_taskp = NULL; - if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) - goto direct; - - /* - * Task is now in BPF scheduler's custody. Set %SCX_TASK_IN_CUSTODY - * so ops.dequeue() is called when it leaves custody. - */ - p->scx.flags |= SCX_TASK_IN_CUSTODY; - - /* - * If not directly dispatched, QUEUEING isn't clear yet and dispatch or - * dequeue may be waiting. The store_release matches their load_acquire. - */ - atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_QUEUED | qseq); - return; - -direct: - direct_dispatch(sch, p, enq_flags); - return; -local_norefill: - dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, enq_flags); - return; -local: - dsq = &rq->scx.local_dsq; - goto enqueue; -global: - dsq = find_global_dsq(sch, task_cpu(p)); - goto enqueue; -bypass: - dsq = bypass_enq_target_dsq(sch, task_cpu(p)); - goto enqueue; - -enqueue: - /* - * For task-ordering, slice refill must be treated as implying the end - * of the current slice. Otherwise, the longer @p stays on the CPU, the - * higher priority it becomes from scx_prio_less()'s POV. - */ - touch_core_sched(rq, p); - refill_task_slice_dfl(sch, p); - clear_direct_dispatch(p); - dispatch_enqueue(sch, rq, dsq, p, enq_flags); -} - -static bool task_runnable(const struct task_struct *p) -{ - return !list_empty(&p->scx.runnable_node); -} - -static void set_task_runnable(struct rq *rq, struct task_struct *p) -{ - lockdep_assert_rq_held(rq); - - if (p->scx.flags & SCX_TASK_RESET_RUNNABLE_AT) { - p->scx.runnable_at = jiffies; - p->scx.flags &= ~SCX_TASK_RESET_RUNNABLE_AT; - } - - /* - * list_add_tail() must be used. scx_bypass() depends on tasks being - * appended to the runnable_list. - */ - list_add_tail(&p->scx.runnable_node, &rq->scx.runnable_list); -} - -static void clr_task_runnable(struct task_struct *p, bool reset_runnable_at) -{ - list_del_init(&p->scx.runnable_node); - if (reset_runnable_at) - p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; -} - -static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_flags) -{ - struct scx_sched *sch = scx_task_sched(p); - int sticky_cpu = p->scx.sticky_cpu; - u64 enq_flags = core_enq_flags | rq->scx.extra_enq_flags; - - if (enq_flags & ENQUEUE_WAKEUP) - rq->scx.flags |= SCX_RQ_IN_WAKEUP; - - /* - * Restoring a running task will be immediately followed by - * set_next_task_scx() which expects the task to not be on the BPF - * scheduler as tasks can only start running through local DSQs. Force - * direct-dispatch into the local DSQ by setting the sticky_cpu. - */ - if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) - sticky_cpu = cpu_of(rq); - - if (p->scx.flags & SCX_TASK_QUEUED) { - WARN_ON_ONCE(!task_runnable(p)); - goto out; - } - - set_task_runnable(rq, p); - p->scx.flags |= SCX_TASK_QUEUED; - rq->scx.nr_running++; - add_nr_running(rq, 1); - - if (SCX_HAS_OP(sch, runnable) && !task_on_rq_migrating(p)) - SCX_CALL_OP_TASK(sch, runnable, rq, p, enq_flags); - - if (enq_flags & SCX_ENQ_WAKEUP) - touch_core_sched(rq, p); - - /* Start dl_server if this is the first task being enqueued */ - if (rq->scx.nr_running == 1) - dl_server_start(&rq->ext_server); - - do_enqueue_task(rq, p, enq_flags, sticky_cpu); - - if (sticky_cpu >= 0) - p->scx.sticky_cpu = -1; -out: - rq->scx.flags &= ~SCX_RQ_IN_WAKEUP; - - if ((enq_flags & SCX_ENQ_CPU_SELECTED) && - unlikely(cpu_of(rq) != p->scx.selected_cpu)) - __scx_add_event(sch, SCX_EV_SELECT_CPU_FALLBACK, 1); -} - -static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags) -{ - struct scx_sched *sch = scx_task_sched(p); - unsigned long opss; - - /* dequeue is always temporary, don't reset runnable_at */ - clr_task_runnable(p, false); - -retry: - /* acquire ensures that we see the preceding updates on QUEUED */ - opss = atomic_long_read_acquire(&p->scx.ops_state); - - switch (opss & SCX_OPSS_STATE_MASK) { - case SCX_OPSS_NONE: - break; - case SCX_OPSS_QUEUEING: - /* - * QUEUEING is started and finished while holding @p's rq lock. - * As we're holding the rq lock now, we shouldn't see QUEUEING. - */ - BUG(); - case SCX_OPSS_QUEUED: - /* - * A queued task must always be in BPF scheduler's custody. If - * SCX_TASK_IN_CUSTODY is clear, finish_dispatch() on another - * CPU has already passed call_task_dequeue() (which clears the - * flag), but has not yet written SCX_OPSS_NONE. That final - * store does not require this rq's lock, so retrying with - * cpu_relax() is bounded: we will observe NONE (or DISPATCHING, - * handled by the fallthrough) on a subsequent iteration. - */ - if (unlikely(!(READ_ONCE(p->scx.flags) & SCX_TASK_IN_CUSTODY))) { - cpu_relax(); - goto retry; - } - - if (atomic_long_try_cmpxchg(&p->scx.ops_state, &opss, - SCX_OPSS_NONE)) - break; - fallthrough; - case SCX_OPSS_DISPATCHING: - /* - * If @p is being dispatched from the BPF scheduler to a DSQ, - * wait for the transfer to complete so that @p doesn't get - * added to its DSQ after dequeueing is complete. - * - * As we're waiting on DISPATCHING with the rq locked, the - * dispatching side shouldn't try to lock the rq while - * DISPATCHING is set. See dispatch_to_local_dsq(). - * - * DISPATCHING shouldn't have qseq set and control can reach - * here with NONE @opss from the above QUEUED case block. - * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss. - */ - wait_ops_state(p, SCX_OPSS_DISPATCHING); - BUG_ON(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE); - break; - } - - /* - * Call ops.dequeue() if the task is still in BPF custody. - * - * The code that clears ops_state to %SCX_OPSS_NONE does not always - * clear %SCX_TASK_IN_CUSTODY: in dispatch_to_local_dsq(), when - * we're moving a task that was in %SCX_OPSS_DISPATCHING to a - * remote CPU's local DSQ, we only set ops_state to %SCX_OPSS_NONE - * so that a concurrent dequeue can proceed, but we clear - * %SCX_TASK_IN_CUSTODY only when we later enqueue or move the - * task. So we can see NONE + IN_CUSTODY here and we must handle - * it. Similarly, after waiting on %SCX_OPSS_DISPATCHING we see - * NONE but the task may still have %SCX_TASK_IN_CUSTODY set until - * it is enqueued on the destination. - */ - call_task_dequeue(sch, rq, p, deq_flags); -} - -static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_flags) -{ - struct scx_sched *sch = scx_task_sched(p); - u64 deq_flags = core_deq_flags; - - /* - * Set %SCX_DEQ_SCHED_CHANGE when the dequeue is due to a property - * change (not sleep or core-sched pick). - */ - if (!(deq_flags & (DEQUEUE_SLEEP | SCX_DEQ_CORE_SCHED_EXEC))) - deq_flags |= SCX_DEQ_SCHED_CHANGE; - - if (!(p->scx.flags & SCX_TASK_QUEUED)) { - WARN_ON_ONCE(task_runnable(p)); - return true; - } - - ops_dequeue(rq, p, deq_flags); - - /* - * A currently running task which is going off @rq first gets dequeued - * and then stops running. As we want running <-> stopping transitions - * to be contained within runnable <-> quiescent transitions, trigger - * ->stopping() early here instead of in put_prev_task_scx(). - * - * @p may go through multiple stopping <-> running transitions between - * here and put_prev_task_scx() if task attribute changes occur while - * balance_one() leaves @rq unlocked. However, they don't contain any - * information meaningful to the BPF scheduler and can be suppressed by - * skipping the callbacks if the task is !QUEUED. - */ - if (SCX_HAS_OP(sch, stopping) && task_current(rq, p)) { - update_curr_scx(rq); - SCX_CALL_OP_TASK(sch, stopping, rq, p, false); - } - - if (SCX_HAS_OP(sch, quiescent) && !task_on_rq_migrating(p)) - SCX_CALL_OP_TASK(sch, quiescent, rq, p, deq_flags); - - if (deq_flags & SCX_DEQ_SLEEP) - p->scx.flags |= SCX_TASK_DEQD_FOR_SLEEP; - else - p->scx.flags &= ~SCX_TASK_DEQD_FOR_SLEEP; - - p->scx.flags &= ~SCX_TASK_QUEUED; - rq->scx.nr_running--; - sub_nr_running(rq, 1); - - dispatch_dequeue(rq, p); - clear_direct_dispatch(p); - return true; -} - -static void yield_task_scx(struct rq *rq) -{ - struct task_struct *p = rq->donor; - struct scx_sched *sch = scx_task_sched(p); - - if (SCX_HAS_OP(sch, yield)) - SCX_CALL_OP_2TASKS_RET(sch, yield, rq, p, NULL); - else - p->scx.slice = 0; -} - -static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) -{ - struct task_struct *from = rq->donor; - struct scx_sched *sch = scx_task_sched(from); - - if (SCX_HAS_OP(sch, yield) && sch == scx_task_sched(to)) - return SCX_CALL_OP_2TASKS_RET(sch, yield, rq, from, to); - else - return false; -} - -static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_flags) -{ - /* - * Preemption between SCX tasks is implemented by resetting the victim - * task's slice to 0 and triggering reschedule on the target CPU. - * Nothing to do. - */ - if (p->sched_class == &ext_sched_class) - return; - - /* - * Getting preempted by a higher-priority class. Reenqueue IMMED tasks. - * This captures all preemption cases including: - * - * - A SCX task is currently running. - * - * - @rq is waking from idle due to a SCX task waking to it. - * - * - A higher-priority wakes up while SCX dispatch is in progress. - */ - if (rq->scx.nr_immed) - schedule_reenq_local(rq, 0); -} - -static void move_local_task_to_local_dsq(struct scx_sched *sch, - struct task_struct *p, u64 enq_flags, - struct scx_dispatch_q *src_dsq, - struct rq *dst_rq) -{ - struct scx_dispatch_q *dst_dsq = &dst_rq->scx.local_dsq; - - /* @dsq is locked and @p is on @dst_rq */ - lockdep_assert_held(&src_dsq->lock); - lockdep_assert_rq_held(dst_rq); - - WARN_ON_ONCE(p->scx.holding_cpu >= 0); - - if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) - list_add(&p->scx.dsq_list.node, &dst_dsq->list); - else - list_add_tail(&p->scx.dsq_list.node, &dst_dsq->list); - - dsq_inc_nr(dst_dsq, p, enq_flags); - p->scx.dsq = dst_dsq; - - local_dsq_post_enq(sch, dst_dsq, p, enq_flags); -} - -/** - * move_remote_task_to_local_dsq - Move a task from a foreign rq to a local DSQ - * @p: task to move - * @enq_flags: %SCX_ENQ_* - * @src_rq: rq to move the task from, locked on entry, released on return - * @dst_rq: rq to move the task into, locked on return - * - * Move @p which is currently on @src_rq to @dst_rq's local DSQ. - */ -static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, - struct rq *src_rq, struct rq *dst_rq) -{ - lockdep_assert_rq_held(src_rq); - - /* - * Set sticky_cpu before deactivate_task() to properly mark the - * beginning of an SCX-internal migration. - */ - p->scx.sticky_cpu = cpu_of(dst_rq); - deactivate_task(src_rq, p, 0); - set_task_cpu(p, cpu_of(dst_rq)); - - raw_spin_rq_unlock(src_rq); - raw_spin_rq_lock(dst_rq); - - /* - * We want to pass scx-specific enq_flags but activate_task() will - * truncate the upper 32 bit. As we own @rq, we can pass them through - * @rq->scx.extra_enq_flags instead. - */ - WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)); - WARN_ON_ONCE(dst_rq->scx.extra_enq_flags); - dst_rq->scx.extra_enq_flags = enq_flags; - activate_task(dst_rq, p, 0); - dst_rq->scx.extra_enq_flags = 0; -} - -/* - * Similar to kernel/sched/core.c::is_cpu_allowed(). However, there are two - * differences: - * - * - is_cpu_allowed() asks "Can this task run on this CPU?" while - * task_can_run_on_remote_rq() asks "Can the BPF scheduler migrate the task to - * this CPU?". - * - * While migration is disabled, is_cpu_allowed() has to say "yes" as the task - * must be allowed to finish on the CPU that it's currently on regardless of - * the CPU state. However, task_can_run_on_remote_rq() must say "no" as the - * BPF scheduler shouldn't attempt to migrate a task which has migration - * disabled. - * - * - The BPF scheduler is bypassed while the rq is offline and we can always say - * no to the BPF scheduler initiated migrations while offline. - * - * The caller must ensure that @p and @rq are on different CPUs. - */ -static bool task_can_run_on_remote_rq(struct scx_sched *sch, - struct task_struct *p, struct rq *rq, - bool enforce) -{ - s32 cpu = cpu_of(rq); - - WARN_ON_ONCE(task_cpu(p) == cpu); - - /* - * If @p has migration disabled, @p->cpus_ptr is updated to contain only - * the pinned CPU in migrate_disable_switch() while @p is being switched - * out. However, put_prev_task_scx() is called before @p->cpus_ptr is - * updated and thus another CPU may see @p on a DSQ inbetween leading to - * @p passing the below task_allowed_on_cpu() check while migration is - * disabled. - * - * Test the migration disabled state first as the race window is narrow - * and the BPF scheduler failing to check migration disabled state can - * easily be masked if task_allowed_on_cpu() is done first. - */ - if (unlikely(is_migration_disabled(p))) { - if (enforce) - scx_error(sch, "SCX_DSQ_LOCAL[_ON] cannot move migration disabled %s[%d] from CPU %d to %d", - p->comm, p->pid, task_cpu(p), cpu); - return false; - } - - /* - * We don't require the BPF scheduler to avoid dispatching to offline - * CPUs mostly for convenience but also because CPUs can go offline - * between scx_bpf_dsq_insert() calls and here. Trigger error iff the - * picked CPU is outside the allowed mask. - */ - if (!task_allowed_on_cpu(p, cpu)) { - if (enforce) - scx_error(sch, "SCX_DSQ_LOCAL[_ON] target CPU %d not allowed for %s[%d]", - cpu, p->comm, p->pid); - return false; - } - - if (!scx_rq_online(rq)) { - if (enforce) - __scx_add_event(sch, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE, 1); - return false; - } - - return true; -} - -/** - * unlink_dsq_and_lock_src_rq() - Unlink task from its DSQ and lock its task_rq - * @p: target task - * @dsq: locked DSQ @p is currently on - * @src_rq: rq @p is currently on, stable with @dsq locked - * - * Called with @dsq locked but no rq's locked. We want to move @p to a different - * DSQ, including any local DSQ, but are not locking @src_rq. Locking @src_rq is - * required when transferring into a local DSQ. Even when transferring into a - * non-local DSQ, it's better to use the same mechanism to protect against - * dequeues and maintain the invariant that @p->scx.dsq can only change while - * @src_rq is locked, which e.g. scx_dump_task() depends on. - * - * We want to grab @src_rq but that can deadlock if we try while locking @dsq, - * so we want to unlink @p from @dsq, drop its lock and then lock @src_rq. As - * this may race with dequeue, which can't drop the rq lock or fail, do a little - * dancing from our side. - * - * @p->scx.holding_cpu is set to this CPU before @dsq is unlocked. If @p gets - * dequeued after we unlock @dsq but before locking @src_rq, the holding_cpu - * would be cleared to -1. While other cpus may have updated it to different - * values afterwards, as this operation can't be preempted or recurse, the - * holding_cpu can never become this CPU again before we're done. Thus, we can - * tell whether we lost to dequeue by testing whether the holding_cpu still - * points to this CPU. See dispatch_dequeue() for the counterpart. - * - * On return, @dsq is unlocked and @src_rq is locked. Returns %true if @p is - * still valid. %false if lost to dequeue. - */ -static bool unlink_dsq_and_lock_src_rq(struct task_struct *p, - struct scx_dispatch_q *dsq, - struct rq *src_rq) -{ - s32 cpu = raw_smp_processor_id(); - - lockdep_assert_held(&dsq->lock); - - WARN_ON_ONCE(p->scx.holding_cpu >= 0); - task_unlink_from_dsq(p, dsq); - p->scx.holding_cpu = cpu; - - raw_spin_unlock(&dsq->lock); - raw_spin_rq_lock(src_rq); - - /* task_rq couldn't have changed if we're still the holding cpu */ - return likely(p->scx.holding_cpu == cpu) && - !WARN_ON_ONCE(src_rq != task_rq(p)); -} - -static bool consume_remote_task(struct rq *this_rq, - struct task_struct *p, u64 enq_flags, - struct scx_dispatch_q *dsq, struct rq *src_rq) -{ - raw_spin_rq_unlock(this_rq); - - if (unlink_dsq_and_lock_src_rq(p, dsq, src_rq)) { - move_remote_task_to_local_dsq(p, enq_flags, src_rq, this_rq); - return true; - } else { - raw_spin_rq_unlock(src_rq); - raw_spin_rq_lock(this_rq); - return false; - } -} - -/** - * move_task_between_dsqs() - Move a task from one DSQ to another - * @sch: scx_sched being operated on - * @p: target task - * @enq_flags: %SCX_ENQ_* - * @src_dsq: DSQ @p is currently on, must not be a local DSQ - * @dst_dsq: DSQ @p is being moved to, can be any DSQ - * - * Must be called with @p's task_rq and @src_dsq locked. If @dst_dsq is a local - * DSQ and @p is on a different CPU, @p will be migrated and thus its task_rq - * will change. As @p's task_rq is locked, this function doesn't need to use the - * holding_cpu mechanism. - * - * On return, @src_dsq is unlocked and only @p's new task_rq, which is the - * return value, is locked. - */ -static struct rq *move_task_between_dsqs(struct scx_sched *sch, - struct task_struct *p, u64 enq_flags, - struct scx_dispatch_q *src_dsq, - struct scx_dispatch_q *dst_dsq) -{ - struct rq *src_rq = task_rq(p), *dst_rq; - - BUG_ON(src_dsq->id == SCX_DSQ_LOCAL); - lockdep_assert_held(&src_dsq->lock); - lockdep_assert_rq_held(src_rq); - - if (dst_dsq->id == SCX_DSQ_LOCAL) { - dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); - if (src_rq != dst_rq && - unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { - dst_dsq = find_global_dsq(sch, task_cpu(p)); - dst_rq = src_rq; - enq_flags |= SCX_ENQ_GDSQ_FALLBACK; - } - } else { - /* no need to migrate if destination is a non-local DSQ */ - dst_rq = src_rq; - } - - /* - * Move @p into $dst_dsq. If $dst_dsq is the local DSQ of a different - * CPU, @p will be migrated. - */ - if (dst_dsq->id == SCX_DSQ_LOCAL) { - /* @p is going from a non-local DSQ to a local DSQ */ - if (src_rq == dst_rq) { - task_unlink_from_dsq(p, src_dsq); - move_local_task_to_local_dsq(sch, p, enq_flags, - src_dsq, dst_rq); - raw_spin_unlock(&src_dsq->lock); - } else { - raw_spin_unlock(&src_dsq->lock); - move_remote_task_to_local_dsq(p, enq_flags, - src_rq, dst_rq); - } - } else { - /* - * @p is going from a non-local DSQ to a non-local DSQ. As - * $src_dsq is already locked, do an abbreviated dequeue. - */ - dispatch_dequeue_locked(p, src_dsq); - raw_spin_unlock(&src_dsq->lock); - - dispatch_enqueue(sch, dst_rq, dst_dsq, p, enq_flags); - } - - return dst_rq; -} - -static bool consume_dispatch_q(struct scx_sched *sch, struct rq *rq, - struct scx_dispatch_q *dsq, u64 enq_flags) -{ - struct task_struct *p; -retry: - /* - * The caller can't expect to successfully consume a task if the task's - * addition to @dsq isn't guaranteed to be visible somehow. Test - * @dsq->list without locking and skip if it seems empty. - */ - if (list_empty(&dsq->list)) - return false; - - raw_spin_lock(&dsq->lock); - - nldsq_for_each_task(p, dsq) { - struct rq *task_rq = task_rq(p); - - /* - * This loop can lead to multiple lockup scenarios, e.g. the BPF - * scheduler can put an enormous number of affinitized tasks into - * a contended DSQ, or the outer retry loop can repeatedly race - * against scx_bypass() dequeueing tasks from @dsq trying to put - * the system into the bypass mode. This can easily live-lock the - * machine. If aborting, exit from all non-bypass DSQs. - */ - if (unlikely(READ_ONCE(sch->aborting)) && dsq->id != SCX_DSQ_BYPASS) - break; - - if (rq == task_rq) { - task_unlink_from_dsq(p, dsq); - move_local_task_to_local_dsq(sch, p, enq_flags, dsq, rq); - raw_spin_unlock(&dsq->lock); - return true; - } - - if (task_can_run_on_remote_rq(sch, p, rq, false)) { - if (likely(consume_remote_task(rq, p, enq_flags, dsq, task_rq))) - return true; - goto retry; - } - } - - raw_spin_unlock(&dsq->lock); - return false; -} - -static bool consume_global_dsq(struct scx_sched *sch, struct rq *rq) -{ - int node = cpu_to_node(cpu_of(rq)); - - return consume_dispatch_q(sch, rq, &sch->pnode[node]->global_dsq, 0); -} - -/** - * dispatch_to_local_dsq - Dispatch a task to a local dsq - * @sch: scx_sched being operated on - * @rq: current rq which is locked - * @dst_dsq: destination DSQ - * @p: task to dispatch - * @enq_flags: %SCX_ENQ_* - * - * We're holding @rq lock and want to dispatch @p to @dst_dsq which is a local - * DSQ. This function performs all the synchronization dancing needed because - * local DSQs are protected with rq locks. - * - * The caller must have exclusive ownership of @p (e.g. through - * %SCX_OPSS_DISPATCHING). - */ -static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, - struct scx_dispatch_q *dst_dsq, - struct task_struct *p, u64 enq_flags) -{ - struct rq *src_rq = task_rq(p); - struct rq *dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); - struct rq *locked_rq = rq; - - /* - * We're synchronized against dequeue through DISPATCHING. As @p can't - * be dequeued, its task_rq and cpus_allowed are stable too. - * - * If dispatching to @rq that @p is already on, no lock dancing needed. - */ - if (rq == src_rq && rq == dst_rq) { - dispatch_enqueue(sch, rq, dst_dsq, p, - enq_flags | SCX_ENQ_CLEAR_OPSS); - return; - } - - if (src_rq != dst_rq && - unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { - dispatch_enqueue(sch, rq, find_global_dsq(sch, task_cpu(p)), p, - enq_flags | SCX_ENQ_CLEAR_OPSS | SCX_ENQ_GDSQ_FALLBACK); - return; - } - - /* - * @p is on a possibly remote @src_rq which we need to lock to move the - * task. If dequeue is in progress, it'd be locking @src_rq and waiting - * on DISPATCHING, so we can't grab @src_rq lock while holding - * DISPATCHING. - * - * As DISPATCHING guarantees that @p is wholly ours, we can pretend that - * we're moving from a DSQ and use the same mechanism - mark the task - * under transfer with holding_cpu, release DISPATCHING and then follow - * the same protocol. See unlink_dsq_and_lock_src_rq(). - */ - p->scx.holding_cpu = raw_smp_processor_id(); - - /* store_release ensures that dequeue sees the above */ - atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); - - /* switch to @src_rq lock */ - if (locked_rq != src_rq) { - raw_spin_rq_unlock(locked_rq); - locked_rq = src_rq; - raw_spin_rq_lock(src_rq); - } - - /* task_rq couldn't have changed if we're still the holding cpu */ - if (likely(p->scx.holding_cpu == raw_smp_processor_id()) && - !WARN_ON_ONCE(src_rq != task_rq(p))) { - /* - * If @p is staying on the same rq, there's no need to go - * through the full deactivate/activate cycle. Optimize by - * abbreviating move_remote_task_to_local_dsq(). - */ - if (src_rq == dst_rq) { - p->scx.holding_cpu = -1; - dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p, - enq_flags); - } else { - move_remote_task_to_local_dsq(p, enq_flags, - src_rq, dst_rq); - /* task has been moved to dst_rq, which is now locked */ - locked_rq = dst_rq; - } - - /* if the destination CPU is idle, wake it up */ - if (sched_class_above(p->sched_class, dst_rq->curr->sched_class)) - resched_curr(dst_rq); - } - - /* switch back to @rq lock */ - if (locked_rq != rq) { - raw_spin_rq_unlock(locked_rq); - raw_spin_rq_lock(rq); - } -} - -/** - * finish_dispatch - Asynchronously finish dispatching a task - * @rq: current rq which is locked - * @p: task to finish dispatching - * @qseq_at_dispatch: qseq when @p started getting dispatched - * @dsq_id: destination DSQ ID - * @enq_flags: %SCX_ENQ_* - * - * Dispatching to local DSQs may need to wait for queueing to complete or - * require rq lock dancing. As we don't wanna do either while inside - * ops.dispatch() to avoid locking order inversion, we split dispatching into - * two parts. scx_bpf_dsq_insert() which is called by ops.dispatch() records the - * task and its qseq. Once ops.dispatch() returns, this function is called to - * finish up. - * - * There is no guarantee that @p is still valid for dispatching or even that it - * was valid in the first place. Make sure that the task is still owned by the - * BPF scheduler and claim the ownership before dispatching. - */ -static void finish_dispatch(struct scx_sched *sch, struct rq *rq, - struct task_struct *p, - unsigned long qseq_at_dispatch, - u64 dsq_id, u64 enq_flags) -{ - struct scx_dispatch_q *dsq; - unsigned long opss; - - touch_core_sched_dispatch(rq, p); -retry: - /* - * No need for _acquire here. @p is accessed only after a successful - * try_cmpxchg to DISPATCHING. - */ - opss = atomic_long_read(&p->scx.ops_state); - - switch (opss & SCX_OPSS_STATE_MASK) { - case SCX_OPSS_DISPATCHING: - case SCX_OPSS_NONE: - /* someone else already got to it */ - return; - case SCX_OPSS_QUEUED: - /* - * If qseq doesn't match, @p has gone through at least one - * dispatch/dequeue and re-enqueue cycle between - * scx_bpf_dsq_insert() and here and we have no claim on it. - */ - if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch) - return; - - /* see SCX_EV_INSERT_NOT_OWNED definition */ - if (unlikely(!scx_task_on_sched(sch, p))) { - __scx_add_event(sch, SCX_EV_INSERT_NOT_OWNED, 1); - return; - } - - /* - * While we know @p is accessible, we don't yet have a claim on - * it - the BPF scheduler is allowed to dispatch tasks - * spuriously and there can be a racing dequeue attempt. Let's - * claim @p by atomically transitioning it from QUEUED to - * DISPATCHING. - */ - if (likely(atomic_long_try_cmpxchg(&p->scx.ops_state, &opss, - SCX_OPSS_DISPATCHING))) - break; - goto retry; - case SCX_OPSS_QUEUEING: - /* - * do_enqueue_task() is in the process of transferring the task - * to the BPF scheduler while holding @p's rq lock. As we aren't - * holding any kernel or BPF resource that the enqueue path may - * depend upon, it's safe to wait. - */ - wait_ops_state(p, opss); - goto retry; - } - - BUG_ON(!(p->scx.flags & SCX_TASK_QUEUED)); - - dsq = find_dsq_for_dispatch(sch, this_rq(), dsq_id, task_cpu(p)); - - if (dsq->id == SCX_DSQ_LOCAL) - dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags); - else - dispatch_enqueue(sch, rq, dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); -} - -static void flush_dispatch_buf(struct scx_sched *sch, struct rq *rq) -{ - struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; - u32 u; - - for (u = 0; u < dspc->cursor; u++) { - struct scx_dsp_buf_ent *ent = &dspc->buf[u]; - - finish_dispatch(sch, rq, ent->task, ent->qseq, ent->dsq_id, - ent->enq_flags); - } - - dspc->nr_tasks += dspc->cursor; - dspc->cursor = 0; -} - -static inline void maybe_queue_balance_callback(struct rq *rq) -{ - lockdep_assert_rq_held(rq); - - if (!(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) - return; - - queue_balance_callback(rq, &rq->scx.deferred_bal_cb, - deferred_bal_cb_workfn); - - rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; -} - -/* - * One user of this function is scx_bpf_dispatch() which can be called - * recursively as sub-sched dispatches nest. Always inline to reduce stack usage - * from the call frame. - */ -static __always_inline bool -scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, - struct task_struct *prev, bool nested) -{ - struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; - int nr_loops = SCX_DSP_MAX_LOOPS; - s32 cpu = cpu_of(rq); - bool prev_on_sch = (prev->sched_class == &ext_sched_class) && - scx_task_on_sched(sch, prev); - - if (consume_global_dsq(sch, rq)) - return true; - - if (bypass_dsp_enabled(sch)) { - /* if @sch is bypassing, only the bypass DSQs are active */ - if (scx_bypassing(sch, cpu)) - return consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0); - -#ifdef CONFIG_EXT_SUB_SCHED - /* - * If @sch isn't bypassing but its children are, @sch is - * responsible for making forward progress for both its own - * tasks that aren't bypassing and the bypassing descendants' - * tasks. The following implements a simple built-in behavior - - * let each CPU try to run the bypass DSQ every Nth time. - * - * Later, if necessary, we can add an ops flag to suppress the - * auto-consumption and a kfunc to consume the bypass DSQ and, - * so that the BPF scheduler can fully control scheduling of - * bypassed tasks. - */ - struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); - - if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) && - consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0)) { - __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); - return true; - } -#endif /* CONFIG_EXT_SUB_SCHED */ - } - - if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq)) - return false; - - dspc->rq = rq; - - /* - * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, - * the local DSQ might still end up empty after a successful - * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() - * produced some tasks, retry. The BPF scheduler may depend on this - * looping behavior to simplify its implementation. - */ - do { - dspc->nr_tasks = 0; - - if (nested) { - SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), - prev_on_sch ? prev : NULL); - } else { - /* stash @prev so that nested invocations can access it */ - rq->scx.sub_dispatch_prev = prev; - SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), - prev_on_sch ? prev : NULL); - rq->scx.sub_dispatch_prev = NULL; - } - - flush_dispatch_buf(sch, rq); - - if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) { - rq->scx.flags |= SCX_RQ_BAL_KEEP; - return true; - } - if (rq->scx.local_dsq.nr) - return true; - if (consume_global_dsq(sch, rq)) - return true; - - /* - * ops.dispatch() can trap us in this loop by repeatedly - * dispatching ineligible tasks. Break out once in a while to - * allow the watchdog to run. As IRQ can't be enabled in - * balance(), we want to complete this scheduling cycle and then - * start a new one. IOW, we want to call resched_curr() on the - * next, most likely idle, task, not the current one. Use - * __scx_bpf_kick_cpu() for deferred kicking. - */ - if (unlikely(!--nr_loops)) { - scx_kick_cpu(sch, cpu, 0); - break; - } - } while (dspc->nr_tasks); - - /* - * Prevent the CPU from going idle while bypassed descendants have tasks - * queued. Without this fallback, bypassed tasks could stall if the host - * scheduler's ops.dispatch() doesn't yield any tasks. - */ - if (bypass_dsp_enabled(sch)) - return consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0); - - return false; -} - -static int balance_one(struct rq *rq, struct task_struct *prev) -{ - struct scx_sched *sch = scx_root; - s32 cpu = cpu_of(rq); - - lockdep_assert_rq_held(rq); - rq->scx.flags |= SCX_RQ_IN_BALANCE; - rq->scx.flags &= ~SCX_RQ_BAL_KEEP; - - if ((sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) && - unlikely(rq->scx.cpu_released)) { - /* - * If the previous sched_class for the current CPU was not SCX, - * notify the BPF scheduler that it again has control of the - * core. This callback complements ->cpu_release(), which is - * emitted in switch_class(). - */ - if (sch->ops.cpu_acquire) - SCX_CALL_OP(sch, cpu_acquire, rq, cpu, NULL); - rq->scx.cpu_released = false; - } - - if (prev->sched_class == &ext_sched_class) { - update_curr_scx(rq); - - /* - * If @prev is runnable & has slice left, it has priority and - * fetching more just increases latency for the fetched tasks. - * Tell pick_task_scx() to keep running @prev. If the BPF - * scheduler wants to handle this explicitly, it should - * implement ->cpu_release(). - * - * See scx_disable_workfn() for the explanation on the bypassing - * test. - */ - if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice && - !scx_bypassing(sch, cpu)) { - rq->scx.flags |= SCX_RQ_BAL_KEEP; - goto has_tasks; - } - } - - /* if there already are tasks to run, nothing to do */ - if (rq->scx.local_dsq.nr) - goto has_tasks; - - if (scx_dispatch_sched(sch, rq, prev, false)) - goto has_tasks; - - /* - * Didn't find another task to run. Keep running @prev unless - * %SCX_OPS_ENQ_LAST is in effect. - */ - if ((prev->scx.flags & SCX_TASK_QUEUED) && - (!(sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(sch, cpu))) { - rq->scx.flags |= SCX_RQ_BAL_KEEP; - __scx_add_event(sch, SCX_EV_DISPATCH_KEEP_LAST, 1); - goto has_tasks; - } - rq->scx.flags &= ~SCX_RQ_IN_BALANCE; - return false; - -has_tasks: - /* - * @rq may have extra IMMED tasks without reenq scheduled: - * - * - rq_is_open() can't reliably tell when and how slice is going to be - * modified for $curr and allows IMMED tasks to be queued while - * dispatch is in progress. - * - * - A non-IMMED HEAD task can get queued in front of an IMMED task - * between the IMMED queueing and the subsequent scheduling event. - */ - if (unlikely(rq->scx.local_dsq.nr > 1 && rq->scx.nr_immed)) - schedule_reenq_local(rq, 0); - - rq->scx.flags &= ~SCX_RQ_IN_BALANCE; - return true; -} - -static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) -{ - struct scx_sched *sch = scx_task_sched(p); - - if (p->scx.flags & SCX_TASK_QUEUED) { - /* - * Core-sched might decide to execute @p before it is - * dispatched. Call ops_dequeue() to notify the BPF scheduler. - */ - ops_dequeue(rq, p, SCX_DEQ_CORE_SCHED_EXEC); - dispatch_dequeue(rq, p); - } - - p->se.exec_start = rq_clock_task(rq); - - /* see dequeue_task_scx() on why we skip when !QUEUED */ - if (SCX_HAS_OP(sch, running) && (p->scx.flags & SCX_TASK_QUEUED)) - SCX_CALL_OP_TASK(sch, running, rq, p); - - clr_task_runnable(p, true); - - /* - * @p is getting newly scheduled or got kicked after someone updated its - * slice. Refresh whether tick can be stopped. See scx_can_stop_tick(). - */ - if ((p->scx.slice == SCX_SLICE_INF) != - (bool)(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) { - if (p->scx.slice == SCX_SLICE_INF) - rq->scx.flags |= SCX_RQ_CAN_STOP_TICK; - else - rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK; - - sched_update_tick_dependency(rq); - - /* - * For now, let's refresh the load_avgs just when transitioning - * in and out of nohz. In the future, we might want to add a - * mechanism which calls the following periodically on - * tick-stopped CPUs. - */ - update_other_load_avgs(rq); - } -} - -static enum scx_cpu_preempt_reason -preempt_reason_from_class(const struct sched_class *class) -{ - if (class == &stop_sched_class) - return SCX_CPU_PREEMPT_STOP; - if (class == &dl_sched_class) - return SCX_CPU_PREEMPT_DL; - if (class == &rt_sched_class) - return SCX_CPU_PREEMPT_RT; - return SCX_CPU_PREEMPT_UNKNOWN; -} - -static void switch_class(struct rq *rq, struct task_struct *next) -{ - struct scx_sched *sch = scx_root; - const struct sched_class *next_class = next->sched_class; - - if (!(sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT)) - return; - - /* - * The callback is conceptually meant to convey that the CPU is no - * longer under the control of SCX. Therefore, don't invoke the callback - * if the next class is below SCX (in which case the BPF scheduler has - * actively decided not to schedule any tasks on the CPU). - */ - if (sched_class_above(&ext_sched_class, next_class)) - return; - - /* - * At this point we know that SCX was preempted by a higher priority - * sched_class, so invoke the ->cpu_release() callback if we have not - * done so already. We only send the callback once between SCX being - * preempted, and it regaining control of the CPU. - * - * ->cpu_release() complements ->cpu_acquire(), which is emitted the - * next time that balance_one() is invoked. - */ - if (!rq->scx.cpu_released) { - if (sch->ops.cpu_release) { - struct scx_cpu_release_args args = { - .reason = preempt_reason_from_class(next_class), - .task = next, - }; - - SCX_CALL_OP(sch, cpu_release, rq, cpu_of(rq), &args); - } - rq->scx.cpu_released = true; - } -} - -static void put_prev_task_scx(struct rq *rq, struct task_struct *p, - struct task_struct *next) -{ - struct scx_sched *sch = scx_task_sched(p); - - /* see kick_sync_wait_bal_cb() */ - smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); - - update_curr_scx(rq); - - /* see dequeue_task_scx() on why we skip when !QUEUED */ - if (SCX_HAS_OP(sch, stopping) && (p->scx.flags & SCX_TASK_QUEUED)) - SCX_CALL_OP_TASK(sch, stopping, rq, p, true); - - if (p->scx.flags & SCX_TASK_QUEUED) { - set_task_runnable(rq, p); - - /* - * If @p has slice left and is being put, @p is getting - * preempted by a higher priority scheduler class or core-sched - * forcing a different task. Leave it at the head of the local - * DSQ unless it was an IMMED task. IMMED tasks should not - * linger on a busy CPU, reenqueue them to the BPF scheduler. - */ - if (p->scx.slice && !scx_bypassing(sch, cpu_of(rq))) { - if (p->scx.flags & SCX_TASK_IMMED) { - p->scx.flags |= SCX_TASK_REENQ_PREEMPTED; - do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; - } else { - dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, SCX_ENQ_HEAD); - } - goto switch_class; - } - - /* - * If @p is runnable but we're about to enter a lower - * sched_class, %SCX_OPS_ENQ_LAST must be set. Tell - * ops.enqueue() that @p is the only one available for this cpu, - * which should trigger an explicit follow-up scheduling event. - */ - if (next && sched_class_above(&ext_sched_class, next->sched_class)) { - WARN_ON_ONCE(!(sch->ops.flags & SCX_OPS_ENQ_LAST)); - do_enqueue_task(rq, p, SCX_ENQ_LAST, -1); - } else { - do_enqueue_task(rq, p, 0, -1); - } - } - -switch_class: - if (next && next->sched_class != &ext_sched_class) - switch_class(rq, next); -} - -static void kick_sync_wait_bal_cb(struct rq *rq) -{ - struct scx_kick_syncs __rcu *ks = __this_cpu_read(scx_kick_syncs); - unsigned long *ksyncs = rcu_dereference_sched(ks)->syncs; - bool waited; - s32 cpu; - - /* - * Drop rq lock and enable IRQs while waiting. IRQs must be enabled - * — a target CPU may be waiting for us to process an IPI (e.g. TLB - * flush) while we wait for its kick_sync to advance. - * - * Also, keep advancing our own kick_sync so that new kick_sync waits - * targeting us, which can start after we drop the lock, cannot form - * cyclic dependencies. - */ -retry: - waited = false; - for_each_cpu(cpu, rq->scx.cpus_to_sync) { - /* - * smp_load_acquire() pairs with smp_store_release() on - * kick_sync updates on the target CPUs. - */ - if (cpu == cpu_of(rq) || - smp_load_acquire(&cpu_rq(cpu)->scx.kick_sync) != ksyncs[cpu]) { - cpumask_clear_cpu(cpu, rq->scx.cpus_to_sync); - continue; - } - - raw_spin_rq_unlock_irq(rq); - while (READ_ONCE(cpu_rq(cpu)->scx.kick_sync) == ksyncs[cpu]) { - smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); - cpu_relax(); - } - raw_spin_rq_lock_irq(rq); - waited = true; - } - - if (waited) - goto retry; -} - -static struct task_struct *first_local_task(struct rq *rq) -{ - return list_first_entry_or_null(&rq->scx.local_dsq.list, - struct task_struct, scx.dsq_list.node); -} - -static struct task_struct * -do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) -{ - struct task_struct *prev = rq->curr; - bool keep_prev; - struct task_struct *p; - - /* see kick_sync_wait_bal_cb() */ - smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); - - rq_modified_begin(rq, &ext_sched_class); - - rq_unpin_lock(rq, rf); - balance_one(rq, prev); - rq_repin_lock(rq, rf); - maybe_queue_balance_callback(rq); - - /* - * Defer to a balance callback which can drop rq lock and enable - * IRQs. Waiting directly in the pick path would deadlock against - * CPUs sending us IPIs (e.g. TLB flushes) while we wait for them. - */ - if (unlikely(rq->scx.kick_sync_pending)) { - rq->scx.kick_sync_pending = false; - queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, - kick_sync_wait_bal_cb); - } - - /* - * If any higher-priority sched class enqueued a runnable task on - * this rq during balance_one(), abort and return RETRY_TASK, so - * that the scheduler loop can restart. - * - * If @force_scx is true, always try to pick a SCHED_EXT task, - * regardless of any higher-priority sched classes activity. - */ - if (!force_scx && rq_modified_above(rq, &ext_sched_class)) - return RETRY_TASK; - - keep_prev = rq->scx.flags & SCX_RQ_BAL_KEEP; - if (unlikely(keep_prev && - prev->sched_class != &ext_sched_class)) { - WARN_ON_ONCE(scx_enable_state() == SCX_ENABLED); - keep_prev = false; - } - - /* - * If balance_one() is telling us to keep running @prev, replenish slice - * if necessary and keep running @prev. Otherwise, pop the first one - * from the local DSQ. - */ - if (keep_prev) { - p = prev; - if (!p->scx.slice) - refill_task_slice_dfl(scx_task_sched(p), p); - } else { - p = first_local_task(rq); - if (!p) - return NULL; - - if (unlikely(!p->scx.slice)) { - struct scx_sched *sch = scx_task_sched(p); - - if (!scx_bypassing(sch, cpu_of(rq)) && - !sch->warned_zero_slice) { - printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in %s()\n", - p->comm, p->pid, __func__); - sch->warned_zero_slice = true; - } - refill_task_slice_dfl(sch, p); - } - } - - return p; -} - -static struct task_struct *pick_task_scx(struct rq *rq, struct rq_flags *rf) -{ - return do_pick_task_scx(rq, rf, false); -} - -/* - * Select the next task to run from the ext scheduling class. - * - * Use do_pick_task_scx() directly with @force_scx enabled, since the - * dl_server must always select a sched_ext task. - */ -static struct task_struct * -ext_server_pick_task(struct sched_dl_entity *dl_se, struct rq_flags *rf) -{ - if (!scx_enabled()) - return NULL; - - return do_pick_task_scx(dl_se->rq, rf, true); -} - -/* - * Initialize the ext server deadline entity. - */ -void ext_server_init(struct rq *rq) -{ - struct sched_dl_entity *dl_se = &rq->ext_server; - - init_dl_entity(dl_se); - - dl_server_init(dl_se, rq, ext_server_pick_task); -} - -#ifdef CONFIG_SCHED_CORE -/** - * scx_prio_less - Task ordering for core-sched - * @a: task A - * @b: task B - * @in_fi: in forced idle state - * - * Core-sched is implemented as an additional scheduling layer on top of the - * usual sched_class'es and needs to find out the expected task ordering. For - * SCX, core-sched calls this function to interrogate the task ordering. - * - * Unless overridden by ops.core_sched_before(), @p->scx.core_sched_at is used - * to implement the default task ordering. The older the timestamp, the higher - * priority the task - the global FIFO ordering matching the default scheduling - * behavior. - * - * When ops.core_sched_before() is enabled, @p->scx.core_sched_at is used to - * implement FIFO ordering within each local DSQ. See pick_task_scx(). - */ -bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, - bool in_fi) -{ - struct scx_sched *sch_a = scx_task_sched(a); - struct scx_sched *sch_b = scx_task_sched(b); - - /* - * The const qualifiers are dropped from task_struct pointers when - * calling ops.core_sched_before(). Accesses are controlled by the - * verifier. - */ - if (sch_a == sch_b && SCX_HAS_OP(sch_a, core_sched_before) && - !scx_bypassing(sch_a, task_cpu(a))) - return SCX_CALL_OP_2TASKS_RET(sch_a, core_sched_before, - task_rq(a), - (struct task_struct *)a, - (struct task_struct *)b); - else - return time_after64(a->scx.core_sched_at, b->scx.core_sched_at); -} -#endif /* CONFIG_SCHED_CORE */ - -static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags) -{ - struct scx_sched *sch = scx_task_sched(p); - bool bypassing; - - /* - * sched_exec() calls with %WF_EXEC when @p is about to exec(2) as it - * can be a good migration opportunity with low cache and memory - * footprint. Returning a CPU different than @prev_cpu triggers - * immediate rq migration. However, for SCX, as the current rq - * association doesn't dictate where the task is going to run, this - * doesn't fit well. If necessary, we can later add a dedicated method - * which can decide to preempt self to force it through the regular - * scheduling path. - */ - if (unlikely(wake_flags & WF_EXEC)) - return prev_cpu; - - bypassing = scx_bypassing(sch, task_cpu(p)); - if (likely(SCX_HAS_OP(sch, select_cpu)) && !bypassing) { - s32 cpu; - struct task_struct **ddsp_taskp; - - ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); - WARN_ON_ONCE(*ddsp_taskp); - *ddsp_taskp = p; - - this_rq()->scx.in_select_cpu = true; - cpu = SCX_CALL_OP_TASK_RET(sch, select_cpu, NULL, p, - scx_cpu_arg(prev_cpu), wake_flags); - cpu = scx_cpu_ret(sch, cpu); - this_rq()->scx.in_select_cpu = false; - p->scx.selected_cpu = cpu; - *ddsp_taskp = NULL; - if (scx_cpu_valid(sch, cpu, "from ops.select_cpu()")) - return cpu; - else - return prev_cpu; - } else { - s32 cpu; - - cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, NULL, 0); - if (cpu >= 0) { - refill_task_slice_dfl(sch, p); - p->scx.ddsp_dsq_id = SCX_DSQ_LOCAL; - } else { - cpu = prev_cpu; - } - p->scx.selected_cpu = cpu; - - if (bypassing) - __scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1); - return cpu; - } -} - -static void task_woken_scx(struct rq *rq, struct task_struct *p) -{ - run_deferred(rq); -} - -static void set_cpus_allowed_scx(struct task_struct *p, - struct affinity_context *ac) -{ - struct scx_sched *sch = scx_task_sched(p); - - set_cpus_allowed_common(p, ac); - - if (task_dead_and_done(p)) - return; - - /* - * The effective cpumask is stored in @p->cpus_ptr which may temporarily - * differ from the configured one in @p->cpus_mask. Always tell the bpf - * scheduler the effective one. - * - * Fine-grained memory write control is enforced by BPF making the const - * designation pointless. Cast it away when calling the operation. - */ - if (SCX_HAS_OP(sch, set_cpumask)) - scx_call_op_set_cpumask(sch, task_rq(p), p, (struct cpumask *)p->cpus_ptr); -} - -static void handle_hotplug(struct rq *rq, bool online) -{ - struct scx_sched *sch = scx_root; - s32 cpu = cpu_of(rq); - - atomic_long_inc(&scx_hotplug_seq); - - /* - * scx_root updates are protected by cpus_read_lock() and will stay - * stable here. Note that we can't depend on scx_enabled() test as the - * hotplug ops need to be enabled before __scx_enabled is set. - */ - if (unlikely(!sch)) - return; - - if (scx_enabled()) - scx_idle_update_selcpu_topology(&sch->ops); - - if (online && SCX_HAS_OP(sch, cpu_online)) - SCX_CALL_OP(sch, cpu_online, NULL, scx_cpu_arg(cpu)); - else if (!online && SCX_HAS_OP(sch, cpu_offline)) - SCX_CALL_OP(sch, cpu_offline, NULL, scx_cpu_arg(cpu)); - else - scx_exit(sch, SCX_EXIT_UNREG_KERN, - SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG, - "cpu %d going %s, exiting scheduler", cpu, - online ? "online" : "offline"); -} - -void scx_rq_activate(struct rq *rq) -{ - handle_hotplug(rq, true); -} - -void scx_rq_deactivate(struct rq *rq) -{ - handle_hotplug(rq, false); -} - -static void rq_online_scx(struct rq *rq) -{ - rq->scx.flags |= SCX_RQ_ONLINE; -} - -static void rq_offline_scx(struct rq *rq) -{ - rq->scx.flags &= ~SCX_RQ_ONLINE; -} - -static bool check_rq_for_timeouts(struct rq *rq) -{ - struct scx_sched *sch; - struct task_struct *p; - struct rq_flags rf; - bool timed_out = false; - - rq_lock_irqsave(rq, &rf); - sch = rcu_dereference_bh(scx_root); - if (unlikely(!sch)) - goto out_unlock; - - list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) { - struct scx_sched *sch = scx_task_sched(p); - unsigned long last_runnable = p->scx.runnable_at; - - if (unlikely(time_after(jiffies, - last_runnable + READ_ONCE(sch->watchdog_timeout)))) { - u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); - - __scx_exit(sch, SCX_EXIT_ERROR_STALL, 0, cpu_of(rq), - "%s[%d] failed to run for %u.%03us", - p->comm, p->pid, dur_ms / 1000, - dur_ms % 1000); - timed_out = true; - break; - } - } -out_unlock: - rq_unlock_irqrestore(rq, &rf); - return timed_out; -} - -static void scx_watchdog_workfn(struct work_struct *work) -{ - unsigned long intv; - int cpu; - - WRITE_ONCE(scx_watchdog_timestamp, jiffies); - - for_each_online_cpu(cpu) { - if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) - break; - - cond_resched(); - } - - intv = READ_ONCE(scx_watchdog_interval); - if (intv < ULONG_MAX) - queue_delayed_work(system_dfl_wq, to_delayed_work(work), intv); -} - -void scx_tick(struct rq *rq) -{ - struct scx_sched *root; - unsigned long last_check; - - if (!scx_enabled()) - return; - - root = rcu_dereference_bh(scx_root); - if (unlikely(!root)) - return; - - last_check = READ_ONCE(scx_watchdog_timestamp); - if (unlikely(time_after(jiffies, - last_check + READ_ONCE(root->watchdog_timeout)))) { - u32 dur_ms = jiffies_to_msecs(jiffies - last_check); - - scx_exit(root, SCX_EXIT_ERROR_STALL, 0, - "watchdog failed to check in for %u.%03us", - dur_ms / 1000, dur_ms % 1000); - } - - update_other_load_avgs(rq); -} - -static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) -{ - struct scx_sched *sch = scx_task_sched(curr); - - update_curr_scx(rq); - - /* - * While disabling, always resched and refresh core-sched timestamp as - * we can't trust the slice management or ops.core_sched_before(). - */ - if (scx_bypassing(sch, cpu_of(rq))) { - curr->scx.slice = 0; - touch_core_sched(rq, curr); - } else if (SCX_HAS_OP(sch, tick)) { - SCX_CALL_OP_TASK(sch, tick, rq, curr); - } - - if (!curr->scx.slice) - resched_curr(rq); -} - -#ifdef CONFIG_EXT_GROUP_SCHED -static struct cgroup *tg_cgrp(struct task_group *tg) -{ - /* - * If CGROUP_SCHED is disabled, @tg is NULL. If @tg is an autogroup, - * @tg->css.cgroup is NULL. In both cases, @tg can be treated as the - * root cgroup. - */ - if (tg && tg->css.cgroup) - return tg->css.cgroup; - else - return &cgrp_dfl_root.cgrp; -} - -#define SCX_INIT_TASK_ARGS_CGROUP(tg) .cgroup = tg_cgrp(tg), - -#else /* CONFIG_EXT_GROUP_SCHED */ - -#define SCX_INIT_TASK_ARGS_CGROUP(tg) - -#endif /* CONFIG_EXT_GROUP_SCHED */ - -static int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fork) -{ - int ret; - - p->scx.disallow = false; - - if (SCX_HAS_OP(sch, init_task)) { - struct scx_init_task_args args = { - SCX_INIT_TASK_ARGS_CGROUP(task_group(p)) - .fork = fork, - }; - - ret = SCX_CALL_OP_RET(sch, init_task, NULL, p, &args); - if (unlikely(ret)) { - ret = ops_sanitize_err(sch, "init_task", ret); - return ret; - } - } - - if (p->scx.disallow) { - if (unlikely(scx_parent(sch))) { - scx_error(sch, "non-root ops.init_task() set task->scx.disallow for %s[%d]", - p->comm, p->pid); - } else if (unlikely(fork)) { - scx_error(sch, "ops.init_task() set task->scx.disallow for %s[%d] during fork", - p->comm, p->pid); - } else { - struct rq *rq; - struct rq_flags rf; - - rq = task_rq_lock(p, &rf); - - /* - * We're in the load path and @p->policy will be applied - * right after. Reverting @p->policy here and rejecting - * %SCHED_EXT transitions from scx_check_setscheduler() - * guarantees that if ops.init_task() sets @p->disallow, - * @p can never be in SCX. - */ - if (p->policy == SCHED_EXT) { - p->policy = SCHED_NORMAL; - atomic_long_inc(&scx_nr_rejected); - } - - task_rq_unlock(rq, p, &rf); - } - } - - return 0; -} - -static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p) -{ - struct rq *rq = task_rq(p); - u32 weight; - - lockdep_assert_rq_held(rq); - - /* - * Verify the task is not in BPF scheduler's custody. If flag - * transitions are consistent, the flag should always be clear - * here. - */ - WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY); - - /* - * Set the weight before calling ops.enable() so that the scheduler - * doesn't see a stale value if they inspect the task struct. - */ - if (task_has_idle_policy(p)) - weight = WEIGHT_IDLEPRIO; - else - weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; - - p->scx.weight = sched_weight_to_cgroup(weight); - - if (SCX_HAS_OP(sch, enable)) - SCX_CALL_OP_TASK(sch, enable, rq, p); - - if (SCX_HAS_OP(sch, set_weight)) - SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight); -} - -static void scx_enable_task(struct scx_sched *sch, struct task_struct *p) -{ - __scx_enable_task(sch, p); - scx_set_task_state(p, SCX_TASK_ENABLED); -} - -static void scx_disable_task(struct scx_sched *sch, struct task_struct *p) -{ - struct rq *rq = task_rq(p); - - lockdep_assert_rq_held(rq); - WARN_ON_ONCE(scx_get_task_state(p) != SCX_TASK_ENABLED); - - clear_direct_dispatch(p); - - if (SCX_HAS_OP(sch, disable)) - SCX_CALL_OP_TASK(sch, disable, rq, p); - scx_set_task_state(p, SCX_TASK_READY); - - /* - * Verify the task is not in BPF scheduler's custody. If flag - * transitions are consistent, the flag should always be clear - * here. - */ - WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY); -} - -static void __scx_disable_and_exit_task(struct scx_sched *sch, - struct task_struct *p) -{ - struct scx_exit_task_args args = { - .cancelled = false, - }; - - lockdep_assert_held(&p->pi_lock); - lockdep_assert_rq_held(task_rq(p)); - - switch (scx_get_task_state(p)) { - case SCX_TASK_NONE: - return; - case SCX_TASK_INIT: - args.cancelled = true; - break; - case SCX_TASK_READY: - break; - case SCX_TASK_ENABLED: - scx_disable_task(sch, p); - break; - default: - WARN_ON_ONCE(true); - return; - } - - if (SCX_HAS_OP(sch, exit_task)) - SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args); -} - -/* - * Undo a completed __scx_init_task(sch, p, false) when scx_enable_task() never - * ran. The task state has not been transitioned, so this mirrors the - * SCX_TASK_INIT branch in __scx_disable_and_exit_task(). - */ -static void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct *p) -{ - struct scx_exit_task_args args = { .cancelled = true }; - - lockdep_assert_held(&p->pi_lock); - lockdep_assert_rq_held(task_rq(p)); - - if (SCX_HAS_OP(sch, exit_task)) - SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args); -} - -static void scx_disable_and_exit_task(struct scx_sched *sch, - struct task_struct *p) -{ - __scx_disable_and_exit_task(sch, p); - - /* - * If set, @p exited between __scx_init_task() and scx_enable_task() in - * scx_sub_enable() and is initialized for both the associated sched and - * its parent. Exit for the child too - scx_enable_task() never ran for - * it, so undo only init_task. The flag is only set on the sub-enable - * path, so it's always clear when @p arrives here in %SCX_TASK_NONE. - */ - if (p->scx.flags & SCX_TASK_SUB_INIT) { - if (!WARN_ON_ONCE(!scx_enabling_sub_sched)) - scx_sub_init_cancel_task(scx_enabling_sub_sched, p); - p->scx.flags &= ~SCX_TASK_SUB_INIT; - } - - scx_set_task_sched(p, NULL); - scx_set_task_state(p, SCX_TASK_NONE); -} - -void init_scx_entity(struct sched_ext_entity *scx) -{ - memset(scx, 0, sizeof(*scx)); - INIT_LIST_HEAD(&scx->dsq_list.node); - RB_CLEAR_NODE(&scx->dsq_priq); - scx->sticky_cpu = -1; - scx->holding_cpu = -1; - INIT_LIST_HEAD(&scx->runnable_node); - scx->runnable_at = jiffies; - scx->ddsp_dsq_id = SCX_DSQ_INVALID; - scx->slice = SCX_SLICE_DFL; -} - -/* See scx_tid_alloc / scx_tid_cursor. */ -static u64 scx_alloc_tid(void) -{ - struct scx_tid_alloc *ta; - - guard(preempt)(); - ta = this_cpu_ptr(&scx_tid_alloc); - - if (unlikely(ta->next >= ta->end)) { - ta->next = atomic64_fetch_add(SCX_TID_CHUNK, &scx_tid_cursor); - ta->end = ta->next + SCX_TID_CHUNK; - } - return ta->next++; -} - -static void scx_tid_hash_insert(struct task_struct *p) -{ - int ret; - - lockdep_assert_held(&scx_tasks_lock); - - ret = rhashtable_lookup_insert_fast(&scx_tid_hash, - &p->scx.tid_hash_node, - scx_tid_hash_params); - WARN_ON_ONCE(ret); -} - -void scx_pre_fork(struct task_struct *p) -{ - /* - * BPF scheduler enable/disable paths want to be able to iterate and - * update all tasks which can become complex when racing forks. As - * enable/disable are very cold paths, let's use a percpu_rwsem to - * exclude forks. - */ - percpu_down_read(&scx_fork_rwsem); -} - -int scx_fork(struct task_struct *p, struct kernel_clone_args *kargs) -{ - s32 ret; - - percpu_rwsem_assert_held(&scx_fork_rwsem); - - p->scx.tid = scx_alloc_tid(); - - if (scx_init_task_enabled) { -#ifdef CONFIG_EXT_SUB_SCHED - struct scx_sched *sch = kargs->cset->dfl_cgrp->scx_sched; -#else - struct scx_sched *sch = scx_root; -#endif - scx_set_task_state(p, SCX_TASK_INIT_BEGIN); - ret = __scx_init_task(sch, p, true); - if (unlikely(ret)) { - scx_set_task_state(p, SCX_TASK_NONE); - return ret; - } - scx_set_task_state(p, SCX_TASK_INIT); - scx_set_task_sched(p, sch); - } - - return 0; -} - -void scx_post_fork(struct task_struct *p) -{ - if (scx_init_task_enabled) { - scx_set_task_state(p, SCX_TASK_READY); - - /* - * Enable the task immediately if it's running on sched_ext. - * Otherwise, it'll be enabled in switching_to_scx() if and - * when it's ever configured to run with a SCHED_EXT policy. - */ - if (p->sched_class == &ext_sched_class) { - struct rq_flags rf; - struct rq *rq; - - rq = task_rq_lock(p, &rf); - scx_enable_task(scx_task_sched(p), p); - task_rq_unlock(rq, p, &rf); - } - } - - scoped_guard(raw_spinlock_irq, &scx_tasks_lock) { - list_add_tail(&p->scx.tasks_node, &scx_tasks); - if (scx_tid_to_task_enabled()) - scx_tid_hash_insert(p); - } - - percpu_up_read(&scx_fork_rwsem); -} - -void scx_cancel_fork(struct task_struct *p) -{ - if (scx_enabled()) { - struct rq *rq; - struct rq_flags rf; - - rq = task_rq_lock(p, &rf); - WARN_ON_ONCE(scx_get_task_state(p) >= SCX_TASK_READY); - scx_disable_and_exit_task(scx_task_sched(p), p); - task_rq_unlock(rq, p, &rf); - } - - percpu_up_read(&scx_fork_rwsem); -} - -/** - * task_dead_and_done - Is a task dead and done running? - * @p: target task - * - * Once sched_ext_dead() removes the dead task from scx_tasks and exits it, the - * task no longer exists from SCX's POV. However, certain sched_class ops may be - * invoked on these dead tasks leading to failures - e.g. sched_setscheduler() - * may try to switch a task which finished sched_ext_dead() back into SCX - * triggering invalid SCX task state transitions and worse. - * - * Once a task has finished the final switch, sched_ext_dead() is the only thing - * that needs to happen on the task. Use this test to short-circuit sched_class - * operations which may be called on dead tasks. - */ -static bool task_dead_and_done(struct task_struct *p) -{ - struct rq *rq = task_rq(p); - - lockdep_assert_rq_held(rq); - - /* - * In do_task_dead(), a dying task sets %TASK_DEAD with preemption - * disabled and __schedule(). If @p has %TASK_DEAD set and off CPU, @p - * won't ever run again. - */ - return unlikely(READ_ONCE(p->__state) == TASK_DEAD) && - !task_on_cpu(rq, p); -} - -void sched_ext_dead(struct task_struct *p) -{ - /* - * By the time control reaches here, @p has %TASK_DEAD set, switched out - * for the last time and then dropped the rq lock - task_dead_and_done() - * should be returning %true nullifying the straggling sched_class ops. - * Remove from scx_tasks and exit @p. - */ - scoped_guard(raw_spinlock_irqsave, &scx_tasks_lock) { - list_del_init(&p->scx.tasks_node); - if (scx_tid_to_task_enabled()) - rhashtable_remove_fast(&scx_tid_hash, - &p->scx.tid_hash_node, - scx_tid_hash_params); - } - - /* - * @p is off scx_tasks and wholly ours. scx_root_enable()'s READY -> - * ENABLED transitions can't race us. Disable ops for @p. - * - * %SCX_TASK_DEAD synchronizes against cgroup task iteration - see - * scx_task_iter_next_locked(). NONE tasks need no marking: cgroup - * iteration is only used from sub-sched paths, which require root - * enabled. Root enable transitions every live task to at least READY. - * - * %INIT_BEGIN means ops.init_task() is running for @p. Don't call - * into ops; transition to %DEAD so the post-init recheck unwinds - * via scx_sub_init_cancel_task(). - */ - if (scx_get_task_state(p) != SCX_TASK_NONE) { - struct rq_flags rf; - struct rq *rq; - - rq = task_rq_lock(p, &rf); - if (scx_get_task_state(p) != SCX_TASK_INIT_BEGIN) - scx_disable_and_exit_task(scx_task_sched(p), p); - scx_set_task_state(p, SCX_TASK_DEAD); - task_rq_unlock(rq, p, &rf); - } -} - -static void reweight_task_scx(struct rq *rq, struct task_struct *p, - const struct load_weight *lw) -{ - struct scx_sched *sch = scx_task_sched(p); - - lockdep_assert_rq_held(task_rq(p)); - - if (task_dead_and_done(p)) - return; - - p->scx.weight = sched_weight_to_cgroup(scale_load_down(lw->weight)); - if (SCX_HAS_OP(sch, set_weight)) - SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight); -} - -static void prio_changed_scx(struct rq *rq, struct task_struct *p, u64 oldprio) -{ -} - -static void switching_to_scx(struct rq *rq, struct task_struct *p) -{ - struct scx_sched *sch = scx_task_sched(p); - - if (task_dead_and_done(p)) - return; - - scx_enable_task(sch, p); - - /* - * set_cpus_allowed_scx() is not called while @p is associated with a - * different scheduler class. Keep the BPF scheduler up-to-date. - */ - if (SCX_HAS_OP(sch, set_cpumask)) - scx_call_op_set_cpumask(sch, rq, p, (struct cpumask *)p->cpus_ptr); -} - -static void switched_from_scx(struct rq *rq, struct task_struct *p) -{ - if (task_dead_and_done(p)) - return; - - /* - * %NONE means SCX is no longer tracking @p at the task level (e.g. - * scx_fail_parent() handed @p back to the parent at NONE pending the - * parent's own teardown). There is nothing to disable; calling - * scx_disable_task() would WARN on the non-%ENABLED state and trigger a - * NONE -> READY validation failure. - */ - if (scx_get_task_state(p) == SCX_TASK_NONE) - return; - - scx_disable_task(scx_task_sched(p), p); -} - -static void switched_to_scx(struct rq *rq, struct task_struct *p) {} - -int scx_check_setscheduler(struct task_struct *p, int policy) -{ - lockdep_assert_rq_held(task_rq(p)); - - /* if disallow, reject transitioning into SCX */ - if (scx_enabled() && READ_ONCE(p->scx.disallow) && - p->policy != policy && policy == SCHED_EXT) - return -EACCES; - - return 0; -} - -static void process_ddsp_deferred_locals(struct rq *rq) -{ - struct task_struct *p; - - lockdep_assert_rq_held(rq); - - /* - * Now that @rq can be unlocked, execute the deferred enqueueing of - * tasks directly dispatched to the local DSQs of other CPUs. See - * direct_dispatch(). Keep popping from the head instead of using - * list_for_each_entry_safe() as dispatch_local_dsq() may unlock @rq - * temporarily. - */ - while ((p = list_first_entry_or_null(&rq->scx.ddsp_deferred_locals, - struct task_struct, scx.dsq_list.node))) { - struct scx_sched *sch = scx_task_sched(p); - struct scx_dispatch_q *dsq; - u64 dsq_id = p->scx.ddsp_dsq_id; - u64 enq_flags = p->scx.ddsp_enq_flags; - - list_del_init(&p->scx.dsq_list.node); - clear_direct_dispatch(p); - - dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p)); - if (!WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL)) - dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags); - } -} - -/* - * Determine whether @p should be reenqueued from a local DSQ. - * - * @reenq_flags is mutable and accumulates state across the DSQ walk: - * - * - %SCX_REENQ_TSR_NOT_FIRST: Set after the first task is visited. "First" - * tracks position in the DSQ list, not among IMMED tasks. A non-IMMED task at - * the head consumes the first slot. - * - * - %SCX_REENQ_TSR_RQ_OPEN: Set by reenq_local() before the walk if - * rq_is_open() is true. - * - * An IMMED task is kept (returns %false) only if it's the first task in the DSQ - * AND the current task is done — i.e. it will execute immediately. All other - * IMMED tasks are reenqueued. This means if a non-IMMED task sits at the head, - * every IMMED task behind it gets reenqueued. - * - * Reenqueued tasks go through ops.enqueue() with %SCX_ENQ_REENQ | - * %SCX_TASK_REENQ_IMMED. If the BPF scheduler dispatches back to the same local - * DSQ with %SCX_ENQ_IMMED while the CPU is still unavailable, this triggers - * another reenq cycle. Repetitions are bounded by %SCX_REENQ_LOCAL_MAX_REPEAT - * in process_deferred_reenq_locals(). - */ -static bool local_task_should_reenq(struct task_struct *p, u64 *reenq_flags, u32 *reason) -{ - bool first; - - first = !(*reenq_flags & SCX_REENQ_TSR_NOT_FIRST); - *reenq_flags |= SCX_REENQ_TSR_NOT_FIRST; - - *reason = SCX_TASK_REENQ_KFUNC; - - if ((p->scx.flags & SCX_TASK_IMMED) && - (!first || !(*reenq_flags & SCX_REENQ_TSR_RQ_OPEN))) { - __scx_add_event(scx_task_sched(p), SCX_EV_REENQ_IMMED, 1); - *reason = SCX_TASK_REENQ_IMMED; - return true; - } - - return *reenq_flags & SCX_REENQ_ANY; -} - -static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) -{ - LIST_HEAD(tasks); - u32 nr_enqueued = 0; - struct task_struct *p, *n; - - lockdep_assert_rq_held(rq); - - if (WARN_ON_ONCE(reenq_flags & __SCX_REENQ_TSR_MASK)) - reenq_flags &= ~__SCX_REENQ_TSR_MASK; - if (rq_is_open(rq, 0)) - reenq_flags |= SCX_REENQ_TSR_RQ_OPEN; - - /* - * The BPF scheduler may choose to dispatch tasks back to - * @rq->scx.local_dsq. Move all candidate tasks off to a private list - * first to avoid processing the same tasks repeatedly. - */ - list_for_each_entry_safe(p, n, &rq->scx.local_dsq.list, - scx.dsq_list.node) { - struct scx_sched *task_sch = scx_task_sched(p); - u32 reason; - - /* - * If @p is being migrated, @p's current CPU may not agree with - * its allowed CPUs and the migration_cpu_stop is about to - * deactivate and re-activate @p anyway. Skip re-enqueueing. - * - * While racing sched property changes may also dequeue and - * re-enqueue a migrating task while its current CPU and allowed - * CPUs disagree, they use %ENQUEUE_RESTORE which is bypassed to - * the current local DSQ for running tasks and thus are not - * visible to the BPF scheduler. - */ - if (p->migration_pending) - continue; - - if (!scx_is_descendant(task_sch, sch)) - continue; - - if (!local_task_should_reenq(p, &reenq_flags, &reason)) - continue; - - dispatch_dequeue(rq, p); - - if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; - p->scx.flags |= reason; - - list_add_tail(&p->scx.dsq_list.node, &tasks); - } - - list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) { - list_del_init(&p->scx.dsq_list.node); - - do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); - - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; - nr_enqueued++; - } - - return nr_enqueued; -} - -static void process_deferred_reenq_locals(struct rq *rq) -{ - u64 seq = ++rq->scx.deferred_reenq_locals_seq; - - lockdep_assert_rq_held(rq); - - while (true) { - struct scx_sched *sch; - u64 reenq_flags; - bool skip = false; - - scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) { - struct scx_deferred_reenq_local *drl = - list_first_entry_or_null(&rq->scx.deferred_reenq_locals, - struct scx_deferred_reenq_local, - node); - struct scx_sched_pcpu *sch_pcpu; - - if (!drl) - return; - - sch_pcpu = container_of(drl, struct scx_sched_pcpu, - deferred_reenq_local); - sch = sch_pcpu->sch; - - reenq_flags = drl->flags; - WRITE_ONCE(drl->flags, 0); - list_del_init(&drl->node); - - if (likely(drl->seq != seq)) { - drl->seq = seq; - drl->cnt = 0; - } else { - if (unlikely(++drl->cnt > SCX_REENQ_LOCAL_MAX_REPEAT)) { - scx_error(sch, "SCX_ENQ_REENQ on SCX_DSQ_LOCAL repeated %u times", - drl->cnt); - skip = true; - } - - __scx_add_event(sch, SCX_EV_REENQ_LOCAL_REPEAT, 1); - } - } - - if (!skip) { - /* see schedule_dsq_reenq() */ - smp_mb(); - - reenq_local(sch, rq, reenq_flags); - } - } -} - -static bool user_task_should_reenq(struct task_struct *p, u64 reenq_flags, u32 *reason) -{ - *reason = SCX_TASK_REENQ_KFUNC; - return reenq_flags & SCX_REENQ_ANY; -} - -static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flags) -{ - struct rq *locked_rq = rq; - struct scx_sched *sch = dsq->sched; - struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, dsq, 0); - struct task_struct *p; - s32 nr_enqueued = 0; - - lockdep_assert_rq_held(rq); - - raw_spin_lock(&dsq->lock); - - while (likely(!READ_ONCE(sch->bypass_depth))) { - struct rq *task_rq; - u32 reason; - - p = nldsq_cursor_next_task(&cursor, dsq); - if (!p) - break; - - if (!user_task_should_reenq(p, reenq_flags, &reason)) - continue; - - task_rq = task_rq(p); - - if (locked_rq != task_rq) { - if (locked_rq) - raw_spin_rq_unlock(locked_rq); - if (unlikely(!raw_spin_rq_trylock(task_rq))) { - raw_spin_unlock(&dsq->lock); - raw_spin_rq_lock(task_rq); - raw_spin_lock(&dsq->lock); - } - locked_rq = task_rq; - - /* did we lose @p while switching locks? */ - if (nldsq_cursor_lost_task(&cursor, task_rq, dsq, p)) - continue; - } - - /* @p is on @dsq, its rq and @dsq are locked */ - dispatch_dequeue_locked(p, dsq); - raw_spin_unlock(&dsq->lock); - - if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; - p->scx.flags |= reason; - - do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1); - - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; - - if (!(++nr_enqueued % SCX_TASK_ITER_BATCH)) { - raw_spin_rq_unlock(locked_rq); - locked_rq = NULL; - cpu_relax(); - } - - raw_spin_lock(&dsq->lock); - } - - list_del_init(&cursor.node); - raw_spin_unlock(&dsq->lock); - - if (locked_rq != rq) { - if (locked_rq) - raw_spin_rq_unlock(locked_rq); - raw_spin_rq_lock(rq); - } -} - -static void process_deferred_reenq_users(struct rq *rq) -{ - lockdep_assert_rq_held(rq); - - while (true) { - struct scx_dispatch_q *dsq; - u64 reenq_flags; - - scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) { - struct scx_deferred_reenq_user *dru = - list_first_entry_or_null(&rq->scx.deferred_reenq_users, - struct scx_deferred_reenq_user, - node); - struct scx_dsq_pcpu *dsq_pcpu; - - if (!dru) - return; - - dsq_pcpu = container_of(dru, struct scx_dsq_pcpu, - deferred_reenq_user); - dsq = dsq_pcpu->dsq; - reenq_flags = dru->flags; - WRITE_ONCE(dru->flags, 0); - list_del_init(&dru->node); - } - - /* see schedule_dsq_reenq() */ - smp_mb(); - - BUG_ON(dsq->id & SCX_DSQ_FLAG_BUILTIN); - reenq_user(rq, dsq, reenq_flags); - } -} - -static void run_deferred(struct rq *rq) -{ - process_ddsp_deferred_locals(rq); - - if (!list_empty(&rq->scx.deferred_reenq_locals)) - process_deferred_reenq_locals(rq); - - if (!list_empty(&rq->scx.deferred_reenq_users)) - process_deferred_reenq_users(rq); -} - -#ifdef CONFIG_NO_HZ_FULL -bool scx_can_stop_tick(struct rq *rq) -{ - struct task_struct *p = rq->curr; - struct scx_sched *sch = scx_task_sched(p); - - if (p->sched_class != &ext_sched_class) - return true; - - if (scx_bypassing(sch, cpu_of(rq))) - return false; - - /* - * @rq can dispatch from different DSQs, so we can't tell whether it - * needs the tick or not by looking at nr_running. Allow stopping ticks - * iff the BPF scheduler indicated so. See set_next_task_scx(). - */ - return rq->scx.flags & SCX_RQ_CAN_STOP_TICK; -} -#endif - -#ifdef CONFIG_EXT_GROUP_SCHED - -DEFINE_STATIC_PERCPU_RWSEM(scx_cgroup_ops_rwsem); -static bool scx_cgroup_enabled; - -void scx_tg_init(struct task_group *tg) -{ - tg->scx.weight = CGROUP_WEIGHT_DFL; - tg->scx.bw_period_us = default_bw_period_us(); - tg->scx.bw_quota_us = RUNTIME_INF; - tg->scx.idle = false; -} - -int scx_tg_online(struct task_group *tg) -{ - struct scx_sched *sch = scx_root; - int ret = 0; - - WARN_ON_ONCE(tg->scx.flags & (SCX_TG_ONLINE | SCX_TG_INITED)); - - if (scx_cgroup_enabled) { - if (SCX_HAS_OP(sch, cgroup_init)) { - struct scx_cgroup_init_args args = - { .weight = tg->scx.weight, - .bw_period_us = tg->scx.bw_period_us, - .bw_quota_us = tg->scx.bw_quota_us, - .bw_burst_us = tg->scx.bw_burst_us }; - - ret = SCX_CALL_OP_RET(sch, cgroup_init, - NULL, tg->css.cgroup, &args); - if (ret) - ret = ops_sanitize_err(sch, "cgroup_init", ret); - } - if (ret == 0) - tg->scx.flags |= SCX_TG_ONLINE | SCX_TG_INITED; - } else { - tg->scx.flags |= SCX_TG_ONLINE; - } - - return ret; -} - -void scx_tg_offline(struct task_group *tg) -{ - struct scx_sched *sch = scx_root; - - WARN_ON_ONCE(!(tg->scx.flags & SCX_TG_ONLINE)); - - if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_exit) && - (tg->scx.flags & SCX_TG_INITED)) - SCX_CALL_OP(sch, cgroup_exit, NULL, tg->css.cgroup); - tg->scx.flags &= ~(SCX_TG_ONLINE | SCX_TG_INITED); -} - -int scx_cgroup_can_attach(struct cgroup_taskset *tset) -{ - struct scx_sched *sch = scx_root; - struct cgroup_subsys_state *css; - struct task_struct *p; - int ret; - - if (!scx_cgroup_enabled) - return 0; - - cgroup_taskset_for_each(p, css, tset) { - struct cgroup *from = tg_cgrp(task_group(p)); - struct cgroup *to = tg_cgrp(css_tg(css)); - - WARN_ON_ONCE(p->scx.cgrp_moving_from); - - /* - * sched_move_task() omits identity migrations. Let's match the - * behavior so that ops.cgroup_prep_move() and ops.cgroup_move() - * always match one-to-one. - */ - if (from == to) - continue; - - if (SCX_HAS_OP(sch, cgroup_prep_move)) { - ret = SCX_CALL_OP_RET(sch, cgroup_prep_move, NULL, - p, from, css->cgroup); - if (ret) - goto err; - } - - p->scx.cgrp_moving_from = from; - } - - return 0; - -err: - cgroup_taskset_for_each(p, css, tset) { - if (SCX_HAS_OP(sch, cgroup_cancel_move) && - p->scx.cgrp_moving_from) - SCX_CALL_OP(sch, cgroup_cancel_move, NULL, - p, p->scx.cgrp_moving_from, css->cgroup); - p->scx.cgrp_moving_from = NULL; - } - - return ops_sanitize_err(sch, "cgroup_prep_move", ret); -} - -void scx_cgroup_move_task(struct task_struct *p) -{ - struct scx_sched *sch = scx_root; - - if (!scx_cgroup_enabled) - return; - - /* - * scx_cgroup_can_attach() sets cgrp_moving_from only when the task's - * cgroup changes. Migration keys off css rather than cgroup identity, - * so it can hand an unchanged-cgroup task here with cgrp_moving_from - * NULL. Nothing to report to the BPF scheduler then, so skip it and - * keep prep_move and move paired. - */ - if (SCX_HAS_OP(sch, cgroup_move) && p->scx.cgrp_moving_from) - SCX_CALL_OP_TASK(sch, cgroup_move, task_rq(p), - p, p->scx.cgrp_moving_from, - tg_cgrp(task_group(p))); - p->scx.cgrp_moving_from = NULL; -} - -void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) -{ - struct scx_sched *sch = scx_root; - struct cgroup_subsys_state *css; - struct task_struct *p; - - if (!scx_cgroup_enabled) - return; - - cgroup_taskset_for_each(p, css, tset) { - if (SCX_HAS_OP(sch, cgroup_cancel_move) && - p->scx.cgrp_moving_from) - SCX_CALL_OP(sch, cgroup_cancel_move, NULL, - p, p->scx.cgrp_moving_from, css->cgroup); - p->scx.cgrp_moving_from = NULL; - } -} - -void scx_group_set_weight(struct task_group *tg, unsigned long weight) -{ - struct scx_sched *sch; - - percpu_down_read(&scx_cgroup_ops_rwsem); - sch = scx_root; - - if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_weight) && - tg->scx.weight != weight) - SCX_CALL_OP(sch, cgroup_set_weight, NULL, tg_cgrp(tg), weight); - - tg->scx.weight = weight; - - percpu_up_read(&scx_cgroup_ops_rwsem); -} - -void scx_group_set_idle(struct task_group *tg, bool idle) -{ - struct scx_sched *sch; - - percpu_down_read(&scx_cgroup_ops_rwsem); - sch = scx_root; - - if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_idle)) - SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle); - - /* Update the task group's idle state */ - tg->scx.idle = idle; - - percpu_up_read(&scx_cgroup_ops_rwsem); -} - -void scx_group_set_bandwidth(struct task_group *tg, - u64 period_us, u64 quota_us, u64 burst_us) -{ - struct scx_sched *sch; - - percpu_down_read(&scx_cgroup_ops_rwsem); - sch = scx_root; - - if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_bandwidth) && - (tg->scx.bw_period_us != period_us || - tg->scx.bw_quota_us != quota_us || - tg->scx.bw_burst_us != burst_us)) - SCX_CALL_OP(sch, cgroup_set_bandwidth, NULL, - tg_cgrp(tg), period_us, quota_us, burst_us); - - tg->scx.bw_period_us = period_us; - tg->scx.bw_quota_us = quota_us; - tg->scx.bw_burst_us = burst_us; - - percpu_up_read(&scx_cgroup_ops_rwsem); -} -#endif /* CONFIG_EXT_GROUP_SCHED */ - -#if defined(CONFIG_EXT_GROUP_SCHED) || defined(CONFIG_EXT_SUB_SCHED) -static struct cgroup *root_cgroup(void) -{ - return &cgrp_dfl_root.cgrp; -} - -static void scx_cgroup_lock(void) -{ -#ifdef CONFIG_EXT_GROUP_SCHED - percpu_down_write(&scx_cgroup_ops_rwsem); -#endif - cgroup_lock(); -} - -static void scx_cgroup_unlock(void) -{ - cgroup_unlock(); -#ifdef CONFIG_EXT_GROUP_SCHED - percpu_up_write(&scx_cgroup_ops_rwsem); -#endif -} -#else /* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */ -static inline struct cgroup *root_cgroup(void) { return NULL; } -static inline void scx_cgroup_lock(void) {} -static inline void scx_cgroup_unlock(void) {} -#endif /* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */ - -#ifdef CONFIG_EXT_SUB_SCHED -static struct cgroup *sch_cgroup(struct scx_sched *sch) -{ - return sch->cgrp; -} - -/* for each descendant of @cgrp including self, set ->scx_sched to @sch */ -static void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) -{ - struct cgroup *pos; - struct cgroup_subsys_state *css; - - cgroup_for_each_live_descendant_pre(pos, css, cgrp) - rcu_assign_pointer(pos->scx_sched, sch); -} -#else /* CONFIG_EXT_SUB_SCHED */ -static inline struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; } -static inline void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {} -#endif /* CONFIG_EXT_SUB_SCHED */ - -/* - * Omitted operations: - * - * - migrate_task_rq: Unnecessary as task to cpu mapping is transient. - * - * - task_fork/dead: We need fork/dead notifications for all tasks regardless of - * their current sched_class. Call them directly from sched core instead. - */ -DEFINE_SCHED_CLASS(ext) = { - .enqueue_task = enqueue_task_scx, - .dequeue_task = dequeue_task_scx, - .yield_task = yield_task_scx, - .yield_to_task = yield_to_task_scx, - - .wakeup_preempt = wakeup_preempt_scx, - - .pick_task = pick_task_scx, - - .put_prev_task = put_prev_task_scx, - .set_next_task = set_next_task_scx, - - .select_task_rq = select_task_rq_scx, - .task_woken = task_woken_scx, - .set_cpus_allowed = set_cpus_allowed_scx, - - .rq_online = rq_online_scx, - .rq_offline = rq_offline_scx, - - .task_tick = task_tick_scx, - - .switching_to = switching_to_scx, - .switched_from = switched_from_scx, - .switched_to = switched_to_scx, - .reweight_task = reweight_task_scx, - .prio_changed = prio_changed_scx, - - .update_curr = update_curr_scx, - -#ifdef CONFIG_UCLAMP_TASK - .uclamp_enabled = 1, -#endif -}; - -static s32 init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, - struct scx_sched *sch) -{ - s32 cpu; - - memset(dsq, 0, sizeof(*dsq)); - - raw_spin_lock_init(&dsq->lock); - INIT_LIST_HEAD(&dsq->list); - dsq->id = dsq_id; - dsq->sched = sch; - - dsq->pcpu = alloc_percpu(struct scx_dsq_pcpu); - if (!dsq->pcpu) - return -ENOMEM; - - for_each_possible_cpu(cpu) { - struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu); - - pcpu->dsq = dsq; - INIT_LIST_HEAD(&pcpu->deferred_reenq_user.node); - } - - return 0; -} - -static void exit_dsq(struct scx_dispatch_q *dsq) -{ - s32 cpu; - - for_each_possible_cpu(cpu) { - struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu); - struct scx_deferred_reenq_user *dru = &pcpu->deferred_reenq_user; - struct rq *rq = cpu_rq(cpu); - - /* - * There must have been a RCU grace period since the last - * insertion and @dsq should be off the deferred list by now. - */ - if (WARN_ON_ONCE(!list_empty(&dru->node))) { - guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); - list_del_init(&dru->node); - } - } - - free_percpu(dsq->pcpu); -} - -static void free_dsq_rcufn(struct rcu_head *rcu) -{ - struct scx_dispatch_q *dsq = container_of(rcu, struct scx_dispatch_q, rcu); - - exit_dsq(dsq); - kfree(dsq); -} - -static void free_dsq_irq_workfn(struct irq_work *irq_work) -{ - struct llist_node *to_free = llist_del_all(&dsqs_to_free); - struct scx_dispatch_q *dsq, *tmp_dsq; - - llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) - call_rcu(&dsq->rcu, free_dsq_rcufn); -} - -static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); - -static void destroy_dsq(struct scx_sched *sch, u64 dsq_id) -{ - struct scx_dispatch_q *dsq; - unsigned long flags; - - rcu_read_lock(); - - dsq = find_user_dsq(sch, dsq_id); - if (!dsq) - goto out_unlock_rcu; - - raw_spin_lock_irqsave(&dsq->lock, flags); - - if (dsq->nr) { - scx_error(sch, "attempting to destroy in-use dsq 0x%016llx (nr=%u)", - dsq->id, dsq->nr); - goto out_unlock_dsq; - } - - if (rhashtable_remove_fast(&sch->dsq_hash, &dsq->hash_node, - dsq_hash_params)) - goto out_unlock_dsq; - - /* - * Mark dead by invalidating ->id to prevent dispatch_enqueue() from - * queueing more tasks. As this function can be called from anywhere, - * freeing is bounced through an irq work to avoid nesting RCU - * operations inside scheduler locks. - */ - dsq->id = SCX_DSQ_INVALID; - if (llist_add(&dsq->free_node, &dsqs_to_free)) - irq_work_queue(&free_dsq_irq_work); - -out_unlock_dsq: - raw_spin_unlock_irqrestore(&dsq->lock, flags); -out_unlock_rcu: - rcu_read_unlock(); -} - -#ifdef CONFIG_EXT_GROUP_SCHED -static void scx_cgroup_exit(struct scx_sched *sch) -{ - struct cgroup_subsys_state *css; - - scx_cgroup_enabled = false; - - /* - * scx_tg_on/offline() are excluded through cgroup_lock(). If we walk - * cgroups and exit all the inited ones, all online cgroups are exited. - */ - css_for_each_descendant_post(css, &root_task_group.css) { - struct task_group *tg = css_tg(css); - - if (!(tg->scx.flags & SCX_TG_INITED)) - continue; - tg->scx.flags &= ~SCX_TG_INITED; - - if (!sch->ops.cgroup_exit) - continue; - - SCX_CALL_OP(sch, cgroup_exit, NULL, css->cgroup); - } -} - -static int scx_cgroup_init(struct scx_sched *sch) -{ - struct cgroup_subsys_state *css; - int ret; - - /* - * scx_tg_on/offline() are excluded through cgroup_lock(). If we walk - * cgroups and init, all online cgroups are initialized. - */ - css_for_each_descendant_pre(css, &root_task_group.css) { - struct task_group *tg = css_tg(css); - struct scx_cgroup_init_args args = { - .weight = tg->scx.weight, - .bw_period_us = tg->scx.bw_period_us, - .bw_quota_us = tg->scx.bw_quota_us, - .bw_burst_us = tg->scx.bw_burst_us, - }; - - if ((tg->scx.flags & - (SCX_TG_ONLINE | SCX_TG_INITED)) != SCX_TG_ONLINE) - continue; - - if (!sch->ops.cgroup_init) { - tg->scx.flags |= SCX_TG_INITED; - continue; - } - - ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, - css->cgroup, &args); - if (ret) { - scx_error(sch, "ops.cgroup_init() failed (%d)", ret); - return ret; - } - tg->scx.flags |= SCX_TG_INITED; - } - - WARN_ON_ONCE(scx_cgroup_enabled); - scx_cgroup_enabled = true; - - return 0; -} - -#else -static void scx_cgroup_exit(struct scx_sched *sch) {} -static int scx_cgroup_init(struct scx_sched *sch) { return 0; } -#endif - - -/******************************************************************************** - * Sysfs interface and ops enable/disable. - */ - -#define SCX_ATTR(_name) \ - static struct kobj_attribute scx_attr_##_name = { \ - .attr = { .name = __stringify(_name), .mode = 0444 }, \ - .show = scx_attr_##_name##_show, \ - } - -static ssize_t scx_attr_state_show(struct kobject *kobj, - struct kobj_attribute *ka, char *buf) -{ - return sysfs_emit(buf, "%s\n", scx_enable_state_str[scx_enable_state()]); -} -SCX_ATTR(state); - -static ssize_t scx_attr_switch_all_show(struct kobject *kobj, - struct kobj_attribute *ka, char *buf) -{ - return sysfs_emit(buf, "%d\n", READ_ONCE(scx_switching_all)); -} -SCX_ATTR(switch_all); - -static ssize_t scx_attr_nr_rejected_show(struct kobject *kobj, - struct kobj_attribute *ka, char *buf) -{ - return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_nr_rejected)); -} -SCX_ATTR(nr_rejected); - -static ssize_t scx_attr_hotplug_seq_show(struct kobject *kobj, - struct kobj_attribute *ka, char *buf) -{ - return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_hotplug_seq)); -} -SCX_ATTR(hotplug_seq); - -static ssize_t scx_attr_enable_seq_show(struct kobject *kobj, - struct kobj_attribute *ka, char *buf) -{ - return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_enable_seq)); -} -SCX_ATTR(enable_seq); - -static struct attribute *scx_global_attrs[] = { - &scx_attr_state.attr, - &scx_attr_switch_all.attr, - &scx_attr_nr_rejected.attr, - &scx_attr_hotplug_seq.attr, - &scx_attr_enable_seq.attr, - NULL, -}; - -static const struct attribute_group scx_global_attr_group = { - .attrs = scx_global_attrs, -}; - -static void free_pnode(struct scx_sched_pnode *pnode); -static void free_exit_info(struct scx_exit_info *ei); - -static s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch) -{ - size_t size = struct_size_t(struct scx_cmask, bits, - SCX_CMASK_NR_WORDS(num_possible_cpus())); - int cpu; - - if (!sch->is_cid_type || !sch->arena_pool) - return 0; - - sch->set_cmask_scratch = alloc_percpu(struct scx_cmask *); - if (!sch->set_cmask_scratch) - return -ENOMEM; - - for_each_possible_cpu(cpu) { - struct scx_cmask **slot = per_cpu_ptr(sch->set_cmask_scratch, cpu); - - *slot = scx_arena_alloc(sch, size); - if (!*slot) - return -ENOMEM; - scx_cmask_init(*slot, 0, num_possible_cpus()); - } - return 0; -} - -static void scx_set_cmask_scratch_free(struct scx_sched *sch) -{ - size_t size = struct_size_t(struct scx_cmask, bits, - SCX_CMASK_NR_WORDS(num_possible_cpus())); - int cpu; - - if (!sch->set_cmask_scratch) - return; - - for_each_possible_cpu(cpu) { - struct scx_cmask **slot = per_cpu_ptr(sch->set_cmask_scratch, cpu); - - scx_arena_free(sch, *slot, size); - } - free_percpu(sch->set_cmask_scratch); - sch->set_cmask_scratch = NULL; -} - -static void scx_sched_free_rcu_work(struct work_struct *work) -{ - struct rcu_work *rcu_work = to_rcu_work(work); - struct scx_sched *sch = container_of(rcu_work, struct scx_sched, rcu_work); - struct rhashtable_iter rht_iter; - struct scx_dispatch_q *dsq; - int cpu, node; - - irq_work_sync(&sch->disable_irq_work); - kthread_destroy_worker(sch->helper); - timer_shutdown_sync(&sch->bypass_lb_timer); - free_cpumask_var(sch->bypass_lb_donee_cpumask); - free_cpumask_var(sch->bypass_lb_resched_cpumask); - -#ifdef CONFIG_EXT_SUB_SCHED - kfree(sch->cgrp_path); - if (sch_cgroup(sch)) - cgroup_put(sch_cgroup(sch)); - if (sch->sub_kset) - kobject_put(&sch->sub_kset->kobj); -#endif /* CONFIG_EXT_SUB_SCHED */ - - for_each_possible_cpu(cpu) { - struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); - - /* - * $sch would have entered bypass mode before the RCU grace - * period. As that blocks new deferrals, all - * deferred_reenq_local_node's must be off-list by now. - */ - WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local.node)); - - exit_dsq(bypass_dsq(sch, cpu)); - } - - free_percpu(sch->pcpu); - - for_each_node_state(node, N_POSSIBLE) - free_pnode(sch->pnode[node]); - kfree(sch->pnode); - - rhashtable_walk_enter(&sch->dsq_hash, &rht_iter); - do { - rhashtable_walk_start(&rht_iter); - - while (!IS_ERR_OR_NULL((dsq = rhashtable_walk_next(&rht_iter)))) - destroy_dsq(sch, dsq->id); - - rhashtable_walk_stop(&rht_iter); - } while (dsq == ERR_PTR(-EAGAIN)); - rhashtable_walk_exit(&rht_iter); - - rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL); - free_exit_info(sch->exit_info); - scx_set_cmask_scratch_free(sch); - scx_arena_pool_destroy(sch); - if (sch->arena_map) - bpf_map_put(sch->arena_map); - kfree(sch); -} - -static void scx_kobj_release(struct kobject *kobj) -{ - struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); - - INIT_RCU_WORK(&sch->rcu_work, scx_sched_free_rcu_work); - queue_rcu_work(system_dfl_wq, &sch->rcu_work); -} - -static ssize_t scx_attr_ops_show(struct kobject *kobj, - struct kobj_attribute *ka, char *buf) -{ - struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); - - return sysfs_emit(buf, "%s\n", sch->ops.name); -} -SCX_ATTR(ops); - -#define scx_attr_event_show(buf, at, events, kind) ({ \ - sysfs_emit_at(buf, at, "%s %llu\n", #kind, (events)->kind); \ -}) - -static ssize_t scx_attr_events_show(struct kobject *kobj, - struct kobj_attribute *ka, char *buf) -{ - struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); - struct scx_event_stats events; - int at = 0; - - scx_read_events(sch, &events); - at += scx_attr_event_show(buf, at, &events, SCX_EV_SELECT_CPU_FALLBACK); - at += scx_attr_event_show(buf, at, &events, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); - at += scx_attr_event_show(buf, at, &events, SCX_EV_DISPATCH_KEEP_LAST); - at += scx_attr_event_show(buf, at, &events, SCX_EV_ENQ_SKIP_EXITING); - at += scx_attr_event_show(buf, at, &events, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); - at += scx_attr_event_show(buf, at, &events, SCX_EV_REENQ_IMMED); - at += scx_attr_event_show(buf, at, &events, SCX_EV_REENQ_LOCAL_REPEAT); - at += scx_attr_event_show(buf, at, &events, SCX_EV_REFILL_SLICE_DFL); - at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_DURATION); - at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_DISPATCH); - at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_ACTIVATE); - at += scx_attr_event_show(buf, at, &events, SCX_EV_INSERT_NOT_OWNED); - at += scx_attr_event_show(buf, at, &events, SCX_EV_SUB_BYPASS_DISPATCH); - return at; -} -SCX_ATTR(events); - -static struct attribute *scx_sched_attrs[] = { - &scx_attr_ops.attr, - &scx_attr_events.attr, - NULL, -}; -ATTRIBUTE_GROUPS(scx_sched); - -static const struct kobj_type scx_ktype = { - .release = scx_kobj_release, - .sysfs_ops = &kobj_sysfs_ops, - .default_groups = scx_sched_groups, -}; - -static int scx_uevent(const struct kobject *kobj, struct kobj_uevent_env *env) -{ - const struct scx_sched *sch; - - /* - * scx_uevent() can be reached by both scx_sched kobjects (scx_ktype) - * and sub-scheduler kset kobjects (kset_ktype) through the parent - * chain walk. Filter out the latter to avoid invalid casts. - */ - if (kobj->ktype != &scx_ktype) - return 0; - - sch = container_of(kobj, struct scx_sched, kobj); - - return add_uevent_var(env, "SCXOPS=%s", sch->ops.name); -} - -static const struct kset_uevent_ops scx_uevent_ops = { - .uevent = scx_uevent, -}; - -/* - * Used by sched_fork() and __setscheduler_prio() to pick the matching - * sched_class. dl/rt are already handled. - */ -bool task_should_scx(int policy) -{ - /* if disabled, nothing should be on it */ - if (!scx_enabled()) - return false; - - /* scx is taking over all SCHED_OTHER and SCHED_EXT tasks */ - if (READ_ONCE(scx_switching_all)) - return true; - - /* - * scx is tearing down - keep new SCHED_EXT tasks out. - * - * Must come after scx_switching_all test, which serves as a proxy - * for __scx_switched_all. While __scx_switched_all is set, we must - * return true via the branch above: a fork routed to fair would - * stall because next_active_class() skips fair. - * - * This can develop into a deadlock - scx holds scx_enable_mutex across - * kthread_create() in scx_alloc_and_add_sched(); if the new kthread is - * the stalled task, the disable path can never grab the mutex to clear - * scx_switching_all. - */ - if (unlikely(scx_enable_state() == SCX_DISABLING)) - return false; - - return policy == SCHED_EXT; -} - -bool scx_allow_ttwu_queue(const struct task_struct *p) -{ - struct scx_sched *sch; - - if (!scx_enabled()) - return true; - - sch = scx_task_sched(p); - if (unlikely(!sch)) - return true; - - if (sch->ops.flags & SCX_OPS_ALLOW_QUEUED_WAKEUP) - return true; - - if (unlikely(p->sched_class != &ext_sched_class)) - return true; - - return false; -} - -/** - * handle_lockup - sched_ext common lockup handler - * @fmt: format string - * - * Called on system stall or lockup condition and initiates abort of sched_ext - * if enabled, which may resolve the reported lockup. - * - * Returns %true if sched_ext is enabled and abort was initiated, which may - * resolve the lockup. %false if sched_ext is not enabled or abort was already - * initiated by someone else. - */ -static __printf(1, 2) bool handle_lockup(const char *fmt, ...) -{ - struct scx_sched *sch; - va_list args; - bool ret; - - guard(rcu)(); - - sch = rcu_dereference(scx_root); - if (unlikely(!sch)) - return false; - - switch (scx_enable_state()) { - case SCX_ENABLING: - case SCX_ENABLED: - va_start(args, fmt); - ret = scx_verror(sch, fmt, args); - va_end(args); - return ret; - default: - return false; - } -} - -/** - * scx_rcu_cpu_stall - sched_ext RCU CPU stall handler - * - * While there are various reasons why RCU CPU stalls can occur on a system - * that may not be caused by the current BPF scheduler, try kicking out the - * current scheduler in an attempt to recover the system to a good state before - * issuing panics. - * - * Returns %true if sched_ext is enabled and abort was initiated, which may - * resolve the reported RCU stall. %false if sched_ext is not enabled or someone - * else already initiated abort. - */ -bool scx_rcu_cpu_stall(void) -{ - return handle_lockup("RCU CPU stall detected!"); -} - -/** - * scx_softlockup - sched_ext softlockup handler - * @dur_s: number of seconds of CPU stuck due to soft lockup - * - * On some multi-socket setups (e.g. 2x Intel 8480c), the BPF scheduler can - * live-lock the system by making many CPUs target the same DSQ to the point - * where soft-lockup detection triggers. This function is called from - * soft-lockup watchdog when the triggering point is close and tries to unjam - * the system and aborting the BPF scheduler. - */ -void scx_softlockup(u32 dur_s) -{ - if (!handle_lockup("soft lockup - CPU %d stuck for %us", smp_processor_id(), dur_s)) - return; - - printk_deferred(KERN_ERR "sched_ext: Soft lockup - CPU %d stuck for %us, disabling BPF scheduler\n", - smp_processor_id(), dur_s); -} - -/* - * scx_hardlockup() runs from NMI and eventually calls scx_claim_exit(), - * which takes scx_sched_lock. scx_sched_lock isn't NMI-safe and grabbing - * it from NMI context can lead to deadlocks. Defer via irq_work; the - * disable path runs off irq_work anyway. - */ -static atomic_t scx_hardlockup_cpu = ATOMIC_INIT(-1); - -static void scx_hardlockup_irq_workfn(struct irq_work *work) -{ - int cpu = atomic_xchg(&scx_hardlockup_cpu, -1); - - if (cpu >= 0 && handle_lockup("hard lockup - CPU %d", cpu)) - printk_deferred(KERN_ERR "sched_ext: Hard lockup - CPU %d, disabling BPF scheduler\n", - cpu); -} - -static DEFINE_IRQ_WORK(scx_hardlockup_irq_work, scx_hardlockup_irq_workfn); - -/** - * scx_hardlockup - sched_ext hardlockup handler - * - * A poorly behaving BPF scheduler can trigger hard lockup by e.g. putting - * numerous affinitized tasks in a single queue and directing all CPUs at it. - * Try kicking out the current scheduler in an attempt to recover the system to - * a good state before taking more drastic actions. - * - * Queues an irq_work; the handle_lockup() call happens in IRQ context (see - * scx_hardlockup_irq_workfn). - * - * Returns %true if sched_ext is enabled and the work was queued, %false - * otherwise. - */ -bool scx_hardlockup(int cpu) -{ - if (!rcu_access_pointer(scx_root)) - return false; - - atomic_cmpxchg(&scx_hardlockup_cpu, -1, cpu); - irq_work_queue(&scx_hardlockup_irq_work); - return true; -} - -static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, - struct cpumask *donee_mask, struct cpumask *resched_mask, - u32 nr_donor_target, u32 nr_donee_target) -{ - struct rq *donor_rq = cpu_rq(donor); - struct scx_dispatch_q *donor_dsq = bypass_dsq(sch, donor); - struct task_struct *p, *n; - struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, donor_dsq, 0); - s32 delta = READ_ONCE(donor_dsq->nr) - nr_donor_target; - u32 nr_balanced = 0, min_delta_us; - - /* - * All we want to guarantee is reasonable forward progress. No reason to - * fine tune. Assuming every task on @donor_dsq runs their full slice, - * consider offloading iff the total queued duration is over the - * threshold. - */ - min_delta_us = READ_ONCE(scx_bypass_lb_intv_us) / SCX_BYPASS_LB_MIN_DELTA_DIV; - if (delta < DIV_ROUND_UP(min_delta_us, READ_ONCE(scx_slice_bypass_us))) - return 0; - - raw_spin_rq_lock_irq(donor_rq); - raw_spin_lock(&donor_dsq->lock); - list_add(&cursor.node, &donor_dsq->list); -resume: - n = container_of(&cursor, struct task_struct, scx.dsq_list); - n = nldsq_next_task(donor_dsq, n, false); - - while ((p = n)) { - struct scx_dispatch_q *donee_dsq; - int donee; - - n = nldsq_next_task(donor_dsq, n, false); - - if (donor_dsq->nr <= nr_donor_target) - break; - - if (cpumask_empty(donee_mask)) - break; - - /* - * If an earlier pass placed @p on @donor_dsq from a different - * CPU and the donee hasn't consumed it yet, @p is still on the - * previous CPU and task_rq(@p) != @donor_rq. @p can't be moved - * without its rq locked. Skip. - */ - if (task_rq(p) != donor_rq) - continue; - - donee = cpumask_any_and_distribute(donee_mask, p->cpus_ptr); - if (donee >= nr_cpu_ids) - continue; - - donee_dsq = bypass_dsq(sch, donee); - - /* - * $p's rq is not locked but $p's DSQ lock protects its - * scheduling properties making this test safe. - */ - if (!task_can_run_on_remote_rq(sch, p, cpu_rq(donee), false)) - continue; - - /* - * Moving $p from one non-local DSQ to another. The source rq - * and DSQ are already locked. Do an abbreviated dequeue and - * then perform enqueue without unlocking $donor_dsq. - * - * We don't want to drop and reacquire the lock on each - * iteration as @donor_dsq can be very long and potentially - * highly contended. Donee DSQs are less likely to be contended. - * The nested locking is safe as only this LB moves tasks - * between bypass DSQs. - */ - dispatch_dequeue_locked(p, donor_dsq); - dispatch_enqueue(sch, cpu_rq(donee), donee_dsq, p, SCX_ENQ_NESTED); - - /* - * $donee might have been idle and need to be woken up. No need - * to be clever. Kick every CPU that receives tasks. - */ - cpumask_set_cpu(donee, resched_mask); - - if (READ_ONCE(donee_dsq->nr) >= nr_donee_target) - cpumask_clear_cpu(donee, donee_mask); - - nr_balanced++; - if (!(nr_balanced % SCX_BYPASS_LB_BATCH) && n) { - list_move_tail(&cursor.node, &n->scx.dsq_list.node); - raw_spin_unlock(&donor_dsq->lock); - raw_spin_rq_unlock_irq(donor_rq); - cpu_relax(); - raw_spin_rq_lock_irq(donor_rq); - raw_spin_lock(&donor_dsq->lock); - goto resume; - } - } - - list_del_init(&cursor.node); - raw_spin_unlock(&donor_dsq->lock); - raw_spin_rq_unlock_irq(donor_rq); - - return nr_balanced; -} - -static void bypass_lb_node(struct scx_sched *sch, int node) -{ - const struct cpumask *node_mask = cpumask_of_node(node); - struct cpumask *donee_mask = sch->bypass_lb_donee_cpumask; - struct cpumask *resched_mask = sch->bypass_lb_resched_cpumask; - u32 nr_tasks = 0, nr_cpus = 0, nr_balanced = 0; - u32 nr_target, nr_donor_target; - u32 before_min = U32_MAX, before_max = 0; - u32 after_min = U32_MAX, after_max = 0; - int cpu; - - /* count the target tasks and CPUs */ - for_each_cpu_and(cpu, cpu_online_mask, node_mask) { - u32 nr = READ_ONCE(bypass_dsq(sch, cpu)->nr); - - nr_tasks += nr; - nr_cpus++; - - before_min = min(nr, before_min); - before_max = max(nr, before_max); - } - - if (!nr_cpus) - return; - - /* - * We don't want CPUs to have more than $nr_donor_target tasks and - * balancing to fill donee CPUs upto $nr_target. Once targets are - * calculated, find the donee CPUs. - */ - nr_target = DIV_ROUND_UP(nr_tasks, nr_cpus); - nr_donor_target = DIV_ROUND_UP(nr_target * SCX_BYPASS_LB_DONOR_PCT, 100); - - cpumask_clear(donee_mask); - for_each_cpu_and(cpu, cpu_online_mask, node_mask) { - if (READ_ONCE(bypass_dsq(sch, cpu)->nr) < nr_target) - cpumask_set_cpu(cpu, donee_mask); - } - - /* iterate !donee CPUs and see if they should be offloaded */ - cpumask_clear(resched_mask); - for_each_cpu_and(cpu, cpu_online_mask, node_mask) { - if (cpumask_empty(donee_mask)) - break; - if (cpumask_test_cpu(cpu, donee_mask)) - continue; - if (READ_ONCE(bypass_dsq(sch, cpu)->nr) <= nr_donor_target) - continue; - - nr_balanced += bypass_lb_cpu(sch, cpu, donee_mask, resched_mask, - nr_donor_target, nr_target); - } - - for_each_cpu(cpu, resched_mask) - resched_cpu(cpu); - - for_each_cpu_and(cpu, cpu_online_mask, node_mask) { - u32 nr = READ_ONCE(bypass_dsq(sch, cpu)->nr); - - after_min = min(nr, after_min); - after_max = max(nr, after_max); - - } - - trace_sched_ext_bypass_lb(node, nr_cpus, nr_tasks, nr_balanced, - before_min, before_max, after_min, after_max); -} - -/* - * In bypass mode, all tasks are put on the per-CPU bypass DSQs. If the machine - * is over-saturated and the BPF scheduler skewed tasks into few CPUs, some - * bypass DSQs can be overloaded. If there are enough tasks to saturate other - * lightly loaded CPUs, such imbalance can lead to very high execution latency - * on the overloaded CPUs and thus to hung tasks and RCU stalls. To avoid such - * outcomes, a simple load balancing mechanism is implemented by the following - * timer which runs periodically while bypass mode is in effect. - */ -static void scx_bypass_lb_timerfn(struct timer_list *timer) -{ - struct scx_sched *sch = container_of(timer, struct scx_sched, bypass_lb_timer); - int node; - u32 intv_us; - - if (!bypass_dsp_enabled(sch)) - return; - - for_each_node_with_cpus(node) - bypass_lb_node(sch, node); - - intv_us = READ_ONCE(scx_bypass_lb_intv_us); - if (intv_us) - mod_timer(timer, jiffies + usecs_to_jiffies(intv_us)); -} - -static bool inc_bypass_depth(struct scx_sched *sch) -{ - lockdep_assert_held(&scx_bypass_lock); - - WARN_ON_ONCE(sch->bypass_depth < 0); - WRITE_ONCE(sch->bypass_depth, sch->bypass_depth + 1); - if (sch->bypass_depth != 1) - return false; - - WRITE_ONCE(sch->slice_dfl, READ_ONCE(scx_slice_bypass_us) * NSEC_PER_USEC); - sch->bypass_timestamp = ktime_get_ns(); - scx_add_event(sch, SCX_EV_BYPASS_ACTIVATE, 1); - return true; -} - -static bool dec_bypass_depth(struct scx_sched *sch) -{ - lockdep_assert_held(&scx_bypass_lock); - - WARN_ON_ONCE(sch->bypass_depth < 1); - WRITE_ONCE(sch->bypass_depth, sch->bypass_depth - 1); - if (sch->bypass_depth != 0) - return false; - - WRITE_ONCE(sch->slice_dfl, SCX_SLICE_DFL); - scx_add_event(sch, SCX_EV_BYPASS_DURATION, - ktime_get_ns() - sch->bypass_timestamp); - return true; -} - -static void enable_bypass_dsp(struct scx_sched *sch) -{ - struct scx_sched *host = scx_parent(sch) ?: sch; - u32 intv_us = READ_ONCE(scx_bypass_lb_intv_us); - s32 ret; - - /* - * @sch->bypass_depth transitioning from 0 to 1 triggers enabling. - * Shouldn't stagger. - */ - if (WARN_ON_ONCE(test_and_set_bit(0, &sch->bypass_dsp_claim))) - return; - - /* - * When a sub-sched bypasses, its tasks are queued on the bypass DSQs of - * the nearest non-bypassing ancestor or root. As enable_bypass_dsp() is - * called iff @sch is not already bypassed due to an ancestor bypassing, - * we can assume that the parent is not bypassing and thus will be the - * host of the bypass DSQs. - * - * While the situation may change in the future, the following - * guarantees that the nearest non-bypassing ancestor or root has bypass - * dispatch enabled while a descendant is bypassing, which is all that's - * required. - * - * bypass_dsp_enabled() test is used to determine whether to enter the - * bypass dispatch handling path from both bypassing and hosting scheds. - * Bump enable depth on both @sch and bypass dispatch host. - */ - ret = atomic_inc_return(&sch->bypass_dsp_enable_depth); - WARN_ON_ONCE(ret <= 0); - - if (host != sch) { - ret = atomic_inc_return(&host->bypass_dsp_enable_depth); - WARN_ON_ONCE(ret <= 0); - } - - /* - * The LB timer will stop running if bypass dispatch is disabled. Start - * after enabling bypass dispatch. - */ - if (intv_us && !timer_pending(&host->bypass_lb_timer)) - mod_timer(&host->bypass_lb_timer, - jiffies + usecs_to_jiffies(intv_us)); -} - -/* may be called without holding scx_bypass_lock */ -static void disable_bypass_dsp(struct scx_sched *sch) -{ - s32 ret; - - if (!test_and_clear_bit(0, &sch->bypass_dsp_claim)) - return; - - ret = atomic_dec_return(&sch->bypass_dsp_enable_depth); - WARN_ON_ONCE(ret < 0); - - if (scx_parent(sch)) { - ret = atomic_dec_return(&scx_parent(sch)->bypass_dsp_enable_depth); - WARN_ON_ONCE(ret < 0); - } -} - -/** - * scx_bypass - [Un]bypass scx_ops and guarantee forward progress - * @sch: sched to bypass - * @bypass: true for bypass, false for unbypass - * - * Bypassing guarantees that all runnable tasks make forward progress without - * trusting the BPF scheduler. We can't grab any mutexes or rwsems as they might - * be held by tasks that the BPF scheduler is forgetting to run, which - * unfortunately also excludes toggling the static branches. - * - * Let's work around by overriding a couple ops and modifying behaviors based on - * the DISABLING state and then cycling the queued tasks through dequeue/enqueue - * to force global FIFO scheduling. - * - * - ops.select_cpu() is ignored and the default select_cpu() is used. - * - * - ops.enqueue() is ignored and tasks are queued in simple global FIFO order. - * %SCX_OPS_ENQ_LAST is also ignored. - * - * - ops.dispatch() is ignored. - * - * - balance_one() does not set %SCX_RQ_BAL_KEEP on non-zero slice as slice - * can't be trusted. Whenever a tick triggers, the running task is rotated to - * the tail of the queue with core_sched_at touched. - * - * - pick_next_task() suppresses zero slice warning. - * - * - scx_kick_cpu() is disabled to avoid irq_work malfunction during PM - * operations. - * - * - scx_prio_less() reverts to the default core_sched_at order. - */ -static void scx_bypass(struct scx_sched *sch, bool bypass) -{ - struct scx_sched *pos; - unsigned long flags; - int cpu; - - raw_spin_lock_irqsave(&scx_bypass_lock, flags); - - if (bypass) { - if (!inc_bypass_depth(sch)) - goto unlock; - - enable_bypass_dsp(sch); - } else { - if (!dec_bypass_depth(sch)) - goto unlock; - } - - /* - * Bypass state is propagated to all descendants - an scx_sched bypasses - * if itself or any of its ancestors are in bypass mode. - */ - raw_spin_lock(&scx_sched_lock); - scx_for_each_descendant_pre(pos, sch) { - if (pos == sch) - continue; - if (bypass) - inc_bypass_depth(pos); - else - dec_bypass_depth(pos); - } - raw_spin_unlock(&scx_sched_lock); - - /* - * No task property is changing. We just need to make sure all currently - * queued tasks are re-queued according to the new scx_bypassing() - * state. As an optimization, walk each rq's runnable_list instead of - * the scx_tasks list. - * - * This function can't trust the scheduler and thus can't use - * cpus_read_lock(). Walk all possible CPUs instead of online. - */ - for_each_possible_cpu(cpu) { - struct rq *rq = cpu_rq(cpu); - struct task_struct *p, *n; - - raw_spin_rq_lock(rq); - raw_spin_lock(&scx_sched_lock); - - scx_for_each_descendant_pre(pos, sch) { - struct scx_sched_pcpu *pcpu = per_cpu_ptr(pos->pcpu, cpu); - - if (pos->bypass_depth) - pcpu->flags |= SCX_SCHED_PCPU_BYPASSING; - else - pcpu->flags &= ~SCX_SCHED_PCPU_BYPASSING; - } - - raw_spin_unlock(&scx_sched_lock); - - /* - * We need to guarantee that no tasks are on the BPF scheduler - * while bypassing. Either we see enabled or the enable path - * sees scx_bypassing() before moving tasks to SCX. - */ - if (!scx_enabled()) { - raw_spin_rq_unlock(rq); - continue; - } - - /* - * The use of list_for_each_entry_safe_reverse() is required - * because each task is going to be removed from and added back - * to the runnable_list during iteration. Because they're added - * to the tail of the list, safe reverse iteration can still - * visit all nodes. - */ - list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list, - scx.runnable_node) { - if (!scx_is_descendant(scx_task_sched(p), sch)) - continue; - - /* cycling deq/enq is enough, see the function comment */ - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { - /* nothing */ ; - } - } - - /* resched to restore ticks and idle state */ - if (cpu_online(cpu) || cpu == smp_processor_id()) - resched_curr(rq); - - raw_spin_rq_unlock(rq); - } - - /* disarming must come after moving all tasks out of the bypass DSQs */ - if (!bypass) - disable_bypass_dsp(sch); -unlock: - raw_spin_unlock_irqrestore(&scx_bypass_lock, flags); -} - -static void free_exit_info(struct scx_exit_info *ei) -{ - kvfree(ei->dump); - kfree(ei->msg); - kfree(ei->bt); - kfree(ei); -} - -static struct scx_exit_info *alloc_exit_info(size_t exit_dump_len) -{ - struct scx_exit_info *ei; - - ei = kzalloc_obj(*ei); - if (!ei) - return NULL; - - ei->exit_cpu = -1; - ei->bt = kzalloc_objs(ei->bt[0], SCX_EXIT_BT_LEN); - ei->msg = kzalloc(SCX_EXIT_MSG_LEN, GFP_KERNEL); - ei->dump = kvzalloc(exit_dump_len, GFP_KERNEL); - - if (!ei->bt || !ei->msg || !ei->dump) { - free_exit_info(ei); - return NULL; - } - - return ei; -} - -static const char *scx_exit_reason(enum scx_exit_kind kind) -{ - switch (kind) { - case SCX_EXIT_UNREG: - return "unregistered from user space"; - case SCX_EXIT_UNREG_BPF: - return "unregistered from BPF"; - case SCX_EXIT_UNREG_KERN: - return "unregistered from the main kernel"; - case SCX_EXIT_SYSRQ: - return "disabled by sysrq-S"; - case SCX_EXIT_PARENT: - return "parent exiting"; - case SCX_EXIT_ERROR: - return "runtime error"; - case SCX_EXIT_ERROR_BPF: - return "scx_bpf_error"; - case SCX_EXIT_ERROR_STALL: - return "runnable task stall"; - default: - return ""; - } -} - -static void free_kick_syncs(void) -{ - int cpu; - - for_each_possible_cpu(cpu) { - struct scx_kick_syncs **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu); - struct scx_kick_syncs *to_free; - - to_free = rcu_replace_pointer(*ksyncs, NULL, true); - if (to_free) - kvfree_rcu(to_free, rcu); - } -} - -static void refresh_watchdog(void) -{ - struct scx_sched *sch; - unsigned long intv = ULONG_MAX; - - /* take the shortest timeout and use its half for watchdog interval */ - rcu_read_lock(); - list_for_each_entry_rcu(sch, &scx_sched_all, all) - intv = max(min(intv, sch->watchdog_timeout / 2), 1); - rcu_read_unlock(); - - WRITE_ONCE(scx_watchdog_timestamp, jiffies); - WRITE_ONCE(scx_watchdog_interval, intv); - - if (intv < ULONG_MAX) - mod_delayed_work(system_dfl_wq, &scx_watchdog_work, intv); - else - cancel_delayed_work_sync(&scx_watchdog_work); -} - -static s32 scx_link_sched(struct scx_sched *sch) -{ - const char *err_msg = ""; - s32 ret = 0; - - scoped_guard(raw_spinlock_irq, &scx_sched_lock) { -#ifdef CONFIG_EXT_SUB_SCHED - struct scx_sched *parent = scx_parent(sch); - - if (parent) { - /* - * scx_claim_exit() propagates exit_kind transition to - * its sub-scheds while holding scx_sched_lock - either - * we can see the parent's non-NONE exit_kind or the - * parent can shoot us down. - */ - if (atomic_read(&parent->exit_kind) != SCX_EXIT_NONE) { - err_msg = "parent disabled"; - ret = -ENOENT; - break; - } - - ret = rhashtable_lookup_insert_fast(&scx_sched_hash, - &sch->hash_node, scx_sched_hash_params); - if (ret) { - err_msg = "failed to insert into scx_sched_hash"; - break; - } - - list_add_tail(&sch->sibling, &parent->children); - } -#endif /* CONFIG_EXT_SUB_SCHED */ - - list_add_tail_rcu(&sch->all, &scx_sched_all); - } - - /* - * scx_error() takes scx_sched_lock via scx_claim_exit(), so it must run after - * the guard above is released. - */ - if (ret) { - scx_error(sch, "%s (%d)", err_msg, ret); - return ret; - } - - refresh_watchdog(); - return 0; -} - -static void scx_unlink_sched(struct scx_sched *sch) -{ - scoped_guard(raw_spinlock_irq, &scx_sched_lock) { -#ifdef CONFIG_EXT_SUB_SCHED - if (scx_parent(sch)) { - rhashtable_remove_fast(&scx_sched_hash, &sch->hash_node, - scx_sched_hash_params); - list_del_init(&sch->sibling); - } -#endif /* CONFIG_EXT_SUB_SCHED */ - list_del_rcu(&sch->all); - } - - refresh_watchdog(); -} - -/* - * Called to disable future dumps and wait for in-progress one while disabling - * @sch. Once @sch becomes empty during disable, there's no point in dumping it. - * This prevents calling dump ops on a dead sch. - */ -static void scx_disable_dump(struct scx_sched *sch) -{ - guard(raw_spinlock_irqsave)(&scx_dump_lock); - sch->dump_disabled = true; -} - -static void scx_log_sched_disable(struct scx_sched *sch) -{ - struct scx_exit_info *ei = sch->exit_info; - const char *type = scx_parent(sch) ? "sub-scheduler" : "scheduler"; - - if (ei->kind >= SCX_EXIT_ERROR) { - pr_err("sched_ext: BPF %s \"%s\" disabled (%s)\n", type, - sch->ops.name, ei->reason); - - if (ei->msg[0] != '\0') - pr_err("sched_ext: %s: %s\n", sch->ops.name, ei->msg); -#ifdef CONFIG_STACKTRACE - stack_trace_print(ei->bt, ei->bt_len, 2); -#endif - } else { - pr_info("sched_ext: BPF %s \"%s\" disabled (%s)\n", type, - sch->ops.name, ei->reason); - } -} - -#ifdef CONFIG_EXT_SUB_SCHED -static DECLARE_WAIT_QUEUE_HEAD(scx_unlink_waitq); - -static void drain_descendants(struct scx_sched *sch) -{ - /* - * Child scheds that finished the critical part of disabling will take - * themselves off @sch->children. Wait for it to drain. As propagation - * is recursive, empty @sch->children means that all proper descendant - * scheds reached unlinking stage. - */ - wait_event(scx_unlink_waitq, list_empty(&sch->children)); -} - -static void scx_fail_parent(struct scx_sched *sch, - struct task_struct *failed, s32 fail_code) -{ - struct scx_sched *parent = scx_parent(sch); - struct scx_task_iter sti; - struct task_struct *p; - - scx_error(parent, "ops.init_task() failed (%d) for %s[%d] while disabling a sub-scheduler", - fail_code, failed->comm, failed->pid); - - /* - * Once $parent is bypassed, it's safe to put SCX_TASK_NONE tasks into - * it. This may cause downstream failures on the BPF side but $parent is - * dying anyway. - */ - scx_bypass(parent, true); - - scx_task_iter_start(&sti, sch->cgrp); - while ((p = scx_task_iter_next_locked(&sti))) { - if (scx_task_on_sched(parent, p)) - continue; - - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { - scx_disable_and_exit_task(sch, p); - scx_set_task_sched(p, parent); - } - } - scx_task_iter_stop(&sti); -} - -static void scx_sub_disable(struct scx_sched *sch) -{ - struct scx_sched *parent = scx_parent(sch); - struct scx_task_iter sti; - struct task_struct *p; - int ret; - - /* - * Guarantee forward progress and wait for descendants to be disabled. - * To limit disruptions, $parent is not bypassed. Tasks are fully - * prepped and then inserted back into $parent. - */ - scx_bypass(sch, true); - drain_descendants(sch); - - /* - * Here, every runnable task is guaranteed to make forward progress and - * we can safely use blocking synchronization constructs. Actually - * disable ops. - */ - mutex_lock(&scx_enable_mutex); - percpu_down_write(&scx_fork_rwsem); - scx_cgroup_lock(); - - set_cgroup_sched(sch_cgroup(sch), parent); - - scx_task_iter_start(&sti, sch->cgrp); - while ((p = scx_task_iter_next_locked(&sti))) { - struct rq *rq; - struct rq_flags rf; - - /* filter out duplicate visits */ - if (scx_task_on_sched(parent, p)) - continue; - - /* - * By the time control reaches here, all descendant schedulers - * should already have been disabled. - */ - WARN_ON_ONCE(!scx_task_on_sched(sch, p)); - - /* - * @p is pinned by the iter: css_task_iter_next() takes a - * reference and holds it until the next iter_next() call, so - * @p->usage is guaranteed > 0. - */ - get_task_struct(p); - - scx_task_iter_unlock(&sti); - - /* - * $p is READY or ENABLED on @sch. Initialize for $parent, - * disable and exit from @sch, and then switch over to $parent. - * - * If a task fails to initialize for $parent, the only available - * action is disabling $parent too. While this allows disabling - * of a child sched to cause the parent scheduler to fail, the - * failure can only originate from ops.init_task() of the - * parent. A child can't directly affect the parent through its - * own failures. - */ - ret = __scx_init_task(parent, p, false); - if (ret) { - scx_fail_parent(sch, p, ret); - put_task_struct(p); - break; - } - - rq = task_rq_lock(p, &rf); - - if (scx_get_task_state(p) == SCX_TASK_DEAD) { - /* - * sched_ext_dead() raced us between __scx_init_task() - * and this rq lock and ran exit_task() on @sch (the - * sched @p was on at that point), not on $parent. - * $parent's just-completed init is owed an exit_task() - * and we issue it here. - */ - scx_sub_init_cancel_task(parent, p); - task_rq_unlock(rq, p, &rf); - put_task_struct(p); - continue; - } - - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { - /* - * $p is initialized for $parent and still attached to - * @sch. Disable and exit for @sch, switch over to - * $parent, override the state to READY to account for - * $p having already been initialized, and then enable. - */ - scx_disable_and_exit_task(sch, p); - scx_set_task_state(p, SCX_TASK_INIT_BEGIN); - scx_set_task_state(p, SCX_TASK_INIT); - scx_set_task_sched(p, parent); - scx_set_task_state(p, SCX_TASK_READY); - scx_enable_task(parent, p); - } - - task_rq_unlock(rq, p, &rf); - put_task_struct(p); - } - scx_task_iter_stop(&sti); - - scx_disable_dump(sch); - - scx_cgroup_unlock(); - percpu_up_write(&scx_fork_rwsem); - - /* - * All tasks are moved off of @sch but there may still be on-going - * operations (e.g. ops.select_cpu()). Drain them by flushing RCU. Use - * the expedited version as ancestors may be waiting in bypass mode. - * Also, tell the parent that there is no need to keep running bypass - * DSQs for us. - */ - synchronize_rcu_expedited(); - disable_bypass_dsp(sch); - - scx_unlink_sched(sch); - - mutex_unlock(&scx_enable_mutex); - - /* - * @sch is now unlinked from the parent's children list. Notify and call - * ops.sub_detach/exit(). Note that ops.sub_detach/exit() must be called - * after unlinking and releasing all locks. See scx_claim_exit(). - */ - wake_up_all(&scx_unlink_waitq); - - if (parent->ops.sub_detach && sch->sub_attached) { - struct scx_sub_detach_args sub_detach_args = { - .ops = &sch->ops, - .cgroup_path = sch->cgrp_path, - }; - SCX_CALL_OP(parent, sub_detach, NULL, - &sub_detach_args); - } - - scx_log_sched_disable(sch); - - if (sch->ops.exit) - SCX_CALL_OP(sch, exit, NULL, sch->exit_info); - if (sch->sub_kset) - kobject_del(&sch->sub_kset->kobj); - kobject_del(&sch->kobj); -} -#else /* CONFIG_EXT_SUB_SCHED */ -static inline void drain_descendants(struct scx_sched *sch) { } -static inline void scx_sub_disable(struct scx_sched *sch) { } -#endif /* CONFIG_EXT_SUB_SCHED */ - -static void scx_root_disable(struct scx_sched *sch) -{ - struct scx_task_iter sti; - struct task_struct *p; - bool was_switched_all; - int cpu; - - /* guarantee forward progress and wait for descendants to be disabled */ - scx_bypass(sch, true); - drain_descendants(sch); - - switch (scx_set_enable_state(SCX_DISABLING)) { - case SCX_DISABLING: - WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); - break; - case SCX_DISABLED: - pr_warn("sched_ext: ops error detected without ops (%s)\n", - sch->exit_info->msg); - WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_DISABLING); - goto done; - default: - break; - } - - /* - * Here, every runnable task is guaranteed to make forward progress and - * we can safely use blocking synchronization constructs. Actually - * disable ops. - */ - mutex_lock(&scx_enable_mutex); - - was_switched_all = scx_switched_all(); - - static_branch_disable(&__scx_switched_all); - WRITE_ONCE(scx_switching_all, false); - - /* - * Shut down cgroup support before tasks so that the cgroup attach path - * doesn't race against scx_disable_and_exit_task(). - */ - scx_cgroup_lock(); - scx_cgroup_exit(sch); - scx_cgroup_unlock(); - - /* - * The BPF scheduler is going away. All tasks including %TASK_DEAD ones - * must be switched out and exited synchronously. - */ - percpu_down_write(&scx_fork_rwsem); - - scx_init_task_enabled = false; - - scx_task_iter_start(&sti, NULL); - while ((p = scx_task_iter_next_locked(&sti))) { - unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; - const struct sched_class *old_class = p->sched_class; - const struct sched_class *new_class = scx_setscheduler_class(p); - - update_rq_clock(task_rq(p)); - - if (old_class != new_class) - queue_flags |= DEQUEUE_CLASS; - - scoped_guard (sched_change, p, queue_flags) { - p->sched_class = new_class; - } - - scx_disable_and_exit_task(scx_task_sched(p), p); - } - scx_task_iter_stop(&sti); - - scx_disable_dump(sch); - - scx_cgroup_lock(); - set_cgroup_sched(sch_cgroup(sch), NULL); - scx_cgroup_unlock(); - - percpu_up_write(&scx_fork_rwsem); - - /* - * Invalidate all the rq clocks to prevent getting outdated - * rq clocks from a previous scx scheduler. - * - * Also re-balance the dl_server bandwidth reservations: detach - * ext_server (no more sched_ext tasks) and reinstate fair_server if it - * was previously detached because we were running in full mode. - * - * Unlike the enable path, this runs on a recovery path that cannot - * fail, so we use dl_server_swap_bw() to atomically free ext_server's - * bandwidth and reclaim it for fair_server under the same dl_b lock. - * - * The swap can still fail with -EBUSY if someone bumped ext_server's - * runtime via debugfs between enable and disable; in that narrow case - * both servers end up detached and we just WARN. - */ - for_each_possible_cpu(cpu) { - struct rq *rq = cpu_rq(cpu); - - scx_rq_clock_invalidate(rq); - - scoped_guard(rq_lock_irqsave, rq) { - update_rq_clock(rq); - if (was_switched_all) { - if (WARN_ON_ONCE(dl_server_swap_bw(&rq->ext_server, - &rq->fair_server))) - pr_warn("failed to re-attach fair_server on CPU %d\n", cpu); - } else { - dl_server_detach_bw(&rq->ext_server); - } - } - } - - /* no task is on scx, turn off all the switches and flush in-progress calls */ - static_branch_disable(&__scx_enabled); - static_branch_disable(&__scx_is_cid_type); - if (sch->ops.flags & SCX_OPS_TID_TO_TASK) - static_branch_disable(&__scx_tid_to_task_enabled); - bitmap_zero(sch->has_op, SCX_OPI_END); - scx_idle_disable(); - synchronize_rcu(); - if (sch->ops.flags & SCX_OPS_TID_TO_TASK) - rhashtable_free_and_destroy(&scx_tid_hash, NULL, NULL); - - scx_log_sched_disable(sch); - - if (sch->ops.exit) - SCX_CALL_OP(sch, exit, NULL, sch->exit_info); - - scx_unlink_sched(sch); - - /* - * scx_root clearing must be inside cpus_read_lock(). See - * handle_hotplug(). - */ - cpus_read_lock(); - RCU_INIT_POINTER(scx_root, NULL); - cpus_read_unlock(); - - /* - * Delete the kobject from the hierarchy synchronously. Otherwise, sysfs - * could observe an object of the same name still in the hierarchy when - * the next scheduler is loaded. - */ -#ifdef CONFIG_EXT_SUB_SCHED - if (sch->sub_kset) - kobject_del(&sch->sub_kset->kobj); -#endif - kobject_del(&sch->kobj); - - free_kick_syncs(); - - mutex_unlock(&scx_enable_mutex); - - WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_DISABLING); -done: - scx_bypass(sch, false); -} - -/* - * Claim the exit on @sch. The caller must ensure that the helper kthread work - * is kicked before the current task can be preempted. Once exit_kind is - * claimed, scx_error() can no longer trigger, so if the current task gets - * preempted and the BPF scheduler fails to schedule it back, the helper work - * will never be kicked and the whole system can wedge. - */ -static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind) -{ - int none = SCX_EXIT_NONE; - - lockdep_assert_preemption_disabled(); - - if (WARN_ON_ONCE(kind == SCX_EXIT_NONE || kind == SCX_EXIT_DONE)) - kind = SCX_EXIT_ERROR; - - if (!atomic_try_cmpxchg(&sch->exit_kind, &none, kind)) - return false; - - /* - * Some CPUs may be trapped in the dispatch paths. Set the aborting - * flag to break potential live-lock scenarios, ensuring we can - * successfully reach scx_bypass(). - */ - WRITE_ONCE(sch->aborting, true); - - /* - * Propagate exits to descendants immediately. Each has a dedicated - * helper kthread and can run in parallel. While most of disabling is - * serialized, running them in separate threads allows parallelizing - * ops.exit(), which can take arbitrarily long prolonging bypass mode. - * - * To guarantee forward progress, this propagation must be in-line so - * that ->aborting is synchronously asserted for all sub-scheds. The - * propagation is also the interlocking point against sub-sched - * attachment. See scx_link_sched(). - * - * This doesn't cause recursions as propagation only takes place for - * non-propagation exits. - */ - if (kind != SCX_EXIT_PARENT) { - scoped_guard (raw_spinlock_irqsave, &scx_sched_lock) { - struct scx_sched *pos; - scx_for_each_descendant_pre(pos, sch) - scx_disable(pos, SCX_EXIT_PARENT); - } - } - - return true; -} - -static void scx_disable_workfn(struct kthread_work *work) -{ - struct scx_sched *sch = container_of(work, struct scx_sched, disable_work); - struct scx_exit_info *ei = sch->exit_info; - int kind; - - kind = atomic_read(&sch->exit_kind); - while (true) { - if (kind == SCX_EXIT_DONE) /* already disabled? */ - return; - WARN_ON_ONCE(kind == SCX_EXIT_NONE); - if (atomic_try_cmpxchg(&sch->exit_kind, &kind, SCX_EXIT_DONE)) - break; - } - ei->kind = kind; - ei->reason = scx_exit_reason(ei->kind); - - if (scx_parent(sch)) - scx_sub_disable(sch); - else - scx_root_disable(sch); -} - -static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind) -{ - guard(preempt)(); - if (scx_claim_exit(sch, kind)) - irq_work_queue(&sch->disable_irq_work); -} - -/** - * scx_flush_disable_work - flush the disable work and wait for it to finish - * @sch: the scheduler - * - * sch->disable_work might still not queued, causing kthread_flush_work() - * as a noop. Syncing the irq_work first is required to guarantee the - * kthread work has been queued before waiting for it. - */ -static void scx_flush_disable_work(struct scx_sched *sch) -{ - int kind; - - do { - irq_work_sync(&sch->disable_irq_work); - kthread_flush_work(&sch->disable_work); - kind = atomic_read(&sch->exit_kind); - } while (kind != SCX_EXIT_NONE && kind != SCX_EXIT_DONE); -} - -static void dump_newline(struct seq_buf *s) -{ - trace_sched_ext_dump(""); - - /* @s may be zero sized and seq_buf triggers WARN if so */ - if (s->size) - seq_buf_putc(s, '\n'); -} - -static __printf(2, 3) void dump_line(struct seq_buf *s, const char *fmt, ...) -{ - va_list args; - -#ifdef CONFIG_TRACEPOINTS - if (trace_sched_ext_dump_enabled()) { - /* protected by scx_dump_lock */ - static char line_buf[SCX_EXIT_MSG_LEN]; - - va_start(args, fmt); - vscnprintf(line_buf, sizeof(line_buf), fmt, args); - va_end(args); - - trace_call__sched_ext_dump(line_buf); - } -#endif - /* @s may be zero sized and seq_buf triggers WARN if so */ - if (s->size) { - va_start(args, fmt); - seq_buf_vprintf(s, fmt, args); - va_end(args); - - seq_buf_putc(s, '\n'); - } -} - -static void dump_stack_trace(struct seq_buf *s, const char *prefix, - const unsigned long *bt, unsigned int len) -{ - unsigned int i; - - for (i = 0; i < len; i++) - dump_line(s, "%s%pS", prefix, (void *)bt[i]); -} - -static void ops_dump_init(struct seq_buf *s, const char *prefix) -{ - struct scx_dump_data *dd = &scx_dump_data; - - lockdep_assert_irqs_disabled(); - - dd->cpu = smp_processor_id(); /* allow scx_bpf_dump() */ - dd->first = true; - dd->cursor = 0; - dd->s = s; - dd->prefix = prefix; -} - -static void ops_dump_flush(void) -{ - struct scx_dump_data *dd = &scx_dump_data; - char *line = dd->buf.line; - - if (!dd->cursor) - return; - - /* - * There's something to flush and this is the first line. Insert a blank - * line to distinguish ops dump. - */ - if (dd->first) { - dump_newline(dd->s); - dd->first = false; - } - - /* - * There may be multiple lines in $line. Scan and emit each line - * separately. - */ - while (true) { - char *end = line; - char c; - - while (*end != '\n' && *end != '\0') - end++; - - /* - * If $line overflowed, it may not have newline at the end. - * Always emit with a newline. - */ - c = *end; - *end = '\0'; - dump_line(dd->s, "%s%s", dd->prefix, line); - if (c == '\0') - break; - - /* move to the next line */ - end++; - if (*end == '\0') - break; - line = end; - } - - dd->cursor = 0; -} - -static void ops_dump_exit(void) -{ - ops_dump_flush(); - scx_dump_data.cpu = -1; -} - -static void scx_dump_task(struct scx_sched *sch, struct seq_buf *s, struct scx_dump_ctx *dctx, - struct rq *rq, struct task_struct *p, char marker) -{ - static unsigned long bt[SCX_EXIT_BT_LEN]; - struct scx_sched *task_sch = scx_task_sched(p); - const char *own_marker; - char sch_id_buf[32]; - char dsq_id_buf[19] = "(n/a)"; - unsigned long ops_state = atomic_long_read(&p->scx.ops_state); - unsigned int bt_len = 0; - - own_marker = task_sch == sch ? "*" : ""; - - if (task_sch->level == 0) - scnprintf(sch_id_buf, sizeof(sch_id_buf), "root"); - else - scnprintf(sch_id_buf, sizeof(sch_id_buf), "sub%d-%llu", - task_sch->level, task_sch->ops.sub_cgroup_id); - - if (p->scx.dsq) - scnprintf(dsq_id_buf, sizeof(dsq_id_buf), "0x%llx", - (unsigned long long)p->scx.dsq->id); - - dump_newline(s); - dump_line(s, " %c%c %s[%d] %s%s %+ldms", - marker, task_state_to_char(p), p->comm, p->pid, - own_marker, sch_id_buf, - jiffies_delta_msecs(p->scx.runnable_at, dctx->at_jiffies)); - dump_line(s, " scx_state/flags=%u/0x%x dsq_flags=0x%x ops_state/qseq=%lu/%lu", - scx_get_task_state(p) >> SCX_TASK_STATE_SHIFT, - p->scx.flags & ~SCX_TASK_STATE_MASK, - p->scx.dsq_flags, ops_state & SCX_OPSS_STATE_MASK, - ops_state >> SCX_OPSS_QSEQ_SHIFT); - dump_line(s, " sticky/holding_cpu=%d/%d dsq_id=%s", - p->scx.sticky_cpu, p->scx.holding_cpu, dsq_id_buf); - dump_line(s, " dsq_vtime=%llu slice=%llu weight=%u", - p->scx.dsq_vtime, p->scx.slice, p->scx.weight); - dump_line(s, " cpus=%*pb no_mig=%u", cpumask_pr_args(p->cpus_ptr), - p->migration_disabled); - - if (SCX_HAS_OP(sch, dump_task)) { - ops_dump_init(s, " "); - SCX_CALL_OP(sch, dump_task, rq, dctx, p); - ops_dump_exit(); - } - -#ifdef CONFIG_STACKTRACE - bt_len = stack_trace_save_tsk(p, bt, SCX_EXIT_BT_LEN, 1); -#endif - if (bt_len) { - dump_newline(s); - dump_stack_trace(s, " ", bt, bt_len); - } -} - -static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, - struct scx_dump_ctx *dctx, int cpu, - bool dump_all_tasks) -{ - struct rq *rq = cpu_rq(cpu); - struct rq_flags rf; - struct task_struct *p; - struct seq_buf ns; - size_t avail, used; - char *buf; - bool idle; - - rq_lock_irqsave(rq, &rf); - - idle = list_empty(&rq->scx.runnable_list) && - rq->curr->sched_class == &idle_sched_class; - - if (idle && !SCX_HAS_OP(sch, dump_cpu)) - goto next; - - /* - * We don't yet know whether ops.dump_cpu() will produce output - * and we may want to skip the default CPU dump if it doesn't. - * Use a nested seq_buf to generate the standard dump so that we - * can decide whether to commit later. - */ - avail = seq_buf_get_buf(s, &buf); - seq_buf_init(&ns, buf, avail); - - dump_newline(&ns); - dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu ksync=%lu", - cpu, rq->scx.nr_running, rq->scx.flags, - rq->scx.cpu_released, rq->scx.ops_qseq, - rq->scx.kick_sync); - dump_line(&ns, " curr=%s[%d] class=%ps", - rq->curr->comm, rq->curr->pid, - rq->curr->sched_class); - if (!cpumask_empty(rq->scx.cpus_to_kick)) - dump_line(&ns, " cpus_to_kick : %*pb", - cpumask_pr_args(rq->scx.cpus_to_kick)); - if (!cpumask_empty(rq->scx.cpus_to_kick_if_idle)) - dump_line(&ns, " idle_to_kick : %*pb", - cpumask_pr_args(rq->scx.cpus_to_kick_if_idle)); - if (!cpumask_empty(rq->scx.cpus_to_preempt)) - dump_line(&ns, " cpus_to_preempt: %*pb", - cpumask_pr_args(rq->scx.cpus_to_preempt)); - if (!cpumask_empty(rq->scx.cpus_to_wait)) - dump_line(&ns, " cpus_to_wait : %*pb", - cpumask_pr_args(rq->scx.cpus_to_wait)); - if (!cpumask_empty(rq->scx.cpus_to_sync)) - dump_line(&ns, " cpus_to_sync : %*pb", - cpumask_pr_args(rq->scx.cpus_to_sync)); - - used = seq_buf_used(&ns); - if (SCX_HAS_OP(sch, dump_cpu)) { - ops_dump_init(&ns, " "); - SCX_CALL_OP(sch, dump_cpu, rq, dctx, scx_cpu_arg(cpu), idle); - ops_dump_exit(); - } - - /* - * If idle && nothing generated by ops.dump_cpu(), there's - * nothing interesting. Skip. - */ - if (idle && used == seq_buf_used(&ns)) - goto next; - - /* - * $s may already have overflowed when $ns was created. If so, - * calling commit on it will trigger BUG. - */ - if (avail) { - seq_buf_commit(s, seq_buf_used(&ns)); - if (seq_buf_has_overflowed(&ns)) - seq_buf_set_overflow(s); - } - - if (rq->curr->sched_class == &ext_sched_class && - (dump_all_tasks || scx_task_on_sched(sch, rq->curr))) - scx_dump_task(sch, s, dctx, rq, rq->curr, '*'); - - list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) - if (dump_all_tasks || scx_task_on_sched(sch, p)) - scx_dump_task(sch, s, dctx, rq, p, ' '); -next: - rq_unlock_irqrestore(rq, &rf); -} - -/* - * Dump scheduler state. If @dump_all_tasks is true, dump all tasks regardless - * of which scheduler they belong to. If false, only dump tasks owned by @sch. - * For SysRq-D dumps, @dump_all_tasks=false since all schedulers are dumped - * separately. For error dumps, @dump_all_tasks=true since only the failing - * scheduler is dumped. - */ -static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei, - size_t dump_len, bool dump_all_tasks) -{ - static const char trunc_marker[] = "\n\n~~~~ TRUNCATED ~~~~\n"; - struct scx_dump_ctx dctx = { - .kind = ei->kind, - .exit_code = ei->exit_code, - .reason = ei->reason, - .at_ns = ktime_get_ns(), - .at_jiffies = jiffies, - }; - struct seq_buf s; - struct scx_event_stats events; - int cpu; - - guard(raw_spinlock_irqsave)(&scx_dump_lock); - - if (sch->dump_disabled) - return; - - seq_buf_init(&s, ei->dump, dump_len); - -#ifdef CONFIG_EXT_SUB_SCHED - if (sch->level == 0) - dump_line(&s, "%s: root", sch->ops.name); - else - dump_line(&s, "%s: sub%d-%llu %s", - sch->ops.name, sch->level, sch->ops.sub_cgroup_id, - sch->cgrp_path); -#endif - if (ei->kind == SCX_EXIT_NONE) { - dump_line(&s, "Debug dump triggered by %s", ei->reason); - } else { - if (ei->exit_cpu >= 0) - dump_line(&s, "%s[%d] triggered exit kind %d on CPU %d:", - current->comm, current->pid, ei->kind, - ei->exit_cpu); - else - dump_line(&s, "%s[%d] triggered exit kind %d:", - current->comm, current->pid, ei->kind); - dump_line(&s, " %s (%s)", ei->reason, ei->msg); - dump_newline(&s); - dump_line(&s, "Backtrace:"); - dump_stack_trace(&s, " ", ei->bt, ei->bt_len); - } - - if (SCX_HAS_OP(sch, dump)) { - ops_dump_init(&s, ""); - SCX_CALL_OP(sch, dump, NULL, &dctx); - ops_dump_exit(); - } - - dump_newline(&s); - dump_line(&s, "CPU states"); - dump_line(&s, "----------"); - - /* - * Dump the exit CPU first so it isn't lost to dump truncation, then - * walk the rest in order, skipping the one already dumped. - */ - if (ei->exit_cpu >= 0) - scx_dump_cpu(sch, &s, &dctx, ei->exit_cpu, dump_all_tasks); - for_each_possible_cpu(cpu) { - if (cpu != ei->exit_cpu) - scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks); - } - - dump_newline(&s); - dump_line(&s, "Event counters"); - dump_line(&s, "--------------"); - - scx_read_events(sch, &events); - scx_dump_event(s, &events, SCX_EV_SELECT_CPU_FALLBACK); - scx_dump_event(s, &events, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); - scx_dump_event(s, &events, SCX_EV_DISPATCH_KEEP_LAST); - scx_dump_event(s, &events, SCX_EV_ENQ_SKIP_EXITING); - scx_dump_event(s, &events, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); - scx_dump_event(s, &events, SCX_EV_REENQ_IMMED); - scx_dump_event(s, &events, SCX_EV_REENQ_LOCAL_REPEAT); - scx_dump_event(s, &events, SCX_EV_REFILL_SLICE_DFL); - scx_dump_event(s, &events, SCX_EV_BYPASS_DURATION); - scx_dump_event(s, &events, SCX_EV_BYPASS_DISPATCH); - scx_dump_event(s, &events, SCX_EV_BYPASS_ACTIVATE); - scx_dump_event(s, &events, SCX_EV_INSERT_NOT_OWNED); - scx_dump_event(s, &events, SCX_EV_SUB_BYPASS_DISPATCH); - - if (seq_buf_has_overflowed(&s) && dump_len >= sizeof(trunc_marker)) - memcpy(ei->dump + dump_len - sizeof(trunc_marker), - trunc_marker, sizeof(trunc_marker)); -} - -static void scx_disable_irq_workfn(struct irq_work *irq_work) -{ - struct scx_sched *sch = container_of(irq_work, struct scx_sched, disable_irq_work); - struct scx_exit_info *ei = sch->exit_info; - - if (ei->kind >= SCX_EXIT_ERROR) - scx_dump_state(sch, ei, sch->ops.exit_dump_len, true); - - kthread_queue_work(sch->helper, &sch->disable_work); -} - -bool scx_vexit(struct scx_sched *sch, - enum scx_exit_kind kind, s64 exit_code, s32 exit_cpu, - const char *fmt, va_list args) -{ - struct scx_exit_info *ei = sch->exit_info; - - guard(preempt)(); - - if (!scx_claim_exit(sch, kind)) - return false; - - ei->exit_code = exit_code; -#ifdef CONFIG_STACKTRACE - if (kind >= SCX_EXIT_ERROR) - ei->bt_len = stack_trace_save(ei->bt, SCX_EXIT_BT_LEN, 1); -#endif - vscnprintf(ei->msg, SCX_EXIT_MSG_LEN, fmt, args); - - /* - * Set ei->kind and ->reason for scx_dump_state(). They'll be set again - * in scx_disable_workfn(). - */ - ei->kind = kind; - ei->reason = scx_exit_reason(ei->kind); - ei->exit_cpu = exit_cpu; - - irq_work_queue(&sch->disable_irq_work); - return true; -} - -static int alloc_kick_syncs(void) -{ - int cpu; - - /* - * Allocate per-CPU arrays sized by nr_cpu_ids. Use kvzalloc as size - * can exceed percpu allocator limits on large machines. - */ - for_each_possible_cpu(cpu) { - struct scx_kick_syncs **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu); - struct scx_kick_syncs *new_ksyncs; - - WARN_ON_ONCE(rcu_access_pointer(*ksyncs)); - - new_ksyncs = kvzalloc_node(struct_size(new_ksyncs, syncs, nr_cpu_ids), - GFP_KERNEL, cpu_to_node(cpu)); - if (!new_ksyncs) { - free_kick_syncs(); - return -ENOMEM; - } - - rcu_assign_pointer(*ksyncs, new_ksyncs); - } - - return 0; -} - -static void free_pnode(struct scx_sched_pnode *pnode) -{ - if (!pnode) - return; - exit_dsq(&pnode->global_dsq); - kfree(pnode); -} - -static struct scx_sched_pnode *alloc_pnode(struct scx_sched *sch, int node) -{ - struct scx_sched_pnode *pnode; - - pnode = kzalloc_node(sizeof(*pnode), GFP_KERNEL, node); - if (!pnode) - return NULL; - - if (init_dsq(&pnode->global_dsq, SCX_DSQ_GLOBAL, sch)) { - kfree(pnode); - return NULL; - } - - return pnode; -} - -/* - * scx_enable() is offloaded to a dedicated system-wide RT kthread to avoid - * starvation. During the READY -> ENABLED task switching loop, the calling - * thread's sched_class gets switched from fair to ext. As fair has higher - * priority than ext, the calling thread can be indefinitely starved under - * fair-class saturation, leading to a system hang. - */ -struct scx_enable_cmd { - struct kthread_work work; - union { - struct sched_ext_ops *ops; - struct sched_ext_ops_cid *ops_cid; - }; - bool is_cid_type; - struct bpf_map *arena_map; /* arena ref to transfer to sch */ - int ret; -}; - -/* - * Allocate and initialize a new scx_sched. @cgrp's reference is always - * consumed whether the function succeeds or fails. - */ -static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, - struct cgroup *cgrp, - struct scx_sched *parent) -{ - struct sched_ext_ops *ops = cmd->ops; - struct scx_sched *sch; - s32 level = parent ? parent->level + 1 : 0; - s32 node, cpu, ret, bypass_fail_cpu = nr_cpu_ids; - - sch = kzalloc_flex(*sch, ancestors, level + 1); - if (!sch) { - ret = -ENOMEM; - goto err_put_cgrp; - } - - sch->exit_info = alloc_exit_info(ops->exit_dump_len); - if (!sch->exit_info) { - ret = -ENOMEM; - goto err_free_sch; - } - - ret = rhashtable_init(&sch->dsq_hash, &dsq_hash_params); - if (ret < 0) - goto err_free_ei; - - sch->pnode = kzalloc_objs(sch->pnode[0], nr_node_ids); - if (!sch->pnode) { - ret = -ENOMEM; - goto err_free_hash; - } - - for_each_node_state(node, N_POSSIBLE) { - sch->pnode[node] = alloc_pnode(sch, node); - if (!sch->pnode[node]) { - ret = -ENOMEM; - goto err_free_pnode; - } - } - - sch->dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; - sch->pcpu = __alloc_percpu(struct_size_t(struct scx_sched_pcpu, - dsp_ctx.buf, sch->dsp_max_batch), - __alignof__(struct scx_sched_pcpu)); - if (!sch->pcpu) { - ret = -ENOMEM; - goto err_free_pnode; - } - - for_each_possible_cpu(cpu) { - ret = init_dsq(bypass_dsq(sch, cpu), SCX_DSQ_BYPASS, sch); - if (ret) { - bypass_fail_cpu = cpu; - goto err_free_pcpu; - } - } - - for_each_possible_cpu(cpu) { - struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); - - pcpu->sch = sch; - INIT_LIST_HEAD(&pcpu->deferred_reenq_local.node); - } - - sch->helper = kthread_run_worker(0, "sched_ext_helper"); - if (IS_ERR(sch->helper)) { - ret = PTR_ERR(sch->helper); - goto err_free_pcpu; - } - - sched_set_fifo(sch->helper->task); - - if (parent) - memcpy(sch->ancestors, parent->ancestors, - level * sizeof(parent->ancestors[0])); - sch->ancestors[level] = sch; - sch->level = level; - - if (ops->timeout_ms) - sch->watchdog_timeout = msecs_to_jiffies(ops->timeout_ms); - else - sch->watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT; - - sch->slice_dfl = SCX_SLICE_DFL; - atomic_set(&sch->exit_kind, SCX_EXIT_NONE); - sch->disable_irq_work = IRQ_WORK_INIT_HARD(scx_disable_irq_workfn); - kthread_init_work(&sch->disable_work, scx_disable_workfn); - timer_setup(&sch->bypass_lb_timer, scx_bypass_lb_timerfn, 0); - - if (!alloc_cpumask_var(&sch->bypass_lb_donee_cpumask, GFP_KERNEL)) { - ret = -ENOMEM; - goto err_stop_helper; - } - if (!alloc_cpumask_var(&sch->bypass_lb_resched_cpumask, GFP_KERNEL)) { - ret = -ENOMEM; - goto err_free_lb_cpumask; - } - /* - * Copy ops through the right union view. For cid-form the source is - * struct sched_ext_ops_cid which lacks the trailing cpu_acquire/ - * cpu_release; those stay zero from kzalloc. - */ - if (cmd->is_cid_type) { - sch->ops_cid = *cmd->ops_cid; - sch->is_cid_type = true; - } else { - sch->ops = *cmd->ops; - } - - rcu_assign_pointer(ops->priv, sch); - - sch->kobj.kset = scx_kset; - INIT_LIST_HEAD(&sch->all); - -#ifdef CONFIG_EXT_SUB_SCHED - char *buf = kzalloc(PATH_MAX, GFP_KERNEL); - if (!buf) { - ret = -ENOMEM; - goto err_free_lb_resched; - } - cgroup_path(cgrp, buf, PATH_MAX); - sch->cgrp_path = kstrdup(buf, GFP_KERNEL); - kfree(buf); - if (!sch->cgrp_path) { - ret = -ENOMEM; - goto err_free_lb_resched; - } - - sch->cgrp = cgrp; - INIT_LIST_HEAD(&sch->children); - INIT_LIST_HEAD(&sch->sibling); - - if (parent) - ret = kobject_init_and_add(&sch->kobj, &scx_ktype, - &parent->sub_kset->kobj, - "sub-%llu", cgroup_id(cgrp)); - else - ret = kobject_init_and_add(&sch->kobj, &scx_ktype, NULL, "root"); - - if (ret < 0) { - RCU_INIT_POINTER(ops->priv, NULL); - kobject_put(&sch->kobj); - return ERR_PTR(ret); - } - - if (ops->sub_attach) { - sch->sub_kset = kset_create_and_add("sub", NULL, &sch->kobj); - if (!sch->sub_kset) { - RCU_INIT_POINTER(ops->priv, NULL); - kobject_put(&sch->kobj); - return ERR_PTR(-ENOMEM); - } - } -#else /* CONFIG_EXT_SUB_SCHED */ - ret = kobject_init_and_add(&sch->kobj, &scx_ktype, NULL, "root"); - if (ret < 0) { - RCU_INIT_POINTER(ops->priv, NULL); - kobject_put(&sch->kobj); - return ERR_PTR(ret); - } -#endif /* CONFIG_EXT_SUB_SCHED */ - - /* - * Consume the arena_map ref bpf_scx_reg_cid() took. Defer to here so - * earlier failure paths leave cmd->arena_map set and bpf_scx_reg_cid - * drops the ref. After this point, sch owns the ref and any cleanup - * runs through scx_sched_free_rcu_work() which puts it. - */ - sch->arena_map = cmd->arena_map; - /* BPF arena is only available on MMU && 64BIT */ -#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) - if (sch->arena_map) - sch->arena_kern_base = bpf_arena_map_kern_vm_start(sch->arena_map); -#endif - cmd->arena_map = NULL; - return sch; - -#ifdef CONFIG_EXT_SUB_SCHED -err_free_lb_resched: - RCU_INIT_POINTER(ops->priv, NULL); - free_cpumask_var(sch->bypass_lb_resched_cpumask); -#endif -err_free_lb_cpumask: - free_cpumask_var(sch->bypass_lb_donee_cpumask); -err_stop_helper: - kthread_destroy_worker(sch->helper); -err_free_pcpu: - for_each_possible_cpu(cpu) { - if (cpu == bypass_fail_cpu) - break; - exit_dsq(bypass_dsq(sch, cpu)); - } - free_percpu(sch->pcpu); -err_free_pnode: - for_each_node_state(node, N_POSSIBLE) - free_pnode(sch->pnode[node]); - kfree(sch->pnode); -err_free_hash: - rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL); -err_free_ei: - free_exit_info(sch->exit_info); -err_free_sch: - kfree(sch); -err_put_cgrp: -#ifdef CONFIG_EXT_SUB_SCHED - cgroup_put(cgrp); -#endif - return ERR_PTR(ret); -} - -static int check_hotplug_seq(struct scx_sched *sch, - const struct sched_ext_ops *ops) -{ - unsigned long long global_hotplug_seq; - - /* - * If a hotplug event has occurred between when a scheduler was - * initialized, and when we were able to attach, exit and notify user - * space about it. - */ - if (ops->hotplug_seq) { - global_hotplug_seq = atomic_long_read(&scx_hotplug_seq); - if (ops->hotplug_seq != global_hotplug_seq) { - scx_exit(sch, SCX_EXIT_UNREG_KERN, - SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG, - "expected hotplug seq %llu did not match actual %llu", - ops->hotplug_seq, global_hotplug_seq); - return -EBUSY; - } - } - - return 0; -} - -static int validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) -{ - /* - * It doesn't make sense to specify the SCX_OPS_ENQ_LAST flag if the - * ops.enqueue() callback isn't implemented. - */ - if ((ops->flags & SCX_OPS_ENQ_LAST) && !ops->enqueue) { - scx_error(sch, "SCX_OPS_ENQ_LAST requires ops.enqueue() to be implemented"); - return -EINVAL; - } - - /* - * SCX_OPS_TID_TO_TASK is enabled by the root scheduler. A sub-sched - * may set it to declare a dependency; reject if the root hasn't - * enabled it. - */ - if ((ops->flags & SCX_OPS_TID_TO_TASK) && scx_parent(sch) && - !(scx_root->ops.flags & SCX_OPS_TID_TO_TASK)) { - scx_error(sch, "SCX_OPS_TID_TO_TASK requires root scheduler to enable it"); - return -EINVAL; - } - - /* - * SCX_OPS_BUILTIN_IDLE_PER_NODE requires built-in CPU idle - * selection policy to be enabled. - */ - if ((ops->flags & SCX_OPS_BUILTIN_IDLE_PER_NODE) && - (ops->update_idle && !(ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE))) { - scx_error(sch, "SCX_OPS_BUILTIN_IDLE_PER_NODE requires CPU idle selection enabled"); - return -EINVAL; - } - - /* - * cid-form's struct is shorter and doesn't include the cpu_acquire / - * cpu_release tail; reading those fields off a cid-form @ops would - * run past the BPF allocation. Skip for cid-form. - */ - if (!sch->is_cid_type && (ops->cpu_acquire || ops->cpu_release)) - pr_warn("ops->cpu_acquire/release() are deprecated, use sched_switch TP instead\n"); - - /* - * Sub-scheduler support is tied to the cid-form struct_ops. A sub-sched - * attaches through a cid-form-only interface (sub_attach/sub_detach), - * and a root that accepts sub-scheds must expose cid-form state to - * them. Reject cpu-form schedulers on either side. - */ - if (!sch->is_cid_type) { - if (scx_parent(sch)) { - scx_error(sch, "sub-sched requires cid-form struct_ops"); - return -EINVAL; - } - if (ops->sub_attach || ops->sub_detach) { - scx_error(sch, "sub_attach/sub_detach requires cid-form struct_ops"); - return -EINVAL; - } - } - - return 0; -} - -static void scx_root_enable_workfn(struct kthread_work *work) -{ - struct scx_enable_cmd *cmd = container_of(work, struct scx_enable_cmd, work); - struct sched_ext_ops *ops = cmd->ops; - struct cgroup *cgrp = root_cgroup(); - struct scx_sched *sch; - struct scx_task_iter sti; - struct task_struct *p; - int i, cpu, ret; - - mutex_lock(&scx_enable_mutex); - - if (scx_enable_state() != SCX_DISABLED) { - ret = -EBUSY; - goto err_unlock; - } - - /* - * @ops->priv binds @ops to its scx_sched instance. It is set here by - * scx_alloc_and_add_sched() and cleared at the tail of bpf_scx_unreg(), - * which runs after scx_root_disable() has dropped scx_enable_mutex. If - * it's still non-NULL here, a previous attachment on @ops has not - * finished tearing down; proceeding would let the in-flight unreg's - * RCU_INIT_POINTER(NULL) clobber the @ops->priv we are about to assign. - */ - if (rcu_access_pointer(ops->priv)) { - ret = -EBUSY; - goto err_unlock; - } - - ret = alloc_kick_syncs(); - if (ret) - goto err_unlock; - - if (ops->flags & SCX_OPS_TID_TO_TASK) { - ret = rhashtable_init(&scx_tid_hash, &scx_tid_hash_params); - if (ret) - goto err_free_ksyncs; - } - -#ifdef CONFIG_EXT_SUB_SCHED - cgroup_get(cgrp); -#endif - sch = scx_alloc_and_add_sched(cmd, cgrp, NULL); - if (IS_ERR(sch)) { - ret = PTR_ERR(sch); - goto err_free_tid_hash; - } - - if (sch->is_cid_type) - static_branch_enable(&__scx_is_cid_type); - - /* - * Transition to ENABLING and clear exit info to arm the disable path. - * Failure triggers full disabling from here on. - */ - WARN_ON_ONCE(scx_set_enable_state(SCX_ENABLING) != SCX_DISABLED); - WARN_ON_ONCE(scx_root); - - atomic_long_set(&scx_nr_rejected, 0); - - for_each_possible_cpu(cpu) { - struct rq *rq = cpu_rq(cpu); - - rq->scx.local_dsq.sched = sch; - rq->scx.cpuperf_target = SCX_CPUPERF_ONE; - } - - /* - * Keep CPUs stable during enable so that the BPF scheduler can track - * online CPUs by watching ->on/offline_cpu() after ->init(). - */ - cpus_read_lock(); - - /* - * Build the cid mapping before publishing scx_root. The cid kfuncs - * dereference the cid arrays unconditionally once scx_prog_sched() - * returns non-NULL; the rcu_assign_pointer() below pairs with their - * rcu_dereference() to make the populated arrays visible. - */ - ret = scx_cid_init(sch); - if (ret) { - cpus_read_unlock(); - goto err_disable; - } - - /* - * Make the scheduler instance visible. Must be inside cpus_read_lock(). - * See handle_hotplug(). - */ - rcu_assign_pointer(scx_root, sch); - - ret = scx_link_sched(sch); - if (ret) { - cpus_read_unlock(); - goto err_disable; - } - - scx_idle_enable(ops); - - if (sch->ops.init) { - ret = SCX_CALL_OP_RET(sch, init, NULL); - if (ret) { - ret = ops_sanitize_err(sch, "init", ret); - cpus_read_unlock(); - scx_error(sch, "ops.init() failed (%d)", ret); - goto err_disable; - } - sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; - } - - ret = scx_arena_pool_init(sch); - if (ret) { - cpus_read_unlock(); - goto err_disable; - } - - ret = scx_set_cmask_scratch_alloc(sch); - if (ret) { - cpus_read_unlock(); - goto err_disable; - } - - for (i = SCX_OPI_CPU_HOTPLUG_BEGIN; i < SCX_OPI_CPU_HOTPLUG_END; i++) - if (((void (**)(void))ops)[i]) - set_bit(i, sch->has_op); - - ret = check_hotplug_seq(sch, ops); - if (ret) { - cpus_read_unlock(); - goto err_disable; - } - scx_idle_update_selcpu_topology(ops); - - cpus_read_unlock(); - - ret = validate_ops(sch, ops); - if (ret) - goto err_disable; - - /* - * Attach the ext_server bandwidth reservation before anything is - * committed so that we can fail the enable if the root domain cannot - * accommodate it. The matching fair_server detach is deferred to the - * tail of this function, after the switch is fully committed and can no - * longer fail. - * - * On failure, err_disable funnels into scx_root_disable() which - * detaches ext_server, so partially-attached state is cleaned up - * automatically. - */ - for_each_possible_cpu(cpu) { - struct rq *rq = cpu_rq(cpu); - - scoped_guard(rq_lock_irqsave, rq) { - update_rq_clock(rq); - ret = dl_server_attach_bw(&rq->ext_server); - } - if (ret) { - pr_warn("sched_ext: failed to attach ext_server on CPU %d (%d)\n", - cpu, ret); - goto err_disable; - } - } - - /* - * Once __scx_enabled is set, %current can be switched to SCX anytime. - * This can lead to stalls as some BPF schedulers (e.g. userspace - * scheduling) may not function correctly before all tasks are switched. - * Init in bypass mode to guarantee forward progress. - */ - scx_bypass(sch, true); - - for (i = SCX_OPI_NORMAL_BEGIN; i < SCX_OPI_NORMAL_END; i++) - if (((void (**)(void))ops)[i]) - set_bit(i, sch->has_op); - - if (sch->ops.cpu_acquire || sch->ops.cpu_release) - sch->ops.flags |= SCX_OPS_HAS_CPU_PREEMPT; - - /* - * Lock out forks, cgroup on/offlining and moves before opening the - * floodgate so that they don't wander into the operations prematurely. - */ - percpu_down_write(&scx_fork_rwsem); - - WARN_ON_ONCE(scx_init_task_enabled); - scx_init_task_enabled = true; - - /* flip under fork_rwsem; the iter below covers existing tasks */ - if (ops->flags & SCX_OPS_TID_TO_TASK) - static_branch_enable(&__scx_tid_to_task_enabled); - - /* - * Enable ops for every task. Fork is excluded by scx_fork_rwsem - * preventing new tasks from being added. No need to exclude tasks - * leaving as sched_ext_free() can handle both prepped and enabled - * tasks. Prep all tasks first and then enable them with preemption - * disabled. - * - * All cgroups should be initialized before scx_init_task() so that the - * BPF scheduler can reliably track each task's cgroup membership from - * scx_init_task(). Lock out cgroup on/offlining and task migrations - * while tasks are being initialized so that scx_cgroup_can_attach() - * never sees uninitialized tasks. - */ - scx_cgroup_lock(); - set_cgroup_sched(sch_cgroup(sch), sch); - ret = scx_cgroup_init(sch); - if (ret) - goto err_disable_unlock_all; - - scx_task_iter_start(&sti, NULL); - while ((p = scx_task_iter_next_locked(&sti))) { - /* - * @p is in scx_tasks under scx_tasks_lock, and SCX_TASK_DEAD - * tasks are filtered by scx_task_iter_next_locked(). - * sched_ext_dead() removes @p from scx_tasks under the same - * lock before put_task_struct_rcu_user() runs, so @p->usage - * is guaranteed > 0 here. - */ - get_task_struct(p); - - /* - * Set %INIT_BEGIN under the iter's rq lock so that a concurrent - * sched_ext_dead() does not call ops.exit_task() on @p while - * ops.init_task() is running. If sched_ext_dead() runs before - * this store, it has already removed @p from scx_tasks and the - * iter won't visit @p; if it runs after, it observes - * %INIT_BEGIN and transitions to %DEAD without calling ops, - * leaving the post-init recheck below to unwind. - */ - scx_set_task_state(p, SCX_TASK_INIT_BEGIN); - scx_task_iter_unlock(&sti); - - ret = __scx_init_task(sch, p, false); - - scx_task_iter_relock(&sti, p); - - if (unlikely(ret)) { - if (scx_get_task_state(p) != SCX_TASK_DEAD) - scx_set_task_state(p, SCX_TASK_NONE); - scx_task_iter_stop(&sti); - scx_error(sch, "ops.init_task() failed (%d) for %s[%d]", - ret, p->comm, p->pid); - put_task_struct(p); - goto err_disable_unlock_all; - } - - if (scx_get_task_state(p) == SCX_TASK_DEAD) { - /* - * sched_ext_dead() observed %INIT_BEGIN and set %DEAD. - * ops.exit_task() is owed to the sched __scx_init_task() - * ran against; call it now. - */ - scx_sub_init_cancel_task(sch, p); - } else { - scx_set_task_state(p, SCX_TASK_INIT); - scx_set_task_sched(p, sch); - scx_set_task_state(p, SCX_TASK_READY); - } - - /* - * Insert into the tid hash. scx_tasks_lock is held by the iter; - * list_empty() guards against sched_ext_dead() having taken @p - * off the list while init ran unlocked. - */ - if (scx_tid_to_task_enabled() && !list_empty(&p->scx.tasks_node)) - scx_tid_hash_insert(p); - - put_task_struct(p); - } - scx_task_iter_stop(&sti); - scx_cgroup_unlock(); - percpu_up_write(&scx_fork_rwsem); - - /* - * All tasks are READY. It's safe to turn on scx_enabled() and switch - * all eligible tasks. - */ - WRITE_ONCE(scx_switching_all, !(ops->flags & SCX_OPS_SWITCH_PARTIAL)); - static_branch_enable(&__scx_enabled); - - /* - * We're fully committed and can't fail. The task READY -> ENABLED - * transitions here are synchronized against sched_ext_free() through - * scx_tasks_lock. - */ - percpu_down_write(&scx_fork_rwsem); - scx_task_iter_start(&sti, NULL); - while ((p = scx_task_iter_next_locked(&sti))) { - unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE; - const struct sched_class *old_class = p->sched_class; - const struct sched_class *new_class = scx_setscheduler_class(p); - - if (scx_get_task_state(p) != SCX_TASK_READY) - continue; - - if (old_class != new_class) - queue_flags |= DEQUEUE_CLASS; - - scoped_guard (sched_change, p, queue_flags) { - p->scx.slice = READ_ONCE(sch->slice_dfl); - p->sched_class = new_class; - } - } - scx_task_iter_stop(&sti); - percpu_up_write(&scx_fork_rwsem); - - scx_bypass(sch, false); - - if (!scx_tryset_enable_state(SCX_ENABLED, SCX_ENABLING)) { - WARN_ON_ONCE(atomic_read(&sch->exit_kind) == SCX_EXIT_NONE); - goto err_disable; - } - - if (!(ops->flags & SCX_OPS_SWITCH_PARTIAL)) - static_branch_enable(&__scx_switched_all); - - /* - * Detach the fair_server bandwidth reservation now that the switch - * is fully committed. In full mode (!SCX_OPS_SWITCH_PARTIAL) no - * task will ever run in the fair class, so give that bandwidth - * back to the RT class. The matching ext_server attach already - * happened earlier; this only releases bandwidth and cannot fail. - * - * In partial mode keep fair_server attached. - */ - if (scx_switched_all()) { - for_each_possible_cpu(cpu) { - struct rq *rq = cpu_rq(cpu); - - guard(rq_lock_irqsave)(rq); - update_rq_clock(rq); - dl_server_detach_bw(&rq->fair_server); - } - } - - pr_info("sched_ext: BPF scheduler \"%s\" enabled%s\n", - sch->ops.name, scx_switched_all() ? "" : " (partial)"); - kobject_uevent(&sch->kobj, KOBJ_ADD); - mutex_unlock(&scx_enable_mutex); - - atomic_long_inc(&scx_enable_seq); - - cmd->ret = 0; - return; - -err_free_tid_hash: - if (ops->flags & SCX_OPS_TID_TO_TASK) - rhashtable_free_and_destroy(&scx_tid_hash, NULL, NULL); -err_free_ksyncs: - free_kick_syncs(); -err_unlock: - mutex_unlock(&scx_enable_mutex); - cmd->ret = ret; - return; - -err_disable_unlock_all: - scx_cgroup_unlock(); - percpu_up_write(&scx_fork_rwsem); - /* we'll soon enter disable path, keep bypass on */ -err_disable: - mutex_unlock(&scx_enable_mutex); - /* - * Returning an error code here would not pass all the error information - * to userspace. Record errno using scx_error() for cases scx_error() - * wasn't already invoked and exit indicating success so that the error - * is notified through ops.exit() with all the details. - * - * Flush scx_disable_work to ensure that error is reported before init - * completion. sch's base reference will be put by bpf_scx_unreg(). - */ - scx_error(sch, "scx_root_enable() failed (%d)", ret); - scx_flush_disable_work(sch); - cmd->ret = 0; -} - -#ifdef CONFIG_EXT_SUB_SCHED -/* verify that a scheduler can be attached to @cgrp and return the parent */ -static struct scx_sched *find_parent_sched(struct cgroup *cgrp) -{ - struct scx_sched *parent = cgrp->scx_sched; - struct scx_sched *pos; - - lockdep_assert_held(&scx_sched_lock); - - /* can't attach twice to the same cgroup */ - if (parent->cgrp == cgrp) - return ERR_PTR(-EBUSY); - - /* does $parent allow sub-scheds? */ - if (!parent->ops.sub_attach) - return ERR_PTR(-EOPNOTSUPP); - - /* can't insert between $parent and its exiting children */ - list_for_each_entry(pos, &parent->children, sibling) - if (cgroup_is_descendant(pos->cgrp, cgrp)) - return ERR_PTR(-EBUSY); - - return parent; -} - -static bool assert_task_ready_or_enabled(struct task_struct *p) -{ - u32 state = scx_get_task_state(p); - - switch (state) { - case SCX_TASK_READY: - case SCX_TASK_ENABLED: - return true; - default: - WARN_ONCE(true, "sched_ext: Invalid task state %d for %s[%d] during enabling sub sched", - state, p->comm, p->pid); - return false; - } -} - -static void scx_sub_enable_workfn(struct kthread_work *work) -{ - struct scx_enable_cmd *cmd = container_of(work, struct scx_enable_cmd, work); - struct sched_ext_ops *ops = cmd->ops; - struct cgroup *cgrp; - struct scx_sched *parent, *sch; - struct scx_task_iter sti; - struct task_struct *p; - s32 i, ret; - - mutex_lock(&scx_enable_mutex); - - if (!scx_enabled()) { - ret = -ENODEV; - goto out_unlock; - } - - /* See scx_root_enable_workfn() for the @ops->priv check. */ - if (rcu_access_pointer(ops->priv)) { - ret = -EBUSY; - goto out_unlock; - } - - cgrp = cgroup_get_from_id(ops->sub_cgroup_id); - if (IS_ERR(cgrp)) { - ret = PTR_ERR(cgrp); - goto out_unlock; - } - - raw_spin_lock_irq(&scx_sched_lock); - parent = find_parent_sched(cgrp); - if (IS_ERR(parent)) { - raw_spin_unlock_irq(&scx_sched_lock); - ret = PTR_ERR(parent); - goto out_put_cgrp; - } - kobject_get(&parent->kobj); - raw_spin_unlock_irq(&scx_sched_lock); - - /* scx_alloc_and_add_sched() consumes @cgrp whether it succeeds or not */ - sch = scx_alloc_and_add_sched(cmd, cgrp, parent); - kobject_put(&parent->kobj); - if (IS_ERR(sch)) { - ret = PTR_ERR(sch); - goto out_unlock; - } - - ret = scx_link_sched(sch); - if (ret) - goto err_disable; - - if (sch->level >= SCX_SUB_MAX_DEPTH) { - scx_error(sch, "max nesting depth %d violated", - SCX_SUB_MAX_DEPTH); - goto err_disable; - } - - if (sch->ops.init) { - ret = SCX_CALL_OP_RET(sch, init, NULL); - if (ret) { - ret = ops_sanitize_err(sch, "init", ret); - scx_error(sch, "ops.init() failed (%d)", ret); - goto err_disable; - } - sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; - } - - ret = scx_arena_pool_init(sch); - if (ret) - goto err_disable; - - ret = scx_set_cmask_scratch_alloc(sch); - if (ret) - goto err_disable; - - if (validate_ops(sch, ops)) - goto err_disable; - - struct scx_sub_attach_args sub_attach_args = { - .ops = &sch->ops, - .cgroup_path = sch->cgrp_path, - }; - - ret = SCX_CALL_OP_RET(parent, sub_attach, NULL, - &sub_attach_args); - if (ret) { - ret = ops_sanitize_err(sch, "sub_attach", ret); - scx_error(sch, "parent rejected (%d)", ret); - goto err_disable; - } - sch->sub_attached = true; - - scx_bypass(sch, true); - - for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++) - if (((void (**)(void))ops)[i]) - set_bit(i, sch->has_op); - - percpu_down_write(&scx_fork_rwsem); - scx_cgroup_lock(); - - /* - * Set cgroup->scx_sched's and check CSS_ONLINE. Either we see - * !CSS_ONLINE or scx_cgroup_lifetime_notify() sees and shoots us down. - */ - set_cgroup_sched(sch_cgroup(sch), sch); - if (!(cgrp->self.flags & CSS_ONLINE)) { - scx_error(sch, "cgroup is not online"); - goto err_unlock_and_disable; - } - - /* - * Initialize tasks for the new child $sch without exiting them for - * $parent so that the tasks can always be reverted back to $parent - * sched on child init failure. - */ - WARN_ON_ONCE(scx_enabling_sub_sched); - scx_enabling_sub_sched = sch; - - scx_task_iter_start(&sti, sch->cgrp); - while ((p = scx_task_iter_next_locked(&sti))) { - struct rq *rq; - struct rq_flags rf; - - /* - * Task iteration may visit the same task twice when racing - * against exiting. Use %SCX_TASK_SUB_INIT to mark tasks which - * finished __scx_init_task() and skip if set. - * - * A task may exit and get freed between __scx_init_task() - * completion and scx_enable_task(). In such cases, - * scx_disable_and_exit_task() must exit the task for both the - * parent and child scheds. - */ - if (p->scx.flags & SCX_TASK_SUB_INIT) - continue; - - /* @p is pinned by the iter; see scx_sub_disable() */ - get_task_struct(p); - - if (!assert_task_ready_or_enabled(p)) { - ret = -EINVAL; - goto abort; - } - - scx_task_iter_unlock(&sti); - - /* - * As $p is still on $parent, it can't be transitioned to INIT. - * Let's worry about task state later. Use __scx_init_task(). - */ - ret = __scx_init_task(sch, p, false); - if (ret) - goto abort; - - rq = task_rq_lock(p, &rf); - - if (scx_get_task_state(p) == SCX_TASK_DEAD) { - /* - * sched_ext_dead() raced us between __scx_init_task() - * and this rq lock and ran exit_task() on $parent (the - * sched @p was on at that point), not on @sch. @sch's - * just-completed init is owed an exit_task() and we - * issue it here. - */ - scx_sub_init_cancel_task(sch, p); - task_rq_unlock(rq, p, &rf); - put_task_struct(p); - continue; - } - - p->scx.flags |= SCX_TASK_SUB_INIT; - task_rq_unlock(rq, p, &rf); - - put_task_struct(p); - } - scx_task_iter_stop(&sti); - - /* - * All tasks are prepped. Disable/exit tasks for $parent and enable for - * the new @sch. - */ - scx_task_iter_start(&sti, sch->cgrp); - while ((p = scx_task_iter_next_locked(&sti))) { - /* - * Use clearing of %SCX_TASK_SUB_INIT to detect and skip - * duplicate iterations. - */ - if (!(p->scx.flags & SCX_TASK_SUB_INIT)) - continue; - - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { - /* - * $p must be either READY or ENABLED. If ENABLED, - * __scx_disabled_and_exit_task() first disables and - * makes it READY. However, after exiting $p, it will - * leave $p as READY. - */ - assert_task_ready_or_enabled(p); - __scx_disable_and_exit_task(parent, p); - - /* - * $p is now only initialized for @sch and READY, which - * is what we want. Assign it to @sch and enable. - */ - scx_set_task_sched(p, sch); - scx_enable_task(sch, p); - - p->scx.flags &= ~SCX_TASK_SUB_INIT; - } - } - scx_task_iter_stop(&sti); - - scx_enabling_sub_sched = NULL; - - scx_cgroup_unlock(); - percpu_up_write(&scx_fork_rwsem); - - scx_bypass(sch, false); - - pr_info("sched_ext: BPF sub-scheduler \"%s\" enabled\n", sch->ops.name); - kobject_uevent(&sch->kobj, KOBJ_ADD); - ret = 0; - goto out_unlock; - -out_put_cgrp: - cgroup_put(cgrp); -out_unlock: - mutex_unlock(&scx_enable_mutex); - cmd->ret = ret; - return; - -abort: - put_task_struct(p); - scx_task_iter_stop(&sti); - - /* - * Undo __scx_init_task() for tasks we marked. scx_enable_task() never - * ran for @sch on them, so calling scx_disable_task() here would invoke - * ops.disable() without a matching ops.enable(). scx_enabling_sub_sched - * must stay set until SUB_INIT is cleared from every marked task - - * scx_disable_and_exit_task() reads it when a task exits concurrently. - */ - scx_task_iter_start(&sti, sch->cgrp); - while ((p = scx_task_iter_next_locked(&sti))) { - if (p->scx.flags & SCX_TASK_SUB_INIT) { - scx_sub_init_cancel_task(sch, p); - p->scx.flags &= ~SCX_TASK_SUB_INIT; - } - } - scx_task_iter_stop(&sti); - scx_enabling_sub_sched = NULL; -err_unlock_and_disable: - /* we'll soon enter disable path, keep bypass on */ - scx_cgroup_unlock(); - percpu_up_write(&scx_fork_rwsem); -err_disable: - mutex_unlock(&scx_enable_mutex); - scx_flush_disable_work(sch); - cmd->ret = 0; -} - -static s32 scx_cgroup_lifetime_notify(struct notifier_block *nb, - unsigned long action, void *data) -{ - struct cgroup *cgrp = data; - struct cgroup *parent = cgroup_parent(cgrp); - - if (!cgroup_on_dfl(cgrp)) - return NOTIFY_OK; - - switch (action) { - case CGROUP_LIFETIME_ONLINE: - /* inherit ->scx_sched from $parent */ - if (parent) - rcu_assign_pointer(cgrp->scx_sched, parent->scx_sched); - break; - case CGROUP_LIFETIME_OFFLINE: - /* if there is a sched attached, shoot it down */ - if (cgrp->scx_sched && cgrp->scx_sched->cgrp == cgrp) - scx_exit(cgrp->scx_sched, SCX_EXIT_UNREG_KERN, - SCX_ECODE_RSN_CGROUP_OFFLINE, - "cgroup %llu going offline", cgroup_id(cgrp)); - break; - } - - return NOTIFY_OK; -} - -static struct notifier_block scx_cgroup_lifetime_nb = { - .notifier_call = scx_cgroup_lifetime_notify, -}; - -static s32 __init scx_cgroup_lifetime_notifier_init(void) -{ - return blocking_notifier_chain_register(&cgroup_lifetime_notifier, - &scx_cgroup_lifetime_nb); -} -core_initcall(scx_cgroup_lifetime_notifier_init); -#endif /* CONFIG_EXT_SUB_SCHED */ - -static s32 scx_enable(struct scx_enable_cmd *cmd, struct bpf_link *link) -{ - static struct kthread_worker *helper; - static DEFINE_MUTEX(helper_mutex); - - if (housekeeping_enabled(HK_TYPE_DOMAIN_BOOT)) { - pr_err("sched_ext: Not compatible with \"isolcpus=\" domain isolation\n"); - return -EINVAL; - } - - if (!READ_ONCE(helper)) { - mutex_lock(&helper_mutex); - if (!helper) { - struct kthread_worker *w = - kthread_run_worker(0, "scx_enable_helper"); - if (IS_ERR_OR_NULL(w)) { - mutex_unlock(&helper_mutex); - return -ENOMEM; - } - sched_set_fifo(w->task); - WRITE_ONCE(helper, w); - } - mutex_unlock(&helper_mutex); - } - -#ifdef CONFIG_EXT_SUB_SCHED - if (cmd->ops->sub_cgroup_id > 1) - kthread_init_work(&cmd->work, scx_sub_enable_workfn); - else -#endif /* CONFIG_EXT_SUB_SCHED */ - kthread_init_work(&cmd->work, scx_root_enable_workfn); - - kthread_queue_work(READ_ONCE(helper), &cmd->work); - kthread_flush_work(&cmd->work); - return cmd->ret; -} - - -/******************************************************************************** - * bpf_struct_ops plumbing. - */ -#include -#include -#include - -static const struct btf_type *task_struct_type; - -static bool bpf_scx_is_valid_access(int off, int size, - enum bpf_access_type type, - const struct bpf_prog *prog, - struct bpf_insn_access_aux *info) -{ - if (type != BPF_READ) - return false; - if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) - return false; - if (off % size != 0) - return false; - - return btf_ctx_access(off, size, type, prog, info); -} - -static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, - const struct bpf_reg_state *reg, int off, - int size) -{ - const struct btf_type *t; - - t = btf_type_by_id(reg->btf, reg->btf_id); - if (t == task_struct_type) { - /* - * COMPAT: Will be removed in v6.23. - */ - if ((off >= offsetof(struct task_struct, scx.slice) && - off + size <= offsetofend(struct task_struct, scx.slice)) || - (off >= offsetof(struct task_struct, scx.dsq_vtime) && - off + size <= offsetofend(struct task_struct, scx.dsq_vtime))) { - pr_warn("sched_ext: Writing directly to p->scx.slice/dsq_vtime is deprecated, use scx_bpf_task_set_slice/dsq_vtime()"); - return SCALAR_VALUE; - } - - if (off >= offsetof(struct task_struct, scx.disallow) && - off + size <= offsetofend(struct task_struct, scx.disallow)) - return SCALAR_VALUE; - } - - return -EACCES; -} - -static const struct bpf_verifier_ops bpf_scx_verifier_ops = { - .get_func_proto = bpf_base_func_proto, - .is_valid_access = bpf_scx_is_valid_access, - .btf_struct_access = bpf_scx_btf_struct_access, -}; - -static int bpf_scx_init_member(const struct btf_type *t, - const struct btf_member *member, - void *kdata, const void *udata) -{ - const struct sched_ext_ops *uops = udata; - struct sched_ext_ops *ops = kdata; - u32 moff = __btf_member_bit_offset(t, member) / 8; - int ret; - - switch (moff) { - case offsetof(struct sched_ext_ops, dispatch_max_batch): - if (*(u32 *)(udata + moff) > INT_MAX) - return -E2BIG; - ops->dispatch_max_batch = *(u32 *)(udata + moff); - return 1; - case offsetof(struct sched_ext_ops, flags): - if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS) - return -EINVAL; - ops->flags = *(u64 *)(udata + moff); - return 1; - case offsetof(struct sched_ext_ops, name): - ret = bpf_obj_name_cpy(ops->name, uops->name, - sizeof(ops->name)); - if (ret < 0) - return ret; - if (ret == 0) - return -EINVAL; - return 1; - case offsetof(struct sched_ext_ops, timeout_ms): - if (msecs_to_jiffies(*(u32 *)(udata + moff)) > - SCX_WATCHDOG_MAX_TIMEOUT) - return -E2BIG; - ops->timeout_ms = *(u32 *)(udata + moff); - return 1; - case offsetof(struct sched_ext_ops, exit_dump_len): - ops->exit_dump_len = - *(u32 *)(udata + moff) ?: SCX_EXIT_DUMP_DFL_LEN; - return 1; - case offsetof(struct sched_ext_ops, hotplug_seq): - ops->hotplug_seq = *(u64 *)(udata + moff); - return 1; -#ifdef CONFIG_EXT_SUB_SCHED - case offsetof(struct sched_ext_ops, sub_cgroup_id): - ops->sub_cgroup_id = *(u64 *)(udata + moff); - return 1; -#endif /* CONFIG_EXT_SUB_SCHED */ - } - - return 0; -} - -#ifdef CONFIG_EXT_SUB_SCHED -static void scx_pstack_recursion_on_dispatch(struct bpf_prog *prog) -{ - struct scx_sched *sch; - - guard(rcu)(); - sch = scx_prog_sched(prog->aux); - if (unlikely(!sch)) - return; - - scx_error(sch, "dispatch recursion detected"); -} -#endif /* CONFIG_EXT_SUB_SCHED */ - -static int bpf_scx_check_member(const struct btf_type *t, - const struct btf_member *member, - const struct bpf_prog *prog) -{ - u32 moff = __btf_member_bit_offset(t, member) / 8; - - switch (moff) { - case offsetof(struct sched_ext_ops, init_task): -#ifdef CONFIG_EXT_GROUP_SCHED - case offsetof(struct sched_ext_ops, cgroup_init): - case offsetof(struct sched_ext_ops, cgroup_exit): - case offsetof(struct sched_ext_ops, cgroup_prep_move): -#endif - case offsetof(struct sched_ext_ops, cpu_online): - case offsetof(struct sched_ext_ops, cpu_offline): - case offsetof(struct sched_ext_ops, init): - case offsetof(struct sched_ext_ops, exit): - case offsetof(struct sched_ext_ops, sub_attach): - case offsetof(struct sched_ext_ops, sub_detach): - break; - default: - if (prog->sleepable) - return -EINVAL; - } - -#ifdef CONFIG_EXT_SUB_SCHED - /* - * Enable private stack for operations that can nest along the - * hierarchy. - * - * XXX - Ideally, we should only do this for scheds that allow - * sub-scheds and sub-scheds themselves but I don't know how to access - * struct_ops from here. - */ - switch (moff) { - case offsetof(struct sched_ext_ops, dispatch): - prog->aux->priv_stack_requested = true; - prog->aux->recursion_detected = scx_pstack_recursion_on_dispatch; - } -#endif /* CONFIG_EXT_SUB_SCHED */ - - return 0; -} - -static int bpf_scx_reg(void *kdata, struct bpf_link *link) -{ - struct scx_enable_cmd cmd = { .ops = kdata }; - - return scx_enable(&cmd, link); -} - -struct scx_arena_scan { - struct bpf_map *arena; - int err; -}; - -/* - * The verifier enforces one arena per BPF program, so each struct_ops - * member prog contributes at most one arena via bpf_prog_arena(). - * Require all non-NULL contributions to match. - */ -static int scx_arena_scan_prog(struct bpf_prog *prog, void *data) -{ - struct scx_arena_scan *s = data; - struct bpf_map *arena = NULL; - - /* arena.o, which defines these, is built only on MMU && 64BIT */ -#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) - arena = bpf_prog_arena(prog); -#endif - if (!arena) - return 0; - if (s->arena && s->arena != arena) { - s->err = -EINVAL; - return 1; - } - s->arena = arena; - return 0; -} - -static int bpf_scx_reg_cid(void *kdata, struct bpf_link *link) -{ - struct scx_enable_cmd cmd = { .ops_cid = kdata, .is_cid_type = true }; - struct scx_arena_scan scan = {}; - int ret; - - bpf_struct_ops_for_each_prog(kdata, scx_arena_scan_prog, &scan); - if (scan.err) { - pr_err("sched_ext: cid-form scheduler uses multiple arena maps\n"); - return scan.err; - } - if (!scan.arena) { - pr_err("sched_ext: cid-form scheduler must use a BPF arena map\n"); - return -EINVAL; - } - - bpf_map_inc(scan.arena); - cmd.arena_map = scan.arena; - ret = scx_enable(&cmd, link); - if (cmd.arena_map) /* not consumed by scx_alloc_and_add_sched() */ - bpf_map_put(cmd.arena_map); - return ret; -} - -static void bpf_scx_unreg(void *kdata, struct bpf_link *link) -{ - struct sched_ext_ops *ops = kdata; - struct scx_sched *sch = rcu_dereference_protected(ops->priv, true); - - scx_disable(sch, SCX_EXIT_UNREG); - scx_flush_disable_work(sch); - RCU_INIT_POINTER(ops->priv, NULL); - kobject_put(&sch->kobj); -} - -static int bpf_scx_init(struct btf *btf) -{ - task_struct_type = btf_type_by_id(btf, btf_tracing_ids[BTF_TRACING_TYPE_TASK]); - - return 0; -} - -static int bpf_scx_update(void *kdata, void *old_kdata, struct bpf_link *link) -{ - /* - * sched_ext does not support updating the actively-loaded BPF - * scheduler, as registering a BPF scheduler can always fail if the - * scheduler returns an error code for e.g. ops.init(), ops.init_task(), - * etc. Similarly, we can always race with unregistration happening - * elsewhere, such as with sysrq. - */ - return -EOPNOTSUPP; -} - -static int bpf_scx_validate(void *kdata) -{ - return 0; -} - -static s32 sched_ext_ops__select_cpu(struct task_struct *p, s32 prev_cpu, u64 wake_flags) { return -EINVAL; } -static void sched_ext_ops__enqueue(struct task_struct *p, u64 enq_flags) {} -static void sched_ext_ops__dequeue(struct task_struct *p, u64 enq_flags) {} -static void sched_ext_ops__dispatch(s32 prev_cpu, struct task_struct *prev__nullable) {} -static void sched_ext_ops__tick(struct task_struct *p) {} -static void sched_ext_ops__runnable(struct task_struct *p, u64 enq_flags) {} -static void sched_ext_ops__running(struct task_struct *p) {} -static void sched_ext_ops__stopping(struct task_struct *p, bool runnable) {} -static void sched_ext_ops__quiescent(struct task_struct *p, u64 deq_flags) {} -static bool sched_ext_ops__yield(struct task_struct *from, struct task_struct *to__nullable) { return false; } -static bool sched_ext_ops__core_sched_before(struct task_struct *a, struct task_struct *b) { return false; } -static void sched_ext_ops__set_weight(struct task_struct *p, u32 weight) {} -static void sched_ext_ops__set_cpumask(struct task_struct *p, const struct cpumask *mask) {} -static void sched_ext_ops__update_idle(s32 cpu, bool idle) {} -static void sched_ext_ops__cpu_acquire(s32 cpu, struct scx_cpu_acquire_args *args) {} -static void sched_ext_ops__cpu_release(s32 cpu, struct scx_cpu_release_args *args) {} -static s32 sched_ext_ops__init_task(struct task_struct *p, struct scx_init_task_args *args) { return -EINVAL; } -static void sched_ext_ops__exit_task(struct task_struct *p, struct scx_exit_task_args *args) {} -static void sched_ext_ops__enable(struct task_struct *p) {} -static void sched_ext_ops__disable(struct task_struct *p) {} -#ifdef CONFIG_EXT_GROUP_SCHED -static s32 sched_ext_ops__cgroup_init(struct cgroup *cgrp, struct scx_cgroup_init_args *args) { return -EINVAL; } -static void sched_ext_ops__cgroup_exit(struct cgroup *cgrp) {} -static s32 sched_ext_ops__cgroup_prep_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) { return -EINVAL; } -static void sched_ext_ops__cgroup_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) {} -static void sched_ext_ops__cgroup_cancel_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) {} -static void sched_ext_ops__cgroup_set_weight(struct cgroup *cgrp, u32 weight) {} -static void sched_ext_ops__cgroup_set_bandwidth(struct cgroup *cgrp, u64 period_us, u64 quota_us, u64 burst_us) {} -static void sched_ext_ops__cgroup_set_idle(struct cgroup *cgrp, bool idle) {} -#endif /* CONFIG_EXT_GROUP_SCHED */ -static s32 sched_ext_ops__sub_attach(struct scx_sub_attach_args *args) { return -EINVAL; } -static void sched_ext_ops__sub_detach(struct scx_sub_detach_args *args) {} -static void sched_ext_ops__cpu_online(s32 cpu) {} -static void sched_ext_ops__cpu_offline(s32 cpu) {} -static s32 sched_ext_ops__init(void) { return -EINVAL; } -static void sched_ext_ops__exit(struct scx_exit_info *info) {} -static void sched_ext_ops__dump(struct scx_dump_ctx *ctx) {} -static void sched_ext_ops__dump_cpu(struct scx_dump_ctx *ctx, s32 cpu, bool idle) {} -static void sched_ext_ops__dump_task(struct scx_dump_ctx *ctx, struct task_struct *p) {} - -static struct sched_ext_ops __bpf_ops_sched_ext_ops = { - .select_cpu = sched_ext_ops__select_cpu, - .enqueue = sched_ext_ops__enqueue, - .dequeue = sched_ext_ops__dequeue, - .dispatch = sched_ext_ops__dispatch, - .tick = sched_ext_ops__tick, - .runnable = sched_ext_ops__runnable, - .running = sched_ext_ops__running, - .stopping = sched_ext_ops__stopping, - .quiescent = sched_ext_ops__quiescent, - .yield = sched_ext_ops__yield, - .core_sched_before = sched_ext_ops__core_sched_before, - .set_weight = sched_ext_ops__set_weight, - .set_cpumask = sched_ext_ops__set_cpumask, - .update_idle = sched_ext_ops__update_idle, - .cpu_acquire = sched_ext_ops__cpu_acquire, - .cpu_release = sched_ext_ops__cpu_release, - .init_task = sched_ext_ops__init_task, - .exit_task = sched_ext_ops__exit_task, - .enable = sched_ext_ops__enable, - .disable = sched_ext_ops__disable, -#ifdef CONFIG_EXT_GROUP_SCHED - .cgroup_init = sched_ext_ops__cgroup_init, - .cgroup_exit = sched_ext_ops__cgroup_exit, - .cgroup_prep_move = sched_ext_ops__cgroup_prep_move, - .cgroup_move = sched_ext_ops__cgroup_move, - .cgroup_cancel_move = sched_ext_ops__cgroup_cancel_move, - .cgroup_set_weight = sched_ext_ops__cgroup_set_weight, - .cgroup_set_bandwidth = sched_ext_ops__cgroup_set_bandwidth, - .cgroup_set_idle = sched_ext_ops__cgroup_set_idle, -#endif - .sub_attach = sched_ext_ops__sub_attach, - .sub_detach = sched_ext_ops__sub_detach, - .cpu_online = sched_ext_ops__cpu_online, - .cpu_offline = sched_ext_ops__cpu_offline, - .init = sched_ext_ops__init, - .exit = sched_ext_ops__exit, - .dump = sched_ext_ops__dump, - .dump_cpu = sched_ext_ops__dump_cpu, - .dump_task = sched_ext_ops__dump_task, -}; - -static struct bpf_struct_ops bpf_sched_ext_ops = { - .verifier_ops = &bpf_scx_verifier_ops, - .reg = bpf_scx_reg, - .unreg = bpf_scx_unreg, - .check_member = bpf_scx_check_member, - .init_member = bpf_scx_init_member, - .init = bpf_scx_init, - .update = bpf_scx_update, - .validate = bpf_scx_validate, - .name = "sched_ext_ops", - .owner = THIS_MODULE, - .cfi_stubs = &__bpf_ops_sched_ext_ops -}; - -/* - * cid-form cfi stubs. Stubs whose signatures match the cpu-form (param types - * identical, only param names differ across structs) are reused; only - * set_cmask needs a fresh stub since the second argument type differs. - */ -static void sched_ext_ops_cid__set_cmask(struct task_struct *p, - const struct scx_cmask *cmask) {} - -static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { - .select_cid = sched_ext_ops__select_cpu, - .enqueue = sched_ext_ops__enqueue, - .dequeue = sched_ext_ops__dequeue, - .dispatch = sched_ext_ops__dispatch, - .tick = sched_ext_ops__tick, - .runnable = sched_ext_ops__runnable, - .running = sched_ext_ops__running, - .stopping = sched_ext_ops__stopping, - .quiescent = sched_ext_ops__quiescent, - .yield = sched_ext_ops__yield, - .core_sched_before = sched_ext_ops__core_sched_before, - .set_weight = sched_ext_ops__set_weight, - .set_cmask = sched_ext_ops_cid__set_cmask, - .update_idle = sched_ext_ops__update_idle, - .init_task = sched_ext_ops__init_task, - .exit_task = sched_ext_ops__exit_task, - .enable = sched_ext_ops__enable, - .disable = sched_ext_ops__disable, -#ifdef CONFIG_EXT_GROUP_SCHED - .cgroup_init = sched_ext_ops__cgroup_init, - .cgroup_exit = sched_ext_ops__cgroup_exit, - .cgroup_prep_move = sched_ext_ops__cgroup_prep_move, - .cgroup_move = sched_ext_ops__cgroup_move, - .cgroup_cancel_move = sched_ext_ops__cgroup_cancel_move, - .cgroup_set_weight = sched_ext_ops__cgroup_set_weight, - .cgroup_set_bandwidth = sched_ext_ops__cgroup_set_bandwidth, - .cgroup_set_idle = sched_ext_ops__cgroup_set_idle, -#endif - .sub_attach = sched_ext_ops__sub_attach, - .sub_detach = sched_ext_ops__sub_detach, - .cid_online = sched_ext_ops__cpu_online, - .cid_offline = sched_ext_ops__cpu_offline, - .init = sched_ext_ops__init, - .exit = sched_ext_ops__exit, - .dump = sched_ext_ops__dump, - .dump_cid = sched_ext_ops__dump_cpu, - .dump_task = sched_ext_ops__dump_task, -}; - -/* - * The cid-form struct_ops shares all bpf_struct_ops hooks with the cpu form. - * init_member, check_member, reg, unreg, etc. process kdata as the byte block - * verified to match by the BUILD_BUG_ON checks in scx_init(). - */ -static struct bpf_struct_ops bpf_sched_ext_ops_cid = { - .verifier_ops = &bpf_scx_verifier_ops, - .reg = bpf_scx_reg_cid, - .unreg = bpf_scx_unreg, - .check_member = bpf_scx_check_member, - .init_member = bpf_scx_init_member, - .init = bpf_scx_init, - .update = bpf_scx_update, - .validate = bpf_scx_validate, - .name = "sched_ext_ops_cid", - .owner = THIS_MODULE, - .cfi_stubs = &__bpf_ops_sched_ext_ops_cid -}; - - -/******************************************************************************** - * System integration and init. - */ - -static void sysrq_handle_sched_ext_reset(u8 key) -{ - struct scx_sched *sch; - - sch = rcu_dereference(scx_root); - if (likely(sch)) - scx_disable(sch, SCX_EXIT_SYSRQ); - else - pr_info("sched_ext: BPF schedulers not loaded\n"); -} - -static const struct sysrq_key_op sysrq_sched_ext_reset_op = { - .handler = sysrq_handle_sched_ext_reset, - .help_msg = "reset-sched-ext(S)", - .action_msg = "Disable sched_ext and revert all tasks to CFS", - .enable_mask = SYSRQ_ENABLE_RTNICE, -}; - -static void sysrq_handle_sched_ext_dump(u8 key) -{ - struct scx_exit_info ei = { - .kind = SCX_EXIT_NONE, - .exit_cpu = -1, - .reason = "SysRq-D", - }; - struct scx_sched *sch; - - list_for_each_entry_rcu(sch, &scx_sched_all, all) - scx_dump_state(sch, &ei, 0, false); -} - -static const struct sysrq_key_op sysrq_sched_ext_dump_op = { - .handler = sysrq_handle_sched_ext_dump, - .help_msg = "dump-sched-ext(D)", - .action_msg = "Trigger sched_ext debug dump", - .enable_mask = SYSRQ_ENABLE_RTNICE, -}; - -static bool can_skip_idle_kick(struct rq *rq) -{ - lockdep_assert_rq_held(rq); - - /* - * We can skip idle kicking if @rq is going to go through at least one - * full SCX scheduling cycle before going idle. Just checking whether - * curr is not idle is insufficient because we could be racing - * balance_one() trying to pull the next task from a remote rq, which - * may fail, and @rq may become idle afterwards. - * - * The race window is small and we don't and can't guarantee that @rq is - * only kicked while idle anyway. Skip only when sure. - */ - return !is_idle_task(rq->curr) && !(rq->scx.flags & SCX_RQ_IN_BALANCE); -} - -static bool kick_one_cpu(s32 cpu, struct rq *this_rq, unsigned long *ksyncs) -{ - struct rq *rq = cpu_rq(cpu); - struct scx_rq *this_scx = &this_rq->scx; - const struct sched_class *cur_class; - bool should_wait = false; - unsigned long flags; - - raw_spin_rq_lock_irqsave(rq, flags); - cur_class = rq->curr->sched_class; - - /* - * During CPU hotplug, a CPU may depend on kicking itself to make - * forward progress. Allow kicking self regardless of online state. If - * @cpu is running a higher class task, we have no control over @cpu. - * Skip kicking. - */ - if ((cpu_online(cpu) || cpu == cpu_of(this_rq)) && - !sched_class_above(cur_class, &ext_sched_class)) { - if (cpumask_test_cpu(cpu, this_scx->cpus_to_preempt)) { - if (cur_class == &ext_sched_class) - rq->curr->scx.slice = 0; - cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt); - } - - if (cpumask_test_cpu(cpu, this_scx->cpus_to_wait)) { - if (cur_class == &ext_sched_class) { - cpumask_set_cpu(cpu, this_scx->cpus_to_sync); - ksyncs[cpu] = rq->scx.kick_sync; - should_wait = true; - } - cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); - } - - resched_curr(rq); - } else { - cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt); - cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); - } - - raw_spin_rq_unlock_irqrestore(rq, flags); - - return should_wait; -} - -static void kick_one_cpu_if_idle(s32 cpu, struct rq *this_rq) -{ - struct rq *rq = cpu_rq(cpu); - unsigned long flags; - - raw_spin_rq_lock_irqsave(rq, flags); - - if (!can_skip_idle_kick(rq) && - (cpu_online(cpu) || cpu == cpu_of(this_rq))) - resched_curr(rq); - - raw_spin_rq_unlock_irqrestore(rq, flags); -} - -static void kick_cpus_irq_workfn(struct irq_work *irq_work) -{ - struct rq *this_rq = this_rq(); - struct scx_rq *this_scx = &this_rq->scx; - struct scx_kick_syncs __rcu *ksyncs_pcpu = __this_cpu_read(scx_kick_syncs); - bool should_wait = false; - unsigned long *ksyncs; - s32 cpu; - - /* can race with free_kick_syncs() during scheduler disable */ - if (unlikely(!ksyncs_pcpu)) - return; - - ksyncs = rcu_dereference_bh(ksyncs_pcpu)->syncs; - - for_each_cpu(cpu, this_scx->cpus_to_kick) { - should_wait |= kick_one_cpu(cpu, this_rq, ksyncs); - cpumask_clear_cpu(cpu, this_scx->cpus_to_kick); - cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle); - } - - for_each_cpu(cpu, this_scx->cpus_to_kick_if_idle) { - kick_one_cpu_if_idle(cpu, this_rq); - cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle); - } - - /* - * Can't wait in hardirq — kick_sync can't advance, deadlocking if - * CPUs wait for each other. Defer to kick_sync_wait_bal_cb(). - */ - if (should_wait) { - raw_spin_rq_lock(this_rq); - this_scx->kick_sync_pending = true; - resched_curr(this_rq); - raw_spin_rq_unlock(this_rq); - } -} - -/** - * print_scx_info - print out sched_ext scheduler state - * @log_lvl: the log level to use when printing - * @p: target task - * - * If a sched_ext scheduler is enabled, print the name and state of the - * scheduler. If @p is on sched_ext, print further information about the task. - * - * This function can be safely called on any task as long as the task_struct - * itself is accessible. While safe, this function isn't synchronized and may - * print out mixups or garbages of limited length. - */ -void print_scx_info(const char *log_lvl, struct task_struct *p) -{ - struct scx_sched *sch; - enum scx_enable_state state = scx_enable_state(); - const char *all = READ_ONCE(scx_switching_all) ? "+all" : ""; - char runnable_at_buf[22] = "?"; - struct sched_class *class; - unsigned long runnable_at; - - guard(rcu)(); - - sch = scx_task_sched_rcu(p); - - if (!sch) - return; - - /* - * Carefully check if the task was running on sched_ext, and then - * carefully copy the time it's been runnable, and its state. - */ - if (copy_from_kernel_nofault(&class, &p->sched_class, sizeof(class)) || - class != &ext_sched_class) { - printk("%sSched_ext: %s (%s%s)", log_lvl, sch->ops.name, - scx_enable_state_str[state], all); - return; - } - - if (!copy_from_kernel_nofault(&runnable_at, &p->scx.runnable_at, - sizeof(runnable_at))) - scnprintf(runnable_at_buf, sizeof(runnable_at_buf), "%+ldms", - jiffies_delta_msecs(runnable_at, jiffies)); - - /* print everything onto one line to conserve console space */ - printk("%sSched_ext: %s (%s%s), task: runnable_at=%s", - log_lvl, sch->ops.name, scx_enable_state_str[state], all, - runnable_at_buf); -} - -static int scx_pm_handler(struct notifier_block *nb, unsigned long event, void *ptr) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = rcu_dereference(scx_root); - if (!sch) - return NOTIFY_OK; - - /* - * SCX schedulers often have userspace components which are sometimes - * involved in critial scheduling paths. PM operations involve freezing - * userspace which can lead to scheduling misbehaviors including stalls. - * Let's bypass while PM operations are in progress. - */ - switch (event) { - case PM_HIBERNATION_PREPARE: - case PM_SUSPEND_PREPARE: - case PM_RESTORE_PREPARE: - scx_bypass(sch, true); - break; - case PM_POST_HIBERNATION: - case PM_POST_SUSPEND: - case PM_POST_RESTORE: - scx_bypass(sch, false); - break; - } - - return NOTIFY_OK; -} - -static struct notifier_block scx_pm_notifier = { - .notifier_call = scx_pm_handler, -}; - -void __init init_sched_ext_class(void) -{ - s32 cpu, v; - - /* - * The following is to prevent the compiler from optimizing out the enum - * definitions so that BPF scheduler implementations can use them - * through the generated vmlinux.h. - */ - WRITE_ONCE(v, SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | SCX_KICK_PREEMPT | - SCX_TG_ONLINE); - - scx_idle_init_masks(); - - for_each_possible_cpu(cpu) { - struct rq *rq = cpu_rq(cpu); - int n = cpu_to_node(cpu); - - /* local_dsq's sch will be set during scx_root_enable() */ - BUG_ON(init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL)); - - INIT_LIST_HEAD(&rq->scx.runnable_list); - INIT_LIST_HEAD(&rq->scx.ddsp_deferred_locals); - - BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_kick, GFP_KERNEL, n)); - BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_kick_if_idle, GFP_KERNEL, n)); - BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_preempt, GFP_KERNEL, n)); - BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_wait, GFP_KERNEL, n)); - BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_sync, GFP_KERNEL, n)); - raw_spin_lock_init(&rq->scx.deferred_reenq_lock); - INIT_LIST_HEAD(&rq->scx.deferred_reenq_locals); - INIT_LIST_HEAD(&rq->scx.deferred_reenq_users); - rq->scx.deferred_irq_work = IRQ_WORK_INIT_HARD(deferred_irq_workfn); - rq->scx.kick_cpus_irq_work = IRQ_WORK_INIT_HARD(kick_cpus_irq_workfn); - - if (cpu_online(cpu)) - cpu_rq(cpu)->scx.flags |= SCX_RQ_ONLINE; - } - - register_sysrq_key('S', &sysrq_sched_ext_reset_op); - register_sysrq_key('D', &sysrq_sched_ext_dump_op); - INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn); - -#ifdef CONFIG_EXT_SUB_SCHED - BUG_ON(rhashtable_init(&scx_sched_hash, &scx_sched_hash_params)); -#endif /* CONFIG_EXT_SUB_SCHED */ -} - - -/******************************************************************************** - * Helpers that can be called from the BPF scheduler. - */ -static bool scx_vet_enq_flags(struct scx_sched *sch, u64 dsq_id, u64 *enq_flags) -{ - bool is_local = dsq_id == SCX_DSQ_LOCAL || - (dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON; - - if (*enq_flags & SCX_ENQ_IMMED) { - if (unlikely(!is_local)) { - scx_error(sch, "SCX_ENQ_IMMED on a non-local DSQ 0x%llx", dsq_id); - return false; - } - } else if ((sch->ops.flags & SCX_OPS_ALWAYS_ENQ_IMMED) && is_local) { - *enq_flags |= SCX_ENQ_IMMED; - } - - return true; -} - -static bool scx_dsq_insert_preamble(struct scx_sched *sch, struct task_struct *p, - u64 dsq_id, u64 *enq_flags) -{ - lockdep_assert_irqs_disabled(); - - if (unlikely(!p)) { - scx_error(sch, "called with NULL task"); - return false; - } - - if (unlikely(*enq_flags & __SCX_ENQ_INTERNAL_MASK)) { - scx_error(sch, "invalid enq_flags 0x%llx", *enq_flags); - return false; - } - - /* see SCX_EV_INSERT_NOT_OWNED definition */ - if (unlikely(!scx_task_on_sched(sch, p))) { - __scx_add_event(sch, SCX_EV_INSERT_NOT_OWNED, 1); - return false; - } - - if (!scx_vet_enq_flags(sch, dsq_id, enq_flags)) - return false; - - return true; -} - -static void scx_dsq_insert_commit(struct scx_sched *sch, struct task_struct *p, - u64 dsq_id, u64 enq_flags) -{ - struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; - struct task_struct *ddsp_task; - - ddsp_task = __this_cpu_read(direct_dispatch_task); - if (ddsp_task) { - mark_direct_dispatch(sch, ddsp_task, p, dsq_id, enq_flags); - return; - } - - if (unlikely(dspc->cursor >= sch->dsp_max_batch)) { - scx_error(sch, "dispatch buffer overflow"); - return; - } - - dspc->buf[dspc->cursor++] = (struct scx_dsp_buf_ent){ - .task = p, - .qseq = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_QSEQ_MASK, - .dsq_id = dsq_id, - .enq_flags = enq_flags, - }; -} - -__bpf_kfunc_start_defs(); - -/** - * scx_bpf_dsq_insert - Insert a task into the FIFO queue of a DSQ - * @p: task_struct to insert - * @dsq_id: DSQ to insert into - * @slice: duration @p can run for in nsecs, 0 to keep the current value - * @enq_flags: SCX_ENQ_* - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Insert @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe to - * call this function spuriously. Can be called from ops.enqueue(), - * ops.select_cpu(), and ops.dispatch(). - * - * When called from ops.select_cpu() or ops.enqueue(), it's for direct dispatch - * and @p must match the task being enqueued. - * - * When called from ops.select_cpu(), @enq_flags and @dsp_id are stored, and @p - * will be directly inserted into the corresponding dispatch queue after - * ops.select_cpu() returns. If @p is inserted into SCX_DSQ_LOCAL, it will be - * inserted into the local DSQ of the CPU returned by ops.select_cpu(). - * @enq_flags are OR'd with the enqueue flags on the enqueue path before the - * task is inserted. - * - * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id - * and this function can be called upto ops.dispatch_max_batch times to insert - * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the - * remaining slots. scx_bpf_dsq_move_to_local() flushes the batch and resets the - * counter. - * - * This function doesn't have any locking restrictions and may be called under - * BPF locks (in the future when BPF introduces more flexible locking). - * - * @p is allowed to run for @slice. The scheduling path is triggered on slice - * exhaustion. If zero, the current residual slice is maintained. If - * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with - * scx_bpf_kick_cpu() to trigger scheduling. - * - * Returns %true on successful insertion, %false on failure. On the root - * scheduler, %false return triggers scheduler abort and the caller doesn't need - * to check the return value. - */ -__bpf_kfunc bool scx_bpf_dsq_insert___v2(struct task_struct *p, u64 dsq_id, - u64 slice, u64 enq_flags, - const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return false; - - if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags)) - return false; - - if (slice) - p->scx.slice = slice; - else - p->scx.slice = p->scx.slice ?: 1; - - scx_dsq_insert_commit(sch, p, dsq_id, enq_flags); - - return true; -} - -/* - * COMPAT: Will be removed in v6.23 along with the ___v2 suffix. - */ -__bpf_kfunc void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, - u64 slice, u64 enq_flags, - const struct bpf_prog_aux *aux) -{ - scx_bpf_dsq_insert___v2(p, dsq_id, slice, enq_flags, aux); -} - -static bool scx_dsq_insert_vtime(struct scx_sched *sch, struct task_struct *p, - u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) -{ - if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags)) - return false; - - if (slice) - p->scx.slice = slice; - else - p->scx.slice = p->scx.slice ?: 1; - - p->scx.dsq_vtime = vtime; - - scx_dsq_insert_commit(sch, p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); - - return true; -} - -struct scx_bpf_dsq_insert_vtime_args { - /* @p can't be packed together as KF_RCU is not transitive */ - u64 dsq_id; - u64 slice; - u64 vtime; - u64 enq_flags; -}; - -/** - * __scx_bpf_dsq_insert_vtime - Arg-wrapped vtime DSQ insertion - * @p: task_struct to insert - * @args: struct containing the rest of the arguments - * @args->dsq_id: DSQ to insert into - * @args->slice: duration @p can run for in nsecs, 0 to keep the current value - * @args->vtime: @p's ordering inside the vtime-sorted queue of the target DSQ - * @args->enq_flags: SCX_ENQ_* - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Wrapper kfunc that takes arguments via struct to work around BPF's 5 argument - * limit. BPF programs should use scx_bpf_dsq_insert_vtime() which is provided - * as an inline wrapper in common.bpf.h. - * - * Insert @p into the vtime priority queue of the DSQ identified by - * @args->dsq_id. Tasks queued into the priority queue are ordered by - * @args->vtime. All other aspects are identical to scx_bpf_dsq_insert(). - * - * @args->vtime ordering is according to time_before64() which considers - * wrapping. A numerically larger vtime may indicate an earlier position in the - * ordering and vice-versa. - * - * A DSQ can only be used as a FIFO or priority queue at any given time and this - * function must not be called on a DSQ which already has one or more FIFO tasks - * queued and vice-versa. Also, the built-in DSQs (SCX_DSQ_LOCAL and - * SCX_DSQ_GLOBAL) cannot be used as priority queues. - * - * Returns %true on successful insertion, %false on failure. On the root - * scheduler, %false return triggers scheduler abort and the caller doesn't need - * to check the return value. - */ -__bpf_kfunc bool -__scx_bpf_dsq_insert_vtime(struct task_struct *p, - struct scx_bpf_dsq_insert_vtime_args *args, - const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return false; - - return scx_dsq_insert_vtime(sch, p, args->dsq_id, args->slice, - args->vtime, args->enq_flags); -} - -/* - * COMPAT: Will be removed in v6.23. - */ -__bpf_kfunc void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id, - u64 slice, u64 vtime, u64 enq_flags) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = rcu_dereference(scx_root); - if (unlikely(!sch)) - return; - -#ifdef CONFIG_EXT_SUB_SCHED - /* - * Disallow if any sub-scheds are attached. There is no way to tell - * which scheduler called us, just error out @p's scheduler. - */ - if (unlikely(!list_empty(&sch->children))) { - scx_error(scx_task_sched(p), "__scx_bpf_dsq_insert_vtime() must be used"); - return; - } -#endif - - scx_dsq_insert_vtime(sch, p, dsq_id, slice, vtime, enq_flags); -} - -__bpf_kfunc_end_defs(); - -BTF_KFUNCS_START(scx_kfunc_ids_enqueue_dispatch) -BTF_ID_FLAGS(func, scx_bpf_dsq_insert, KF_IMPLICIT_ARGS | KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_dsq_insert___v2, KF_IMPLICIT_ARGS | KF_RCU) -BTF_ID_FLAGS(func, __scx_bpf_dsq_insert_vtime, KF_IMPLICIT_ARGS | KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_dsq_insert_vtime, KF_RCU) -BTF_KFUNCS_END(scx_kfunc_ids_enqueue_dispatch) - -static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { - .owner = THIS_MODULE, - .set = &scx_kfunc_ids_enqueue_dispatch, - .filter = scx_kfunc_context_filter, -}; - -static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, - struct task_struct *p, u64 dsq_id, u64 enq_flags) -{ - struct scx_dispatch_q *src_dsq = kit->dsq, *dst_dsq; - struct scx_sched *sch; - struct rq *this_rq, *src_rq, *locked_rq; - bool dispatched = false; - bool in_balance; - unsigned long flags; - - /* - * The verifier considers an iterator slot initialized on any - * KF_ITER_NEW return, so a BPF program may legally reach here after - * bpf_iter_scx_dsq_new() failed and left @kit->dsq NULL. - */ - if (unlikely(!src_dsq)) - return false; - - sch = src_dsq->sched; - - if (!scx_vet_enq_flags(sch, dsq_id, &enq_flags)) - return false; - - /* - * If the BPF scheduler keeps calling this function repeatedly, it can - * cause similar live-lock conditions as consume_dispatch_q(). - */ - if (unlikely(READ_ONCE(sch->aborting))) - return false; - - if (unlikely(!scx_task_on_sched(sch, p))) { - scx_error(sch, "scx_bpf_dsq_move[_vtime]() on %s[%d] but the task belongs to a different scheduler", - p->comm, p->pid); - return false; - } - - /* - * Can be called from either ops.dispatch() locking this_rq() or any - * context where no rq lock is held. If latter, lock @p's task_rq which - * we'll likely need anyway. - */ - src_rq = task_rq(p); - - local_irq_save(flags); - this_rq = this_rq(); - in_balance = this_rq->scx.flags & SCX_RQ_IN_BALANCE; - - if (in_balance) { - if (this_rq != src_rq) { - raw_spin_rq_unlock(this_rq); - raw_spin_rq_lock(src_rq); - } - } else { - raw_spin_rq_lock(src_rq); - } - - locked_rq = src_rq; - raw_spin_lock(&src_dsq->lock); - - /* did someone else get to it while we dropped the locks? */ - if (nldsq_cursor_lost_task(&kit->cursor, src_rq, src_dsq, p)) { - raw_spin_unlock(&src_dsq->lock); - goto out; - } - - /* @p is still on $src_dsq and stable, determine the destination */ - dst_dsq = find_dsq_for_dispatch(sch, this_rq, dsq_id, task_cpu(p)); - - /* - * Apply vtime and slice updates before moving so that the new time is - * visible before inserting into $dst_dsq. @p is still on $src_dsq but - * this is safe as we're locking it. - */ - if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_VTIME) - p->scx.dsq_vtime = kit->vtime; - if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_SLICE) - p->scx.slice = kit->slice; - - /* execute move */ - locked_rq = move_task_between_dsqs(sch, p, enq_flags, src_dsq, dst_dsq); - dispatched = true; -out: - if (in_balance) { - if (this_rq != locked_rq) { - raw_spin_rq_unlock(locked_rq); - raw_spin_rq_lock(this_rq); - } - } else { - raw_spin_rq_unlock_irqrestore(locked_rq, flags); - } - - kit->cursor.flags &= ~(__SCX_DSQ_ITER_HAS_SLICE | - __SCX_DSQ_ITER_HAS_VTIME); - return dispatched; -} - -__bpf_kfunc_start_defs(); - -/** - * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Can only be called from ops.dispatch(). - */ -__bpf_kfunc u32 scx_bpf_dispatch_nr_slots(const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return 0; - - return sch->dsp_max_batch - __this_cpu_read(sch->pcpu->dsp_ctx.cursor); -} - -/** - * scx_bpf_dispatch_cancel - Cancel the latest dispatch - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Cancel the latest dispatch. Can be called multiple times to cancel further - * dispatches. Can only be called from ops.dispatch(). - */ -__bpf_kfunc void scx_bpf_dispatch_cancel(const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - struct scx_dsp_ctx *dspc; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return; - - dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; - - if (dspc->cursor > 0) - dspc->cursor--; - else - scx_error(sch, "dispatch buffer underflow"); -} - -/** - * scx_bpf_dsq_move_to_local - move a task from a DSQ to the current CPU's local DSQ - * @dsq_id: DSQ to move task from. Must be a user-created DSQ - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * @enq_flags: %SCX_ENQ_* - * - * Move a task from the non-local DSQ identified by @dsq_id to the current CPU's - * local DSQ for execution with @enq_flags applied. Can only be called from - * ops.dispatch(). - * - * Built-in DSQs (%SCX_DSQ_GLOBAL and %SCX_DSQ_LOCAL*) are not supported as - * sources. Local DSQs support reenqueueing (a task can be picked up for - * execution, dequeued for property changes, or reenqueued), but the BPF - * scheduler cannot directly iterate or move tasks from them. %SCX_DSQ_GLOBAL - * is similar but also doesn't support reenqueueing, as it maps to multiple - * per-node DSQs making the scope difficult to define; this may change in the - * future. - * - * This function flushes the in-flight dispatches from scx_bpf_dsq_insert() - * before trying to move from the specified DSQ. It may also grab rq locks and - * thus can't be called under any BPF locks. - * - * Returns %true if a task has been moved, %false if there isn't any task to - * move. - */ -__bpf_kfunc bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags, - const struct bpf_prog_aux *aux) -{ - struct scx_dispatch_q *dsq; - struct scx_sched *sch; - struct scx_dsp_ctx *dspc; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return false; - - if (!scx_vet_enq_flags(sch, SCX_DSQ_LOCAL, &enq_flags)) - return false; - - dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; - - flush_dispatch_buf(sch, dspc->rq); - - dsq = find_user_dsq(sch, dsq_id); - if (unlikely(!dsq)) { - scx_error(sch, "invalid DSQ ID 0x%016llx", dsq_id); - return false; - } - - if (consume_dispatch_q(sch, dspc->rq, dsq, enq_flags)) { - /* - * A successfully consumed task can be dequeued before it starts - * running while the CPU is trying to migrate other dispatched - * tasks. Bump nr_tasks to tell balance_one() to retry on empty - * local DSQ. - */ - dspc->nr_tasks++; - return true; - } else { - return false; - } -} - -/* - * COMPAT: ___v2 was introduced in v7.1. Remove this and ___v2 tag in the future. - */ -__bpf_kfunc bool scx_bpf_dsq_move_to_local(u64 dsq_id, const struct bpf_prog_aux *aux) -{ - return scx_bpf_dsq_move_to_local___v2(dsq_id, 0, aux); -} - -/** - * scx_bpf_dsq_move_set_slice - Override slice when moving between DSQs - * @it__iter: DSQ iterator in progress - * @slice: duration the moved task can run for in nsecs - * - * Override the slice of the next task that will be moved from @it__iter using - * scx_bpf_dsq_move[_vtime](). If this function is not called, the previous - * slice duration is kept. - */ -__bpf_kfunc void scx_bpf_dsq_move_set_slice(struct bpf_iter_scx_dsq *it__iter, - u64 slice) -{ - struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter; - - kit->slice = slice; - kit->cursor.flags |= __SCX_DSQ_ITER_HAS_SLICE; -} - -/** - * scx_bpf_dsq_move_set_vtime - Override vtime when moving between DSQs - * @it__iter: DSQ iterator in progress - * @vtime: task's ordering inside the vtime-sorted queue of the target DSQ - * - * Override the vtime of the next task that will be moved from @it__iter using - * scx_bpf_dsq_move_vtime(). If this function is not called, the previous slice - * vtime is kept. If scx_bpf_dsq_move() is used to dispatch the next task, the - * override is ignored and cleared. - */ -__bpf_kfunc void scx_bpf_dsq_move_set_vtime(struct bpf_iter_scx_dsq *it__iter, - u64 vtime) -{ - struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter; - - kit->vtime = vtime; - kit->cursor.flags |= __SCX_DSQ_ITER_HAS_VTIME; -} - -/** - * scx_bpf_dsq_move - Move a task from DSQ iteration to a DSQ - * @it__iter: DSQ iterator in progress - * @p: task to transfer - * @dsq_id: DSQ to move @p to - * @enq_flags: SCX_ENQ_* - * - * Transfer @p which is on the DSQ currently iterated by @it__iter to the DSQ - * specified by @dsq_id. All DSQs - local DSQs, global DSQ and user DSQs - can - * be the destination. - * - * For the transfer to be successful, @p must still be on the DSQ and have been - * queued before the DSQ iteration started. This function doesn't care whether - * @p was obtained from the DSQ iteration. @p just has to be on the DSQ and have - * been queued before the iteration started. - * - * @p's slice is kept by default. Use scx_bpf_dsq_move_set_slice() to update. - * - * Can be called from ops.dispatch() or any BPF context which doesn't hold a rq - * lock (e.g. BPF timers or SYSCALL programs). - * - * Returns %true if @p has been consumed, %false if @p had already been - * consumed, dequeued, or, for sub-scheds, @dsq_id points to a disallowed local - * DSQ. - */ -__bpf_kfunc bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter, - struct task_struct *p, u64 dsq_id, - u64 enq_flags) -{ - return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter, - p, dsq_id, enq_flags); -} - -/** - * scx_bpf_dsq_move_vtime - Move a task from DSQ iteration to a PRIQ DSQ - * @it__iter: DSQ iterator in progress - * @p: task to transfer - * @dsq_id: DSQ to move @p to - * @enq_flags: SCX_ENQ_* - * - * Transfer @p which is on the DSQ currently iterated by @it__iter to the - * priority queue of the DSQ specified by @dsq_id. The destination must be a - * user DSQ as only user DSQs support priority queue. - * - * @p's slice and vtime are kept by default. Use scx_bpf_dsq_move_set_slice() - * and scx_bpf_dsq_move_set_vtime() to update. - * - * All other aspects are identical to scx_bpf_dsq_move(). See - * scx_bpf_dsq_insert_vtime() for more information on @vtime. - */ -__bpf_kfunc bool scx_bpf_dsq_move_vtime(struct bpf_iter_scx_dsq *it__iter, - struct task_struct *p, u64 dsq_id, - u64 enq_flags) -{ - return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter, - p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); -} - -#ifdef CONFIG_EXT_SUB_SCHED -/** - * scx_bpf_sub_dispatch - Trigger dispatching on a child scheduler - * @cgroup_id: cgroup ID of the child scheduler to dispatch - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Allows a parent scheduler to trigger dispatching on one of its direct - * child schedulers. The child scheduler runs its dispatch operation to - * move tasks from dispatch queues to the local runqueue. - * - * Returns: true on success, false if cgroup_id is invalid, not a direct - * child, or caller lacks dispatch permission. - */ -__bpf_kfunc bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux) -{ - struct rq *this_rq = this_rq(); - struct scx_sched *parent, *child; - - guard(rcu)(); - parent = scx_prog_sched(aux); - if (unlikely(!parent)) - return false; - - child = scx_find_sub_sched(cgroup_id); - - if (unlikely(!child)) - return false; - - if (unlikely(scx_parent(child) != parent)) { - scx_error(parent, "trying to dispatch a distant sub-sched on cgroup %llu", - cgroup_id); - return false; - } - - return scx_dispatch_sched(child, this_rq, this_rq->scx.sub_dispatch_prev, - true); -} -#endif /* CONFIG_EXT_SUB_SCHED */ - -__bpf_kfunc_end_defs(); - -BTF_KFUNCS_START(scx_kfunc_ids_dispatch) -BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_dispatch_cancel, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_dsq_move_to_local, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_dsq_move_to_local___v2, KF_IMPLICIT_ARGS) -/* scx_bpf_dsq_move*() also in scx_kfunc_ids_unlocked: callable from unlocked contexts */ -BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU) -#ifdef CONFIG_EXT_SUB_SCHED -BTF_ID_FLAGS(func, scx_bpf_sub_dispatch, KF_IMPLICIT_ARGS) -#endif -BTF_KFUNCS_END(scx_kfunc_ids_dispatch) - -static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = { - .owner = THIS_MODULE, - .set = &scx_kfunc_ids_dispatch, - .filter = scx_kfunc_context_filter, -}; - -__bpf_kfunc_start_defs(); - -/** - * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Iterate over all of the tasks currently enqueued on the local DSQ of the - * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of - * processed tasks. Can only be called from ops.cpu_release(). - */ -__bpf_kfunc u32 scx_bpf_reenqueue_local(const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - struct rq *rq; - - guard(rcu)(); - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return 0; - - rq = cpu_rq(smp_processor_id()); - lockdep_assert_rq_held(rq); - - return reenq_local(sch, rq, SCX_REENQ_ANY); -} - -__bpf_kfunc_end_defs(); - -BTF_KFUNCS_START(scx_kfunc_ids_cpu_release) -BTF_ID_FLAGS(func, scx_bpf_reenqueue_local, KF_IMPLICIT_ARGS) -BTF_KFUNCS_END(scx_kfunc_ids_cpu_release) - -static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = { - .owner = THIS_MODULE, - .set = &scx_kfunc_ids_cpu_release, - .filter = scx_kfunc_context_filter, -}; - -__bpf_kfunc_start_defs(); - -/** - * scx_bpf_create_dsq - Create a custom DSQ - * @dsq_id: DSQ to create - * @node: NUMA node to allocate from - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Create a custom DSQ identified by @dsq_id. Can be called from any sleepable - * scx callback, and any BPF_PROG_TYPE_SYSCALL prog. - */ -__bpf_kfunc s32 scx_bpf_create_dsq(u64 dsq_id, s32 node, const struct bpf_prog_aux *aux) -{ - struct scx_dispatch_q *dsq; - struct scx_sched *sch; - s32 ret; - - if (unlikely(node >= (int)nr_node_ids || - (node < 0 && node != NUMA_NO_NODE))) - return -EINVAL; - - if (unlikely(dsq_id & SCX_DSQ_FLAG_BUILTIN)) - return -EINVAL; - - dsq = kmalloc_node(sizeof(*dsq), GFP_KERNEL, node); - if (!dsq) - return -ENOMEM; - - /* - * init_dsq() must be called in GFP_KERNEL context. Init it with NULL - * @sch and update afterwards. - */ - ret = init_dsq(dsq, dsq_id, NULL); - if (ret) { - kfree(dsq); - return ret; - } - - rcu_read_lock(); - - sch = scx_prog_sched(aux); - if (sch) { - dsq->sched = sch; - ret = rhashtable_lookup_insert_fast(&sch->dsq_hash, &dsq->hash_node, - dsq_hash_params); - } else { - ret = -ENODEV; - } - - rcu_read_unlock(); - if (ret) { - exit_dsq(dsq); - kfree(dsq); - } - return ret; -} - -__bpf_kfunc_end_defs(); - -BTF_KFUNCS_START(scx_kfunc_ids_unlocked) -BTF_ID_FLAGS(func, scx_bpf_create_dsq, KF_IMPLICIT_ARGS | KF_SLEEPABLE) -/* also in scx_kfunc_ids_dispatch: also callable from ops.dispatch() */ -BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU) -/* also in scx_kfunc_ids_select_cpu: also callable from ops.select_cpu()/ops.enqueue() */ -BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS | KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_IMPLICIT_ARGS | KF_RCU) -BTF_KFUNCS_END(scx_kfunc_ids_unlocked) - -static const struct btf_kfunc_id_set scx_kfunc_set_unlocked = { - .owner = THIS_MODULE, - .set = &scx_kfunc_ids_unlocked, - .filter = scx_kfunc_context_filter, -}; - -__bpf_kfunc_start_defs(); - -/** - * scx_bpf_task_set_slice - Set task's time slice - * @p: task of interest - * @slice: time slice to set in nsecs - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Set @p's time slice to @slice. Returns %true on success, %false if the - * calling scheduler doesn't have authority over @p. - */ -__bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, - const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - sch = scx_prog_sched(aux); - if (unlikely(!sch || !scx_task_on_sched(sch, p))) - return false; - - p->scx.slice = slice; - return true; -} - -/** - * scx_bpf_task_set_dsq_vtime - Set task's virtual time for DSQ ordering - * @p: task of interest - * @vtime: virtual time to set - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Set @p's virtual time to @vtime. Returns %true on success, %false if the - * calling scheduler doesn't have authority over @p. - */ -__bpf_kfunc bool scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime, - const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - sch = scx_prog_sched(aux); - if (unlikely(!sch || !scx_task_on_sched(sch, p))) - return false; - - p->scx.dsq_vtime = vtime; - return true; -} - -static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) -{ - struct rq *this_rq; - unsigned long irq_flags; - - local_irq_save(irq_flags); - - this_rq = this_rq(); - - /* - * While bypassing for PM ops, IRQ handling may not be online which can - * lead to irq_work_queue() malfunction such as infinite busy wait for - * IRQ status update. Suppress kicking. - */ - if (scx_bypassing(sch, cpu_of(this_rq))) - goto out; - - /* - * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting - * rq locks. We can probably be smarter and avoid bouncing if called - * from ops which don't hold a rq lock. - */ - if (flags & SCX_KICK_IDLE) { - struct rq *target_rq = cpu_rq(cpu); - - if (unlikely(flags & (SCX_KICK_PREEMPT | SCX_KICK_WAIT))) - scx_error(sch, "PREEMPT/WAIT cannot be used with SCX_KICK_IDLE"); - - if (raw_spin_rq_trylock(target_rq)) { - if (can_skip_idle_kick(target_rq)) { - raw_spin_rq_unlock(target_rq); - goto out; - } - raw_spin_rq_unlock(target_rq); - } - cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick_if_idle); - } else { - cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick); - - if (flags & SCX_KICK_PREEMPT) - cpumask_set_cpu(cpu, this_rq->scx.cpus_to_preempt); - if (flags & SCX_KICK_WAIT) - cpumask_set_cpu(cpu, this_rq->scx.cpus_to_wait); - } - - irq_work_queue(&this_rq->scx.kick_cpus_irq_work); -out: - local_irq_restore(irq_flags); -} - -/** - * scx_bpf_kick_cpu - Trigger reschedule on a CPU - * @cpu: cpu to kick - * @flags: %SCX_KICK_* flags - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or - * trigger rescheduling on a busy CPU. This can be called from any online - * scx_ops operation and the actual kicking is performed asynchronously through - * an irq work. - */ -__bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - sch = scx_prog_sched(aux); - if (likely(sch) && scx_cpu_valid(sch, cpu, NULL)) - scx_kick_cpu(sch, cpu, flags); -} - -/** - * scx_bpf_kick_cid - Trigger reschedule on the CPU mapped to @cid - * @cid: cid to kick - * @flags: %SCX_KICK_* flags - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * cid-addressed equivalent of scx_bpf_kick_cpu(). Return 0 on success, - * -errno otherwise. - */ -__bpf_kfunc s32 scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - s32 cpu; - - guard(rcu)(); - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return -ENODEV; - cpu = scx_cid_to_cpu(sch, cid); - if (cpu < 0) - return cpu; - scx_kick_cpu(sch, cpu, flags); - return 0; -} - -/** - * scx_bpf_dsq_nr_queued - Return the number of queued tasks - * @dsq_id: id of the DSQ - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Return the number of tasks in the DSQ matching @dsq_id. If not found, - * -%ENOENT is returned. - */ -__bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - struct scx_dispatch_q *dsq; - s32 ret; - - preempt_disable(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) { - ret = -ENODEV; - goto out; - } - - if (dsq_id == SCX_DSQ_LOCAL) { - ret = READ_ONCE(this_rq()->scx.local_dsq.nr); - goto out; - } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { - s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK); - - if (scx_cpu_valid(sch, cpu, NULL)) { - ret = READ_ONCE(cpu_rq(cpu)->scx.local_dsq.nr); - goto out; - } - } else { - dsq = find_user_dsq(sch, dsq_id); - if (dsq) { - ret = READ_ONCE(dsq->nr); - goto out; - } - } - ret = -ENOENT; -out: - preempt_enable(); - return ret; -} - -/** - * scx_bpf_destroy_dsq - Destroy a custom DSQ - * @dsq_id: DSQ to destroy - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with - * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is - * empty and no further tasks are dispatched to it. Ignored if called on a DSQ - * which doesn't exist. Can be called from any online scx_ops operations. - */ -__bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - sch = scx_prog_sched(aux); - if (sch) - destroy_dsq(sch, dsq_id); -} - -/** - * bpf_iter_scx_dsq_new - Create a DSQ iterator - * @it: iterator to initialize - * @dsq_id: DSQ to iterate - * @flags: %SCX_DSQ_ITER_* - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Initialize BPF iterator @it which can be used with bpf_for_each() to walk - * tasks in the DSQ specified by @dsq_id. Iteration using @it only includes - * tasks which are already queued when this function is invoked. - */ -__bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, - u64 flags, const struct bpf_prog_aux *aux) -{ - struct bpf_iter_scx_dsq_kern *kit = (void *)it; - struct scx_sched *sch; - - BUILD_BUG_ON(sizeof(struct bpf_iter_scx_dsq_kern) > - sizeof(struct bpf_iter_scx_dsq)); - BUILD_BUG_ON(__alignof__(struct bpf_iter_scx_dsq_kern) != - __alignof__(struct bpf_iter_scx_dsq)); - BUILD_BUG_ON(__SCX_DSQ_ITER_ALL_FLAGS & - ((1U << __SCX_DSQ_LNODE_PRIV_SHIFT) - 1)); - - /* - * next() and destroy() will be called regardless of the return value. - * Always clear $kit->dsq. - */ - kit->dsq = NULL; - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return -ENODEV; - - if (flags & ~__SCX_DSQ_ITER_USER_FLAGS) - return -EINVAL; - - kit->dsq = find_user_dsq(sch, dsq_id); - if (!kit->dsq) - return -ENOENT; - - kit->cursor = INIT_DSQ_LIST_CURSOR(kit->cursor, kit->dsq, flags); - - return 0; -} - -/** - * bpf_iter_scx_dsq_next - Progress a DSQ iterator - * @it: iterator to progress - * - * Return the next task. See bpf_iter_scx_dsq_new(). - */ -__bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it) -{ - struct bpf_iter_scx_dsq_kern *kit = (void *)it; - - if (!kit->dsq) - return NULL; - - guard(raw_spinlock_irqsave)(&kit->dsq->lock); - - return nldsq_cursor_next_task(&kit->cursor, kit->dsq); -} - -/** - * bpf_iter_scx_dsq_destroy - Destroy a DSQ iterator - * @it: iterator to destroy - * - * Undo scx_iter_scx_dsq_new(). - */ -__bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) -{ - struct bpf_iter_scx_dsq_kern *kit = (void *)it; - - if (!kit->dsq) - return; - - if (!list_empty(&kit->cursor.node)) { - unsigned long flags; - - raw_spin_lock_irqsave(&kit->dsq->lock, flags); - list_del_init(&kit->cursor.node); - raw_spin_unlock_irqrestore(&kit->dsq->lock, flags); - } - kit->dsq = NULL; -} - -/** - * scx_bpf_dsq_peek - Lockless peek at the first element. - * @dsq_id: DSQ to examine. - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Read the first element in the DSQ. This is semantically equivalent to using - * the DSQ iterator, but is lockfree. Of course, like any lockless operation, - * this provides only a point-in-time snapshot, and the contents may change - * by the time any subsequent locking operation reads the queue. - * - * Returns the pointer, or NULL indicates an empty queue OR internal error. - */ -__bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id, - const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - struct scx_dispatch_q *dsq; - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return NULL; - - if (unlikely(dsq_id & SCX_DSQ_FLAG_BUILTIN)) { - scx_error(sch, "peek disallowed on builtin DSQ 0x%llx", dsq_id); - return NULL; - } - - dsq = find_user_dsq(sch, dsq_id); - if (unlikely(!dsq)) { - scx_error(sch, "peek on non-existent DSQ 0x%llx", dsq_id); - return NULL; - } - - return rcu_dereference(dsq->first_task); -} - -/** - * scx_bpf_dsq_reenq - Re-enqueue tasks on a DSQ - * @dsq_id: DSQ to re-enqueue - * @reenq_flags: %SCX_RENQ_* - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Iterate over all of the tasks currently enqueued on the DSQ identified by - * @dsq_id, and re-enqueue them in the BPF scheduler. The following DSQs are - * supported: - * - * - Local DSQs (%SCX_DSQ_LOCAL or %SCX_DSQ_LOCAL_ON | $cpu) - * - User DSQs - * - * Re-enqueues are performed asynchronously. Can be called from anywhere. - */ -__bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags, - const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - struct scx_dispatch_q *dsq; - - guard(preempt)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return; - - if (unlikely(reenq_flags & ~__SCX_REENQ_USER_MASK)) { - scx_error(sch, "invalid SCX_REENQ flags 0x%llx", reenq_flags); - return; - } - - /* not specifying any filter bits is the same as %SCX_REENQ_ANY */ - if (!(reenq_flags & __SCX_REENQ_FILTER_MASK)) - reenq_flags |= SCX_REENQ_ANY; - - dsq = find_dsq_for_dispatch(sch, this_rq(), dsq_id, smp_processor_id()); - schedule_dsq_reenq(sch, dsq, reenq_flags, scx_locked_rq()); -} - -/** - * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Iterate over all of the tasks currently enqueued on the local DSQ of the - * caller's CPU, and re-enqueue them in the BPF scheduler. Can be called from - * anywhere. - * - * This is now a special case of scx_bpf_dsq_reenq() and may be removed in the - * future. - */ -__bpf_kfunc void scx_bpf_reenqueue_local___v2(const struct bpf_prog_aux *aux) -{ - scx_bpf_dsq_reenq(SCX_DSQ_LOCAL, 0, aux); -} - -__bpf_kfunc_end_defs(); - -__printf(5, 0) -static s32 __bstr_format(struct scx_sched *sch, u64 *data_buf, char *line_buf, - size_t line_size, char *fmt, unsigned long long *data, - u32 data__sz) -{ - struct bpf_bprintf_data bprintf_data = { .get_bin_args = true }; - s32 ret; - - if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 || - (data__sz && !data)) { - scx_error(sch, "invalid data=%p and data__sz=%u", (void *)data, data__sz); - return -EINVAL; - } - - ret = copy_from_kernel_nofault(data_buf, data, data__sz); - if (ret < 0) { - scx_error(sch, "failed to read data fields (%d)", ret); - return ret; - } - - ret = bpf_bprintf_prepare(fmt, UINT_MAX, data_buf, data__sz / 8, - &bprintf_data); - if (ret < 0) { - scx_error(sch, "format preparation failed (%d)", ret); - return ret; - } - - ret = bstr_printf(line_buf, line_size, fmt, - bprintf_data.bin_args); - bpf_bprintf_cleanup(&bprintf_data); - if (ret < 0) { - scx_error(sch, "(\"%s\", %p, %u) failed to format", fmt, data, data__sz); - return ret; - } - - return ret; -} - -__printf(3, 0) -static s32 bstr_format(struct scx_sched *sch, struct scx_bstr_buf *buf, - char *fmt, unsigned long long *data, u32 data__sz) -{ - return __bstr_format(sch, buf->data, buf->line, sizeof(buf->line), - fmt, data, data__sz); -} - -__bpf_kfunc_start_defs(); - -/** - * scx_bpf_exit_bstr - Gracefully exit the BPF scheduler. - * @exit_code: Exit value to pass to user space via struct scx_exit_info. - * @fmt: error message format string - * @data: format string parameters packaged using ___bpf_fill() macro - * @data__sz: @data len, must end in '__sz' for the verifier - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Indicate that the BPF scheduler wants to exit gracefully, and initiate ops - * disabling. - */ -__printf(2, 0) -__bpf_kfunc void scx_bpf_exit_bstr(s64 exit_code, char *fmt, - unsigned long long *data, u32 data__sz, - const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - unsigned long flags; - - raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags); - sch = scx_prog_sched(aux); - if (likely(sch) && - bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0) - scx_exit(sch, SCX_EXIT_UNREG_BPF, exit_code, "%s", scx_exit_bstr_buf.line); - raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags); -} - -/** - * scx_bpf_error_bstr - Indicate fatal error - * @fmt: error message format string - * @data: format string parameters packaged using ___bpf_fill() macro - * @data__sz: @data len, must end in '__sz' for the verifier - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Indicate that the BPF scheduler encountered a fatal error and initiate ops - * disabling. - */ -__printf(1, 0) -__bpf_kfunc void scx_bpf_error_bstr(char *fmt, unsigned long long *data, - u32 data__sz, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - unsigned long flags; - - raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags); - sch = scx_prog_sched(aux); - if (likely(sch) && - bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0) - scx_exit(sch, SCX_EXIT_ERROR_BPF, 0, "%s", scx_exit_bstr_buf.line); - raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags); -} - -/** - * scx_bpf_dump_bstr - Generate extra debug dump specific to the BPF scheduler - * @fmt: format string - * @data: format string parameters packaged using ___bpf_fill() macro - * @data__sz: @data len, must end in '__sz' for the verifier - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * To be called through scx_bpf_dump() helper from ops.dump(), dump_cpu() and - * dump_task() to generate extra debug dump specific to the BPF scheduler. - * - * The extra dump may be multiple lines. A single line may be split over - * multiple calls. The last line is automatically terminated. - */ -__printf(1, 0) -__bpf_kfunc void scx_bpf_dump_bstr(char *fmt, unsigned long long *data, - u32 data__sz, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - struct scx_dump_data *dd = &scx_dump_data; - struct scx_bstr_buf *buf = &dd->buf; - s32 ret; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return; - - if (raw_smp_processor_id() != dd->cpu) { - scx_error(sch, "scx_bpf_dump() must only be called from ops.dump() and friends"); - return; - } - - /* append the formatted string to the line buf */ - ret = __bstr_format(sch, buf->data, buf->line + dd->cursor, - sizeof(buf->line) - dd->cursor, fmt, data, data__sz); - if (ret < 0) { - dump_line(dd->s, "%s[!] (\"%s\", %p, %u) failed to format (%d)", - dd->prefix, fmt, data, data__sz, ret); - return; - } - - dd->cursor += ret; - dd->cursor = min_t(s32, dd->cursor, sizeof(buf->line)); - - if (!dd->cursor) - return; - - /* - * If the line buf overflowed or ends in a newline, flush it into the - * dump. This is to allow the caller to generate a single line over - * multiple calls. As ops_dump_flush() can also handle multiple lines in - * the line buf, the only case which can lead to an unexpected - * truncation is when the caller keeps generating newlines in the middle - * instead of the end consecutively. Don't do that. - */ - if (dd->cursor >= sizeof(buf->line) || buf->line[dd->cursor - 1] == '\n') - ops_dump_flush(); -} - -/** - * scx_bpf_cpuperf_cap - Query the maximum relative capacity of a CPU - * @cpu: CPU of interest - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Return the maximum relative capacity of @cpu in relation to the most - * performant CPU in the system. The return value is in the range [1, - * %SCX_CPUPERF_ONE]. See scx_bpf_cpuperf_cur(). - */ -__bpf_kfunc u32 scx_bpf_cpuperf_cap(s32 cpu, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (likely(sch) && scx_cpu_valid(sch, cpu, NULL)) - return arch_scale_cpu_capacity(cpu); - else - return SCX_CPUPERF_ONE; -} - -/** - * scx_bpf_cidperf_cap - Query the maximum relative capacity of the CPU at @cid - * @cid: cid of the CPU to query - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * cid-addressed equivalent of scx_bpf_cpuperf_cap(). - */ -__bpf_kfunc u32 scx_bpf_cidperf_cap(s32 cid, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - s32 cpu; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return SCX_CPUPERF_ONE; - cpu = scx_cid_to_cpu(sch, cid); - if (cpu < 0) - return SCX_CPUPERF_ONE; - return arch_scale_cpu_capacity(cpu); -} - -/** - * scx_bpf_cpuperf_cur - Query the current relative performance of a CPU - * @cpu: CPU of interest - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Return the current relative performance of @cpu in relation to its maximum. - * The return value is in the range [1, %SCX_CPUPERF_ONE]. - * - * The current performance level of a CPU in relation to the maximum performance - * available in the system can be calculated as follows: - * - * scx_bpf_cpuperf_cap() * scx_bpf_cpuperf_cur() / %SCX_CPUPERF_ONE - * - * The result is in the range [1, %SCX_CPUPERF_ONE]. - */ -__bpf_kfunc u32 scx_bpf_cpuperf_cur(s32 cpu, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (likely(sch) && scx_cpu_valid(sch, cpu, NULL)) - return arch_scale_freq_capacity(cpu); - else - return SCX_CPUPERF_ONE; -} - -/** - * scx_bpf_cidperf_cur - Query the current performance of the CPU at @cid - * @cid: cid of the CPU to query - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * cid-addressed equivalent of scx_bpf_cpuperf_cur(). - */ -__bpf_kfunc u32 scx_bpf_cidperf_cur(s32 cid, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - s32 cpu; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return SCX_CPUPERF_ONE; - cpu = scx_cid_to_cpu(sch, cid); - if (cpu < 0) - return SCX_CPUPERF_ONE; - return arch_scale_freq_capacity(cpu); -} - -/** - * scx_bpf_cpuperf_set - Set the relative performance target of a CPU - * @cpu: CPU of interest - * @perf: target performance level [0, %SCX_CPUPERF_ONE] - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Set the target performance level of @cpu to @perf. @perf is in linear - * relative scale between 0 and %SCX_CPUPERF_ONE. This determines how the - * schedutil cpufreq governor chooses the target frequency. - * - * The actual performance level chosen, CPU grouping, and the overhead and - * latency of the operations are dependent on the hardware and cpufreq driver in - * use. Consult hardware and cpufreq documentation for more information. The - * current performance level can be monitored using scx_bpf_cpuperf_cur(). - */ -__bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return; - - if (unlikely(perf > SCX_CPUPERF_ONE)) { - scx_error(sch, "Invalid cpuperf target %u for CPU %d", perf, cpu); - return; - } - - if (scx_cpu_valid(sch, cpu, NULL)) { - struct rq *rq = cpu_rq(cpu), *locked_rq = scx_locked_rq(); - struct rq_flags rf; - - /* - * When called with an rq lock held, restrict the operation - * to the corresponding CPU to prevent ABBA deadlocks. - */ - if (locked_rq && rq != locked_rq) { - scx_error(sch, "Invalid target CPU %d", cpu); - return; - } - - /* - * If no rq lock is held, allow to operate on any CPU by - * acquiring the corresponding rq lock. - */ - if (!locked_rq) { - rq_lock_irqsave(rq, &rf); - update_rq_clock(rq); - } - - rq->scx.cpuperf_target = perf; - cpufreq_update_util(rq, 0); - - if (!locked_rq) - rq_unlock_irqrestore(rq, &rf); - } -} - -/** - * scx_bpf_cidperf_set - Set the performance target of the CPU at @cid - * @cid: cid of the CPU to target - * @perf: target performance level [0, %SCX_CPUPERF_ONE] - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * cid-addressed equivalent of scx_bpf_cpuperf_set(). - */ -__bpf_kfunc void scx_bpf_cidperf_set(s32 cid, u32 perf, - const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - s32 cpu; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return; - cpu = scx_cid_to_cpu(sch, cid); - if (cpu < 0) - return; - scx_bpf_cpuperf_set(cpu, perf, aux); -} - -/** - * scx_bpf_nr_node_ids - Return the number of possible node IDs - * - * All valid node IDs in the system are smaller than the returned value. - */ -__bpf_kfunc u32 scx_bpf_nr_node_ids(void) -{ - return nr_node_ids; -} - -/** - * scx_bpf_nr_cpu_ids - Return the number of possible CPU IDs - * - * All valid CPU IDs in the system are smaller than the returned value. - */ -__bpf_kfunc u32 scx_bpf_nr_cpu_ids(void) -{ - return nr_cpu_ids; -} - -/** - * scx_bpf_nr_cids - Return the size of the cid space - * - * Equals num_possible_cpus(). All valid cids are in [0, return value). - */ -__bpf_kfunc u32 scx_bpf_nr_cids(void) -{ - return num_possible_cpus(); -} - -/** - * scx_bpf_nr_online_cids - Return current count of online CPUs in cid space - * - * Return num_online_cpus(). The standard model restarts the scheduler on - * hotplug, which lets schedulers treat [0, nr_online_cids) as the online - * range. Schedulers that prefer to handle hotplug without a restart should - * install a custom mapping via scx_bpf_cid_override() and track onlining - * through the ops.cid_online / ops.cid_offline callbacks. - */ -__bpf_kfunc u32 scx_bpf_nr_online_cids(void) -{ - return num_online_cpus(); -} - -/** - * scx_bpf_this_cid - Return the cid of the CPU this program is running on - * - * cid-addressed equivalent of bpf_get_smp_processor_id() for scx programs. - * The current cpu is trivially valid, so this is just a table lookup. Return - * -EINVAL if called from a non-SCX program before any scheduler has ever - * been enabled (the cid table is still unallocated at that point). - */ -__bpf_kfunc s32 scx_bpf_this_cid(void) -{ - s16 *tbl = READ_ONCE(scx_cpu_to_cid_tbl); - - if (!tbl) - return -EINVAL; - return tbl[raw_smp_processor_id()]; -} - -/** - * scx_bpf_get_possible_cpumask - Get a referenced kptr to cpu_possible_mask - */ -__bpf_kfunc const struct cpumask *scx_bpf_get_possible_cpumask(void) -{ - return cpu_possible_mask; -} - -/** - * scx_bpf_get_online_cpumask - Get a referenced kptr to cpu_online_mask - */ -__bpf_kfunc const struct cpumask *scx_bpf_get_online_cpumask(void) -{ - return cpu_online_mask; -} - -/** - * scx_bpf_put_cpumask - Release a possible/online cpumask - * @cpumask: cpumask to release - */ -__bpf_kfunc void scx_bpf_put_cpumask(const struct cpumask *cpumask) -{ - /* - * Empty function body because we aren't actually acquiring or releasing - * a reference to a global cpumask, which is read-only in the caller and - * is never released. The acquire / release semantics here are just used - * to make the cpumask is a trusted pointer in the caller. - */ -} - -/** - * scx_bpf_task_running - Is task currently running? - * @p: task of interest - */ -__bpf_kfunc bool scx_bpf_task_running(const struct task_struct *p) -{ - return task_rq(p)->curr == p; -} - -/** - * scx_bpf_task_cpu - CPU a task is currently associated with - * @p: task of interest - */ -__bpf_kfunc s32 scx_bpf_task_cpu(const struct task_struct *p) -{ - return task_cpu(p); -} - -/** - * scx_bpf_task_cid - cid a task is currently associated with - * @p: task of interest - * - * cid-addressed equivalent of scx_bpf_task_cpu(). task_cpu(p) is always a - * valid cpu, so this is just a table lookup. Return -EINVAL if called from - * a non-SCX program before any scheduler has ever been enabled. - */ -__bpf_kfunc s32 scx_bpf_task_cid(const struct task_struct *p) -{ - s16 *tbl = READ_ONCE(scx_cpu_to_cid_tbl); - - if (!tbl) - return -EINVAL; - return tbl[task_cpu(p)]; -} - -/** - * scx_bpf_cpu_rq - Fetch the rq of a CPU - * @cpu: CPU of the rq - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - */ -__bpf_kfunc struct rq *scx_bpf_cpu_rq(s32 cpu, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return NULL; - - if (!scx_cpu_valid(sch, cpu, NULL)) - return NULL; - - if (!sch->warned_deprecated_rq) { - printk_deferred(KERN_WARNING "sched_ext: %s() is deprecated; " - "use scx_bpf_locked_rq() when holding rq lock " - "or scx_bpf_cpu_curr() to read remote curr safely.\n", __func__); - sch->warned_deprecated_rq = true; - } - - return cpu_rq(cpu); -} - -/** - * scx_bpf_locked_rq - Return the rq currently locked by SCX - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Returns the rq if a rq lock is currently held by SCX. - * Otherwise emits an error and returns NULL. - */ -__bpf_kfunc struct rq *scx_bpf_locked_rq(const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - struct rq *rq; - - guard(preempt)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return NULL; - - rq = scx_locked_rq(); - if (!rq) { - scx_error(sch, "accessing rq without holding rq lock"); - return NULL; - } - - return rq; -} - -/** - * scx_bpf_cpu_curr - Return remote CPU's curr task - * @cpu: CPU of interest - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Callers must hold RCU read lock (KF_RCU). - */ -__bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return NULL; - - if (!scx_cpu_valid(sch, cpu, NULL)) - return NULL; - - return rcu_dereference(cpu_rq(cpu)->curr); -} - -/** - * scx_bpf_cid_curr - Return the curr task on the CPU at @cid - * @cid: cid of interest - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * cid-addressed equivalent of scx_bpf_cpu_curr(). Callers must hold RCU - * read lock (KF_RCU). - */ -__bpf_kfunc struct task_struct *scx_bpf_cid_curr(s32 cid, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - s32 cpu; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return NULL; - cpu = scx_cid_to_cpu(sch, cid); - if (cpu < 0) - return NULL; - return rcu_dereference(cpu_rq(cpu)->curr); -} - -/** - * scx_bpf_tid_to_task - Look up a task by its scx tid - * @tid: task ID previously read from p->scx.tid - * - * Returns the task with the given tid, or NULL if no such task exists. The - * returned pointer is valid until the end of the current RCU read section - * (KF_RCU_PROTECTED). Requires SCX_OPS_TID_TO_TASK to be set on the root - * scheduler; otherwise an error is raised and NULL returned. - */ -__bpf_kfunc struct task_struct *scx_bpf_tid_to_task(u64 tid) -{ - struct sched_ext_entity *scx; - - if (!scx_tid_to_task_enabled()) { - struct scx_sched *sch = rcu_dereference(scx_root); - - if (sch) - scx_error(sch, "scx_bpf_tid_to_task() called without SCX_OPS_TID_TO_TASK"); - return NULL; - } - - scx = rhashtable_lookup(&scx_tid_hash, &tid, scx_tid_hash_params); - if (!scx) - return NULL; - - return container_of(scx, struct task_struct, scx); -} - -/** - * scx_bpf_now - Returns a high-performance monotonically non-decreasing - * clock for the current CPU. The clock returned is in nanoseconds. - * - * It provides the following properties: - * - * 1) High performance: Many BPF schedulers call bpf_ktime_get_ns() frequently - * to account for execution time and track tasks' runtime properties. - * Unfortunately, in some hardware platforms, bpf_ktime_get_ns() -- which - * eventually reads a hardware timestamp counter -- is neither performant nor - * scalable. scx_bpf_now() aims to provide a high-performance clock by - * using the rq clock in the scheduler core whenever possible. - * - * 2) High enough resolution for the BPF scheduler use cases: In most BPF - * scheduler use cases, the required clock resolution is lower than the most - * accurate hardware clock (e.g., rdtsc in x86). scx_bpf_now() basically - * uses the rq clock in the scheduler core whenever it is valid. It considers - * that the rq clock is valid from the time the rq clock is updated - * (update_rq_clock) until the rq is unlocked (rq_unpin_lock). - * - * 3) Monotonically non-decreasing clock for the same CPU: scx_bpf_now() - * guarantees the clock never goes backward when comparing them in the same - * CPU. On the other hand, when comparing clocks in different CPUs, there - * is no such guarantee -- the clock can go backward. It provides a - * monotonically *non-decreasing* clock so that it would provide the same - * clock values in two different scx_bpf_now() calls in the same CPU - * during the same period of when the rq clock is valid. - */ -__bpf_kfunc u64 scx_bpf_now(void) -{ - struct rq *rq; - u64 clock; - - preempt_disable(); - - rq = this_rq(); - if (smp_load_acquire(&rq->scx.flags) & SCX_RQ_CLK_VALID) { - /* - * If the rq clock is valid, use the cached rq clock. - * - * Note that scx_bpf_now() is re-entrant between a process - * context and an interrupt context (e.g., timer interrupt). - * However, we don't need to consider the race between them - * because such race is not observable from a caller. - */ - clock = READ_ONCE(rq->scx.clock); - } else { - /* - * Otherwise, return a fresh rq clock. - * - * The rq clock is updated outside of the rq lock. - * In this case, keep the updated rq clock invalid so the next - * kfunc call outside the rq lock gets a fresh rq clock. - */ - clock = sched_clock_cpu(cpu_of(rq)); - } - - preempt_enable(); - - return clock; -} - -static void scx_read_events(struct scx_sched *sch, struct scx_event_stats *events) -{ - struct scx_event_stats *e_cpu; - int cpu; - - /* Aggregate per-CPU event counters into @events. */ - memset(events, 0, sizeof(*events)); - for_each_possible_cpu(cpu) { - e_cpu = &per_cpu_ptr(sch->pcpu, cpu)->event_stats; - scx_agg_event(events, e_cpu, SCX_EV_SELECT_CPU_FALLBACK); - scx_agg_event(events, e_cpu, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); - scx_agg_event(events, e_cpu, SCX_EV_DISPATCH_KEEP_LAST); - scx_agg_event(events, e_cpu, SCX_EV_ENQ_SKIP_EXITING); - scx_agg_event(events, e_cpu, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); - scx_agg_event(events, e_cpu, SCX_EV_REENQ_IMMED); - scx_agg_event(events, e_cpu, SCX_EV_REENQ_LOCAL_REPEAT); - scx_agg_event(events, e_cpu, SCX_EV_REFILL_SLICE_DFL); - scx_agg_event(events, e_cpu, SCX_EV_BYPASS_DURATION); - scx_agg_event(events, e_cpu, SCX_EV_BYPASS_DISPATCH); - scx_agg_event(events, e_cpu, SCX_EV_BYPASS_ACTIVATE); - scx_agg_event(events, e_cpu, SCX_EV_INSERT_NOT_OWNED); - scx_agg_event(events, e_cpu, SCX_EV_SUB_BYPASS_DISPATCH); - } -} - -/* - * scx_bpf_events - Get a system-wide event counter to - * @events: output buffer from a BPF program - * @events__sz: @events len, must end in '__sz'' for the verifier - */ -__bpf_kfunc void scx_bpf_events(struct scx_event_stats *events, - size_t events__sz) -{ - struct scx_sched *sch; - struct scx_event_stats e_sys; - - rcu_read_lock(); - sch = rcu_dereference(scx_root); - if (sch) - scx_read_events(sch, &e_sys); - else - memset(&e_sys, 0, sizeof(e_sys)); - rcu_read_unlock(); - - /* - * We cannot entirely trust a BPF-provided size since a BPF program - * might be compiled against a different vmlinux.h, of which - * scx_event_stats would be larger (a newer vmlinux.h) or smaller - * (an older vmlinux.h). Hence, we use the smaller size to avoid - * memory corruption. - */ - events__sz = min(events__sz, sizeof(*events)); - memcpy(events, &e_sys, events__sz); -} - -#ifdef CONFIG_CGROUP_SCHED -/** - * scx_bpf_task_cgroup - Return the sched cgroup of a task - * @p: task of interest - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with - * from the scheduler's POV. SCX operations should use this function to - * determine @p's current cgroup as, unlike following @p->cgroups, - * @p->sched_task_group is stable for the duration of the SCX op. See - * SCX_CALL_OP_TASK() for details. - */ -__bpf_kfunc struct cgroup *scx_bpf_task_cgroup(struct task_struct *p, - const struct bpf_prog_aux *aux) -{ - struct task_group *tg = p->sched_task_group; - struct cgroup *cgrp = &cgrp_dfl_root.cgrp; - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - goto out; - - if (!scx_kf_arg_task_ok(sch, p)) - goto out; - - cgrp = tg_cgrp(tg); - -out: - cgroup_get(cgrp); - return cgrp; -} -#endif /* CONFIG_CGROUP_SCHED */ - -__bpf_kfunc_end_defs(); - -BTF_KFUNCS_START(scx_kfunc_ids_any) -BTF_ID_FLAGS(func, scx_bpf_task_set_slice, KF_IMPLICIT_ARGS | KF_RCU); -BTF_ID_FLAGS(func, scx_bpf_task_set_dsq_vtime, KF_IMPLICIT_ARGS | KF_RCU); -BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_kick_cid, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_destroy_dsq, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_dsq_peek, KF_IMPLICIT_ARGS | KF_RCU_PROTECTED | KF_RET_NULL) -BTF_ID_FLAGS(func, scx_bpf_dsq_reenq, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_reenqueue_local___v2, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, bpf_iter_scx_dsq_new, KF_IMPLICIT_ARGS | KF_ITER_NEW | KF_RCU_PROTECTED) -BTF_ID_FLAGS(func, bpf_iter_scx_dsq_next, KF_ITER_NEXT | KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_iter_scx_dsq_destroy, KF_ITER_DESTROY) -BTF_ID_FLAGS(func, scx_bpf_exit_bstr, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_dump_bstr, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_cidperf_cap, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_cidperf_cur, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_cidperf_set, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_nr_node_ids) -BTF_ID_FLAGS(func, scx_bpf_nr_cpu_ids) -BTF_ID_FLAGS(func, scx_bpf_nr_cids) -BTF_ID_FLAGS(func, scx_bpf_nr_online_cids) -BTF_ID_FLAGS(func, scx_bpf_this_cid) -BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE) -BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE) -BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE) -BTF_ID_FLAGS(func, scx_bpf_task_running, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_task_cid, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_cpu_rq, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_locked_rq, KF_IMPLICIT_ARGS | KF_RET_NULL) -BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) -BTF_ID_FLAGS(func, scx_bpf_cid_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) -BTF_ID_FLAGS(func, scx_bpf_tid_to_task, KF_RET_NULL | KF_RCU_PROTECTED) -BTF_ID_FLAGS(func, scx_bpf_now) -BTF_ID_FLAGS(func, scx_bpf_events) -#ifdef CONFIG_CGROUP_SCHED -BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_IMPLICIT_ARGS | KF_RCU | KF_ACQUIRE) -#endif -BTF_KFUNCS_END(scx_kfunc_ids_any) - -static const struct btf_kfunc_id_set scx_kfunc_set_any = { - .owner = THIS_MODULE, - .set = &scx_kfunc_ids_any, - .filter = scx_kfunc_context_filter, -}; - -/* - * cpu-form kfuncs that are forbidden from cid-form schedulers - * (bpf_sched_ext_ops_cid). Programs targeting the cid struct_ops type must - * use the cid-form alternative (cid/cmask kfuncs). - * - * Membership overlaps with scx_kfunc_ids_{any,idle,select_cpu}; the filter - * tests this set independently and rejects matches before the per-op - * allow-list check runs. - * - * pahole/resolve_btfids scans every BTF_ID_FLAGS() at build time and - * intersects flags across duplicate entries, so each entry must carry the - * same flags as the kfunc's primary declaration; otherwise the flags get - * dropped globally. - */ -BTF_KFUNCS_START(scx_kfunc_ids_cpu_only) -BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_cpu_rq, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) -BTF_ID_FLAGS(func, scx_bpf_cpu_node, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE) -BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE) -BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE) -BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_IMPLICIT_ARGS | KF_RCU) -BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS | KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_IMPLICIT_ARGS | KF_ACQUIRE) -BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE) -BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_IMPLICIT_ARGS | KF_ACQUIRE) -BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE) -BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) -BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu, KF_IMPLICIT_ARGS | KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu_node, KF_IMPLICIT_ARGS | KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu, KF_IMPLICIT_ARGS | KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu_node, KF_IMPLICIT_ARGS | KF_RCU) -BTF_KFUNCS_END(scx_kfunc_ids_cpu_only) - -/* - * Per-op kfunc allow flags. Each bit corresponds to a context-sensitive kfunc - * group; an op may permit zero or more groups, with the union expressed in - * scx_kf_allow_flags[]. The verifier-time filter (scx_kfunc_context_filter()) - * consults this table to decide whether a context-sensitive kfunc is callable - * from a given SCX op. - */ -enum scx_kf_allow_flags { - SCX_KF_ALLOW_UNLOCKED = 1 << 0, - SCX_KF_ALLOW_INIT = 1 << 1, - SCX_KF_ALLOW_CPU_RELEASE = 1 << 2, - SCX_KF_ALLOW_DISPATCH = 1 << 3, - SCX_KF_ALLOW_ENQUEUE = 1 << 4, - SCX_KF_ALLOW_SELECT_CPU = 1 << 5, -}; - -/* - * Map each SCX op to the union of kfunc groups it permits, indexed by - * SCX_OP_IDX(op). Ops not listed only permit kfuncs that are not - * context-sensitive. - */ -static const u32 scx_kf_allow_flags[] = { - [SCX_OP_IDX(select_cpu)] = SCX_KF_ALLOW_SELECT_CPU | SCX_KF_ALLOW_ENQUEUE, - [SCX_OP_IDX(enqueue)] = SCX_KF_ALLOW_SELECT_CPU | SCX_KF_ALLOW_ENQUEUE, - [SCX_OP_IDX(dispatch)] = SCX_KF_ALLOW_ENQUEUE | SCX_KF_ALLOW_DISPATCH, - [SCX_OP_IDX(cpu_release)] = SCX_KF_ALLOW_CPU_RELEASE, - [SCX_OP_IDX(init_task)] = SCX_KF_ALLOW_UNLOCKED, - [SCX_OP_IDX(dump)] = SCX_KF_ALLOW_UNLOCKED, -#ifdef CONFIG_EXT_GROUP_SCHED - [SCX_OP_IDX(cgroup_init)] = SCX_KF_ALLOW_UNLOCKED, - [SCX_OP_IDX(cgroup_exit)] = SCX_KF_ALLOW_UNLOCKED, - [SCX_OP_IDX(cgroup_prep_move)] = SCX_KF_ALLOW_UNLOCKED, - [SCX_OP_IDX(cgroup_cancel_move)] = SCX_KF_ALLOW_UNLOCKED, - [SCX_OP_IDX(cgroup_set_weight)] = SCX_KF_ALLOW_UNLOCKED, - [SCX_OP_IDX(cgroup_set_bandwidth)] = SCX_KF_ALLOW_UNLOCKED, - [SCX_OP_IDX(cgroup_set_idle)] = SCX_KF_ALLOW_UNLOCKED, -#endif /* CONFIG_EXT_GROUP_SCHED */ - [SCX_OP_IDX(sub_attach)] = SCX_KF_ALLOW_UNLOCKED, - [SCX_OP_IDX(sub_detach)] = SCX_KF_ALLOW_UNLOCKED, - [SCX_OP_IDX(cpu_online)] = SCX_KF_ALLOW_UNLOCKED, - [SCX_OP_IDX(cpu_offline)] = SCX_KF_ALLOW_UNLOCKED, - [SCX_OP_IDX(init)] = SCX_KF_ALLOW_UNLOCKED | SCX_KF_ALLOW_INIT, - [SCX_OP_IDX(exit)] = SCX_KF_ALLOW_UNLOCKED, -}; - -/* - * Verifier-time filter for SCX kfuncs. Registered via the .filter field on - * each per-group btf_kfunc_id_set. The BPF core invokes this for every kfunc - * call in the registered hook (BPF_PROG_TYPE_STRUCT_OPS or - * BPF_PROG_TYPE_SYSCALL), regardless of which set originally introduced the - * kfunc - so the filter must short-circuit on kfuncs it doesn't govern by - * falling through to "allow" when none of the SCX sets contain the kfunc. - */ -int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) -{ - bool in_unlocked = btf_id_set8_contains(&scx_kfunc_ids_unlocked, kfunc_id); - bool in_init = btf_id_set8_contains(&scx_kfunc_ids_init, kfunc_id); - bool in_select_cpu = btf_id_set8_contains(&scx_kfunc_ids_select_cpu, kfunc_id); - bool in_enqueue = btf_id_set8_contains(&scx_kfunc_ids_enqueue_dispatch, kfunc_id); - bool in_dispatch = btf_id_set8_contains(&scx_kfunc_ids_dispatch, kfunc_id); - bool in_cpu_release = btf_id_set8_contains(&scx_kfunc_ids_cpu_release, kfunc_id); - bool in_idle = btf_id_set8_contains(&scx_kfunc_ids_idle, kfunc_id); - bool in_any = btf_id_set8_contains(&scx_kfunc_ids_any, kfunc_id); - bool in_cpu_only = btf_id_set8_contains(&scx_kfunc_ids_cpu_only, kfunc_id); - u32 moff, flags; - - /* Not an SCX kfunc - allow. */ - if (!(in_unlocked || in_init || in_select_cpu || in_enqueue || in_dispatch || - in_cpu_release || in_idle || in_any)) - return 0; - - /* SYSCALL progs (e.g. BPF test_run()) may call unlocked and select_cpu kfuncs. */ - if (prog->type == BPF_PROG_TYPE_SYSCALL) - return (in_unlocked || in_select_cpu || in_idle || in_any) ? 0 : -EACCES; - - if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) - return (in_any || in_idle) ? 0 : -EACCES; - - /* - * add_subprog_and_kfunc() collects all kfunc calls, including dead code - * guarded by bpf_ksym_exists(), before check_attach_btf_id() sets - * prog->aux->st_ops. Allow all kfuncs when st_ops is not yet set; - * do_check_main() re-runs the filter with st_ops set and enforces the - * actual restrictions. - */ - if (!prog->aux->st_ops) - return 0; - - /* - * Non-SCX struct_ops: SCX kfuncs are not permitted. - * - * Both bpf_sched_ext_ops (cpu-form) and bpf_sched_ext_ops_cid - * (cid-form) are valid SCX struct_ops. Member offsets match between - * the two (verified by BUILD_BUG_ON in scx_init()), so the shared - * scx_kf_allow_flags[] table indexed by SCX_MOFF_IDX(moff) applies to - * both. - */ - if (prog->aux->st_ops != &bpf_sched_ext_ops && - prog->aux->st_ops != &bpf_sched_ext_ops_cid) - return -EACCES; - - /* - * cid-form schedulers must use cid/cmask kfuncs. cid and cpu are both - * small s32s and trivially confused, so cpu-only kfuncs are rejected at - * load time. The reverse (cpu-form calling cid-form kfuncs) is - * intentionally permissive to ease gradual cpumask -> cid migration. - */ - if (prog->aux->st_ops == &bpf_sched_ext_ops_cid && in_cpu_only) - return -EACCES; - - /* SCX struct_ops: check the per-op allow list. */ - if (in_any || in_idle) - return 0; - - moff = prog->aux->attach_st_ops_member_off; - flags = scx_kf_allow_flags[SCX_MOFF_IDX(moff)]; - - if ((flags & SCX_KF_ALLOW_UNLOCKED) && in_unlocked) - return 0; - if ((flags & SCX_KF_ALLOW_INIT) && in_init) - return 0; - if ((flags & SCX_KF_ALLOW_CPU_RELEASE) && in_cpu_release) - return 0; - if ((flags & SCX_KF_ALLOW_DISPATCH) && in_dispatch) - return 0; - if ((flags & SCX_KF_ALLOW_ENQUEUE) && in_enqueue) - return 0; - if ((flags & SCX_KF_ALLOW_SELECT_CPU) && in_select_cpu) - return 0; - - return -EACCES; -} - -static int __init scx_init(void) -{ - int ret; - - /* - * sched_ext_ops_cid mirrors sched_ext_ops up to and including @priv. - * Both bpf_scx_init_member() and bpf_scx_check_member() use offsets - * from struct sched_ext_ops; sched_ext_ops_cid relies on those offsets - * matching for the shared fields. Catch any drift at boot. - */ -#define CID_OFFSET_MATCH(cpu_field, cid_field) \ - BUILD_BUG_ON(offsetof(struct sched_ext_ops, cpu_field) != \ - offsetof(struct sched_ext_ops_cid, cid_field)) - /* data fields used by bpf_scx_init_member() */ - CID_OFFSET_MATCH(dispatch_max_batch, dispatch_max_batch); - CID_OFFSET_MATCH(flags, flags); - CID_OFFSET_MATCH(name, name); - CID_OFFSET_MATCH(timeout_ms, timeout_ms); - CID_OFFSET_MATCH(exit_dump_len, exit_dump_len); - CID_OFFSET_MATCH(hotplug_seq, hotplug_seq); - CID_OFFSET_MATCH(sub_cgroup_id, sub_cgroup_id); - /* shared callbacks: the union view requires byte-for-byte offset match */ - CID_OFFSET_MATCH(enqueue, enqueue); - CID_OFFSET_MATCH(dequeue, dequeue); - CID_OFFSET_MATCH(dispatch, dispatch); - CID_OFFSET_MATCH(tick, tick); - CID_OFFSET_MATCH(runnable, runnable); - CID_OFFSET_MATCH(running, running); - CID_OFFSET_MATCH(stopping, stopping); - CID_OFFSET_MATCH(quiescent, quiescent); - CID_OFFSET_MATCH(yield, yield); - CID_OFFSET_MATCH(core_sched_before, core_sched_before); - CID_OFFSET_MATCH(set_weight, set_weight); - CID_OFFSET_MATCH(update_idle, update_idle); - CID_OFFSET_MATCH(init_task, init_task); - CID_OFFSET_MATCH(exit_task, exit_task); - CID_OFFSET_MATCH(enable, enable); - CID_OFFSET_MATCH(disable, disable); - CID_OFFSET_MATCH(dump, dump); - CID_OFFSET_MATCH(dump_task, dump_task); - CID_OFFSET_MATCH(sub_attach, sub_attach); - CID_OFFSET_MATCH(sub_detach, sub_detach); - CID_OFFSET_MATCH(init, init); - CID_OFFSET_MATCH(exit, exit); -#ifdef CONFIG_EXT_GROUP_SCHED - CID_OFFSET_MATCH(cgroup_init, cgroup_init); - CID_OFFSET_MATCH(cgroup_exit, cgroup_exit); - CID_OFFSET_MATCH(cgroup_prep_move, cgroup_prep_move); - CID_OFFSET_MATCH(cgroup_move, cgroup_move); - CID_OFFSET_MATCH(cgroup_cancel_move, cgroup_cancel_move); - CID_OFFSET_MATCH(cgroup_set_weight, cgroup_set_weight); - CID_OFFSET_MATCH(cgroup_set_bandwidth, cgroup_set_bandwidth); - CID_OFFSET_MATCH(cgroup_set_idle, cgroup_set_idle); -#endif - /* renamed callbacks must occupy the same slot as their cpu-form sibling */ - CID_OFFSET_MATCH(select_cpu, select_cid); - CID_OFFSET_MATCH(set_cpumask, set_cmask); - CID_OFFSET_MATCH(cpu_online, cid_online); - CID_OFFSET_MATCH(cpu_offline, cid_offline); - CID_OFFSET_MATCH(dump_cpu, dump_cid); - /* @priv tail must align since both share the same data block */ - CID_OFFSET_MATCH(priv, priv); - /* - * cid-form must end exactly at @priv - validate_ops() skips - * cpu_acquire/cpu_release for cid-form because reading those fields - * past the BPF allocation would be UB. - */ - BUILD_BUG_ON(offsetof(struct sched_ext_ops_cid, __end) != - offsetofend(struct sched_ext_ops, priv)); -#undef CID_OFFSET_MATCH - - /* - * kfunc registration can't be done from init_sched_ext_class() as - * register_btf_kfunc_id_set() needs most of the system to be up. - * - * Some kfuncs are context-sensitive and can only be called from - * specific SCX ops. They are grouped into per-context BTF sets, each - * registered with scx_kfunc_context_filter as its .filter callback. The - * BPF core dedups identical filter pointers per hook - * (btf_populate_kfunc_set()), so the filter is invoked exactly once per - * kfunc lookup; it consults scx_kf_allow_flags[] to enforce per-op - * restrictions at verify time. - */ - if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, - &scx_kfunc_set_enqueue_dispatch)) || - (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, - &scx_kfunc_set_dispatch)) || - (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, - &scx_kfunc_set_cpu_release)) || - (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, - &scx_kfunc_set_unlocked)) || - (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, - &scx_kfunc_set_unlocked)) || - (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, - &scx_kfunc_set_any)) || - (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, - &scx_kfunc_set_any)) || - (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, - &scx_kfunc_set_any))) { - pr_err("sched_ext: Failed to register kfunc sets (%d)\n", ret); - return ret; - } - - ret = scx_idle_init(); - if (ret) { - pr_err("sched_ext: Failed to initialize idle tracking (%d)\n", ret); - return ret; - } - - ret = scx_cid_kfunc_init(); - if (ret) { - pr_err("sched_ext: Failed to register cid kfuncs (%d)\n", ret); - return ret; - } - - ret = register_bpf_struct_ops(&bpf_sched_ext_ops, sched_ext_ops); - if (ret) { - pr_err("sched_ext: Failed to register struct_ops (%d)\n", ret); - return ret; - } - - ret = register_bpf_struct_ops(&bpf_sched_ext_ops_cid, sched_ext_ops_cid); - if (ret) { - pr_err("sched_ext: Failed to register cid struct_ops (%d)\n", ret); - return ret; - } - - ret = register_pm_notifier(&scx_pm_notifier); - if (ret) { - pr_err("sched_ext: Failed to register PM notifier (%d)\n", ret); - return ret; - } - - scx_kset = kset_create_and_add("sched_ext", &scx_uevent_ops, kernel_kobj); - if (!scx_kset) { - pr_err("sched_ext: Failed to create /sys/kernel/sched_ext\n"); - return -ENOMEM; - } - - ret = sysfs_create_group(&scx_kset->kobj, &scx_global_attr_group); - if (ret < 0) { - pr_err("sched_ext: Failed to add global attributes\n"); - return ret; - } - - return 0; -} -__initcall(scx_init); diff --git a/kernel/sched/ext.h b/kernel/sched/ext.h deleted file mode 100644 index 0b7fc46aee08..000000000000 --- a/kernel/sched/ext.h +++ /dev/null @@ -1,95 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst - * - * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. - * Copyright (c) 2022 Tejun Heo - * Copyright (c) 2022 David Vernet - */ -#ifdef CONFIG_SCHED_CLASS_EXT - -void scx_tick(struct rq *rq); -void init_scx_entity(struct sched_ext_entity *scx); -void scx_pre_fork(struct task_struct *p); -int scx_fork(struct task_struct *p, struct kernel_clone_args *kargs); -void scx_post_fork(struct task_struct *p); -void scx_cancel_fork(struct task_struct *p); -bool scx_can_stop_tick(struct rq *rq); -void scx_rq_activate(struct rq *rq); -void scx_rq_deactivate(struct rq *rq); -int scx_check_setscheduler(struct task_struct *p, int policy); -bool task_should_scx(int policy); -bool scx_allow_ttwu_queue(const struct task_struct *p); -void init_sched_ext_class(void); - -static inline u32 scx_cpuperf_target(s32 cpu) -{ - if (scx_enabled()) - return cpu_rq(cpu)->scx.cpuperf_target; - else - return 0; -} - -static inline bool task_on_scx(const struct task_struct *p) -{ - return scx_enabled() && p->sched_class == &ext_sched_class; -} - -#ifdef CONFIG_SCHED_CORE -bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, - bool in_fi); -#endif - -#else /* CONFIG_SCHED_CLASS_EXT */ - -static inline void scx_tick(struct rq *rq) {} -static inline void scx_pre_fork(struct task_struct *p) {} -static inline int scx_fork(struct task_struct *p, struct kernel_clone_args *kargs) { return 0; } -static inline void scx_post_fork(struct task_struct *p) {} -static inline void scx_cancel_fork(struct task_struct *p) {} -static inline u32 scx_cpuperf_target(s32 cpu) { return 0; } -static inline bool scx_can_stop_tick(struct rq *rq) { return true; } -static inline void scx_rq_activate(struct rq *rq) {} -static inline void scx_rq_deactivate(struct rq *rq) {} -static inline int scx_check_setscheduler(struct task_struct *p, int policy) { return 0; } -static inline bool task_on_scx(const struct task_struct *p) { return false; } -static inline bool scx_allow_ttwu_queue(const struct task_struct *p) { return true; } -static inline void init_sched_ext_class(void) {} - -#endif /* CONFIG_SCHED_CLASS_EXT */ - -#ifdef CONFIG_SCHED_CLASS_EXT -void __scx_update_idle(struct rq *rq, bool idle, bool do_notify); - -static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) -{ - if (scx_enabled()) - __scx_update_idle(rq, idle, do_notify); -} -#else -static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) {} -#endif - -#ifdef CONFIG_CGROUP_SCHED -#ifdef CONFIG_EXT_GROUP_SCHED -void scx_tg_init(struct task_group *tg); -int scx_tg_online(struct task_group *tg); -void scx_tg_offline(struct task_group *tg); -int scx_cgroup_can_attach(struct cgroup_taskset *tset); -void scx_cgroup_move_task(struct task_struct *p); -void scx_cgroup_cancel_attach(struct cgroup_taskset *tset); -void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight); -void scx_group_set_idle(struct task_group *tg, bool idle); -void scx_group_set_bandwidth(struct task_group *tg, u64 period_us, u64 quota_us, u64 burst_us); -#else /* CONFIG_EXT_GROUP_SCHED */ -static inline void scx_tg_init(struct task_group *tg) {} -static inline int scx_tg_online(struct task_group *tg) { return 0; } -static inline void scx_tg_offline(struct task_group *tg) {} -static inline int scx_cgroup_can_attach(struct cgroup_taskset *tset) { return 0; } -static inline void scx_cgroup_move_task(struct task_struct *p) {} -static inline void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) {} -static inline void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight) {} -static inline void scx_group_set_idle(struct task_group *tg, bool idle) {} -static inline void scx_group_set_bandwidth(struct task_group *tg, u64 period_us, u64 quota_us, u64 burst_us) {} -#endif /* CONFIG_EXT_GROUP_SCHED */ -#endif /* CONFIG_CGROUP_SCHED */ diff --git a/kernel/sched/ext/arena.c b/kernel/sched/ext/arena.c new file mode 100644 index 000000000000..493c2424f842 --- /dev/null +++ b/kernel/sched/ext/arena.c @@ -0,0 +1,131 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * scx_arena_pool: kernel-side sub-allocator over BPF-arena pages. + * + * Each chunk added to @sch->arena_pool comes from one + * bpf_arena_alloc_pages_sleepable() call and is registered at the + * kernel-side mapping address. Callers translate to the BPF-arena form + * themselves if needed. + * + * Allocations grow the pool on demand. Underlying arena pages are released + * when the arena map itself is torn down. + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2026 Tejun Heo + */ + +enum scx_arena_consts { + SCX_ARENA_MIN_ORDER = 3, /* 8-byte minimum sub-allocation */ + SCX_ARENA_GROW_PAGES = 4, /* per growth */ +}; + +s32 scx_arena_pool_init(struct scx_sched *sch) +{ + if (!sch->arena_map) + return 0; + + sch->arena_pool = gen_pool_create(SCX_ARENA_MIN_ORDER, NUMA_NO_NODE); + if (!sch->arena_pool) + return -ENOMEM; + return 0; +} + +static void scx_arena_clear_chunk(struct gen_pool *pool, struct gen_pool_chunk *chunk, + void *data) +{ + int order = pool->min_alloc_order; + size_t chunk_sz = chunk->end_addr - chunk->start_addr + 1; + unsigned long end_bit = chunk_sz >> order; + unsigned long b, e; + + for_each_set_bitrange(b, e, chunk->bits, end_bit) + gen_pool_free(pool, chunk->start_addr + (b << order), + (e - b) << order); +} + +/* + * Tear down the pool. Outstanding gen_pool allocations are freed via + * scx_arena_clear_chunk() so gen_pool_destroy() doesn't BUG. The underlying + * arena pages are released when the arena map itself is torn down. + */ +void scx_arena_pool_destroy(struct scx_sched *sch) +{ + if (!sch->arena_pool) + return; + gen_pool_for_each_chunk(sch->arena_pool, scx_arena_clear_chunk, NULL); + gen_pool_destroy(sch->arena_pool); + sch->arena_pool = NULL; +} + +/* + * Grow the pool by @page_cnt pages. bpf_arena_alloc_pages_sleepable() and + * gen_pool_add() (which calls vzalloc(GFP_KERNEL)) require a sleepable + * context. + */ +static int scx_arena_grow(struct scx_sched *sch, u32 page_cnt) +{ + u64 kern_vm_start; + u32 uaddr32; + void *p; + int ret; + + if (!sch->arena_map || !sch->arena_pool) + return -EINVAL; + + p = bpf_arena_alloc_pages_sleepable(sch->arena_map, NULL, + page_cnt, NUMA_NO_NODE, 0); + if (!p) + return -ENOMEM; + + uaddr32 = (u32)(unsigned long)p; + /* arena.o, which defines these, is built only on MMU && 64BIT */ +#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) + kern_vm_start = bpf_arena_map_kern_vm_start(sch->arena_map); +#else + kern_vm_start = 0; +#endif + + ret = gen_pool_add(sch->arena_pool, kern_vm_start + uaddr32, + page_cnt * PAGE_SIZE, NUMA_NO_NODE); + if (ret) { + bpf_arena_free_pages_non_sleepable(sch->arena_map, p, page_cnt); + return ret; + } + return 0; +} + +/* + * Allocate @size bytes from the arena pool. Returns kernel VA on success, NULL + * on failure. May grow the pool via scx_arena_grow() which sleeps. Caller must + * be in a GFP_KERNEL context. + */ +void *scx_arena_alloc(struct scx_sched *sch, size_t size) +{ + unsigned long kern_va; + u32 page_cnt; + + might_sleep(); + + if (!sch->arena_pool) + return NULL; + + while (true) { + kern_va = gen_pool_alloc(sch->arena_pool, size); + if (kern_va) + break; + page_cnt = max_t(u32, SCX_ARENA_GROW_PAGES, + (size + PAGE_SIZE - 1) >> PAGE_SHIFT); + if (scx_arena_grow(sch, page_cnt)) + return NULL; + } + + return (void *)kern_va; +} + +void scx_arena_free(struct scx_sched *sch, void *kern_va, size_t size) +{ + if (sch->arena_pool && kern_va) + gen_pool_free(sch->arena_pool, (unsigned long)kern_va, size); +} diff --git a/kernel/sched/ext/arena.h b/kernel/sched/ext/arena.h new file mode 100644 index 000000000000..4f3610160102 --- /dev/null +++ b/kernel/sched/ext/arena.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Copyright (c) 2025 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2025 Tejun Heo + */ +#ifndef _KERNEL_SCHED_EXT_ARENA_H +#define _KERNEL_SCHED_EXT_ARENA_H + +struct scx_sched; + +s32 scx_arena_pool_init(struct scx_sched *sch); +void scx_arena_pool_destroy(struct scx_sched *sch); +void *scx_arena_alloc(struct scx_sched *sch, size_t size); +void scx_arena_free(struct scx_sched *sch, void *kern_va, size_t size); + +#endif /* _KERNEL_SCHED_EXT_ARENA_H */ diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c new file mode 100644 index 000000000000..aeaea88f34c5 --- /dev/null +++ b/kernel/sched/ext/cid.c @@ -0,0 +1,707 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2026 Tejun Heo + */ +#include + +/* + * cid tables. + * + * Pointers are published once on first enable and never revoked. The default + * mapping is populated before ops.init() runs; scx_bpf_cid_override() commits + * before it returns. As long as the BPF scheduler only uses the tables from + * those points onward, it sees a consistent view. + */ +s16 *scx_cid_to_cpu_tbl; +s16 *scx_cpu_to_cid_tbl; +struct scx_cid_topo *scx_cid_topo; + +#define SCX_CID_TOPO_NEG (struct scx_cid_topo) { \ + .core_cid = -1, .core_idx = -1, .llc_cid = -1, .llc_idx = -1, \ + .node_cid = -1, .node_idx = -1, \ +} + +/* + * Return @cpu's LLC shared_cpu_map. If cacheinfo isn't populated (offline or + * !present), record @cpu in @fallbacks and return its node mask instead - the + * worst that can happen is that the cpu's LLC becomes coarser than reality. + */ +static const struct cpumask *cpu_llc_mask(int cpu, struct cpumask *fallbacks) +{ + struct cpu_cacheinfo *ci = get_cpu_cacheinfo(cpu); + + if (!ci || !ci->info_list || !ci->num_leaves) { + cpumask_set_cpu(cpu, fallbacks); + return cpumask_of_node(cpu_to_node(cpu)); + } + return &ci->info_list[ci->num_leaves - 1].shared_cpu_map; +} + +/* Allocate the cid tables once on first enable; never freed. */ +static s32 scx_cid_arrays_alloc(void) +{ + u32 npossible = num_possible_cpus(); + s16 *cid_to_cpu, *cpu_to_cid; + struct scx_cid_topo *cid_topo; + + if (scx_cid_to_cpu_tbl) + return 0; + + cid_to_cpu = kzalloc_objs(*scx_cid_to_cpu_tbl, npossible, GFP_KERNEL); + cpu_to_cid = kzalloc_objs(*scx_cpu_to_cid_tbl, nr_cpu_ids, GFP_KERNEL); + cid_topo = kmalloc_objs(*scx_cid_topo, npossible, GFP_KERNEL); + + if (!cid_to_cpu || !cpu_to_cid || !cid_topo) { + kfree(cid_to_cpu); + kfree(cpu_to_cid); + kfree(cid_topo); + return -ENOMEM; + } + + WRITE_ONCE(scx_cid_to_cpu_tbl, cid_to_cpu); + WRITE_ONCE(scx_cpu_to_cid_tbl, cpu_to_cid); + WRITE_ONCE(scx_cid_topo, cid_topo); + return 0; +} + +/** + * scx_cid_init - build the cid mapping + * @sch: the scx_sched being initialized; used as the scx_error() target + * + * See "Topological CPU IDs" in cid.h for the model. Walk online cpus by + * intersection at each level (parent_scratch & this_level_mask), which keeps + * containment correct by construction and naturally splits a physical LLC + * straddling two NUMA nodes into two LLC units. The caller must hold + * cpus_read_lock. + */ +s32 scx_cid_init(struct scx_sched *sch) +{ + cpumask_var_t to_walk __free(free_cpumask_var) = CPUMASK_VAR_NULL; + cpumask_var_t node_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; + cpumask_var_t llc_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; + cpumask_var_t core_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; + cpumask_var_t llc_fallback __free(free_cpumask_var) = CPUMASK_VAR_NULL; + cpumask_var_t online_no_topo __free(free_cpumask_var) = CPUMASK_VAR_NULL; + u32 next_cid = 0; + s32 next_node_idx = 0, next_llc_idx = 0, next_core_idx = 0; + s32 cpu, ret; + + /* CMASK_MAX_WORDS in cid.bpf.h covers NR_CPUS up to 8192 */ + BUILD_BUG_ON(NR_CPUS > 8192); + + lockdep_assert_cpus_held(); + + ret = scx_cid_arrays_alloc(); + if (ret) + return ret; + + if (!zalloc_cpumask_var(&to_walk, GFP_KERNEL) || + !zalloc_cpumask_var(&node_scratch, GFP_KERNEL) || + !zalloc_cpumask_var(&llc_scratch, GFP_KERNEL) || + !zalloc_cpumask_var(&core_scratch, GFP_KERNEL) || + !zalloc_cpumask_var(&llc_fallback, GFP_KERNEL) || + !zalloc_cpumask_var(&online_no_topo, GFP_KERNEL)) + return -ENOMEM; + + /* -1 sentinels for sparse-possible cpu id holes (0 is a valid cid) */ + for (cpu = 0; cpu < nr_cpu_ids; cpu++) + scx_cpu_to_cid_tbl[cpu] = -1; + + cpumask_copy(to_walk, cpu_online_mask); + + while (!cpumask_empty(to_walk)) { + s32 next_cpu = cpumask_first(to_walk); + s32 nid = cpu_to_node(next_cpu); + s32 node_cid = next_cid; + s32 node_idx; + + /* + * No NUMA info: skip and let the tail loop assign a no-topo + * cid. cpumask_of_node(-1) is undefined. + */ + if (nid < 0) { + cpumask_clear_cpu(next_cpu, to_walk); + continue; + } + + node_idx = next_node_idx++; + + /* node_scratch = to_walk & this node */ + cpumask_and(node_scratch, to_walk, cpumask_of_node(nid)); + if (WARN_ON_ONCE(!cpumask_test_cpu(next_cpu, node_scratch))) + return -EINVAL; + + while (!cpumask_empty(node_scratch)) { + s32 ncpu = cpumask_first(node_scratch); + const struct cpumask *llc_mask = cpu_llc_mask(ncpu, llc_fallback); + s32 llc_cid = next_cid; + s32 llc_idx = next_llc_idx++; + + /* llc_scratch = node_scratch & this llc */ + cpumask_and(llc_scratch, node_scratch, llc_mask); + if (WARN_ON_ONCE(!cpumask_test_cpu(ncpu, llc_scratch))) + return -EINVAL; + + while (!cpumask_empty(llc_scratch)) { + s32 lcpu = cpumask_first(llc_scratch); + const struct cpumask *sib = topology_sibling_cpumask(lcpu); + s32 core_cid = next_cid; + s32 core_idx = next_core_idx++; + s32 ccpu; + + /* core_scratch = llc_scratch & this core */ + cpumask_and(core_scratch, llc_scratch, sib); + if (WARN_ON_ONCE(!cpumask_test_cpu(lcpu, core_scratch))) + return -EINVAL; + + for_each_cpu(ccpu, core_scratch) { + s32 cid = next_cid++; + + scx_cid_to_cpu_tbl[cid] = ccpu; + scx_cpu_to_cid_tbl[ccpu] = cid; + scx_cid_topo[cid] = (struct scx_cid_topo){ + .core_cid = core_cid, + .core_idx = core_idx, + .llc_cid = llc_cid, + .llc_idx = llc_idx, + .node_cid = node_cid, + .node_idx = node_idx, + }; + + cpumask_clear_cpu(ccpu, llc_scratch); + cpumask_clear_cpu(ccpu, node_scratch); + cpumask_clear_cpu(ccpu, to_walk); + } + } + } + } + + /* + * No-topo section: any possible cpu without a cid - normally just the + * not-online ones. Collect any currently-online cpus that land here in + * @online_no_topo so we can warn about them at the end. + */ + for_each_cpu(cpu, cpu_possible_mask) { + s32 cid; + + if (__scx_cpu_to_cid(cpu) != -1) + continue; + if (cpu_online(cpu)) + cpumask_set_cpu(cpu, online_no_topo); + + cid = next_cid++; + scx_cid_to_cpu_tbl[cid] = cpu; + scx_cpu_to_cid_tbl[cpu] = cid; + scx_cid_topo[cid] = SCX_CID_TOPO_NEG; + } + + if (!cpumask_empty(llc_fallback)) + pr_warn("scx_cid: cpus without cacheinfo, using node mask as llc: %*pbl\n", + cpumask_pr_args(llc_fallback)); + if (!cpumask_empty(online_no_topo)) + pr_warn("scx_cid: online cpus with no usable topology: %*pbl\n", + cpumask_pr_args(online_no_topo)); + + return 0; +} + +/** + * scx_cmask_clear - Zero every bit in @m's active range + * @m: cmask to clear + * + * Storage past the active range is left as is. + */ +void scx_cmask_clear(struct scx_cmask *m) +{ + u32 nr_words; + + if (!m->nr_cids) + return; + nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1; + memset(m->bits, 0, nr_words * sizeof(u64)); +} + +/** + * scx_cmask_fill - Set every bit in @m's active range + * @m: cmask to fill + * + * Counterpart to scx_cmask_clear(). Storage past the active range is left as is. + */ +void scx_cmask_fill(struct scx_cmask *m) +{ + u32 nr_words, head_bits, tail_bits; + + if (!m->nr_cids) + return; + nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1; + memset(m->bits, 0xff, nr_words * sizeof(u64)); + + /* clear word-0 bits below base */ + head_bits = m->base & 63; + if (head_bits) + m->bits[0] &= ~((1ULL << head_bits) - 1); + + /* clear last-word bits at or past base + nr_cids */ + tail_bits = (m->base + m->nr_cids) & 63; + if (tail_bits) + m->bits[nr_words - 1] &= (1ULL << tail_bits) - 1; +} + +/** + * scx_cpumask_to_cmask - Translate a kernel cpumask into a cmask + * @src: source cpumask + * @dst: cmask to write + * + * Clear @dst's active range and set the bit for each cid whose cpu is in + * @src and lies within that range. Out-of-range cids are silently ignored. + */ +void scx_cpumask_to_cmask(const struct cpumask *src, struct scx_cmask *dst) +{ + s32 cpu; + + scx_cmask_clear(dst); + for_each_cpu(cpu, src) { + s32 cid = __scx_cpu_to_cid(cpu); + + if (cid >= 0) + __scx_cmask_set(cid, dst); + } +} + +__bpf_kfunc_start_defs(); + +/** + * scx_bpf_cid_override - Install an explicit cpu->cid mapping + * @cpu_to_cid: array of nr_cpu_ids s32 entries (cid for each cpu) + * @cpu_to_cid__sz: must be nr_cpu_ids * sizeof(s32) bytes + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * May only be called from ops.init() of the root scheduler. Replace the + * topology-probed cid mapping with the caller-provided one. Each possible cpu + * must map to a unique cid in [0, num_possible_cpus()). Topo info is cleared. + * On invalid input, trigger scx_error() to abort the scheduler. + */ +__bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid, u32 cpu_to_cid__sz, + const struct bpf_prog_aux *aux) +{ + cpumask_var_t seen __free(free_cpumask_var) = CPUMASK_VAR_NULL; + struct scx_sched *sch; + bool alloced; + s32 cpu, cid; + + /* GFP_KERNEL alloc must happen before the rcu read section */ + alloced = zalloc_cpumask_var(&seen, GFP_KERNEL); + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return; + + if (!alloced) { + scx_error(sch, "scx_bpf_cid_override: failed to allocate cpumask"); + return; + } + + if (scx_parent(sch)) { + scx_error(sch, "scx_bpf_cid_override() only allowed from root sched"); + return; + } + + if (cpu_to_cid__sz != nr_cpu_ids * sizeof(s32)) { + scx_error(sch, "scx_bpf_cid_override: expected %zu bytes, got %u", + nr_cpu_ids * sizeof(s32), cpu_to_cid__sz); + return; + } + + for_each_possible_cpu(cpu) { + s32 c = cpu_to_cid[cpu]; + + if (!cid_valid(sch, c)) + return; + if (cpumask_test_and_set_cpu(c, seen)) { + scx_error(sch, "cid %d assigned to multiple cpus", c); + return; + } + scx_cpu_to_cid_tbl[cpu] = c; + scx_cid_to_cpu_tbl[c] = cpu; + } + + /* Invalidate stale topo info - the override carries no topology. */ + for (cid = 0; cid < num_possible_cpus(); cid++) + scx_cid_topo[cid] = SCX_CID_TOPO_NEG; +} + +/** + * scx_bpf_cid_to_cpu - Return the raw CPU id for @cid + * @cid: cid to look up + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Return the raw CPU id for @cid. Trigger scx_error() and return -EINVAL if + * @cid is invalid. The cid<->cpu mapping is static for the lifetime of the + * loaded scheduler, so the BPF side can cache the result to avoid repeated + * kfunc invocations. + */ +__bpf_kfunc s32 scx_bpf_cid_to_cpu(s32 cid, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return -EINVAL; + return scx_cid_to_cpu(sch, cid); +} + +/** + * scx_bpf_cpu_to_cid - Return the cid for @cpu + * @cpu: cpu to look up + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Return the cid for @cpu. Trigger scx_error() and return -EINVAL if @cpu is + * invalid. The cid<->cpu mapping is static for the lifetime of the loaded + * scheduler, so the BPF side can cache the result to avoid repeated kfunc + * invocations. + */ +__bpf_kfunc s32 scx_bpf_cpu_to_cid(s32 cpu, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return -EINVAL; + return scx_cpu_to_cid(sch, cpu); +} + +/* + * Set ops on cmasks. cmask_walk_op2() shares one walk across mutating + * (and/or/copy/andnot) and predicate (subset/intersects) two-cmask forms; + * cmask_walk_op1() does the same shape over a single cmask range. Every public + * entry passes a compile-time-constant @op; cmask_walk_op{1,2}() and + * cmask_word_op{1,2}() are __always_inline so the inner switch collapses to the + * selected op and cmask_op2_is_pred() folds the predicate early-exit out of + * mutating ops. + * + * Two-cmask ops only touch @dst bits inside the intersection of the two ranges; + * bits outside stay untouched. In particular, scx_cmask_copy() does NOT zero + * @dst bits that lie outside @src's range. + * + * The _RACY variants are otherwise identical to their non-racy counterpart but + * read @src word-by-word via data_race(). Memory ordering with concurrent + * writers is the caller's responsibility. + */ +enum cmask_op2 { + /* mutating */ + CMASK_OP2_AND, + CMASK_OP2_OR, + CMASK_OP2_OR_RACY, + CMASK_OP2_COPY, + CMASK_OP2_COPY_RACY, + CMASK_OP2_ANDNOT, + /* predicates - short-circuit when the per-word result is true */ + CMASK_OP2_SUBSET, + CMASK_OP2_INTERSECTS, +}; + +static __always_inline bool cmask_op2_is_pred(const enum cmask_op2 op) +{ + return op == CMASK_OP2_SUBSET || op == CMASK_OP2_INTERSECTS; +} + +static __always_inline bool cmask_word_op2(u64 *av, const u64 *bp, u64 mask, + const enum cmask_op2 op) +{ + switch (op) { + case CMASK_OP2_AND: + *av &= ~mask | *bp; + return false; + case CMASK_OP2_OR: + *av |= *bp & mask; + return false; + case CMASK_OP2_OR_RACY: + *av |= data_race(*bp) & mask; + return false; + case CMASK_OP2_COPY: + *av = (*av & ~mask) | (*bp & mask); + return false; + case CMASK_OP2_COPY_RACY: + *av = (*av & ~mask) | (data_race(*bp) & mask); + return false; + case CMASK_OP2_ANDNOT: + *av &= ~(*bp & mask); + return false; + case CMASK_OP2_SUBSET: + /* stop on the first bit in @sub not set in @super */ + return (*bp & ~*av) & mask; + case CMASK_OP2_INTERSECTS: + return (*av & *bp) & mask; + } + unreachable(); +} + +/* + * Walk the intersection of [@a_base, @a_base + @a_nr_cids) with [@b_base, + * @b_base + @b_nr_cids) word by word, applying @op. Mutating ops walk all words + * and return false; predicates return true on the first word whose per-word + * test is true. Empty intersection returns false (matches "no bits to consider" + * for both mutate and predicate). + * + * Base/nr_cids are taken as parameters so callers with snapshotted bounds can + * drive the walk with values independent of the cmask's header. + */ +static __always_inline bool cmask_walk_op2(u64 *a_bits, u32 a_base, u32 a_nr_cids, + const u64 *b_bits, u32 b_base, u32 b_nr_cids, + const enum cmask_op2 op) +{ + u32 lo = max(a_base, b_base); + u32 hi = min(a_base + a_nr_cids, b_base + b_nr_cids); + u32 a_word_off = a_base / 64; + u32 b_word_off = b_base / 64; + u32 lo_word = lo / 64; + u32 hi_word = (hi - 1) / 64; + u64 head_mask = GENMASK_U64(63, lo & 63); + u64 tail_mask = GENMASK_U64((hi - 1) & 63, 0); + u32 w; + + if (lo >= hi) + return false; + + if (lo_word == hi_word) + return cmask_word_op2(&a_bits[lo_word - a_word_off], + &b_bits[lo_word - b_word_off], + head_mask & tail_mask, op); + + if (cmask_word_op2(&a_bits[lo_word - a_word_off], + &b_bits[lo_word - b_word_off], head_mask, op) && + cmask_op2_is_pred(op)) + return true; + + for (w = lo_word + 1; w < hi_word; w++) + if (cmask_word_op2(&a_bits[w - a_word_off], + &b_bits[w - b_word_off], ~0ULL, op) && + cmask_op2_is_pred(op)) + return true; + + return cmask_word_op2(&a_bits[hi_word - a_word_off], + &b_bits[hi_word - b_word_off], tail_mask, op); +} + +enum cmask_op1 { + CMASK_OP1_ANY_SET, +}; + +static __always_inline bool cmask_word_op1(const u64 *ap, u64 mask, + const enum cmask_op1 op) +{ + switch (op) { + case CMASK_OP1_ANY_SET: + return *ap & mask; + } + unreachable(); +} + +/* + * Walk [@a_base, @a_base + @a_nr_cids) of @a_bits word by word, applying @op. + * Returns true on the first word whose per-word test is true; returns false if + * no word matches or the range is empty. All current op1s short-circuit on + * per-word true; if a non-predicate op1 lands here, add a cmask_op1_is_pred() + * guard analogous to cmask_op2_is_pred(). + */ +static __always_inline bool cmask_walk_op1(const u64 *a_bits, u32 a_base, + u32 a_nr_cids, + const enum cmask_op1 op) +{ + u32 lo = a_base; + u32 hi = a_base + a_nr_cids; + u32 a_word_off = a_base / 64; + u32 lo_word = lo / 64; + u32 hi_word = (hi - 1) / 64; + u64 head_mask = GENMASK_U64(63, lo & 63); + u64 tail_mask = GENMASK_U64((hi - 1) & 63, 0); + u32 w; + + if (lo >= hi) + return false; + + if (lo_word == hi_word) + return cmask_word_op1(&a_bits[lo_word - a_word_off], + head_mask & tail_mask, op); + + if (cmask_word_op1(&a_bits[lo_word - a_word_off], head_mask, op)) + return true; + for (w = lo_word + 1; w < hi_word; w++) + if (cmask_word_op1(&a_bits[w - a_word_off], ~0ULL, op)) + return true; + return cmask_word_op1(&a_bits[hi_word - a_word_off], tail_mask, op); +} + +void scx_cmask_and(struct scx_cmask *dst, const struct scx_cmask *src) +{ + cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, + src->bits, src->base, src->nr_cids, CMASK_OP2_AND); +} + +void scx_cmask_or(struct scx_cmask *dst, const struct scx_cmask *src) +{ + cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, + src->bits, src->base, src->nr_cids, CMASK_OP2_OR); +} + +/** + * scx_cmask_or_racy - OR @src into @dst, reading @src without locking + * + * @src is read word-by-word through data_race(). Same per-bit independence + * rationale as scx_cmask_copy_racy(). Memory ordering with writers is the + * caller's responsibility. + */ +void scx_cmask_or_racy(struct scx_cmask *dst, const struct scx_cmask *src) +{ + cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, + src->bits, src->base, src->nr_cids, CMASK_OP2_OR_RACY); +} + +void scx_cmask_copy(struct scx_cmask *dst, const struct scx_cmask *src) +{ + cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, + src->bits, src->base, src->nr_cids, CMASK_OP2_COPY); +} + +/** + * scx_cmask_copy_racy - Snapshot @src into @dst without locking + * + * @src is read word-by-word through data_race(). Head/tail masking matches + * scx_cmask_copy(). Each bit in a cmask is independent, so partial updates + * just leave some bits fresher than others. Memory ordering with writers is + * the caller's responsibility. + */ +void scx_cmask_copy_racy(struct scx_cmask *dst, const struct scx_cmask *src) +{ + cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, + src->bits, src->base, src->nr_cids, CMASK_OP2_COPY_RACY); +} + +void scx_cmask_andnot(struct scx_cmask *dst, const struct scx_cmask *src) +{ + cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, + src->bits, src->base, src->nr_cids, CMASK_OP2_ANDNOT); +} + +/* + * Return true if @cm has any bit set in [@lo, @hi). Caller must ensure + * [@lo, @hi) is contained in @cm's range. + */ +static bool cmask_any_set_in_range(const struct scx_cmask *cm, u32 lo, u32 hi) +{ + if (lo >= hi) + return false; + return cmask_walk_op1(&cm->bits[lo / 64 - cm->base / 64], lo, hi - lo, + CMASK_OP1_ANY_SET); +} + +/** + * scx_cmask_subset - test whether @sub is a subset of @super + * @sub: cmask to test + * @super: cmask to test against + * + * Return true iff every set bit of @sub is also set in @super. + */ +bool scx_cmask_subset(const struct scx_cmask *sub, const struct scx_cmask *super) +{ + u32 super_end = super->base + super->nr_cids; + u32 sub_end = sub->base + sub->nr_cids; + + /* + * Set bits in @sub outside @super's range can't be in @super, so any + * such bit means not a subset. The walk below only visits words + * common to both ranges, so these need a separate scan. + */ + if (sub->base < super->base && + cmask_any_set_in_range(sub, sub->base, min(super->base, sub_end))) + return false; + if (sub_end > super_end && + cmask_any_set_in_range(sub, max(sub->base, super_end), sub_end)) + return false; + + return !cmask_walk_op2((u64 *)super->bits, super->base, super->nr_cids, + sub->bits, sub->base, sub->nr_cids, CMASK_OP2_SUBSET); +} + +bool scx_cmask_intersects(const struct scx_cmask *a, const struct scx_cmask *b) +{ + return cmask_walk_op2((u64 *)a->bits, a->base, a->nr_cids, + b->bits, b->base, b->nr_cids, CMASK_OP2_INTERSECTS); +} + +/** + * scx_cmask_empty - Test whether @m has no bits set + * @m: cmask to test + * + * Return true iff @m's active range has no bits set. + */ +bool scx_cmask_empty(const struct scx_cmask *m) +{ + return !cmask_any_set_in_range(m, m->base, m->base + m->nr_cids); +} + +/** + * scx_bpf_cid_topo - Copy out per-cid topology info + * @cid: cid to look up + * @out__uninit: where to copy the topology info; fully written by this call + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Fill @out__uninit with the topology info for @cid. Trigger scx_error() if + * @cid is out of range. If @cid is valid but in the no-topo section, all fields + * are set to -1. + */ +__bpf_kfunc void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out__uninit, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch) || !cid_valid(sch, cid)) { + *out__uninit = SCX_CID_TOPO_NEG; + return; + } + + *out__uninit = READ_ONCE(scx_cid_topo)[cid]; +} + +__bpf_kfunc_end_defs(); + +BTF_KFUNCS_START(scx_kfunc_ids_init) +BTF_ID_FLAGS(func, scx_bpf_cid_override, KF_IMPLICIT_ARGS | KF_SLEEPABLE) +BTF_KFUNCS_END(scx_kfunc_ids_init) + +static const struct btf_kfunc_id_set scx_kfunc_set_init = { + .owner = THIS_MODULE, + .set = &scx_kfunc_ids_init, + .filter = scx_kfunc_context_filter, +}; + +BTF_KFUNCS_START(scx_kfunc_ids_cid) +BTF_ID_FLAGS(func, scx_bpf_cid_to_cpu, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cpu_to_cid, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cid_topo, KF_IMPLICIT_ARGS) +BTF_KFUNCS_END(scx_kfunc_ids_cid) + +static const struct btf_kfunc_id_set scx_kfunc_set_cid = { + .owner = THIS_MODULE, + .set = &scx_kfunc_ids_cid, +}; + +int scx_cid_kfunc_init(void) +{ + return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_init) ?: + register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_cid) ?: + register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &scx_kfunc_set_cid) ?: + register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_cid); +} diff --git a/kernel/sched/ext/cid.h b/kernel/sched/ext/cid.h new file mode 100644 index 000000000000..6e657fd147b0 --- /dev/null +++ b/kernel/sched/ext/cid.h @@ -0,0 +1,271 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Topological CPU IDs (cids) + * -------------------------- + * + * Raw cpu numbers are clumsy for sharding work and communication across + * topology units, especially from BPF: the space can be sparse, numerical + * closeness doesn't imply topological closeness (x86 hyperthreading often puts + * SMT siblings far apart), and a range of cpu ids doesn't mean anything. + * Sub-scheds make this acute - cpu allocation, revocation and other state are + * constantly communicated across sub-scheds, and passing whole cpumasks scales + * poorly with cpu count. cpumasks are also awkward in BPF: a variable-length + * kernel type sized for the maximum NR_CPUS (4k), with verbose helper sequences + * for every op. + * + * cids give every cpu a dense, topology-ordered id. CPUs sharing a core, LLC or + * NUMA node get contiguous cid ranges, so a topology unit becomes a (start, + * length) slice of cid space. Communication can pass a slice instead of a + * cpumask, and BPF code can process, for example, a u64 word's worth of cids at + * a time. + * + * The mapping is built once at root scheduler enable time by walking the + * topology of online cpus only. Going by online cpus is out of necessity: + * depending on the arch, topology info isn't reliably available for offline + * cpus. The expected usage model is restarting the scheduler on hotplug events + * so the mapping is rebuilt against the new online set. A scheduler that wants + * to handle hotplug without a restart can provide its own cid and shard mapping + * through the override interface. + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2026 Tejun Heo + */ +#ifndef _KERNEL_SCHED_EXT_CID_H +#define _KERNEL_SCHED_EXT_CID_H + +struct scx_sched; + +/* + * Cid space (total is always num_possible_cpus()) is laid out with + * topology-annotated cids first, then no-topo cids at the tail. The + * topology-annotated block covers the cpus that were online when scx_cid_init() + * ran and remains valid even after those cpus go offline. The tail block covers + * possible-but-not-online cpus and carries all-(-1) topo info (see + * scx_cid_topo); callers detect it via the -1 sentinels. + * + * See the comment above the table definitions in cid.c for the + * memory-ordering and visibility contract. + */ +extern s16 *scx_cid_to_cpu_tbl; +extern s16 *scx_cpu_to_cid_tbl; +extern struct scx_cid_topo *scx_cid_topo; +extern struct btf_id_set8 scx_kfunc_ids_init; + +void scx_cmask_clear(struct scx_cmask *m); +void scx_cmask_fill(struct scx_cmask *m); +void scx_cmask_and(struct scx_cmask *dst, const struct scx_cmask *src); +void scx_cmask_or(struct scx_cmask *dst, const struct scx_cmask *src); +void scx_cmask_or_racy(struct scx_cmask *dst, const struct scx_cmask *src); +void scx_cmask_copy(struct scx_cmask *dst, const struct scx_cmask *src); +void scx_cmask_copy_racy(struct scx_cmask *dst, const struct scx_cmask *src); +void scx_cmask_andnot(struct scx_cmask *dst, const struct scx_cmask *src); +bool scx_cmask_subset(const struct scx_cmask *sub, const struct scx_cmask *super); +bool scx_cmask_intersects(const struct scx_cmask *a, const struct scx_cmask *b); +bool scx_cmask_empty(const struct scx_cmask *m); +s32 scx_cid_init(struct scx_sched *sch); +int scx_cid_kfunc_init(void); +void scx_cpumask_to_cmask(const struct cpumask *src, struct scx_cmask *dst); + +/** + * cid_valid - Verify a cid value, to be used on ops input args + * @sch: scx_sched to abort on error + * @cid: cid which came from a BPF ops + * + * Return true if @cid is in [0, num_possible_cpus()). On failure, trigger + * scx_error() and return false. + */ +static inline bool cid_valid(struct scx_sched *sch, s32 cid) +{ + if (likely(cid >= 0 && cid < num_possible_cpus())) + return true; + scx_error(sch, "invalid cid %d", cid); + return false; +} + +/** + * __scx_cid_to_cpu - Unchecked cid->cpu table lookup + * @cid: cid to look up. Must be in [0, num_possible_cpus()). + * + * Intended for callsites that have already validated @cid and that hold a + * non-NULL @sch from scx_prog_sched() - a live sched implies the table has + * been allocated, so no NULL check is needed here. + */ +static inline s32 __scx_cid_to_cpu(s32 cid) +{ + /* READ_ONCE pairs with WRITE_ONCE in scx_cid_arrays_alloc() */ + return READ_ONCE(scx_cid_to_cpu_tbl)[cid]; +} + +/** + * __scx_cpu_to_cid - Unchecked cpu->cid table lookup + * @cpu: cpu to look up. Must be a valid possible cpu id. + * + * Same usage constraints as __scx_cid_to_cpu(). + */ +static inline s32 __scx_cpu_to_cid(s32 cpu) +{ + return READ_ONCE(scx_cpu_to_cid_tbl)[cpu]; +} + +/** + * scx_cid_to_cpu - Translate @cid to its cpu + * @sch: scx_sched for error reporting + * @cid: cid to look up + * + * Return the cpu for @cid or a negative errno on failure. Invalid cid triggers + * scx_error() on @sch. The cid arrays are allocated on first scheduler enable + * and never freed, so the returned cpu is stable for the lifetime of the loaded + * scheduler. + */ +static inline s32 scx_cid_to_cpu(struct scx_sched *sch, s32 cid) +{ + if (!cid_valid(sch, cid)) + return -EINVAL; + return __scx_cid_to_cpu(cid); +} + +/** + * scx_cpu_to_cid - Translate @cpu to its cid + * @sch: scx_sched for error reporting + * @cpu: cpu to look up + * + * Return the cid for @cpu or a negative errno on failure. Invalid cpu triggers + * scx_error() on @sch. Same lifetime guarantee as scx_cid_to_cpu(). + */ +static inline s32 scx_cpu_to_cid(struct scx_sched *sch, s32 cpu) +{ + if (!scx_cpu_valid(sch, cpu, NULL)) + return -EINVAL; + return __scx_cpu_to_cid(cpu); +} + +/** + * scx_is_cid_type - Test whether the active scheduler hierarchy is cid-form + */ +static inline bool scx_is_cid_type(void) +{ + return static_branch_unlikely(&__scx_is_cid_type); +} + +static inline bool __scx_cmask_contains(u32 cid, const struct scx_cmask *m) +{ + return likely(cid >= m->base && cid < m->base + m->nr_cids); +} + +/* Word in bits[] covering @cid. @cid must satisfy __scx_cmask_contains(). */ +static inline u64 *__scx_cmask_word(u32 cid, const struct scx_cmask *m) +{ + return (u64 *)&m->bits[cid / 64 - m->base / 64]; +} + +/** + * __scx_cmask_init - Initialize @m with explicit storage capacity + * @m: cmask to initialize + * @base: first cid of the active range + * @nr_cids: number of cids in the active range + * @alloc_cids: storage capacity in cids, at least @nr_cids + * + * Use when storage is sized larger than the initial active range. All of + * bits[] is zeroed. + */ +static inline void __scx_cmask_init(struct scx_cmask *m, u32 base, u32 nr_cids, + u32 alloc_cids) +{ + if (WARN_ON_ONCE(alloc_cids < nr_cids)) + nr_cids = alloc_cids; + + m->base = base; + m->nr_cids = nr_cids; + m->alloc_words = SCX_CMASK_NR_WORDS(alloc_cids); + memset(m->bits, 0, m->alloc_words * sizeof(u64)); +} + +/** + * scx_cmask_init - Initialize @m on tight storage + * @m: cmask to initialize + * @base: first cid of the active range + * @nr_cids: number of cids in the active range + * + * All of bits[] is zeroed. + */ +static inline void scx_cmask_init(struct scx_cmask *m, u32 base, u32 nr_cids) +{ + __scx_cmask_init(m, base, nr_cids, nr_cids); +} + +/** + * scx_cmask_reframe - Reshape @m's active range without resizing storage + * @m: cmask to reframe + * @base: new active range base + * @nr_cids: new active range length, must fit within @m->alloc_words + * + * Body bits within the new range become garbage - only the head and tail + * words are zeroed to keep the padding invariant. + */ +static inline void scx_cmask_reframe(struct scx_cmask *m, u32 base, u32 nr_cids) +{ + if (WARN_ON_ONCE(SCX_CMASK_NR_WORDS(nr_cids) > m->alloc_words)) + return; + + if (nr_cids) { + u32 last_word = ((base & 63) + nr_cids - 1) / 64; + + m->bits[0] = 0; + m->bits[last_word] = 0; + } + + m->base = base; + m->nr_cids = nr_cids; +} + +static inline void __scx_cmask_set(u32 cid, struct scx_cmask *m) +{ + if (!__scx_cmask_contains(cid, m)) + return; + *__scx_cmask_word(cid, m) |= BIT_U64(cid & 63); +} + +/** + * scx_cmask_test - test whether @cid is set in @m + * @cid: cid to test + * @m: cmask to test + * + * Return %false if @cid is outside @m's active range. Otherwise return the + * bit's value. Read via READ_ONCE so callers can race set/clear writers. + */ +static inline bool scx_cmask_test(u32 cid, const struct scx_cmask *m) +{ + if (!__scx_cmask_contains(cid, m)) + return false; + return READ_ONCE(*__scx_cmask_word(cid, m)) & BIT_U64(cid & 63); +} + +/* + * Words of bits[] the active range spans, 0 if empty. Tighter than the storage + * SCX_CMASK_NR_WORDS() sizes for the worst-case base alignment. + */ +static inline u32 scx_cmask_nr_used_words(const struct scx_cmask *m) +{ + if (!m->nr_cids) + return 0; + return ((m->base & 63) + m->nr_cids - 1) / 64 + 1; +} + +/** + * scx_cmask_for_each_cid - iterate set cids in @m + * @cid: s32 loop var that receives each set cid in turn + * @m: cmask to iterate + * + * Visits set bits within @m's active range in ascending order. Scans only the + * words the active range spans, where head and tail padding is kept zero, so + * no per-cid range check is needed. + */ +#define scx_cmask_for_each_cid(cid, m) \ + for (u64 __bs = (m)->base & ~63u, __wi = 0, \ + __nw = scx_cmask_nr_used_words(m); \ + __wi < __nw; __wi++) \ + for (u64 __w = READ_ONCE((m)->bits[__wi]); \ + __w && ((cid) = __bs + __wi * 64 + __ffs64(__w), true); \ + __w &= __w - 1) + +#endif /* _KERNEL_SCHED_EXT_CID_H */ diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c new file mode 100644 index 000000000000..00fe6cc6d7e2 --- /dev/null +++ b/kernel/sched/ext/ext.c @@ -0,0 +1,10882 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2022 Tejun Heo + * Copyright (c) 2022 David Vernet + */ + +static DEFINE_RAW_SPINLOCK(scx_sched_lock); + +/* + * NOTE: sched_ext is in the process of growing multiple scheduler support and + * scx_root usage is in a transitional state. Naked dereferences are safe if the + * caller is one of the tasks attached to SCX and explicit RCU dereference is + * necessary otherwise. Naked scx_root dereferences trigger sparse warnings but + * are used as temporary markers to indicate that the dereferences need to be + * updated to point to the associated scheduler instances rather than scx_root. + */ +struct scx_sched __rcu *scx_root; + +/* + * All scheds, writers must hold both scx_enable_mutex and scx_sched_lock. + * Readers can hold either or rcu_read_lock(). + */ +static LIST_HEAD(scx_sched_all); + +#ifdef CONFIG_EXT_SUB_SCHED +static const struct rhashtable_params scx_sched_hash_params = { + .key_len = sizeof_field(struct scx_sched, ops.sub_cgroup_id), + .key_offset = offsetof(struct scx_sched, ops.sub_cgroup_id), + .head_offset = offsetof(struct scx_sched, hash_node), + .insecure_elasticity = true, /* inserted under scx_sched_lock */ +}; + +static struct rhashtable scx_sched_hash; +#endif + +/* see SCX_OPS_TID_TO_TASK */ +static const struct rhashtable_params scx_tid_hash_params = { + .key_len = sizeof_field(struct sched_ext_entity, tid), + .key_offset = offsetof(struct sched_ext_entity, tid), + .head_offset = offsetof(struct sched_ext_entity, tid_hash_node), + .insecure_elasticity = true, /* inserted/removed under scx_tasks_lock */ +}; +static struct rhashtable scx_tid_hash; + +/* + * During exit, a task may schedule after losing its PIDs. When disabling the + * BPF scheduler, we need to be able to iterate tasks in every state to + * guarantee system safety. Maintain a dedicated task list which contains every + * task between its fork and eventual free. + */ +static DEFINE_RAW_SPINLOCK(scx_tasks_lock); +static LIST_HEAD(scx_tasks); + +/* ops enable/disable */ +static DEFINE_MUTEX(scx_enable_mutex); +DEFINE_STATIC_KEY_FALSE(__scx_enabled); +DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); +static atomic_t scx_enable_state_var = ATOMIC_INIT(SCX_DISABLED); +static DEFINE_RAW_SPINLOCK(scx_bypass_lock); +static bool scx_init_task_enabled; +static bool scx_switching_all; +DEFINE_STATIC_KEY_FALSE(__scx_switched_all); +static DEFINE_STATIC_KEY_FALSE(__scx_tid_to_task_enabled); + +/* + * True once SCX_OPS_TID_TO_TASK has been negotiated with the root scheduler + * and the tid->task table is live. Wraps the static key so callers don't + * take the address, and hints "likely enabled" for the common case where + * the feature is in use. + */ +static inline bool scx_tid_to_task_enabled(void) +{ + return static_branch_likely(&__scx_tid_to_task_enabled); +} + +static atomic_long_t scx_nr_rejected = ATOMIC_LONG_INIT(0); +static atomic_long_t scx_hotplug_seq = ATOMIC_LONG_INIT(0); + +/* Global cursor for the per-CPU tid allocator. Starts at 1; tid 0 is reserved. */ +static atomic64_t scx_tid_cursor = ATOMIC64_INIT(1); + +#ifdef CONFIG_EXT_SUB_SCHED +/* + * The sub sched being enabled. Used by scx_disable_and_exit_task() to exit + * tasks for the sub-sched being enabled. Use a global variable instead of a + * per-task field as all enables are serialized. + */ +static struct scx_sched *scx_enabling_sub_sched; +#else +#define scx_enabling_sub_sched (struct scx_sched *)NULL +#endif /* CONFIG_EXT_SUB_SCHED */ + +/* + * A monotonically increasing sequence number that is incremented every time a + * scheduler is enabled. This can be used to check if any custom sched_ext + * scheduler has ever been used in the system. + */ +static atomic_long_t scx_enable_seq = ATOMIC_LONG_INIT(0); + +/* + * Watchdog interval. All scx_sched's share a single watchdog timer and the + * interval is half of the shortest sch->watchdog_timeout. + */ +static unsigned long scx_watchdog_interval; + +/* + * The last time the delayed work was run. This delayed work relies on + * ksoftirqd being able to run to service timer interrupts, so it's possible + * that this work itself could get wedged. To account for this, we check that + * it's not stalled in the timer tick, and trigger an error if it is. + */ +static unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; + +static struct delayed_work scx_watchdog_work; + +/* + * For %SCX_KICK_WAIT: Each CPU has a pointer to an array of kick_sync sequence + * numbers. The arrays are allocated with kvzalloc() as size can exceed percpu + * allocator limits on large machines. O(nr_cpu_ids^2) allocation, allocated + * lazily when enabling and freed when disabling to avoid waste when sched_ext + * isn't active. + */ +struct scx_kick_syncs { + struct rcu_head rcu; + unsigned long syncs[]; +}; + +static DEFINE_PER_CPU(struct scx_kick_syncs __rcu *, scx_kick_syncs); + +/* + * Per-CPU buffered allocator state for p->scx.tid. Each CPU pulls a chunk of + * SCX_TID_CHUNK ids from scx_tid_cursor and hands them out locally without + * further synchronization. See scx_alloc_tid(). + */ +struct scx_tid_alloc { + u64 next; + u64 end; +}; +static DEFINE_PER_CPU(struct scx_tid_alloc, scx_tid_alloc); + +/* + * Direct dispatch marker. + * + * Non-NULL values are used for direct dispatch from enqueue path. A valid + * pointer points to the task currently being enqueued. An ERR_PTR value is used + * to indicate that direct dispatch has already happened. + */ +static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); + +static const struct rhashtable_params dsq_hash_params = { + .key_len = sizeof_field(struct scx_dispatch_q, id), + .key_offset = offsetof(struct scx_dispatch_q, id), + .head_offset = offsetof(struct scx_dispatch_q, hash_node), +}; + +static LLIST_HEAD(dsqs_to_free); + +/* string formatting from BPF */ +struct scx_bstr_buf { + u64 data[MAX_BPRINTF_VARARGS]; + char line[SCX_EXIT_MSG_LEN]; +}; + +static DEFINE_RAW_SPINLOCK(scx_exit_bstr_buf_lock); +static struct scx_bstr_buf scx_exit_bstr_buf; + +/* ops debug dump */ +static DEFINE_RAW_SPINLOCK(scx_dump_lock); + +struct scx_dump_data { + s32 cpu; + bool first; + s32 cursor; + struct seq_buf *s; + const char *prefix; + struct scx_bstr_buf buf; +}; + +static struct scx_dump_data scx_dump_data = { + .cpu = -1, +}; + +/* /sys/kernel/sched_ext interface */ +static struct kset *scx_kset; + +/* + * Parameters that can be adjusted through /sys/module/sched_ext/parameters. + * There usually is no reason to modify these as normal scheduler operation + * shouldn't be affected by them. The knobs are primarily for debugging. + */ +static unsigned int scx_slice_bypass_us = SCX_SLICE_BYPASS / NSEC_PER_USEC; +static unsigned int scx_bypass_lb_intv_us = SCX_BYPASS_LB_DFL_INTV_US; + +static int set_slice_us(const char *val, const struct kernel_param *kp) +{ + return param_set_uint_minmax(val, kp, 100, 100 * USEC_PER_MSEC); +} + +static const struct kernel_param_ops slice_us_param_ops = { + .set = set_slice_us, + .get = param_get_uint, +}; + +static int set_bypass_lb_intv_us(const char *val, const struct kernel_param *kp) +{ + return param_set_uint_minmax(val, kp, 0, 10 * USEC_PER_SEC); +} + +static const struct kernel_param_ops bypass_lb_intv_us_param_ops = { + .set = set_bypass_lb_intv_us, + .get = param_get_uint, +}; + +#undef MODULE_PARAM_PREFIX +#define MODULE_PARAM_PREFIX "sched_ext." + +module_param_cb(slice_bypass_us, &slice_us_param_ops, &scx_slice_bypass_us, 0600); +MODULE_PARM_DESC(slice_bypass_us, "bypass slice in microseconds, applied on [un]load (100us to 100ms)"); +module_param_cb(bypass_lb_intv_us, &bypass_lb_intv_us_param_ops, &scx_bypass_lb_intv_us, 0600); +MODULE_PARM_DESC(bypass_lb_intv_us, "bypass load balance interval in microseconds (0 (disable) to 10s)"); + +#undef MODULE_PARAM_PREFIX + +#define CREATE_TRACE_POINTS +#include + +static void run_deferred(struct rq *rq); +static bool task_dead_and_done(struct task_struct *p); +static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); +static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind); + +__printf(5, 6) bool __scx_exit(struct scx_sched *sch, + enum scx_exit_kind kind, s64 exit_code, + s32 exit_cpu, const char *fmt, ...) +{ + va_list args; + bool ret; + + va_start(args, fmt); + ret = scx_vexit(sch, kind, exit_code, exit_cpu, fmt, args); + va_end(args); + + return ret; +} + +#define SCX_HAS_OP(sch, op) test_bit(SCX_OP_IDX(op), (sch)->has_op) + +static long jiffies_delta_msecs(unsigned long at, unsigned long now) +{ + if (time_after(at, now)) + return jiffies_to_msecs(at - now); + else + return -(long)jiffies_to_msecs(now - at); +} + +static bool u32_before(u32 a, u32 b) +{ + return (s32)(a - b) < 0; +} + +#ifdef CONFIG_EXT_SUB_SCHED +/** + * scx_parent - Find the parent sched + * @sch: sched to find the parent of + * + * Returns the parent scheduler or %NULL if @sch is root. + */ +static struct scx_sched *scx_parent(struct scx_sched *sch) +{ + if (sch->level) + return sch->ancestors[sch->level - 1]; + else + return NULL; +} + +/** + * scx_next_descendant_pre - find the next descendant for pre-order walk + * @pos: the current position (%NULL to initiate traversal) + * @root: sched whose descendants to walk + * + * To be used by scx_for_each_descendant_pre(). Find the next descendant to + * visit for pre-order traversal of @root's descendants. @root is included in + * the iteration and the first node to be visited. + */ +static struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, + struct scx_sched *root) +{ + struct scx_sched *next; + + lockdep_assert(lockdep_is_held(&scx_enable_mutex) || + lockdep_is_held(&scx_sched_lock)); + + /* if first iteration, visit @root */ + if (!pos) + return root; + + /* visit the first child if exists */ + next = list_first_entry_or_null(&pos->children, struct scx_sched, sibling); + if (next) + return next; + + /* no child, visit my or the closest ancestor's next sibling */ + while (pos != root) { + if (!list_is_last(&pos->sibling, &scx_parent(pos)->children)) + return list_next_entry(pos, sibling); + pos = scx_parent(pos); + } + + return NULL; +} + +static struct scx_sched *scx_find_sub_sched(u64 cgroup_id) +{ + return rhashtable_lookup(&scx_sched_hash, &cgroup_id, + scx_sched_hash_params); +} + +static void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) +{ + rcu_assign_pointer(p->scx.sched, sch); +} +#else /* CONFIG_EXT_SUB_SCHED */ +static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; } +static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; } +static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {} +#endif /* CONFIG_EXT_SUB_SCHED */ + +/** + * scx_is_descendant - Test whether sched is a descendant + * @sch: sched to test + * @ancestor: ancestor sched to test against + * + * Test whether @sch is a descendant of @ancestor. + */ +static bool scx_is_descendant(struct scx_sched *sch, struct scx_sched *ancestor) +{ + if (sch->level < ancestor->level) + return false; + return sch->ancestors[ancestor->level] == ancestor; +} + +/** + * scx_for_each_descendant_pre - pre-order walk of a sched's descendants + * @pos: iteration cursor + * @root: sched to walk the descendants of + * + * Walk @root's descendants. @root is included in the iteration and the first + * node to be visited. Must be called with either scx_enable_mutex or + * scx_sched_lock held. + */ +#define scx_for_each_descendant_pre(pos, root) \ + for ((pos) = scx_next_descendant_pre(NULL, (root)); (pos); \ + (pos) = scx_next_descendant_pre((pos), (root))) + +static struct scx_dispatch_q *find_global_dsq(struct scx_sched *sch, s32 cpu) +{ + return &sch->pnode[cpu_to_node(cpu)]->global_dsq; +} + +static struct scx_dispatch_q *find_user_dsq(struct scx_sched *sch, u64 dsq_id) +{ + return rhashtable_lookup(&sch->dsq_hash, &dsq_id, dsq_hash_params); +} + +static const struct sched_class *scx_setscheduler_class(struct task_struct *p) +{ + if (p->sched_class == &stop_sched_class) + return &stop_sched_class; + + return __setscheduler_class(p->policy, p->prio); +} + +static struct scx_dispatch_q *bypass_dsq(struct scx_sched *sch, s32 cpu) +{ + return &per_cpu_ptr(sch->pcpu, cpu)->bypass_dsq; +} + +static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 cpu) +{ +#ifdef CONFIG_EXT_SUB_SCHED + /* + * If @sch is a sub-sched which is bypassing, its tasks should go into + * the bypass DSQs of the nearest ancestor which is not bypassing. The + * not-bypassing ancestor is responsible for scheduling all tasks from + * bypassing sub-trees. If all ancestors including root are bypassing, + * all tasks should go to the root's bypass DSQs. + * + * Whenever a sched starts bypassing, all runnable tasks in its subtree + * are re-enqueued after scx_bypassing() is turned on, guaranteeing that + * all tasks are transferred to the right DSQs. + */ + while (scx_parent(sch) && scx_bypassing(sch, cpu)) + sch = scx_parent(sch); +#endif /* CONFIG_EXT_SUB_SCHED */ + + return bypass_dsq(sch, cpu); +} + +/** + * bypass_dsp_enabled - Check if bypass dispatch path is enabled + * @sch: scheduler to check + * + * When a descendant scheduler enters bypass mode, bypassed tasks are scheduled + * by the nearest non-bypassing ancestor, or the root scheduler if all ancestors + * are bypassing. In the former case, the ancestor is not itself bypassing but + * its bypass DSQs will be populated with bypassed tasks from descendants. Thus, + * the ancestor's bypass dispatch path must be active even though its own + * bypass_depth remains zero. + * + * This function checks bypass_dsp_enable_depth which is managed separately from + * bypass_depth to enable this decoupling. See enable_bypass_dsp() and + * disable_bypass_dsp(). + */ +static bool bypass_dsp_enabled(struct scx_sched *sch) +{ + return unlikely(atomic_read(&sch->bypass_dsp_enable_depth)); +} + +/** + * rq_is_open - Is the rq available for immediate execution of an SCX task? + * @rq: rq to test + * @enq_flags: optional %SCX_ENQ_* of the task being enqueued + * + * Returns %true if @rq is currently open for executing an SCX task. After a + * %false return, @rq is guaranteed to invoke SCX dispatch path at least once + * before going to idle and not inserting a task into @rq's local DSQ after a + * %false return doesn't cause @rq to stall. + */ +static bool rq_is_open(struct rq *rq, u64 enq_flags) +{ + lockdep_assert_rq_held(rq); + + /* + * A higher-priority class task is either running or in the process of + * waking up on @rq. + */ + if (sched_class_above(rq->next_class, &ext_sched_class)) + return false; + + /* + * @rq is either in transition to or in idle and there is no + * higher-priority class task waking up on it. + */ + if (sched_class_above(&ext_sched_class, rq->next_class)) + return true; + + /* + * @rq is either picking, in transition to, or running an SCX task. + */ + + /* + * If we're in the dispatch path holding rq lock, $curr may or may not + * be ready depending on whether the on-going dispatch decides to extend + * $curr's slice. We say yes here and resolve it at the end of dispatch. + * See balance_one(). + */ + if (rq->scx.flags & SCX_RQ_IN_BALANCE) + return true; + + /* + * %SCX_ENQ_PREEMPT clears $curr's slice if on SCX and kicks dispatch, + * so allow it to avoid spuriously triggering reenq on a combined + * PREEMPT|IMMED insertion. + */ + if (enq_flags & SCX_ENQ_PREEMPT) + return true; + + /* + * @rq is either in transition to or running an SCX task and can't go + * idle without another SCX dispatch cycle. + */ + return false; +} + +/* + * Track the rq currently locked. + * + * This allows kfuncs to safely operate on rq from any scx ops callback, + * knowing which rq is already locked. + */ +DEFINE_PER_CPU(struct rq *, scx_locked_rq_state); + +static inline void update_locked_rq(struct rq *rq) +{ + /* + * Check whether @rq is actually locked. This can help expose bugs + * or incorrect assumptions about the context in which a kfunc or + * callback is executed. + */ + if (rq) + lockdep_assert_rq_held(rq); + __this_cpu_write(scx_locked_rq_state, rq); +} + +/* + * SCX ops can recurse via scx_bpf_sub_dispatch() - the inner call must not + * clobber the outer's scx_locked_rq_state. Save it on entry, restore on exit. + */ +#define SCX_CALL_OP(sch, op, locked_rq, args...) \ +do { \ + struct rq *__prev_locked_rq; \ + \ + if (locked_rq) { \ + __prev_locked_rq = scx_locked_rq(); \ + update_locked_rq(locked_rq); \ + } \ + (sch)->ops.op(args); \ + if (locked_rq) \ + update_locked_rq(__prev_locked_rq); \ +} while (0) + +/* + * Flipped on enable per sch->is_cid_type. Declared in internal.h so + * subsystem inlines can read it. + */ +DEFINE_STATIC_KEY_FALSE(__scx_is_cid_type); + +/* + * scx_cpu_arg() wraps a cpu arg being handed to an SCX op. For cid-form + * schedulers it resolves to the matching cid; for cpu-form it passes @cpu + * through. scx_cpu_ret() is the inverse for a cpu/cid returned from an op + * (currently only ops.select_cpu); it validates the BPF-supplied cid and + * triggers scx_error() on @sch if invalid. + */ +static s32 scx_cpu_arg(s32 cpu) +{ + if (scx_is_cid_type()) + return __scx_cpu_to_cid(cpu); + return cpu; +} + +static s32 scx_cpu_ret(struct scx_sched *sch, s32 cpu_or_cid) +{ + if (cpu_or_cid < 0 || !scx_is_cid_type()) + return cpu_or_cid; + return scx_cid_to_cpu(sch, cpu_or_cid); +} + +#define SCX_CALL_OP_RET(sch, op, locked_rq, args...) \ +({ \ + struct rq *__prev_locked_rq; \ + __typeof__((sch)->ops.op(args)) __ret; \ + \ + if (locked_rq) { \ + __prev_locked_rq = scx_locked_rq(); \ + update_locked_rq(locked_rq); \ + } \ + __ret = (sch)->ops.op(args); \ + if (locked_rq) \ + update_locked_rq(__prev_locked_rq); \ + __ret; \ +}) + +/* + * SCX_CALL_OP_TASK*() invokes an SCX op that takes one or two task arguments + * and records them in current->scx.kf_tasks[] for the duration of the call. A + * kfunc invoked from inside such an op can then use + * scx_kf_arg_task_ok() to verify that its task argument is one of + * those subject tasks. + * + * Every SCX_CALL_OP_TASK*() call site invokes its op with @p's rq lock held - + * either via the @locked_rq argument here, or (for ops.select_cpu()) via @p's + * pi_lock held by try_to_wake_up() with rq tracking via scx_rq.in_select_cpu. + * So if kf_tasks[] is set, @p's scheduler-protected fields are stable. + * + * kf_tasks[] can not stack, so task-based SCX ops must not nest. The + * WARN_ON_ONCE() in each macro catches a re-entry of any of the three variants + * while a previous one is still in progress. + */ +#define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...) \ +do { \ + WARN_ON_ONCE(current->scx.kf_tasks[0]); \ + current->scx.kf_tasks[0] = task; \ + SCX_CALL_OP((sch), op, locked_rq, task, ##args); \ + current->scx.kf_tasks[0] = NULL; \ +} while (0) + +#define SCX_CALL_OP_TASK_RET(sch, op, locked_rq, task, args...) \ +({ \ + __typeof__((sch)->ops.op(task, ##args)) __ret; \ + WARN_ON_ONCE(current->scx.kf_tasks[0]); \ + current->scx.kf_tasks[0] = task; \ + __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task, ##args); \ + current->scx.kf_tasks[0] = NULL; \ + __ret; \ +}) + +#define SCX_CALL_OP_2TASKS_RET(sch, op, locked_rq, task0, task1, args...) \ +({ \ + __typeof__((sch)->ops.op(task0, task1, ##args)) __ret; \ + WARN_ON_ONCE(current->scx.kf_tasks[0]); \ + current->scx.kf_tasks[0] = task0; \ + current->scx.kf_tasks[1] = task1; \ + __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task0, task1, ##args); \ + current->scx.kf_tasks[0] = NULL; \ + current->scx.kf_tasks[1] = NULL; \ + __ret; \ +}) + +/** + * scx_call_op_set_cpumask - invoke ops.set_cpumask / ops_cid.set_cmask for @task + * @sch: scx_sched being invoked + * @rq: rq to update as the currently-locked rq, or NULL + * @task: task whose affinity is changing + * @cpumask: new cpumask + * + * For cid-form schedulers, translate @cpumask to a cmask via the per-cpu + * scratch in cid.c and dispatch through the ops_cid union view. Caller + * must hold @rq's rq lock so this_cpu_ptr is stable across the call. + */ +static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, + struct task_struct *task, + const struct cpumask *cpumask) +{ + WARN_ON_ONCE(current->scx.kf_tasks[0]); + current->scx.kf_tasks[0] = task; + if (rq) + update_locked_rq(rq); + + if (scx_is_cid_type()) { + struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch); + /* + * Build the per-CPU arena cmask and hand BPF its arena address. + * Caller holds the rq lock with IRQs disabled, which makes us + * the sole user of the scratch area. + */ + scx_cpumask_to_cmask(cpumask, kern_va); + sch->ops_cid.set_cmask(task, scx_kaddr_to_arena(sch, kern_va)); + } else { + sch->ops.set_cpumask(task, cpumask); + } + + if (rq) + update_locked_rq(NULL); + current->scx.kf_tasks[0] = NULL; +} + +/* see SCX_CALL_OP_TASK() */ +static __always_inline bool scx_kf_arg_task_ok(struct scx_sched *sch, + struct task_struct *p) +{ + if (unlikely((p != current->scx.kf_tasks[0] && + p != current->scx.kf_tasks[1]))) { + scx_error(sch, "called on a task not being operated on"); + return false; + } + + return true; +} + +enum scx_dsq_iter_flags { + /* iterate in the reverse dispatch order */ + SCX_DSQ_ITER_REV = 1U << 16, + + __SCX_DSQ_ITER_HAS_SLICE = 1U << 30, + __SCX_DSQ_ITER_HAS_VTIME = 1U << 31, + + __SCX_DSQ_ITER_USER_FLAGS = SCX_DSQ_ITER_REV, + __SCX_DSQ_ITER_ALL_FLAGS = __SCX_DSQ_ITER_USER_FLAGS | + __SCX_DSQ_ITER_HAS_SLICE | + __SCX_DSQ_ITER_HAS_VTIME, +}; + +/** + * nldsq_next_task - Iterate to the next task in a non-local DSQ + * @dsq: non-local dsq being iterated + * @cur: current position, %NULL to start iteration + * @rev: walk backwards + * + * Returns %NULL when iteration is finished. + */ +static struct task_struct *nldsq_next_task(struct scx_dispatch_q *dsq, + struct task_struct *cur, bool rev) +{ + struct list_head *list_node; + struct scx_dsq_list_node *dsq_lnode; + + lockdep_assert_held(&dsq->lock); + + if (cur) + list_node = &cur->scx.dsq_list.node; + else + list_node = &dsq->list; + + /* find the next task, need to skip BPF iteration cursors */ + do { + if (rev) + list_node = list_node->prev; + else + list_node = list_node->next; + + if (list_node == &dsq->list) + return NULL; + + dsq_lnode = container_of(list_node, struct scx_dsq_list_node, + node); + } while (dsq_lnode->flags & SCX_DSQ_LNODE_ITER_CURSOR); + + return container_of(dsq_lnode, struct task_struct, scx.dsq_list); +} + +#define nldsq_for_each_task(p, dsq) \ + for ((p) = nldsq_next_task((dsq), NULL, false); (p); \ + (p) = nldsq_next_task((dsq), (p), false)) + +/** + * nldsq_cursor_next_task - Iterate to the next task given a cursor in a non-local DSQ + * @cursor: scx_dsq_list_node initialized with INIT_DSQ_LIST_CURSOR() + * @dsq: non-local dsq being iterated + * + * Find the next task in a cursor based iteration. The caller must have + * initialized @cursor using INIT_DSQ_LIST_CURSOR() and can release the DSQ lock + * between the iteration steps. + * + * Only tasks which were queued before @cursor was initialized are visible. This + * bounds the iteration and guarantees that vtime never jumps in the other + * direction while iterating. + */ +static struct task_struct *nldsq_cursor_next_task(struct scx_dsq_list_node *cursor, + struct scx_dispatch_q *dsq) +{ + bool rev = cursor->flags & SCX_DSQ_ITER_REV; + struct task_struct *p; + + lockdep_assert_held(&dsq->lock); + BUG_ON(!(cursor->flags & SCX_DSQ_LNODE_ITER_CURSOR)); + + if (list_empty(&cursor->node)) + p = NULL; + else + p = container_of(cursor, struct task_struct, scx.dsq_list); + + /* skip cursors and tasks that were queued after @cursor init */ + do { + p = nldsq_next_task(dsq, p, rev); + } while (p && unlikely(u32_before(cursor->priv, p->scx.dsq_seq))); + + if (p) { + if (rev) + list_move_tail(&cursor->node, &p->scx.dsq_list.node); + else + list_move(&cursor->node, &p->scx.dsq_list.node); + } else { + list_del_init(&cursor->node); + } + + return p; +} + +/** + * nldsq_cursor_lost_task - Test whether someone else took the task since iteration + * @cursor: scx_dsq_list_node initialized with INIT_DSQ_LIST_CURSOR() + * @rq: rq @p was on + * @dsq: dsq @p was on + * @p: target task + * + * @p is a task returned by nldsq_cursor_next_task(). The locks may have been + * dropped and re-acquired inbetween. Verify that no one else took or is in the + * process of taking @p from @dsq. + * + * On %false return, the caller can assume full ownership of @p. + */ +static bool nldsq_cursor_lost_task(struct scx_dsq_list_node *cursor, + struct rq *rq, struct scx_dispatch_q *dsq, + struct task_struct *p) +{ + lockdep_assert_rq_held(rq); + lockdep_assert_held(&dsq->lock); + + /* + * @p could have already left $src_dsq, got re-enqueud, or be in the + * process of being consumed by someone else. + */ + if (unlikely(p->scx.dsq != dsq || + u32_before(cursor->priv, p->scx.dsq_seq) || + p->scx.holding_cpu >= 0)) + return true; + + /* if @p has stayed on @dsq, its rq couldn't have changed */ + if (WARN_ON_ONCE(rq != task_rq(p))) + return true; + + return false; +} + +/* + * BPF DSQ iterator. Tasks in a non-local DSQ can be iterated in [reverse] + * dispatch order. BPF-visible iterator is opaque and larger to allow future + * changes without breaking backward compatibility. Can be used with + * bpf_for_each(). See bpf_iter_scx_dsq_*(). + */ +struct bpf_iter_scx_dsq_kern { + struct scx_dsq_list_node cursor; + struct scx_dispatch_q *dsq; + u64 slice; + u64 vtime; +} __attribute__((aligned(8))); + +struct bpf_iter_scx_dsq { + u64 __opaque[6]; +} __attribute__((aligned(8))); + + +static u32 scx_get_task_state(const struct task_struct *p) +{ + return p->scx.flags & SCX_TASK_STATE_MASK; +} + +static void scx_set_task_state(struct task_struct *p, u32 state) +{ + u32 prev_state = scx_get_task_state(p); + bool warn = false; + + switch (state) { + case SCX_TASK_NONE: + warn = prev_state == SCX_TASK_DEAD; + break; + case SCX_TASK_INIT_BEGIN: + warn = prev_state != SCX_TASK_NONE; + break; + case SCX_TASK_INIT: + warn = prev_state != SCX_TASK_INIT_BEGIN; + p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; + break; + case SCX_TASK_READY: + warn = !(prev_state == SCX_TASK_INIT || + prev_state == SCX_TASK_ENABLED); + break; + case SCX_TASK_ENABLED: + warn = prev_state != SCX_TASK_READY; + break; + case SCX_TASK_DEAD: + warn = !(prev_state == SCX_TASK_NONE || + prev_state == SCX_TASK_INIT_BEGIN); + break; + default: + WARN_ONCE(1, "sched_ext: Invalid task state %d -> %d for %s[%d]", + prev_state, state, p->comm, p->pid); + return; + } + + WARN_ONCE(warn, "sched_ext: Invalid task state transition 0x%x -> 0x%x for %s[%d]", + prev_state, state, p->comm, p->pid); + + p->scx.flags &= ~SCX_TASK_STATE_MASK; + p->scx.flags |= state; +} + +/* + * SCX task iterator. + */ +struct scx_task_iter { + struct sched_ext_entity cursor; + struct task_struct *locked_task; + struct rq *rq; + struct rq_flags rf; + u32 cnt; + bool list_locked; +#ifdef CONFIG_EXT_SUB_SCHED + struct cgroup *cgrp; + struct cgroup_subsys_state *css_pos; + struct css_task_iter css_iter; +#endif +}; + +/** + * scx_task_iter_start - Lock scx_tasks_lock and start a task iteration + * @iter: iterator to init + * @cgrp: Optional root of cgroup subhierarchy to iterate + * + * Initialize @iter. Once initialized, @iter must eventually be stopped with + * scx_task_iter_stop(). + * + * If @cgrp is %NULL, scx_tasks is used for iteration and this function returns + * with scx_tasks_lock held and @iter->cursor inserted into scx_tasks. + * + * If @cgrp is not %NULL, @cgrp and its descendants' tasks are walked using + * @iter->css_iter. The caller must be holding cgroup_lock() to prevent cgroup + * task migrations. + * + * The two modes of iterations are largely independent and it's likely that + * scx_tasks can be removed in favor of always using cgroup iteration if + * CONFIG_SCHED_CLASS_EXT depends on CONFIG_CGROUPS. + * + * scx_tasks_lock and the rq lock may be released using scx_task_iter_unlock() + * between this and the first next() call or between any two next() calls. If + * the locks are released between two next() calls, the caller is responsible + * for ensuring that the task being iterated remains accessible either through + * RCU read lock or obtaining a reference count. + * + * All tasks which existed when the iteration started are guaranteed to be + * visited as long as they are not dead. + */ +static void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp) +{ + memset(iter, 0, sizeof(*iter)); + +#ifdef CONFIG_EXT_SUB_SCHED + if (cgrp) { + lockdep_assert_held(&cgroup_mutex); + iter->cgrp = cgrp; + iter->css_pos = css_next_descendant_pre(NULL, &iter->cgrp->self); + css_task_iter_start(iter->css_pos, CSS_TASK_ITER_WITH_DEAD, + &iter->css_iter); + return; + } +#endif + raw_spin_lock_irq(&scx_tasks_lock); + + iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; + list_add(&iter->cursor.tasks_node, &scx_tasks); + iter->list_locked = true; +} + +static void __scx_task_iter_rq_unlock(struct scx_task_iter *iter) +{ + if (iter->locked_task) { + __balance_callbacks(iter->rq, &iter->rf); + task_rq_unlock(iter->rq, iter->locked_task, &iter->rf); + iter->locked_task = NULL; + } +} + +/** + * scx_task_iter_unlock - Unlock rq and scx_tasks_lock held by a task iterator + * @iter: iterator to unlock + * + * If @iter is in the middle of a locked iteration, it may be locking the rq of + * the task currently being visited in addition to scx_tasks_lock. Unlock both. + * This function can be safely called anytime during an iteration. The next + * iterator operation will automatically restore the necessary locking. + */ +static void scx_task_iter_unlock(struct scx_task_iter *iter) +{ + __scx_task_iter_rq_unlock(iter); + if (iter->list_locked) { + iter->list_locked = false; + raw_spin_unlock_irq(&scx_tasks_lock); + } +} + +static void __scx_task_iter_maybe_relock(struct scx_task_iter *iter) +{ + if (!iter->list_locked) { + raw_spin_lock_irq(&scx_tasks_lock); + iter->list_locked = true; + } +} + +/** + * scx_task_iter_relock - Re-acquire scx_tasks_lock and, optionally, @p's rq + * @iter: iterator to relock + * @p: task whose rq to lock, or %NULL for scx_tasks_lock only + * + * Counterpart to scx_task_iter_unlock(). Locking @p's rq is optional. Once + * re-acquired, both locks are managed by the iterator from here on. + */ +static void scx_task_iter_relock(struct scx_task_iter *iter, + struct task_struct *p) +{ + __scx_task_iter_maybe_relock(iter); + if (p) { + iter->rq = task_rq_lock(p, &iter->rf); + iter->locked_task = p; + } +} + +/** + * scx_task_iter_stop - Stop a task iteration and unlock scx_tasks_lock + * @iter: iterator to exit + * + * Exit a previously initialized @iter. Must be called with scx_tasks_lock held + * which is released on return. If the iterator holds a task's rq lock, that rq + * lock is also released. See scx_task_iter_start() for details. + */ +static void scx_task_iter_stop(struct scx_task_iter *iter) +{ +#ifdef CONFIG_EXT_SUB_SCHED + if (iter->cgrp) { + if (iter->css_pos) + css_task_iter_end(&iter->css_iter); + __scx_task_iter_rq_unlock(iter); + return; + } +#endif + __scx_task_iter_maybe_relock(iter); + list_del_init(&iter->cursor.tasks_node); + scx_task_iter_unlock(iter); +} + +/** + * scx_task_iter_next - Next task + * @iter: iterator to walk + * + * Visit the next task. See scx_task_iter_start() for details. Locks are dropped + * and re-acquired every %SCX_TASK_ITER_BATCH iterations to avoid causing stalls + * by holding scx_tasks_lock for too long. + */ +static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) +{ + struct list_head *cursor = &iter->cursor.tasks_node; + struct sched_ext_entity *pos; + + if (!(++iter->cnt % SCX_TASK_ITER_BATCH)) { + scx_task_iter_unlock(iter); + cond_resched(); + } + +#ifdef CONFIG_EXT_SUB_SCHED + if (iter->cgrp) { + while (iter->css_pos) { + struct task_struct *p; + + p = css_task_iter_next(&iter->css_iter); + if (p) + return p; + + css_task_iter_end(&iter->css_iter); + iter->css_pos = css_next_descendant_pre(iter->css_pos, + &iter->cgrp->self); + if (iter->css_pos) + css_task_iter_start(iter->css_pos, CSS_TASK_ITER_WITH_DEAD, + &iter->css_iter); + } + return NULL; + } +#endif + __scx_task_iter_maybe_relock(iter); + + list_for_each_entry(pos, cursor, tasks_node) { + if (&pos->tasks_node == &scx_tasks) + return NULL; + if (!(pos->flags & SCX_TASK_CURSOR)) { + list_move(cursor, &pos->tasks_node); + return container_of(pos, struct task_struct, scx); + } + } + + /* can't happen, should always terminate at scx_tasks above */ + BUG(); +} + +/** + * scx_task_iter_next_locked - Next non-idle task with its rq locked + * @iter: iterator to walk + * + * Visit the non-idle task with its rq lock held. Allows callers to specify + * whether they would like to filter out dead tasks. See scx_task_iter_start() + * for details. + */ +static struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter) +{ + struct task_struct *p; + + __scx_task_iter_rq_unlock(iter); + + while ((p = scx_task_iter_next(iter))) { + /* + * scx_task_iter is used to prepare and move tasks into SCX + * while loading the BPF scheduler and vice-versa while + * unloading. The init_tasks ("swappers") should be excluded + * from the iteration because: + * + * - It's unsafe to use __setschduler_prio() on an init_task to + * determine the sched_class to use as it won't preserve its + * idle_sched_class. + * + * - ops.init/exit_task() can easily be confused if called with + * init_tasks as they, e.g., share PID 0. + * + * As init_tasks are never scheduled through SCX, they can be + * skipped safely. Note that is_idle_task() which tests %PF_IDLE + * doesn't work here: + * + * - %PF_IDLE may not be set for an init_task whose CPU hasn't + * yet been onlined. + * + * - %PF_IDLE can be set on tasks that are not init_tasks. See + * play_idle_precise() used by CONFIG_IDLE_INJECT. + * + * Test for idle_sched_class as only init_tasks are on it. + */ + if (p->sched_class == &idle_sched_class) + continue; + + iter->rq = task_rq_lock(p, &iter->rf); + iter->locked_task = p; + + /* + * cgroup_task_dead() removes the dead tasks from cset->tasks + * after sched_ext_dead() and cgroup iteration may see tasks + * which already finished sched_ext_dead(). %SCX_TASK_DEAD is + * set by sched_ext_dead() under @p's rq lock. Test it to + * avoid visiting tasks which are already dead from SCX POV. + */ + if (scx_get_task_state(p) == SCX_TASK_DEAD) { + __scx_task_iter_rq_unlock(iter); + continue; + } + + return p; + } + return NULL; +} + +/** + * scx_add_event - Increase an event counter for 'name' by 'cnt' + * @sch: scx_sched to account events for + * @name: an event name defined in struct scx_event_stats + * @cnt: the number of the event occurred + * + * This can be used when preemption is not disabled. + */ +#define scx_add_event(sch, name, cnt) do { \ + this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \ + trace_sched_ext_event(#name, (cnt)); \ +} while(0) + +/** + * __scx_add_event - Increase an event counter for 'name' by 'cnt' + * @sch: scx_sched to account events for + * @name: an event name defined in struct scx_event_stats + * @cnt: the number of the event occurred + * + * This should be used only when preemption is disabled. + */ +#define __scx_add_event(sch, name, cnt) do { \ + __this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \ + trace_sched_ext_event(#name, cnt); \ +} while(0) + +/** + * scx_agg_event - Aggregate an event counter 'kind' from 'src_e' to 'dst_e' + * @dst_e: destination event stats + * @src_e: source event stats + * @kind: a kind of event to be aggregated + */ +#define scx_agg_event(dst_e, src_e, kind) do { \ + (dst_e)->kind += READ_ONCE((src_e)->kind); \ +} while(0) + +/** + * scx_dump_event - Dump an event 'kind' in 'events' to 's' + * @s: output seq_buf + * @events: event stats + * @kind: a kind of event to dump + */ +#define scx_dump_event(s, events, kind) do { \ + dump_line(&(s), "%40s: %16lld", #kind, (events)->kind); \ +} while (0) + + +static void scx_read_events(struct scx_sched *sch, + struct scx_event_stats *events); + +static enum scx_enable_state scx_enable_state(void) +{ + return atomic_read(&scx_enable_state_var); +} + +static enum scx_enable_state scx_set_enable_state(enum scx_enable_state to) +{ + return atomic_xchg(&scx_enable_state_var, to); +} + +static bool scx_tryset_enable_state(enum scx_enable_state to, + enum scx_enable_state from) +{ + int from_v = from; + + return atomic_try_cmpxchg(&scx_enable_state_var, &from_v, to); +} + +/** + * wait_ops_state - Busy-wait the specified ops state to end + * @p: target task + * @opss: state to wait the end of + * + * Busy-wait for @p to transition out of @opss. This can only be used when the + * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also + * has load_acquire semantics to ensure that the caller can see the updates made + * in the enqueueing and dispatching paths. + */ +static void wait_ops_state(struct task_struct *p, unsigned long opss) +{ + do { + cpu_relax(); + } while (atomic_long_read_acquire(&p->scx.ops_state) == opss); +} + +static inline bool __cpu_valid(s32 cpu) +{ + return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu)); +} + +/** + * scx_cpu_valid - Verify a cpu number, to be used on ops input args + * @sch: scx_sched to abort on error + * @cpu: cpu number which came from a BPF ops + * @where: extra information reported on error + * + * @cpu is a cpu number which came from the BPF scheduler and can be any value. + * Verify that it is in range and one of the possible cpus. If invalid, trigger + * an ops error. + */ +bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where) +{ + if (__cpu_valid(cpu)) { + return true; + } else { + scx_error(sch, "invalid CPU %d%s%s", cpu, where ? " " : "", where ?: ""); + return false; + } +} + +/** + * ops_sanitize_err - Sanitize a -errno value + * @sch: scx_sched to error out on error + * @ops_name: operation to blame on failure + * @err: -errno value to sanitize + * + * Verify @err is a valid -errno. If not, trigger scx_error() and return + * -%EPROTO. This is necessary because returning a rogue -errno up the chain can + * cause misbehaviors. For an example, a large negative return from + * ops.init_task() triggers an oops when passed up the call chain because the + * value fails IS_ERR() test after being encoded with ERR_PTR() and then is + * handled as a pointer. + */ +static int ops_sanitize_err(struct scx_sched *sch, const char *ops_name, s32 err) +{ + if (err < 0 && err >= -MAX_ERRNO) + return err; + + scx_error(sch, "ops.%s() returned an invalid errno %d", ops_name, err); + return -EPROTO; +} + +static void deferred_bal_cb_workfn(struct rq *rq) +{ + run_deferred(rq); +} + +static void deferred_irq_workfn(struct irq_work *irq_work) +{ + struct rq *rq = container_of(irq_work, struct rq, scx.deferred_irq_work); + + raw_spin_rq_lock(rq); + run_deferred(rq); + raw_spin_rq_unlock(rq); +} + +/** + * schedule_deferred - Schedule execution of deferred actions on an rq + * @rq: target rq + * + * Schedule execution of deferred actions on @rq. Deferred actions are executed + * with @rq locked but unpinned, and thus can unlock @rq to e.g. migrate tasks + * to other rqs. + */ +static void schedule_deferred(struct rq *rq) +{ + /* + * This is the fallback when schedule_deferred_locked() can't use + * the cheaper balance callback or wakeup hook paths (the target + * CPU is not in balance or wakeup). Currently, this is primarily + * hit by reenqueue operations targeting a remote CPU. + * + * Queue on the target CPU. The deferred work can run from any CPU + * correctly - the _locked() path already processes remote rqs from + * the calling CPU - but targeting the owning CPU allows IPI delivery + * without waiting for the calling CPU to re-enable IRQs and is + * cheaper as the reenqueue runs locally. + */ + irq_work_queue_on(&rq->scx.deferred_irq_work, cpu_of(rq)); +} + +/** + * schedule_deferred_locked - Schedule execution of deferred actions on an rq + * @rq: target rq + * + * Schedule execution of deferred actions on @rq. Equivalent to + * schedule_deferred() but requires @rq to be locked and can be more efficient. + */ +static void schedule_deferred_locked(struct rq *rq) +{ + lockdep_assert_rq_held(rq); + + /* + * If in the middle of waking up a task, task_woken_scx() will be called + * afterwards which will then run the deferred actions, no need to + * schedule anything. + */ + if (rq->scx.flags & SCX_RQ_IN_WAKEUP) + return; + + /* Don't do anything if there already is a deferred operation. */ + if (rq->scx.flags & SCX_RQ_BAL_CB_PENDING) + return; + + /* + * If in balance, the balance callbacks will be called before rq lock is + * released. Schedule one. + * + * + * We can't directly insert the callback into the + * rq's list: The call can drop its lock and make the pending balance + * callback visible to unrelated code paths that call rq_pin_lock(). + * + * Just let balance_one() know that it must do it itself. + */ + if (rq->scx.flags & SCX_RQ_IN_BALANCE) { + rq->scx.flags |= SCX_RQ_BAL_CB_PENDING; + return; + } + + /* + * No scheduler hooks available. Use the generic irq_work path. The + * above WAKEUP and BALANCE paths should cover most of the cases and the + * time to IRQ re-enable shouldn't be long. + */ + schedule_deferred(rq); +} + +static void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, + u64 reenq_flags, struct rq *locked_rq) +{ + struct rq *rq; + + /* + * Allowing reenqueues doesn't make sense while bypassing. This also + * blocks from new reenqueues to be scheduled on dead scheds. + */ + if (unlikely(READ_ONCE(sch->bypass_depth))) + return; + + if (dsq->id == SCX_DSQ_LOCAL) { + rq = container_of(dsq, struct rq, scx.local_dsq); + + struct scx_sched_pcpu *sch_pcpu = per_cpu_ptr(sch->pcpu, cpu_of(rq)); + struct scx_deferred_reenq_local *drl = &sch_pcpu->deferred_reenq_local; + + /* + * Pairs with smp_mb() in process_deferred_reenq_locals() and + * guarantees that there is a reenq_local() afterwards. + */ + smp_mb(); + + if (list_empty(&drl->node) || + (READ_ONCE(drl->flags) & reenq_flags) != reenq_flags) { + + guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); + + if (list_empty(&drl->node)) + list_move_tail(&drl->node, &rq->scx.deferred_reenq_locals); + WRITE_ONCE(drl->flags, drl->flags | reenq_flags); + } + } else if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN)) { + rq = this_rq(); + + struct scx_dsq_pcpu *dsq_pcpu = per_cpu_ptr(dsq->pcpu, cpu_of(rq)); + struct scx_deferred_reenq_user *dru = &dsq_pcpu->deferred_reenq_user; + + /* + * Pairs with smp_mb() in process_deferred_reenq_users() and + * guarantees that there is a reenq_user() afterwards. + */ + smp_mb(); + + if (list_empty(&dru->node) || + (READ_ONCE(dru->flags) & reenq_flags) != reenq_flags) { + + guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); + + if (list_empty(&dru->node)) + list_move_tail(&dru->node, &rq->scx.deferred_reenq_users); + WRITE_ONCE(dru->flags, dru->flags | reenq_flags); + } + } else { + scx_error(sch, "DSQ 0x%llx not allowed for reenq", dsq->id); + return; + } + + if (rq == locked_rq) + schedule_deferred_locked(rq); + else + schedule_deferred(rq); +} + +static void schedule_reenq_local(struct rq *rq, u64 reenq_flags) +{ + struct scx_sched *root = rcu_dereference_sched(scx_root); + + if (WARN_ON_ONCE(!root)) + return; + + schedule_dsq_reenq(root, &rq->scx.local_dsq, reenq_flags, rq); +} + +/** + * touch_core_sched - Update timestamp used for core-sched task ordering + * @rq: rq to read clock from, must be locked + * @p: task to update the timestamp for + * + * Update @p->scx.core_sched_at timestamp. This is used by scx_prio_less() to + * implement global or local-DSQ FIFO ordering for core-sched. Should be called + * when a task becomes runnable and its turn on the CPU ends (e.g. slice + * exhaustion). + */ +static void touch_core_sched(struct rq *rq, struct task_struct *p) +{ + lockdep_assert_rq_held(rq); + +#ifdef CONFIG_SCHED_CORE + /* + * It's okay to update the timestamp spuriously. Use + * sched_core_disabled() which is cheaper than enabled(). + * + * As this is used to determine ordering between tasks of sibling CPUs, + * it may be better to use per-core dispatch sequence instead. + */ + if (!sched_core_disabled()) + p->scx.core_sched_at = sched_clock_cpu(cpu_of(rq)); +#endif +} + +/** + * touch_core_sched_dispatch - Update core-sched timestamp on dispatch + * @rq: rq to read clock from, must be locked + * @p: task being dispatched + * + * If the BPF scheduler implements custom core-sched ordering via + * ops.core_sched_before(), @p->scx.core_sched_at is used to implement FIFO + * ordering within each local DSQ. This function is called from dispatch paths + * and updates @p->scx.core_sched_at if custom core-sched ordering is in effect. + */ +static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) +{ + lockdep_assert_rq_held(rq); + +#ifdef CONFIG_SCHED_CORE + if (unlikely(SCX_HAS_OP(scx_root, core_sched_before))) + touch_core_sched(rq, p); +#endif +} + +static void update_curr_scx(struct rq *rq) +{ + struct task_struct *curr = rq->curr; + s64 delta_exec; + + delta_exec = update_curr_common(rq); + if (unlikely(delta_exec <= 0)) + return; + + if (curr->scx.slice != SCX_SLICE_INF) { + curr->scx.slice -= min_t(u64, curr->scx.slice, delta_exec); + if (!curr->scx.slice) + touch_core_sched(rq, curr); + } + + dl_server_update(&rq->ext_server, delta_exec); +} + +static bool scx_dsq_priq_less(struct rb_node *node_a, + const struct rb_node *node_b) +{ + const struct task_struct *a = + container_of(node_a, struct task_struct, scx.dsq_priq); + const struct task_struct *b = + container_of(node_b, struct task_struct, scx.dsq_priq); + + return time_before64(a->scx.dsq_vtime, b->scx.dsq_vtime); +} + +static void dsq_inc_nr(struct scx_dispatch_q *dsq, struct task_struct *p, u64 enq_flags) +{ + /* scx_bpf_dsq_nr_queued() reads ->nr without locking, use WRITE_ONCE() */ + WRITE_ONCE(dsq->nr, dsq->nr + 1); + + /* + * Once @p reaches a local DSQ, it can only leave it by being dispatched + * to the CPU or dequeued. In both cases, the only way @p can go back to + * the BPF sched is through enqueueing. If being inserted into a local + * DSQ with IMMED, persist the state until the next enqueueing event in + * do_enqueue_task() so that we can maintain IMMED protection through + * e.g. SAVE/RESTORE cycles and slice extensions. + */ + if (enq_flags & SCX_ENQ_IMMED) { + if (unlikely(dsq->id != SCX_DSQ_LOCAL)) { + WARN_ON_ONCE(!(enq_flags & SCX_ENQ_GDSQ_FALLBACK)); + return; + } + p->scx.flags |= SCX_TASK_IMMED; + } + + if (p->scx.flags & SCX_TASK_IMMED) { + struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); + + if (WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL)) + return; + + rq->scx.nr_immed++; + + /* + * If @rq already had other tasks or the current task is not + * done yet, @p can't go on the CPU immediately. Re-enqueue. + */ + if (unlikely(dsq->nr > 1 || !rq_is_open(rq, enq_flags))) + schedule_reenq_local(rq, 0); + } +} + +static void dsq_dec_nr(struct scx_dispatch_q *dsq, struct task_struct *p) +{ + /* see dsq_inc_nr() */ + WRITE_ONCE(dsq->nr, dsq->nr - 1); + + if (p->scx.flags & SCX_TASK_IMMED) { + struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); + + if (WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL) || + WARN_ON_ONCE(rq->scx.nr_immed <= 0)) + return; + + rq->scx.nr_immed--; + } +} + +static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p) +{ + p->scx.slice = READ_ONCE(sch->slice_dfl); + __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1); +} + +/* + * Return true if @p is moving due to an internal SCX migration, false + * otherwise. + */ +static inline bool task_scx_migrating(struct task_struct *p) +{ + /* + * We only need to check sticky_cpu: it is set to the destination + * CPU in move_remote_task_to_local_dsq() before deactivate_task() + * and cleared when the task is enqueued on the destination, so it + * is only non-negative during an internal SCX migration. + */ + return p->scx.sticky_cpu >= 0; +} + +/* + * Call ops.dequeue() if the task is in BPF custody and not migrating. + * Clears %SCX_TASK_IN_CUSTODY when the callback is invoked. + */ +static void call_task_dequeue(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 deq_flags) +{ + if (!(p->scx.flags & SCX_TASK_IN_CUSTODY) || task_scx_migrating(p)) + return; + + if (SCX_HAS_OP(sch, dequeue)) + SCX_CALL_OP_TASK(sch, dequeue, rq, p, deq_flags); + + p->scx.flags &= ~SCX_TASK_IN_CUSTODY; +} + +static void local_dsq_post_enq(struct scx_sched *sch, struct scx_dispatch_q *dsq, + struct task_struct *p, u64 enq_flags) +{ + struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); + + call_task_dequeue(sch, rq, p, 0); + + /* + * Note that @rq's lock may be dropped between this enqueue and @p + * actually getting on CPU. This gives higher-class tasks (e.g. RT) + * an opportunity to wake up on @rq and prevent @p from running. + * Here are some concrete examples: + * + * Example 1: + * + * We dispatch two tasks from a single ops.dispatch(): + * - First, a local task to this CPU's local DSQ; + * - Second, a local/remote task to a remote CPU's local DSQ. + * We must drop the local rq lock in order to finish the second + * dispatch. In that time, an RT task can wake up on the local rq. + * + * Example 2: + * + * We dispatch a local/remote task to a remote CPU's local DSQ. + * We must drop the remote rq lock before the dispatched task can run, + * which gives an RT task an opportunity to wake up on the remote rq. + * + * Both examples work the same if we replace dispatching with moving + * the tasks from a user-created DSQ. + * + * We must detect these wakeups so that we can re-enqueue IMMED tasks + * from @rq's local DSQ. scx_wakeup_preempt() serves exactly this + * purpose, but for it to be invoked, we must ensure that we bump + * @rq->next_class to &ext_sched_class if it's currently idle. + * + * wakeup_preempt() does the bumping, and since we only invoke it if + * @rq->next_class is below &ext_sched_class, it will also + * resched_curr(rq). + */ + if (sched_class_above(p->sched_class, rq->next_class)) + wakeup_preempt(rq, p, 0); + + /* + * If @rq is in balance, the CPU is already vacant and looking for the + * next task to run. No need to preempt or trigger resched after moving + * @p into its local DSQ. + * Note that the wakeup_preempt() above may have already triggered + * a resched if @rq->next_class was idle. It's harmless, since + * need_resched is cleared immediately after task pick. + */ + if (rq->scx.flags & SCX_RQ_IN_BALANCE) + return; + + if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && + rq->curr->sched_class == &ext_sched_class) { + rq->curr->scx.slice = 0; + resched_curr(rq); + } +} + +static void dispatch_enqueue(struct scx_sched *sch, struct rq *rq, + struct scx_dispatch_q *dsq, struct task_struct *p, + u64 enq_flags) +{ + bool is_local = dsq->id == SCX_DSQ_LOCAL; + + WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); + WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || + !RB_EMPTY_NODE(&p->scx.dsq_priq)); + + if (!is_local) { + raw_spin_lock_nested(&dsq->lock, + (enq_flags & SCX_ENQ_NESTED) ? SINGLE_DEPTH_NESTING : 0); + + if (unlikely(dsq->id == SCX_DSQ_INVALID)) { + scx_error(sch, "attempting to dispatch to a destroyed dsq"); + /* fall back to the global dsq */ + raw_spin_unlock(&dsq->lock); + dsq = find_global_dsq(sch, task_cpu(p)); + raw_spin_lock(&dsq->lock); + } + } + + if (unlikely((dsq->id & SCX_DSQ_FLAG_BUILTIN) && + (enq_flags & SCX_ENQ_DSQ_PRIQ))) { + /* + * SCX_DSQ_LOCAL and SCX_DSQ_GLOBAL DSQs always consume from + * their FIFO queues. To avoid confusion and accidentally + * starving vtime-dispatched tasks by FIFO-dispatched tasks, we + * disallow any internal DSQ from doing vtime ordering of + * tasks. + */ + scx_error(sch, "cannot use vtime ordering for built-in DSQs"); + enq_flags &= ~SCX_ENQ_DSQ_PRIQ; + } + + if (enq_flags & SCX_ENQ_DSQ_PRIQ) { + struct rb_node *rbp; + + /* + * A PRIQ DSQ shouldn't be using FIFO enqueueing. As tasks are + * linked to both the rbtree and list on PRIQs, this can only be + * tested easily when adding the first task. + */ + if (unlikely(RB_EMPTY_ROOT(&dsq->priq) && + nldsq_next_task(dsq, NULL, false))) + scx_error(sch, "DSQ ID 0x%016llx already had FIFO-enqueued tasks", + dsq->id); + + p->scx.dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; + rb_add(&p->scx.dsq_priq, &dsq->priq, scx_dsq_priq_less); + + /* + * Find the previous task and insert after it on the list so + * that @dsq->list is vtime ordered. + */ + rbp = rb_prev(&p->scx.dsq_priq); + if (rbp) { + struct task_struct *prev = + container_of(rbp, struct task_struct, + scx.dsq_priq); + list_add(&p->scx.dsq_list.node, &prev->scx.dsq_list.node); + /* first task unchanged - no update needed */ + } else { + list_add(&p->scx.dsq_list.node, &dsq->list); + /* not builtin and new task is at head - use fastpath */ + rcu_assign_pointer(dsq->first_task, p); + } + } else { + /* a FIFO DSQ shouldn't be using PRIQ enqueuing */ + if (unlikely(!RB_EMPTY_ROOT(&dsq->priq))) + scx_error(sch, "DSQ ID 0x%016llx already had PRIQ-enqueued tasks", + dsq->id); + + if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) { + list_add(&p->scx.dsq_list.node, &dsq->list); + /* new task inserted at head - use fastpath */ + if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN)) + rcu_assign_pointer(dsq->first_task, p); + } else { + /* + * dsq->list can contain parked BPF iterator cursors, so + * list_empty() here isn't a reliable proxy for "no real + * task in the DSQ". Test dsq->first_task directly. + */ + list_add_tail(&p->scx.dsq_list.node, &dsq->list); + if (!dsq->first_task && !(dsq->id & SCX_DSQ_FLAG_BUILTIN)) + rcu_assign_pointer(dsq->first_task, p); + } + } + + /* seq records the order tasks are queued, used by BPF DSQ iterator */ + WRITE_ONCE(dsq->seq, dsq->seq + 1); + p->scx.dsq_seq = dsq->seq; + + dsq_inc_nr(dsq, p, enq_flags); + p->scx.dsq = dsq; + + /* + * Update custody and call ops.dequeue() before clearing ops_state: + * once ops_state is cleared, waiters in ops_dequeue() can proceed + * and dequeue_task_scx() will RMW p->scx.flags. If we clear + * ops_state first, both sides would modify p->scx.flags + * concurrently in a non-atomic way. + */ + if (is_local) { + local_dsq_post_enq(sch, dsq, p, enq_flags); + } else { + /* + * Task on global/bypass DSQ: leave custody, task on + * non-terminal DSQ: enter custody. + */ + if (dsq->id == SCX_DSQ_GLOBAL || dsq->id == SCX_DSQ_BYPASS) + call_task_dequeue(sch, rq, p, 0); + else + p->scx.flags |= SCX_TASK_IN_CUSTODY; + + raw_spin_unlock(&dsq->lock); + } + + /* + * We're transitioning out of QUEUEING or DISPATCHING. store_release to + * match waiters' load_acquire. + */ + if (enq_flags & SCX_ENQ_CLEAR_OPSS) + atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); +} + +static void task_unlink_from_dsq(struct task_struct *p, + struct scx_dispatch_q *dsq) +{ + WARN_ON_ONCE(list_empty(&p->scx.dsq_list.node)); + + if (p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { + rb_erase(&p->scx.dsq_priq, &dsq->priq); + RB_CLEAR_NODE(&p->scx.dsq_priq); + p->scx.dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; + } + + list_del_init(&p->scx.dsq_list.node); + dsq_dec_nr(dsq, p); + + if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN) && dsq->first_task == p) { + struct task_struct *first_task; + + first_task = nldsq_next_task(dsq, NULL, false); + rcu_assign_pointer(dsq->first_task, first_task); + } +} + +static void dispatch_dequeue(struct rq *rq, struct task_struct *p) +{ + struct scx_dispatch_q *dsq = p->scx.dsq; + bool is_local = dsq == &rq->scx.local_dsq; + + lockdep_assert_rq_held(rq); + + if (!dsq) { + /* + * If !dsq && on-list, @p is on @rq's ddsp_deferred_locals. + * Unlinking is all that's needed to cancel. + */ + if (unlikely(!list_empty(&p->scx.dsq_list.node))) + list_del_init(&p->scx.dsq_list.node); + + /* + * When dispatching directly from the BPF scheduler to a local + * DSQ, the task isn't associated with any DSQ but + * @p->scx.holding_cpu may be set under the protection of + * %SCX_OPSS_DISPATCHING. + */ + if (p->scx.holding_cpu >= 0) + p->scx.holding_cpu = -1; + + return; + } + + if (!is_local) + raw_spin_lock(&dsq->lock); + + /* + * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx.dsq_* can't + * change underneath us. + */ + if (p->scx.holding_cpu < 0) { + /* @p must still be on @dsq, dequeue */ + task_unlink_from_dsq(p, dsq); + } else { + /* + * We're racing against dispatch_to_local_dsq() which already + * removed @p from @dsq and set @p->scx.holding_cpu. Clear the + * holding_cpu which tells dispatch_to_local_dsq() that it lost + * the race. + */ + WARN_ON_ONCE(!list_empty(&p->scx.dsq_list.node)); + p->scx.holding_cpu = -1; + } + p->scx.dsq = NULL; + + if (!is_local) + raw_spin_unlock(&dsq->lock); +} + +/* + * Abbreviated version of dispatch_dequeue() that can be used when both @p's rq + * and dsq are locked. + */ +static void dispatch_dequeue_locked(struct task_struct *p, + struct scx_dispatch_q *dsq) +{ + lockdep_assert_rq_held(task_rq(p)); + lockdep_assert_held(&dsq->lock); + + task_unlink_from_dsq(p, dsq); + p->scx.dsq = NULL; +} + +static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch, + struct rq *rq, u64 dsq_id, + s32 tcpu) +{ + struct scx_dispatch_q *dsq; + + if (dsq_id == SCX_DSQ_LOCAL) + return &rq->scx.local_dsq; + + if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { + s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK); + + if (!scx_cpu_valid(sch, cpu, "in SCX_DSQ_LOCAL_ON dispatch verdict")) + return find_global_dsq(sch, tcpu); + + return &cpu_rq(cpu)->scx.local_dsq; + } + + if (dsq_id == SCX_DSQ_GLOBAL) + dsq = find_global_dsq(sch, tcpu); + else + dsq = find_user_dsq(sch, dsq_id); + + if (unlikely(!dsq)) { + scx_error(sch, "non-existent DSQ 0x%llx", dsq_id); + return find_global_dsq(sch, tcpu); + } + + return dsq; +} + +static void mark_direct_dispatch(struct scx_sched *sch, + struct task_struct *ddsp_task, + struct task_struct *p, u64 dsq_id, + u64 enq_flags) +{ + /* + * Mark that dispatch already happened from ops.select_cpu() or + * ops.enqueue() by spoiling direct_dispatch_task with a non-NULL value + * which can never match a valid task pointer. + */ + __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH)); + + /* @p must match the task on the enqueue path */ + if (unlikely(p != ddsp_task)) { + if (IS_ERR(ddsp_task)) + scx_error(sch, "%s[%d] already direct-dispatched", + p->comm, p->pid); + else + scx_error(sch, "scheduling for %s[%d] but trying to direct-dispatch %s[%d]", + ddsp_task->comm, ddsp_task->pid, + p->comm, p->pid); + return; + } + + WARN_ON_ONCE(p->scx.ddsp_dsq_id != SCX_DSQ_INVALID); + WARN_ON_ONCE(p->scx.ddsp_enq_flags); + + p->scx.ddsp_dsq_id = dsq_id; + p->scx.ddsp_enq_flags = enq_flags; +} + +/* + * Clear @p direct dispatch state when leaving the scheduler. + * + * Direct dispatch state must be cleared in the following cases: + * - direct_dispatch(): cleared on the synchronous enqueue path, deferred + * dispatch keeps the state until consumed + * - process_ddsp_deferred_locals(): cleared after consuming deferred state, + * - do_enqueue_task(): cleared on enqueue fallbacks where the dispatch + * verdict is ignored (local/global/bypass) + * - dequeue_task_scx(): cleared after dispatch_dequeue(), covering deferred + * cancellation and holding_cpu races + * - scx_disable_task(): cleared for queued wakeup tasks, which are excluded by + * the scx_bypass() loop, so that stale state is not reused by a subsequent + * scheduler instance + */ +static inline void clear_direct_dispatch(struct task_struct *p) +{ + p->scx.ddsp_dsq_id = SCX_DSQ_INVALID; + p->scx.ddsp_enq_flags = 0; +} + +static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, + u64 enq_flags) +{ + struct rq *rq = task_rq(p); + struct scx_dispatch_q *dsq = + find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, task_cpu(p)); + u64 ddsp_enq_flags; + + touch_core_sched_dispatch(rq, p); + + p->scx.ddsp_enq_flags |= enq_flags; + + /* + * We are in the enqueue path with @rq locked and pinned, and thus can't + * double lock a remote rq and enqueue to its local DSQ. For + * DSQ_LOCAL_ON verdicts targeting the local DSQ of a remote CPU, defer + * the enqueue so that it's executed when @rq can be unlocked. + */ + if (dsq->id == SCX_DSQ_LOCAL && dsq != &rq->scx.local_dsq) { + unsigned long opss; + + opss = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_STATE_MASK; + + switch (opss & SCX_OPSS_STATE_MASK) { + case SCX_OPSS_NONE: + break; + case SCX_OPSS_QUEUEING: + /* + * As @p was never passed to the BPF side, _release is + * not strictly necessary. Still do it for consistency. + */ + atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); + break; + default: + WARN_ONCE(true, "sched_ext: %s[%d] has invalid ops state 0x%lx in direct_dispatch()", + p->comm, p->pid, opss); + atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); + break; + } + + WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); + list_add_tail(&p->scx.dsq_list.node, + &rq->scx.ddsp_deferred_locals); + schedule_deferred_locked(rq); + return; + } + + ddsp_enq_flags = p->scx.ddsp_enq_flags; + clear_direct_dispatch(p); + + dispatch_enqueue(sch, rq, dsq, p, ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS); +} + +static bool scx_rq_online(struct rq *rq) +{ + /* + * Test both cpu_active() and %SCX_RQ_ONLINE. %SCX_RQ_ONLINE indicates + * the online state as seen from the BPF scheduler. cpu_active() test + * guarantees that, if this function returns %true, %SCX_RQ_ONLINE will + * stay set until the current scheduling operation is complete even if + * we aren't locking @rq. + */ + return likely((rq->scx.flags & SCX_RQ_ONLINE) && cpu_active(cpu_of(rq))); +} + +static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, + int sticky_cpu) +{ + struct scx_sched *sch = scx_task_sched(p); + struct task_struct **ddsp_taskp; + struct scx_dispatch_q *dsq; + unsigned long qseq; + + WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_QUEUED)); + + /* internal movements - rq migration / RESTORE */ + if (sticky_cpu == cpu_of(rq)) + goto local_norefill; + + /* + * Clear persistent TASK_IMMED for fresh enqueues, see dsq_inc_nr(). + * Note that exiting and migration-disabled tasks that skip + * ops.enqueue() below will lose IMMED protection unless + * %SCX_OPS_ENQ_EXITING / %SCX_OPS_ENQ_MIGRATION_DISABLED are set. + */ + p->scx.flags &= ~SCX_TASK_IMMED; + + /* + * If !scx_rq_online(), we already told the BPF scheduler that the CPU + * is offline and are just running the hotplug path. Don't bother the + * BPF scheduler. + */ + if (!scx_rq_online(rq)) + goto local; + + if (scx_bypassing(sch, cpu_of(rq))) { + __scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1); + goto bypass; + } + + if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) + goto direct; + + /* see %SCX_OPS_ENQ_EXITING */ + if (!(sch->ops.flags & SCX_OPS_ENQ_EXITING) && + unlikely(p->flags & PF_EXITING)) { + __scx_add_event(sch, SCX_EV_ENQ_SKIP_EXITING, 1); + goto local; + } + + /* see %SCX_OPS_ENQ_MIGRATION_DISABLED */ + if (!(sch->ops.flags & SCX_OPS_ENQ_MIGRATION_DISABLED) && + is_migration_disabled(p)) { + __scx_add_event(sch, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED, 1); + goto local; + } + + if (unlikely(!SCX_HAS_OP(sch, enqueue))) + goto global; + + /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ + qseq = rq->scx.ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; + + WARN_ON_ONCE(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE); + atomic_long_set(&p->scx.ops_state, SCX_OPSS_QUEUEING | qseq); + + ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); + WARN_ON_ONCE(*ddsp_taskp); + *ddsp_taskp = p; + + SCX_CALL_OP_TASK(sch, enqueue, rq, p, enq_flags); + + *ddsp_taskp = NULL; + if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) + goto direct; + + /* + * Task is now in BPF scheduler's custody. Set %SCX_TASK_IN_CUSTODY + * so ops.dequeue() is called when it leaves custody. + */ + p->scx.flags |= SCX_TASK_IN_CUSTODY; + + /* + * If not directly dispatched, QUEUEING isn't clear yet and dispatch or + * dequeue may be waiting. The store_release matches their load_acquire. + */ + atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_QUEUED | qseq); + return; + +direct: + direct_dispatch(sch, p, enq_flags); + return; +local_norefill: + dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, enq_flags); + return; +local: + dsq = &rq->scx.local_dsq; + goto enqueue; +global: + dsq = find_global_dsq(sch, task_cpu(p)); + goto enqueue; +bypass: + dsq = bypass_enq_target_dsq(sch, task_cpu(p)); + goto enqueue; + +enqueue: + /* + * For task-ordering, slice refill must be treated as implying the end + * of the current slice. Otherwise, the longer @p stays on the CPU, the + * higher priority it becomes from scx_prio_less()'s POV. + */ + touch_core_sched(rq, p); + refill_task_slice_dfl(sch, p); + clear_direct_dispatch(p); + dispatch_enqueue(sch, rq, dsq, p, enq_flags); +} + +static bool task_runnable(const struct task_struct *p) +{ + return !list_empty(&p->scx.runnable_node); +} + +static void set_task_runnable(struct rq *rq, struct task_struct *p) +{ + lockdep_assert_rq_held(rq); + + if (p->scx.flags & SCX_TASK_RESET_RUNNABLE_AT) { + p->scx.runnable_at = jiffies; + p->scx.flags &= ~SCX_TASK_RESET_RUNNABLE_AT; + } + + /* + * list_add_tail() must be used. scx_bypass() depends on tasks being + * appended to the runnable_list. + */ + list_add_tail(&p->scx.runnable_node, &rq->scx.runnable_list); +} + +static void clr_task_runnable(struct task_struct *p, bool reset_runnable_at) +{ + list_del_init(&p->scx.runnable_node); + if (reset_runnable_at) + p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; +} + +static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_flags) +{ + struct scx_sched *sch = scx_task_sched(p); + int sticky_cpu = p->scx.sticky_cpu; + u64 enq_flags = core_enq_flags | rq->scx.extra_enq_flags; + + if (enq_flags & ENQUEUE_WAKEUP) + rq->scx.flags |= SCX_RQ_IN_WAKEUP; + + /* + * Restoring a running task will be immediately followed by + * set_next_task_scx() which expects the task to not be on the BPF + * scheduler as tasks can only start running through local DSQs. Force + * direct-dispatch into the local DSQ by setting the sticky_cpu. + */ + if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) + sticky_cpu = cpu_of(rq); + + if (p->scx.flags & SCX_TASK_QUEUED) { + WARN_ON_ONCE(!task_runnable(p)); + goto out; + } + + set_task_runnable(rq, p); + p->scx.flags |= SCX_TASK_QUEUED; + rq->scx.nr_running++; + add_nr_running(rq, 1); + + if (SCX_HAS_OP(sch, runnable) && !task_on_rq_migrating(p)) + SCX_CALL_OP_TASK(sch, runnable, rq, p, enq_flags); + + if (enq_flags & SCX_ENQ_WAKEUP) + touch_core_sched(rq, p); + + /* Start dl_server if this is the first task being enqueued */ + if (rq->scx.nr_running == 1) + dl_server_start(&rq->ext_server); + + do_enqueue_task(rq, p, enq_flags, sticky_cpu); + + if (sticky_cpu >= 0) + p->scx.sticky_cpu = -1; +out: + rq->scx.flags &= ~SCX_RQ_IN_WAKEUP; + + if ((enq_flags & SCX_ENQ_CPU_SELECTED) && + unlikely(cpu_of(rq) != p->scx.selected_cpu)) + __scx_add_event(sch, SCX_EV_SELECT_CPU_FALLBACK, 1); +} + +static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags) +{ + struct scx_sched *sch = scx_task_sched(p); + unsigned long opss; + + /* dequeue is always temporary, don't reset runnable_at */ + clr_task_runnable(p, false); + +retry: + /* acquire ensures that we see the preceding updates on QUEUED */ + opss = atomic_long_read_acquire(&p->scx.ops_state); + + switch (opss & SCX_OPSS_STATE_MASK) { + case SCX_OPSS_NONE: + break; + case SCX_OPSS_QUEUEING: + /* + * QUEUEING is started and finished while holding @p's rq lock. + * As we're holding the rq lock now, we shouldn't see QUEUEING. + */ + BUG(); + case SCX_OPSS_QUEUED: + /* + * A queued task must always be in BPF scheduler's custody. If + * SCX_TASK_IN_CUSTODY is clear, finish_dispatch() on another + * CPU has already passed call_task_dequeue() (which clears the + * flag), but has not yet written SCX_OPSS_NONE. That final + * store does not require this rq's lock, so retrying with + * cpu_relax() is bounded: we will observe NONE (or DISPATCHING, + * handled by the fallthrough) on a subsequent iteration. + */ + if (unlikely(!(READ_ONCE(p->scx.flags) & SCX_TASK_IN_CUSTODY))) { + cpu_relax(); + goto retry; + } + + if (atomic_long_try_cmpxchg(&p->scx.ops_state, &opss, + SCX_OPSS_NONE)) + break; + fallthrough; + case SCX_OPSS_DISPATCHING: + /* + * If @p is being dispatched from the BPF scheduler to a DSQ, + * wait for the transfer to complete so that @p doesn't get + * added to its DSQ after dequeueing is complete. + * + * As we're waiting on DISPATCHING with the rq locked, the + * dispatching side shouldn't try to lock the rq while + * DISPATCHING is set. See dispatch_to_local_dsq(). + * + * DISPATCHING shouldn't have qseq set and control can reach + * here with NONE @opss from the above QUEUED case block. + * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss. + */ + wait_ops_state(p, SCX_OPSS_DISPATCHING); + BUG_ON(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE); + break; + } + + /* + * Call ops.dequeue() if the task is still in BPF custody. + * + * The code that clears ops_state to %SCX_OPSS_NONE does not always + * clear %SCX_TASK_IN_CUSTODY: in dispatch_to_local_dsq(), when + * we're moving a task that was in %SCX_OPSS_DISPATCHING to a + * remote CPU's local DSQ, we only set ops_state to %SCX_OPSS_NONE + * so that a concurrent dequeue can proceed, but we clear + * %SCX_TASK_IN_CUSTODY only when we later enqueue or move the + * task. So we can see NONE + IN_CUSTODY here and we must handle + * it. Similarly, after waiting on %SCX_OPSS_DISPATCHING we see + * NONE but the task may still have %SCX_TASK_IN_CUSTODY set until + * it is enqueued on the destination. + */ + call_task_dequeue(sch, rq, p, deq_flags); +} + +static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_flags) +{ + struct scx_sched *sch = scx_task_sched(p); + u64 deq_flags = core_deq_flags; + + /* + * Set %SCX_DEQ_SCHED_CHANGE when the dequeue is due to a property + * change (not sleep or core-sched pick). + */ + if (!(deq_flags & (DEQUEUE_SLEEP | SCX_DEQ_CORE_SCHED_EXEC))) + deq_flags |= SCX_DEQ_SCHED_CHANGE; + + if (!(p->scx.flags & SCX_TASK_QUEUED)) { + WARN_ON_ONCE(task_runnable(p)); + return true; + } + + ops_dequeue(rq, p, deq_flags); + + /* + * A currently running task which is going off @rq first gets dequeued + * and then stops running. As we want running <-> stopping transitions + * to be contained within runnable <-> quiescent transitions, trigger + * ->stopping() early here instead of in put_prev_task_scx(). + * + * @p may go through multiple stopping <-> running transitions between + * here and put_prev_task_scx() if task attribute changes occur while + * balance_one() leaves @rq unlocked. However, they don't contain any + * information meaningful to the BPF scheduler and can be suppressed by + * skipping the callbacks if the task is !QUEUED. + */ + if (SCX_HAS_OP(sch, stopping) && task_current(rq, p)) { + update_curr_scx(rq); + SCX_CALL_OP_TASK(sch, stopping, rq, p, false); + } + + if (SCX_HAS_OP(sch, quiescent) && !task_on_rq_migrating(p)) + SCX_CALL_OP_TASK(sch, quiescent, rq, p, deq_flags); + + if (deq_flags & SCX_DEQ_SLEEP) + p->scx.flags |= SCX_TASK_DEQD_FOR_SLEEP; + else + p->scx.flags &= ~SCX_TASK_DEQD_FOR_SLEEP; + + p->scx.flags &= ~SCX_TASK_QUEUED; + rq->scx.nr_running--; + sub_nr_running(rq, 1); + + dispatch_dequeue(rq, p); + clear_direct_dispatch(p); + return true; +} + +static void yield_task_scx(struct rq *rq) +{ + struct task_struct *p = rq->donor; + struct scx_sched *sch = scx_task_sched(p); + + if (SCX_HAS_OP(sch, yield)) + SCX_CALL_OP_2TASKS_RET(sch, yield, rq, p, NULL); + else + p->scx.slice = 0; +} + +static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) +{ + struct task_struct *from = rq->donor; + struct scx_sched *sch = scx_task_sched(from); + + if (SCX_HAS_OP(sch, yield) && sch == scx_task_sched(to)) + return SCX_CALL_OP_2TASKS_RET(sch, yield, rq, from, to); + else + return false; +} + +static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_flags) +{ + /* + * Preemption between SCX tasks is implemented by resetting the victim + * task's slice to 0 and triggering reschedule on the target CPU. + * Nothing to do. + */ + if (p->sched_class == &ext_sched_class) + return; + + /* + * Getting preempted by a higher-priority class. Reenqueue IMMED tasks. + * This captures all preemption cases including: + * + * - A SCX task is currently running. + * + * - @rq is waking from idle due to a SCX task waking to it. + * + * - A higher-priority wakes up while SCX dispatch is in progress. + */ + if (rq->scx.nr_immed) + schedule_reenq_local(rq, 0); +} + +static void move_local_task_to_local_dsq(struct scx_sched *sch, + struct task_struct *p, u64 enq_flags, + struct scx_dispatch_q *src_dsq, + struct rq *dst_rq) +{ + struct scx_dispatch_q *dst_dsq = &dst_rq->scx.local_dsq; + + /* @dsq is locked and @p is on @dst_rq */ + lockdep_assert_held(&src_dsq->lock); + lockdep_assert_rq_held(dst_rq); + + WARN_ON_ONCE(p->scx.holding_cpu >= 0); + + if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) + list_add(&p->scx.dsq_list.node, &dst_dsq->list); + else + list_add_tail(&p->scx.dsq_list.node, &dst_dsq->list); + + dsq_inc_nr(dst_dsq, p, enq_flags); + p->scx.dsq = dst_dsq; + + local_dsq_post_enq(sch, dst_dsq, p, enq_flags); +} + +/** + * move_remote_task_to_local_dsq - Move a task from a foreign rq to a local DSQ + * @p: task to move + * @enq_flags: %SCX_ENQ_* + * @src_rq: rq to move the task from, locked on entry, released on return + * @dst_rq: rq to move the task into, locked on return + * + * Move @p which is currently on @src_rq to @dst_rq's local DSQ. + */ +static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, + struct rq *src_rq, struct rq *dst_rq) +{ + lockdep_assert_rq_held(src_rq); + + /* + * Set sticky_cpu before deactivate_task() to properly mark the + * beginning of an SCX-internal migration. + */ + p->scx.sticky_cpu = cpu_of(dst_rq); + deactivate_task(src_rq, p, 0); + set_task_cpu(p, cpu_of(dst_rq)); + + raw_spin_rq_unlock(src_rq); + raw_spin_rq_lock(dst_rq); + + /* + * We want to pass scx-specific enq_flags but activate_task() will + * truncate the upper 32 bit. As we own @rq, we can pass them through + * @rq->scx.extra_enq_flags instead. + */ + WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)); + WARN_ON_ONCE(dst_rq->scx.extra_enq_flags); + dst_rq->scx.extra_enq_flags = enq_flags; + activate_task(dst_rq, p, 0); + dst_rq->scx.extra_enq_flags = 0; +} + +/* + * Similar to kernel/sched/core.c::is_cpu_allowed(). However, there are two + * differences: + * + * - is_cpu_allowed() asks "Can this task run on this CPU?" while + * task_can_run_on_remote_rq() asks "Can the BPF scheduler migrate the task to + * this CPU?". + * + * While migration is disabled, is_cpu_allowed() has to say "yes" as the task + * must be allowed to finish on the CPU that it's currently on regardless of + * the CPU state. However, task_can_run_on_remote_rq() must say "no" as the + * BPF scheduler shouldn't attempt to migrate a task which has migration + * disabled. + * + * - The BPF scheduler is bypassed while the rq is offline and we can always say + * no to the BPF scheduler initiated migrations while offline. + * + * The caller must ensure that @p and @rq are on different CPUs. + */ +static bool task_can_run_on_remote_rq(struct scx_sched *sch, + struct task_struct *p, struct rq *rq, + bool enforce) +{ + s32 cpu = cpu_of(rq); + + WARN_ON_ONCE(task_cpu(p) == cpu); + + /* + * If @p has migration disabled, @p->cpus_ptr is updated to contain only + * the pinned CPU in migrate_disable_switch() while @p is being switched + * out. However, put_prev_task_scx() is called before @p->cpus_ptr is + * updated and thus another CPU may see @p on a DSQ inbetween leading to + * @p passing the below task_allowed_on_cpu() check while migration is + * disabled. + * + * Test the migration disabled state first as the race window is narrow + * and the BPF scheduler failing to check migration disabled state can + * easily be masked if task_allowed_on_cpu() is done first. + */ + if (unlikely(is_migration_disabled(p))) { + if (enforce) + scx_error(sch, "SCX_DSQ_LOCAL[_ON] cannot move migration disabled %s[%d] from CPU %d to %d", + p->comm, p->pid, task_cpu(p), cpu); + return false; + } + + /* + * We don't require the BPF scheduler to avoid dispatching to offline + * CPUs mostly for convenience but also because CPUs can go offline + * between scx_bpf_dsq_insert() calls and here. Trigger error iff the + * picked CPU is outside the allowed mask. + */ + if (!task_allowed_on_cpu(p, cpu)) { + if (enforce) + scx_error(sch, "SCX_DSQ_LOCAL[_ON] target CPU %d not allowed for %s[%d]", + cpu, p->comm, p->pid); + return false; + } + + if (!scx_rq_online(rq)) { + if (enforce) + __scx_add_event(sch, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE, 1); + return false; + } + + return true; +} + +/** + * unlink_dsq_and_lock_src_rq() - Unlink task from its DSQ and lock its task_rq + * @p: target task + * @dsq: locked DSQ @p is currently on + * @src_rq: rq @p is currently on, stable with @dsq locked + * + * Called with @dsq locked but no rq's locked. We want to move @p to a different + * DSQ, including any local DSQ, but are not locking @src_rq. Locking @src_rq is + * required when transferring into a local DSQ. Even when transferring into a + * non-local DSQ, it's better to use the same mechanism to protect against + * dequeues and maintain the invariant that @p->scx.dsq can only change while + * @src_rq is locked, which e.g. scx_dump_task() depends on. + * + * We want to grab @src_rq but that can deadlock if we try while locking @dsq, + * so we want to unlink @p from @dsq, drop its lock and then lock @src_rq. As + * this may race with dequeue, which can't drop the rq lock or fail, do a little + * dancing from our side. + * + * @p->scx.holding_cpu is set to this CPU before @dsq is unlocked. If @p gets + * dequeued after we unlock @dsq but before locking @src_rq, the holding_cpu + * would be cleared to -1. While other cpus may have updated it to different + * values afterwards, as this operation can't be preempted or recurse, the + * holding_cpu can never become this CPU again before we're done. Thus, we can + * tell whether we lost to dequeue by testing whether the holding_cpu still + * points to this CPU. See dispatch_dequeue() for the counterpart. + * + * On return, @dsq is unlocked and @src_rq is locked. Returns %true if @p is + * still valid. %false if lost to dequeue. + */ +static bool unlink_dsq_and_lock_src_rq(struct task_struct *p, + struct scx_dispatch_q *dsq, + struct rq *src_rq) +{ + s32 cpu = raw_smp_processor_id(); + + lockdep_assert_held(&dsq->lock); + + WARN_ON_ONCE(p->scx.holding_cpu >= 0); + task_unlink_from_dsq(p, dsq); + p->scx.holding_cpu = cpu; + + raw_spin_unlock(&dsq->lock); + raw_spin_rq_lock(src_rq); + + /* task_rq couldn't have changed if we're still the holding cpu */ + return likely(p->scx.holding_cpu == cpu) && + !WARN_ON_ONCE(src_rq != task_rq(p)); +} + +static bool consume_remote_task(struct rq *this_rq, + struct task_struct *p, u64 enq_flags, + struct scx_dispatch_q *dsq, struct rq *src_rq) +{ + raw_spin_rq_unlock(this_rq); + + if (unlink_dsq_and_lock_src_rq(p, dsq, src_rq)) { + move_remote_task_to_local_dsq(p, enq_flags, src_rq, this_rq); + return true; + } else { + raw_spin_rq_unlock(src_rq); + raw_spin_rq_lock(this_rq); + return false; + } +} + +/** + * move_task_between_dsqs() - Move a task from one DSQ to another + * @sch: scx_sched being operated on + * @p: target task + * @enq_flags: %SCX_ENQ_* + * @src_dsq: DSQ @p is currently on, must not be a local DSQ + * @dst_dsq: DSQ @p is being moved to, can be any DSQ + * + * Must be called with @p's task_rq and @src_dsq locked. If @dst_dsq is a local + * DSQ and @p is on a different CPU, @p will be migrated and thus its task_rq + * will change. As @p's task_rq is locked, this function doesn't need to use the + * holding_cpu mechanism. + * + * On return, @src_dsq is unlocked and only @p's new task_rq, which is the + * return value, is locked. + */ +static struct rq *move_task_between_dsqs(struct scx_sched *sch, + struct task_struct *p, u64 enq_flags, + struct scx_dispatch_q *src_dsq, + struct scx_dispatch_q *dst_dsq) +{ + struct rq *src_rq = task_rq(p), *dst_rq; + + BUG_ON(src_dsq->id == SCX_DSQ_LOCAL); + lockdep_assert_held(&src_dsq->lock); + lockdep_assert_rq_held(src_rq); + + if (dst_dsq->id == SCX_DSQ_LOCAL) { + dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); + if (src_rq != dst_rq && + unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { + dst_dsq = find_global_dsq(sch, task_cpu(p)); + dst_rq = src_rq; + enq_flags |= SCX_ENQ_GDSQ_FALLBACK; + } + } else { + /* no need to migrate if destination is a non-local DSQ */ + dst_rq = src_rq; + } + + /* + * Move @p into $dst_dsq. If $dst_dsq is the local DSQ of a different + * CPU, @p will be migrated. + */ + if (dst_dsq->id == SCX_DSQ_LOCAL) { + /* @p is going from a non-local DSQ to a local DSQ */ + if (src_rq == dst_rq) { + task_unlink_from_dsq(p, src_dsq); + move_local_task_to_local_dsq(sch, p, enq_flags, + src_dsq, dst_rq); + raw_spin_unlock(&src_dsq->lock); + } else { + raw_spin_unlock(&src_dsq->lock); + move_remote_task_to_local_dsq(p, enq_flags, + src_rq, dst_rq); + } + } else { + /* + * @p is going from a non-local DSQ to a non-local DSQ. As + * $src_dsq is already locked, do an abbreviated dequeue. + */ + dispatch_dequeue_locked(p, src_dsq); + raw_spin_unlock(&src_dsq->lock); + + dispatch_enqueue(sch, dst_rq, dst_dsq, p, enq_flags); + } + + return dst_rq; +} + +static bool consume_dispatch_q(struct scx_sched *sch, struct rq *rq, + struct scx_dispatch_q *dsq, u64 enq_flags) +{ + struct task_struct *p; +retry: + /* + * The caller can't expect to successfully consume a task if the task's + * addition to @dsq isn't guaranteed to be visible somehow. Test + * @dsq->list without locking and skip if it seems empty. + */ + if (list_empty(&dsq->list)) + return false; + + raw_spin_lock(&dsq->lock); + + nldsq_for_each_task(p, dsq) { + struct rq *task_rq = task_rq(p); + + /* + * This loop can lead to multiple lockup scenarios, e.g. the BPF + * scheduler can put an enormous number of affinitized tasks into + * a contended DSQ, or the outer retry loop can repeatedly race + * against scx_bypass() dequeueing tasks from @dsq trying to put + * the system into the bypass mode. This can easily live-lock the + * machine. If aborting, exit from all non-bypass DSQs. + */ + if (unlikely(READ_ONCE(sch->aborting)) && dsq->id != SCX_DSQ_BYPASS) + break; + + if (rq == task_rq) { + task_unlink_from_dsq(p, dsq); + move_local_task_to_local_dsq(sch, p, enq_flags, dsq, rq); + raw_spin_unlock(&dsq->lock); + return true; + } + + if (task_can_run_on_remote_rq(sch, p, rq, false)) { + if (likely(consume_remote_task(rq, p, enq_flags, dsq, task_rq))) + return true; + goto retry; + } + } + + raw_spin_unlock(&dsq->lock); + return false; +} + +static bool consume_global_dsq(struct scx_sched *sch, struct rq *rq) +{ + int node = cpu_to_node(cpu_of(rq)); + + return consume_dispatch_q(sch, rq, &sch->pnode[node]->global_dsq, 0); +} + +/** + * dispatch_to_local_dsq - Dispatch a task to a local dsq + * @sch: scx_sched being operated on + * @rq: current rq which is locked + * @dst_dsq: destination DSQ + * @p: task to dispatch + * @enq_flags: %SCX_ENQ_* + * + * We're holding @rq lock and want to dispatch @p to @dst_dsq which is a local + * DSQ. This function performs all the synchronization dancing needed because + * local DSQs are protected with rq locks. + * + * The caller must have exclusive ownership of @p (e.g. through + * %SCX_OPSS_DISPATCHING). + */ +static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, + struct scx_dispatch_q *dst_dsq, + struct task_struct *p, u64 enq_flags) +{ + struct rq *src_rq = task_rq(p); + struct rq *dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); + struct rq *locked_rq = rq; + + /* + * We're synchronized against dequeue through DISPATCHING. As @p can't + * be dequeued, its task_rq and cpus_allowed are stable too. + * + * If dispatching to @rq that @p is already on, no lock dancing needed. + */ + if (rq == src_rq && rq == dst_rq) { + dispatch_enqueue(sch, rq, dst_dsq, p, + enq_flags | SCX_ENQ_CLEAR_OPSS); + return; + } + + if (src_rq != dst_rq && + unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { + dispatch_enqueue(sch, rq, find_global_dsq(sch, task_cpu(p)), p, + enq_flags | SCX_ENQ_CLEAR_OPSS | SCX_ENQ_GDSQ_FALLBACK); + return; + } + + /* + * @p is on a possibly remote @src_rq which we need to lock to move the + * task. If dequeue is in progress, it'd be locking @src_rq and waiting + * on DISPATCHING, so we can't grab @src_rq lock while holding + * DISPATCHING. + * + * As DISPATCHING guarantees that @p is wholly ours, we can pretend that + * we're moving from a DSQ and use the same mechanism - mark the task + * under transfer with holding_cpu, release DISPATCHING and then follow + * the same protocol. See unlink_dsq_and_lock_src_rq(). + */ + p->scx.holding_cpu = raw_smp_processor_id(); + + /* store_release ensures that dequeue sees the above */ + atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); + + /* switch to @src_rq lock */ + if (locked_rq != src_rq) { + raw_spin_rq_unlock(locked_rq); + locked_rq = src_rq; + raw_spin_rq_lock(src_rq); + } + + /* task_rq couldn't have changed if we're still the holding cpu */ + if (likely(p->scx.holding_cpu == raw_smp_processor_id()) && + !WARN_ON_ONCE(src_rq != task_rq(p))) { + /* + * If @p is staying on the same rq, there's no need to go + * through the full deactivate/activate cycle. Optimize by + * abbreviating move_remote_task_to_local_dsq(). + */ + if (src_rq == dst_rq) { + p->scx.holding_cpu = -1; + dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p, + enq_flags); + } else { + move_remote_task_to_local_dsq(p, enq_flags, + src_rq, dst_rq); + /* task has been moved to dst_rq, which is now locked */ + locked_rq = dst_rq; + } + + /* if the destination CPU is idle, wake it up */ + if (sched_class_above(p->sched_class, dst_rq->curr->sched_class)) + resched_curr(dst_rq); + } + + /* switch back to @rq lock */ + if (locked_rq != rq) { + raw_spin_rq_unlock(locked_rq); + raw_spin_rq_lock(rq); + } +} + +/** + * finish_dispatch - Asynchronously finish dispatching a task + * @rq: current rq which is locked + * @p: task to finish dispatching + * @qseq_at_dispatch: qseq when @p started getting dispatched + * @dsq_id: destination DSQ ID + * @enq_flags: %SCX_ENQ_* + * + * Dispatching to local DSQs may need to wait for queueing to complete or + * require rq lock dancing. As we don't wanna do either while inside + * ops.dispatch() to avoid locking order inversion, we split dispatching into + * two parts. scx_bpf_dsq_insert() which is called by ops.dispatch() records the + * task and its qseq. Once ops.dispatch() returns, this function is called to + * finish up. + * + * There is no guarantee that @p is still valid for dispatching or even that it + * was valid in the first place. Make sure that the task is still owned by the + * BPF scheduler and claim the ownership before dispatching. + */ +static void finish_dispatch(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, + unsigned long qseq_at_dispatch, + u64 dsq_id, u64 enq_flags) +{ + struct scx_dispatch_q *dsq; + unsigned long opss; + + touch_core_sched_dispatch(rq, p); +retry: + /* + * No need for _acquire here. @p is accessed only after a successful + * try_cmpxchg to DISPATCHING. + */ + opss = atomic_long_read(&p->scx.ops_state); + + switch (opss & SCX_OPSS_STATE_MASK) { + case SCX_OPSS_DISPATCHING: + case SCX_OPSS_NONE: + /* someone else already got to it */ + return; + case SCX_OPSS_QUEUED: + /* + * If qseq doesn't match, @p has gone through at least one + * dispatch/dequeue and re-enqueue cycle between + * scx_bpf_dsq_insert() and here and we have no claim on it. + */ + if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch) + return; + + /* see SCX_EV_INSERT_NOT_OWNED definition */ + if (unlikely(!scx_task_on_sched(sch, p))) { + __scx_add_event(sch, SCX_EV_INSERT_NOT_OWNED, 1); + return; + } + + /* + * While we know @p is accessible, we don't yet have a claim on + * it - the BPF scheduler is allowed to dispatch tasks + * spuriously and there can be a racing dequeue attempt. Let's + * claim @p by atomically transitioning it from QUEUED to + * DISPATCHING. + */ + if (likely(atomic_long_try_cmpxchg(&p->scx.ops_state, &opss, + SCX_OPSS_DISPATCHING))) + break; + goto retry; + case SCX_OPSS_QUEUEING: + /* + * do_enqueue_task() is in the process of transferring the task + * to the BPF scheduler while holding @p's rq lock. As we aren't + * holding any kernel or BPF resource that the enqueue path may + * depend upon, it's safe to wait. + */ + wait_ops_state(p, opss); + goto retry; + } + + BUG_ON(!(p->scx.flags & SCX_TASK_QUEUED)); + + dsq = find_dsq_for_dispatch(sch, this_rq(), dsq_id, task_cpu(p)); + + if (dsq->id == SCX_DSQ_LOCAL) + dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags); + else + dispatch_enqueue(sch, rq, dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); +} + +static void flush_dispatch_buf(struct scx_sched *sch, struct rq *rq) +{ + struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; + u32 u; + + for (u = 0; u < dspc->cursor; u++) { + struct scx_dsp_buf_ent *ent = &dspc->buf[u]; + + finish_dispatch(sch, rq, ent->task, ent->qseq, ent->dsq_id, + ent->enq_flags); + } + + dspc->nr_tasks += dspc->cursor; + dspc->cursor = 0; +} + +static inline void maybe_queue_balance_callback(struct rq *rq) +{ + lockdep_assert_rq_held(rq); + + if (!(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) + return; + + queue_balance_callback(rq, &rq->scx.deferred_bal_cb, + deferred_bal_cb_workfn); + + rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; +} + +/* + * One user of this function is scx_bpf_dispatch() which can be called + * recursively as sub-sched dispatches nest. Always inline to reduce stack usage + * from the call frame. + */ +static __always_inline bool +scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, + struct task_struct *prev, bool nested) +{ + struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; + int nr_loops = SCX_DSP_MAX_LOOPS; + s32 cpu = cpu_of(rq); + bool prev_on_sch = (prev->sched_class == &ext_sched_class) && + scx_task_on_sched(sch, prev); + + if (consume_global_dsq(sch, rq)) + return true; + + if (bypass_dsp_enabled(sch)) { + /* if @sch is bypassing, only the bypass DSQs are active */ + if (scx_bypassing(sch, cpu)) + return consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0); + +#ifdef CONFIG_EXT_SUB_SCHED + /* + * If @sch isn't bypassing but its children are, @sch is + * responsible for making forward progress for both its own + * tasks that aren't bypassing and the bypassing descendants' + * tasks. The following implements a simple built-in behavior - + * let each CPU try to run the bypass DSQ every Nth time. + * + * Later, if necessary, we can add an ops flag to suppress the + * auto-consumption and a kfunc to consume the bypass DSQ and, + * so that the BPF scheduler can fully control scheduling of + * bypassed tasks. + */ + struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); + + if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) && + consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0)) { + __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); + return true; + } +#endif /* CONFIG_EXT_SUB_SCHED */ + } + + if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq)) + return false; + + dspc->rq = rq; + + /* + * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, + * the local DSQ might still end up empty after a successful + * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() + * produced some tasks, retry. The BPF scheduler may depend on this + * looping behavior to simplify its implementation. + */ + do { + dspc->nr_tasks = 0; + + if (nested) { + SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), + prev_on_sch ? prev : NULL); + } else { + /* stash @prev so that nested invocations can access it */ + rq->scx.sub_dispatch_prev = prev; + SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), + prev_on_sch ? prev : NULL); + rq->scx.sub_dispatch_prev = NULL; + } + + flush_dispatch_buf(sch, rq); + + if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) { + rq->scx.flags |= SCX_RQ_BAL_KEEP; + return true; + } + if (rq->scx.local_dsq.nr) + return true; + if (consume_global_dsq(sch, rq)) + return true; + + /* + * ops.dispatch() can trap us in this loop by repeatedly + * dispatching ineligible tasks. Break out once in a while to + * allow the watchdog to run. As IRQ can't be enabled in + * balance(), we want to complete this scheduling cycle and then + * start a new one. IOW, we want to call resched_curr() on the + * next, most likely idle, task, not the current one. Use + * __scx_bpf_kick_cpu() for deferred kicking. + */ + if (unlikely(!--nr_loops)) { + scx_kick_cpu(sch, cpu, 0); + break; + } + } while (dspc->nr_tasks); + + /* + * Prevent the CPU from going idle while bypassed descendants have tasks + * queued. Without this fallback, bypassed tasks could stall if the host + * scheduler's ops.dispatch() doesn't yield any tasks. + */ + if (bypass_dsp_enabled(sch)) + return consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0); + + return false; +} + +static int balance_one(struct rq *rq, struct task_struct *prev) +{ + struct scx_sched *sch = scx_root; + s32 cpu = cpu_of(rq); + + lockdep_assert_rq_held(rq); + rq->scx.flags |= SCX_RQ_IN_BALANCE; + rq->scx.flags &= ~SCX_RQ_BAL_KEEP; + + if ((sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) && + unlikely(rq->scx.cpu_released)) { + /* + * If the previous sched_class for the current CPU was not SCX, + * notify the BPF scheduler that it again has control of the + * core. This callback complements ->cpu_release(), which is + * emitted in switch_class(). + */ + if (sch->ops.cpu_acquire) + SCX_CALL_OP(sch, cpu_acquire, rq, cpu, NULL); + rq->scx.cpu_released = false; + } + + if (prev->sched_class == &ext_sched_class) { + update_curr_scx(rq); + + /* + * If @prev is runnable & has slice left, it has priority and + * fetching more just increases latency for the fetched tasks. + * Tell pick_task_scx() to keep running @prev. If the BPF + * scheduler wants to handle this explicitly, it should + * implement ->cpu_release(). + * + * See scx_disable_workfn() for the explanation on the bypassing + * test. + */ + if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice && + !scx_bypassing(sch, cpu)) { + rq->scx.flags |= SCX_RQ_BAL_KEEP; + goto has_tasks; + } + } + + /* if there already are tasks to run, nothing to do */ + if (rq->scx.local_dsq.nr) + goto has_tasks; + + if (scx_dispatch_sched(sch, rq, prev, false)) + goto has_tasks; + + /* + * Didn't find another task to run. Keep running @prev unless + * %SCX_OPS_ENQ_LAST is in effect. + */ + if ((prev->scx.flags & SCX_TASK_QUEUED) && + (!(sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(sch, cpu))) { + rq->scx.flags |= SCX_RQ_BAL_KEEP; + __scx_add_event(sch, SCX_EV_DISPATCH_KEEP_LAST, 1); + goto has_tasks; + } + rq->scx.flags &= ~SCX_RQ_IN_BALANCE; + return false; + +has_tasks: + /* + * @rq may have extra IMMED tasks without reenq scheduled: + * + * - rq_is_open() can't reliably tell when and how slice is going to be + * modified for $curr and allows IMMED tasks to be queued while + * dispatch is in progress. + * + * - A non-IMMED HEAD task can get queued in front of an IMMED task + * between the IMMED queueing and the subsequent scheduling event. + */ + if (unlikely(rq->scx.local_dsq.nr > 1 && rq->scx.nr_immed)) + schedule_reenq_local(rq, 0); + + rq->scx.flags &= ~SCX_RQ_IN_BALANCE; + return true; +} + +static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) +{ + struct scx_sched *sch = scx_task_sched(p); + + if (p->scx.flags & SCX_TASK_QUEUED) { + /* + * Core-sched might decide to execute @p before it is + * dispatched. Call ops_dequeue() to notify the BPF scheduler. + */ + ops_dequeue(rq, p, SCX_DEQ_CORE_SCHED_EXEC); + dispatch_dequeue(rq, p); + } + + p->se.exec_start = rq_clock_task(rq); + + /* see dequeue_task_scx() on why we skip when !QUEUED */ + if (SCX_HAS_OP(sch, running) && (p->scx.flags & SCX_TASK_QUEUED)) + SCX_CALL_OP_TASK(sch, running, rq, p); + + clr_task_runnable(p, true); + + /* + * @p is getting newly scheduled or got kicked after someone updated its + * slice. Refresh whether tick can be stopped. See scx_can_stop_tick(). + */ + if ((p->scx.slice == SCX_SLICE_INF) != + (bool)(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) { + if (p->scx.slice == SCX_SLICE_INF) + rq->scx.flags |= SCX_RQ_CAN_STOP_TICK; + else + rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK; + + sched_update_tick_dependency(rq); + + /* + * For now, let's refresh the load_avgs just when transitioning + * in and out of nohz. In the future, we might want to add a + * mechanism which calls the following periodically on + * tick-stopped CPUs. + */ + update_other_load_avgs(rq); + } +} + +static enum scx_cpu_preempt_reason +preempt_reason_from_class(const struct sched_class *class) +{ + if (class == &stop_sched_class) + return SCX_CPU_PREEMPT_STOP; + if (class == &dl_sched_class) + return SCX_CPU_PREEMPT_DL; + if (class == &rt_sched_class) + return SCX_CPU_PREEMPT_RT; + return SCX_CPU_PREEMPT_UNKNOWN; +} + +static void switch_class(struct rq *rq, struct task_struct *next) +{ + struct scx_sched *sch = scx_root; + const struct sched_class *next_class = next->sched_class; + + if (!(sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT)) + return; + + /* + * The callback is conceptually meant to convey that the CPU is no + * longer under the control of SCX. Therefore, don't invoke the callback + * if the next class is below SCX (in which case the BPF scheduler has + * actively decided not to schedule any tasks on the CPU). + */ + if (sched_class_above(&ext_sched_class, next_class)) + return; + + /* + * At this point we know that SCX was preempted by a higher priority + * sched_class, so invoke the ->cpu_release() callback if we have not + * done so already. We only send the callback once between SCX being + * preempted, and it regaining control of the CPU. + * + * ->cpu_release() complements ->cpu_acquire(), which is emitted the + * next time that balance_one() is invoked. + */ + if (!rq->scx.cpu_released) { + if (sch->ops.cpu_release) { + struct scx_cpu_release_args args = { + .reason = preempt_reason_from_class(next_class), + .task = next, + }; + + SCX_CALL_OP(sch, cpu_release, rq, cpu_of(rq), &args); + } + rq->scx.cpu_released = true; + } +} + +static void put_prev_task_scx(struct rq *rq, struct task_struct *p, + struct task_struct *next) +{ + struct scx_sched *sch = scx_task_sched(p); + + /* see kick_sync_wait_bal_cb() */ + smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); + + update_curr_scx(rq); + + /* see dequeue_task_scx() on why we skip when !QUEUED */ + if (SCX_HAS_OP(sch, stopping) && (p->scx.flags & SCX_TASK_QUEUED)) + SCX_CALL_OP_TASK(sch, stopping, rq, p, true); + + if (p->scx.flags & SCX_TASK_QUEUED) { + set_task_runnable(rq, p); + + /* + * If @p has slice left and is being put, @p is getting + * preempted by a higher priority scheduler class or core-sched + * forcing a different task. Leave it at the head of the local + * DSQ unless it was an IMMED task. IMMED tasks should not + * linger on a busy CPU, reenqueue them to the BPF scheduler. + */ + if (p->scx.slice && !scx_bypassing(sch, cpu_of(rq))) { + if (p->scx.flags & SCX_TASK_IMMED) { + p->scx.flags |= SCX_TASK_REENQ_PREEMPTED; + do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + } else { + dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, SCX_ENQ_HEAD); + } + goto switch_class; + } + + /* + * If @p is runnable but we're about to enter a lower + * sched_class, %SCX_OPS_ENQ_LAST must be set. Tell + * ops.enqueue() that @p is the only one available for this cpu, + * which should trigger an explicit follow-up scheduling event. + */ + if (next && sched_class_above(&ext_sched_class, next->sched_class)) { + WARN_ON_ONCE(!(sch->ops.flags & SCX_OPS_ENQ_LAST)); + do_enqueue_task(rq, p, SCX_ENQ_LAST, -1); + } else { + do_enqueue_task(rq, p, 0, -1); + } + } + +switch_class: + if (next && next->sched_class != &ext_sched_class) + switch_class(rq, next); +} + +static void kick_sync_wait_bal_cb(struct rq *rq) +{ + struct scx_kick_syncs __rcu *ks = __this_cpu_read(scx_kick_syncs); + unsigned long *ksyncs = rcu_dereference_sched(ks)->syncs; + bool waited; + s32 cpu; + + /* + * Drop rq lock and enable IRQs while waiting. IRQs must be enabled + * — a target CPU may be waiting for us to process an IPI (e.g. TLB + * flush) while we wait for its kick_sync to advance. + * + * Also, keep advancing our own kick_sync so that new kick_sync waits + * targeting us, which can start after we drop the lock, cannot form + * cyclic dependencies. + */ +retry: + waited = false; + for_each_cpu(cpu, rq->scx.cpus_to_sync) { + /* + * smp_load_acquire() pairs with smp_store_release() on + * kick_sync updates on the target CPUs. + */ + if (cpu == cpu_of(rq) || + smp_load_acquire(&cpu_rq(cpu)->scx.kick_sync) != ksyncs[cpu]) { + cpumask_clear_cpu(cpu, rq->scx.cpus_to_sync); + continue; + } + + raw_spin_rq_unlock_irq(rq); + while (READ_ONCE(cpu_rq(cpu)->scx.kick_sync) == ksyncs[cpu]) { + smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); + cpu_relax(); + } + raw_spin_rq_lock_irq(rq); + waited = true; + } + + if (waited) + goto retry; +} + +static struct task_struct *first_local_task(struct rq *rq) +{ + return list_first_entry_or_null(&rq->scx.local_dsq.list, + struct task_struct, scx.dsq_list.node); +} + +static struct task_struct * +do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) +{ + struct task_struct *prev = rq->curr; + bool keep_prev; + struct task_struct *p; + + /* see kick_sync_wait_bal_cb() */ + smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); + + rq_modified_begin(rq, &ext_sched_class); + + rq_unpin_lock(rq, rf); + balance_one(rq, prev); + rq_repin_lock(rq, rf); + maybe_queue_balance_callback(rq); + + /* + * Defer to a balance callback which can drop rq lock and enable + * IRQs. Waiting directly in the pick path would deadlock against + * CPUs sending us IPIs (e.g. TLB flushes) while we wait for them. + */ + if (unlikely(rq->scx.kick_sync_pending)) { + rq->scx.kick_sync_pending = false; + queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, + kick_sync_wait_bal_cb); + } + + /* + * If any higher-priority sched class enqueued a runnable task on + * this rq during balance_one(), abort and return RETRY_TASK, so + * that the scheduler loop can restart. + * + * If @force_scx is true, always try to pick a SCHED_EXT task, + * regardless of any higher-priority sched classes activity. + */ + if (!force_scx && rq_modified_above(rq, &ext_sched_class)) + return RETRY_TASK; + + keep_prev = rq->scx.flags & SCX_RQ_BAL_KEEP; + if (unlikely(keep_prev && + prev->sched_class != &ext_sched_class)) { + WARN_ON_ONCE(scx_enable_state() == SCX_ENABLED); + keep_prev = false; + } + + /* + * If balance_one() is telling us to keep running @prev, replenish slice + * if necessary and keep running @prev. Otherwise, pop the first one + * from the local DSQ. + */ + if (keep_prev) { + p = prev; + if (!p->scx.slice) + refill_task_slice_dfl(scx_task_sched(p), p); + } else { + p = first_local_task(rq); + if (!p) + return NULL; + + if (unlikely(!p->scx.slice)) { + struct scx_sched *sch = scx_task_sched(p); + + if (!scx_bypassing(sch, cpu_of(rq)) && + !sch->warned_zero_slice) { + printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in %s()\n", + p->comm, p->pid, __func__); + sch->warned_zero_slice = true; + } + refill_task_slice_dfl(sch, p); + } + } + + return p; +} + +static struct task_struct *pick_task_scx(struct rq *rq, struct rq_flags *rf) +{ + return do_pick_task_scx(rq, rf, false); +} + +/* + * Select the next task to run from the ext scheduling class. + * + * Use do_pick_task_scx() directly with @force_scx enabled, since the + * dl_server must always select a sched_ext task. + */ +static struct task_struct * +ext_server_pick_task(struct sched_dl_entity *dl_se, struct rq_flags *rf) +{ + if (!scx_enabled()) + return NULL; + + return do_pick_task_scx(dl_se->rq, rf, true); +} + +/* + * Initialize the ext server deadline entity. + */ +void ext_server_init(struct rq *rq) +{ + struct sched_dl_entity *dl_se = &rq->ext_server; + + init_dl_entity(dl_se); + + dl_server_init(dl_se, rq, ext_server_pick_task); +} + +#ifdef CONFIG_SCHED_CORE +/** + * scx_prio_less - Task ordering for core-sched + * @a: task A + * @b: task B + * @in_fi: in forced idle state + * + * Core-sched is implemented as an additional scheduling layer on top of the + * usual sched_class'es and needs to find out the expected task ordering. For + * SCX, core-sched calls this function to interrogate the task ordering. + * + * Unless overridden by ops.core_sched_before(), @p->scx.core_sched_at is used + * to implement the default task ordering. The older the timestamp, the higher + * priority the task - the global FIFO ordering matching the default scheduling + * behavior. + * + * When ops.core_sched_before() is enabled, @p->scx.core_sched_at is used to + * implement FIFO ordering within each local DSQ. See pick_task_scx(). + */ +bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, + bool in_fi) +{ + struct scx_sched *sch_a = scx_task_sched(a); + struct scx_sched *sch_b = scx_task_sched(b); + + /* + * The const qualifiers are dropped from task_struct pointers when + * calling ops.core_sched_before(). Accesses are controlled by the + * verifier. + */ + if (sch_a == sch_b && SCX_HAS_OP(sch_a, core_sched_before) && + !scx_bypassing(sch_a, task_cpu(a))) + return SCX_CALL_OP_2TASKS_RET(sch_a, core_sched_before, + task_rq(a), + (struct task_struct *)a, + (struct task_struct *)b); + else + return time_after64(a->scx.core_sched_at, b->scx.core_sched_at); +} +#endif /* CONFIG_SCHED_CORE */ + +static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags) +{ + struct scx_sched *sch = scx_task_sched(p); + bool bypassing; + + /* + * sched_exec() calls with %WF_EXEC when @p is about to exec(2) as it + * can be a good migration opportunity with low cache and memory + * footprint. Returning a CPU different than @prev_cpu triggers + * immediate rq migration. However, for SCX, as the current rq + * association doesn't dictate where the task is going to run, this + * doesn't fit well. If necessary, we can later add a dedicated method + * which can decide to preempt self to force it through the regular + * scheduling path. + */ + if (unlikely(wake_flags & WF_EXEC)) + return prev_cpu; + + bypassing = scx_bypassing(sch, task_cpu(p)); + if (likely(SCX_HAS_OP(sch, select_cpu)) && !bypassing) { + s32 cpu; + struct task_struct **ddsp_taskp; + + ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); + WARN_ON_ONCE(*ddsp_taskp); + *ddsp_taskp = p; + + this_rq()->scx.in_select_cpu = true; + cpu = SCX_CALL_OP_TASK_RET(sch, select_cpu, NULL, p, + scx_cpu_arg(prev_cpu), wake_flags); + cpu = scx_cpu_ret(sch, cpu); + this_rq()->scx.in_select_cpu = false; + p->scx.selected_cpu = cpu; + *ddsp_taskp = NULL; + if (scx_cpu_valid(sch, cpu, "from ops.select_cpu()")) + return cpu; + else + return prev_cpu; + } else { + s32 cpu; + + cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, NULL, 0); + if (cpu >= 0) { + refill_task_slice_dfl(sch, p); + p->scx.ddsp_dsq_id = SCX_DSQ_LOCAL; + } else { + cpu = prev_cpu; + } + p->scx.selected_cpu = cpu; + + if (bypassing) + __scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1); + return cpu; + } +} + +static void task_woken_scx(struct rq *rq, struct task_struct *p) +{ + run_deferred(rq); +} + +static void set_cpus_allowed_scx(struct task_struct *p, + struct affinity_context *ac) +{ + struct scx_sched *sch = scx_task_sched(p); + + set_cpus_allowed_common(p, ac); + + if (task_dead_and_done(p)) + return; + + /* + * The effective cpumask is stored in @p->cpus_ptr which may temporarily + * differ from the configured one in @p->cpus_mask. Always tell the bpf + * scheduler the effective one. + * + * Fine-grained memory write control is enforced by BPF making the const + * designation pointless. Cast it away when calling the operation. + */ + if (SCX_HAS_OP(sch, set_cpumask)) + scx_call_op_set_cpumask(sch, task_rq(p), p, (struct cpumask *)p->cpus_ptr); +} + +static void handle_hotplug(struct rq *rq, bool online) +{ + struct scx_sched *sch = scx_root; + s32 cpu = cpu_of(rq); + + atomic_long_inc(&scx_hotplug_seq); + + /* + * scx_root updates are protected by cpus_read_lock() and will stay + * stable here. Note that we can't depend on scx_enabled() test as the + * hotplug ops need to be enabled before __scx_enabled is set. + */ + if (unlikely(!sch)) + return; + + if (scx_enabled()) + scx_idle_update_selcpu_topology(&sch->ops); + + if (online && SCX_HAS_OP(sch, cpu_online)) + SCX_CALL_OP(sch, cpu_online, NULL, scx_cpu_arg(cpu)); + else if (!online && SCX_HAS_OP(sch, cpu_offline)) + SCX_CALL_OP(sch, cpu_offline, NULL, scx_cpu_arg(cpu)); + else + scx_exit(sch, SCX_EXIT_UNREG_KERN, + SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG, + "cpu %d going %s, exiting scheduler", cpu, + online ? "online" : "offline"); +} + +void scx_rq_activate(struct rq *rq) +{ + handle_hotplug(rq, true); +} + +void scx_rq_deactivate(struct rq *rq) +{ + handle_hotplug(rq, false); +} + +static void rq_online_scx(struct rq *rq) +{ + rq->scx.flags |= SCX_RQ_ONLINE; +} + +static void rq_offline_scx(struct rq *rq) +{ + rq->scx.flags &= ~SCX_RQ_ONLINE; +} + +static bool check_rq_for_timeouts(struct rq *rq) +{ + struct scx_sched *sch; + struct task_struct *p; + struct rq_flags rf; + bool timed_out = false; + + rq_lock_irqsave(rq, &rf); + sch = rcu_dereference_bh(scx_root); + if (unlikely(!sch)) + goto out_unlock; + + list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) { + struct scx_sched *sch = scx_task_sched(p); + unsigned long last_runnable = p->scx.runnable_at; + + if (unlikely(time_after(jiffies, + last_runnable + READ_ONCE(sch->watchdog_timeout)))) { + u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); + + __scx_exit(sch, SCX_EXIT_ERROR_STALL, 0, cpu_of(rq), + "%s[%d] failed to run for %u.%03us", + p->comm, p->pid, dur_ms / 1000, + dur_ms % 1000); + timed_out = true; + break; + } + } +out_unlock: + rq_unlock_irqrestore(rq, &rf); + return timed_out; +} + +static void scx_watchdog_workfn(struct work_struct *work) +{ + unsigned long intv; + int cpu; + + WRITE_ONCE(scx_watchdog_timestamp, jiffies); + + for_each_online_cpu(cpu) { + if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) + break; + + cond_resched(); + } + + intv = READ_ONCE(scx_watchdog_interval); + if (intv < ULONG_MAX) + queue_delayed_work(system_dfl_wq, to_delayed_work(work), intv); +} + +void scx_tick(struct rq *rq) +{ + struct scx_sched *root; + unsigned long last_check; + + if (!scx_enabled()) + return; + + root = rcu_dereference_bh(scx_root); + if (unlikely(!root)) + return; + + last_check = READ_ONCE(scx_watchdog_timestamp); + if (unlikely(time_after(jiffies, + last_check + READ_ONCE(root->watchdog_timeout)))) { + u32 dur_ms = jiffies_to_msecs(jiffies - last_check); + + scx_exit(root, SCX_EXIT_ERROR_STALL, 0, + "watchdog failed to check in for %u.%03us", + dur_ms / 1000, dur_ms % 1000); + } + + update_other_load_avgs(rq); +} + +static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) +{ + struct scx_sched *sch = scx_task_sched(curr); + + update_curr_scx(rq); + + /* + * While disabling, always resched and refresh core-sched timestamp as + * we can't trust the slice management or ops.core_sched_before(). + */ + if (scx_bypassing(sch, cpu_of(rq))) { + curr->scx.slice = 0; + touch_core_sched(rq, curr); + } else if (SCX_HAS_OP(sch, tick)) { + SCX_CALL_OP_TASK(sch, tick, rq, curr); + } + + if (!curr->scx.slice) + resched_curr(rq); +} + +#ifdef CONFIG_EXT_GROUP_SCHED +static struct cgroup *tg_cgrp(struct task_group *tg) +{ + /* + * If CGROUP_SCHED is disabled, @tg is NULL. If @tg is an autogroup, + * @tg->css.cgroup is NULL. In both cases, @tg can be treated as the + * root cgroup. + */ + if (tg && tg->css.cgroup) + return tg->css.cgroup; + else + return &cgrp_dfl_root.cgrp; +} + +#define SCX_INIT_TASK_ARGS_CGROUP(tg) .cgroup = tg_cgrp(tg), + +#else /* CONFIG_EXT_GROUP_SCHED */ + +#define SCX_INIT_TASK_ARGS_CGROUP(tg) + +#endif /* CONFIG_EXT_GROUP_SCHED */ + +static int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fork) +{ + int ret; + + p->scx.disallow = false; + + if (SCX_HAS_OP(sch, init_task)) { + struct scx_init_task_args args = { + SCX_INIT_TASK_ARGS_CGROUP(task_group(p)) + .fork = fork, + }; + + ret = SCX_CALL_OP_RET(sch, init_task, NULL, p, &args); + if (unlikely(ret)) { + ret = ops_sanitize_err(sch, "init_task", ret); + return ret; + } + } + + if (p->scx.disallow) { + if (unlikely(scx_parent(sch))) { + scx_error(sch, "non-root ops.init_task() set task->scx.disallow for %s[%d]", + p->comm, p->pid); + } else if (unlikely(fork)) { + scx_error(sch, "ops.init_task() set task->scx.disallow for %s[%d] during fork", + p->comm, p->pid); + } else { + struct rq *rq; + struct rq_flags rf; + + rq = task_rq_lock(p, &rf); + + /* + * We're in the load path and @p->policy will be applied + * right after. Reverting @p->policy here and rejecting + * %SCHED_EXT transitions from scx_check_setscheduler() + * guarantees that if ops.init_task() sets @p->disallow, + * @p can never be in SCX. + */ + if (p->policy == SCHED_EXT) { + p->policy = SCHED_NORMAL; + atomic_long_inc(&scx_nr_rejected); + } + + task_rq_unlock(rq, p, &rf); + } + } + + return 0; +} + +static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p) +{ + struct rq *rq = task_rq(p); + u32 weight; + + lockdep_assert_rq_held(rq); + + /* + * Verify the task is not in BPF scheduler's custody. If flag + * transitions are consistent, the flag should always be clear + * here. + */ + WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY); + + /* + * Set the weight before calling ops.enable() so that the scheduler + * doesn't see a stale value if they inspect the task struct. + */ + if (task_has_idle_policy(p)) + weight = WEIGHT_IDLEPRIO; + else + weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; + + p->scx.weight = sched_weight_to_cgroup(weight); + + if (SCX_HAS_OP(sch, enable)) + SCX_CALL_OP_TASK(sch, enable, rq, p); + + if (SCX_HAS_OP(sch, set_weight)) + SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight); +} + +static void scx_enable_task(struct scx_sched *sch, struct task_struct *p) +{ + __scx_enable_task(sch, p); + scx_set_task_state(p, SCX_TASK_ENABLED); +} + +static void scx_disable_task(struct scx_sched *sch, struct task_struct *p) +{ + struct rq *rq = task_rq(p); + + lockdep_assert_rq_held(rq); + WARN_ON_ONCE(scx_get_task_state(p) != SCX_TASK_ENABLED); + + clear_direct_dispatch(p); + + if (SCX_HAS_OP(sch, disable)) + SCX_CALL_OP_TASK(sch, disable, rq, p); + scx_set_task_state(p, SCX_TASK_READY); + + /* + * Verify the task is not in BPF scheduler's custody. If flag + * transitions are consistent, the flag should always be clear + * here. + */ + WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY); +} + +static void __scx_disable_and_exit_task(struct scx_sched *sch, + struct task_struct *p) +{ + struct scx_exit_task_args args = { + .cancelled = false, + }; + + lockdep_assert_held(&p->pi_lock); + lockdep_assert_rq_held(task_rq(p)); + + switch (scx_get_task_state(p)) { + case SCX_TASK_NONE: + return; + case SCX_TASK_INIT: + args.cancelled = true; + break; + case SCX_TASK_READY: + break; + case SCX_TASK_ENABLED: + scx_disable_task(sch, p); + break; + default: + WARN_ON_ONCE(true); + return; + } + + if (SCX_HAS_OP(sch, exit_task)) + SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args); +} + +/* + * Undo a completed __scx_init_task(sch, p, false) when scx_enable_task() never + * ran. The task state has not been transitioned, so this mirrors the + * SCX_TASK_INIT branch in __scx_disable_and_exit_task(). + */ +static void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct *p) +{ + struct scx_exit_task_args args = { .cancelled = true }; + + lockdep_assert_held(&p->pi_lock); + lockdep_assert_rq_held(task_rq(p)); + + if (SCX_HAS_OP(sch, exit_task)) + SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args); +} + +static void scx_disable_and_exit_task(struct scx_sched *sch, + struct task_struct *p) +{ + __scx_disable_and_exit_task(sch, p); + + /* + * If set, @p exited between __scx_init_task() and scx_enable_task() in + * scx_sub_enable() and is initialized for both the associated sched and + * its parent. Exit for the child too - scx_enable_task() never ran for + * it, so undo only init_task. The flag is only set on the sub-enable + * path, so it's always clear when @p arrives here in %SCX_TASK_NONE. + */ + if (p->scx.flags & SCX_TASK_SUB_INIT) { + if (!WARN_ON_ONCE(!scx_enabling_sub_sched)) + scx_sub_init_cancel_task(scx_enabling_sub_sched, p); + p->scx.flags &= ~SCX_TASK_SUB_INIT; + } + + scx_set_task_sched(p, NULL); + scx_set_task_state(p, SCX_TASK_NONE); +} + +void init_scx_entity(struct sched_ext_entity *scx) +{ + memset(scx, 0, sizeof(*scx)); + INIT_LIST_HEAD(&scx->dsq_list.node); + RB_CLEAR_NODE(&scx->dsq_priq); + scx->sticky_cpu = -1; + scx->holding_cpu = -1; + INIT_LIST_HEAD(&scx->runnable_node); + scx->runnable_at = jiffies; + scx->ddsp_dsq_id = SCX_DSQ_INVALID; + scx->slice = SCX_SLICE_DFL; +} + +/* See scx_tid_alloc / scx_tid_cursor. */ +static u64 scx_alloc_tid(void) +{ + struct scx_tid_alloc *ta; + + guard(preempt)(); + ta = this_cpu_ptr(&scx_tid_alloc); + + if (unlikely(ta->next >= ta->end)) { + ta->next = atomic64_fetch_add(SCX_TID_CHUNK, &scx_tid_cursor); + ta->end = ta->next + SCX_TID_CHUNK; + } + return ta->next++; +} + +static void scx_tid_hash_insert(struct task_struct *p) +{ + int ret; + + lockdep_assert_held(&scx_tasks_lock); + + ret = rhashtable_lookup_insert_fast(&scx_tid_hash, + &p->scx.tid_hash_node, + scx_tid_hash_params); + WARN_ON_ONCE(ret); +} + +void scx_pre_fork(struct task_struct *p) +{ + /* + * BPF scheduler enable/disable paths want to be able to iterate and + * update all tasks which can become complex when racing forks. As + * enable/disable are very cold paths, let's use a percpu_rwsem to + * exclude forks. + */ + percpu_down_read(&scx_fork_rwsem); +} + +int scx_fork(struct task_struct *p, struct kernel_clone_args *kargs) +{ + s32 ret; + + percpu_rwsem_assert_held(&scx_fork_rwsem); + + p->scx.tid = scx_alloc_tid(); + + if (scx_init_task_enabled) { +#ifdef CONFIG_EXT_SUB_SCHED + struct scx_sched *sch = kargs->cset->dfl_cgrp->scx_sched; +#else + struct scx_sched *sch = scx_root; +#endif + scx_set_task_state(p, SCX_TASK_INIT_BEGIN); + ret = __scx_init_task(sch, p, true); + if (unlikely(ret)) { + scx_set_task_state(p, SCX_TASK_NONE); + return ret; + } + scx_set_task_state(p, SCX_TASK_INIT); + scx_set_task_sched(p, sch); + } + + return 0; +} + +void scx_post_fork(struct task_struct *p) +{ + if (scx_init_task_enabled) { + scx_set_task_state(p, SCX_TASK_READY); + + /* + * Enable the task immediately if it's running on sched_ext. + * Otherwise, it'll be enabled in switching_to_scx() if and + * when it's ever configured to run with a SCHED_EXT policy. + */ + if (p->sched_class == &ext_sched_class) { + struct rq_flags rf; + struct rq *rq; + + rq = task_rq_lock(p, &rf); + scx_enable_task(scx_task_sched(p), p); + task_rq_unlock(rq, p, &rf); + } + } + + scoped_guard(raw_spinlock_irq, &scx_tasks_lock) { + list_add_tail(&p->scx.tasks_node, &scx_tasks); + if (scx_tid_to_task_enabled()) + scx_tid_hash_insert(p); + } + + percpu_up_read(&scx_fork_rwsem); +} + +void scx_cancel_fork(struct task_struct *p) +{ + if (scx_enabled()) { + struct rq *rq; + struct rq_flags rf; + + rq = task_rq_lock(p, &rf); + WARN_ON_ONCE(scx_get_task_state(p) >= SCX_TASK_READY); + scx_disable_and_exit_task(scx_task_sched(p), p); + task_rq_unlock(rq, p, &rf); + } + + percpu_up_read(&scx_fork_rwsem); +} + +/** + * task_dead_and_done - Is a task dead and done running? + * @p: target task + * + * Once sched_ext_dead() removes the dead task from scx_tasks and exits it, the + * task no longer exists from SCX's POV. However, certain sched_class ops may be + * invoked on these dead tasks leading to failures - e.g. sched_setscheduler() + * may try to switch a task which finished sched_ext_dead() back into SCX + * triggering invalid SCX task state transitions and worse. + * + * Once a task has finished the final switch, sched_ext_dead() is the only thing + * that needs to happen on the task. Use this test to short-circuit sched_class + * operations which may be called on dead tasks. + */ +static bool task_dead_and_done(struct task_struct *p) +{ + struct rq *rq = task_rq(p); + + lockdep_assert_rq_held(rq); + + /* + * In do_task_dead(), a dying task sets %TASK_DEAD with preemption + * disabled and __schedule(). If @p has %TASK_DEAD set and off CPU, @p + * won't ever run again. + */ + return unlikely(READ_ONCE(p->__state) == TASK_DEAD) && + !task_on_cpu(rq, p); +} + +void sched_ext_dead(struct task_struct *p) +{ + /* + * By the time control reaches here, @p has %TASK_DEAD set, switched out + * for the last time and then dropped the rq lock - task_dead_and_done() + * should be returning %true nullifying the straggling sched_class ops. + * Remove from scx_tasks and exit @p. + */ + scoped_guard(raw_spinlock_irqsave, &scx_tasks_lock) { + list_del_init(&p->scx.tasks_node); + if (scx_tid_to_task_enabled()) + rhashtable_remove_fast(&scx_tid_hash, + &p->scx.tid_hash_node, + scx_tid_hash_params); + } + + /* + * @p is off scx_tasks and wholly ours. scx_root_enable()'s READY -> + * ENABLED transitions can't race us. Disable ops for @p. + * + * %SCX_TASK_DEAD synchronizes against cgroup task iteration - see + * scx_task_iter_next_locked(). NONE tasks need no marking: cgroup + * iteration is only used from sub-sched paths, which require root + * enabled. Root enable transitions every live task to at least READY. + * + * %INIT_BEGIN means ops.init_task() is running for @p. Don't call + * into ops; transition to %DEAD so the post-init recheck unwinds + * via scx_sub_init_cancel_task(). + */ + if (scx_get_task_state(p) != SCX_TASK_NONE) { + struct rq_flags rf; + struct rq *rq; + + rq = task_rq_lock(p, &rf); + if (scx_get_task_state(p) != SCX_TASK_INIT_BEGIN) + scx_disable_and_exit_task(scx_task_sched(p), p); + scx_set_task_state(p, SCX_TASK_DEAD); + task_rq_unlock(rq, p, &rf); + } +} + +static void reweight_task_scx(struct rq *rq, struct task_struct *p, + const struct load_weight *lw) +{ + struct scx_sched *sch = scx_task_sched(p); + + lockdep_assert_rq_held(task_rq(p)); + + if (task_dead_and_done(p)) + return; + + p->scx.weight = sched_weight_to_cgroup(scale_load_down(lw->weight)); + if (SCX_HAS_OP(sch, set_weight)) + SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight); +} + +static void prio_changed_scx(struct rq *rq, struct task_struct *p, u64 oldprio) +{ +} + +static void switching_to_scx(struct rq *rq, struct task_struct *p) +{ + struct scx_sched *sch = scx_task_sched(p); + + if (task_dead_and_done(p)) + return; + + scx_enable_task(sch, p); + + /* + * set_cpus_allowed_scx() is not called while @p is associated with a + * different scheduler class. Keep the BPF scheduler up-to-date. + */ + if (SCX_HAS_OP(sch, set_cpumask)) + scx_call_op_set_cpumask(sch, rq, p, (struct cpumask *)p->cpus_ptr); +} + +static void switched_from_scx(struct rq *rq, struct task_struct *p) +{ + if (task_dead_and_done(p)) + return; + + /* + * %NONE means SCX is no longer tracking @p at the task level (e.g. + * scx_fail_parent() handed @p back to the parent at NONE pending the + * parent's own teardown). There is nothing to disable; calling + * scx_disable_task() would WARN on the non-%ENABLED state and trigger a + * NONE -> READY validation failure. + */ + if (scx_get_task_state(p) == SCX_TASK_NONE) + return; + + scx_disable_task(scx_task_sched(p), p); +} + +static void switched_to_scx(struct rq *rq, struct task_struct *p) {} + +int scx_check_setscheduler(struct task_struct *p, int policy) +{ + lockdep_assert_rq_held(task_rq(p)); + + /* if disallow, reject transitioning into SCX */ + if (scx_enabled() && READ_ONCE(p->scx.disallow) && + p->policy != policy && policy == SCHED_EXT) + return -EACCES; + + return 0; +} + +static void process_ddsp_deferred_locals(struct rq *rq) +{ + struct task_struct *p; + + lockdep_assert_rq_held(rq); + + /* + * Now that @rq can be unlocked, execute the deferred enqueueing of + * tasks directly dispatched to the local DSQs of other CPUs. See + * direct_dispatch(). Keep popping from the head instead of using + * list_for_each_entry_safe() as dispatch_local_dsq() may unlock @rq + * temporarily. + */ + while ((p = list_first_entry_or_null(&rq->scx.ddsp_deferred_locals, + struct task_struct, scx.dsq_list.node))) { + struct scx_sched *sch = scx_task_sched(p); + struct scx_dispatch_q *dsq; + u64 dsq_id = p->scx.ddsp_dsq_id; + u64 enq_flags = p->scx.ddsp_enq_flags; + + list_del_init(&p->scx.dsq_list.node); + clear_direct_dispatch(p); + + dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p)); + if (!WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL)) + dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags); + } +} + +/* + * Determine whether @p should be reenqueued from a local DSQ. + * + * @reenq_flags is mutable and accumulates state across the DSQ walk: + * + * - %SCX_REENQ_TSR_NOT_FIRST: Set after the first task is visited. "First" + * tracks position in the DSQ list, not among IMMED tasks. A non-IMMED task at + * the head consumes the first slot. + * + * - %SCX_REENQ_TSR_RQ_OPEN: Set by reenq_local() before the walk if + * rq_is_open() is true. + * + * An IMMED task is kept (returns %false) only if it's the first task in the DSQ + * AND the current task is done — i.e. it will execute immediately. All other + * IMMED tasks are reenqueued. This means if a non-IMMED task sits at the head, + * every IMMED task behind it gets reenqueued. + * + * Reenqueued tasks go through ops.enqueue() with %SCX_ENQ_REENQ | + * %SCX_TASK_REENQ_IMMED. If the BPF scheduler dispatches back to the same local + * DSQ with %SCX_ENQ_IMMED while the CPU is still unavailable, this triggers + * another reenq cycle. Repetitions are bounded by %SCX_REENQ_LOCAL_MAX_REPEAT + * in process_deferred_reenq_locals(). + */ +static bool local_task_should_reenq(struct task_struct *p, u64 *reenq_flags, u32 *reason) +{ + bool first; + + first = !(*reenq_flags & SCX_REENQ_TSR_NOT_FIRST); + *reenq_flags |= SCX_REENQ_TSR_NOT_FIRST; + + *reason = SCX_TASK_REENQ_KFUNC; + + if ((p->scx.flags & SCX_TASK_IMMED) && + (!first || !(*reenq_flags & SCX_REENQ_TSR_RQ_OPEN))) { + __scx_add_event(scx_task_sched(p), SCX_EV_REENQ_IMMED, 1); + *reason = SCX_TASK_REENQ_IMMED; + return true; + } + + return *reenq_flags & SCX_REENQ_ANY; +} + +static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) +{ + LIST_HEAD(tasks); + u32 nr_enqueued = 0; + struct task_struct *p, *n; + + lockdep_assert_rq_held(rq); + + if (WARN_ON_ONCE(reenq_flags & __SCX_REENQ_TSR_MASK)) + reenq_flags &= ~__SCX_REENQ_TSR_MASK; + if (rq_is_open(rq, 0)) + reenq_flags |= SCX_REENQ_TSR_RQ_OPEN; + + /* + * The BPF scheduler may choose to dispatch tasks back to + * @rq->scx.local_dsq. Move all candidate tasks off to a private list + * first to avoid processing the same tasks repeatedly. + */ + list_for_each_entry_safe(p, n, &rq->scx.local_dsq.list, + scx.dsq_list.node) { + struct scx_sched *task_sch = scx_task_sched(p); + u32 reason; + + /* + * If @p is being migrated, @p's current CPU may not agree with + * its allowed CPUs and the migration_cpu_stop is about to + * deactivate and re-activate @p anyway. Skip re-enqueueing. + * + * While racing sched property changes may also dequeue and + * re-enqueue a migrating task while its current CPU and allowed + * CPUs disagree, they use %ENQUEUE_RESTORE which is bypassed to + * the current local DSQ for running tasks and thus are not + * visible to the BPF scheduler. + */ + if (p->migration_pending) + continue; + + if (!scx_is_descendant(task_sch, sch)) + continue; + + if (!local_task_should_reenq(p, &reenq_flags, &reason)) + continue; + + dispatch_dequeue(rq, p); + + if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + p->scx.flags |= reason; + + list_add_tail(&p->scx.dsq_list.node, &tasks); + } + + list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) { + list_del_init(&p->scx.dsq_list.node); + + do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); + + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + nr_enqueued++; + } + + return nr_enqueued; +} + +static void process_deferred_reenq_locals(struct rq *rq) +{ + u64 seq = ++rq->scx.deferred_reenq_locals_seq; + + lockdep_assert_rq_held(rq); + + while (true) { + struct scx_sched *sch; + u64 reenq_flags; + bool skip = false; + + scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) { + struct scx_deferred_reenq_local *drl = + list_first_entry_or_null(&rq->scx.deferred_reenq_locals, + struct scx_deferred_reenq_local, + node); + struct scx_sched_pcpu *sch_pcpu; + + if (!drl) + return; + + sch_pcpu = container_of(drl, struct scx_sched_pcpu, + deferred_reenq_local); + sch = sch_pcpu->sch; + + reenq_flags = drl->flags; + WRITE_ONCE(drl->flags, 0); + list_del_init(&drl->node); + + if (likely(drl->seq != seq)) { + drl->seq = seq; + drl->cnt = 0; + } else { + if (unlikely(++drl->cnt > SCX_REENQ_LOCAL_MAX_REPEAT)) { + scx_error(sch, "SCX_ENQ_REENQ on SCX_DSQ_LOCAL repeated %u times", + drl->cnt); + skip = true; + } + + __scx_add_event(sch, SCX_EV_REENQ_LOCAL_REPEAT, 1); + } + } + + if (!skip) { + /* see schedule_dsq_reenq() */ + smp_mb(); + + reenq_local(sch, rq, reenq_flags); + } + } +} + +static bool user_task_should_reenq(struct task_struct *p, u64 reenq_flags, u32 *reason) +{ + *reason = SCX_TASK_REENQ_KFUNC; + return reenq_flags & SCX_REENQ_ANY; +} + +static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flags) +{ + struct rq *locked_rq = rq; + struct scx_sched *sch = dsq->sched; + struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, dsq, 0); + struct task_struct *p; + s32 nr_enqueued = 0; + + lockdep_assert_rq_held(rq); + + raw_spin_lock(&dsq->lock); + + while (likely(!READ_ONCE(sch->bypass_depth))) { + struct rq *task_rq; + u32 reason; + + p = nldsq_cursor_next_task(&cursor, dsq); + if (!p) + break; + + if (!user_task_should_reenq(p, reenq_flags, &reason)) + continue; + + task_rq = task_rq(p); + + if (locked_rq != task_rq) { + if (locked_rq) + raw_spin_rq_unlock(locked_rq); + if (unlikely(!raw_spin_rq_trylock(task_rq))) { + raw_spin_unlock(&dsq->lock); + raw_spin_rq_lock(task_rq); + raw_spin_lock(&dsq->lock); + } + locked_rq = task_rq; + + /* did we lose @p while switching locks? */ + if (nldsq_cursor_lost_task(&cursor, task_rq, dsq, p)) + continue; + } + + /* @p is on @dsq, its rq and @dsq are locked */ + dispatch_dequeue_locked(p, dsq); + raw_spin_unlock(&dsq->lock); + + if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + p->scx.flags |= reason; + + do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1); + + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + + if (!(++nr_enqueued % SCX_TASK_ITER_BATCH)) { + raw_spin_rq_unlock(locked_rq); + locked_rq = NULL; + cpu_relax(); + } + + raw_spin_lock(&dsq->lock); + } + + list_del_init(&cursor.node); + raw_spin_unlock(&dsq->lock); + + if (locked_rq != rq) { + if (locked_rq) + raw_spin_rq_unlock(locked_rq); + raw_spin_rq_lock(rq); + } +} + +static void process_deferred_reenq_users(struct rq *rq) +{ + lockdep_assert_rq_held(rq); + + while (true) { + struct scx_dispatch_q *dsq; + u64 reenq_flags; + + scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) { + struct scx_deferred_reenq_user *dru = + list_first_entry_or_null(&rq->scx.deferred_reenq_users, + struct scx_deferred_reenq_user, + node); + struct scx_dsq_pcpu *dsq_pcpu; + + if (!dru) + return; + + dsq_pcpu = container_of(dru, struct scx_dsq_pcpu, + deferred_reenq_user); + dsq = dsq_pcpu->dsq; + reenq_flags = dru->flags; + WRITE_ONCE(dru->flags, 0); + list_del_init(&dru->node); + } + + /* see schedule_dsq_reenq() */ + smp_mb(); + + BUG_ON(dsq->id & SCX_DSQ_FLAG_BUILTIN); + reenq_user(rq, dsq, reenq_flags); + } +} + +static void run_deferred(struct rq *rq) +{ + process_ddsp_deferred_locals(rq); + + if (!list_empty(&rq->scx.deferred_reenq_locals)) + process_deferred_reenq_locals(rq); + + if (!list_empty(&rq->scx.deferred_reenq_users)) + process_deferred_reenq_users(rq); +} + +#ifdef CONFIG_NO_HZ_FULL +bool scx_can_stop_tick(struct rq *rq) +{ + struct task_struct *p = rq->curr; + struct scx_sched *sch = scx_task_sched(p); + + if (p->sched_class != &ext_sched_class) + return true; + + if (scx_bypassing(sch, cpu_of(rq))) + return false; + + /* + * @rq can dispatch from different DSQs, so we can't tell whether it + * needs the tick or not by looking at nr_running. Allow stopping ticks + * iff the BPF scheduler indicated so. See set_next_task_scx(). + */ + return rq->scx.flags & SCX_RQ_CAN_STOP_TICK; +} +#endif + +#ifdef CONFIG_EXT_GROUP_SCHED + +DEFINE_STATIC_PERCPU_RWSEM(scx_cgroup_ops_rwsem); +static bool scx_cgroup_enabled; + +void scx_tg_init(struct task_group *tg) +{ + tg->scx.weight = CGROUP_WEIGHT_DFL; + tg->scx.bw_period_us = default_bw_period_us(); + tg->scx.bw_quota_us = RUNTIME_INF; + tg->scx.idle = false; +} + +int scx_tg_online(struct task_group *tg) +{ + struct scx_sched *sch = scx_root; + int ret = 0; + + WARN_ON_ONCE(tg->scx.flags & (SCX_TG_ONLINE | SCX_TG_INITED)); + + if (scx_cgroup_enabled) { + if (SCX_HAS_OP(sch, cgroup_init)) { + struct scx_cgroup_init_args args = + { .weight = tg->scx.weight, + .bw_period_us = tg->scx.bw_period_us, + .bw_quota_us = tg->scx.bw_quota_us, + .bw_burst_us = tg->scx.bw_burst_us }; + + ret = SCX_CALL_OP_RET(sch, cgroup_init, + NULL, tg->css.cgroup, &args); + if (ret) + ret = ops_sanitize_err(sch, "cgroup_init", ret); + } + if (ret == 0) + tg->scx.flags |= SCX_TG_ONLINE | SCX_TG_INITED; + } else { + tg->scx.flags |= SCX_TG_ONLINE; + } + + return ret; +} + +void scx_tg_offline(struct task_group *tg) +{ + struct scx_sched *sch = scx_root; + + WARN_ON_ONCE(!(tg->scx.flags & SCX_TG_ONLINE)); + + if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_exit) && + (tg->scx.flags & SCX_TG_INITED)) + SCX_CALL_OP(sch, cgroup_exit, NULL, tg->css.cgroup); + tg->scx.flags &= ~(SCX_TG_ONLINE | SCX_TG_INITED); +} + +int scx_cgroup_can_attach(struct cgroup_taskset *tset) +{ + struct scx_sched *sch = scx_root; + struct cgroup_subsys_state *css; + struct task_struct *p; + int ret; + + if (!scx_cgroup_enabled) + return 0; + + cgroup_taskset_for_each(p, css, tset) { + struct cgroup *from = tg_cgrp(task_group(p)); + struct cgroup *to = tg_cgrp(css_tg(css)); + + WARN_ON_ONCE(p->scx.cgrp_moving_from); + + /* + * sched_move_task() omits identity migrations. Let's match the + * behavior so that ops.cgroup_prep_move() and ops.cgroup_move() + * always match one-to-one. + */ + if (from == to) + continue; + + if (SCX_HAS_OP(sch, cgroup_prep_move)) { + ret = SCX_CALL_OP_RET(sch, cgroup_prep_move, NULL, + p, from, css->cgroup); + if (ret) + goto err; + } + + p->scx.cgrp_moving_from = from; + } + + return 0; + +err: + cgroup_taskset_for_each(p, css, tset) { + if (SCX_HAS_OP(sch, cgroup_cancel_move) && + p->scx.cgrp_moving_from) + SCX_CALL_OP(sch, cgroup_cancel_move, NULL, + p, p->scx.cgrp_moving_from, css->cgroup); + p->scx.cgrp_moving_from = NULL; + } + + return ops_sanitize_err(sch, "cgroup_prep_move", ret); +} + +void scx_cgroup_move_task(struct task_struct *p) +{ + struct scx_sched *sch = scx_root; + + if (!scx_cgroup_enabled) + return; + + /* + * scx_cgroup_can_attach() sets cgrp_moving_from only when the task's + * cgroup changes. Migration keys off css rather than cgroup identity, + * so it can hand an unchanged-cgroup task here with cgrp_moving_from + * NULL. Nothing to report to the BPF scheduler then, so skip it and + * keep prep_move and move paired. + */ + if (SCX_HAS_OP(sch, cgroup_move) && p->scx.cgrp_moving_from) + SCX_CALL_OP_TASK(sch, cgroup_move, task_rq(p), + p, p->scx.cgrp_moving_from, + tg_cgrp(task_group(p))); + p->scx.cgrp_moving_from = NULL; +} + +void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) +{ + struct scx_sched *sch = scx_root; + struct cgroup_subsys_state *css; + struct task_struct *p; + + if (!scx_cgroup_enabled) + return; + + cgroup_taskset_for_each(p, css, tset) { + if (SCX_HAS_OP(sch, cgroup_cancel_move) && + p->scx.cgrp_moving_from) + SCX_CALL_OP(sch, cgroup_cancel_move, NULL, + p, p->scx.cgrp_moving_from, css->cgroup); + p->scx.cgrp_moving_from = NULL; + } +} + +void scx_group_set_weight(struct task_group *tg, unsigned long weight) +{ + struct scx_sched *sch; + + percpu_down_read(&scx_cgroup_ops_rwsem); + sch = scx_root; + + if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_weight) && + tg->scx.weight != weight) + SCX_CALL_OP(sch, cgroup_set_weight, NULL, tg_cgrp(tg), weight); + + tg->scx.weight = weight; + + percpu_up_read(&scx_cgroup_ops_rwsem); +} + +void scx_group_set_idle(struct task_group *tg, bool idle) +{ + struct scx_sched *sch; + + percpu_down_read(&scx_cgroup_ops_rwsem); + sch = scx_root; + + if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_idle)) + SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle); + + /* Update the task group's idle state */ + tg->scx.idle = idle; + + percpu_up_read(&scx_cgroup_ops_rwsem); +} + +void scx_group_set_bandwidth(struct task_group *tg, + u64 period_us, u64 quota_us, u64 burst_us) +{ + struct scx_sched *sch; + + percpu_down_read(&scx_cgroup_ops_rwsem); + sch = scx_root; + + if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_bandwidth) && + (tg->scx.bw_period_us != period_us || + tg->scx.bw_quota_us != quota_us || + tg->scx.bw_burst_us != burst_us)) + SCX_CALL_OP(sch, cgroup_set_bandwidth, NULL, + tg_cgrp(tg), period_us, quota_us, burst_us); + + tg->scx.bw_period_us = period_us; + tg->scx.bw_quota_us = quota_us; + tg->scx.bw_burst_us = burst_us; + + percpu_up_read(&scx_cgroup_ops_rwsem); +} +#endif /* CONFIG_EXT_GROUP_SCHED */ + +#if defined(CONFIG_EXT_GROUP_SCHED) || defined(CONFIG_EXT_SUB_SCHED) +static struct cgroup *root_cgroup(void) +{ + return &cgrp_dfl_root.cgrp; +} + +static void scx_cgroup_lock(void) +{ +#ifdef CONFIG_EXT_GROUP_SCHED + percpu_down_write(&scx_cgroup_ops_rwsem); +#endif + cgroup_lock(); +} + +static void scx_cgroup_unlock(void) +{ + cgroup_unlock(); +#ifdef CONFIG_EXT_GROUP_SCHED + percpu_up_write(&scx_cgroup_ops_rwsem); +#endif +} +#else /* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */ +static inline struct cgroup *root_cgroup(void) { return NULL; } +static inline void scx_cgroup_lock(void) {} +static inline void scx_cgroup_unlock(void) {} +#endif /* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */ + +#ifdef CONFIG_EXT_SUB_SCHED +static struct cgroup *sch_cgroup(struct scx_sched *sch) +{ + return sch->cgrp; +} + +/* for each descendant of @cgrp including self, set ->scx_sched to @sch */ +static void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) +{ + struct cgroup *pos; + struct cgroup_subsys_state *css; + + cgroup_for_each_live_descendant_pre(pos, css, cgrp) + rcu_assign_pointer(pos->scx_sched, sch); +} +#else /* CONFIG_EXT_SUB_SCHED */ +static inline struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; } +static inline void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {} +#endif /* CONFIG_EXT_SUB_SCHED */ + +/* + * Omitted operations: + * + * - migrate_task_rq: Unnecessary as task to cpu mapping is transient. + * + * - task_fork/dead: We need fork/dead notifications for all tasks regardless of + * their current sched_class. Call them directly from sched core instead. + */ +DEFINE_SCHED_CLASS(ext) = { + .enqueue_task = enqueue_task_scx, + .dequeue_task = dequeue_task_scx, + .yield_task = yield_task_scx, + .yield_to_task = yield_to_task_scx, + + .wakeup_preempt = wakeup_preempt_scx, + + .pick_task = pick_task_scx, + + .put_prev_task = put_prev_task_scx, + .set_next_task = set_next_task_scx, + + .select_task_rq = select_task_rq_scx, + .task_woken = task_woken_scx, + .set_cpus_allowed = set_cpus_allowed_scx, + + .rq_online = rq_online_scx, + .rq_offline = rq_offline_scx, + + .task_tick = task_tick_scx, + + .switching_to = switching_to_scx, + .switched_from = switched_from_scx, + .switched_to = switched_to_scx, + .reweight_task = reweight_task_scx, + .prio_changed = prio_changed_scx, + + .update_curr = update_curr_scx, + +#ifdef CONFIG_UCLAMP_TASK + .uclamp_enabled = 1, +#endif +}; + +static s32 init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, + struct scx_sched *sch) +{ + s32 cpu; + + memset(dsq, 0, sizeof(*dsq)); + + raw_spin_lock_init(&dsq->lock); + INIT_LIST_HEAD(&dsq->list); + dsq->id = dsq_id; + dsq->sched = sch; + + dsq->pcpu = alloc_percpu(struct scx_dsq_pcpu); + if (!dsq->pcpu) + return -ENOMEM; + + for_each_possible_cpu(cpu) { + struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu); + + pcpu->dsq = dsq; + INIT_LIST_HEAD(&pcpu->deferred_reenq_user.node); + } + + return 0; +} + +static void exit_dsq(struct scx_dispatch_q *dsq) +{ + s32 cpu; + + for_each_possible_cpu(cpu) { + struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu); + struct scx_deferred_reenq_user *dru = &pcpu->deferred_reenq_user; + struct rq *rq = cpu_rq(cpu); + + /* + * There must have been a RCU grace period since the last + * insertion and @dsq should be off the deferred list by now. + */ + if (WARN_ON_ONCE(!list_empty(&dru->node))) { + guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); + list_del_init(&dru->node); + } + } + + free_percpu(dsq->pcpu); +} + +static void free_dsq_rcufn(struct rcu_head *rcu) +{ + struct scx_dispatch_q *dsq = container_of(rcu, struct scx_dispatch_q, rcu); + + exit_dsq(dsq); + kfree(dsq); +} + +static void free_dsq_irq_workfn(struct irq_work *irq_work) +{ + struct llist_node *to_free = llist_del_all(&dsqs_to_free); + struct scx_dispatch_q *dsq, *tmp_dsq; + + llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) + call_rcu(&dsq->rcu, free_dsq_rcufn); +} + +static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); + +static void destroy_dsq(struct scx_sched *sch, u64 dsq_id) +{ + struct scx_dispatch_q *dsq; + unsigned long flags; + + rcu_read_lock(); + + dsq = find_user_dsq(sch, dsq_id); + if (!dsq) + goto out_unlock_rcu; + + raw_spin_lock_irqsave(&dsq->lock, flags); + + if (dsq->nr) { + scx_error(sch, "attempting to destroy in-use dsq 0x%016llx (nr=%u)", + dsq->id, dsq->nr); + goto out_unlock_dsq; + } + + if (rhashtable_remove_fast(&sch->dsq_hash, &dsq->hash_node, + dsq_hash_params)) + goto out_unlock_dsq; + + /* + * Mark dead by invalidating ->id to prevent dispatch_enqueue() from + * queueing more tasks. As this function can be called from anywhere, + * freeing is bounced through an irq work to avoid nesting RCU + * operations inside scheduler locks. + */ + dsq->id = SCX_DSQ_INVALID; + if (llist_add(&dsq->free_node, &dsqs_to_free)) + irq_work_queue(&free_dsq_irq_work); + +out_unlock_dsq: + raw_spin_unlock_irqrestore(&dsq->lock, flags); +out_unlock_rcu: + rcu_read_unlock(); +} + +#ifdef CONFIG_EXT_GROUP_SCHED +static void scx_cgroup_exit(struct scx_sched *sch) +{ + struct cgroup_subsys_state *css; + + scx_cgroup_enabled = false; + + /* + * scx_tg_on/offline() are excluded through cgroup_lock(). If we walk + * cgroups and exit all the inited ones, all online cgroups are exited. + */ + css_for_each_descendant_post(css, &root_task_group.css) { + struct task_group *tg = css_tg(css); + + if (!(tg->scx.flags & SCX_TG_INITED)) + continue; + tg->scx.flags &= ~SCX_TG_INITED; + + if (!sch->ops.cgroup_exit) + continue; + + SCX_CALL_OP(sch, cgroup_exit, NULL, css->cgroup); + } +} + +static int scx_cgroup_init(struct scx_sched *sch) +{ + struct cgroup_subsys_state *css; + int ret; + + /* + * scx_tg_on/offline() are excluded through cgroup_lock(). If we walk + * cgroups and init, all online cgroups are initialized. + */ + css_for_each_descendant_pre(css, &root_task_group.css) { + struct task_group *tg = css_tg(css); + struct scx_cgroup_init_args args = { + .weight = tg->scx.weight, + .bw_period_us = tg->scx.bw_period_us, + .bw_quota_us = tg->scx.bw_quota_us, + .bw_burst_us = tg->scx.bw_burst_us, + }; + + if ((tg->scx.flags & + (SCX_TG_ONLINE | SCX_TG_INITED)) != SCX_TG_ONLINE) + continue; + + if (!sch->ops.cgroup_init) { + tg->scx.flags |= SCX_TG_INITED; + continue; + } + + ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, + css->cgroup, &args); + if (ret) { + scx_error(sch, "ops.cgroup_init() failed (%d)", ret); + return ret; + } + tg->scx.flags |= SCX_TG_INITED; + } + + WARN_ON_ONCE(scx_cgroup_enabled); + scx_cgroup_enabled = true; + + return 0; +} + +#else +static void scx_cgroup_exit(struct scx_sched *sch) {} +static int scx_cgroup_init(struct scx_sched *sch) { return 0; } +#endif + + +/******************************************************************************** + * Sysfs interface and ops enable/disable. + */ + +#define SCX_ATTR(_name) \ + static struct kobj_attribute scx_attr_##_name = { \ + .attr = { .name = __stringify(_name), .mode = 0444 }, \ + .show = scx_attr_##_name##_show, \ + } + +static ssize_t scx_attr_state_show(struct kobject *kobj, + struct kobj_attribute *ka, char *buf) +{ + return sysfs_emit(buf, "%s\n", scx_enable_state_str[scx_enable_state()]); +} +SCX_ATTR(state); + +static ssize_t scx_attr_switch_all_show(struct kobject *kobj, + struct kobj_attribute *ka, char *buf) +{ + return sysfs_emit(buf, "%d\n", READ_ONCE(scx_switching_all)); +} +SCX_ATTR(switch_all); + +static ssize_t scx_attr_nr_rejected_show(struct kobject *kobj, + struct kobj_attribute *ka, char *buf) +{ + return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_nr_rejected)); +} +SCX_ATTR(nr_rejected); + +static ssize_t scx_attr_hotplug_seq_show(struct kobject *kobj, + struct kobj_attribute *ka, char *buf) +{ + return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_hotplug_seq)); +} +SCX_ATTR(hotplug_seq); + +static ssize_t scx_attr_enable_seq_show(struct kobject *kobj, + struct kobj_attribute *ka, char *buf) +{ + return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_enable_seq)); +} +SCX_ATTR(enable_seq); + +static struct attribute *scx_global_attrs[] = { + &scx_attr_state.attr, + &scx_attr_switch_all.attr, + &scx_attr_nr_rejected.attr, + &scx_attr_hotplug_seq.attr, + &scx_attr_enable_seq.attr, + NULL, +}; + +static const struct attribute_group scx_global_attr_group = { + .attrs = scx_global_attrs, +}; + +static void free_pnode(struct scx_sched_pnode *pnode); +static void free_exit_info(struct scx_exit_info *ei); + +static s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch) +{ + size_t size = struct_size_t(struct scx_cmask, bits, + SCX_CMASK_NR_WORDS(num_possible_cpus())); + int cpu; + + if (!sch->is_cid_type || !sch->arena_pool) + return 0; + + sch->set_cmask_scratch = alloc_percpu(struct scx_cmask *); + if (!sch->set_cmask_scratch) + return -ENOMEM; + + for_each_possible_cpu(cpu) { + struct scx_cmask **slot = per_cpu_ptr(sch->set_cmask_scratch, cpu); + + *slot = scx_arena_alloc(sch, size); + if (!*slot) + return -ENOMEM; + scx_cmask_init(*slot, 0, num_possible_cpus()); + } + return 0; +} + +static void scx_set_cmask_scratch_free(struct scx_sched *sch) +{ + size_t size = struct_size_t(struct scx_cmask, bits, + SCX_CMASK_NR_WORDS(num_possible_cpus())); + int cpu; + + if (!sch->set_cmask_scratch) + return; + + for_each_possible_cpu(cpu) { + struct scx_cmask **slot = per_cpu_ptr(sch->set_cmask_scratch, cpu); + + scx_arena_free(sch, *slot, size); + } + free_percpu(sch->set_cmask_scratch); + sch->set_cmask_scratch = NULL; +} + +static void scx_sched_free_rcu_work(struct work_struct *work) +{ + struct rcu_work *rcu_work = to_rcu_work(work); + struct scx_sched *sch = container_of(rcu_work, struct scx_sched, rcu_work); + struct rhashtable_iter rht_iter; + struct scx_dispatch_q *dsq; + int cpu, node; + + irq_work_sync(&sch->disable_irq_work); + kthread_destroy_worker(sch->helper); + timer_shutdown_sync(&sch->bypass_lb_timer); + free_cpumask_var(sch->bypass_lb_donee_cpumask); + free_cpumask_var(sch->bypass_lb_resched_cpumask); + +#ifdef CONFIG_EXT_SUB_SCHED + kfree(sch->cgrp_path); + if (sch_cgroup(sch)) + cgroup_put(sch_cgroup(sch)); + if (sch->sub_kset) + kobject_put(&sch->sub_kset->kobj); +#endif /* CONFIG_EXT_SUB_SCHED */ + + for_each_possible_cpu(cpu) { + struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); + + /* + * $sch would have entered bypass mode before the RCU grace + * period. As that blocks new deferrals, all + * deferred_reenq_local_node's must be off-list by now. + */ + WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local.node)); + + exit_dsq(bypass_dsq(sch, cpu)); + } + + free_percpu(sch->pcpu); + + for_each_node_state(node, N_POSSIBLE) + free_pnode(sch->pnode[node]); + kfree(sch->pnode); + + rhashtable_walk_enter(&sch->dsq_hash, &rht_iter); + do { + rhashtable_walk_start(&rht_iter); + + while (!IS_ERR_OR_NULL((dsq = rhashtable_walk_next(&rht_iter)))) + destroy_dsq(sch, dsq->id); + + rhashtable_walk_stop(&rht_iter); + } while (dsq == ERR_PTR(-EAGAIN)); + rhashtable_walk_exit(&rht_iter); + + rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL); + free_exit_info(sch->exit_info); + scx_set_cmask_scratch_free(sch); + scx_arena_pool_destroy(sch); + if (sch->arena_map) + bpf_map_put(sch->arena_map); + kfree(sch); +} + +static void scx_kobj_release(struct kobject *kobj) +{ + struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); + + INIT_RCU_WORK(&sch->rcu_work, scx_sched_free_rcu_work); + queue_rcu_work(system_dfl_wq, &sch->rcu_work); +} + +static ssize_t scx_attr_ops_show(struct kobject *kobj, + struct kobj_attribute *ka, char *buf) +{ + struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); + + return sysfs_emit(buf, "%s\n", sch->ops.name); +} +SCX_ATTR(ops); + +#define scx_attr_event_show(buf, at, events, kind) ({ \ + sysfs_emit_at(buf, at, "%s %llu\n", #kind, (events)->kind); \ +}) + +static ssize_t scx_attr_events_show(struct kobject *kobj, + struct kobj_attribute *ka, char *buf) +{ + struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); + struct scx_event_stats events; + int at = 0; + + scx_read_events(sch, &events); + at += scx_attr_event_show(buf, at, &events, SCX_EV_SELECT_CPU_FALLBACK); + at += scx_attr_event_show(buf, at, &events, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); + at += scx_attr_event_show(buf, at, &events, SCX_EV_DISPATCH_KEEP_LAST); + at += scx_attr_event_show(buf, at, &events, SCX_EV_ENQ_SKIP_EXITING); + at += scx_attr_event_show(buf, at, &events, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); + at += scx_attr_event_show(buf, at, &events, SCX_EV_REENQ_IMMED); + at += scx_attr_event_show(buf, at, &events, SCX_EV_REENQ_LOCAL_REPEAT); + at += scx_attr_event_show(buf, at, &events, SCX_EV_REFILL_SLICE_DFL); + at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_DURATION); + at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_DISPATCH); + at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_ACTIVATE); + at += scx_attr_event_show(buf, at, &events, SCX_EV_INSERT_NOT_OWNED); + at += scx_attr_event_show(buf, at, &events, SCX_EV_SUB_BYPASS_DISPATCH); + return at; +} +SCX_ATTR(events); + +static struct attribute *scx_sched_attrs[] = { + &scx_attr_ops.attr, + &scx_attr_events.attr, + NULL, +}; +ATTRIBUTE_GROUPS(scx_sched); + +static const struct kobj_type scx_ktype = { + .release = scx_kobj_release, + .sysfs_ops = &kobj_sysfs_ops, + .default_groups = scx_sched_groups, +}; + +static int scx_uevent(const struct kobject *kobj, struct kobj_uevent_env *env) +{ + const struct scx_sched *sch; + + /* + * scx_uevent() can be reached by both scx_sched kobjects (scx_ktype) + * and sub-scheduler kset kobjects (kset_ktype) through the parent + * chain walk. Filter out the latter to avoid invalid casts. + */ + if (kobj->ktype != &scx_ktype) + return 0; + + sch = container_of(kobj, struct scx_sched, kobj); + + return add_uevent_var(env, "SCXOPS=%s", sch->ops.name); +} + +static const struct kset_uevent_ops scx_uevent_ops = { + .uevent = scx_uevent, +}; + +/* + * Used by sched_fork() and __setscheduler_prio() to pick the matching + * sched_class. dl/rt are already handled. + */ +bool task_should_scx(int policy) +{ + /* if disabled, nothing should be on it */ + if (!scx_enabled()) + return false; + + /* scx is taking over all SCHED_OTHER and SCHED_EXT tasks */ + if (READ_ONCE(scx_switching_all)) + return true; + + /* + * scx is tearing down - keep new SCHED_EXT tasks out. + * + * Must come after scx_switching_all test, which serves as a proxy + * for __scx_switched_all. While __scx_switched_all is set, we must + * return true via the branch above: a fork routed to fair would + * stall because next_active_class() skips fair. + * + * This can develop into a deadlock - scx holds scx_enable_mutex across + * kthread_create() in scx_alloc_and_add_sched(); if the new kthread is + * the stalled task, the disable path can never grab the mutex to clear + * scx_switching_all. + */ + if (unlikely(scx_enable_state() == SCX_DISABLING)) + return false; + + return policy == SCHED_EXT; +} + +bool scx_allow_ttwu_queue(const struct task_struct *p) +{ + struct scx_sched *sch; + + if (!scx_enabled()) + return true; + + sch = scx_task_sched(p); + if (unlikely(!sch)) + return true; + + if (sch->ops.flags & SCX_OPS_ALLOW_QUEUED_WAKEUP) + return true; + + if (unlikely(p->sched_class != &ext_sched_class)) + return true; + + return false; +} + +/** + * handle_lockup - sched_ext common lockup handler + * @fmt: format string + * + * Called on system stall or lockup condition and initiates abort of sched_ext + * if enabled, which may resolve the reported lockup. + * + * Returns %true if sched_ext is enabled and abort was initiated, which may + * resolve the lockup. %false if sched_ext is not enabled or abort was already + * initiated by someone else. + */ +static __printf(1, 2) bool handle_lockup(const char *fmt, ...) +{ + struct scx_sched *sch; + va_list args; + bool ret; + + guard(rcu)(); + + sch = rcu_dereference(scx_root); + if (unlikely(!sch)) + return false; + + switch (scx_enable_state()) { + case SCX_ENABLING: + case SCX_ENABLED: + va_start(args, fmt); + ret = scx_verror(sch, fmt, args); + va_end(args); + return ret; + default: + return false; + } +} + +/** + * scx_rcu_cpu_stall - sched_ext RCU CPU stall handler + * + * While there are various reasons why RCU CPU stalls can occur on a system + * that may not be caused by the current BPF scheduler, try kicking out the + * current scheduler in an attempt to recover the system to a good state before + * issuing panics. + * + * Returns %true if sched_ext is enabled and abort was initiated, which may + * resolve the reported RCU stall. %false if sched_ext is not enabled or someone + * else already initiated abort. + */ +bool scx_rcu_cpu_stall(void) +{ + return handle_lockup("RCU CPU stall detected!"); +} + +/** + * scx_softlockup - sched_ext softlockup handler + * @dur_s: number of seconds of CPU stuck due to soft lockup + * + * On some multi-socket setups (e.g. 2x Intel 8480c), the BPF scheduler can + * live-lock the system by making many CPUs target the same DSQ to the point + * where soft-lockup detection triggers. This function is called from + * soft-lockup watchdog when the triggering point is close and tries to unjam + * the system and aborting the BPF scheduler. + */ +void scx_softlockup(u32 dur_s) +{ + if (!handle_lockup("soft lockup - CPU %d stuck for %us", smp_processor_id(), dur_s)) + return; + + printk_deferred(KERN_ERR "sched_ext: Soft lockup - CPU %d stuck for %us, disabling BPF scheduler\n", + smp_processor_id(), dur_s); +} + +/* + * scx_hardlockup() runs from NMI and eventually calls scx_claim_exit(), + * which takes scx_sched_lock. scx_sched_lock isn't NMI-safe and grabbing + * it from NMI context can lead to deadlocks. Defer via irq_work; the + * disable path runs off irq_work anyway. + */ +static atomic_t scx_hardlockup_cpu = ATOMIC_INIT(-1); + +static void scx_hardlockup_irq_workfn(struct irq_work *work) +{ + int cpu = atomic_xchg(&scx_hardlockup_cpu, -1); + + if (cpu >= 0 && handle_lockup("hard lockup - CPU %d", cpu)) + printk_deferred(KERN_ERR "sched_ext: Hard lockup - CPU %d, disabling BPF scheduler\n", + cpu); +} + +static DEFINE_IRQ_WORK(scx_hardlockup_irq_work, scx_hardlockup_irq_workfn); + +/** + * scx_hardlockup - sched_ext hardlockup handler + * + * A poorly behaving BPF scheduler can trigger hard lockup by e.g. putting + * numerous affinitized tasks in a single queue and directing all CPUs at it. + * Try kicking out the current scheduler in an attempt to recover the system to + * a good state before taking more drastic actions. + * + * Queues an irq_work; the handle_lockup() call happens in IRQ context (see + * scx_hardlockup_irq_workfn). + * + * Returns %true if sched_ext is enabled and the work was queued, %false + * otherwise. + */ +bool scx_hardlockup(int cpu) +{ + if (!rcu_access_pointer(scx_root)) + return false; + + atomic_cmpxchg(&scx_hardlockup_cpu, -1, cpu); + irq_work_queue(&scx_hardlockup_irq_work); + return true; +} + +static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, + struct cpumask *donee_mask, struct cpumask *resched_mask, + u32 nr_donor_target, u32 nr_donee_target) +{ + struct rq *donor_rq = cpu_rq(donor); + struct scx_dispatch_q *donor_dsq = bypass_dsq(sch, donor); + struct task_struct *p, *n; + struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, donor_dsq, 0); + s32 delta = READ_ONCE(donor_dsq->nr) - nr_donor_target; + u32 nr_balanced = 0, min_delta_us; + + /* + * All we want to guarantee is reasonable forward progress. No reason to + * fine tune. Assuming every task on @donor_dsq runs their full slice, + * consider offloading iff the total queued duration is over the + * threshold. + */ + min_delta_us = READ_ONCE(scx_bypass_lb_intv_us) / SCX_BYPASS_LB_MIN_DELTA_DIV; + if (delta < DIV_ROUND_UP(min_delta_us, READ_ONCE(scx_slice_bypass_us))) + return 0; + + raw_spin_rq_lock_irq(donor_rq); + raw_spin_lock(&donor_dsq->lock); + list_add(&cursor.node, &donor_dsq->list); +resume: + n = container_of(&cursor, struct task_struct, scx.dsq_list); + n = nldsq_next_task(donor_dsq, n, false); + + while ((p = n)) { + struct scx_dispatch_q *donee_dsq; + int donee; + + n = nldsq_next_task(donor_dsq, n, false); + + if (donor_dsq->nr <= nr_donor_target) + break; + + if (cpumask_empty(donee_mask)) + break; + + /* + * If an earlier pass placed @p on @donor_dsq from a different + * CPU and the donee hasn't consumed it yet, @p is still on the + * previous CPU and task_rq(@p) != @donor_rq. @p can't be moved + * without its rq locked. Skip. + */ + if (task_rq(p) != donor_rq) + continue; + + donee = cpumask_any_and_distribute(donee_mask, p->cpus_ptr); + if (donee >= nr_cpu_ids) + continue; + + donee_dsq = bypass_dsq(sch, donee); + + /* + * $p's rq is not locked but $p's DSQ lock protects its + * scheduling properties making this test safe. + */ + if (!task_can_run_on_remote_rq(sch, p, cpu_rq(donee), false)) + continue; + + /* + * Moving $p from one non-local DSQ to another. The source rq + * and DSQ are already locked. Do an abbreviated dequeue and + * then perform enqueue without unlocking $donor_dsq. + * + * We don't want to drop and reacquire the lock on each + * iteration as @donor_dsq can be very long and potentially + * highly contended. Donee DSQs are less likely to be contended. + * The nested locking is safe as only this LB moves tasks + * between bypass DSQs. + */ + dispatch_dequeue_locked(p, donor_dsq); + dispatch_enqueue(sch, cpu_rq(donee), donee_dsq, p, SCX_ENQ_NESTED); + + /* + * $donee might have been idle and need to be woken up. No need + * to be clever. Kick every CPU that receives tasks. + */ + cpumask_set_cpu(donee, resched_mask); + + if (READ_ONCE(donee_dsq->nr) >= nr_donee_target) + cpumask_clear_cpu(donee, donee_mask); + + nr_balanced++; + if (!(nr_balanced % SCX_BYPASS_LB_BATCH) && n) { + list_move_tail(&cursor.node, &n->scx.dsq_list.node); + raw_spin_unlock(&donor_dsq->lock); + raw_spin_rq_unlock_irq(donor_rq); + cpu_relax(); + raw_spin_rq_lock_irq(donor_rq); + raw_spin_lock(&donor_dsq->lock); + goto resume; + } + } + + list_del_init(&cursor.node); + raw_spin_unlock(&donor_dsq->lock); + raw_spin_rq_unlock_irq(donor_rq); + + return nr_balanced; +} + +static void bypass_lb_node(struct scx_sched *sch, int node) +{ + const struct cpumask *node_mask = cpumask_of_node(node); + struct cpumask *donee_mask = sch->bypass_lb_donee_cpumask; + struct cpumask *resched_mask = sch->bypass_lb_resched_cpumask; + u32 nr_tasks = 0, nr_cpus = 0, nr_balanced = 0; + u32 nr_target, nr_donor_target; + u32 before_min = U32_MAX, before_max = 0; + u32 after_min = U32_MAX, after_max = 0; + int cpu; + + /* count the target tasks and CPUs */ + for_each_cpu_and(cpu, cpu_online_mask, node_mask) { + u32 nr = READ_ONCE(bypass_dsq(sch, cpu)->nr); + + nr_tasks += nr; + nr_cpus++; + + before_min = min(nr, before_min); + before_max = max(nr, before_max); + } + + if (!nr_cpus) + return; + + /* + * We don't want CPUs to have more than $nr_donor_target tasks and + * balancing to fill donee CPUs upto $nr_target. Once targets are + * calculated, find the donee CPUs. + */ + nr_target = DIV_ROUND_UP(nr_tasks, nr_cpus); + nr_donor_target = DIV_ROUND_UP(nr_target * SCX_BYPASS_LB_DONOR_PCT, 100); + + cpumask_clear(donee_mask); + for_each_cpu_and(cpu, cpu_online_mask, node_mask) { + if (READ_ONCE(bypass_dsq(sch, cpu)->nr) < nr_target) + cpumask_set_cpu(cpu, donee_mask); + } + + /* iterate !donee CPUs and see if they should be offloaded */ + cpumask_clear(resched_mask); + for_each_cpu_and(cpu, cpu_online_mask, node_mask) { + if (cpumask_empty(donee_mask)) + break; + if (cpumask_test_cpu(cpu, donee_mask)) + continue; + if (READ_ONCE(bypass_dsq(sch, cpu)->nr) <= nr_donor_target) + continue; + + nr_balanced += bypass_lb_cpu(sch, cpu, donee_mask, resched_mask, + nr_donor_target, nr_target); + } + + for_each_cpu(cpu, resched_mask) + resched_cpu(cpu); + + for_each_cpu_and(cpu, cpu_online_mask, node_mask) { + u32 nr = READ_ONCE(bypass_dsq(sch, cpu)->nr); + + after_min = min(nr, after_min); + after_max = max(nr, after_max); + + } + + trace_sched_ext_bypass_lb(node, nr_cpus, nr_tasks, nr_balanced, + before_min, before_max, after_min, after_max); +} + +/* + * In bypass mode, all tasks are put on the per-CPU bypass DSQs. If the machine + * is over-saturated and the BPF scheduler skewed tasks into few CPUs, some + * bypass DSQs can be overloaded. If there are enough tasks to saturate other + * lightly loaded CPUs, such imbalance can lead to very high execution latency + * on the overloaded CPUs and thus to hung tasks and RCU stalls. To avoid such + * outcomes, a simple load balancing mechanism is implemented by the following + * timer which runs periodically while bypass mode is in effect. + */ +static void scx_bypass_lb_timerfn(struct timer_list *timer) +{ + struct scx_sched *sch = container_of(timer, struct scx_sched, bypass_lb_timer); + int node; + u32 intv_us; + + if (!bypass_dsp_enabled(sch)) + return; + + for_each_node_with_cpus(node) + bypass_lb_node(sch, node); + + intv_us = READ_ONCE(scx_bypass_lb_intv_us); + if (intv_us) + mod_timer(timer, jiffies + usecs_to_jiffies(intv_us)); +} + +static bool inc_bypass_depth(struct scx_sched *sch) +{ + lockdep_assert_held(&scx_bypass_lock); + + WARN_ON_ONCE(sch->bypass_depth < 0); + WRITE_ONCE(sch->bypass_depth, sch->bypass_depth + 1); + if (sch->bypass_depth != 1) + return false; + + WRITE_ONCE(sch->slice_dfl, READ_ONCE(scx_slice_bypass_us) * NSEC_PER_USEC); + sch->bypass_timestamp = ktime_get_ns(); + scx_add_event(sch, SCX_EV_BYPASS_ACTIVATE, 1); + return true; +} + +static bool dec_bypass_depth(struct scx_sched *sch) +{ + lockdep_assert_held(&scx_bypass_lock); + + WARN_ON_ONCE(sch->bypass_depth < 1); + WRITE_ONCE(sch->bypass_depth, sch->bypass_depth - 1); + if (sch->bypass_depth != 0) + return false; + + WRITE_ONCE(sch->slice_dfl, SCX_SLICE_DFL); + scx_add_event(sch, SCX_EV_BYPASS_DURATION, + ktime_get_ns() - sch->bypass_timestamp); + return true; +} + +static void enable_bypass_dsp(struct scx_sched *sch) +{ + struct scx_sched *host = scx_parent(sch) ?: sch; + u32 intv_us = READ_ONCE(scx_bypass_lb_intv_us); + s32 ret; + + /* + * @sch->bypass_depth transitioning from 0 to 1 triggers enabling. + * Shouldn't stagger. + */ + if (WARN_ON_ONCE(test_and_set_bit(0, &sch->bypass_dsp_claim))) + return; + + /* + * When a sub-sched bypasses, its tasks are queued on the bypass DSQs of + * the nearest non-bypassing ancestor or root. As enable_bypass_dsp() is + * called iff @sch is not already bypassed due to an ancestor bypassing, + * we can assume that the parent is not bypassing and thus will be the + * host of the bypass DSQs. + * + * While the situation may change in the future, the following + * guarantees that the nearest non-bypassing ancestor or root has bypass + * dispatch enabled while a descendant is bypassing, which is all that's + * required. + * + * bypass_dsp_enabled() test is used to determine whether to enter the + * bypass dispatch handling path from both bypassing and hosting scheds. + * Bump enable depth on both @sch and bypass dispatch host. + */ + ret = atomic_inc_return(&sch->bypass_dsp_enable_depth); + WARN_ON_ONCE(ret <= 0); + + if (host != sch) { + ret = atomic_inc_return(&host->bypass_dsp_enable_depth); + WARN_ON_ONCE(ret <= 0); + } + + /* + * The LB timer will stop running if bypass dispatch is disabled. Start + * after enabling bypass dispatch. + */ + if (intv_us && !timer_pending(&host->bypass_lb_timer)) + mod_timer(&host->bypass_lb_timer, + jiffies + usecs_to_jiffies(intv_us)); +} + +/* may be called without holding scx_bypass_lock */ +static void disable_bypass_dsp(struct scx_sched *sch) +{ + s32 ret; + + if (!test_and_clear_bit(0, &sch->bypass_dsp_claim)) + return; + + ret = atomic_dec_return(&sch->bypass_dsp_enable_depth); + WARN_ON_ONCE(ret < 0); + + if (scx_parent(sch)) { + ret = atomic_dec_return(&scx_parent(sch)->bypass_dsp_enable_depth); + WARN_ON_ONCE(ret < 0); + } +} + +/** + * scx_bypass - [Un]bypass scx_ops and guarantee forward progress + * @sch: sched to bypass + * @bypass: true for bypass, false for unbypass + * + * Bypassing guarantees that all runnable tasks make forward progress without + * trusting the BPF scheduler. We can't grab any mutexes or rwsems as they might + * be held by tasks that the BPF scheduler is forgetting to run, which + * unfortunately also excludes toggling the static branches. + * + * Let's work around by overriding a couple ops and modifying behaviors based on + * the DISABLING state and then cycling the queued tasks through dequeue/enqueue + * to force global FIFO scheduling. + * + * - ops.select_cpu() is ignored and the default select_cpu() is used. + * + * - ops.enqueue() is ignored and tasks are queued in simple global FIFO order. + * %SCX_OPS_ENQ_LAST is also ignored. + * + * - ops.dispatch() is ignored. + * + * - balance_one() does not set %SCX_RQ_BAL_KEEP on non-zero slice as slice + * can't be trusted. Whenever a tick triggers, the running task is rotated to + * the tail of the queue with core_sched_at touched. + * + * - pick_next_task() suppresses zero slice warning. + * + * - scx_kick_cpu() is disabled to avoid irq_work malfunction during PM + * operations. + * + * - scx_prio_less() reverts to the default core_sched_at order. + */ +static void scx_bypass(struct scx_sched *sch, bool bypass) +{ + struct scx_sched *pos; + unsigned long flags; + int cpu; + + raw_spin_lock_irqsave(&scx_bypass_lock, flags); + + if (bypass) { + if (!inc_bypass_depth(sch)) + goto unlock; + + enable_bypass_dsp(sch); + } else { + if (!dec_bypass_depth(sch)) + goto unlock; + } + + /* + * Bypass state is propagated to all descendants - an scx_sched bypasses + * if itself or any of its ancestors are in bypass mode. + */ + raw_spin_lock(&scx_sched_lock); + scx_for_each_descendant_pre(pos, sch) { + if (pos == sch) + continue; + if (bypass) + inc_bypass_depth(pos); + else + dec_bypass_depth(pos); + } + raw_spin_unlock(&scx_sched_lock); + + /* + * No task property is changing. We just need to make sure all currently + * queued tasks are re-queued according to the new scx_bypassing() + * state. As an optimization, walk each rq's runnable_list instead of + * the scx_tasks list. + * + * This function can't trust the scheduler and thus can't use + * cpus_read_lock(). Walk all possible CPUs instead of online. + */ + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + struct task_struct *p, *n; + + raw_spin_rq_lock(rq); + raw_spin_lock(&scx_sched_lock); + + scx_for_each_descendant_pre(pos, sch) { + struct scx_sched_pcpu *pcpu = per_cpu_ptr(pos->pcpu, cpu); + + if (pos->bypass_depth) + pcpu->flags |= SCX_SCHED_PCPU_BYPASSING; + else + pcpu->flags &= ~SCX_SCHED_PCPU_BYPASSING; + } + + raw_spin_unlock(&scx_sched_lock); + + /* + * We need to guarantee that no tasks are on the BPF scheduler + * while bypassing. Either we see enabled or the enable path + * sees scx_bypassing() before moving tasks to SCX. + */ + if (!scx_enabled()) { + raw_spin_rq_unlock(rq); + continue; + } + + /* + * The use of list_for_each_entry_safe_reverse() is required + * because each task is going to be removed from and added back + * to the runnable_list during iteration. Because they're added + * to the tail of the list, safe reverse iteration can still + * visit all nodes. + */ + list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list, + scx.runnable_node) { + if (!scx_is_descendant(scx_task_sched(p), sch)) + continue; + + /* cycling deq/enq is enough, see the function comment */ + scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + /* nothing */ ; + } + } + + /* resched to restore ticks and idle state */ + if (cpu_online(cpu) || cpu == smp_processor_id()) + resched_curr(rq); + + raw_spin_rq_unlock(rq); + } + + /* disarming must come after moving all tasks out of the bypass DSQs */ + if (!bypass) + disable_bypass_dsp(sch); +unlock: + raw_spin_unlock_irqrestore(&scx_bypass_lock, flags); +} + +static void free_exit_info(struct scx_exit_info *ei) +{ + kvfree(ei->dump); + kfree(ei->msg); + kfree(ei->bt); + kfree(ei); +} + +static struct scx_exit_info *alloc_exit_info(size_t exit_dump_len) +{ + struct scx_exit_info *ei; + + ei = kzalloc_obj(*ei); + if (!ei) + return NULL; + + ei->exit_cpu = -1; + ei->bt = kzalloc_objs(ei->bt[0], SCX_EXIT_BT_LEN); + ei->msg = kzalloc(SCX_EXIT_MSG_LEN, GFP_KERNEL); + ei->dump = kvzalloc(exit_dump_len, GFP_KERNEL); + + if (!ei->bt || !ei->msg || !ei->dump) { + free_exit_info(ei); + return NULL; + } + + return ei; +} + +static const char *scx_exit_reason(enum scx_exit_kind kind) +{ + switch (kind) { + case SCX_EXIT_UNREG: + return "unregistered from user space"; + case SCX_EXIT_UNREG_BPF: + return "unregistered from BPF"; + case SCX_EXIT_UNREG_KERN: + return "unregistered from the main kernel"; + case SCX_EXIT_SYSRQ: + return "disabled by sysrq-S"; + case SCX_EXIT_PARENT: + return "parent exiting"; + case SCX_EXIT_ERROR: + return "runtime error"; + case SCX_EXIT_ERROR_BPF: + return "scx_bpf_error"; + case SCX_EXIT_ERROR_STALL: + return "runnable task stall"; + default: + return ""; + } +} + +static void free_kick_syncs(void) +{ + int cpu; + + for_each_possible_cpu(cpu) { + struct scx_kick_syncs **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu); + struct scx_kick_syncs *to_free; + + to_free = rcu_replace_pointer(*ksyncs, NULL, true); + if (to_free) + kvfree_rcu(to_free, rcu); + } +} + +static void refresh_watchdog(void) +{ + struct scx_sched *sch; + unsigned long intv = ULONG_MAX; + + /* take the shortest timeout and use its half for watchdog interval */ + rcu_read_lock(); + list_for_each_entry_rcu(sch, &scx_sched_all, all) + intv = max(min(intv, sch->watchdog_timeout / 2), 1); + rcu_read_unlock(); + + WRITE_ONCE(scx_watchdog_timestamp, jiffies); + WRITE_ONCE(scx_watchdog_interval, intv); + + if (intv < ULONG_MAX) + mod_delayed_work(system_dfl_wq, &scx_watchdog_work, intv); + else + cancel_delayed_work_sync(&scx_watchdog_work); +} + +static s32 scx_link_sched(struct scx_sched *sch) +{ + const char *err_msg = ""; + s32 ret = 0; + + scoped_guard(raw_spinlock_irq, &scx_sched_lock) { +#ifdef CONFIG_EXT_SUB_SCHED + struct scx_sched *parent = scx_parent(sch); + + if (parent) { + /* + * scx_claim_exit() propagates exit_kind transition to + * its sub-scheds while holding scx_sched_lock - either + * we can see the parent's non-NONE exit_kind or the + * parent can shoot us down. + */ + if (atomic_read(&parent->exit_kind) != SCX_EXIT_NONE) { + err_msg = "parent disabled"; + ret = -ENOENT; + break; + } + + ret = rhashtable_lookup_insert_fast(&scx_sched_hash, + &sch->hash_node, scx_sched_hash_params); + if (ret) { + err_msg = "failed to insert into scx_sched_hash"; + break; + } + + list_add_tail(&sch->sibling, &parent->children); + } +#endif /* CONFIG_EXT_SUB_SCHED */ + + list_add_tail_rcu(&sch->all, &scx_sched_all); + } + + /* + * scx_error() takes scx_sched_lock via scx_claim_exit(), so it must run after + * the guard above is released. + */ + if (ret) { + scx_error(sch, "%s (%d)", err_msg, ret); + return ret; + } + + refresh_watchdog(); + return 0; +} + +static void scx_unlink_sched(struct scx_sched *sch) +{ + scoped_guard(raw_spinlock_irq, &scx_sched_lock) { +#ifdef CONFIG_EXT_SUB_SCHED + if (scx_parent(sch)) { + rhashtable_remove_fast(&scx_sched_hash, &sch->hash_node, + scx_sched_hash_params); + list_del_init(&sch->sibling); + } +#endif /* CONFIG_EXT_SUB_SCHED */ + list_del_rcu(&sch->all); + } + + refresh_watchdog(); +} + +/* + * Called to disable future dumps and wait for in-progress one while disabling + * @sch. Once @sch becomes empty during disable, there's no point in dumping it. + * This prevents calling dump ops on a dead sch. + */ +static void scx_disable_dump(struct scx_sched *sch) +{ + guard(raw_spinlock_irqsave)(&scx_dump_lock); + sch->dump_disabled = true; +} + +static void scx_log_sched_disable(struct scx_sched *sch) +{ + struct scx_exit_info *ei = sch->exit_info; + const char *type = scx_parent(sch) ? "sub-scheduler" : "scheduler"; + + if (ei->kind >= SCX_EXIT_ERROR) { + pr_err("sched_ext: BPF %s \"%s\" disabled (%s)\n", type, + sch->ops.name, ei->reason); + + if (ei->msg[0] != '\0') + pr_err("sched_ext: %s: %s\n", sch->ops.name, ei->msg); +#ifdef CONFIG_STACKTRACE + stack_trace_print(ei->bt, ei->bt_len, 2); +#endif + } else { + pr_info("sched_ext: BPF %s \"%s\" disabled (%s)\n", type, + sch->ops.name, ei->reason); + } +} + +#ifdef CONFIG_EXT_SUB_SCHED +static DECLARE_WAIT_QUEUE_HEAD(scx_unlink_waitq); + +static void drain_descendants(struct scx_sched *sch) +{ + /* + * Child scheds that finished the critical part of disabling will take + * themselves off @sch->children. Wait for it to drain. As propagation + * is recursive, empty @sch->children means that all proper descendant + * scheds reached unlinking stage. + */ + wait_event(scx_unlink_waitq, list_empty(&sch->children)); +} + +static void scx_fail_parent(struct scx_sched *sch, + struct task_struct *failed, s32 fail_code) +{ + struct scx_sched *parent = scx_parent(sch); + struct scx_task_iter sti; + struct task_struct *p; + + scx_error(parent, "ops.init_task() failed (%d) for %s[%d] while disabling a sub-scheduler", + fail_code, failed->comm, failed->pid); + + /* + * Once $parent is bypassed, it's safe to put SCX_TASK_NONE tasks into + * it. This may cause downstream failures on the BPF side but $parent is + * dying anyway. + */ + scx_bypass(parent, true); + + scx_task_iter_start(&sti, sch->cgrp); + while ((p = scx_task_iter_next_locked(&sti))) { + if (scx_task_on_sched(parent, p)) + continue; + + scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + scx_disable_and_exit_task(sch, p); + scx_set_task_sched(p, parent); + } + } + scx_task_iter_stop(&sti); +} + +static void scx_sub_disable(struct scx_sched *sch) +{ + struct scx_sched *parent = scx_parent(sch); + struct scx_task_iter sti; + struct task_struct *p; + int ret; + + /* + * Guarantee forward progress and wait for descendants to be disabled. + * To limit disruptions, $parent is not bypassed. Tasks are fully + * prepped and then inserted back into $parent. + */ + scx_bypass(sch, true); + drain_descendants(sch); + + /* + * Here, every runnable task is guaranteed to make forward progress and + * we can safely use blocking synchronization constructs. Actually + * disable ops. + */ + mutex_lock(&scx_enable_mutex); + percpu_down_write(&scx_fork_rwsem); + scx_cgroup_lock(); + + set_cgroup_sched(sch_cgroup(sch), parent); + + scx_task_iter_start(&sti, sch->cgrp); + while ((p = scx_task_iter_next_locked(&sti))) { + struct rq *rq; + struct rq_flags rf; + + /* filter out duplicate visits */ + if (scx_task_on_sched(parent, p)) + continue; + + /* + * By the time control reaches here, all descendant schedulers + * should already have been disabled. + */ + WARN_ON_ONCE(!scx_task_on_sched(sch, p)); + + /* + * @p is pinned by the iter: css_task_iter_next() takes a + * reference and holds it until the next iter_next() call, so + * @p->usage is guaranteed > 0. + */ + get_task_struct(p); + + scx_task_iter_unlock(&sti); + + /* + * $p is READY or ENABLED on @sch. Initialize for $parent, + * disable and exit from @sch, and then switch over to $parent. + * + * If a task fails to initialize for $parent, the only available + * action is disabling $parent too. While this allows disabling + * of a child sched to cause the parent scheduler to fail, the + * failure can only originate from ops.init_task() of the + * parent. A child can't directly affect the parent through its + * own failures. + */ + ret = __scx_init_task(parent, p, false); + if (ret) { + scx_fail_parent(sch, p, ret); + put_task_struct(p); + break; + } + + rq = task_rq_lock(p, &rf); + + if (scx_get_task_state(p) == SCX_TASK_DEAD) { + /* + * sched_ext_dead() raced us between __scx_init_task() + * and this rq lock and ran exit_task() on @sch (the + * sched @p was on at that point), not on $parent. + * $parent's just-completed init is owed an exit_task() + * and we issue it here. + */ + scx_sub_init_cancel_task(parent, p); + task_rq_unlock(rq, p, &rf); + put_task_struct(p); + continue; + } + + scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + /* + * $p is initialized for $parent and still attached to + * @sch. Disable and exit for @sch, switch over to + * $parent, override the state to READY to account for + * $p having already been initialized, and then enable. + */ + scx_disable_and_exit_task(sch, p); + scx_set_task_state(p, SCX_TASK_INIT_BEGIN); + scx_set_task_state(p, SCX_TASK_INIT); + scx_set_task_sched(p, parent); + scx_set_task_state(p, SCX_TASK_READY); + scx_enable_task(parent, p); + } + + task_rq_unlock(rq, p, &rf); + put_task_struct(p); + } + scx_task_iter_stop(&sti); + + scx_disable_dump(sch); + + scx_cgroup_unlock(); + percpu_up_write(&scx_fork_rwsem); + + /* + * All tasks are moved off of @sch but there may still be on-going + * operations (e.g. ops.select_cpu()). Drain them by flushing RCU. Use + * the expedited version as ancestors may be waiting in bypass mode. + * Also, tell the parent that there is no need to keep running bypass + * DSQs for us. + */ + synchronize_rcu_expedited(); + disable_bypass_dsp(sch); + + scx_unlink_sched(sch); + + mutex_unlock(&scx_enable_mutex); + + /* + * @sch is now unlinked from the parent's children list. Notify and call + * ops.sub_detach/exit(). Note that ops.sub_detach/exit() must be called + * after unlinking and releasing all locks. See scx_claim_exit(). + */ + wake_up_all(&scx_unlink_waitq); + + if (parent->ops.sub_detach && sch->sub_attached) { + struct scx_sub_detach_args sub_detach_args = { + .ops = &sch->ops, + .cgroup_path = sch->cgrp_path, + }; + SCX_CALL_OP(parent, sub_detach, NULL, + &sub_detach_args); + } + + scx_log_sched_disable(sch); + + if (sch->ops.exit) + SCX_CALL_OP(sch, exit, NULL, sch->exit_info); + if (sch->sub_kset) + kobject_del(&sch->sub_kset->kobj); + kobject_del(&sch->kobj); +} +#else /* CONFIG_EXT_SUB_SCHED */ +static inline void drain_descendants(struct scx_sched *sch) { } +static inline void scx_sub_disable(struct scx_sched *sch) { } +#endif /* CONFIG_EXT_SUB_SCHED */ + +static void scx_root_disable(struct scx_sched *sch) +{ + struct scx_task_iter sti; + struct task_struct *p; + bool was_switched_all; + int cpu; + + /* guarantee forward progress and wait for descendants to be disabled */ + scx_bypass(sch, true); + drain_descendants(sch); + + switch (scx_set_enable_state(SCX_DISABLING)) { + case SCX_DISABLING: + WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); + break; + case SCX_DISABLED: + pr_warn("sched_ext: ops error detected without ops (%s)\n", + sch->exit_info->msg); + WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_DISABLING); + goto done; + default: + break; + } + + /* + * Here, every runnable task is guaranteed to make forward progress and + * we can safely use blocking synchronization constructs. Actually + * disable ops. + */ + mutex_lock(&scx_enable_mutex); + + was_switched_all = scx_switched_all(); + + static_branch_disable(&__scx_switched_all); + WRITE_ONCE(scx_switching_all, false); + + /* + * Shut down cgroup support before tasks so that the cgroup attach path + * doesn't race against scx_disable_and_exit_task(). + */ + scx_cgroup_lock(); + scx_cgroup_exit(sch); + scx_cgroup_unlock(); + + /* + * The BPF scheduler is going away. All tasks including %TASK_DEAD ones + * must be switched out and exited synchronously. + */ + percpu_down_write(&scx_fork_rwsem); + + scx_init_task_enabled = false; + + scx_task_iter_start(&sti, NULL); + while ((p = scx_task_iter_next_locked(&sti))) { + unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; + const struct sched_class *old_class = p->sched_class; + const struct sched_class *new_class = scx_setscheduler_class(p); + + update_rq_clock(task_rq(p)); + + if (old_class != new_class) + queue_flags |= DEQUEUE_CLASS; + + scoped_guard (sched_change, p, queue_flags) { + p->sched_class = new_class; + } + + scx_disable_and_exit_task(scx_task_sched(p), p); + } + scx_task_iter_stop(&sti); + + scx_disable_dump(sch); + + scx_cgroup_lock(); + set_cgroup_sched(sch_cgroup(sch), NULL); + scx_cgroup_unlock(); + + percpu_up_write(&scx_fork_rwsem); + + /* + * Invalidate all the rq clocks to prevent getting outdated + * rq clocks from a previous scx scheduler. + * + * Also re-balance the dl_server bandwidth reservations: detach + * ext_server (no more sched_ext tasks) and reinstate fair_server if it + * was previously detached because we were running in full mode. + * + * Unlike the enable path, this runs on a recovery path that cannot + * fail, so we use dl_server_swap_bw() to atomically free ext_server's + * bandwidth and reclaim it for fair_server under the same dl_b lock. + * + * The swap can still fail with -EBUSY if someone bumped ext_server's + * runtime via debugfs between enable and disable; in that narrow case + * both servers end up detached and we just WARN. + */ + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + scx_rq_clock_invalidate(rq); + + scoped_guard(rq_lock_irqsave, rq) { + update_rq_clock(rq); + if (was_switched_all) { + if (WARN_ON_ONCE(dl_server_swap_bw(&rq->ext_server, + &rq->fair_server))) + pr_warn("failed to re-attach fair_server on CPU %d\n", cpu); + } else { + dl_server_detach_bw(&rq->ext_server); + } + } + } + + /* no task is on scx, turn off all the switches and flush in-progress calls */ + static_branch_disable(&__scx_enabled); + static_branch_disable(&__scx_is_cid_type); + if (sch->ops.flags & SCX_OPS_TID_TO_TASK) + static_branch_disable(&__scx_tid_to_task_enabled); + bitmap_zero(sch->has_op, SCX_OPI_END); + scx_idle_disable(); + synchronize_rcu(); + if (sch->ops.flags & SCX_OPS_TID_TO_TASK) + rhashtable_free_and_destroy(&scx_tid_hash, NULL, NULL); + + scx_log_sched_disable(sch); + + if (sch->ops.exit) + SCX_CALL_OP(sch, exit, NULL, sch->exit_info); + + scx_unlink_sched(sch); + + /* + * scx_root clearing must be inside cpus_read_lock(). See + * handle_hotplug(). + */ + cpus_read_lock(); + RCU_INIT_POINTER(scx_root, NULL); + cpus_read_unlock(); + + /* + * Delete the kobject from the hierarchy synchronously. Otherwise, sysfs + * could observe an object of the same name still in the hierarchy when + * the next scheduler is loaded. + */ +#ifdef CONFIG_EXT_SUB_SCHED + if (sch->sub_kset) + kobject_del(&sch->sub_kset->kobj); +#endif + kobject_del(&sch->kobj); + + free_kick_syncs(); + + mutex_unlock(&scx_enable_mutex); + + WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_DISABLING); +done: + scx_bypass(sch, false); +} + +/* + * Claim the exit on @sch. The caller must ensure that the helper kthread work + * is kicked before the current task can be preempted. Once exit_kind is + * claimed, scx_error() can no longer trigger, so if the current task gets + * preempted and the BPF scheduler fails to schedule it back, the helper work + * will never be kicked and the whole system can wedge. + */ +static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind) +{ + int none = SCX_EXIT_NONE; + + lockdep_assert_preemption_disabled(); + + if (WARN_ON_ONCE(kind == SCX_EXIT_NONE || kind == SCX_EXIT_DONE)) + kind = SCX_EXIT_ERROR; + + if (!atomic_try_cmpxchg(&sch->exit_kind, &none, kind)) + return false; + + /* + * Some CPUs may be trapped in the dispatch paths. Set the aborting + * flag to break potential live-lock scenarios, ensuring we can + * successfully reach scx_bypass(). + */ + WRITE_ONCE(sch->aborting, true); + + /* + * Propagate exits to descendants immediately. Each has a dedicated + * helper kthread and can run in parallel. While most of disabling is + * serialized, running them in separate threads allows parallelizing + * ops.exit(), which can take arbitrarily long prolonging bypass mode. + * + * To guarantee forward progress, this propagation must be in-line so + * that ->aborting is synchronously asserted for all sub-scheds. The + * propagation is also the interlocking point against sub-sched + * attachment. See scx_link_sched(). + * + * This doesn't cause recursions as propagation only takes place for + * non-propagation exits. + */ + if (kind != SCX_EXIT_PARENT) { + scoped_guard (raw_spinlock_irqsave, &scx_sched_lock) { + struct scx_sched *pos; + scx_for_each_descendant_pre(pos, sch) + scx_disable(pos, SCX_EXIT_PARENT); + } + } + + return true; +} + +static void scx_disable_workfn(struct kthread_work *work) +{ + struct scx_sched *sch = container_of(work, struct scx_sched, disable_work); + struct scx_exit_info *ei = sch->exit_info; + int kind; + + kind = atomic_read(&sch->exit_kind); + while (true) { + if (kind == SCX_EXIT_DONE) /* already disabled? */ + return; + WARN_ON_ONCE(kind == SCX_EXIT_NONE); + if (atomic_try_cmpxchg(&sch->exit_kind, &kind, SCX_EXIT_DONE)) + break; + } + ei->kind = kind; + ei->reason = scx_exit_reason(ei->kind); + + if (scx_parent(sch)) + scx_sub_disable(sch); + else + scx_root_disable(sch); +} + +static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind) +{ + guard(preempt)(); + if (scx_claim_exit(sch, kind)) + irq_work_queue(&sch->disable_irq_work); +} + +/** + * scx_flush_disable_work - flush the disable work and wait for it to finish + * @sch: the scheduler + * + * sch->disable_work might still not queued, causing kthread_flush_work() + * as a noop. Syncing the irq_work first is required to guarantee the + * kthread work has been queued before waiting for it. + */ +static void scx_flush_disable_work(struct scx_sched *sch) +{ + int kind; + + do { + irq_work_sync(&sch->disable_irq_work); + kthread_flush_work(&sch->disable_work); + kind = atomic_read(&sch->exit_kind); + } while (kind != SCX_EXIT_NONE && kind != SCX_EXIT_DONE); +} + +static void dump_newline(struct seq_buf *s) +{ + trace_sched_ext_dump(""); + + /* @s may be zero sized and seq_buf triggers WARN if so */ + if (s->size) + seq_buf_putc(s, '\n'); +} + +static __printf(2, 3) void dump_line(struct seq_buf *s, const char *fmt, ...) +{ + va_list args; + +#ifdef CONFIG_TRACEPOINTS + if (trace_sched_ext_dump_enabled()) { + /* protected by scx_dump_lock */ + static char line_buf[SCX_EXIT_MSG_LEN]; + + va_start(args, fmt); + vscnprintf(line_buf, sizeof(line_buf), fmt, args); + va_end(args); + + trace_call__sched_ext_dump(line_buf); + } +#endif + /* @s may be zero sized and seq_buf triggers WARN if so */ + if (s->size) { + va_start(args, fmt); + seq_buf_vprintf(s, fmt, args); + va_end(args); + + seq_buf_putc(s, '\n'); + } +} + +static void dump_stack_trace(struct seq_buf *s, const char *prefix, + const unsigned long *bt, unsigned int len) +{ + unsigned int i; + + for (i = 0; i < len; i++) + dump_line(s, "%s%pS", prefix, (void *)bt[i]); +} + +static void ops_dump_init(struct seq_buf *s, const char *prefix) +{ + struct scx_dump_data *dd = &scx_dump_data; + + lockdep_assert_irqs_disabled(); + + dd->cpu = smp_processor_id(); /* allow scx_bpf_dump() */ + dd->first = true; + dd->cursor = 0; + dd->s = s; + dd->prefix = prefix; +} + +static void ops_dump_flush(void) +{ + struct scx_dump_data *dd = &scx_dump_data; + char *line = dd->buf.line; + + if (!dd->cursor) + return; + + /* + * There's something to flush and this is the first line. Insert a blank + * line to distinguish ops dump. + */ + if (dd->first) { + dump_newline(dd->s); + dd->first = false; + } + + /* + * There may be multiple lines in $line. Scan and emit each line + * separately. + */ + while (true) { + char *end = line; + char c; + + while (*end != '\n' && *end != '\0') + end++; + + /* + * If $line overflowed, it may not have newline at the end. + * Always emit with a newline. + */ + c = *end; + *end = '\0'; + dump_line(dd->s, "%s%s", dd->prefix, line); + if (c == '\0') + break; + + /* move to the next line */ + end++; + if (*end == '\0') + break; + line = end; + } + + dd->cursor = 0; +} + +static void ops_dump_exit(void) +{ + ops_dump_flush(); + scx_dump_data.cpu = -1; +} + +static void scx_dump_task(struct scx_sched *sch, struct seq_buf *s, struct scx_dump_ctx *dctx, + struct rq *rq, struct task_struct *p, char marker) +{ + static unsigned long bt[SCX_EXIT_BT_LEN]; + struct scx_sched *task_sch = scx_task_sched(p); + const char *own_marker; + char sch_id_buf[32]; + char dsq_id_buf[19] = "(n/a)"; + unsigned long ops_state = atomic_long_read(&p->scx.ops_state); + unsigned int bt_len = 0; + + own_marker = task_sch == sch ? "*" : ""; + + if (task_sch->level == 0) + scnprintf(sch_id_buf, sizeof(sch_id_buf), "root"); + else + scnprintf(sch_id_buf, sizeof(sch_id_buf), "sub%d-%llu", + task_sch->level, task_sch->ops.sub_cgroup_id); + + if (p->scx.dsq) + scnprintf(dsq_id_buf, sizeof(dsq_id_buf), "0x%llx", + (unsigned long long)p->scx.dsq->id); + + dump_newline(s); + dump_line(s, " %c%c %s[%d] %s%s %+ldms", + marker, task_state_to_char(p), p->comm, p->pid, + own_marker, sch_id_buf, + jiffies_delta_msecs(p->scx.runnable_at, dctx->at_jiffies)); + dump_line(s, " scx_state/flags=%u/0x%x dsq_flags=0x%x ops_state/qseq=%lu/%lu", + scx_get_task_state(p) >> SCX_TASK_STATE_SHIFT, + p->scx.flags & ~SCX_TASK_STATE_MASK, + p->scx.dsq_flags, ops_state & SCX_OPSS_STATE_MASK, + ops_state >> SCX_OPSS_QSEQ_SHIFT); + dump_line(s, " sticky/holding_cpu=%d/%d dsq_id=%s", + p->scx.sticky_cpu, p->scx.holding_cpu, dsq_id_buf); + dump_line(s, " dsq_vtime=%llu slice=%llu weight=%u", + p->scx.dsq_vtime, p->scx.slice, p->scx.weight); + dump_line(s, " cpus=%*pb no_mig=%u", cpumask_pr_args(p->cpus_ptr), + p->migration_disabled); + + if (SCX_HAS_OP(sch, dump_task)) { + ops_dump_init(s, " "); + SCX_CALL_OP(sch, dump_task, rq, dctx, p); + ops_dump_exit(); + } + +#ifdef CONFIG_STACKTRACE + bt_len = stack_trace_save_tsk(p, bt, SCX_EXIT_BT_LEN, 1); +#endif + if (bt_len) { + dump_newline(s); + dump_stack_trace(s, " ", bt, bt_len); + } +} + +static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, + struct scx_dump_ctx *dctx, int cpu, + bool dump_all_tasks) +{ + struct rq *rq = cpu_rq(cpu); + struct rq_flags rf; + struct task_struct *p; + struct seq_buf ns; + size_t avail, used; + char *buf; + bool idle; + + rq_lock_irqsave(rq, &rf); + + idle = list_empty(&rq->scx.runnable_list) && + rq->curr->sched_class == &idle_sched_class; + + if (idle && !SCX_HAS_OP(sch, dump_cpu)) + goto next; + + /* + * We don't yet know whether ops.dump_cpu() will produce output + * and we may want to skip the default CPU dump if it doesn't. + * Use a nested seq_buf to generate the standard dump so that we + * can decide whether to commit later. + */ + avail = seq_buf_get_buf(s, &buf); + seq_buf_init(&ns, buf, avail); + + dump_newline(&ns); + dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu ksync=%lu", + cpu, rq->scx.nr_running, rq->scx.flags, + rq->scx.cpu_released, rq->scx.ops_qseq, + rq->scx.kick_sync); + dump_line(&ns, " curr=%s[%d] class=%ps", + rq->curr->comm, rq->curr->pid, + rq->curr->sched_class); + if (!cpumask_empty(rq->scx.cpus_to_kick)) + dump_line(&ns, " cpus_to_kick : %*pb", + cpumask_pr_args(rq->scx.cpus_to_kick)); + if (!cpumask_empty(rq->scx.cpus_to_kick_if_idle)) + dump_line(&ns, " idle_to_kick : %*pb", + cpumask_pr_args(rq->scx.cpus_to_kick_if_idle)); + if (!cpumask_empty(rq->scx.cpus_to_preempt)) + dump_line(&ns, " cpus_to_preempt: %*pb", + cpumask_pr_args(rq->scx.cpus_to_preempt)); + if (!cpumask_empty(rq->scx.cpus_to_wait)) + dump_line(&ns, " cpus_to_wait : %*pb", + cpumask_pr_args(rq->scx.cpus_to_wait)); + if (!cpumask_empty(rq->scx.cpus_to_sync)) + dump_line(&ns, " cpus_to_sync : %*pb", + cpumask_pr_args(rq->scx.cpus_to_sync)); + + used = seq_buf_used(&ns); + if (SCX_HAS_OP(sch, dump_cpu)) { + ops_dump_init(&ns, " "); + SCX_CALL_OP(sch, dump_cpu, rq, dctx, scx_cpu_arg(cpu), idle); + ops_dump_exit(); + } + + /* + * If idle && nothing generated by ops.dump_cpu(), there's + * nothing interesting. Skip. + */ + if (idle && used == seq_buf_used(&ns)) + goto next; + + /* + * $s may already have overflowed when $ns was created. If so, + * calling commit on it will trigger BUG. + */ + if (avail) { + seq_buf_commit(s, seq_buf_used(&ns)); + if (seq_buf_has_overflowed(&ns)) + seq_buf_set_overflow(s); + } + + if (rq->curr->sched_class == &ext_sched_class && + (dump_all_tasks || scx_task_on_sched(sch, rq->curr))) + scx_dump_task(sch, s, dctx, rq, rq->curr, '*'); + + list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) + if (dump_all_tasks || scx_task_on_sched(sch, p)) + scx_dump_task(sch, s, dctx, rq, p, ' '); +next: + rq_unlock_irqrestore(rq, &rf); +} + +/* + * Dump scheduler state. If @dump_all_tasks is true, dump all tasks regardless + * of which scheduler they belong to. If false, only dump tasks owned by @sch. + * For SysRq-D dumps, @dump_all_tasks=false since all schedulers are dumped + * separately. For error dumps, @dump_all_tasks=true since only the failing + * scheduler is dumped. + */ +static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei, + size_t dump_len, bool dump_all_tasks) +{ + static const char trunc_marker[] = "\n\n~~~~ TRUNCATED ~~~~\n"; + struct scx_dump_ctx dctx = { + .kind = ei->kind, + .exit_code = ei->exit_code, + .reason = ei->reason, + .at_ns = ktime_get_ns(), + .at_jiffies = jiffies, + }; + struct seq_buf s; + struct scx_event_stats events; + int cpu; + + guard(raw_spinlock_irqsave)(&scx_dump_lock); + + if (sch->dump_disabled) + return; + + seq_buf_init(&s, ei->dump, dump_len); + +#ifdef CONFIG_EXT_SUB_SCHED + if (sch->level == 0) + dump_line(&s, "%s: root", sch->ops.name); + else + dump_line(&s, "%s: sub%d-%llu %s", + sch->ops.name, sch->level, sch->ops.sub_cgroup_id, + sch->cgrp_path); +#endif + if (ei->kind == SCX_EXIT_NONE) { + dump_line(&s, "Debug dump triggered by %s", ei->reason); + } else { + if (ei->exit_cpu >= 0) + dump_line(&s, "%s[%d] triggered exit kind %d on CPU %d:", + current->comm, current->pid, ei->kind, + ei->exit_cpu); + else + dump_line(&s, "%s[%d] triggered exit kind %d:", + current->comm, current->pid, ei->kind); + dump_line(&s, " %s (%s)", ei->reason, ei->msg); + dump_newline(&s); + dump_line(&s, "Backtrace:"); + dump_stack_trace(&s, " ", ei->bt, ei->bt_len); + } + + if (SCX_HAS_OP(sch, dump)) { + ops_dump_init(&s, ""); + SCX_CALL_OP(sch, dump, NULL, &dctx); + ops_dump_exit(); + } + + dump_newline(&s); + dump_line(&s, "CPU states"); + dump_line(&s, "----------"); + + /* + * Dump the exit CPU first so it isn't lost to dump truncation, then + * walk the rest in order, skipping the one already dumped. + */ + if (ei->exit_cpu >= 0) + scx_dump_cpu(sch, &s, &dctx, ei->exit_cpu, dump_all_tasks); + for_each_possible_cpu(cpu) { + if (cpu != ei->exit_cpu) + scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks); + } + + dump_newline(&s); + dump_line(&s, "Event counters"); + dump_line(&s, "--------------"); + + scx_read_events(sch, &events); + scx_dump_event(s, &events, SCX_EV_SELECT_CPU_FALLBACK); + scx_dump_event(s, &events, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); + scx_dump_event(s, &events, SCX_EV_DISPATCH_KEEP_LAST); + scx_dump_event(s, &events, SCX_EV_ENQ_SKIP_EXITING); + scx_dump_event(s, &events, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); + scx_dump_event(s, &events, SCX_EV_REENQ_IMMED); + scx_dump_event(s, &events, SCX_EV_REENQ_LOCAL_REPEAT); + scx_dump_event(s, &events, SCX_EV_REFILL_SLICE_DFL); + scx_dump_event(s, &events, SCX_EV_BYPASS_DURATION); + scx_dump_event(s, &events, SCX_EV_BYPASS_DISPATCH); + scx_dump_event(s, &events, SCX_EV_BYPASS_ACTIVATE); + scx_dump_event(s, &events, SCX_EV_INSERT_NOT_OWNED); + scx_dump_event(s, &events, SCX_EV_SUB_BYPASS_DISPATCH); + + if (seq_buf_has_overflowed(&s) && dump_len >= sizeof(trunc_marker)) + memcpy(ei->dump + dump_len - sizeof(trunc_marker), + trunc_marker, sizeof(trunc_marker)); +} + +static void scx_disable_irq_workfn(struct irq_work *irq_work) +{ + struct scx_sched *sch = container_of(irq_work, struct scx_sched, disable_irq_work); + struct scx_exit_info *ei = sch->exit_info; + + if (ei->kind >= SCX_EXIT_ERROR) + scx_dump_state(sch, ei, sch->ops.exit_dump_len, true); + + kthread_queue_work(sch->helper, &sch->disable_work); +} + +bool scx_vexit(struct scx_sched *sch, + enum scx_exit_kind kind, s64 exit_code, s32 exit_cpu, + const char *fmt, va_list args) +{ + struct scx_exit_info *ei = sch->exit_info; + + guard(preempt)(); + + if (!scx_claim_exit(sch, kind)) + return false; + + ei->exit_code = exit_code; +#ifdef CONFIG_STACKTRACE + if (kind >= SCX_EXIT_ERROR) + ei->bt_len = stack_trace_save(ei->bt, SCX_EXIT_BT_LEN, 1); +#endif + vscnprintf(ei->msg, SCX_EXIT_MSG_LEN, fmt, args); + + /* + * Set ei->kind and ->reason for scx_dump_state(). They'll be set again + * in scx_disable_workfn(). + */ + ei->kind = kind; + ei->reason = scx_exit_reason(ei->kind); + ei->exit_cpu = exit_cpu; + + irq_work_queue(&sch->disable_irq_work); + return true; +} + +static int alloc_kick_syncs(void) +{ + int cpu; + + /* + * Allocate per-CPU arrays sized by nr_cpu_ids. Use kvzalloc as size + * can exceed percpu allocator limits on large machines. + */ + for_each_possible_cpu(cpu) { + struct scx_kick_syncs **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu); + struct scx_kick_syncs *new_ksyncs; + + WARN_ON_ONCE(rcu_access_pointer(*ksyncs)); + + new_ksyncs = kvzalloc_node(struct_size(new_ksyncs, syncs, nr_cpu_ids), + GFP_KERNEL, cpu_to_node(cpu)); + if (!new_ksyncs) { + free_kick_syncs(); + return -ENOMEM; + } + + rcu_assign_pointer(*ksyncs, new_ksyncs); + } + + return 0; +} + +static void free_pnode(struct scx_sched_pnode *pnode) +{ + if (!pnode) + return; + exit_dsq(&pnode->global_dsq); + kfree(pnode); +} + +static struct scx_sched_pnode *alloc_pnode(struct scx_sched *sch, int node) +{ + struct scx_sched_pnode *pnode; + + pnode = kzalloc_node(sizeof(*pnode), GFP_KERNEL, node); + if (!pnode) + return NULL; + + if (init_dsq(&pnode->global_dsq, SCX_DSQ_GLOBAL, sch)) { + kfree(pnode); + return NULL; + } + + return pnode; +} + +/* + * scx_enable() is offloaded to a dedicated system-wide RT kthread to avoid + * starvation. During the READY -> ENABLED task switching loop, the calling + * thread's sched_class gets switched from fair to ext. As fair has higher + * priority than ext, the calling thread can be indefinitely starved under + * fair-class saturation, leading to a system hang. + */ +struct scx_enable_cmd { + struct kthread_work work; + union { + struct sched_ext_ops *ops; + struct sched_ext_ops_cid *ops_cid; + }; + bool is_cid_type; + struct bpf_map *arena_map; /* arena ref to transfer to sch */ + int ret; +}; + +/* + * Allocate and initialize a new scx_sched. @cgrp's reference is always + * consumed whether the function succeeds or fails. + */ +static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, + struct cgroup *cgrp, + struct scx_sched *parent) +{ + struct sched_ext_ops *ops = cmd->ops; + struct scx_sched *sch; + s32 level = parent ? parent->level + 1 : 0; + s32 node, cpu, ret, bypass_fail_cpu = nr_cpu_ids; + + sch = kzalloc_flex(*sch, ancestors, level + 1); + if (!sch) { + ret = -ENOMEM; + goto err_put_cgrp; + } + + sch->exit_info = alloc_exit_info(ops->exit_dump_len); + if (!sch->exit_info) { + ret = -ENOMEM; + goto err_free_sch; + } + + ret = rhashtable_init(&sch->dsq_hash, &dsq_hash_params); + if (ret < 0) + goto err_free_ei; + + sch->pnode = kzalloc_objs(sch->pnode[0], nr_node_ids); + if (!sch->pnode) { + ret = -ENOMEM; + goto err_free_hash; + } + + for_each_node_state(node, N_POSSIBLE) { + sch->pnode[node] = alloc_pnode(sch, node); + if (!sch->pnode[node]) { + ret = -ENOMEM; + goto err_free_pnode; + } + } + + sch->dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; + sch->pcpu = __alloc_percpu(struct_size_t(struct scx_sched_pcpu, + dsp_ctx.buf, sch->dsp_max_batch), + __alignof__(struct scx_sched_pcpu)); + if (!sch->pcpu) { + ret = -ENOMEM; + goto err_free_pnode; + } + + for_each_possible_cpu(cpu) { + ret = init_dsq(bypass_dsq(sch, cpu), SCX_DSQ_BYPASS, sch); + if (ret) { + bypass_fail_cpu = cpu; + goto err_free_pcpu; + } + } + + for_each_possible_cpu(cpu) { + struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); + + pcpu->sch = sch; + INIT_LIST_HEAD(&pcpu->deferred_reenq_local.node); + } + + sch->helper = kthread_run_worker(0, "sched_ext_helper"); + if (IS_ERR(sch->helper)) { + ret = PTR_ERR(sch->helper); + goto err_free_pcpu; + } + + sched_set_fifo(sch->helper->task); + + if (parent) + memcpy(sch->ancestors, parent->ancestors, + level * sizeof(parent->ancestors[0])); + sch->ancestors[level] = sch; + sch->level = level; + + if (ops->timeout_ms) + sch->watchdog_timeout = msecs_to_jiffies(ops->timeout_ms); + else + sch->watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT; + + sch->slice_dfl = SCX_SLICE_DFL; + atomic_set(&sch->exit_kind, SCX_EXIT_NONE); + sch->disable_irq_work = IRQ_WORK_INIT_HARD(scx_disable_irq_workfn); + kthread_init_work(&sch->disable_work, scx_disable_workfn); + timer_setup(&sch->bypass_lb_timer, scx_bypass_lb_timerfn, 0); + + if (!alloc_cpumask_var(&sch->bypass_lb_donee_cpumask, GFP_KERNEL)) { + ret = -ENOMEM; + goto err_stop_helper; + } + if (!alloc_cpumask_var(&sch->bypass_lb_resched_cpumask, GFP_KERNEL)) { + ret = -ENOMEM; + goto err_free_lb_cpumask; + } + /* + * Copy ops through the right union view. For cid-form the source is + * struct sched_ext_ops_cid which lacks the trailing cpu_acquire/ + * cpu_release; those stay zero from kzalloc. + */ + if (cmd->is_cid_type) { + sch->ops_cid = *cmd->ops_cid; + sch->is_cid_type = true; + } else { + sch->ops = *cmd->ops; + } + + rcu_assign_pointer(ops->priv, sch); + + sch->kobj.kset = scx_kset; + INIT_LIST_HEAD(&sch->all); + +#ifdef CONFIG_EXT_SUB_SCHED + char *buf = kzalloc(PATH_MAX, GFP_KERNEL); + if (!buf) { + ret = -ENOMEM; + goto err_free_lb_resched; + } + cgroup_path(cgrp, buf, PATH_MAX); + sch->cgrp_path = kstrdup(buf, GFP_KERNEL); + kfree(buf); + if (!sch->cgrp_path) { + ret = -ENOMEM; + goto err_free_lb_resched; + } + + sch->cgrp = cgrp; + INIT_LIST_HEAD(&sch->children); + INIT_LIST_HEAD(&sch->sibling); + + if (parent) + ret = kobject_init_and_add(&sch->kobj, &scx_ktype, + &parent->sub_kset->kobj, + "sub-%llu", cgroup_id(cgrp)); + else + ret = kobject_init_and_add(&sch->kobj, &scx_ktype, NULL, "root"); + + if (ret < 0) { + RCU_INIT_POINTER(ops->priv, NULL); + kobject_put(&sch->kobj); + return ERR_PTR(ret); + } + + if (ops->sub_attach) { + sch->sub_kset = kset_create_and_add("sub", NULL, &sch->kobj); + if (!sch->sub_kset) { + RCU_INIT_POINTER(ops->priv, NULL); + kobject_put(&sch->kobj); + return ERR_PTR(-ENOMEM); + } + } +#else /* CONFIG_EXT_SUB_SCHED */ + ret = kobject_init_and_add(&sch->kobj, &scx_ktype, NULL, "root"); + if (ret < 0) { + RCU_INIT_POINTER(ops->priv, NULL); + kobject_put(&sch->kobj); + return ERR_PTR(ret); + } +#endif /* CONFIG_EXT_SUB_SCHED */ + + /* + * Consume the arena_map ref bpf_scx_reg_cid() took. Defer to here so + * earlier failure paths leave cmd->arena_map set and bpf_scx_reg_cid + * drops the ref. After this point, sch owns the ref and any cleanup + * runs through scx_sched_free_rcu_work() which puts it. + */ + sch->arena_map = cmd->arena_map; + /* BPF arena is only available on MMU && 64BIT */ +#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) + if (sch->arena_map) + sch->arena_kern_base = bpf_arena_map_kern_vm_start(sch->arena_map); +#endif + cmd->arena_map = NULL; + return sch; + +#ifdef CONFIG_EXT_SUB_SCHED +err_free_lb_resched: + RCU_INIT_POINTER(ops->priv, NULL); + free_cpumask_var(sch->bypass_lb_resched_cpumask); +#endif +err_free_lb_cpumask: + free_cpumask_var(sch->bypass_lb_donee_cpumask); +err_stop_helper: + kthread_destroy_worker(sch->helper); +err_free_pcpu: + for_each_possible_cpu(cpu) { + if (cpu == bypass_fail_cpu) + break; + exit_dsq(bypass_dsq(sch, cpu)); + } + free_percpu(sch->pcpu); +err_free_pnode: + for_each_node_state(node, N_POSSIBLE) + free_pnode(sch->pnode[node]); + kfree(sch->pnode); +err_free_hash: + rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL); +err_free_ei: + free_exit_info(sch->exit_info); +err_free_sch: + kfree(sch); +err_put_cgrp: +#ifdef CONFIG_EXT_SUB_SCHED + cgroup_put(cgrp); +#endif + return ERR_PTR(ret); +} + +static int check_hotplug_seq(struct scx_sched *sch, + const struct sched_ext_ops *ops) +{ + unsigned long long global_hotplug_seq; + + /* + * If a hotplug event has occurred between when a scheduler was + * initialized, and when we were able to attach, exit and notify user + * space about it. + */ + if (ops->hotplug_seq) { + global_hotplug_seq = atomic_long_read(&scx_hotplug_seq); + if (ops->hotplug_seq != global_hotplug_seq) { + scx_exit(sch, SCX_EXIT_UNREG_KERN, + SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG, + "expected hotplug seq %llu did not match actual %llu", + ops->hotplug_seq, global_hotplug_seq); + return -EBUSY; + } + } + + return 0; +} + +static int validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) +{ + /* + * It doesn't make sense to specify the SCX_OPS_ENQ_LAST flag if the + * ops.enqueue() callback isn't implemented. + */ + if ((ops->flags & SCX_OPS_ENQ_LAST) && !ops->enqueue) { + scx_error(sch, "SCX_OPS_ENQ_LAST requires ops.enqueue() to be implemented"); + return -EINVAL; + } + + /* + * SCX_OPS_TID_TO_TASK is enabled by the root scheduler. A sub-sched + * may set it to declare a dependency; reject if the root hasn't + * enabled it. + */ + if ((ops->flags & SCX_OPS_TID_TO_TASK) && scx_parent(sch) && + !(scx_root->ops.flags & SCX_OPS_TID_TO_TASK)) { + scx_error(sch, "SCX_OPS_TID_TO_TASK requires root scheduler to enable it"); + return -EINVAL; + } + + /* + * SCX_OPS_BUILTIN_IDLE_PER_NODE requires built-in CPU idle + * selection policy to be enabled. + */ + if ((ops->flags & SCX_OPS_BUILTIN_IDLE_PER_NODE) && + (ops->update_idle && !(ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE))) { + scx_error(sch, "SCX_OPS_BUILTIN_IDLE_PER_NODE requires CPU idle selection enabled"); + return -EINVAL; + } + + /* + * cid-form's struct is shorter and doesn't include the cpu_acquire / + * cpu_release tail; reading those fields off a cid-form @ops would + * run past the BPF allocation. Skip for cid-form. + */ + if (!sch->is_cid_type && (ops->cpu_acquire || ops->cpu_release)) + pr_warn("ops->cpu_acquire/release() are deprecated, use sched_switch TP instead\n"); + + /* + * Sub-scheduler support is tied to the cid-form struct_ops. A sub-sched + * attaches through a cid-form-only interface (sub_attach/sub_detach), + * and a root that accepts sub-scheds must expose cid-form state to + * them. Reject cpu-form schedulers on either side. + */ + if (!sch->is_cid_type) { + if (scx_parent(sch)) { + scx_error(sch, "sub-sched requires cid-form struct_ops"); + return -EINVAL; + } + if (ops->sub_attach || ops->sub_detach) { + scx_error(sch, "sub_attach/sub_detach requires cid-form struct_ops"); + return -EINVAL; + } + } + + return 0; +} + +static void scx_root_enable_workfn(struct kthread_work *work) +{ + struct scx_enable_cmd *cmd = container_of(work, struct scx_enable_cmd, work); + struct sched_ext_ops *ops = cmd->ops; + struct cgroup *cgrp = root_cgroup(); + struct scx_sched *sch; + struct scx_task_iter sti; + struct task_struct *p; + int i, cpu, ret; + + mutex_lock(&scx_enable_mutex); + + if (scx_enable_state() != SCX_DISABLED) { + ret = -EBUSY; + goto err_unlock; + } + + /* + * @ops->priv binds @ops to its scx_sched instance. It is set here by + * scx_alloc_and_add_sched() and cleared at the tail of bpf_scx_unreg(), + * which runs after scx_root_disable() has dropped scx_enable_mutex. If + * it's still non-NULL here, a previous attachment on @ops has not + * finished tearing down; proceeding would let the in-flight unreg's + * RCU_INIT_POINTER(NULL) clobber the @ops->priv we are about to assign. + */ + if (rcu_access_pointer(ops->priv)) { + ret = -EBUSY; + goto err_unlock; + } + + ret = alloc_kick_syncs(); + if (ret) + goto err_unlock; + + if (ops->flags & SCX_OPS_TID_TO_TASK) { + ret = rhashtable_init(&scx_tid_hash, &scx_tid_hash_params); + if (ret) + goto err_free_ksyncs; + } + +#ifdef CONFIG_EXT_SUB_SCHED + cgroup_get(cgrp); +#endif + sch = scx_alloc_and_add_sched(cmd, cgrp, NULL); + if (IS_ERR(sch)) { + ret = PTR_ERR(sch); + goto err_free_tid_hash; + } + + if (sch->is_cid_type) + static_branch_enable(&__scx_is_cid_type); + + /* + * Transition to ENABLING and clear exit info to arm the disable path. + * Failure triggers full disabling from here on. + */ + WARN_ON_ONCE(scx_set_enable_state(SCX_ENABLING) != SCX_DISABLED); + WARN_ON_ONCE(scx_root); + + atomic_long_set(&scx_nr_rejected, 0); + + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + rq->scx.local_dsq.sched = sch; + rq->scx.cpuperf_target = SCX_CPUPERF_ONE; + } + + /* + * Keep CPUs stable during enable so that the BPF scheduler can track + * online CPUs by watching ->on/offline_cpu() after ->init(). + */ + cpus_read_lock(); + + /* + * Build the cid mapping before publishing scx_root. The cid kfuncs + * dereference the cid arrays unconditionally once scx_prog_sched() + * returns non-NULL; the rcu_assign_pointer() below pairs with their + * rcu_dereference() to make the populated arrays visible. + */ + ret = scx_cid_init(sch); + if (ret) { + cpus_read_unlock(); + goto err_disable; + } + + /* + * Make the scheduler instance visible. Must be inside cpus_read_lock(). + * See handle_hotplug(). + */ + rcu_assign_pointer(scx_root, sch); + + ret = scx_link_sched(sch); + if (ret) { + cpus_read_unlock(); + goto err_disable; + } + + scx_idle_enable(ops); + + if (sch->ops.init) { + ret = SCX_CALL_OP_RET(sch, init, NULL); + if (ret) { + ret = ops_sanitize_err(sch, "init", ret); + cpus_read_unlock(); + scx_error(sch, "ops.init() failed (%d)", ret); + goto err_disable; + } + sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; + } + + ret = scx_arena_pool_init(sch); + if (ret) { + cpus_read_unlock(); + goto err_disable; + } + + ret = scx_set_cmask_scratch_alloc(sch); + if (ret) { + cpus_read_unlock(); + goto err_disable; + } + + for (i = SCX_OPI_CPU_HOTPLUG_BEGIN; i < SCX_OPI_CPU_HOTPLUG_END; i++) + if (((void (**)(void))ops)[i]) + set_bit(i, sch->has_op); + + ret = check_hotplug_seq(sch, ops); + if (ret) { + cpus_read_unlock(); + goto err_disable; + } + scx_idle_update_selcpu_topology(ops); + + cpus_read_unlock(); + + ret = validate_ops(sch, ops); + if (ret) + goto err_disable; + + /* + * Attach the ext_server bandwidth reservation before anything is + * committed so that we can fail the enable if the root domain cannot + * accommodate it. The matching fair_server detach is deferred to the + * tail of this function, after the switch is fully committed and can no + * longer fail. + * + * On failure, err_disable funnels into scx_root_disable() which + * detaches ext_server, so partially-attached state is cleaned up + * automatically. + */ + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + scoped_guard(rq_lock_irqsave, rq) { + update_rq_clock(rq); + ret = dl_server_attach_bw(&rq->ext_server); + } + if (ret) { + pr_warn("sched_ext: failed to attach ext_server on CPU %d (%d)\n", + cpu, ret); + goto err_disable; + } + } + + /* + * Once __scx_enabled is set, %current can be switched to SCX anytime. + * This can lead to stalls as some BPF schedulers (e.g. userspace + * scheduling) may not function correctly before all tasks are switched. + * Init in bypass mode to guarantee forward progress. + */ + scx_bypass(sch, true); + + for (i = SCX_OPI_NORMAL_BEGIN; i < SCX_OPI_NORMAL_END; i++) + if (((void (**)(void))ops)[i]) + set_bit(i, sch->has_op); + + if (sch->ops.cpu_acquire || sch->ops.cpu_release) + sch->ops.flags |= SCX_OPS_HAS_CPU_PREEMPT; + + /* + * Lock out forks, cgroup on/offlining and moves before opening the + * floodgate so that they don't wander into the operations prematurely. + */ + percpu_down_write(&scx_fork_rwsem); + + WARN_ON_ONCE(scx_init_task_enabled); + scx_init_task_enabled = true; + + /* flip under fork_rwsem; the iter below covers existing tasks */ + if (ops->flags & SCX_OPS_TID_TO_TASK) + static_branch_enable(&__scx_tid_to_task_enabled); + + /* + * Enable ops for every task. Fork is excluded by scx_fork_rwsem + * preventing new tasks from being added. No need to exclude tasks + * leaving as sched_ext_free() can handle both prepped and enabled + * tasks. Prep all tasks first and then enable them with preemption + * disabled. + * + * All cgroups should be initialized before scx_init_task() so that the + * BPF scheduler can reliably track each task's cgroup membership from + * scx_init_task(). Lock out cgroup on/offlining and task migrations + * while tasks are being initialized so that scx_cgroup_can_attach() + * never sees uninitialized tasks. + */ + scx_cgroup_lock(); + set_cgroup_sched(sch_cgroup(sch), sch); + ret = scx_cgroup_init(sch); + if (ret) + goto err_disable_unlock_all; + + scx_task_iter_start(&sti, NULL); + while ((p = scx_task_iter_next_locked(&sti))) { + /* + * @p is in scx_tasks under scx_tasks_lock, and SCX_TASK_DEAD + * tasks are filtered by scx_task_iter_next_locked(). + * sched_ext_dead() removes @p from scx_tasks under the same + * lock before put_task_struct_rcu_user() runs, so @p->usage + * is guaranteed > 0 here. + */ + get_task_struct(p); + + /* + * Set %INIT_BEGIN under the iter's rq lock so that a concurrent + * sched_ext_dead() does not call ops.exit_task() on @p while + * ops.init_task() is running. If sched_ext_dead() runs before + * this store, it has already removed @p from scx_tasks and the + * iter won't visit @p; if it runs after, it observes + * %INIT_BEGIN and transitions to %DEAD without calling ops, + * leaving the post-init recheck below to unwind. + */ + scx_set_task_state(p, SCX_TASK_INIT_BEGIN); + scx_task_iter_unlock(&sti); + + ret = __scx_init_task(sch, p, false); + + scx_task_iter_relock(&sti, p); + + if (unlikely(ret)) { + if (scx_get_task_state(p) != SCX_TASK_DEAD) + scx_set_task_state(p, SCX_TASK_NONE); + scx_task_iter_stop(&sti); + scx_error(sch, "ops.init_task() failed (%d) for %s[%d]", + ret, p->comm, p->pid); + put_task_struct(p); + goto err_disable_unlock_all; + } + + if (scx_get_task_state(p) == SCX_TASK_DEAD) { + /* + * sched_ext_dead() observed %INIT_BEGIN and set %DEAD. + * ops.exit_task() is owed to the sched __scx_init_task() + * ran against; call it now. + */ + scx_sub_init_cancel_task(sch, p); + } else { + scx_set_task_state(p, SCX_TASK_INIT); + scx_set_task_sched(p, sch); + scx_set_task_state(p, SCX_TASK_READY); + } + + /* + * Insert into the tid hash. scx_tasks_lock is held by the iter; + * list_empty() guards against sched_ext_dead() having taken @p + * off the list while init ran unlocked. + */ + if (scx_tid_to_task_enabled() && !list_empty(&p->scx.tasks_node)) + scx_tid_hash_insert(p); + + put_task_struct(p); + } + scx_task_iter_stop(&sti); + scx_cgroup_unlock(); + percpu_up_write(&scx_fork_rwsem); + + /* + * All tasks are READY. It's safe to turn on scx_enabled() and switch + * all eligible tasks. + */ + WRITE_ONCE(scx_switching_all, !(ops->flags & SCX_OPS_SWITCH_PARTIAL)); + static_branch_enable(&__scx_enabled); + + /* + * We're fully committed and can't fail. The task READY -> ENABLED + * transitions here are synchronized against sched_ext_free() through + * scx_tasks_lock. + */ + percpu_down_write(&scx_fork_rwsem); + scx_task_iter_start(&sti, NULL); + while ((p = scx_task_iter_next_locked(&sti))) { + unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE; + const struct sched_class *old_class = p->sched_class; + const struct sched_class *new_class = scx_setscheduler_class(p); + + if (scx_get_task_state(p) != SCX_TASK_READY) + continue; + + if (old_class != new_class) + queue_flags |= DEQUEUE_CLASS; + + scoped_guard (sched_change, p, queue_flags) { + p->scx.slice = READ_ONCE(sch->slice_dfl); + p->sched_class = new_class; + } + } + scx_task_iter_stop(&sti); + percpu_up_write(&scx_fork_rwsem); + + scx_bypass(sch, false); + + if (!scx_tryset_enable_state(SCX_ENABLED, SCX_ENABLING)) { + WARN_ON_ONCE(atomic_read(&sch->exit_kind) == SCX_EXIT_NONE); + goto err_disable; + } + + if (!(ops->flags & SCX_OPS_SWITCH_PARTIAL)) + static_branch_enable(&__scx_switched_all); + + /* + * Detach the fair_server bandwidth reservation now that the switch + * is fully committed. In full mode (!SCX_OPS_SWITCH_PARTIAL) no + * task will ever run in the fair class, so give that bandwidth + * back to the RT class. The matching ext_server attach already + * happened earlier; this only releases bandwidth and cannot fail. + * + * In partial mode keep fair_server attached. + */ + if (scx_switched_all()) { + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + guard(rq_lock_irqsave)(rq); + update_rq_clock(rq); + dl_server_detach_bw(&rq->fair_server); + } + } + + pr_info("sched_ext: BPF scheduler \"%s\" enabled%s\n", + sch->ops.name, scx_switched_all() ? "" : " (partial)"); + kobject_uevent(&sch->kobj, KOBJ_ADD); + mutex_unlock(&scx_enable_mutex); + + atomic_long_inc(&scx_enable_seq); + + cmd->ret = 0; + return; + +err_free_tid_hash: + if (ops->flags & SCX_OPS_TID_TO_TASK) + rhashtable_free_and_destroy(&scx_tid_hash, NULL, NULL); +err_free_ksyncs: + free_kick_syncs(); +err_unlock: + mutex_unlock(&scx_enable_mutex); + cmd->ret = ret; + return; + +err_disable_unlock_all: + scx_cgroup_unlock(); + percpu_up_write(&scx_fork_rwsem); + /* we'll soon enter disable path, keep bypass on */ +err_disable: + mutex_unlock(&scx_enable_mutex); + /* + * Returning an error code here would not pass all the error information + * to userspace. Record errno using scx_error() for cases scx_error() + * wasn't already invoked and exit indicating success so that the error + * is notified through ops.exit() with all the details. + * + * Flush scx_disable_work to ensure that error is reported before init + * completion. sch's base reference will be put by bpf_scx_unreg(). + */ + scx_error(sch, "scx_root_enable() failed (%d)", ret); + scx_flush_disable_work(sch); + cmd->ret = 0; +} + +#ifdef CONFIG_EXT_SUB_SCHED +/* verify that a scheduler can be attached to @cgrp and return the parent */ +static struct scx_sched *find_parent_sched(struct cgroup *cgrp) +{ + struct scx_sched *parent = cgrp->scx_sched; + struct scx_sched *pos; + + lockdep_assert_held(&scx_sched_lock); + + /* can't attach twice to the same cgroup */ + if (parent->cgrp == cgrp) + return ERR_PTR(-EBUSY); + + /* does $parent allow sub-scheds? */ + if (!parent->ops.sub_attach) + return ERR_PTR(-EOPNOTSUPP); + + /* can't insert between $parent and its exiting children */ + list_for_each_entry(pos, &parent->children, sibling) + if (cgroup_is_descendant(pos->cgrp, cgrp)) + return ERR_PTR(-EBUSY); + + return parent; +} + +static bool assert_task_ready_or_enabled(struct task_struct *p) +{ + u32 state = scx_get_task_state(p); + + switch (state) { + case SCX_TASK_READY: + case SCX_TASK_ENABLED: + return true; + default: + WARN_ONCE(true, "sched_ext: Invalid task state %d for %s[%d] during enabling sub sched", + state, p->comm, p->pid); + return false; + } +} + +static void scx_sub_enable_workfn(struct kthread_work *work) +{ + struct scx_enable_cmd *cmd = container_of(work, struct scx_enable_cmd, work); + struct sched_ext_ops *ops = cmd->ops; + struct cgroup *cgrp; + struct scx_sched *parent, *sch; + struct scx_task_iter sti; + struct task_struct *p; + s32 i, ret; + + mutex_lock(&scx_enable_mutex); + + if (!scx_enabled()) { + ret = -ENODEV; + goto out_unlock; + } + + /* See scx_root_enable_workfn() for the @ops->priv check. */ + if (rcu_access_pointer(ops->priv)) { + ret = -EBUSY; + goto out_unlock; + } + + cgrp = cgroup_get_from_id(ops->sub_cgroup_id); + if (IS_ERR(cgrp)) { + ret = PTR_ERR(cgrp); + goto out_unlock; + } + + raw_spin_lock_irq(&scx_sched_lock); + parent = find_parent_sched(cgrp); + if (IS_ERR(parent)) { + raw_spin_unlock_irq(&scx_sched_lock); + ret = PTR_ERR(parent); + goto out_put_cgrp; + } + kobject_get(&parent->kobj); + raw_spin_unlock_irq(&scx_sched_lock); + + /* scx_alloc_and_add_sched() consumes @cgrp whether it succeeds or not */ + sch = scx_alloc_and_add_sched(cmd, cgrp, parent); + kobject_put(&parent->kobj); + if (IS_ERR(sch)) { + ret = PTR_ERR(sch); + goto out_unlock; + } + + ret = scx_link_sched(sch); + if (ret) + goto err_disable; + + if (sch->level >= SCX_SUB_MAX_DEPTH) { + scx_error(sch, "max nesting depth %d violated", + SCX_SUB_MAX_DEPTH); + goto err_disable; + } + + if (sch->ops.init) { + ret = SCX_CALL_OP_RET(sch, init, NULL); + if (ret) { + ret = ops_sanitize_err(sch, "init", ret); + scx_error(sch, "ops.init() failed (%d)", ret); + goto err_disable; + } + sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; + } + + ret = scx_arena_pool_init(sch); + if (ret) + goto err_disable; + + ret = scx_set_cmask_scratch_alloc(sch); + if (ret) + goto err_disable; + + if (validate_ops(sch, ops)) + goto err_disable; + + struct scx_sub_attach_args sub_attach_args = { + .ops = &sch->ops, + .cgroup_path = sch->cgrp_path, + }; + + ret = SCX_CALL_OP_RET(parent, sub_attach, NULL, + &sub_attach_args); + if (ret) { + ret = ops_sanitize_err(sch, "sub_attach", ret); + scx_error(sch, "parent rejected (%d)", ret); + goto err_disable; + } + sch->sub_attached = true; + + scx_bypass(sch, true); + + for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++) + if (((void (**)(void))ops)[i]) + set_bit(i, sch->has_op); + + percpu_down_write(&scx_fork_rwsem); + scx_cgroup_lock(); + + /* + * Set cgroup->scx_sched's and check CSS_ONLINE. Either we see + * !CSS_ONLINE or scx_cgroup_lifetime_notify() sees and shoots us down. + */ + set_cgroup_sched(sch_cgroup(sch), sch); + if (!(cgrp->self.flags & CSS_ONLINE)) { + scx_error(sch, "cgroup is not online"); + goto err_unlock_and_disable; + } + + /* + * Initialize tasks for the new child $sch without exiting them for + * $parent so that the tasks can always be reverted back to $parent + * sched on child init failure. + */ + WARN_ON_ONCE(scx_enabling_sub_sched); + scx_enabling_sub_sched = sch; + + scx_task_iter_start(&sti, sch->cgrp); + while ((p = scx_task_iter_next_locked(&sti))) { + struct rq *rq; + struct rq_flags rf; + + /* + * Task iteration may visit the same task twice when racing + * against exiting. Use %SCX_TASK_SUB_INIT to mark tasks which + * finished __scx_init_task() and skip if set. + * + * A task may exit and get freed between __scx_init_task() + * completion and scx_enable_task(). In such cases, + * scx_disable_and_exit_task() must exit the task for both the + * parent and child scheds. + */ + if (p->scx.flags & SCX_TASK_SUB_INIT) + continue; + + /* @p is pinned by the iter; see scx_sub_disable() */ + get_task_struct(p); + + if (!assert_task_ready_or_enabled(p)) { + ret = -EINVAL; + goto abort; + } + + scx_task_iter_unlock(&sti); + + /* + * As $p is still on $parent, it can't be transitioned to INIT. + * Let's worry about task state later. Use __scx_init_task(). + */ + ret = __scx_init_task(sch, p, false); + if (ret) + goto abort; + + rq = task_rq_lock(p, &rf); + + if (scx_get_task_state(p) == SCX_TASK_DEAD) { + /* + * sched_ext_dead() raced us between __scx_init_task() + * and this rq lock and ran exit_task() on $parent (the + * sched @p was on at that point), not on @sch. @sch's + * just-completed init is owed an exit_task() and we + * issue it here. + */ + scx_sub_init_cancel_task(sch, p); + task_rq_unlock(rq, p, &rf); + put_task_struct(p); + continue; + } + + p->scx.flags |= SCX_TASK_SUB_INIT; + task_rq_unlock(rq, p, &rf); + + put_task_struct(p); + } + scx_task_iter_stop(&sti); + + /* + * All tasks are prepped. Disable/exit tasks for $parent and enable for + * the new @sch. + */ + scx_task_iter_start(&sti, sch->cgrp); + while ((p = scx_task_iter_next_locked(&sti))) { + /* + * Use clearing of %SCX_TASK_SUB_INIT to detect and skip + * duplicate iterations. + */ + if (!(p->scx.flags & SCX_TASK_SUB_INIT)) + continue; + + scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + /* + * $p must be either READY or ENABLED. If ENABLED, + * __scx_disabled_and_exit_task() first disables and + * makes it READY. However, after exiting $p, it will + * leave $p as READY. + */ + assert_task_ready_or_enabled(p); + __scx_disable_and_exit_task(parent, p); + + /* + * $p is now only initialized for @sch and READY, which + * is what we want. Assign it to @sch and enable. + */ + scx_set_task_sched(p, sch); + scx_enable_task(sch, p); + + p->scx.flags &= ~SCX_TASK_SUB_INIT; + } + } + scx_task_iter_stop(&sti); + + scx_enabling_sub_sched = NULL; + + scx_cgroup_unlock(); + percpu_up_write(&scx_fork_rwsem); + + scx_bypass(sch, false); + + pr_info("sched_ext: BPF sub-scheduler \"%s\" enabled\n", sch->ops.name); + kobject_uevent(&sch->kobj, KOBJ_ADD); + ret = 0; + goto out_unlock; + +out_put_cgrp: + cgroup_put(cgrp); +out_unlock: + mutex_unlock(&scx_enable_mutex); + cmd->ret = ret; + return; + +abort: + put_task_struct(p); + scx_task_iter_stop(&sti); + + /* + * Undo __scx_init_task() for tasks we marked. scx_enable_task() never + * ran for @sch on them, so calling scx_disable_task() here would invoke + * ops.disable() without a matching ops.enable(). scx_enabling_sub_sched + * must stay set until SUB_INIT is cleared from every marked task - + * scx_disable_and_exit_task() reads it when a task exits concurrently. + */ + scx_task_iter_start(&sti, sch->cgrp); + while ((p = scx_task_iter_next_locked(&sti))) { + if (p->scx.flags & SCX_TASK_SUB_INIT) { + scx_sub_init_cancel_task(sch, p); + p->scx.flags &= ~SCX_TASK_SUB_INIT; + } + } + scx_task_iter_stop(&sti); + scx_enabling_sub_sched = NULL; +err_unlock_and_disable: + /* we'll soon enter disable path, keep bypass on */ + scx_cgroup_unlock(); + percpu_up_write(&scx_fork_rwsem); +err_disable: + mutex_unlock(&scx_enable_mutex); + scx_flush_disable_work(sch); + cmd->ret = 0; +} + +static s32 scx_cgroup_lifetime_notify(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct cgroup *cgrp = data; + struct cgroup *parent = cgroup_parent(cgrp); + + if (!cgroup_on_dfl(cgrp)) + return NOTIFY_OK; + + switch (action) { + case CGROUP_LIFETIME_ONLINE: + /* inherit ->scx_sched from $parent */ + if (parent) + rcu_assign_pointer(cgrp->scx_sched, parent->scx_sched); + break; + case CGROUP_LIFETIME_OFFLINE: + /* if there is a sched attached, shoot it down */ + if (cgrp->scx_sched && cgrp->scx_sched->cgrp == cgrp) + scx_exit(cgrp->scx_sched, SCX_EXIT_UNREG_KERN, + SCX_ECODE_RSN_CGROUP_OFFLINE, + "cgroup %llu going offline", cgroup_id(cgrp)); + break; + } + + return NOTIFY_OK; +} + +static struct notifier_block scx_cgroup_lifetime_nb = { + .notifier_call = scx_cgroup_lifetime_notify, +}; + +static s32 __init scx_cgroup_lifetime_notifier_init(void) +{ + return blocking_notifier_chain_register(&cgroup_lifetime_notifier, + &scx_cgroup_lifetime_nb); +} +core_initcall(scx_cgroup_lifetime_notifier_init); +#endif /* CONFIG_EXT_SUB_SCHED */ + +static s32 scx_enable(struct scx_enable_cmd *cmd, struct bpf_link *link) +{ + static struct kthread_worker *helper; + static DEFINE_MUTEX(helper_mutex); + + if (housekeeping_enabled(HK_TYPE_DOMAIN_BOOT)) { + pr_err("sched_ext: Not compatible with \"isolcpus=\" domain isolation\n"); + return -EINVAL; + } + + if (!READ_ONCE(helper)) { + mutex_lock(&helper_mutex); + if (!helper) { + struct kthread_worker *w = + kthread_run_worker(0, "scx_enable_helper"); + if (IS_ERR_OR_NULL(w)) { + mutex_unlock(&helper_mutex); + return -ENOMEM; + } + sched_set_fifo(w->task); + WRITE_ONCE(helper, w); + } + mutex_unlock(&helper_mutex); + } + +#ifdef CONFIG_EXT_SUB_SCHED + if (cmd->ops->sub_cgroup_id > 1) + kthread_init_work(&cmd->work, scx_sub_enable_workfn); + else +#endif /* CONFIG_EXT_SUB_SCHED */ + kthread_init_work(&cmd->work, scx_root_enable_workfn); + + kthread_queue_work(READ_ONCE(helper), &cmd->work); + kthread_flush_work(&cmd->work); + return cmd->ret; +} + + +/******************************************************************************** + * bpf_struct_ops plumbing. + */ +#include +#include +#include + +static const struct btf_type *task_struct_type; + +static bool bpf_scx_is_valid_access(int off, int size, + enum bpf_access_type type, + const struct bpf_prog *prog, + struct bpf_insn_access_aux *info) +{ + if (type != BPF_READ) + return false; + if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) + return false; + if (off % size != 0) + return false; + + return btf_ctx_access(off, size, type, prog, info); +} + +static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, + const struct bpf_reg_state *reg, int off, + int size) +{ + const struct btf_type *t; + + t = btf_type_by_id(reg->btf, reg->btf_id); + if (t == task_struct_type) { + /* + * COMPAT: Will be removed in v6.23. + */ + if ((off >= offsetof(struct task_struct, scx.slice) && + off + size <= offsetofend(struct task_struct, scx.slice)) || + (off >= offsetof(struct task_struct, scx.dsq_vtime) && + off + size <= offsetofend(struct task_struct, scx.dsq_vtime))) { + pr_warn("sched_ext: Writing directly to p->scx.slice/dsq_vtime is deprecated, use scx_bpf_task_set_slice/dsq_vtime()"); + return SCALAR_VALUE; + } + + if (off >= offsetof(struct task_struct, scx.disallow) && + off + size <= offsetofend(struct task_struct, scx.disallow)) + return SCALAR_VALUE; + } + + return -EACCES; +} + +static const struct bpf_verifier_ops bpf_scx_verifier_ops = { + .get_func_proto = bpf_base_func_proto, + .is_valid_access = bpf_scx_is_valid_access, + .btf_struct_access = bpf_scx_btf_struct_access, +}; + +static int bpf_scx_init_member(const struct btf_type *t, + const struct btf_member *member, + void *kdata, const void *udata) +{ + const struct sched_ext_ops *uops = udata; + struct sched_ext_ops *ops = kdata; + u32 moff = __btf_member_bit_offset(t, member) / 8; + int ret; + + switch (moff) { + case offsetof(struct sched_ext_ops, dispatch_max_batch): + if (*(u32 *)(udata + moff) > INT_MAX) + return -E2BIG; + ops->dispatch_max_batch = *(u32 *)(udata + moff); + return 1; + case offsetof(struct sched_ext_ops, flags): + if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS) + return -EINVAL; + ops->flags = *(u64 *)(udata + moff); + return 1; + case offsetof(struct sched_ext_ops, name): + ret = bpf_obj_name_cpy(ops->name, uops->name, + sizeof(ops->name)); + if (ret < 0) + return ret; + if (ret == 0) + return -EINVAL; + return 1; + case offsetof(struct sched_ext_ops, timeout_ms): + if (msecs_to_jiffies(*(u32 *)(udata + moff)) > + SCX_WATCHDOG_MAX_TIMEOUT) + return -E2BIG; + ops->timeout_ms = *(u32 *)(udata + moff); + return 1; + case offsetof(struct sched_ext_ops, exit_dump_len): + ops->exit_dump_len = + *(u32 *)(udata + moff) ?: SCX_EXIT_DUMP_DFL_LEN; + return 1; + case offsetof(struct sched_ext_ops, hotplug_seq): + ops->hotplug_seq = *(u64 *)(udata + moff); + return 1; +#ifdef CONFIG_EXT_SUB_SCHED + case offsetof(struct sched_ext_ops, sub_cgroup_id): + ops->sub_cgroup_id = *(u64 *)(udata + moff); + return 1; +#endif /* CONFIG_EXT_SUB_SCHED */ + } + + return 0; +} + +#ifdef CONFIG_EXT_SUB_SCHED +static void scx_pstack_recursion_on_dispatch(struct bpf_prog *prog) +{ + struct scx_sched *sch; + + guard(rcu)(); + sch = scx_prog_sched(prog->aux); + if (unlikely(!sch)) + return; + + scx_error(sch, "dispatch recursion detected"); +} +#endif /* CONFIG_EXT_SUB_SCHED */ + +static int bpf_scx_check_member(const struct btf_type *t, + const struct btf_member *member, + const struct bpf_prog *prog) +{ + u32 moff = __btf_member_bit_offset(t, member) / 8; + + switch (moff) { + case offsetof(struct sched_ext_ops, init_task): +#ifdef CONFIG_EXT_GROUP_SCHED + case offsetof(struct sched_ext_ops, cgroup_init): + case offsetof(struct sched_ext_ops, cgroup_exit): + case offsetof(struct sched_ext_ops, cgroup_prep_move): +#endif + case offsetof(struct sched_ext_ops, cpu_online): + case offsetof(struct sched_ext_ops, cpu_offline): + case offsetof(struct sched_ext_ops, init): + case offsetof(struct sched_ext_ops, exit): + case offsetof(struct sched_ext_ops, sub_attach): + case offsetof(struct sched_ext_ops, sub_detach): + break; + default: + if (prog->sleepable) + return -EINVAL; + } + +#ifdef CONFIG_EXT_SUB_SCHED + /* + * Enable private stack for operations that can nest along the + * hierarchy. + * + * XXX - Ideally, we should only do this for scheds that allow + * sub-scheds and sub-scheds themselves but I don't know how to access + * struct_ops from here. + */ + switch (moff) { + case offsetof(struct sched_ext_ops, dispatch): + prog->aux->priv_stack_requested = true; + prog->aux->recursion_detected = scx_pstack_recursion_on_dispatch; + } +#endif /* CONFIG_EXT_SUB_SCHED */ + + return 0; +} + +static int bpf_scx_reg(void *kdata, struct bpf_link *link) +{ + struct scx_enable_cmd cmd = { .ops = kdata }; + + return scx_enable(&cmd, link); +} + +struct scx_arena_scan { + struct bpf_map *arena; + int err; +}; + +/* + * The verifier enforces one arena per BPF program, so each struct_ops + * member prog contributes at most one arena via bpf_prog_arena(). + * Require all non-NULL contributions to match. + */ +static int scx_arena_scan_prog(struct bpf_prog *prog, void *data) +{ + struct scx_arena_scan *s = data; + struct bpf_map *arena = NULL; + + /* arena.o, which defines these, is built only on MMU && 64BIT */ +#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) + arena = bpf_prog_arena(prog); +#endif + if (!arena) + return 0; + if (s->arena && s->arena != arena) { + s->err = -EINVAL; + return 1; + } + s->arena = arena; + return 0; +} + +static int bpf_scx_reg_cid(void *kdata, struct bpf_link *link) +{ + struct scx_enable_cmd cmd = { .ops_cid = kdata, .is_cid_type = true }; + struct scx_arena_scan scan = {}; + int ret; + + bpf_struct_ops_for_each_prog(kdata, scx_arena_scan_prog, &scan); + if (scan.err) { + pr_err("sched_ext: cid-form scheduler uses multiple arena maps\n"); + return scan.err; + } + if (!scan.arena) { + pr_err("sched_ext: cid-form scheduler must use a BPF arena map\n"); + return -EINVAL; + } + + bpf_map_inc(scan.arena); + cmd.arena_map = scan.arena; + ret = scx_enable(&cmd, link); + if (cmd.arena_map) /* not consumed by scx_alloc_and_add_sched() */ + bpf_map_put(cmd.arena_map); + return ret; +} + +static void bpf_scx_unreg(void *kdata, struct bpf_link *link) +{ + struct sched_ext_ops *ops = kdata; + struct scx_sched *sch = rcu_dereference_protected(ops->priv, true); + + scx_disable(sch, SCX_EXIT_UNREG); + scx_flush_disable_work(sch); + RCU_INIT_POINTER(ops->priv, NULL); + kobject_put(&sch->kobj); +} + +static int bpf_scx_init(struct btf *btf) +{ + task_struct_type = btf_type_by_id(btf, btf_tracing_ids[BTF_TRACING_TYPE_TASK]); + + return 0; +} + +static int bpf_scx_update(void *kdata, void *old_kdata, struct bpf_link *link) +{ + /* + * sched_ext does not support updating the actively-loaded BPF + * scheduler, as registering a BPF scheduler can always fail if the + * scheduler returns an error code for e.g. ops.init(), ops.init_task(), + * etc. Similarly, we can always race with unregistration happening + * elsewhere, such as with sysrq. + */ + return -EOPNOTSUPP; +} + +static int bpf_scx_validate(void *kdata) +{ + return 0; +} + +static s32 sched_ext_ops__select_cpu(struct task_struct *p, s32 prev_cpu, u64 wake_flags) { return -EINVAL; } +static void sched_ext_ops__enqueue(struct task_struct *p, u64 enq_flags) {} +static void sched_ext_ops__dequeue(struct task_struct *p, u64 enq_flags) {} +static void sched_ext_ops__dispatch(s32 prev_cpu, struct task_struct *prev__nullable) {} +static void sched_ext_ops__tick(struct task_struct *p) {} +static void sched_ext_ops__runnable(struct task_struct *p, u64 enq_flags) {} +static void sched_ext_ops__running(struct task_struct *p) {} +static void sched_ext_ops__stopping(struct task_struct *p, bool runnable) {} +static void sched_ext_ops__quiescent(struct task_struct *p, u64 deq_flags) {} +static bool sched_ext_ops__yield(struct task_struct *from, struct task_struct *to__nullable) { return false; } +static bool sched_ext_ops__core_sched_before(struct task_struct *a, struct task_struct *b) { return false; } +static void sched_ext_ops__set_weight(struct task_struct *p, u32 weight) {} +static void sched_ext_ops__set_cpumask(struct task_struct *p, const struct cpumask *mask) {} +static void sched_ext_ops__update_idle(s32 cpu, bool idle) {} +static void sched_ext_ops__cpu_acquire(s32 cpu, struct scx_cpu_acquire_args *args) {} +static void sched_ext_ops__cpu_release(s32 cpu, struct scx_cpu_release_args *args) {} +static s32 sched_ext_ops__init_task(struct task_struct *p, struct scx_init_task_args *args) { return -EINVAL; } +static void sched_ext_ops__exit_task(struct task_struct *p, struct scx_exit_task_args *args) {} +static void sched_ext_ops__enable(struct task_struct *p) {} +static void sched_ext_ops__disable(struct task_struct *p) {} +#ifdef CONFIG_EXT_GROUP_SCHED +static s32 sched_ext_ops__cgroup_init(struct cgroup *cgrp, struct scx_cgroup_init_args *args) { return -EINVAL; } +static void sched_ext_ops__cgroup_exit(struct cgroup *cgrp) {} +static s32 sched_ext_ops__cgroup_prep_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) { return -EINVAL; } +static void sched_ext_ops__cgroup_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) {} +static void sched_ext_ops__cgroup_cancel_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) {} +static void sched_ext_ops__cgroup_set_weight(struct cgroup *cgrp, u32 weight) {} +static void sched_ext_ops__cgroup_set_bandwidth(struct cgroup *cgrp, u64 period_us, u64 quota_us, u64 burst_us) {} +static void sched_ext_ops__cgroup_set_idle(struct cgroup *cgrp, bool idle) {} +#endif /* CONFIG_EXT_GROUP_SCHED */ +static s32 sched_ext_ops__sub_attach(struct scx_sub_attach_args *args) { return -EINVAL; } +static void sched_ext_ops__sub_detach(struct scx_sub_detach_args *args) {} +static void sched_ext_ops__cpu_online(s32 cpu) {} +static void sched_ext_ops__cpu_offline(s32 cpu) {} +static s32 sched_ext_ops__init(void) { return -EINVAL; } +static void sched_ext_ops__exit(struct scx_exit_info *info) {} +static void sched_ext_ops__dump(struct scx_dump_ctx *ctx) {} +static void sched_ext_ops__dump_cpu(struct scx_dump_ctx *ctx, s32 cpu, bool idle) {} +static void sched_ext_ops__dump_task(struct scx_dump_ctx *ctx, struct task_struct *p) {} + +static struct sched_ext_ops __bpf_ops_sched_ext_ops = { + .select_cpu = sched_ext_ops__select_cpu, + .enqueue = sched_ext_ops__enqueue, + .dequeue = sched_ext_ops__dequeue, + .dispatch = sched_ext_ops__dispatch, + .tick = sched_ext_ops__tick, + .runnable = sched_ext_ops__runnable, + .running = sched_ext_ops__running, + .stopping = sched_ext_ops__stopping, + .quiescent = sched_ext_ops__quiescent, + .yield = sched_ext_ops__yield, + .core_sched_before = sched_ext_ops__core_sched_before, + .set_weight = sched_ext_ops__set_weight, + .set_cpumask = sched_ext_ops__set_cpumask, + .update_idle = sched_ext_ops__update_idle, + .cpu_acquire = sched_ext_ops__cpu_acquire, + .cpu_release = sched_ext_ops__cpu_release, + .init_task = sched_ext_ops__init_task, + .exit_task = sched_ext_ops__exit_task, + .enable = sched_ext_ops__enable, + .disable = sched_ext_ops__disable, +#ifdef CONFIG_EXT_GROUP_SCHED + .cgroup_init = sched_ext_ops__cgroup_init, + .cgroup_exit = sched_ext_ops__cgroup_exit, + .cgroup_prep_move = sched_ext_ops__cgroup_prep_move, + .cgroup_move = sched_ext_ops__cgroup_move, + .cgroup_cancel_move = sched_ext_ops__cgroup_cancel_move, + .cgroup_set_weight = sched_ext_ops__cgroup_set_weight, + .cgroup_set_bandwidth = sched_ext_ops__cgroup_set_bandwidth, + .cgroup_set_idle = sched_ext_ops__cgroup_set_idle, +#endif + .sub_attach = sched_ext_ops__sub_attach, + .sub_detach = sched_ext_ops__sub_detach, + .cpu_online = sched_ext_ops__cpu_online, + .cpu_offline = sched_ext_ops__cpu_offline, + .init = sched_ext_ops__init, + .exit = sched_ext_ops__exit, + .dump = sched_ext_ops__dump, + .dump_cpu = sched_ext_ops__dump_cpu, + .dump_task = sched_ext_ops__dump_task, +}; + +static struct bpf_struct_ops bpf_sched_ext_ops = { + .verifier_ops = &bpf_scx_verifier_ops, + .reg = bpf_scx_reg, + .unreg = bpf_scx_unreg, + .check_member = bpf_scx_check_member, + .init_member = bpf_scx_init_member, + .init = bpf_scx_init, + .update = bpf_scx_update, + .validate = bpf_scx_validate, + .name = "sched_ext_ops", + .owner = THIS_MODULE, + .cfi_stubs = &__bpf_ops_sched_ext_ops +}; + +/* + * cid-form cfi stubs. Stubs whose signatures match the cpu-form (param types + * identical, only param names differ across structs) are reused; only + * set_cmask needs a fresh stub since the second argument type differs. + */ +static void sched_ext_ops_cid__set_cmask(struct task_struct *p, + const struct scx_cmask *cmask) {} + +static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { + .select_cid = sched_ext_ops__select_cpu, + .enqueue = sched_ext_ops__enqueue, + .dequeue = sched_ext_ops__dequeue, + .dispatch = sched_ext_ops__dispatch, + .tick = sched_ext_ops__tick, + .runnable = sched_ext_ops__runnable, + .running = sched_ext_ops__running, + .stopping = sched_ext_ops__stopping, + .quiescent = sched_ext_ops__quiescent, + .yield = sched_ext_ops__yield, + .core_sched_before = sched_ext_ops__core_sched_before, + .set_weight = sched_ext_ops__set_weight, + .set_cmask = sched_ext_ops_cid__set_cmask, + .update_idle = sched_ext_ops__update_idle, + .init_task = sched_ext_ops__init_task, + .exit_task = sched_ext_ops__exit_task, + .enable = sched_ext_ops__enable, + .disable = sched_ext_ops__disable, +#ifdef CONFIG_EXT_GROUP_SCHED + .cgroup_init = sched_ext_ops__cgroup_init, + .cgroup_exit = sched_ext_ops__cgroup_exit, + .cgroup_prep_move = sched_ext_ops__cgroup_prep_move, + .cgroup_move = sched_ext_ops__cgroup_move, + .cgroup_cancel_move = sched_ext_ops__cgroup_cancel_move, + .cgroup_set_weight = sched_ext_ops__cgroup_set_weight, + .cgroup_set_bandwidth = sched_ext_ops__cgroup_set_bandwidth, + .cgroup_set_idle = sched_ext_ops__cgroup_set_idle, +#endif + .sub_attach = sched_ext_ops__sub_attach, + .sub_detach = sched_ext_ops__sub_detach, + .cid_online = sched_ext_ops__cpu_online, + .cid_offline = sched_ext_ops__cpu_offline, + .init = sched_ext_ops__init, + .exit = sched_ext_ops__exit, + .dump = sched_ext_ops__dump, + .dump_cid = sched_ext_ops__dump_cpu, + .dump_task = sched_ext_ops__dump_task, +}; + +/* + * The cid-form struct_ops shares all bpf_struct_ops hooks with the cpu form. + * init_member, check_member, reg, unreg, etc. process kdata as the byte block + * verified to match by the BUILD_BUG_ON checks in scx_init(). + */ +static struct bpf_struct_ops bpf_sched_ext_ops_cid = { + .verifier_ops = &bpf_scx_verifier_ops, + .reg = bpf_scx_reg_cid, + .unreg = bpf_scx_unreg, + .check_member = bpf_scx_check_member, + .init_member = bpf_scx_init_member, + .init = bpf_scx_init, + .update = bpf_scx_update, + .validate = bpf_scx_validate, + .name = "sched_ext_ops_cid", + .owner = THIS_MODULE, + .cfi_stubs = &__bpf_ops_sched_ext_ops_cid +}; + + +/******************************************************************************** + * System integration and init. + */ + +static void sysrq_handle_sched_ext_reset(u8 key) +{ + struct scx_sched *sch; + + sch = rcu_dereference(scx_root); + if (likely(sch)) + scx_disable(sch, SCX_EXIT_SYSRQ); + else + pr_info("sched_ext: BPF schedulers not loaded\n"); +} + +static const struct sysrq_key_op sysrq_sched_ext_reset_op = { + .handler = sysrq_handle_sched_ext_reset, + .help_msg = "reset-sched-ext(S)", + .action_msg = "Disable sched_ext and revert all tasks to CFS", + .enable_mask = SYSRQ_ENABLE_RTNICE, +}; + +static void sysrq_handle_sched_ext_dump(u8 key) +{ + struct scx_exit_info ei = { + .kind = SCX_EXIT_NONE, + .exit_cpu = -1, + .reason = "SysRq-D", + }; + struct scx_sched *sch; + + list_for_each_entry_rcu(sch, &scx_sched_all, all) + scx_dump_state(sch, &ei, 0, false); +} + +static const struct sysrq_key_op sysrq_sched_ext_dump_op = { + .handler = sysrq_handle_sched_ext_dump, + .help_msg = "dump-sched-ext(D)", + .action_msg = "Trigger sched_ext debug dump", + .enable_mask = SYSRQ_ENABLE_RTNICE, +}; + +static bool can_skip_idle_kick(struct rq *rq) +{ + lockdep_assert_rq_held(rq); + + /* + * We can skip idle kicking if @rq is going to go through at least one + * full SCX scheduling cycle before going idle. Just checking whether + * curr is not idle is insufficient because we could be racing + * balance_one() trying to pull the next task from a remote rq, which + * may fail, and @rq may become idle afterwards. + * + * The race window is small and we don't and can't guarantee that @rq is + * only kicked while idle anyway. Skip only when sure. + */ + return !is_idle_task(rq->curr) && !(rq->scx.flags & SCX_RQ_IN_BALANCE); +} + +static bool kick_one_cpu(s32 cpu, struct rq *this_rq, unsigned long *ksyncs) +{ + struct rq *rq = cpu_rq(cpu); + struct scx_rq *this_scx = &this_rq->scx; + const struct sched_class *cur_class; + bool should_wait = false; + unsigned long flags; + + raw_spin_rq_lock_irqsave(rq, flags); + cur_class = rq->curr->sched_class; + + /* + * During CPU hotplug, a CPU may depend on kicking itself to make + * forward progress. Allow kicking self regardless of online state. If + * @cpu is running a higher class task, we have no control over @cpu. + * Skip kicking. + */ + if ((cpu_online(cpu) || cpu == cpu_of(this_rq)) && + !sched_class_above(cur_class, &ext_sched_class)) { + if (cpumask_test_cpu(cpu, this_scx->cpus_to_preempt)) { + if (cur_class == &ext_sched_class) + rq->curr->scx.slice = 0; + cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt); + } + + if (cpumask_test_cpu(cpu, this_scx->cpus_to_wait)) { + if (cur_class == &ext_sched_class) { + cpumask_set_cpu(cpu, this_scx->cpus_to_sync); + ksyncs[cpu] = rq->scx.kick_sync; + should_wait = true; + } + cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); + } + + resched_curr(rq); + } else { + cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt); + cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); + } + + raw_spin_rq_unlock_irqrestore(rq, flags); + + return should_wait; +} + +static void kick_one_cpu_if_idle(s32 cpu, struct rq *this_rq) +{ + struct rq *rq = cpu_rq(cpu); + unsigned long flags; + + raw_spin_rq_lock_irqsave(rq, flags); + + if (!can_skip_idle_kick(rq) && + (cpu_online(cpu) || cpu == cpu_of(this_rq))) + resched_curr(rq); + + raw_spin_rq_unlock_irqrestore(rq, flags); +} + +static void kick_cpus_irq_workfn(struct irq_work *irq_work) +{ + struct rq *this_rq = this_rq(); + struct scx_rq *this_scx = &this_rq->scx; + struct scx_kick_syncs __rcu *ksyncs_pcpu = __this_cpu_read(scx_kick_syncs); + bool should_wait = false; + unsigned long *ksyncs; + s32 cpu; + + /* can race with free_kick_syncs() during scheduler disable */ + if (unlikely(!ksyncs_pcpu)) + return; + + ksyncs = rcu_dereference_bh(ksyncs_pcpu)->syncs; + + for_each_cpu(cpu, this_scx->cpus_to_kick) { + should_wait |= kick_one_cpu(cpu, this_rq, ksyncs); + cpumask_clear_cpu(cpu, this_scx->cpus_to_kick); + cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle); + } + + for_each_cpu(cpu, this_scx->cpus_to_kick_if_idle) { + kick_one_cpu_if_idle(cpu, this_rq); + cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle); + } + + /* + * Can't wait in hardirq — kick_sync can't advance, deadlocking if + * CPUs wait for each other. Defer to kick_sync_wait_bal_cb(). + */ + if (should_wait) { + raw_spin_rq_lock(this_rq); + this_scx->kick_sync_pending = true; + resched_curr(this_rq); + raw_spin_rq_unlock(this_rq); + } +} + +/** + * print_scx_info - print out sched_ext scheduler state + * @log_lvl: the log level to use when printing + * @p: target task + * + * If a sched_ext scheduler is enabled, print the name and state of the + * scheduler. If @p is on sched_ext, print further information about the task. + * + * This function can be safely called on any task as long as the task_struct + * itself is accessible. While safe, this function isn't synchronized and may + * print out mixups or garbages of limited length. + */ +void print_scx_info(const char *log_lvl, struct task_struct *p) +{ + struct scx_sched *sch; + enum scx_enable_state state = scx_enable_state(); + const char *all = READ_ONCE(scx_switching_all) ? "+all" : ""; + char runnable_at_buf[22] = "?"; + struct sched_class *class; + unsigned long runnable_at; + + guard(rcu)(); + + sch = scx_task_sched_rcu(p); + + if (!sch) + return; + + /* + * Carefully check if the task was running on sched_ext, and then + * carefully copy the time it's been runnable, and its state. + */ + if (copy_from_kernel_nofault(&class, &p->sched_class, sizeof(class)) || + class != &ext_sched_class) { + printk("%sSched_ext: %s (%s%s)", log_lvl, sch->ops.name, + scx_enable_state_str[state], all); + return; + } + + if (!copy_from_kernel_nofault(&runnable_at, &p->scx.runnable_at, + sizeof(runnable_at))) + scnprintf(runnable_at_buf, sizeof(runnable_at_buf), "%+ldms", + jiffies_delta_msecs(runnable_at, jiffies)); + + /* print everything onto one line to conserve console space */ + printk("%sSched_ext: %s (%s%s), task: runnable_at=%s", + log_lvl, sch->ops.name, scx_enable_state_str[state], all, + runnable_at_buf); +} + +static int scx_pm_handler(struct notifier_block *nb, unsigned long event, void *ptr) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = rcu_dereference(scx_root); + if (!sch) + return NOTIFY_OK; + + /* + * SCX schedulers often have userspace components which are sometimes + * involved in critial scheduling paths. PM operations involve freezing + * userspace which can lead to scheduling misbehaviors including stalls. + * Let's bypass while PM operations are in progress. + */ + switch (event) { + case PM_HIBERNATION_PREPARE: + case PM_SUSPEND_PREPARE: + case PM_RESTORE_PREPARE: + scx_bypass(sch, true); + break; + case PM_POST_HIBERNATION: + case PM_POST_SUSPEND: + case PM_POST_RESTORE: + scx_bypass(sch, false); + break; + } + + return NOTIFY_OK; +} + +static struct notifier_block scx_pm_notifier = { + .notifier_call = scx_pm_handler, +}; + +void __init init_sched_ext_class(void) +{ + s32 cpu, v; + + /* + * The following is to prevent the compiler from optimizing out the enum + * definitions so that BPF scheduler implementations can use them + * through the generated vmlinux.h. + */ + WRITE_ONCE(v, SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | SCX_KICK_PREEMPT | + SCX_TG_ONLINE); + + scx_idle_init_masks(); + + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + int n = cpu_to_node(cpu); + + /* local_dsq's sch will be set during scx_root_enable() */ + BUG_ON(init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL)); + + INIT_LIST_HEAD(&rq->scx.runnable_list); + INIT_LIST_HEAD(&rq->scx.ddsp_deferred_locals); + + BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_kick, GFP_KERNEL, n)); + BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_kick_if_idle, GFP_KERNEL, n)); + BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_preempt, GFP_KERNEL, n)); + BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_wait, GFP_KERNEL, n)); + BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_sync, GFP_KERNEL, n)); + raw_spin_lock_init(&rq->scx.deferred_reenq_lock); + INIT_LIST_HEAD(&rq->scx.deferred_reenq_locals); + INIT_LIST_HEAD(&rq->scx.deferred_reenq_users); + rq->scx.deferred_irq_work = IRQ_WORK_INIT_HARD(deferred_irq_workfn); + rq->scx.kick_cpus_irq_work = IRQ_WORK_INIT_HARD(kick_cpus_irq_workfn); + + if (cpu_online(cpu)) + cpu_rq(cpu)->scx.flags |= SCX_RQ_ONLINE; + } + + register_sysrq_key('S', &sysrq_sched_ext_reset_op); + register_sysrq_key('D', &sysrq_sched_ext_dump_op); + INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn); + +#ifdef CONFIG_EXT_SUB_SCHED + BUG_ON(rhashtable_init(&scx_sched_hash, &scx_sched_hash_params)); +#endif /* CONFIG_EXT_SUB_SCHED */ +} + + +/******************************************************************************** + * Helpers that can be called from the BPF scheduler. + */ +static bool scx_vet_enq_flags(struct scx_sched *sch, u64 dsq_id, u64 *enq_flags) +{ + bool is_local = dsq_id == SCX_DSQ_LOCAL || + (dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON; + + if (*enq_flags & SCX_ENQ_IMMED) { + if (unlikely(!is_local)) { + scx_error(sch, "SCX_ENQ_IMMED on a non-local DSQ 0x%llx", dsq_id); + return false; + } + } else if ((sch->ops.flags & SCX_OPS_ALWAYS_ENQ_IMMED) && is_local) { + *enq_flags |= SCX_ENQ_IMMED; + } + + return true; +} + +static bool scx_dsq_insert_preamble(struct scx_sched *sch, struct task_struct *p, + u64 dsq_id, u64 *enq_flags) +{ + lockdep_assert_irqs_disabled(); + + if (unlikely(!p)) { + scx_error(sch, "called with NULL task"); + return false; + } + + if (unlikely(*enq_flags & __SCX_ENQ_INTERNAL_MASK)) { + scx_error(sch, "invalid enq_flags 0x%llx", *enq_flags); + return false; + } + + /* see SCX_EV_INSERT_NOT_OWNED definition */ + if (unlikely(!scx_task_on_sched(sch, p))) { + __scx_add_event(sch, SCX_EV_INSERT_NOT_OWNED, 1); + return false; + } + + if (!scx_vet_enq_flags(sch, dsq_id, enq_flags)) + return false; + + return true; +} + +static void scx_dsq_insert_commit(struct scx_sched *sch, struct task_struct *p, + u64 dsq_id, u64 enq_flags) +{ + struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; + struct task_struct *ddsp_task; + + ddsp_task = __this_cpu_read(direct_dispatch_task); + if (ddsp_task) { + mark_direct_dispatch(sch, ddsp_task, p, dsq_id, enq_flags); + return; + } + + if (unlikely(dspc->cursor >= sch->dsp_max_batch)) { + scx_error(sch, "dispatch buffer overflow"); + return; + } + + dspc->buf[dspc->cursor++] = (struct scx_dsp_buf_ent){ + .task = p, + .qseq = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_QSEQ_MASK, + .dsq_id = dsq_id, + .enq_flags = enq_flags, + }; +} + +__bpf_kfunc_start_defs(); + +/** + * scx_bpf_dsq_insert - Insert a task into the FIFO queue of a DSQ + * @p: task_struct to insert + * @dsq_id: DSQ to insert into + * @slice: duration @p can run for in nsecs, 0 to keep the current value + * @enq_flags: SCX_ENQ_* + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Insert @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe to + * call this function spuriously. Can be called from ops.enqueue(), + * ops.select_cpu(), and ops.dispatch(). + * + * When called from ops.select_cpu() or ops.enqueue(), it's for direct dispatch + * and @p must match the task being enqueued. + * + * When called from ops.select_cpu(), @enq_flags and @dsp_id are stored, and @p + * will be directly inserted into the corresponding dispatch queue after + * ops.select_cpu() returns. If @p is inserted into SCX_DSQ_LOCAL, it will be + * inserted into the local DSQ of the CPU returned by ops.select_cpu(). + * @enq_flags are OR'd with the enqueue flags on the enqueue path before the + * task is inserted. + * + * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id + * and this function can be called upto ops.dispatch_max_batch times to insert + * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the + * remaining slots. scx_bpf_dsq_move_to_local() flushes the batch and resets the + * counter. + * + * This function doesn't have any locking restrictions and may be called under + * BPF locks (in the future when BPF introduces more flexible locking). + * + * @p is allowed to run for @slice. The scheduling path is triggered on slice + * exhaustion. If zero, the current residual slice is maintained. If + * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with + * scx_bpf_kick_cpu() to trigger scheduling. + * + * Returns %true on successful insertion, %false on failure. On the root + * scheduler, %false return triggers scheduler abort and the caller doesn't need + * to check the return value. + */ +__bpf_kfunc bool scx_bpf_dsq_insert___v2(struct task_struct *p, u64 dsq_id, + u64 slice, u64 enq_flags, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return false; + + if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags)) + return false; + + if (slice) + p->scx.slice = slice; + else + p->scx.slice = p->scx.slice ?: 1; + + scx_dsq_insert_commit(sch, p, dsq_id, enq_flags); + + return true; +} + +/* + * COMPAT: Will be removed in v6.23 along with the ___v2 suffix. + */ +__bpf_kfunc void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, + u64 slice, u64 enq_flags, + const struct bpf_prog_aux *aux) +{ + scx_bpf_dsq_insert___v2(p, dsq_id, slice, enq_flags, aux); +} + +static bool scx_dsq_insert_vtime(struct scx_sched *sch, struct task_struct *p, + u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) +{ + if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags)) + return false; + + if (slice) + p->scx.slice = slice; + else + p->scx.slice = p->scx.slice ?: 1; + + p->scx.dsq_vtime = vtime; + + scx_dsq_insert_commit(sch, p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); + + return true; +} + +struct scx_bpf_dsq_insert_vtime_args { + /* @p can't be packed together as KF_RCU is not transitive */ + u64 dsq_id; + u64 slice; + u64 vtime; + u64 enq_flags; +}; + +/** + * __scx_bpf_dsq_insert_vtime - Arg-wrapped vtime DSQ insertion + * @p: task_struct to insert + * @args: struct containing the rest of the arguments + * @args->dsq_id: DSQ to insert into + * @args->slice: duration @p can run for in nsecs, 0 to keep the current value + * @args->vtime: @p's ordering inside the vtime-sorted queue of the target DSQ + * @args->enq_flags: SCX_ENQ_* + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Wrapper kfunc that takes arguments via struct to work around BPF's 5 argument + * limit. BPF programs should use scx_bpf_dsq_insert_vtime() which is provided + * as an inline wrapper in common.bpf.h. + * + * Insert @p into the vtime priority queue of the DSQ identified by + * @args->dsq_id. Tasks queued into the priority queue are ordered by + * @args->vtime. All other aspects are identical to scx_bpf_dsq_insert(). + * + * @args->vtime ordering is according to time_before64() which considers + * wrapping. A numerically larger vtime may indicate an earlier position in the + * ordering and vice-versa. + * + * A DSQ can only be used as a FIFO or priority queue at any given time and this + * function must not be called on a DSQ which already has one or more FIFO tasks + * queued and vice-versa. Also, the built-in DSQs (SCX_DSQ_LOCAL and + * SCX_DSQ_GLOBAL) cannot be used as priority queues. + * + * Returns %true on successful insertion, %false on failure. On the root + * scheduler, %false return triggers scheduler abort and the caller doesn't need + * to check the return value. + */ +__bpf_kfunc bool +__scx_bpf_dsq_insert_vtime(struct task_struct *p, + struct scx_bpf_dsq_insert_vtime_args *args, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return false; + + return scx_dsq_insert_vtime(sch, p, args->dsq_id, args->slice, + args->vtime, args->enq_flags); +} + +/* + * COMPAT: Will be removed in v6.23. + */ +__bpf_kfunc void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id, + u64 slice, u64 vtime, u64 enq_flags) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = rcu_dereference(scx_root); + if (unlikely(!sch)) + return; + +#ifdef CONFIG_EXT_SUB_SCHED + /* + * Disallow if any sub-scheds are attached. There is no way to tell + * which scheduler called us, just error out @p's scheduler. + */ + if (unlikely(!list_empty(&sch->children))) { + scx_error(scx_task_sched(p), "__scx_bpf_dsq_insert_vtime() must be used"); + return; + } +#endif + + scx_dsq_insert_vtime(sch, p, dsq_id, slice, vtime, enq_flags); +} + +__bpf_kfunc_end_defs(); + +BTF_KFUNCS_START(scx_kfunc_ids_enqueue_dispatch) +BTF_ID_FLAGS(func, scx_bpf_dsq_insert, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_dsq_insert___v2, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, __scx_bpf_dsq_insert_vtime, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_dsq_insert_vtime, KF_RCU) +BTF_KFUNCS_END(scx_kfunc_ids_enqueue_dispatch) + +static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { + .owner = THIS_MODULE, + .set = &scx_kfunc_ids_enqueue_dispatch, + .filter = scx_kfunc_context_filter, +}; + +static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, + struct task_struct *p, u64 dsq_id, u64 enq_flags) +{ + struct scx_dispatch_q *src_dsq = kit->dsq, *dst_dsq; + struct scx_sched *sch; + struct rq *this_rq, *src_rq, *locked_rq; + bool dispatched = false; + bool in_balance; + unsigned long flags; + + /* + * The verifier considers an iterator slot initialized on any + * KF_ITER_NEW return, so a BPF program may legally reach here after + * bpf_iter_scx_dsq_new() failed and left @kit->dsq NULL. + */ + if (unlikely(!src_dsq)) + return false; + + sch = src_dsq->sched; + + if (!scx_vet_enq_flags(sch, dsq_id, &enq_flags)) + return false; + + /* + * If the BPF scheduler keeps calling this function repeatedly, it can + * cause similar live-lock conditions as consume_dispatch_q(). + */ + if (unlikely(READ_ONCE(sch->aborting))) + return false; + + if (unlikely(!scx_task_on_sched(sch, p))) { + scx_error(sch, "scx_bpf_dsq_move[_vtime]() on %s[%d] but the task belongs to a different scheduler", + p->comm, p->pid); + return false; + } + + /* + * Can be called from either ops.dispatch() locking this_rq() or any + * context where no rq lock is held. If latter, lock @p's task_rq which + * we'll likely need anyway. + */ + src_rq = task_rq(p); + + local_irq_save(flags); + this_rq = this_rq(); + in_balance = this_rq->scx.flags & SCX_RQ_IN_BALANCE; + + if (in_balance) { + if (this_rq != src_rq) { + raw_spin_rq_unlock(this_rq); + raw_spin_rq_lock(src_rq); + } + } else { + raw_spin_rq_lock(src_rq); + } + + locked_rq = src_rq; + raw_spin_lock(&src_dsq->lock); + + /* did someone else get to it while we dropped the locks? */ + if (nldsq_cursor_lost_task(&kit->cursor, src_rq, src_dsq, p)) { + raw_spin_unlock(&src_dsq->lock); + goto out; + } + + /* @p is still on $src_dsq and stable, determine the destination */ + dst_dsq = find_dsq_for_dispatch(sch, this_rq, dsq_id, task_cpu(p)); + + /* + * Apply vtime and slice updates before moving so that the new time is + * visible before inserting into $dst_dsq. @p is still on $src_dsq but + * this is safe as we're locking it. + */ + if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_VTIME) + p->scx.dsq_vtime = kit->vtime; + if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_SLICE) + p->scx.slice = kit->slice; + + /* execute move */ + locked_rq = move_task_between_dsqs(sch, p, enq_flags, src_dsq, dst_dsq); + dispatched = true; +out: + if (in_balance) { + if (this_rq != locked_rq) { + raw_spin_rq_unlock(locked_rq); + raw_spin_rq_lock(this_rq); + } + } else { + raw_spin_rq_unlock_irqrestore(locked_rq, flags); + } + + kit->cursor.flags &= ~(__SCX_DSQ_ITER_HAS_SLICE | + __SCX_DSQ_ITER_HAS_VTIME); + return dispatched; +} + +__bpf_kfunc_start_defs(); + +/** + * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Can only be called from ops.dispatch(). + */ +__bpf_kfunc u32 scx_bpf_dispatch_nr_slots(const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return 0; + + return sch->dsp_max_batch - __this_cpu_read(sch->pcpu->dsp_ctx.cursor); +} + +/** + * scx_bpf_dispatch_cancel - Cancel the latest dispatch + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Cancel the latest dispatch. Can be called multiple times to cancel further + * dispatches. Can only be called from ops.dispatch(). + */ +__bpf_kfunc void scx_bpf_dispatch_cancel(const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + struct scx_dsp_ctx *dspc; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return; + + dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; + + if (dspc->cursor > 0) + dspc->cursor--; + else + scx_error(sch, "dispatch buffer underflow"); +} + +/** + * scx_bpf_dsq_move_to_local - move a task from a DSQ to the current CPU's local DSQ + * @dsq_id: DSQ to move task from. Must be a user-created DSQ + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * @enq_flags: %SCX_ENQ_* + * + * Move a task from the non-local DSQ identified by @dsq_id to the current CPU's + * local DSQ for execution with @enq_flags applied. Can only be called from + * ops.dispatch(). + * + * Built-in DSQs (%SCX_DSQ_GLOBAL and %SCX_DSQ_LOCAL*) are not supported as + * sources. Local DSQs support reenqueueing (a task can be picked up for + * execution, dequeued for property changes, or reenqueued), but the BPF + * scheduler cannot directly iterate or move tasks from them. %SCX_DSQ_GLOBAL + * is similar but also doesn't support reenqueueing, as it maps to multiple + * per-node DSQs making the scope difficult to define; this may change in the + * future. + * + * This function flushes the in-flight dispatches from scx_bpf_dsq_insert() + * before trying to move from the specified DSQ. It may also grab rq locks and + * thus can't be called under any BPF locks. + * + * Returns %true if a task has been moved, %false if there isn't any task to + * move. + */ +__bpf_kfunc bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags, + const struct bpf_prog_aux *aux) +{ + struct scx_dispatch_q *dsq; + struct scx_sched *sch; + struct scx_dsp_ctx *dspc; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return false; + + if (!scx_vet_enq_flags(sch, SCX_DSQ_LOCAL, &enq_flags)) + return false; + + dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; + + flush_dispatch_buf(sch, dspc->rq); + + dsq = find_user_dsq(sch, dsq_id); + if (unlikely(!dsq)) { + scx_error(sch, "invalid DSQ ID 0x%016llx", dsq_id); + return false; + } + + if (consume_dispatch_q(sch, dspc->rq, dsq, enq_flags)) { + /* + * A successfully consumed task can be dequeued before it starts + * running while the CPU is trying to migrate other dispatched + * tasks. Bump nr_tasks to tell balance_one() to retry on empty + * local DSQ. + */ + dspc->nr_tasks++; + return true; + } else { + return false; + } +} + +/* + * COMPAT: ___v2 was introduced in v7.1. Remove this and ___v2 tag in the future. + */ +__bpf_kfunc bool scx_bpf_dsq_move_to_local(u64 dsq_id, const struct bpf_prog_aux *aux) +{ + return scx_bpf_dsq_move_to_local___v2(dsq_id, 0, aux); +} + +/** + * scx_bpf_dsq_move_set_slice - Override slice when moving between DSQs + * @it__iter: DSQ iterator in progress + * @slice: duration the moved task can run for in nsecs + * + * Override the slice of the next task that will be moved from @it__iter using + * scx_bpf_dsq_move[_vtime](). If this function is not called, the previous + * slice duration is kept. + */ +__bpf_kfunc void scx_bpf_dsq_move_set_slice(struct bpf_iter_scx_dsq *it__iter, + u64 slice) +{ + struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter; + + kit->slice = slice; + kit->cursor.flags |= __SCX_DSQ_ITER_HAS_SLICE; +} + +/** + * scx_bpf_dsq_move_set_vtime - Override vtime when moving between DSQs + * @it__iter: DSQ iterator in progress + * @vtime: task's ordering inside the vtime-sorted queue of the target DSQ + * + * Override the vtime of the next task that will be moved from @it__iter using + * scx_bpf_dsq_move_vtime(). If this function is not called, the previous slice + * vtime is kept. If scx_bpf_dsq_move() is used to dispatch the next task, the + * override is ignored and cleared. + */ +__bpf_kfunc void scx_bpf_dsq_move_set_vtime(struct bpf_iter_scx_dsq *it__iter, + u64 vtime) +{ + struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter; + + kit->vtime = vtime; + kit->cursor.flags |= __SCX_DSQ_ITER_HAS_VTIME; +} + +/** + * scx_bpf_dsq_move - Move a task from DSQ iteration to a DSQ + * @it__iter: DSQ iterator in progress + * @p: task to transfer + * @dsq_id: DSQ to move @p to + * @enq_flags: SCX_ENQ_* + * + * Transfer @p which is on the DSQ currently iterated by @it__iter to the DSQ + * specified by @dsq_id. All DSQs - local DSQs, global DSQ and user DSQs - can + * be the destination. + * + * For the transfer to be successful, @p must still be on the DSQ and have been + * queued before the DSQ iteration started. This function doesn't care whether + * @p was obtained from the DSQ iteration. @p just has to be on the DSQ and have + * been queued before the iteration started. + * + * @p's slice is kept by default. Use scx_bpf_dsq_move_set_slice() to update. + * + * Can be called from ops.dispatch() or any BPF context which doesn't hold a rq + * lock (e.g. BPF timers or SYSCALL programs). + * + * Returns %true if @p has been consumed, %false if @p had already been + * consumed, dequeued, or, for sub-scheds, @dsq_id points to a disallowed local + * DSQ. + */ +__bpf_kfunc bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter, + struct task_struct *p, u64 dsq_id, + u64 enq_flags) +{ + return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter, + p, dsq_id, enq_flags); +} + +/** + * scx_bpf_dsq_move_vtime - Move a task from DSQ iteration to a PRIQ DSQ + * @it__iter: DSQ iterator in progress + * @p: task to transfer + * @dsq_id: DSQ to move @p to + * @enq_flags: SCX_ENQ_* + * + * Transfer @p which is on the DSQ currently iterated by @it__iter to the + * priority queue of the DSQ specified by @dsq_id. The destination must be a + * user DSQ as only user DSQs support priority queue. + * + * @p's slice and vtime are kept by default. Use scx_bpf_dsq_move_set_slice() + * and scx_bpf_dsq_move_set_vtime() to update. + * + * All other aspects are identical to scx_bpf_dsq_move(). See + * scx_bpf_dsq_insert_vtime() for more information on @vtime. + */ +__bpf_kfunc bool scx_bpf_dsq_move_vtime(struct bpf_iter_scx_dsq *it__iter, + struct task_struct *p, u64 dsq_id, + u64 enq_flags) +{ + return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter, + p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); +} + +#ifdef CONFIG_EXT_SUB_SCHED +/** + * scx_bpf_sub_dispatch - Trigger dispatching on a child scheduler + * @cgroup_id: cgroup ID of the child scheduler to dispatch + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Allows a parent scheduler to trigger dispatching on one of its direct + * child schedulers. The child scheduler runs its dispatch operation to + * move tasks from dispatch queues to the local runqueue. + * + * Returns: true on success, false if cgroup_id is invalid, not a direct + * child, or caller lacks dispatch permission. + */ +__bpf_kfunc bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux) +{ + struct rq *this_rq = this_rq(); + struct scx_sched *parent, *child; + + guard(rcu)(); + parent = scx_prog_sched(aux); + if (unlikely(!parent)) + return false; + + child = scx_find_sub_sched(cgroup_id); + + if (unlikely(!child)) + return false; + + if (unlikely(scx_parent(child) != parent)) { + scx_error(parent, "trying to dispatch a distant sub-sched on cgroup %llu", + cgroup_id); + return false; + } + + return scx_dispatch_sched(child, this_rq, this_rq->scx.sub_dispatch_prev, + true); +} +#endif /* CONFIG_EXT_SUB_SCHED */ + +__bpf_kfunc_end_defs(); + +BTF_KFUNCS_START(scx_kfunc_ids_dispatch) +BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_dispatch_cancel, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_dsq_move_to_local, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_dsq_move_to_local___v2, KF_IMPLICIT_ARGS) +/* scx_bpf_dsq_move*() also in scx_kfunc_ids_unlocked: callable from unlocked contexts */ +BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU) +#ifdef CONFIG_EXT_SUB_SCHED +BTF_ID_FLAGS(func, scx_bpf_sub_dispatch, KF_IMPLICIT_ARGS) +#endif +BTF_KFUNCS_END(scx_kfunc_ids_dispatch) + +static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = { + .owner = THIS_MODULE, + .set = &scx_kfunc_ids_dispatch, + .filter = scx_kfunc_context_filter, +}; + +__bpf_kfunc_start_defs(); + +/** + * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Iterate over all of the tasks currently enqueued on the local DSQ of the + * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of + * processed tasks. Can only be called from ops.cpu_release(). + */ +__bpf_kfunc u32 scx_bpf_reenqueue_local(const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + struct rq *rq; + + guard(rcu)(); + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return 0; + + rq = cpu_rq(smp_processor_id()); + lockdep_assert_rq_held(rq); + + return reenq_local(sch, rq, SCX_REENQ_ANY); +} + +__bpf_kfunc_end_defs(); + +BTF_KFUNCS_START(scx_kfunc_ids_cpu_release) +BTF_ID_FLAGS(func, scx_bpf_reenqueue_local, KF_IMPLICIT_ARGS) +BTF_KFUNCS_END(scx_kfunc_ids_cpu_release) + +static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = { + .owner = THIS_MODULE, + .set = &scx_kfunc_ids_cpu_release, + .filter = scx_kfunc_context_filter, +}; + +__bpf_kfunc_start_defs(); + +/** + * scx_bpf_create_dsq - Create a custom DSQ + * @dsq_id: DSQ to create + * @node: NUMA node to allocate from + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Create a custom DSQ identified by @dsq_id. Can be called from any sleepable + * scx callback, and any BPF_PROG_TYPE_SYSCALL prog. + */ +__bpf_kfunc s32 scx_bpf_create_dsq(u64 dsq_id, s32 node, const struct bpf_prog_aux *aux) +{ + struct scx_dispatch_q *dsq; + struct scx_sched *sch; + s32 ret; + + if (unlikely(node >= (int)nr_node_ids || + (node < 0 && node != NUMA_NO_NODE))) + return -EINVAL; + + if (unlikely(dsq_id & SCX_DSQ_FLAG_BUILTIN)) + return -EINVAL; + + dsq = kmalloc_node(sizeof(*dsq), GFP_KERNEL, node); + if (!dsq) + return -ENOMEM; + + /* + * init_dsq() must be called in GFP_KERNEL context. Init it with NULL + * @sch and update afterwards. + */ + ret = init_dsq(dsq, dsq_id, NULL); + if (ret) { + kfree(dsq); + return ret; + } + + rcu_read_lock(); + + sch = scx_prog_sched(aux); + if (sch) { + dsq->sched = sch; + ret = rhashtable_lookup_insert_fast(&sch->dsq_hash, &dsq->hash_node, + dsq_hash_params); + } else { + ret = -ENODEV; + } + + rcu_read_unlock(); + if (ret) { + exit_dsq(dsq); + kfree(dsq); + } + return ret; +} + +__bpf_kfunc_end_defs(); + +BTF_KFUNCS_START(scx_kfunc_ids_unlocked) +BTF_ID_FLAGS(func, scx_bpf_create_dsq, KF_IMPLICIT_ARGS | KF_SLEEPABLE) +/* also in scx_kfunc_ids_dispatch: also callable from ops.dispatch() */ +BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU) +/* also in scx_kfunc_ids_select_cpu: also callable from ops.select_cpu()/ops.enqueue() */ +BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_IMPLICIT_ARGS | KF_RCU) +BTF_KFUNCS_END(scx_kfunc_ids_unlocked) + +static const struct btf_kfunc_id_set scx_kfunc_set_unlocked = { + .owner = THIS_MODULE, + .set = &scx_kfunc_ids_unlocked, + .filter = scx_kfunc_context_filter, +}; + +__bpf_kfunc_start_defs(); + +/** + * scx_bpf_task_set_slice - Set task's time slice + * @p: task of interest + * @slice: time slice to set in nsecs + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Set @p's time slice to @slice. Returns %true on success, %false if the + * calling scheduler doesn't have authority over @p. + */ +__bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + sch = scx_prog_sched(aux); + if (unlikely(!sch || !scx_task_on_sched(sch, p))) + return false; + + p->scx.slice = slice; + return true; +} + +/** + * scx_bpf_task_set_dsq_vtime - Set task's virtual time for DSQ ordering + * @p: task of interest + * @vtime: virtual time to set + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Set @p's virtual time to @vtime. Returns %true on success, %false if the + * calling scheduler doesn't have authority over @p. + */ +__bpf_kfunc bool scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + sch = scx_prog_sched(aux); + if (unlikely(!sch || !scx_task_on_sched(sch, p))) + return false; + + p->scx.dsq_vtime = vtime; + return true; +} + +static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) +{ + struct rq *this_rq; + unsigned long irq_flags; + + local_irq_save(irq_flags); + + this_rq = this_rq(); + + /* + * While bypassing for PM ops, IRQ handling may not be online which can + * lead to irq_work_queue() malfunction such as infinite busy wait for + * IRQ status update. Suppress kicking. + */ + if (scx_bypassing(sch, cpu_of(this_rq))) + goto out; + + /* + * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting + * rq locks. We can probably be smarter and avoid bouncing if called + * from ops which don't hold a rq lock. + */ + if (flags & SCX_KICK_IDLE) { + struct rq *target_rq = cpu_rq(cpu); + + if (unlikely(flags & (SCX_KICK_PREEMPT | SCX_KICK_WAIT))) + scx_error(sch, "PREEMPT/WAIT cannot be used with SCX_KICK_IDLE"); + + if (raw_spin_rq_trylock(target_rq)) { + if (can_skip_idle_kick(target_rq)) { + raw_spin_rq_unlock(target_rq); + goto out; + } + raw_spin_rq_unlock(target_rq); + } + cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick_if_idle); + } else { + cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick); + + if (flags & SCX_KICK_PREEMPT) + cpumask_set_cpu(cpu, this_rq->scx.cpus_to_preempt); + if (flags & SCX_KICK_WAIT) + cpumask_set_cpu(cpu, this_rq->scx.cpus_to_wait); + } + + irq_work_queue(&this_rq->scx.kick_cpus_irq_work); +out: + local_irq_restore(irq_flags); +} + +/** + * scx_bpf_kick_cpu - Trigger reschedule on a CPU + * @cpu: cpu to kick + * @flags: %SCX_KICK_* flags + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or + * trigger rescheduling on a busy CPU. This can be called from any online + * scx_ops operation and the actual kicking is performed asynchronously through + * an irq work. + */ +__bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + sch = scx_prog_sched(aux); + if (likely(sch) && scx_cpu_valid(sch, cpu, NULL)) + scx_kick_cpu(sch, cpu, flags); +} + +/** + * scx_bpf_kick_cid - Trigger reschedule on the CPU mapped to @cid + * @cid: cid to kick + * @flags: %SCX_KICK_* flags + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * cid-addressed equivalent of scx_bpf_kick_cpu(). Return 0 on success, + * -errno otherwise. + */ +__bpf_kfunc s32 scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + s32 cpu; + + guard(rcu)(); + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return -ENODEV; + cpu = scx_cid_to_cpu(sch, cid); + if (cpu < 0) + return cpu; + scx_kick_cpu(sch, cpu, flags); + return 0; +} + +/** + * scx_bpf_dsq_nr_queued - Return the number of queued tasks + * @dsq_id: id of the DSQ + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Return the number of tasks in the DSQ matching @dsq_id. If not found, + * -%ENOENT is returned. + */ +__bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + struct scx_dispatch_q *dsq; + s32 ret; + + preempt_disable(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) { + ret = -ENODEV; + goto out; + } + + if (dsq_id == SCX_DSQ_LOCAL) { + ret = READ_ONCE(this_rq()->scx.local_dsq.nr); + goto out; + } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { + s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK); + + if (scx_cpu_valid(sch, cpu, NULL)) { + ret = READ_ONCE(cpu_rq(cpu)->scx.local_dsq.nr); + goto out; + } + } else { + dsq = find_user_dsq(sch, dsq_id); + if (dsq) { + ret = READ_ONCE(dsq->nr); + goto out; + } + } + ret = -ENOENT; +out: + preempt_enable(); + return ret; +} + +/** + * scx_bpf_destroy_dsq - Destroy a custom DSQ + * @dsq_id: DSQ to destroy + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with + * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is + * empty and no further tasks are dispatched to it. Ignored if called on a DSQ + * which doesn't exist. Can be called from any online scx_ops operations. + */ +__bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + sch = scx_prog_sched(aux); + if (sch) + destroy_dsq(sch, dsq_id); +} + +/** + * bpf_iter_scx_dsq_new - Create a DSQ iterator + * @it: iterator to initialize + * @dsq_id: DSQ to iterate + * @flags: %SCX_DSQ_ITER_* + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Initialize BPF iterator @it which can be used with bpf_for_each() to walk + * tasks in the DSQ specified by @dsq_id. Iteration using @it only includes + * tasks which are already queued when this function is invoked. + */ +__bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, + u64 flags, const struct bpf_prog_aux *aux) +{ + struct bpf_iter_scx_dsq_kern *kit = (void *)it; + struct scx_sched *sch; + + BUILD_BUG_ON(sizeof(struct bpf_iter_scx_dsq_kern) > + sizeof(struct bpf_iter_scx_dsq)); + BUILD_BUG_ON(__alignof__(struct bpf_iter_scx_dsq_kern) != + __alignof__(struct bpf_iter_scx_dsq)); + BUILD_BUG_ON(__SCX_DSQ_ITER_ALL_FLAGS & + ((1U << __SCX_DSQ_LNODE_PRIV_SHIFT) - 1)); + + /* + * next() and destroy() will be called regardless of the return value. + * Always clear $kit->dsq. + */ + kit->dsq = NULL; + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return -ENODEV; + + if (flags & ~__SCX_DSQ_ITER_USER_FLAGS) + return -EINVAL; + + kit->dsq = find_user_dsq(sch, dsq_id); + if (!kit->dsq) + return -ENOENT; + + kit->cursor = INIT_DSQ_LIST_CURSOR(kit->cursor, kit->dsq, flags); + + return 0; +} + +/** + * bpf_iter_scx_dsq_next - Progress a DSQ iterator + * @it: iterator to progress + * + * Return the next task. See bpf_iter_scx_dsq_new(). + */ +__bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it) +{ + struct bpf_iter_scx_dsq_kern *kit = (void *)it; + + if (!kit->dsq) + return NULL; + + guard(raw_spinlock_irqsave)(&kit->dsq->lock); + + return nldsq_cursor_next_task(&kit->cursor, kit->dsq); +} + +/** + * bpf_iter_scx_dsq_destroy - Destroy a DSQ iterator + * @it: iterator to destroy + * + * Undo scx_iter_scx_dsq_new(). + */ +__bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) +{ + struct bpf_iter_scx_dsq_kern *kit = (void *)it; + + if (!kit->dsq) + return; + + if (!list_empty(&kit->cursor.node)) { + unsigned long flags; + + raw_spin_lock_irqsave(&kit->dsq->lock, flags); + list_del_init(&kit->cursor.node); + raw_spin_unlock_irqrestore(&kit->dsq->lock, flags); + } + kit->dsq = NULL; +} + +/** + * scx_bpf_dsq_peek - Lockless peek at the first element. + * @dsq_id: DSQ to examine. + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Read the first element in the DSQ. This is semantically equivalent to using + * the DSQ iterator, but is lockfree. Of course, like any lockless operation, + * this provides only a point-in-time snapshot, and the contents may change + * by the time any subsequent locking operation reads the queue. + * + * Returns the pointer, or NULL indicates an empty queue OR internal error. + */ +__bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + struct scx_dispatch_q *dsq; + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return NULL; + + if (unlikely(dsq_id & SCX_DSQ_FLAG_BUILTIN)) { + scx_error(sch, "peek disallowed on builtin DSQ 0x%llx", dsq_id); + return NULL; + } + + dsq = find_user_dsq(sch, dsq_id); + if (unlikely(!dsq)) { + scx_error(sch, "peek on non-existent DSQ 0x%llx", dsq_id); + return NULL; + } + + return rcu_dereference(dsq->first_task); +} + +/** + * scx_bpf_dsq_reenq - Re-enqueue tasks on a DSQ + * @dsq_id: DSQ to re-enqueue + * @reenq_flags: %SCX_RENQ_* + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Iterate over all of the tasks currently enqueued on the DSQ identified by + * @dsq_id, and re-enqueue them in the BPF scheduler. The following DSQs are + * supported: + * + * - Local DSQs (%SCX_DSQ_LOCAL or %SCX_DSQ_LOCAL_ON | $cpu) + * - User DSQs + * + * Re-enqueues are performed asynchronously. Can be called from anywhere. + */ +__bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + struct scx_dispatch_q *dsq; + + guard(preempt)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return; + + if (unlikely(reenq_flags & ~__SCX_REENQ_USER_MASK)) { + scx_error(sch, "invalid SCX_REENQ flags 0x%llx", reenq_flags); + return; + } + + /* not specifying any filter bits is the same as %SCX_REENQ_ANY */ + if (!(reenq_flags & __SCX_REENQ_FILTER_MASK)) + reenq_flags |= SCX_REENQ_ANY; + + dsq = find_dsq_for_dispatch(sch, this_rq(), dsq_id, smp_processor_id()); + schedule_dsq_reenq(sch, dsq, reenq_flags, scx_locked_rq()); +} + +/** + * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Iterate over all of the tasks currently enqueued on the local DSQ of the + * caller's CPU, and re-enqueue them in the BPF scheduler. Can be called from + * anywhere. + * + * This is now a special case of scx_bpf_dsq_reenq() and may be removed in the + * future. + */ +__bpf_kfunc void scx_bpf_reenqueue_local___v2(const struct bpf_prog_aux *aux) +{ + scx_bpf_dsq_reenq(SCX_DSQ_LOCAL, 0, aux); +} + +__bpf_kfunc_end_defs(); + +__printf(5, 0) +static s32 __bstr_format(struct scx_sched *sch, u64 *data_buf, char *line_buf, + size_t line_size, char *fmt, unsigned long long *data, + u32 data__sz) +{ + struct bpf_bprintf_data bprintf_data = { .get_bin_args = true }; + s32 ret; + + if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 || + (data__sz && !data)) { + scx_error(sch, "invalid data=%p and data__sz=%u", (void *)data, data__sz); + return -EINVAL; + } + + ret = copy_from_kernel_nofault(data_buf, data, data__sz); + if (ret < 0) { + scx_error(sch, "failed to read data fields (%d)", ret); + return ret; + } + + ret = bpf_bprintf_prepare(fmt, UINT_MAX, data_buf, data__sz / 8, + &bprintf_data); + if (ret < 0) { + scx_error(sch, "format preparation failed (%d)", ret); + return ret; + } + + ret = bstr_printf(line_buf, line_size, fmt, + bprintf_data.bin_args); + bpf_bprintf_cleanup(&bprintf_data); + if (ret < 0) { + scx_error(sch, "(\"%s\", %p, %u) failed to format", fmt, data, data__sz); + return ret; + } + + return ret; +} + +__printf(3, 0) +static s32 bstr_format(struct scx_sched *sch, struct scx_bstr_buf *buf, + char *fmt, unsigned long long *data, u32 data__sz) +{ + return __bstr_format(sch, buf->data, buf->line, sizeof(buf->line), + fmt, data, data__sz); +} + +__bpf_kfunc_start_defs(); + +/** + * scx_bpf_exit_bstr - Gracefully exit the BPF scheduler. + * @exit_code: Exit value to pass to user space via struct scx_exit_info. + * @fmt: error message format string + * @data: format string parameters packaged using ___bpf_fill() macro + * @data__sz: @data len, must end in '__sz' for the verifier + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Indicate that the BPF scheduler wants to exit gracefully, and initiate ops + * disabling. + */ +__printf(2, 0) +__bpf_kfunc void scx_bpf_exit_bstr(s64 exit_code, char *fmt, + unsigned long long *data, u32 data__sz, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + unsigned long flags; + + raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags); + sch = scx_prog_sched(aux); + if (likely(sch) && + bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0) + scx_exit(sch, SCX_EXIT_UNREG_BPF, exit_code, "%s", scx_exit_bstr_buf.line); + raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags); +} + +/** + * scx_bpf_error_bstr - Indicate fatal error + * @fmt: error message format string + * @data: format string parameters packaged using ___bpf_fill() macro + * @data__sz: @data len, must end in '__sz' for the verifier + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Indicate that the BPF scheduler encountered a fatal error and initiate ops + * disabling. + */ +__printf(1, 0) +__bpf_kfunc void scx_bpf_error_bstr(char *fmt, unsigned long long *data, + u32 data__sz, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + unsigned long flags; + + raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags); + sch = scx_prog_sched(aux); + if (likely(sch) && + bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0) + scx_exit(sch, SCX_EXIT_ERROR_BPF, 0, "%s", scx_exit_bstr_buf.line); + raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags); +} + +/** + * scx_bpf_dump_bstr - Generate extra debug dump specific to the BPF scheduler + * @fmt: format string + * @data: format string parameters packaged using ___bpf_fill() macro + * @data__sz: @data len, must end in '__sz' for the verifier + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * To be called through scx_bpf_dump() helper from ops.dump(), dump_cpu() and + * dump_task() to generate extra debug dump specific to the BPF scheduler. + * + * The extra dump may be multiple lines. A single line may be split over + * multiple calls. The last line is automatically terminated. + */ +__printf(1, 0) +__bpf_kfunc void scx_bpf_dump_bstr(char *fmt, unsigned long long *data, + u32 data__sz, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + struct scx_dump_data *dd = &scx_dump_data; + struct scx_bstr_buf *buf = &dd->buf; + s32 ret; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return; + + if (raw_smp_processor_id() != dd->cpu) { + scx_error(sch, "scx_bpf_dump() must only be called from ops.dump() and friends"); + return; + } + + /* append the formatted string to the line buf */ + ret = __bstr_format(sch, buf->data, buf->line + dd->cursor, + sizeof(buf->line) - dd->cursor, fmt, data, data__sz); + if (ret < 0) { + dump_line(dd->s, "%s[!] (\"%s\", %p, %u) failed to format (%d)", + dd->prefix, fmt, data, data__sz, ret); + return; + } + + dd->cursor += ret; + dd->cursor = min_t(s32, dd->cursor, sizeof(buf->line)); + + if (!dd->cursor) + return; + + /* + * If the line buf overflowed or ends in a newline, flush it into the + * dump. This is to allow the caller to generate a single line over + * multiple calls. As ops_dump_flush() can also handle multiple lines in + * the line buf, the only case which can lead to an unexpected + * truncation is when the caller keeps generating newlines in the middle + * instead of the end consecutively. Don't do that. + */ + if (dd->cursor >= sizeof(buf->line) || buf->line[dd->cursor - 1] == '\n') + ops_dump_flush(); +} + +/** + * scx_bpf_cpuperf_cap - Query the maximum relative capacity of a CPU + * @cpu: CPU of interest + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Return the maximum relative capacity of @cpu in relation to the most + * performant CPU in the system. The return value is in the range [1, + * %SCX_CPUPERF_ONE]. See scx_bpf_cpuperf_cur(). + */ +__bpf_kfunc u32 scx_bpf_cpuperf_cap(s32 cpu, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (likely(sch) && scx_cpu_valid(sch, cpu, NULL)) + return arch_scale_cpu_capacity(cpu); + else + return SCX_CPUPERF_ONE; +} + +/** + * scx_bpf_cidperf_cap - Query the maximum relative capacity of the CPU at @cid + * @cid: cid of the CPU to query + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * cid-addressed equivalent of scx_bpf_cpuperf_cap(). + */ +__bpf_kfunc u32 scx_bpf_cidperf_cap(s32 cid, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + s32 cpu; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return SCX_CPUPERF_ONE; + cpu = scx_cid_to_cpu(sch, cid); + if (cpu < 0) + return SCX_CPUPERF_ONE; + return arch_scale_cpu_capacity(cpu); +} + +/** + * scx_bpf_cpuperf_cur - Query the current relative performance of a CPU + * @cpu: CPU of interest + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Return the current relative performance of @cpu in relation to its maximum. + * The return value is in the range [1, %SCX_CPUPERF_ONE]. + * + * The current performance level of a CPU in relation to the maximum performance + * available in the system can be calculated as follows: + * + * scx_bpf_cpuperf_cap() * scx_bpf_cpuperf_cur() / %SCX_CPUPERF_ONE + * + * The result is in the range [1, %SCX_CPUPERF_ONE]. + */ +__bpf_kfunc u32 scx_bpf_cpuperf_cur(s32 cpu, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (likely(sch) && scx_cpu_valid(sch, cpu, NULL)) + return arch_scale_freq_capacity(cpu); + else + return SCX_CPUPERF_ONE; +} + +/** + * scx_bpf_cidperf_cur - Query the current performance of the CPU at @cid + * @cid: cid of the CPU to query + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * cid-addressed equivalent of scx_bpf_cpuperf_cur(). + */ +__bpf_kfunc u32 scx_bpf_cidperf_cur(s32 cid, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + s32 cpu; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return SCX_CPUPERF_ONE; + cpu = scx_cid_to_cpu(sch, cid); + if (cpu < 0) + return SCX_CPUPERF_ONE; + return arch_scale_freq_capacity(cpu); +} + +/** + * scx_bpf_cpuperf_set - Set the relative performance target of a CPU + * @cpu: CPU of interest + * @perf: target performance level [0, %SCX_CPUPERF_ONE] + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Set the target performance level of @cpu to @perf. @perf is in linear + * relative scale between 0 and %SCX_CPUPERF_ONE. This determines how the + * schedutil cpufreq governor chooses the target frequency. + * + * The actual performance level chosen, CPU grouping, and the overhead and + * latency of the operations are dependent on the hardware and cpufreq driver in + * use. Consult hardware and cpufreq documentation for more information. The + * current performance level can be monitored using scx_bpf_cpuperf_cur(). + */ +__bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return; + + if (unlikely(perf > SCX_CPUPERF_ONE)) { + scx_error(sch, "Invalid cpuperf target %u for CPU %d", perf, cpu); + return; + } + + if (scx_cpu_valid(sch, cpu, NULL)) { + struct rq *rq = cpu_rq(cpu), *locked_rq = scx_locked_rq(); + struct rq_flags rf; + + /* + * When called with an rq lock held, restrict the operation + * to the corresponding CPU to prevent ABBA deadlocks. + */ + if (locked_rq && rq != locked_rq) { + scx_error(sch, "Invalid target CPU %d", cpu); + return; + } + + /* + * If no rq lock is held, allow to operate on any CPU by + * acquiring the corresponding rq lock. + */ + if (!locked_rq) { + rq_lock_irqsave(rq, &rf); + update_rq_clock(rq); + } + + rq->scx.cpuperf_target = perf; + cpufreq_update_util(rq, 0); + + if (!locked_rq) + rq_unlock_irqrestore(rq, &rf); + } +} + +/** + * scx_bpf_cidperf_set - Set the performance target of the CPU at @cid + * @cid: cid of the CPU to target + * @perf: target performance level [0, %SCX_CPUPERF_ONE] + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * cid-addressed equivalent of scx_bpf_cpuperf_set(). + */ +__bpf_kfunc void scx_bpf_cidperf_set(s32 cid, u32 perf, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + s32 cpu; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return; + cpu = scx_cid_to_cpu(sch, cid); + if (cpu < 0) + return; + scx_bpf_cpuperf_set(cpu, perf, aux); +} + +/** + * scx_bpf_nr_node_ids - Return the number of possible node IDs + * + * All valid node IDs in the system are smaller than the returned value. + */ +__bpf_kfunc u32 scx_bpf_nr_node_ids(void) +{ + return nr_node_ids; +} + +/** + * scx_bpf_nr_cpu_ids - Return the number of possible CPU IDs + * + * All valid CPU IDs in the system are smaller than the returned value. + */ +__bpf_kfunc u32 scx_bpf_nr_cpu_ids(void) +{ + return nr_cpu_ids; +} + +/** + * scx_bpf_nr_cids - Return the size of the cid space + * + * Equals num_possible_cpus(). All valid cids are in [0, return value). + */ +__bpf_kfunc u32 scx_bpf_nr_cids(void) +{ + return num_possible_cpus(); +} + +/** + * scx_bpf_nr_online_cids - Return current count of online CPUs in cid space + * + * Return num_online_cpus(). The standard model restarts the scheduler on + * hotplug, which lets schedulers treat [0, nr_online_cids) as the online + * range. Schedulers that prefer to handle hotplug without a restart should + * install a custom mapping via scx_bpf_cid_override() and track onlining + * through the ops.cid_online / ops.cid_offline callbacks. + */ +__bpf_kfunc u32 scx_bpf_nr_online_cids(void) +{ + return num_online_cpus(); +} + +/** + * scx_bpf_this_cid - Return the cid of the CPU this program is running on + * + * cid-addressed equivalent of bpf_get_smp_processor_id() for scx programs. + * The current cpu is trivially valid, so this is just a table lookup. Return + * -EINVAL if called from a non-SCX program before any scheduler has ever + * been enabled (the cid table is still unallocated at that point). + */ +__bpf_kfunc s32 scx_bpf_this_cid(void) +{ + s16 *tbl = READ_ONCE(scx_cpu_to_cid_tbl); + + if (!tbl) + return -EINVAL; + return tbl[raw_smp_processor_id()]; +} + +/** + * scx_bpf_get_possible_cpumask - Get a referenced kptr to cpu_possible_mask + */ +__bpf_kfunc const struct cpumask *scx_bpf_get_possible_cpumask(void) +{ + return cpu_possible_mask; +} + +/** + * scx_bpf_get_online_cpumask - Get a referenced kptr to cpu_online_mask + */ +__bpf_kfunc const struct cpumask *scx_bpf_get_online_cpumask(void) +{ + return cpu_online_mask; +} + +/** + * scx_bpf_put_cpumask - Release a possible/online cpumask + * @cpumask: cpumask to release + */ +__bpf_kfunc void scx_bpf_put_cpumask(const struct cpumask *cpumask) +{ + /* + * Empty function body because we aren't actually acquiring or releasing + * a reference to a global cpumask, which is read-only in the caller and + * is never released. The acquire / release semantics here are just used + * to make the cpumask is a trusted pointer in the caller. + */ +} + +/** + * scx_bpf_task_running - Is task currently running? + * @p: task of interest + */ +__bpf_kfunc bool scx_bpf_task_running(const struct task_struct *p) +{ + return task_rq(p)->curr == p; +} + +/** + * scx_bpf_task_cpu - CPU a task is currently associated with + * @p: task of interest + */ +__bpf_kfunc s32 scx_bpf_task_cpu(const struct task_struct *p) +{ + return task_cpu(p); +} + +/** + * scx_bpf_task_cid - cid a task is currently associated with + * @p: task of interest + * + * cid-addressed equivalent of scx_bpf_task_cpu(). task_cpu(p) is always a + * valid cpu, so this is just a table lookup. Return -EINVAL if called from + * a non-SCX program before any scheduler has ever been enabled. + */ +__bpf_kfunc s32 scx_bpf_task_cid(const struct task_struct *p) +{ + s16 *tbl = READ_ONCE(scx_cpu_to_cid_tbl); + + if (!tbl) + return -EINVAL; + return tbl[task_cpu(p)]; +} + +/** + * scx_bpf_cpu_rq - Fetch the rq of a CPU + * @cpu: CPU of the rq + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + */ +__bpf_kfunc struct rq *scx_bpf_cpu_rq(s32 cpu, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return NULL; + + if (!scx_cpu_valid(sch, cpu, NULL)) + return NULL; + + if (!sch->warned_deprecated_rq) { + printk_deferred(KERN_WARNING "sched_ext: %s() is deprecated; " + "use scx_bpf_locked_rq() when holding rq lock " + "or scx_bpf_cpu_curr() to read remote curr safely.\n", __func__); + sch->warned_deprecated_rq = true; + } + + return cpu_rq(cpu); +} + +/** + * scx_bpf_locked_rq - Return the rq currently locked by SCX + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Returns the rq if a rq lock is currently held by SCX. + * Otherwise emits an error and returns NULL. + */ +__bpf_kfunc struct rq *scx_bpf_locked_rq(const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + struct rq *rq; + + guard(preempt)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return NULL; + + rq = scx_locked_rq(); + if (!rq) { + scx_error(sch, "accessing rq without holding rq lock"); + return NULL; + } + + return rq; +} + +/** + * scx_bpf_cpu_curr - Return remote CPU's curr task + * @cpu: CPU of interest + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Callers must hold RCU read lock (KF_RCU). + */ +__bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return NULL; + + if (!scx_cpu_valid(sch, cpu, NULL)) + return NULL; + + return rcu_dereference(cpu_rq(cpu)->curr); +} + +/** + * scx_bpf_cid_curr - Return the curr task on the CPU at @cid + * @cid: cid of interest + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * cid-addressed equivalent of scx_bpf_cpu_curr(). Callers must hold RCU + * read lock (KF_RCU). + */ +__bpf_kfunc struct task_struct *scx_bpf_cid_curr(s32 cid, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + s32 cpu; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return NULL; + cpu = scx_cid_to_cpu(sch, cid); + if (cpu < 0) + return NULL; + return rcu_dereference(cpu_rq(cpu)->curr); +} + +/** + * scx_bpf_tid_to_task - Look up a task by its scx tid + * @tid: task ID previously read from p->scx.tid + * + * Returns the task with the given tid, or NULL if no such task exists. The + * returned pointer is valid until the end of the current RCU read section + * (KF_RCU_PROTECTED). Requires SCX_OPS_TID_TO_TASK to be set on the root + * scheduler; otherwise an error is raised and NULL returned. + */ +__bpf_kfunc struct task_struct *scx_bpf_tid_to_task(u64 tid) +{ + struct sched_ext_entity *scx; + + if (!scx_tid_to_task_enabled()) { + struct scx_sched *sch = rcu_dereference(scx_root); + + if (sch) + scx_error(sch, "scx_bpf_tid_to_task() called without SCX_OPS_TID_TO_TASK"); + return NULL; + } + + scx = rhashtable_lookup(&scx_tid_hash, &tid, scx_tid_hash_params); + if (!scx) + return NULL; + + return container_of(scx, struct task_struct, scx); +} + +/** + * scx_bpf_now - Returns a high-performance monotonically non-decreasing + * clock for the current CPU. The clock returned is in nanoseconds. + * + * It provides the following properties: + * + * 1) High performance: Many BPF schedulers call bpf_ktime_get_ns() frequently + * to account for execution time and track tasks' runtime properties. + * Unfortunately, in some hardware platforms, bpf_ktime_get_ns() -- which + * eventually reads a hardware timestamp counter -- is neither performant nor + * scalable. scx_bpf_now() aims to provide a high-performance clock by + * using the rq clock in the scheduler core whenever possible. + * + * 2) High enough resolution for the BPF scheduler use cases: In most BPF + * scheduler use cases, the required clock resolution is lower than the most + * accurate hardware clock (e.g., rdtsc in x86). scx_bpf_now() basically + * uses the rq clock in the scheduler core whenever it is valid. It considers + * that the rq clock is valid from the time the rq clock is updated + * (update_rq_clock) until the rq is unlocked (rq_unpin_lock). + * + * 3) Monotonically non-decreasing clock for the same CPU: scx_bpf_now() + * guarantees the clock never goes backward when comparing them in the same + * CPU. On the other hand, when comparing clocks in different CPUs, there + * is no such guarantee -- the clock can go backward. It provides a + * monotonically *non-decreasing* clock so that it would provide the same + * clock values in two different scx_bpf_now() calls in the same CPU + * during the same period of when the rq clock is valid. + */ +__bpf_kfunc u64 scx_bpf_now(void) +{ + struct rq *rq; + u64 clock; + + preempt_disable(); + + rq = this_rq(); + if (smp_load_acquire(&rq->scx.flags) & SCX_RQ_CLK_VALID) { + /* + * If the rq clock is valid, use the cached rq clock. + * + * Note that scx_bpf_now() is re-entrant between a process + * context and an interrupt context (e.g., timer interrupt). + * However, we don't need to consider the race between them + * because such race is not observable from a caller. + */ + clock = READ_ONCE(rq->scx.clock); + } else { + /* + * Otherwise, return a fresh rq clock. + * + * The rq clock is updated outside of the rq lock. + * In this case, keep the updated rq clock invalid so the next + * kfunc call outside the rq lock gets a fresh rq clock. + */ + clock = sched_clock_cpu(cpu_of(rq)); + } + + preempt_enable(); + + return clock; +} + +static void scx_read_events(struct scx_sched *sch, struct scx_event_stats *events) +{ + struct scx_event_stats *e_cpu; + int cpu; + + /* Aggregate per-CPU event counters into @events. */ + memset(events, 0, sizeof(*events)); + for_each_possible_cpu(cpu) { + e_cpu = &per_cpu_ptr(sch->pcpu, cpu)->event_stats; + scx_agg_event(events, e_cpu, SCX_EV_SELECT_CPU_FALLBACK); + scx_agg_event(events, e_cpu, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); + scx_agg_event(events, e_cpu, SCX_EV_DISPATCH_KEEP_LAST); + scx_agg_event(events, e_cpu, SCX_EV_ENQ_SKIP_EXITING); + scx_agg_event(events, e_cpu, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); + scx_agg_event(events, e_cpu, SCX_EV_REENQ_IMMED); + scx_agg_event(events, e_cpu, SCX_EV_REENQ_LOCAL_REPEAT); + scx_agg_event(events, e_cpu, SCX_EV_REFILL_SLICE_DFL); + scx_agg_event(events, e_cpu, SCX_EV_BYPASS_DURATION); + scx_agg_event(events, e_cpu, SCX_EV_BYPASS_DISPATCH); + scx_agg_event(events, e_cpu, SCX_EV_BYPASS_ACTIVATE); + scx_agg_event(events, e_cpu, SCX_EV_INSERT_NOT_OWNED); + scx_agg_event(events, e_cpu, SCX_EV_SUB_BYPASS_DISPATCH); + } +} + +/* + * scx_bpf_events - Get a system-wide event counter to + * @events: output buffer from a BPF program + * @events__sz: @events len, must end in '__sz'' for the verifier + */ +__bpf_kfunc void scx_bpf_events(struct scx_event_stats *events, + size_t events__sz) +{ + struct scx_sched *sch; + struct scx_event_stats e_sys; + + rcu_read_lock(); + sch = rcu_dereference(scx_root); + if (sch) + scx_read_events(sch, &e_sys); + else + memset(&e_sys, 0, sizeof(e_sys)); + rcu_read_unlock(); + + /* + * We cannot entirely trust a BPF-provided size since a BPF program + * might be compiled against a different vmlinux.h, of which + * scx_event_stats would be larger (a newer vmlinux.h) or smaller + * (an older vmlinux.h). Hence, we use the smaller size to avoid + * memory corruption. + */ + events__sz = min(events__sz, sizeof(*events)); + memcpy(events, &e_sys, events__sz); +} + +#ifdef CONFIG_CGROUP_SCHED +/** + * scx_bpf_task_cgroup - Return the sched cgroup of a task + * @p: task of interest + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with + * from the scheduler's POV. SCX operations should use this function to + * determine @p's current cgroup as, unlike following @p->cgroups, + * @p->sched_task_group is stable for the duration of the SCX op. See + * SCX_CALL_OP_TASK() for details. + */ +__bpf_kfunc struct cgroup *scx_bpf_task_cgroup(struct task_struct *p, + const struct bpf_prog_aux *aux) +{ + struct task_group *tg = p->sched_task_group; + struct cgroup *cgrp = &cgrp_dfl_root.cgrp; + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + goto out; + + if (!scx_kf_arg_task_ok(sch, p)) + goto out; + + cgrp = tg_cgrp(tg); + +out: + cgroup_get(cgrp); + return cgrp; +} +#endif /* CONFIG_CGROUP_SCHED */ + +__bpf_kfunc_end_defs(); + +BTF_KFUNCS_START(scx_kfunc_ids_any) +BTF_ID_FLAGS(func, scx_bpf_task_set_slice, KF_IMPLICIT_ARGS | KF_RCU); +BTF_ID_FLAGS(func, scx_bpf_task_set_dsq_vtime, KF_IMPLICIT_ARGS | KF_RCU); +BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_kick_cid, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_destroy_dsq, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_dsq_peek, KF_IMPLICIT_ARGS | KF_RCU_PROTECTED | KF_RET_NULL) +BTF_ID_FLAGS(func, scx_bpf_dsq_reenq, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_reenqueue_local___v2, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, bpf_iter_scx_dsq_new, KF_IMPLICIT_ARGS | KF_ITER_NEW | KF_RCU_PROTECTED) +BTF_ID_FLAGS(func, bpf_iter_scx_dsq_next, KF_ITER_NEXT | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_iter_scx_dsq_destroy, KF_ITER_DESTROY) +BTF_ID_FLAGS(func, scx_bpf_exit_bstr, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_dump_bstr, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cidperf_cap, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cidperf_cur, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cidperf_set, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_nr_node_ids) +BTF_ID_FLAGS(func, scx_bpf_nr_cpu_ids) +BTF_ID_FLAGS(func, scx_bpf_nr_cids) +BTF_ID_FLAGS(func, scx_bpf_nr_online_cids) +BTF_ID_FLAGS(func, scx_bpf_this_cid) +BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE) +BTF_ID_FLAGS(func, scx_bpf_task_running, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_task_cid, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_cpu_rq, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_locked_rq, KF_IMPLICIT_ARGS | KF_RET_NULL) +BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) +BTF_ID_FLAGS(func, scx_bpf_cid_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) +BTF_ID_FLAGS(func, scx_bpf_tid_to_task, KF_RET_NULL | KF_RCU_PROTECTED) +BTF_ID_FLAGS(func, scx_bpf_now) +BTF_ID_FLAGS(func, scx_bpf_events) +#ifdef CONFIG_CGROUP_SCHED +BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_IMPLICIT_ARGS | KF_RCU | KF_ACQUIRE) +#endif +BTF_KFUNCS_END(scx_kfunc_ids_any) + +static const struct btf_kfunc_id_set scx_kfunc_set_any = { + .owner = THIS_MODULE, + .set = &scx_kfunc_ids_any, + .filter = scx_kfunc_context_filter, +}; + +/* + * cpu-form kfuncs that are forbidden from cid-form schedulers + * (bpf_sched_ext_ops_cid). Programs targeting the cid struct_ops type must + * use the cid-form alternative (cid/cmask kfuncs). + * + * Membership overlaps with scx_kfunc_ids_{any,idle,select_cpu}; the filter + * tests this set independently and rejects matches before the per-op + * allow-list check runs. + * + * pahole/resolve_btfids scans every BTF_ID_FLAGS() at build time and + * intersects flags across duplicate entries, so each entry must carry the + * same flags as the kfunc's primary declaration; otherwise the flags get + * dropped globally. + */ +BTF_KFUNCS_START(scx_kfunc_ids_cpu_only) +BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_cpu_rq, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) +BTF_ID_FLAGS(func, scx_bpf_cpu_node, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE) +BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_IMPLICIT_ARGS | KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_IMPLICIT_ARGS | KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) +BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu_node, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu_node, KF_IMPLICIT_ARGS | KF_RCU) +BTF_KFUNCS_END(scx_kfunc_ids_cpu_only) + +/* + * Per-op kfunc allow flags. Each bit corresponds to a context-sensitive kfunc + * group; an op may permit zero or more groups, with the union expressed in + * scx_kf_allow_flags[]. The verifier-time filter (scx_kfunc_context_filter()) + * consults this table to decide whether a context-sensitive kfunc is callable + * from a given SCX op. + */ +enum scx_kf_allow_flags { + SCX_KF_ALLOW_UNLOCKED = 1 << 0, + SCX_KF_ALLOW_INIT = 1 << 1, + SCX_KF_ALLOW_CPU_RELEASE = 1 << 2, + SCX_KF_ALLOW_DISPATCH = 1 << 3, + SCX_KF_ALLOW_ENQUEUE = 1 << 4, + SCX_KF_ALLOW_SELECT_CPU = 1 << 5, +}; + +/* + * Map each SCX op to the union of kfunc groups it permits, indexed by + * SCX_OP_IDX(op). Ops not listed only permit kfuncs that are not + * context-sensitive. + */ +static const u32 scx_kf_allow_flags[] = { + [SCX_OP_IDX(select_cpu)] = SCX_KF_ALLOW_SELECT_CPU | SCX_KF_ALLOW_ENQUEUE, + [SCX_OP_IDX(enqueue)] = SCX_KF_ALLOW_SELECT_CPU | SCX_KF_ALLOW_ENQUEUE, + [SCX_OP_IDX(dispatch)] = SCX_KF_ALLOW_ENQUEUE | SCX_KF_ALLOW_DISPATCH, + [SCX_OP_IDX(cpu_release)] = SCX_KF_ALLOW_CPU_RELEASE, + [SCX_OP_IDX(init_task)] = SCX_KF_ALLOW_UNLOCKED, + [SCX_OP_IDX(dump)] = SCX_KF_ALLOW_UNLOCKED, +#ifdef CONFIG_EXT_GROUP_SCHED + [SCX_OP_IDX(cgroup_init)] = SCX_KF_ALLOW_UNLOCKED, + [SCX_OP_IDX(cgroup_exit)] = SCX_KF_ALLOW_UNLOCKED, + [SCX_OP_IDX(cgroup_prep_move)] = SCX_KF_ALLOW_UNLOCKED, + [SCX_OP_IDX(cgroup_cancel_move)] = SCX_KF_ALLOW_UNLOCKED, + [SCX_OP_IDX(cgroup_set_weight)] = SCX_KF_ALLOW_UNLOCKED, + [SCX_OP_IDX(cgroup_set_bandwidth)] = SCX_KF_ALLOW_UNLOCKED, + [SCX_OP_IDX(cgroup_set_idle)] = SCX_KF_ALLOW_UNLOCKED, +#endif /* CONFIG_EXT_GROUP_SCHED */ + [SCX_OP_IDX(sub_attach)] = SCX_KF_ALLOW_UNLOCKED, + [SCX_OP_IDX(sub_detach)] = SCX_KF_ALLOW_UNLOCKED, + [SCX_OP_IDX(cpu_online)] = SCX_KF_ALLOW_UNLOCKED, + [SCX_OP_IDX(cpu_offline)] = SCX_KF_ALLOW_UNLOCKED, + [SCX_OP_IDX(init)] = SCX_KF_ALLOW_UNLOCKED | SCX_KF_ALLOW_INIT, + [SCX_OP_IDX(exit)] = SCX_KF_ALLOW_UNLOCKED, +}; + +/* + * Verifier-time filter for SCX kfuncs. Registered via the .filter field on + * each per-group btf_kfunc_id_set. The BPF core invokes this for every kfunc + * call in the registered hook (BPF_PROG_TYPE_STRUCT_OPS or + * BPF_PROG_TYPE_SYSCALL), regardless of which set originally introduced the + * kfunc - so the filter must short-circuit on kfuncs it doesn't govern by + * falling through to "allow" when none of the SCX sets contain the kfunc. + */ +int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) +{ + bool in_unlocked = btf_id_set8_contains(&scx_kfunc_ids_unlocked, kfunc_id); + bool in_init = btf_id_set8_contains(&scx_kfunc_ids_init, kfunc_id); + bool in_select_cpu = btf_id_set8_contains(&scx_kfunc_ids_select_cpu, kfunc_id); + bool in_enqueue = btf_id_set8_contains(&scx_kfunc_ids_enqueue_dispatch, kfunc_id); + bool in_dispatch = btf_id_set8_contains(&scx_kfunc_ids_dispatch, kfunc_id); + bool in_cpu_release = btf_id_set8_contains(&scx_kfunc_ids_cpu_release, kfunc_id); + bool in_idle = btf_id_set8_contains(&scx_kfunc_ids_idle, kfunc_id); + bool in_any = btf_id_set8_contains(&scx_kfunc_ids_any, kfunc_id); + bool in_cpu_only = btf_id_set8_contains(&scx_kfunc_ids_cpu_only, kfunc_id); + u32 moff, flags; + + /* Not an SCX kfunc - allow. */ + if (!(in_unlocked || in_init || in_select_cpu || in_enqueue || in_dispatch || + in_cpu_release || in_idle || in_any)) + return 0; + + /* SYSCALL progs (e.g. BPF test_run()) may call unlocked and select_cpu kfuncs. */ + if (prog->type == BPF_PROG_TYPE_SYSCALL) + return (in_unlocked || in_select_cpu || in_idle || in_any) ? 0 : -EACCES; + + if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) + return (in_any || in_idle) ? 0 : -EACCES; + + /* + * add_subprog_and_kfunc() collects all kfunc calls, including dead code + * guarded by bpf_ksym_exists(), before check_attach_btf_id() sets + * prog->aux->st_ops. Allow all kfuncs when st_ops is not yet set; + * do_check_main() re-runs the filter with st_ops set and enforces the + * actual restrictions. + */ + if (!prog->aux->st_ops) + return 0; + + /* + * Non-SCX struct_ops: SCX kfuncs are not permitted. + * + * Both bpf_sched_ext_ops (cpu-form) and bpf_sched_ext_ops_cid + * (cid-form) are valid SCX struct_ops. Member offsets match between + * the two (verified by BUILD_BUG_ON in scx_init()), so the shared + * scx_kf_allow_flags[] table indexed by SCX_MOFF_IDX(moff) applies to + * both. + */ + if (prog->aux->st_ops != &bpf_sched_ext_ops && + prog->aux->st_ops != &bpf_sched_ext_ops_cid) + return -EACCES; + + /* + * cid-form schedulers must use cid/cmask kfuncs. cid and cpu are both + * small s32s and trivially confused, so cpu-only kfuncs are rejected at + * load time. The reverse (cpu-form calling cid-form kfuncs) is + * intentionally permissive to ease gradual cpumask -> cid migration. + */ + if (prog->aux->st_ops == &bpf_sched_ext_ops_cid && in_cpu_only) + return -EACCES; + + /* SCX struct_ops: check the per-op allow list. */ + if (in_any || in_idle) + return 0; + + moff = prog->aux->attach_st_ops_member_off; + flags = scx_kf_allow_flags[SCX_MOFF_IDX(moff)]; + + if ((flags & SCX_KF_ALLOW_UNLOCKED) && in_unlocked) + return 0; + if ((flags & SCX_KF_ALLOW_INIT) && in_init) + return 0; + if ((flags & SCX_KF_ALLOW_CPU_RELEASE) && in_cpu_release) + return 0; + if ((flags & SCX_KF_ALLOW_DISPATCH) && in_dispatch) + return 0; + if ((flags & SCX_KF_ALLOW_ENQUEUE) && in_enqueue) + return 0; + if ((flags & SCX_KF_ALLOW_SELECT_CPU) && in_select_cpu) + return 0; + + return -EACCES; +} + +static int __init scx_init(void) +{ + int ret; + + /* + * sched_ext_ops_cid mirrors sched_ext_ops up to and including @priv. + * Both bpf_scx_init_member() and bpf_scx_check_member() use offsets + * from struct sched_ext_ops; sched_ext_ops_cid relies on those offsets + * matching for the shared fields. Catch any drift at boot. + */ +#define CID_OFFSET_MATCH(cpu_field, cid_field) \ + BUILD_BUG_ON(offsetof(struct sched_ext_ops, cpu_field) != \ + offsetof(struct sched_ext_ops_cid, cid_field)) + /* data fields used by bpf_scx_init_member() */ + CID_OFFSET_MATCH(dispatch_max_batch, dispatch_max_batch); + CID_OFFSET_MATCH(flags, flags); + CID_OFFSET_MATCH(name, name); + CID_OFFSET_MATCH(timeout_ms, timeout_ms); + CID_OFFSET_MATCH(exit_dump_len, exit_dump_len); + CID_OFFSET_MATCH(hotplug_seq, hotplug_seq); + CID_OFFSET_MATCH(sub_cgroup_id, sub_cgroup_id); + /* shared callbacks: the union view requires byte-for-byte offset match */ + CID_OFFSET_MATCH(enqueue, enqueue); + CID_OFFSET_MATCH(dequeue, dequeue); + CID_OFFSET_MATCH(dispatch, dispatch); + CID_OFFSET_MATCH(tick, tick); + CID_OFFSET_MATCH(runnable, runnable); + CID_OFFSET_MATCH(running, running); + CID_OFFSET_MATCH(stopping, stopping); + CID_OFFSET_MATCH(quiescent, quiescent); + CID_OFFSET_MATCH(yield, yield); + CID_OFFSET_MATCH(core_sched_before, core_sched_before); + CID_OFFSET_MATCH(set_weight, set_weight); + CID_OFFSET_MATCH(update_idle, update_idle); + CID_OFFSET_MATCH(init_task, init_task); + CID_OFFSET_MATCH(exit_task, exit_task); + CID_OFFSET_MATCH(enable, enable); + CID_OFFSET_MATCH(disable, disable); + CID_OFFSET_MATCH(dump, dump); + CID_OFFSET_MATCH(dump_task, dump_task); + CID_OFFSET_MATCH(sub_attach, sub_attach); + CID_OFFSET_MATCH(sub_detach, sub_detach); + CID_OFFSET_MATCH(init, init); + CID_OFFSET_MATCH(exit, exit); +#ifdef CONFIG_EXT_GROUP_SCHED + CID_OFFSET_MATCH(cgroup_init, cgroup_init); + CID_OFFSET_MATCH(cgroup_exit, cgroup_exit); + CID_OFFSET_MATCH(cgroup_prep_move, cgroup_prep_move); + CID_OFFSET_MATCH(cgroup_move, cgroup_move); + CID_OFFSET_MATCH(cgroup_cancel_move, cgroup_cancel_move); + CID_OFFSET_MATCH(cgroup_set_weight, cgroup_set_weight); + CID_OFFSET_MATCH(cgroup_set_bandwidth, cgroup_set_bandwidth); + CID_OFFSET_MATCH(cgroup_set_idle, cgroup_set_idle); +#endif + /* renamed callbacks must occupy the same slot as their cpu-form sibling */ + CID_OFFSET_MATCH(select_cpu, select_cid); + CID_OFFSET_MATCH(set_cpumask, set_cmask); + CID_OFFSET_MATCH(cpu_online, cid_online); + CID_OFFSET_MATCH(cpu_offline, cid_offline); + CID_OFFSET_MATCH(dump_cpu, dump_cid); + /* @priv tail must align since both share the same data block */ + CID_OFFSET_MATCH(priv, priv); + /* + * cid-form must end exactly at @priv - validate_ops() skips + * cpu_acquire/cpu_release for cid-form because reading those fields + * past the BPF allocation would be UB. + */ + BUILD_BUG_ON(offsetof(struct sched_ext_ops_cid, __end) != + offsetofend(struct sched_ext_ops, priv)); +#undef CID_OFFSET_MATCH + + /* + * kfunc registration can't be done from init_sched_ext_class() as + * register_btf_kfunc_id_set() needs most of the system to be up. + * + * Some kfuncs are context-sensitive and can only be called from + * specific SCX ops. They are grouped into per-context BTF sets, each + * registered with scx_kfunc_context_filter as its .filter callback. The + * BPF core dedups identical filter pointers per hook + * (btf_populate_kfunc_set()), so the filter is invoked exactly once per + * kfunc lookup; it consults scx_kf_allow_flags[] to enforce per-op + * restrictions at verify time. + */ + if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, + &scx_kfunc_set_enqueue_dispatch)) || + (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, + &scx_kfunc_set_dispatch)) || + (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, + &scx_kfunc_set_cpu_release)) || + (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, + &scx_kfunc_set_unlocked)) || + (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, + &scx_kfunc_set_unlocked)) || + (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, + &scx_kfunc_set_any)) || + (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, + &scx_kfunc_set_any)) || + (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, + &scx_kfunc_set_any))) { + pr_err("sched_ext: Failed to register kfunc sets (%d)\n", ret); + return ret; + } + + ret = scx_idle_init(); + if (ret) { + pr_err("sched_ext: Failed to initialize idle tracking (%d)\n", ret); + return ret; + } + + ret = scx_cid_kfunc_init(); + if (ret) { + pr_err("sched_ext: Failed to register cid kfuncs (%d)\n", ret); + return ret; + } + + ret = register_bpf_struct_ops(&bpf_sched_ext_ops, sched_ext_ops); + if (ret) { + pr_err("sched_ext: Failed to register struct_ops (%d)\n", ret); + return ret; + } + + ret = register_bpf_struct_ops(&bpf_sched_ext_ops_cid, sched_ext_ops_cid); + if (ret) { + pr_err("sched_ext: Failed to register cid struct_ops (%d)\n", ret); + return ret; + } + + ret = register_pm_notifier(&scx_pm_notifier); + if (ret) { + pr_err("sched_ext: Failed to register PM notifier (%d)\n", ret); + return ret; + } + + scx_kset = kset_create_and_add("sched_ext", &scx_uevent_ops, kernel_kobj); + if (!scx_kset) { + pr_err("sched_ext: Failed to create /sys/kernel/sched_ext\n"); + return -ENOMEM; + } + + ret = sysfs_create_group(&scx_kset->kobj, &scx_global_attr_group); + if (ret < 0) { + pr_err("sched_ext: Failed to add global attributes\n"); + return ret; + } + + return 0; +} +__initcall(scx_init); diff --git a/kernel/sched/ext/ext.h b/kernel/sched/ext/ext.h new file mode 100644 index 000000000000..0b7fc46aee08 --- /dev/null +++ b/kernel/sched/ext/ext.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2022 Tejun Heo + * Copyright (c) 2022 David Vernet + */ +#ifdef CONFIG_SCHED_CLASS_EXT + +void scx_tick(struct rq *rq); +void init_scx_entity(struct sched_ext_entity *scx); +void scx_pre_fork(struct task_struct *p); +int scx_fork(struct task_struct *p, struct kernel_clone_args *kargs); +void scx_post_fork(struct task_struct *p); +void scx_cancel_fork(struct task_struct *p); +bool scx_can_stop_tick(struct rq *rq); +void scx_rq_activate(struct rq *rq); +void scx_rq_deactivate(struct rq *rq); +int scx_check_setscheduler(struct task_struct *p, int policy); +bool task_should_scx(int policy); +bool scx_allow_ttwu_queue(const struct task_struct *p); +void init_sched_ext_class(void); + +static inline u32 scx_cpuperf_target(s32 cpu) +{ + if (scx_enabled()) + return cpu_rq(cpu)->scx.cpuperf_target; + else + return 0; +} + +static inline bool task_on_scx(const struct task_struct *p) +{ + return scx_enabled() && p->sched_class == &ext_sched_class; +} + +#ifdef CONFIG_SCHED_CORE +bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, + bool in_fi); +#endif + +#else /* CONFIG_SCHED_CLASS_EXT */ + +static inline void scx_tick(struct rq *rq) {} +static inline void scx_pre_fork(struct task_struct *p) {} +static inline int scx_fork(struct task_struct *p, struct kernel_clone_args *kargs) { return 0; } +static inline void scx_post_fork(struct task_struct *p) {} +static inline void scx_cancel_fork(struct task_struct *p) {} +static inline u32 scx_cpuperf_target(s32 cpu) { return 0; } +static inline bool scx_can_stop_tick(struct rq *rq) { return true; } +static inline void scx_rq_activate(struct rq *rq) {} +static inline void scx_rq_deactivate(struct rq *rq) {} +static inline int scx_check_setscheduler(struct task_struct *p, int policy) { return 0; } +static inline bool task_on_scx(const struct task_struct *p) { return false; } +static inline bool scx_allow_ttwu_queue(const struct task_struct *p) { return true; } +static inline void init_sched_ext_class(void) {} + +#endif /* CONFIG_SCHED_CLASS_EXT */ + +#ifdef CONFIG_SCHED_CLASS_EXT +void __scx_update_idle(struct rq *rq, bool idle, bool do_notify); + +static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) +{ + if (scx_enabled()) + __scx_update_idle(rq, idle, do_notify); +} +#else +static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) {} +#endif + +#ifdef CONFIG_CGROUP_SCHED +#ifdef CONFIG_EXT_GROUP_SCHED +void scx_tg_init(struct task_group *tg); +int scx_tg_online(struct task_group *tg); +void scx_tg_offline(struct task_group *tg); +int scx_cgroup_can_attach(struct cgroup_taskset *tset); +void scx_cgroup_move_task(struct task_struct *p); +void scx_cgroup_cancel_attach(struct cgroup_taskset *tset); +void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight); +void scx_group_set_idle(struct task_group *tg, bool idle); +void scx_group_set_bandwidth(struct task_group *tg, u64 period_us, u64 quota_us, u64 burst_us); +#else /* CONFIG_EXT_GROUP_SCHED */ +static inline void scx_tg_init(struct task_group *tg) {} +static inline int scx_tg_online(struct task_group *tg) { return 0; } +static inline void scx_tg_offline(struct task_group *tg) {} +static inline int scx_cgroup_can_attach(struct cgroup_taskset *tset) { return 0; } +static inline void scx_cgroup_move_task(struct task_struct *p) {} +static inline void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) {} +static inline void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight) {} +static inline void scx_group_set_idle(struct task_group *tg, bool idle) {} +static inline void scx_group_set_bandwidth(struct task_group *tg, u64 period_us, u64 quota_us, u64 burst_us) {} +#endif /* CONFIG_EXT_GROUP_SCHED */ +#endif /* CONFIG_CGROUP_SCHED */ diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c new file mode 100644 index 000000000000..2077373d8da3 --- /dev/null +++ b/kernel/sched/ext/idle.c @@ -0,0 +1,1511 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Built-in idle CPU tracking policy. + * + * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2022 Tejun Heo + * Copyright (c) 2022 David Vernet + * Copyright (c) 2024 Andrea Righi + */ + +/* Enable/disable built-in idle CPU selection policy */ +static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); + +/* Enable/disable per-node idle cpumasks */ +static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_per_node); + +/* Enable/disable LLC aware optimizations */ +static DEFINE_STATIC_KEY_FALSE(scx_selcpu_topo_llc); + +/* Enable/disable NUMA aware optimizations */ +static DEFINE_STATIC_KEY_FALSE(scx_selcpu_topo_numa); + +/* + * cpumasks to track idle CPUs within each NUMA node. + * + * If SCX_OPS_BUILTIN_IDLE_PER_NODE is not enabled, a single global cpumask + * from is used to track all the idle CPUs in the system. + */ +struct scx_idle_cpus { + cpumask_var_t cpu; + cpumask_var_t smt; +}; + +/* + * Global host-wide idle cpumasks (used when SCX_OPS_BUILTIN_IDLE_PER_NODE + * is not enabled). + */ +static struct scx_idle_cpus scx_idle_global_masks; + +/* + * Per-node idle cpumasks. + */ +static struct scx_idle_cpus **scx_idle_node_masks; + +/* + * Local per-CPU cpumasks (used to generate temporary idle cpumasks). + */ +static DEFINE_PER_CPU(cpumask_var_t, local_idle_cpumask); +static DEFINE_PER_CPU(cpumask_var_t, local_llc_idle_cpumask); +static DEFINE_PER_CPU(cpumask_var_t, local_numa_idle_cpumask); + +/* + * Return the idle masks associated to a target @node. + * + * NUMA_NO_NODE identifies the global idle cpumask. + */ +static struct scx_idle_cpus *idle_cpumask(int node) +{ + return node == NUMA_NO_NODE ? &scx_idle_global_masks : scx_idle_node_masks[node]; +} + +/* + * Returns the NUMA node ID associated with a @cpu, or NUMA_NO_NODE if + * per-node idle cpumasks are disabled. + */ +static int scx_cpu_node_if_enabled(int cpu) +{ + if (!static_branch_maybe(CONFIG_NUMA, &scx_builtin_idle_per_node)) + return NUMA_NO_NODE; + + return cpu_to_node(cpu); +} + +static bool scx_idle_test_and_clear_cpu(int cpu) +{ + int node = scx_cpu_node_if_enabled(cpu); + struct cpumask *idle_cpus = idle_cpumask(node)->cpu; + + /* + * SMT mask should be cleared whether we can claim @cpu or not. The SMT + * cluster is not wholly idle either way. This also prevents + * scx_pick_idle_cpu() from getting caught in an infinite loop. + */ + if (sched_smt_active()) { + const struct cpumask *smt = cpu_smt_mask(cpu); + struct cpumask *idle_smts = idle_cpumask(node)->smt; + + /* + * If offline, @cpu is not its own sibling and + * scx_pick_idle_cpu() can get caught in an infinite loop as + * @cpu is never cleared from the idle SMT mask. Ensure that + * @cpu is eventually cleared. + * + * NOTE: Use cpumask_intersects() and cpumask_test_cpu() to + * reduce memory writes, which may help alleviate cache + * coherence pressure. + */ + if (cpumask_intersects(smt, idle_smts)) + cpumask_andnot(idle_smts, idle_smts, smt); + else if (cpumask_test_cpu(cpu, idle_smts)) + __cpumask_clear_cpu(cpu, idle_smts); + } + + return cpumask_test_and_clear_cpu(cpu, idle_cpus); +} + +/* + * Pick an idle CPU in a specific NUMA node. + */ +static s32 pick_idle_cpu_in_node(const struct cpumask *cpus_allowed, int node, u64 flags) +{ + int cpu; + +retry: + if (sched_smt_active()) { + cpu = cpumask_any_and_distribute(idle_cpumask(node)->smt, cpus_allowed); + if (cpu < nr_cpu_ids) + goto found; + + if (flags & SCX_PICK_IDLE_CORE) + return -EBUSY; + } + + cpu = cpumask_any_and_distribute(idle_cpumask(node)->cpu, cpus_allowed); + if (cpu >= nr_cpu_ids) + return -EBUSY; + +found: + if (scx_idle_test_and_clear_cpu(cpu)) + return cpu; + else + goto retry; +} + +#ifdef CONFIG_NUMA +/* + * Tracks nodes that have not yet been visited when searching for an idle + * CPU across all available nodes. + */ +static DEFINE_PER_CPU(nodemask_t, per_cpu_unvisited); + +/* + * Search for an idle CPU across all nodes, excluding @node. + */ +static s32 pick_idle_cpu_from_online_nodes(const struct cpumask *cpus_allowed, int node, u64 flags) +{ + nodemask_t *unvisited; + s32 cpu = -EBUSY; + + preempt_disable(); + unvisited = this_cpu_ptr(&per_cpu_unvisited); + + /* + * Restrict the search to the online nodes (excluding the current + * node that has been visited already). + */ + nodes_copy(*unvisited, node_states[N_ONLINE]); + node_clear(node, *unvisited); + + /* + * Traverse all nodes in order of increasing distance, starting + * from @node. + * + * This loop is O(N^2), with N being the amount of NUMA nodes, + * which might be quite expensive in large NUMA systems. However, + * this complexity comes into play only when a scheduler enables + * SCX_OPS_BUILTIN_IDLE_PER_NODE and it's requesting an idle CPU + * without specifying a target NUMA node, so it shouldn't be a + * bottleneck is most cases. + * + * As a future optimization we may want to cache the list of nodes + * in a per-node array, instead of actually traversing them every + * time. + */ + for_each_node_numadist(node, *unvisited) { + cpu = pick_idle_cpu_in_node(cpus_allowed, node, flags); + if (cpu >= 0) + break; + } + preempt_enable(); + + return cpu; +} +#else +static inline s32 +pick_idle_cpu_from_online_nodes(const struct cpumask *cpus_allowed, int node, u64 flags) +{ + return -EBUSY; +} +#endif + +/* + * Find an idle CPU in the system, starting from @node. + */ +static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed, int node, u64 flags) +{ + s32 cpu; + + /* + * Always search in the starting node first (this is an + * optimization that can save some cycles even when the search is + * not limited to a single node). + */ + cpu = pick_idle_cpu_in_node(cpus_allowed, node, flags); + if (cpu >= 0) + return cpu; + + /* + * Stop the search if we are using only a single global cpumask + * (NUMA_NO_NODE) or if the search is restricted to the first node + * only. + */ + if (node == NUMA_NO_NODE || flags & SCX_PICK_IDLE_IN_NODE) + return -EBUSY; + + /* + * Extend the search to the other online nodes. + */ + return pick_idle_cpu_from_online_nodes(cpus_allowed, node, flags); +} + +/* + * Return the amount of CPUs in the same LLC domain of @cpu (or zero if the LLC + * domain is not defined). + */ +static unsigned int llc_weight(s32 cpu) +{ + struct sched_domain *sd; + + sd = rcu_dereference(per_cpu(sd_llc, cpu)); + if (!sd) + return 0; + + return sd->span_weight; +} + +/* + * Return the cpumask representing the LLC domain of @cpu (or NULL if the LLC + * domain is not defined). + */ +static struct cpumask *llc_span(s32 cpu) +{ + struct sched_domain *sd; + + sd = rcu_dereference(per_cpu(sd_llc, cpu)); + if (!sd) + return NULL; + + return sched_domain_span(sd); +} + +/* + * Return the amount of CPUs in the same NUMA domain of @cpu (or zero if the + * NUMA domain is not defined). + */ +static unsigned int numa_weight(s32 cpu) +{ + struct sched_domain *sd; + struct sched_group *sg; + + sd = rcu_dereference(per_cpu(sd_numa, cpu)); + if (!sd) + return 0; + sg = sd->groups; + if (!sg) + return 0; + + return sg->group_weight; +} + +/* + * Return the cpumask representing the NUMA domain of @cpu (or NULL if the NUMA + * domain is not defined). + */ +static struct cpumask *numa_span(s32 cpu) +{ + struct sched_domain *sd; + struct sched_group *sg; + + sd = rcu_dereference(per_cpu(sd_numa, cpu)); + if (!sd) + return NULL; + sg = sd->groups; + if (!sg) + return NULL; + + return sched_group_span(sg); +} + +/* + * Return true if the LLC domains do not perfectly overlap with the NUMA + * domains, false otherwise. + */ +static bool llc_numa_mismatch(void) +{ + int cpu; + + /* + * We need to scan all online CPUs to verify whether their scheduling + * domains overlap. + * + * While it is rare to encounter architectures with asymmetric NUMA + * topologies, CPU hotplugging or virtualized environments can result + * in asymmetric configurations. + * + * For example: + * + * NUMA 0: + * - LLC 0: cpu0..cpu7 + * - LLC 1: cpu8..cpu15 [offline] + * + * NUMA 1: + * - LLC 0: cpu16..cpu23 + * - LLC 1: cpu24..cpu31 + * + * In this case, if we only check the first online CPU (cpu0), we might + * incorrectly assume that the LLC and NUMA domains are fully + * overlapping, which is incorrect (as NUMA 1 has two distinct LLC + * domains). + */ + for_each_online_cpu(cpu) + if (llc_weight(cpu) != numa_weight(cpu)) + return true; + + return false; +} + +/* + * Initialize topology-aware scheduling. + * + * Detect if the system has multiple LLC or multiple NUMA domains and enable + * cache-aware / NUMA-aware scheduling optimizations in the default CPU idle + * selection policy. + * + * Assumption: the kernel's internal topology representation assumes that each + * CPU belongs to a single LLC domain, and that each LLC domain is entirely + * contained within a single NUMA node. + */ +void scx_idle_update_selcpu_topology(struct sched_ext_ops *ops) +{ + bool enable_llc = false, enable_numa = false; + unsigned int nr_cpus; + s32 cpu = cpumask_first(cpu_online_mask); + + /* + * Enable LLC domain optimization only when there are multiple LLC + * domains among the online CPUs. If all online CPUs are part of a + * single LLC domain, the idle CPU selection logic can choose any + * online CPU without bias. + * + * Note that it is sufficient to check the LLC domain of the first + * online CPU to determine whether a single LLC domain includes all + * CPUs. + */ + rcu_read_lock(); + nr_cpus = llc_weight(cpu); + if (nr_cpus > 0) { + if (nr_cpus < num_online_cpus()) + enable_llc = true; + pr_debug("sched_ext: LLC=%*pb weight=%u\n", + cpumask_pr_args(llc_span(cpu)), llc_weight(cpu)); + } + + /* + * Enable NUMA optimization only when there are multiple NUMA domains + * among the online CPUs and the NUMA domains don't perfectly overlap + * with the LLC domains. + * + * If all CPUs belong to the same NUMA node and the same LLC domain, + * enabling both NUMA and LLC optimizations is unnecessary, as checking + * for an idle CPU in the same domain twice is redundant. + * + * If SCX_OPS_BUILTIN_IDLE_PER_NODE is enabled ignore the NUMA + * optimization, as we would naturally select idle CPUs within + * specific NUMA nodes querying the corresponding per-node cpumask. + */ + if (!(ops->flags & SCX_OPS_BUILTIN_IDLE_PER_NODE)) { + nr_cpus = numa_weight(cpu); + if (nr_cpus > 0) { + if (nr_cpus < num_online_cpus() && llc_numa_mismatch()) + enable_numa = true; + pr_debug("sched_ext: NUMA=%*pb weight=%u\n", + cpumask_pr_args(numa_span(cpu)), nr_cpus); + } + } + rcu_read_unlock(); + + pr_debug("sched_ext: LLC idle selection %s\n", + str_enabled_disabled(enable_llc)); + pr_debug("sched_ext: NUMA idle selection %s\n", + str_enabled_disabled(enable_numa)); + + if (enable_llc) + static_branch_enable_cpuslocked(&scx_selcpu_topo_llc); + else + static_branch_disable_cpuslocked(&scx_selcpu_topo_llc); + if (enable_numa) + static_branch_enable_cpuslocked(&scx_selcpu_topo_numa); + else + static_branch_disable_cpuslocked(&scx_selcpu_topo_numa); +} + +/* + * Return true if @p can run on all possible CPUs, false otherwise. + */ +static inline bool task_affinity_all(const struct task_struct *p) +{ + return p->nr_cpus_allowed >= num_possible_cpus(); +} + +/* + * Built-in CPU idle selection policy: + * + * 1. Prioritize full-idle cores: + * - always prioritize CPUs from fully idle cores (both logical CPUs are + * idle) to avoid interference caused by SMT. + * + * 2. Reuse the same CPU: + * - prefer the last used CPU to take advantage of cached data (L1, L2) and + * branch prediction optimizations. + * + * 3. Prefer @prev_cpu's SMT sibling: + * - if @prev_cpu is busy and no fully idle core is available, try to + * place the task on an idle SMT sibling of @prev_cpu; keeping the + * task on the same core makes migration cheaper, preserves L1 cache + * locality and reduces wakeup latency. + * + * 4. Pick a CPU within the same LLC (Last-Level Cache): + * - if the above conditions aren't met, pick a CPU that shares the same + * LLC, if the LLC domain is a subset of @cpus_allowed, to maintain + * cache locality. + * + * 5. Pick a CPU within the same NUMA node, if enabled: + * - choose a CPU from the same NUMA node, if the node cpumask is a + * subset of @cpus_allowed, to reduce memory access latency. + * + * 6. Pick any idle CPU within the @cpus_allowed domain. + * + * Step 4 and 5 are performed only if the system has, respectively, + * multiple LLCs / multiple NUMA nodes (see scx_selcpu_topo_llc and + * scx_selcpu_topo_numa) and they don't contain the same subset of CPUs. + * + * If %SCX_OPS_BUILTIN_IDLE_PER_NODE is enabled, the search will always + * begin in @prev_cpu's node and proceed to other nodes in order of + * increasing distance. + * + * Return the picked CPU if idle, or a negative value otherwise. + * + * NOTE: tasks that can only run on 1 CPU are excluded by this logic, because + * we never call ops.select_cpu() for them, see select_task_rq(). + */ +s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, + const struct cpumask *cpus_allowed, u64 flags) +{ + const struct cpumask *llc_cpus = NULL, *numa_cpus = NULL; + const struct cpumask *allowed = cpus_allowed ?: p->cpus_ptr; + int node = scx_cpu_node_if_enabled(prev_cpu); + bool is_prev_allowed; + s32 cpu; + + preempt_disable(); + + /* + * Determine the subset of CPUs usable by @p within @cpus_allowed. + */ + if (allowed != p->cpus_ptr) { + struct cpumask *local_cpus = this_cpu_cpumask_var_ptr(local_idle_cpumask); + + if (task_affinity_all(p)) { + allowed = cpus_allowed; + } else if (cpumask_and(local_cpus, cpus_allowed, p->cpus_ptr)) { + allowed = local_cpus; + } else { + cpu = -EBUSY; + goto out_enable; + } + } + + /* + * Check whether @prev_cpu is still within the allowed set. If not, + * we can still try selecting a nearby CPU. + */ + is_prev_allowed = cpumask_test_cpu(prev_cpu, allowed); + + /* + * This is necessary to protect llc_cpus. + */ + rcu_read_lock(); + + /* + * Determine the subset of CPUs that the task can use in its + * current LLC and node. + * + * If the task can run on all CPUs, use the node and LLC cpumasks + * directly. + */ + if (static_branch_maybe(CONFIG_NUMA, &scx_selcpu_topo_numa)) { + struct cpumask *local_cpus = this_cpu_cpumask_var_ptr(local_numa_idle_cpumask); + const struct cpumask *cpus = numa_span(prev_cpu); + + if (allowed == p->cpus_ptr && task_affinity_all(p)) + numa_cpus = cpus; + else if (cpus && cpumask_and(local_cpus, allowed, cpus)) + numa_cpus = local_cpus; + } + + if (static_branch_maybe(CONFIG_SCHED_MC, &scx_selcpu_topo_llc)) { + struct cpumask *local_cpus = this_cpu_cpumask_var_ptr(local_llc_idle_cpumask); + const struct cpumask *cpus = llc_span(prev_cpu); + + if (allowed == p->cpus_ptr && task_affinity_all(p)) + llc_cpus = cpus; + else if (cpus && cpumask_and(local_cpus, allowed, cpus)) + llc_cpus = local_cpus; + } + + /* + * If WAKE_SYNC, try to migrate the wakee to the waker's CPU. + */ + if (wake_flags & SCX_WAKE_SYNC) { + int waker_node; + + /* + * If the waker's CPU is cache affine and prev_cpu is idle, + * then avoid a migration. + */ + cpu = smp_processor_id(); + if (is_prev_allowed && cpus_share_cache(cpu, prev_cpu) && + scx_idle_test_and_clear_cpu(prev_cpu)) { + cpu = prev_cpu; + goto out_unlock; + } + + /* + * If the waker's local DSQ is empty, and the system is under + * utilized, try to wake up @p to the local DSQ of the waker. + * + * Checking only for an empty local DSQ is insufficient as it + * could give the wakee an unfair advantage when the system is + * oversaturated. + * + * Checking only for the presence of idle CPUs is also + * insufficient as the local DSQ of the waker could have tasks + * piled up on it even if there is an idle core elsewhere on + * the system. + */ + waker_node = scx_cpu_node_if_enabled(cpu); + if (!(current->flags & PF_EXITING) && + cpu_rq(cpu)->scx.local_dsq.nr == 0 && + (!(flags & SCX_PICK_IDLE_IN_NODE) || (waker_node == node)) && + !cpumask_empty(idle_cpumask(waker_node)->cpu)) { + if (cpumask_test_cpu(cpu, allowed)) + goto out_unlock; + } + } + + /* + * If CPU has SMT, any wholly idle CPU is likely a better pick than + * partially idle @prev_cpu. + */ + if (sched_smt_active()) { + /* + * Keep using @prev_cpu if it's part of a fully idle core. + */ + if (is_prev_allowed && + cpumask_test_cpu(prev_cpu, idle_cpumask(node)->smt) && + scx_idle_test_and_clear_cpu(prev_cpu)) { + cpu = prev_cpu; + goto out_unlock; + } + + /* + * Search for any fully idle core in the same LLC domain. + */ + if (llc_cpus) { + cpu = pick_idle_cpu_in_node(llc_cpus, node, SCX_PICK_IDLE_CORE); + if (cpu >= 0) + goto out_unlock; + } + + /* + * Search for any fully idle core in the same NUMA node. + */ + if (numa_cpus) { + cpu = pick_idle_cpu_in_node(numa_cpus, node, SCX_PICK_IDLE_CORE); + if (cpu >= 0) + goto out_unlock; + } + + /* + * Search for any full-idle core usable by the task. + * + * If the node-aware idle CPU selection policy is enabled + * (%SCX_OPS_BUILTIN_IDLE_PER_NODE), the search will always + * begin in prev_cpu's node and proceed to other nodes in + * order of increasing distance. + */ + cpu = scx_pick_idle_cpu(allowed, node, flags | SCX_PICK_IDLE_CORE); + if (cpu >= 0) + goto out_unlock; + + /* + * Give up if we're strictly looking for a full-idle SMT + * core. + */ + if (flags & SCX_PICK_IDLE_CORE) { + cpu = -EBUSY; + goto out_unlock; + } + } + + /* + * Use @prev_cpu if it's idle. + */ + if (is_prev_allowed && scx_idle_test_and_clear_cpu(prev_cpu)) { + cpu = prev_cpu; + goto out_unlock; + } + + /* + * Use @prev_cpu's sibling if it's idle. + */ + if (sched_smt_active()) { + for_each_cpu_and(cpu, cpu_smt_mask(prev_cpu), allowed) { + if (cpu == prev_cpu) + continue; + if (scx_idle_test_and_clear_cpu(cpu)) + goto out_unlock; + } + } + + /* + * Search for any idle CPU in the same LLC domain. + */ + if (llc_cpus) { + cpu = pick_idle_cpu_in_node(llc_cpus, node, 0); + if (cpu >= 0) + goto out_unlock; + } + + /* + * Search for any idle CPU in the same NUMA node. + */ + if (numa_cpus) { + cpu = pick_idle_cpu_in_node(numa_cpus, node, 0); + if (cpu >= 0) + goto out_unlock; + } + + /* + * Search for any idle CPU usable by the task. + * + * If the node-aware idle CPU selection policy is enabled + * (%SCX_OPS_BUILTIN_IDLE_PER_NODE), the search will always begin + * in prev_cpu's node and proceed to other nodes in order of + * increasing distance. + */ + cpu = scx_pick_idle_cpu(allowed, node, flags); + +out_unlock: + rcu_read_unlock(); +out_enable: + preempt_enable(); + + return cpu; +} + +/* + * Initialize global and per-node idle cpumasks. + */ +void scx_idle_init_masks(void) +{ + int i; + + /* Allocate global idle cpumasks */ + BUG_ON(!alloc_cpumask_var(&scx_idle_global_masks.cpu, GFP_KERNEL)); + BUG_ON(!alloc_cpumask_var(&scx_idle_global_masks.smt, GFP_KERNEL)); + + /* Allocate per-node idle cpumasks (use nr_node_ids for non-contiguous NUMA nodes) */ + scx_idle_node_masks = kzalloc_objs(*scx_idle_node_masks, nr_node_ids); + BUG_ON(!scx_idle_node_masks); + + for_each_node(i) { + scx_idle_node_masks[i] = kzalloc_node(sizeof(**scx_idle_node_masks), + GFP_KERNEL, i); + BUG_ON(!scx_idle_node_masks[i]); + + BUG_ON(!alloc_cpumask_var_node(&scx_idle_node_masks[i]->cpu, GFP_KERNEL, i)); + BUG_ON(!alloc_cpumask_var_node(&scx_idle_node_masks[i]->smt, GFP_KERNEL, i)); + } + + /* Allocate local per-cpu idle cpumasks */ + for_each_possible_cpu(i) { + BUG_ON(!alloc_cpumask_var_node(&per_cpu(local_idle_cpumask, i), + GFP_KERNEL, cpu_to_node(i))); + BUG_ON(!alloc_cpumask_var_node(&per_cpu(local_llc_idle_cpumask, i), + GFP_KERNEL, cpu_to_node(i))); + BUG_ON(!alloc_cpumask_var_node(&per_cpu(local_numa_idle_cpumask, i), + GFP_KERNEL, cpu_to_node(i))); + } +} + +static void update_builtin_idle(int cpu, bool idle) +{ + int node = scx_cpu_node_if_enabled(cpu); + struct cpumask *idle_cpus = idle_cpumask(node)->cpu; + + assign_cpu(cpu, idle_cpus, idle); + + if (sched_smt_active()) { + const struct cpumask *smt = cpu_smt_mask(cpu); + struct cpumask *idle_smts = idle_cpumask(node)->smt; + + if (idle) { + /* + * idle_smt handling is racy but that's fine as it's + * only for optimization and self-correcting. + */ + if (!cpumask_subset(smt, idle_cpus)) + return; + cpumask_or(idle_smts, idle_smts, smt); + } else { + cpumask_andnot(idle_smts, idle_smts, smt); + } + } +} + +/* + * Update the idle state of a CPU to @idle. + * + * If @do_notify is true, ops.update_idle() is invoked to notify the scx + * scheduler of an actual idle state transition (idle to busy or vice + * versa). If @do_notify is false, only the idle state in the idle masks is + * refreshed without invoking ops.update_idle(). + * + * This distinction is necessary, because an idle CPU can be "reserved" and + * awakened via scx_bpf_pick_idle_cpu() + scx_bpf_kick_cpu(), marking it as + * busy even if no tasks are dispatched. In this case, the CPU may return + * to idle without a true state transition. Refreshing the idle masks + * without invoking ops.update_idle() ensures accurate idle state tracking + * while avoiding unnecessary updates and maintaining balanced state + * transitions. + */ +void __scx_update_idle(struct rq *rq, bool idle, bool do_notify) +{ + struct scx_sched *sch = scx_root; + int cpu = cpu_of(rq); + + lockdep_assert_rq_held(rq); + + /* + * Update the idle masks: + * - for real idle transitions (do_notify == true) + * - for idle-to-idle transitions (indicated by the previous task + * being the idle thread, managed by pick_task_idle()) + * + * Skip updating idle masks if the previous task is not the idle + * thread, since set_next_task_idle() has already handled it when + * transitioning from a task to the idle thread (calling this + * function with do_notify == true). + * + * In this way we can avoid updating the idle masks twice, + * unnecessarily. + */ + if (static_branch_likely(&scx_builtin_idle_enabled)) + if (do_notify || is_idle_task(rq->curr)) + update_builtin_idle(cpu, idle); + + /* + * Trigger ops.update_idle() only when transitioning from a task to + * the idle thread and vice versa. + * + * Idle transitions are indicated by do_notify being set to true, + * managed by put_prev_task_idle()/set_next_task_idle(). + * + * This must come after builtin idle update so that BPF schedulers can + * create interlocking between ops.update_idle() and ops.enqueue() - + * either enqueue() sees the idle bit or update_idle() sees the task + * that enqueue() queued. + */ + if (SCX_HAS_OP(sch, update_idle) && do_notify && + !scx_bypassing(sch, cpu_of(rq))) + SCX_CALL_OP(sch, update_idle, rq, scx_cpu_arg(cpu_of(rq)), idle); +} + +static void reset_idle_masks(struct sched_ext_ops *ops) +{ + int node; + + /* + * Consider all online cpus idle. Should converge to the actual state + * quickly. + */ + if (!(ops->flags & SCX_OPS_BUILTIN_IDLE_PER_NODE)) { + cpumask_copy(idle_cpumask(NUMA_NO_NODE)->cpu, cpu_online_mask); + cpumask_copy(idle_cpumask(NUMA_NO_NODE)->smt, cpu_online_mask); + return; + } + + for_each_node(node) { + const struct cpumask *node_mask = cpumask_of_node(node); + + cpumask_and(idle_cpumask(node)->cpu, cpu_online_mask, node_mask); + cpumask_and(idle_cpumask(node)->smt, cpu_online_mask, node_mask); + } +} + +void scx_idle_enable(struct sched_ext_ops *ops) +{ + if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) + static_branch_enable_cpuslocked(&scx_builtin_idle_enabled); + else + static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); + + if (ops->flags & SCX_OPS_BUILTIN_IDLE_PER_NODE) + static_branch_enable_cpuslocked(&scx_builtin_idle_per_node); + else + static_branch_disable_cpuslocked(&scx_builtin_idle_per_node); + + reset_idle_masks(ops); +} + +void scx_idle_disable(void) +{ + static_branch_disable(&scx_builtin_idle_enabled); + static_branch_disable(&scx_builtin_idle_per_node); +} + +/******************************************************************************** + * Helpers that can be called from the BPF scheduler. + */ + +static int validate_node(struct scx_sched *sch, int node) +{ + if (!static_branch_likely(&scx_builtin_idle_per_node)) { + scx_error(sch, "per-node idle tracking is disabled"); + return -EOPNOTSUPP; + } + + /* Return no entry for NUMA_NO_NODE (not a critical scx error) */ + if (node == NUMA_NO_NODE) + return -ENOENT; + + /* Make sure node is in a valid range */ + if (node < 0 || node >= nr_node_ids) { + scx_error(sch, "invalid node %d", node); + return -EINVAL; + } + + /* Make sure the node is part of the set of possible nodes */ + if (!node_possible(node)) { + scx_error(sch, "unavailable node %d", node); + return -EINVAL; + } + + return node; +} + +__bpf_kfunc_start_defs(); + +static bool check_builtin_idle_enabled(struct scx_sched *sch) +{ + if (static_branch_likely(&scx_builtin_idle_enabled)) + return true; + + scx_error(sch, "built-in idle tracking is disabled"); + return false; +} + +/* + * Determine whether @p is a migration-disabled task in the context of BPF + * code. + * + * We can't simply check whether @p->migration_disabled is set in a + * sched_ext callback, because the BPF prolog (__bpf_prog_enter) may disable + * migration for the current task while running BPF code. + * + * Since the BPF prolog calls migrate_disable() only when CONFIG_PREEMPT_RCU + * is enabled (via rcu_read_lock_dont_migrate()), migration_disabled == 1 for + * the current task is ambiguous only in that case: it could be from the BPF + * prolog rather than a real migrate_disable() call. + * + * Without CONFIG_PREEMPT_RCU, the BPF prolog never calls migrate_disable(), + * so migration_disabled == 1 always means the task is truly + * migration-disabled. + * + * Therefore, when migration_disabled == 1 and CONFIG_PREEMPT_RCU is enabled, + * check whether @p is the current task or not: if it is, then migration was + * not disabled before entering the callback, otherwise migration was disabled. + * + * Returns true if @p is migration-disabled, false otherwise. + */ +static bool is_bpf_migration_disabled(const struct task_struct *p) +{ + if (p->migration_disabled == 1) { + if (IS_ENABLED(CONFIG_PREEMPT_RCU)) + return p != current; + return true; + } + return p->migration_disabled; +} + +static s32 select_cpu_from_kfunc(struct scx_sched *sch, struct task_struct *p, + s32 prev_cpu, u64 wake_flags, + const struct cpumask *allowed, u64 flags) +{ + unsigned long irq_flags; + bool we_locked = false; + s32 cpu; + + if (!scx_cpu_valid(sch, prev_cpu, NULL)) + return -EINVAL; + + if (!check_builtin_idle_enabled(sch)) + return -EBUSY; + + /* + * Accessing p->cpus_ptr / p->nr_cpus_allowed needs either @p's rq + * lock or @p's pi_lock. Three cases: + * + * - inside ops.select_cpu(): try_to_wake_up() holds the wake-up + * task's pi_lock; the wake-up task is recorded in kf_tasks[0] + * by SCX_CALL_OP_TASK_RET(). + * - other rq-locked SCX op: scx_locked_rq() points at the held rq. + * - truly unlocked (UNLOCKED ops, SYSCALL, non-SCX struct_ops): + * nothing held, take pi_lock ourselves. + * + * In the first two cases, BPF schedulers may pass an arbitrary task + * that the held lock doesn't cover. Refuse those. + */ + if (this_rq()->scx.in_select_cpu) { + if (!scx_kf_arg_task_ok(sch, p)) + return -EINVAL; + lockdep_assert_held(&p->pi_lock); + } else if (scx_locked_rq()) { + if (task_rq(p) != scx_locked_rq()) + goto cross_task; + } else { + raw_spin_lock_irqsave(&p->pi_lock, irq_flags); + we_locked = true; + } + + /* + * This may also be called from ops.enqueue(), so we need to handle + * per-CPU tasks as well. For these tasks, we can skip all idle CPU + * selection optimizations and simply check whether the previously + * used CPU is idle and within the allowed cpumask. + */ + if (p->nr_cpus_allowed == 1 || is_bpf_migration_disabled(p)) { + if (cpumask_test_cpu(prev_cpu, allowed ?: p->cpus_ptr) && + scx_idle_test_and_clear_cpu(prev_cpu)) + cpu = prev_cpu; + else + cpu = -EBUSY; + } else { + cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, + allowed ?: p->cpus_ptr, flags); + } + + if (we_locked) + raw_spin_unlock_irqrestore(&p->pi_lock, irq_flags); + + return cpu; + +cross_task: + scx_error(sch, "select_cpu kfunc called cross-task on %s[%d]", + p->comm, p->pid); + return -EINVAL; +} + +/** + * scx_bpf_cpu_node - Return the NUMA node the given @cpu belongs to, or + * trigger an error if @cpu is invalid + * @cpu: target CPU + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + */ +__bpf_kfunc s32 scx_bpf_cpu_node(s32 cpu, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch) || !scx_cpu_valid(sch, cpu, NULL)) + return NUMA_NO_NODE; + return cpu_to_node(cpu); +} + +/** + * scx_bpf_select_cpu_dfl - The default implementation of ops.select_cpu() + * @p: task_struct to select a CPU for + * @prev_cpu: CPU @p was on previously + * @wake_flags: %SCX_WAKE_* flags + * @is_idle: out parameter indicating whether the returned CPU is idle + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Can be called from ops.select_cpu(), ops.enqueue(), or from an unlocked + * context such as a BPF test_run() call, as long as built-in CPU selection + * is enabled: ops.update_idle() is missing or %SCX_OPS_KEEP_BUILTIN_IDLE + * is set. + * + * Returns the picked CPU with *@is_idle indicating whether the picked CPU is + * currently idle and thus a good candidate for direct dispatching. + */ +__bpf_kfunc s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, + u64 wake_flags, bool *is_idle, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + s32 cpu; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return -ENODEV; + + cpu = select_cpu_from_kfunc(sch, p, prev_cpu, wake_flags, NULL, 0); + if (cpu >= 0) { + *is_idle = true; + return cpu; + } + *is_idle = false; + return prev_cpu; +} + +struct scx_bpf_select_cpu_and_args { + /* @p and @cpus_allowed can't be packed together as KF_RCU is not transitive */ + s32 prev_cpu; + u64 wake_flags; + u64 flags; +}; + +/** + * __scx_bpf_select_cpu_and - Arg-wrapped CPU selection with cpumask + * @p: task_struct to select a CPU for + * @cpus_allowed: cpumask of allowed CPUs + * @args: struct containing the rest of the arguments + * @args->prev_cpu: CPU @p was on previously + * @args->wake_flags: %SCX_WAKE_* flags + * @args->flags: %SCX_PICK_IDLE* flags + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Wrapper kfunc that takes arguments via struct to work around BPF's 5 argument + * limit. BPF programs should use scx_bpf_select_cpu_and() which is provided + * as an inline wrapper in common.bpf.h. + * + * Can be called from ops.select_cpu(), ops.enqueue(), or from an unlocked + * context such as a BPF test_run() call, as long as built-in CPU selection + * is enabled: ops.update_idle() is missing or %SCX_OPS_KEEP_BUILTIN_IDLE + * is set. + * + * @p, @args->prev_cpu and @args->wake_flags match ops.select_cpu(). + * + * Returns the selected idle CPU, which will be automatically awakened upon + * returning from ops.select_cpu() and can be used for direct dispatch, or + * a negative value if no idle CPU is available. + */ +__bpf_kfunc s32 +__scx_bpf_select_cpu_and(struct task_struct *p, const struct cpumask *cpus_allowed, + struct scx_bpf_select_cpu_and_args *args, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return -ENODEV; + + return select_cpu_from_kfunc(sch, p, args->prev_cpu, args->wake_flags, + cpus_allowed, args->flags); +} + +/* + * COMPAT: Will be removed in v6.22. + */ +__bpf_kfunc s32 scx_bpf_select_cpu_and(struct task_struct *p, s32 prev_cpu, u64 wake_flags, + const struct cpumask *cpus_allowed, u64 flags) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = rcu_dereference(scx_root); + if (unlikely(!sch)) + return -ENODEV; + +#ifdef CONFIG_EXT_SUB_SCHED + /* + * Disallow if any sub-scheds are attached. There is no way to tell + * which scheduler called us, just error out @p's scheduler. + */ + if (unlikely(!list_empty(&sch->children))) { + scx_error(scx_task_sched(p), "__scx_bpf_select_cpu_and() must be used"); + return -EINVAL; + } +#endif + + return select_cpu_from_kfunc(sch, p, prev_cpu, wake_flags, + cpus_allowed, flags); +} + +/** + * scx_bpf_get_idle_cpumask_node - Get a referenced kptr to the + * idle-tracking per-CPU cpumask of a target NUMA node. + * @node: target NUMA node + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Returns an empty cpumask if idle tracking is not enabled, if @node is + * not valid, or running on a UP kernel. In this case the actual error will + * be reported to the BPF scheduler via scx_error(). + */ +__bpf_kfunc const struct cpumask * +scx_bpf_get_idle_cpumask_node(s32 node, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return cpu_none_mask; + + node = validate_node(sch, node); + if (node < 0) + return cpu_none_mask; + + return idle_cpumask(node)->cpu; +} + +/** + * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking + * per-CPU cpumask. + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Returns an empty mask if idle tracking is not enabled, or running on a + * UP kernel. + */ +__bpf_kfunc const struct cpumask *scx_bpf_get_idle_cpumask(const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return cpu_none_mask; + + if (static_branch_unlikely(&scx_builtin_idle_per_node)) { + scx_error(sch, "SCX_OPS_BUILTIN_IDLE_PER_NODE enabled"); + return cpu_none_mask; + } + + if (!check_builtin_idle_enabled(sch)) + return cpu_none_mask; + + return idle_cpumask(NUMA_NO_NODE)->cpu; +} + +/** + * scx_bpf_get_idle_smtmask_node - Get a referenced kptr to the + * idle-tracking, per-physical-core cpumask of a target NUMA node. Can be + * used to determine if an entire physical core is free. + * @node: target NUMA node + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Returns an empty cpumask if idle tracking is not enabled, if @node is + * not valid, or running on a UP kernel. In this case the actual error will + * be reported to the BPF scheduler via scx_error(). + */ +__bpf_kfunc const struct cpumask * +scx_bpf_get_idle_smtmask_node(s32 node, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return cpu_none_mask; + + node = validate_node(sch, node); + if (node < 0) + return cpu_none_mask; + + if (sched_smt_active()) + return idle_cpumask(node)->smt; + else + return idle_cpumask(node)->cpu; +} + +/** + * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking, + * per-physical-core cpumask. Can be used to determine if an entire physical + * core is free. + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Returns an empty mask if idle tracking is not enabled, or running on a + * UP kernel. + */ +__bpf_kfunc const struct cpumask *scx_bpf_get_idle_smtmask(const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return cpu_none_mask; + + if (static_branch_unlikely(&scx_builtin_idle_per_node)) { + scx_error(sch, "SCX_OPS_BUILTIN_IDLE_PER_NODE enabled"); + return cpu_none_mask; + } + + if (!check_builtin_idle_enabled(sch)) + return cpu_none_mask; + + if (sched_smt_active()) + return idle_cpumask(NUMA_NO_NODE)->smt; + else + return idle_cpumask(NUMA_NO_NODE)->cpu; +} + +/** + * scx_bpf_put_idle_cpumask - Release a previously acquired referenced kptr to + * either the percpu, or SMT idle-tracking cpumask. + * @idle_mask: &cpumask to use + */ +__bpf_kfunc void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask) +{ + /* + * Empty function body because we aren't actually acquiring or releasing + * a reference to a global idle cpumask, which is read-only in the + * caller and is never released. The acquire / release semantics here + * are just used to make the cpumask a trusted pointer in the caller. + */ +} + +/** + * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state + * @cpu: cpu to test and clear idle for + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Returns %true if @cpu was idle and its idle state was successfully cleared. + * %false otherwise. + * + * Unavailable if ops.update_idle() is implemented and + * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. + */ +__bpf_kfunc bool scx_bpf_test_and_clear_cpu_idle(s32 cpu, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return false; + + if (!check_builtin_idle_enabled(sch)) + return false; + + if (!scx_cpu_valid(sch, cpu, NULL)) + return false; + + return scx_idle_test_and_clear_cpu(cpu); +} + +/** + * scx_bpf_pick_idle_cpu_node - Pick and claim an idle cpu from @node + * @cpus_allowed: Allowed cpumask + * @node: target NUMA node + * @flags: %SCX_PICK_IDLE_* flags + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Pick and claim an idle cpu in @cpus_allowed from the NUMA node @node. + * + * Returns the picked idle cpu number on success, or -%EBUSY if no matching + * cpu was found. + * + * The search starts from @node and proceeds to other online NUMA nodes in + * order of increasing distance (unless SCX_PICK_IDLE_IN_NODE is specified, + * in which case the search is limited to the target @node). + * + * Always returns an error if ops.update_idle() is implemented and + * %SCX_OPS_KEEP_BUILTIN_IDLE is not set, or if + * %SCX_OPS_BUILTIN_IDLE_PER_NODE is not set. + */ +__bpf_kfunc s32 scx_bpf_pick_idle_cpu_node(const struct cpumask *cpus_allowed, + s32 node, u64 flags, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return -ENODEV; + + node = validate_node(sch, node); + if (node < 0) + return node; + + return scx_pick_idle_cpu(cpus_allowed, node, flags); +} + +/** + * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu + * @cpus_allowed: Allowed cpumask + * @flags: %SCX_PICK_IDLE_CPU_* flags + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Pick and claim an idle cpu in @cpus_allowed. Returns the picked idle cpu + * number on success. -%EBUSY if no matching cpu was found. + * + * Idle CPU tracking may race against CPU scheduling state transitions. For + * example, this function may return -%EBUSY as CPUs are transitioning into the + * idle state. If the caller then assumes that there will be dispatch events on + * the CPUs as they were all busy, the scheduler may end up stalling with CPUs + * idling while there are pending tasks. Use scx_bpf_pick_any_cpu() and + * scx_bpf_kick_cpu() to guarantee that there will be at least one dispatch + * event in the near future. + * + * Unavailable if ops.update_idle() is implemented and + * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. + * + * Always returns an error if %SCX_OPS_BUILTIN_IDLE_PER_NODE is set, use + * scx_bpf_pick_idle_cpu_node() instead. + */ +__bpf_kfunc s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed, + u64 flags, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return -ENODEV; + + if (static_branch_maybe(CONFIG_NUMA, &scx_builtin_idle_per_node)) { + scx_error(sch, "per-node idle tracking is enabled"); + return -EBUSY; + } + + if (!check_builtin_idle_enabled(sch)) + return -EBUSY; + + return scx_pick_idle_cpu(cpus_allowed, NUMA_NO_NODE, flags); +} + +/** + * scx_bpf_pick_any_cpu_node - Pick and claim an idle cpu if available + * or pick any CPU from @node + * @cpus_allowed: Allowed cpumask + * @node: target NUMA node + * @flags: %SCX_PICK_IDLE_CPU_* flags + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Pick and claim an idle cpu in @cpus_allowed. If none is available, pick any + * CPU in @cpus_allowed. Guaranteed to succeed and returns the picked idle cpu + * number if @cpus_allowed is not empty. -%EBUSY is returned if @cpus_allowed is + * empty. + * + * The search starts from @node and proceeds to other online NUMA nodes in + * order of increasing distance (unless %SCX_PICK_IDLE_IN_NODE is specified, + * in which case the search is limited to the target @node, regardless of + * the CPU idle state). + * + * If ops.update_idle() is implemented and %SCX_OPS_KEEP_BUILTIN_IDLE is not + * set, this function can't tell which CPUs are idle and will always pick any + * CPU. + */ +__bpf_kfunc s32 scx_bpf_pick_any_cpu_node(const struct cpumask *cpus_allowed, + s32 node, u64 flags, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + s32 cpu; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return -ENODEV; + + node = validate_node(sch, node); + if (node < 0) + return node; + + cpu = scx_pick_idle_cpu(cpus_allowed, node, flags); + if (cpu >= 0) + return cpu; + + if (flags & SCX_PICK_IDLE_IN_NODE) + cpu = cpumask_any_and_distribute(cpumask_of_node(node), cpus_allowed); + else + cpu = cpumask_any_distribute(cpus_allowed); + if (cpu < nr_cpu_ids) + return cpu; + else + return -EBUSY; +} + +/** + * scx_bpf_pick_any_cpu - Pick and claim an idle cpu if available or pick any CPU + * @cpus_allowed: Allowed cpumask + * @flags: %SCX_PICK_IDLE_CPU_* flags + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Pick and claim an idle cpu in @cpus_allowed. If none is available, pick any + * CPU in @cpus_allowed. Guaranteed to succeed and returns the picked idle cpu + * number if @cpus_allowed is not empty. -%EBUSY is returned if @cpus_allowed is + * empty. + * + * If ops.update_idle() is implemented and %SCX_OPS_KEEP_BUILTIN_IDLE is not + * set, this function can't tell which CPUs are idle and will always pick any + * CPU. + * + * Always returns an error if %SCX_OPS_BUILTIN_IDLE_PER_NODE is set, use + * scx_bpf_pick_any_cpu_node() instead. + */ +__bpf_kfunc s32 scx_bpf_pick_any_cpu(const struct cpumask *cpus_allowed, + u64 flags, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + s32 cpu; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return -ENODEV; + + if (static_branch_maybe(CONFIG_NUMA, &scx_builtin_idle_per_node)) { + scx_error(sch, "per-node idle tracking is enabled"); + return -EBUSY; + } + + if (static_branch_likely(&scx_builtin_idle_enabled)) { + cpu = scx_pick_idle_cpu(cpus_allowed, NUMA_NO_NODE, flags); + if (cpu >= 0) + return cpu; + } + + cpu = cpumask_any_distribute(cpus_allowed); + if (cpu < nr_cpu_ids) + return cpu; + else + return -EBUSY; +} + +__bpf_kfunc_end_defs(); + +BTF_KFUNCS_START(scx_kfunc_ids_idle) +BTF_ID_FLAGS(func, scx_bpf_cpu_node, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_IMPLICIT_ARGS | KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_IMPLICIT_ARGS | KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) +BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu_node, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu_node, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu, KF_IMPLICIT_ARGS | KF_RCU) +BTF_KFUNCS_END(scx_kfunc_ids_idle) + +static const struct btf_kfunc_id_set scx_kfunc_set_idle = { + .owner = THIS_MODULE, + .set = &scx_kfunc_ids_idle, + .filter = scx_kfunc_context_filter, +}; + +/* + * The select_cpu kfuncs internally call task_rq_lock() when invoked from an + * rq-unlocked context, and thus cannot be safely called from arbitrary tracing + * contexts where @p's pi_lock state is unknown. Keep them out of + * BPF_PROG_TYPE_TRACING by registering them in their own set which is exposed + * only to STRUCT_OPS and SYSCALL programs. + * + * These kfuncs are also members of scx_kfunc_ids_unlocked (see ext.c) because + * they're callable from unlocked contexts in addition to ops.select_cpu() and + * ops.enqueue(). + */ +BTF_KFUNCS_START(scx_kfunc_ids_select_cpu) +BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS | KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_IMPLICIT_ARGS | KF_RCU) +BTF_KFUNCS_END(scx_kfunc_ids_select_cpu) + +static const struct btf_kfunc_id_set scx_kfunc_set_select_cpu = { + .owner = THIS_MODULE, + .set = &scx_kfunc_ids_select_cpu, + .filter = scx_kfunc_context_filter, +}; + +int scx_idle_init(void) +{ + return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_idle) ?: + register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &scx_kfunc_set_idle) ?: + register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_idle) ?: + register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_select_cpu) ?: + register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_select_cpu); +} diff --git a/kernel/sched/ext/idle.h b/kernel/sched/ext/idle.h new file mode 100644 index 000000000000..8d169d3bbdf9 --- /dev/null +++ b/kernel/sched/ext/idle.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2022 Tejun Heo + * Copyright (c) 2022 David Vernet + * Copyright (c) 2024 Andrea Righi + */ +#ifndef _KERNEL_SCHED_EXT_IDLE_H +#define _KERNEL_SCHED_EXT_IDLE_H + +struct sched_ext_ops; + +extern struct btf_id_set8 scx_kfunc_ids_idle; +extern struct btf_id_set8 scx_kfunc_ids_select_cpu; + +void scx_idle_update_selcpu_topology(struct sched_ext_ops *ops); +void scx_idle_init_masks(void); + +s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, + const struct cpumask *cpus_allowed, u64 flags); +void scx_idle_enable(struct sched_ext_ops *ops); +void scx_idle_disable(void); +int scx_idle_init(void); + +#endif /* _KERNEL_SCHED_EXT_IDLE_H */ diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h new file mode 100644 index 000000000000..b04701190b23 --- /dev/null +++ b/kernel/sched/ext/internal.h @@ -0,0 +1,1653 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Copyright (c) 2025 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2025 Tejun Heo + */ +#define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) +#define SCX_MOFF_IDX(moff) ((moff) / sizeof(void (*)(void))) + +enum scx_exit_kind { + SCX_EXIT_NONE, + SCX_EXIT_DONE, + + SCX_EXIT_UNREG = 64, /* user-space initiated unregistration */ + SCX_EXIT_UNREG_BPF, /* BPF-initiated unregistration */ + SCX_EXIT_UNREG_KERN, /* kernel-initiated unregistration */ + SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ + SCX_EXIT_PARENT, /* parent exiting */ + + SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ + SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ + SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ +}; + +/* + * An exit code can be specified when exiting with scx_bpf_exit() or scx_exit(), + * corresponding to exit_kind UNREG_BPF and UNREG_KERN respectively. The codes + * are 64bit of the format: + * + * Bits: [63 .. 48 47 .. 32 31 .. 0] + * [ SYS ACT ] [ SYS RSN ] [ USR ] + * + * SYS ACT: System-defined exit actions + * SYS RSN: System-defined exit reasons + * USR : User-defined exit codes and reasons + * + * Using the above, users may communicate intention and context by ORing system + * actions and/or system reasons with a user-defined exit code. + */ +enum scx_exit_code { + /* Reasons */ + SCX_ECODE_RSN_HOTPLUG = 1LLU << 32, + SCX_ECODE_RSN_CGROUP_OFFLINE = 2LLU << 32, + + /* Actions */ + SCX_ECODE_ACT_RESTART = 1LLU << 48, +}; + +enum scx_exit_flags { + /* + * ops.exit() may be called even if the loading failed before ops.init() + * finishes successfully. This is because ops.exit() allows rich exit + * info communication. The following flag indicates whether ops.init() + * finished successfully. + */ + SCX_EFLAG_INITIALIZED = 1LLU << 0, +}; + +/* + * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is + * being disabled. + */ +struct scx_exit_info { + /* %SCX_EXIT_* - broad category of the exit reason */ + enum scx_exit_kind kind; + + /* + * CPU that initiated the exit, valid once @kind has been set. + * Negative if the exit path didn't identify a CPU. + */ + s32 exit_cpu; + + /* exit code if gracefully exiting */ + s64 exit_code; + + /* %SCX_EFLAG_* */ + u64 flags; + + /* textual representation of the above */ + const char *reason; + + /* backtrace if exiting due to an error */ + unsigned long *bt; + u32 bt_len; + + /* informational message */ + char *msg; + + /* debug dump */ + char *dump; +}; + +/* sched_ext_ops.flags */ +enum scx_ops_flags { + /* + * Keep built-in idle tracking even if ops.update_idle() is implemented. + */ + SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, + + /* + * By default, if there are no other task to run on the CPU, ext core + * keeps running the current task even after its slice expires. If this + * flag is specified, such tasks are passed to ops.enqueue() with + * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. + */ + SCX_OPS_ENQ_LAST = 1LLU << 1, + + /* + * An exiting task may schedule after PF_EXITING is set. In such cases, + * bpf_task_from_pid() may not be able to find the task and if the BPF + * scheduler depends on pid lookup for dispatching, the task will be + * lost leading to various issues including RCU grace period stalls. + * + * To mask this problem, by default, unhashed tasks are automatically + * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't + * depend on pid lookups and wants to handle these tasks directly, the + * following flag can be used. With %SCX_OPS_TID_TO_TASK, + * scx_bpf_tid_to_task() can find exiting tasks reliably. + */ + SCX_OPS_ENQ_EXITING = 1LLU << 2, + + /* + * If set, only tasks with policy set to SCHED_EXT are attached to + * sched_ext. If clear, SCHED_NORMAL tasks are also included. + */ + SCX_OPS_SWITCH_PARTIAL = 1LLU << 3, + + /* + * A migration disabled task can only execute on its current CPU. By + * default, such tasks are automatically put on the CPU's local DSQ with + * the default slice on enqueue. If this ops flag is set, they also go + * through ops.enqueue(). + * + * A migration disabled task never invokes ops.select_cpu() as it can + * only select the current CPU. Also, p->cpus_ptr will only contain its + * current CPU while p->nr_cpus_allowed keeps tracking p->user_cpus_ptr + * and thus may disagree with cpumask_weight(p->cpus_ptr). + */ + SCX_OPS_ENQ_MIGRATION_DISABLED = 1LLU << 4, + + /* + * Queued wakeup (ttwu_queue) is a wakeup optimization that invokes + * ops.enqueue() on the ops.select_cpu() selected or the wakee's + * previous CPU via IPI (inter-processor interrupt) to reduce cacheline + * transfers. When this optimization is enabled, ops.select_cpu() is + * skipped in some cases (when racing against the wakee switching out). + * As the BPF scheduler may depend on ops.select_cpu() being invoked + * during wakeups, queued wakeup is disabled by default. + * + * If this ops flag is set, queued wakeup optimization is enabled and + * the BPF scheduler must be able to handle ops.enqueue() invoked on the + * wakee's CPU without preceding ops.select_cpu() even for tasks which + * may be executed on multiple CPUs. + */ + SCX_OPS_ALLOW_QUEUED_WAKEUP = 1LLU << 5, + + /* + * If set, enable per-node idle cpumasks. If clear, use a single global + * flat idle cpumask. + */ + SCX_OPS_BUILTIN_IDLE_PER_NODE = 1LLU << 6, + + /* + * If set, %SCX_ENQ_IMMED is assumed to be set on all local DSQ + * enqueues. + */ + SCX_OPS_ALWAYS_ENQ_IMMED = 1LLU << 7, + + /* + * Maintain a mapping from p->scx.tid to task_struct so the BPF + * scheduler can recover task pointers from stored tids via + * scx_bpf_tid_to_task(). + * + * Only the root scheduler turns this on. A sub-sched may set the flag + * to declare a dependency on the lookup; if the root scheduler hasn't + * enabled it, attaching the sub-sched is rejected. + */ + SCX_OPS_TID_TO_TASK = 1LLU << 8, + + SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | + SCX_OPS_ENQ_LAST | + SCX_OPS_ENQ_EXITING | + SCX_OPS_ENQ_MIGRATION_DISABLED | + SCX_OPS_ALLOW_QUEUED_WAKEUP | + SCX_OPS_SWITCH_PARTIAL | + SCX_OPS_BUILTIN_IDLE_PER_NODE | + SCX_OPS_ALWAYS_ENQ_IMMED | + SCX_OPS_TID_TO_TASK, + + /* high 8 bits are internal, don't include in SCX_OPS_ALL_FLAGS */ + __SCX_OPS_INTERNAL_MASK = 0xffLLU << 56, + + SCX_OPS_HAS_CPU_PREEMPT = 1LLU << 56, +}; + +/* argument container for ops.init_task() */ +struct scx_init_task_args { + /* + * Set if ops.init_task() is being invoked on the fork path, as opposed + * to the scheduler transition path. + */ + bool fork; +#ifdef CONFIG_EXT_GROUP_SCHED + /* the cgroup the task is joining */ + struct cgroup *cgroup; +#endif +}; + +/* argument container for ops.exit_task() */ +struct scx_exit_task_args { + /* Whether the task exited before running on sched_ext. */ + bool cancelled; +}; + +/* argument container for ops.cgroup_init() */ +struct scx_cgroup_init_args { + /* the weight of the cgroup [1..10000] */ + u32 weight; + + /* bandwidth control parameters from cpu.max and cpu.max.burst */ + u64 bw_period_us; + u64 bw_quota_us; + u64 bw_burst_us; +}; + +enum scx_cpu_preempt_reason { + /* next task is being scheduled by &sched_class_rt */ + SCX_CPU_PREEMPT_RT, + /* next task is being scheduled by &sched_class_dl */ + SCX_CPU_PREEMPT_DL, + /* next task is being scheduled by &sched_class_stop */ + SCX_CPU_PREEMPT_STOP, + /* unknown reason for SCX being preempted */ + SCX_CPU_PREEMPT_UNKNOWN, +}; + +/* + * Argument container for ops.cpu_acquire(). Currently empty, but may be + * expanded in the future. + */ +struct scx_cpu_acquire_args {}; + +/* argument container for ops.cpu_release() */ +struct scx_cpu_release_args { + /* the reason the CPU was preempted */ + enum scx_cpu_preempt_reason reason; + + /* the task that's going to be scheduled on the CPU */ + struct task_struct *task; +}; + +/* informational context provided to dump operations */ +struct scx_dump_ctx { + enum scx_exit_kind kind; + s64 exit_code; + const char *reason; + u64 at_ns; + u64 at_jiffies; +}; + +/* argument container for ops.sub_attach() */ +struct scx_sub_attach_args { + struct sched_ext_ops *ops; + char *cgroup_path; +}; + +/* argument container for ops.sub_detach() */ +struct scx_sub_detach_args { + struct sched_ext_ops *ops; + char *cgroup_path; +}; + +/** + * struct sched_ext_ops - Operation table for BPF scheduler implementation + * + * A BPF scheduler can implement an arbitrary scheduling policy by + * implementing and loading operations in this table. Note that a userland + * scheduling policy can also be implemented using the BPF scheduler + * as a shim layer. + */ +struct sched_ext_ops { + /** + * @select_cpu: Pick the target CPU for a task which is being woken up + * @p: task being woken up + * @prev_cpu: the cpu @p was on before sleeping + * @wake_flags: SCX_WAKE_* + * + * Decision made here isn't final. @p may be moved to any CPU while it + * is getting dispatched for execution later. However, as @p is not on + * the rq at this point, getting the eventual execution CPU right here + * saves a small bit of overhead down the line. + * + * If an idle CPU is returned, the CPU is kicked and will try to + * dispatch. While an explicit custom mechanism can be added, + * select_cpu() serves as the default way to wake up idle CPUs. + * + * @p may be inserted into a DSQ directly by calling + * scx_bpf_dsq_insert(). If so, the ops.enqueue() will be skipped. + * Directly inserting into %SCX_DSQ_LOCAL will put @p in the local DSQ + * of the CPU returned by this operation. + * + * Note that select_cpu() is never called for tasks that can only run + * on a single CPU or tasks with migration disabled, as they don't have + * the option to select a different CPU. See select_task_rq() for + * details. + */ + s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); + + /** + * @enqueue: Enqueue a task on the BPF scheduler + * @p: task being enqueued + * @enq_flags: %SCX_ENQ_* + * + * @p is ready to run. Insert directly into a DSQ by calling + * scx_bpf_dsq_insert() or enqueue on the BPF scheduler. If not directly + * inserted, the bpf scheduler owns @p and if it fails to dispatch @p, + * the task will stall. + * + * If @p was inserted into a DSQ from ops.select_cpu(), this callback is + * skipped. + */ + void (*enqueue)(struct task_struct *p, u64 enq_flags); + + /** + * @dequeue: Remove a task from the BPF scheduler + * @p: task being dequeued + * @deq_flags: %SCX_DEQ_* + * + * Remove @p from the BPF scheduler. This is usually called to isolate + * the task while updating its scheduling properties (e.g. priority). + * + * The ext core keeps track of whether the BPF side owns a given task or + * not and can gracefully ignore spurious dispatches from BPF side, + * which makes it safe to not implement this method. However, depending + * on the scheduling logic, this can lead to confusing behaviors - e.g. + * scheduling position not being updated across a priority change. + */ + void (*dequeue)(struct task_struct *p, u64 deq_flags); + + /** + * @dispatch: Dispatch tasks from the BPF scheduler and/or user DSQs + * @cpu: CPU to dispatch tasks for + * @prev: previous task being switched out + * + * Called when a CPU's local dsq is empty. The operation should dispatch + * one or more tasks from the BPF scheduler into the DSQs using + * scx_bpf_dsq_insert() and/or move from user DSQs into the local DSQ + * using scx_bpf_dsq_move_to_local(). + * + * The maximum number of times scx_bpf_dsq_insert() can be called + * without an intervening scx_bpf_dsq_move_to_local() is specified by + * ops.dispatch_max_batch. See the comments on top of the two functions + * for more details. + * + * When not %NULL, @prev is an SCX task with its slice depleted. If + * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in + * @prev->scx.flags, it is not enqueued yet and will be enqueued after + * ops.dispatch() returns. To keep executing @prev, return without + * dispatching or moving any tasks. Also see %SCX_OPS_ENQ_LAST. + */ + void (*dispatch)(s32 cpu, struct task_struct *prev); + + /** + * @tick: Periodic tick + * @p: task running currently + * + * This operation is called every 1/HZ seconds on CPUs which are + * executing an SCX task. Setting @p->scx.slice to 0 will trigger an + * immediate dispatch cycle on the CPU. + */ + void (*tick)(struct task_struct *p); + + /** + * @runnable: A task is becoming runnable on its associated CPU + * @p: task becoming runnable + * @enq_flags: %SCX_ENQ_* + * + * This and the following three functions can be used to track a task's + * execution state transitions. A task becomes ->runnable() on a CPU, + * and then goes through one or more ->running() and ->stopping() pairs + * as it runs on the CPU, and eventually becomes ->quiescent() when it's + * done running on the CPU. + * + * @p is becoming runnable on the CPU because it's + * + * - waking up (%SCX_ENQ_WAKEUP) + * - being moved from another CPU + * - being restored after temporarily taken off the queue for an + * attribute change. + * + * This and ->enqueue() are related but not coupled. This operation + * notifies @p's state transition and may not be followed by ->enqueue() + * e.g. when @p is being dispatched to a remote CPU, or when @p is + * being enqueued on a CPU experiencing a hotplug event. Likewise, a + * task may be ->enqueue()'d without being preceded by this operation + * e.g. after exhausting its slice. + */ + void (*runnable)(struct task_struct *p, u64 enq_flags); + + /** + * @running: A task is starting to run on its associated CPU + * @p: task starting to run + * + * Note that this callback may be called from a CPU other than the + * one the task is going to run on. This can happen when a task + * property is changed (i.e., affinity), since scx_next_task_scx(), + * which triggers this callback, may run on a CPU different from + * the task's assigned CPU. + * + * Therefore, always use scx_bpf_task_cpu(@p) to determine the + * target CPU the task is going to use. + * + * See ->runnable() for explanation on the task state notifiers. + */ + void (*running)(struct task_struct *p); + + /** + * @stopping: A task is stopping execution + * @p: task stopping to run + * @runnable: is task @p still runnable? + * + * Note that this callback may be called from a CPU other than the + * one the task was running on. This can happen when a task + * property is changed (i.e., affinity), since dequeue_task_scx(), + * which triggers this callback, may run on a CPU different from + * the task's assigned CPU. + * + * Therefore, always use scx_bpf_task_cpu(@p) to retrieve the CPU + * the task was running on. + * + * See ->runnable() for explanation on the task state notifiers. If + * !@runnable, ->quiescent() will be invoked after this operation + * returns. + */ + void (*stopping)(struct task_struct *p, bool runnable); + + /** + * @quiescent: A task is becoming not runnable on its associated CPU + * @p: task becoming not runnable + * @deq_flags: %SCX_DEQ_* + * + * See ->runnable() for explanation on the task state notifiers. + * + * @p is becoming quiescent on the CPU because it's + * + * - sleeping (%SCX_DEQ_SLEEP) + * - being moved to another CPU + * - being temporarily taken off the queue for an attribute change + * (%SCX_DEQ_SAVE) + * + * This and ->dequeue() are related but not coupled. This operation + * notifies @p's state transition and may not be preceded by ->dequeue() + * e.g. when @p is being dispatched to a remote CPU. + */ + void (*quiescent)(struct task_struct *p, u64 deq_flags); + + /** + * @yield: Yield CPU + * @from: yielding task + * @to: optional yield target task + * + * If @to is NULL, @from is yielding the CPU to other runnable tasks. + * The BPF scheduler should ensure that other available tasks are + * dispatched before the yielding task. Return value is ignored in this + * case. + * + * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf + * scheduler can implement the request, return %true; otherwise, %false. + */ + bool (*yield)(struct task_struct *from, struct task_struct *to); + + /** + * @core_sched_before: Task ordering for core-sched + * @a: task A + * @b: task B + * + * Used by core-sched to determine the ordering between two tasks. See + * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on + * core-sched. + * + * Both @a and @b are runnable and may or may not currently be queued on + * the BPF scheduler. Should return %true if @a should run before @b. + * %false if there's no required ordering or @b should run before @a. + * + * If not specified, the default is ordering them according to when they + * became runnable. + */ + bool (*core_sched_before)(struct task_struct *a, struct task_struct *b); + + /** + * @set_weight: Set task weight + * @p: task to set weight for + * @weight: new weight [1..10000] + * + * Update @p's weight to @weight. + */ + void (*set_weight)(struct task_struct *p, u32 weight); + + /** + * @set_cpumask: Set CPU affinity + * @p: task to set CPU affinity for + * @cpumask: cpumask of cpus that @p can run on + * + * Update @p's CPU affinity to @cpumask. + */ + void (*set_cpumask)(struct task_struct *p, + const struct cpumask *cpumask); + + /** + * @update_idle: Update the idle state of a CPU + * @cpu: CPU to update the idle state for + * @idle: whether entering or exiting the idle state + * + * This operation is called when @rq's CPU goes or leaves the idle + * state. By default, implementing this operation disables the built-in + * idle CPU tracking and the following helpers become unavailable: + * + * - scx_bpf_select_cpu_dfl() + * - scx_bpf_select_cpu_and() + * - scx_bpf_test_and_clear_cpu_idle() + * - scx_bpf_pick_idle_cpu() + * + * The user also must implement ops.select_cpu() as the default + * implementation relies on scx_bpf_select_cpu_dfl(). + * + * Specify the %SCX_OPS_KEEP_BUILTIN_IDLE flag to keep the built-in idle + * tracking. + */ + void (*update_idle)(s32 cpu, bool idle); + + /** + * @init_task: Initialize a task to run in a BPF scheduler + * @p: task to initialize for BPF scheduling + * @args: init arguments, see the struct definition + * + * Either we're loading a BPF scheduler or a new task is being forked. + * Initialize @p for BPF scheduling. This operation may block and can + * be used for allocations, and is called exactly once for a task. + * + * Return 0 for success, -errno for failure. An error return while + * loading will abort loading of the BPF scheduler. During a fork, it + * will abort that specific fork. + */ + s32 (*init_task)(struct task_struct *p, struct scx_init_task_args *args); + + /** + * @exit_task: Exit a previously-running task from the system + * @p: task to exit + * @args: exit arguments, see the struct definition + * + * @p is exiting or the BPF scheduler is being unloaded. Perform any + * necessary cleanup for @p. + */ + void (*exit_task)(struct task_struct *p, struct scx_exit_task_args *args); + + /** + * @enable: Enable BPF scheduling for a task + * @p: task to enable BPF scheduling for + * + * Enable @p for BPF scheduling. enable() is called on @p any time it + * enters SCX, and is always paired with a matching disable(). + */ + void (*enable)(struct task_struct *p); + + /** + * @disable: Disable BPF scheduling for a task + * @p: task to disable BPF scheduling for + * + * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. + * Disable BPF scheduling for @p. A disable() call is always matched + * with a prior enable() call. + */ + void (*disable)(struct task_struct *p); + + /** + * @dump: Dump BPF scheduler state on error + * @ctx: debug dump context + * + * Use scx_bpf_dump() to generate BPF scheduler specific debug dump. + */ + void (*dump)(struct scx_dump_ctx *ctx); + + /** + * @dump_cpu: Dump BPF scheduler state for a CPU on error + * @ctx: debug dump context + * @cpu: CPU to generate debug dump for + * @idle: @cpu is currently idle without any runnable tasks + * + * Use scx_bpf_dump() to generate BPF scheduler specific debug dump for + * @cpu. If @idle is %true and this operation doesn't produce any + * output, @cpu is skipped for dump. + */ + void (*dump_cpu)(struct scx_dump_ctx *ctx, s32 cpu, bool idle); + + /** + * @dump_task: Dump BPF scheduler state for a runnable task on error + * @ctx: debug dump context + * @p: runnable task to generate debug dump for + * + * Use scx_bpf_dump() to generate BPF scheduler specific debug dump for + * @p. + */ + void (*dump_task)(struct scx_dump_ctx *ctx, struct task_struct *p); + +#ifdef CONFIG_EXT_GROUP_SCHED + /** + * @cgroup_init: Initialize a cgroup + * @cgrp: cgroup being initialized + * @args: init arguments, see the struct definition + * + * Either the BPF scheduler is being loaded or @cgrp created, initialize + * @cgrp for sched_ext. This operation may block. + * + * Return 0 for success, -errno for failure. An error return while + * loading will abort loading of the BPF scheduler. During cgroup + * creation, it will abort the specific cgroup creation. + */ + s32 (*cgroup_init)(struct cgroup *cgrp, + struct scx_cgroup_init_args *args); + + /** + * @cgroup_exit: Exit a cgroup + * @cgrp: cgroup being exited + * + * Either the BPF scheduler is being unloaded or @cgrp destroyed, exit + * @cgrp for sched_ext. This operation my block. + */ + void (*cgroup_exit)(struct cgroup *cgrp); + + /** + * @cgroup_prep_move: Prepare a task to be moved to a different cgroup + * @p: task being moved + * @from: cgroup @p is being moved from + * @to: cgroup @p is being moved to + * + * Prepare @p for move from cgroup @from to @to. This operation may + * block and can be used for allocations. + * + * Return 0 for success, -errno for failure. An error return aborts the + * migration. + */ + s32 (*cgroup_prep_move)(struct task_struct *p, + struct cgroup *from, struct cgroup *to); + + /** + * @cgroup_move: Commit cgroup move + * @p: task being moved + * @from: cgroup @p is being moved from + * @to: cgroup @p is being moved to + * + * Commit the move. @p is dequeued during this operation. + */ + void (*cgroup_move)(struct task_struct *p, + struct cgroup *from, struct cgroup *to); + + /** + * @cgroup_cancel_move: Cancel cgroup move + * @p: task whose cgroup move is being canceled + * @from: cgroup @p was being moved from + * @to: cgroup @p was being moved to + * + * @p was cgroup_prep_move()'d but failed before reaching cgroup_move(). + * Undo the preparation. + */ + void (*cgroup_cancel_move)(struct task_struct *p, + struct cgroup *from, struct cgroup *to); + + /** + * @cgroup_set_weight: A cgroup's weight is being changed + * @cgrp: cgroup whose weight is being updated + * @weight: new weight [1..10000] + * + * Update @cgrp's weight to @weight. + */ + void (*cgroup_set_weight)(struct cgroup *cgrp, u32 weight); + + /** + * @cgroup_set_bandwidth: A cgroup's bandwidth is being changed + * @cgrp: cgroup whose bandwidth is being updated + * @period_us: bandwidth control period + * @quota_us: bandwidth control quota + * @burst_us: bandwidth control burst + * + * Update @cgrp's bandwidth control parameters. This is from the cpu.max + * cgroup interface. + * + * @quota_us / @period_us determines the CPU bandwidth @cgrp is entitled + * to. For example, if @period_us is 1_000_000 and @quota_us is + * 2_500_000. @cgrp is entitled to 2.5 CPUs. @burst_us can be + * interpreted in the same fashion and specifies how much @cgrp can + * burst temporarily. The specific control mechanism and thus the + * interpretation of @period_us and burstiness is up to the BPF + * scheduler. + */ + void (*cgroup_set_bandwidth)(struct cgroup *cgrp, + u64 period_us, u64 quota_us, u64 burst_us); + + /** + * @cgroup_set_idle: A cgroup's idle state is being changed + * @cgrp: cgroup whose idle state is being updated + * @idle: whether the cgroup is entering or exiting idle state + * + * Update @cgrp's idle state to @idle. This callback is invoked when + * a cgroup transitions between idle and non-idle states, allowing the + * BPF scheduler to adjust its behavior accordingly. + */ + void (*cgroup_set_idle)(struct cgroup *cgrp, bool idle); + +#endif /* CONFIG_EXT_GROUP_SCHED */ + + /** + * @sub_attach: Attach a sub-scheduler + * @args: argument container, see the struct definition + * + * Return 0 to accept the sub-scheduler. -errno to reject. + */ + s32 (*sub_attach)(struct scx_sub_attach_args *args); + + /** + * @sub_detach: Detach a sub-scheduler + * @args: argument container, see the struct definition + */ + void (*sub_detach)(struct scx_sub_detach_args *args); + + /* + * All online ops must come before ops.cpu_online(). + */ + + /** + * @cpu_online: A CPU became online + * @cpu: CPU which just came up + * + * @cpu just came online. @cpu will not call ops.enqueue() or + * ops.dispatch(), nor run tasks associated with other CPUs beforehand. + */ + void (*cpu_online)(s32 cpu); + + /** + * @cpu_offline: A CPU is going offline + * @cpu: CPU which is going offline + * + * @cpu is going offline. @cpu will not call ops.enqueue() or + * ops.dispatch(), nor run tasks associated with other CPUs afterwards. + */ + void (*cpu_offline)(s32 cpu); + + /* + * All CPU hotplug ops must come before ops.init(). + */ + + /** + * @init: Initialize the BPF scheduler + */ + s32 (*init)(void); + + /** + * @exit: Clean up after the BPF scheduler + * @info: Exit info + * + * ops.exit() is also called on ops.init() failure, which is a bit + * unusual. This is to allow rich reporting through @info on how + * ops.init() failed. + */ + void (*exit)(struct scx_exit_info *info); + + /* + * Data fields must comes after all ops fields. + */ + + /** + * @dispatch_max_batch: Max nr of tasks that dispatch() can dispatch + */ + u32 dispatch_max_batch; + + /** + * @flags: %SCX_OPS_* flags + */ + u64 flags; + + /** + * @timeout_ms: The maximum amount of time, in milliseconds, that a + * runnable task should be able to wait before being scheduled. The + * maximum timeout may not exceed the default timeout of 30 seconds. + * + * Defaults to the maximum allowed timeout value of 30 seconds. + */ + u32 timeout_ms; + + /** + * @exit_dump_len: scx_exit_info.dump buffer length. If 0, the default + * value of 32768 is used. + */ + u32 exit_dump_len; + + /** + * @hotplug_seq: A sequence number that may be set by the scheduler to + * detect when a hotplug event has occurred during the loading process. + * If 0, no detection occurs. Otherwise, the scheduler will fail to + * load if the sequence number does not match @scx_hotplug_seq on the + * enable path. + */ + u64 hotplug_seq; + + /** + * @cgroup_id: When >1, attach the scheduler as a sub-scheduler on the + * specified cgroup. + */ + u64 sub_cgroup_id; + + /** + * @name: BPF scheduler's name + * + * Must be a non-zero valid BPF object name including only isalnum(), + * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the + * BPF scheduler is enabled. + */ + char name[SCX_OPS_NAME_LEN]; + + /* internal use only, must be NULL */ + void __rcu *priv; + + /* + * Deprecated callbacks. Kept at the end of the struct so the cid-form + * struct (sched_ext_ops_cid) can omit them without affecting the + * shared field offsets. Use SCX_ENQ_IMMED instead. Sitting past + * SCX_OPI_END means has_op doesn't cover them, so SCX_HAS_OP() cannot + * be used; callers must test sch->ops.cpu_acquire / cpu_release + * directly. + */ + + /** + * @cpu_acquire: A CPU is becoming available to the BPF scheduler + * @cpu: The CPU being acquired by the BPF scheduler. + * @args: Acquire arguments, see the struct definition. + * + * A CPU that was previously released from the BPF scheduler is now once + * again under its control. Deprecated; use SCX_ENQ_IMMED instead. + */ + void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); + + /** + * @cpu_release: A CPU is taken away from the BPF scheduler + * @cpu: The CPU being released by the BPF scheduler. + * @args: Release arguments, see the struct definition. + * + * The specified CPU is no longer under the control of the BPF + * scheduler. This could be because it was preempted by a higher + * priority sched_class, though there may be other reasons as well. The + * caller should consult @args->reason to determine the cause. + * Deprecated; use SCX_ENQ_IMMED instead. + */ + void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); +}; + +/** + * struct sched_ext_ops_cid - cid-form alternative to struct sched_ext_ops + * + * Mirrors struct sched_ext_ops with cpu/cpumask substituted with cid/cmask + * where applicable. Layout up to and including @priv matches sched_ext_ops + * byte-for-byte (verified by BUILD_BUG_ON checks at scx_init() time) so + * shared field offsets work for both struct types in bpf_scx_init_member() + * and bpf_scx_check_member(). The deprecated cpu_acquire/cpu_release + * callbacks at the tail of sched_ext_ops are omitted here entirely. + * + * Differences from sched_ext_ops: + * - select_cpu -> select_cid (returns cid) + * - dispatch -> dispatch (cpu arg is now cid) + * - update_idle -> update_idle (cpu arg is now cid) + * - set_cpumask -> set_cmask (cmask instead of cpumask) + * - cpu_online -> cid_online + * - cpu_offline -> cid_offline + * - dump_cpu -> dump_cid + * - cpu_acquire/cpu_release -> not present (deprecated in sched_ext_ops) + * + * BPF schedulers using this type cannot call cpu-form scx_bpf_* kfuncs; + * use the cid-form variants instead. Enforced at BPF verifier time via + * scx_kfunc_context_filter() branching on prog->aux->st_ops. + * + * See sched_ext_ops for callback documentation. + */ +struct sched_ext_ops_cid { + s32 (*select_cid)(struct task_struct *p, s32 prev_cid, u64 wake_flags); + void (*enqueue)(struct task_struct *p, u64 enq_flags); + void (*dequeue)(struct task_struct *p, u64 deq_flags); + void (*dispatch)(s32 cid, struct task_struct *prev); + void (*tick)(struct task_struct *p); + void (*runnable)(struct task_struct *p, u64 enq_flags); + void (*running)(struct task_struct *p); + void (*stopping)(struct task_struct *p, bool runnable); + void (*quiescent)(struct task_struct *p, u64 deq_flags); + bool (*yield)(struct task_struct *from, struct task_struct *to); + bool (*core_sched_before)(struct task_struct *a, + struct task_struct *b); + void (*set_weight)(struct task_struct *p, u32 weight); + void (*set_cmask)(struct task_struct *p, + const struct scx_cmask *cmask); + void (*update_idle)(s32 cid, bool idle); + s32 (*init_task)(struct task_struct *p, + struct scx_init_task_args *args); + void (*exit_task)(struct task_struct *p, + struct scx_exit_task_args *args); + void (*enable)(struct task_struct *p); + void (*disable)(struct task_struct *p); + void (*dump)(struct scx_dump_ctx *ctx); + void (*dump_cid)(struct scx_dump_ctx *ctx, s32 cid, bool idle); + void (*dump_task)(struct scx_dump_ctx *ctx, struct task_struct *p); +#ifdef CONFIG_EXT_GROUP_SCHED + s32 (*cgroup_init)(struct cgroup *cgrp, + struct scx_cgroup_init_args *args); + void (*cgroup_exit)(struct cgroup *cgrp); + s32 (*cgroup_prep_move)(struct task_struct *p, + struct cgroup *from, struct cgroup *to); + void (*cgroup_move)(struct task_struct *p, + struct cgroup *from, struct cgroup *to); + void (*cgroup_cancel_move)(struct task_struct *p, + struct cgroup *from, struct cgroup *to); + void (*cgroup_set_weight)(struct cgroup *cgrp, u32 weight); + void (*cgroup_set_bandwidth)(struct cgroup *cgrp, + u64 period_us, u64 quota_us, u64 burst_us); + void (*cgroup_set_idle)(struct cgroup *cgrp, bool idle); +#endif /* CONFIG_EXT_GROUP_SCHED */ + s32 (*sub_attach)(struct scx_sub_attach_args *args); + void (*sub_detach)(struct scx_sub_detach_args *args); + void (*cid_online)(s32 cid); + void (*cid_offline)(s32 cid); + s32 (*init)(void); + void (*exit)(struct scx_exit_info *info); + + /* Data fields - must match sched_ext_ops layout exactly */ + u32 dispatch_max_batch; + u64 flags; + u32 timeout_ms; + u32 exit_dump_len; + u64 hotplug_seq; + u64 sub_cgroup_id; + char name[SCX_OPS_NAME_LEN]; + + /* internal use only, must be NULL */ + void __rcu *priv; + + /* layout end anchor for the BUILD_BUG_ON in scx_init(); keep last */ + char __end[0]; +}; + +enum scx_opi { + SCX_OPI_BEGIN = 0, + SCX_OPI_NORMAL_BEGIN = 0, + SCX_OPI_NORMAL_END = SCX_OP_IDX(cpu_online), + SCX_OPI_CPU_HOTPLUG_BEGIN = SCX_OP_IDX(cpu_online), + SCX_OPI_CPU_HOTPLUG_END = SCX_OP_IDX(init), + SCX_OPI_END = SCX_OP_IDX(init), +}; + +/* + * Collection of event counters. Event types are placed in descending order. + */ +struct scx_event_stats { + /* + * If ops.select_cpu() returns a CPU which can't be used by the task, + * the core scheduler code silently picks a fallback CPU. + */ + s64 SCX_EV_SELECT_CPU_FALLBACK; + + /* + * When dispatching to a local DSQ, the CPU may have gone offline in + * the meantime. In this case, the task is bounced to the global DSQ. + */ + s64 SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE; + + /* + * If SCX_OPS_ENQ_LAST is not set, the number of times that a task + * continued to run because there were no other tasks on the CPU. + */ + s64 SCX_EV_DISPATCH_KEEP_LAST; + + /* + * If SCX_OPS_ENQ_EXITING is not set, the number of times that a task + * is dispatched to a local DSQ when exiting. + */ + s64 SCX_EV_ENQ_SKIP_EXITING; + + /* + * If SCX_OPS_ENQ_MIGRATION_DISABLED is not set, the number of times a + * migration disabled task skips ops.enqueue() and is dispatched to its + * local DSQ. + */ + s64 SCX_EV_ENQ_SKIP_MIGRATION_DISABLED; + + /* + * The number of times a task, enqueued on a local DSQ with + * SCX_ENQ_IMMED, was re-enqueued because the CPU was not available for + * immediate execution. + */ + s64 SCX_EV_REENQ_IMMED; + + /* + * The number of times a reenq of local DSQ caused another reenq of + * local DSQ. This can happen when %SCX_ENQ_IMMED races against a higher + * priority class task even if the BPF scheduler always satisfies the + * prerequisites for %SCX_ENQ_IMMED at the time of enqueue. However, + * that scenario is very unlikely and this count going up regularly + * indicates that the BPF scheduler is handling %SCX_ENQ_REENQ + * incorrectly causing recursive reenqueues. + */ + s64 SCX_EV_REENQ_LOCAL_REPEAT; + + /* + * Total number of times a task's time slice was refilled with the + * default value (SCX_SLICE_DFL). + */ + s64 SCX_EV_REFILL_SLICE_DFL; + + /* + * The total duration of bypass modes in nanoseconds. + */ + s64 SCX_EV_BYPASS_DURATION; + + /* + * The number of tasks dispatched in the bypassing mode. + */ + s64 SCX_EV_BYPASS_DISPATCH; + + /* + * The number of times the bypassing mode has been activated. + */ + s64 SCX_EV_BYPASS_ACTIVATE; + + /* + * The number of times the scheduler attempted to insert a task that it + * doesn't own into a DSQ. Such attempts are ignored. + * + * As BPF schedulers are allowed to ignore dequeues, it's difficult to + * tell whether such an attempt is from a scheduler malfunction or an + * ignored dequeue around sub-sched enabling. If this count keeps going + * up regardless of sub-sched enabling, it likely indicates a bug in the + * scheduler. + */ + s64 SCX_EV_INSERT_NOT_OWNED; + + /* + * The number of times tasks from bypassing descendants are scheduled + * from sub_bypass_dsq's. + */ + s64 SCX_EV_SUB_BYPASS_DISPATCH; +}; + +struct scx_sched; + +enum scx_sched_pcpu_flags { + SCX_SCHED_PCPU_BYPASSING = 1LLU << 0, +}; + +/* dispatch buf */ +struct scx_dsp_buf_ent { + struct task_struct *task; + unsigned long qseq; + u64 dsq_id; + u64 enq_flags; +}; + +struct scx_dsp_ctx { + struct rq *rq; + u32 cursor; + u32 nr_tasks; + struct scx_dsp_buf_ent buf[]; +}; + +struct scx_deferred_reenq_local { + struct list_head node; + u64 flags; + u64 seq; + u32 cnt; +}; + +struct scx_sched_pcpu { + struct scx_sched *sch; + u64 flags; /* protected by rq lock */ + + /* + * The event counters are in a per-CPU variable to minimize the + * accounting overhead. A system-wide view on the event counter is + * constructed when requested by scx_bpf_events(). + */ + struct scx_event_stats event_stats; + + struct scx_deferred_reenq_local deferred_reenq_local; + struct scx_dispatch_q bypass_dsq; +#ifdef CONFIG_EXT_SUB_SCHED + u32 bypass_host_seq; +#endif + + /* must be the last entry - contains flex array */ + struct scx_dsp_ctx dsp_ctx; +}; + +struct scx_sched_pnode { + struct scx_dispatch_q global_dsq; +}; + +struct scx_sched { + /* + * cpu-form and cid-form ops share field offsets up to .priv (verified + * by BUILD_BUG_ON in scx_init()). The anonymous union lets the kernel + * access either view of the same storage without function-pointer + * casts: use .ops for cpu-form and shared fields, .ops_cid for the + * cid-renamed callbacks (set_cmask, select_cid, cid_online, ...). + */ + union { + struct sched_ext_ops ops; + struct sched_ext_ops_cid ops_cid; + }; + bool is_cid_type; /* true if registered via bpf_sched_ext_ops_cid */ + + /* + * Arena map auto-discovered from member progs at struct_ops attach. + * cid-form schedulers must use exactly one arena across all member + * progs. NULL on cpu-form. + * + * @arena_pool sub-allocates @arena_map. Each gen_pool chunk is added + * at the kernel-side mapping address. @arena_kern_base is the start + * of the arena's kern_vm range. See scx_arena_to_kaddr() and + * scx_kaddr_to_arena(). + */ + struct bpf_map *arena_map; + struct gen_pool *arena_pool; + uintptr_t arena_kern_base; + + /* + * Per-CPU arena cmask used by scx_call_op_set_cpumask() to hand a cmask + * to ops_cid.set_cmask(). The kernel writes through the stored kern_va + * and hands BPF its arena pointer via scx_kaddr_to_arena(). + */ + struct scx_cmask * __percpu *set_cmask_scratch; + + DECLARE_BITMAP(has_op, SCX_OPI_END); + + /* + * Dispatch queues. + * + * The global DSQ (%SCX_DSQ_GLOBAL) is split per-node for scalability. + * This is to avoid live-locking in bypass mode where all tasks are + * dispatched to %SCX_DSQ_GLOBAL and all CPUs consume from it. If + * per-node split isn't sufficient, it can be further split. + */ + struct rhashtable dsq_hash; + struct scx_sched_pnode **pnode; + struct scx_sched_pcpu __percpu *pcpu; + + u64 slice_dfl; + u64 bypass_timestamp; + s32 bypass_depth; + + /* bypass dispatch path enable state, see bypass_dsp_enabled() */ + unsigned long bypass_dsp_claim; + atomic_t bypass_dsp_enable_depth; + + bool aborting; + bool dump_disabled; /* protected by scx_dump_lock */ + u32 dsp_max_batch; + s32 level; + + /* + * Updates to the following warned bitfields can race causing RMW issues + * but it doesn't really matter. + */ + bool warned_zero_slice:1; + bool warned_deprecated_rq:1; + bool warned_unassoc_progs:1; + + struct list_head all; + +#ifdef CONFIG_EXT_SUB_SCHED + struct rhash_head hash_node; + + struct list_head children; + struct list_head sibling; + struct cgroup *cgrp; + char *cgrp_path; + struct kset *sub_kset; + + bool sub_attached; +#endif /* CONFIG_EXT_SUB_SCHED */ + + /* + * The maximum amount of time in jiffies that a task may be runnable + * without being scheduled on a CPU. If this timeout is exceeded, it + * will trigger scx_error(). + */ + unsigned long watchdog_timeout; + + atomic_t exit_kind; + struct scx_exit_info *exit_info; + + struct kobject kobj; + + struct kthread_worker *helper; + struct irq_work disable_irq_work; + struct kthread_work disable_work; + struct timer_list bypass_lb_timer; + cpumask_var_t bypass_lb_donee_cpumask; + cpumask_var_t bypass_lb_resched_cpumask; + struct rcu_work rcu_work; + + /* all ancestors including self */ + struct scx_sched *ancestors[]; +}; + +/** + * scx_arena_to_kaddr - Translate a BPF-arena pointer to its kernel address + * @sch: scheduler whose arena hosts @bpf_ptr + * @bpf_ptr: BPF-arena pointer, only the low 32 bits are used + * + * The (u32) cast normalizes any input into the arena's 4 GiB kern_vm range, + * which combined with scratch-page fault recovery makes the returned pointer + * safe to dereference up to GUARD_SZ / 2 past the intended object. Accesses + * larger than GUARD_SZ / 2 must be explicitly bounds-checked. + */ +static inline void *scx_arena_to_kaddr(struct scx_sched *sch, const void *bpf_ptr) +{ + return (void *)(sch->arena_kern_base + (u32)(uintptr_t)bpf_ptr); +} + +/** + * scx_kaddr_to_arena - Translate a kernel arena address to its BPF form + * @sch: scheduler whose arena hosts @kaddr + * @kaddr: kernel-side arena address, supplied by trusted kernel code + */ +static inline void *scx_kaddr_to_arena(struct scx_sched *sch, const void *kaddr) +{ + return (void *)((uintptr_t)kaddr - sch->arena_kern_base); +} + +enum scx_wake_flags { + /* expose select WF_* flags as enums */ + SCX_WAKE_FORK = WF_FORK, + SCX_WAKE_TTWU = WF_TTWU, + SCX_WAKE_SYNC = WF_SYNC, +}; + +enum scx_enq_flags { + /* expose select ENQUEUE_* flags as enums */ + SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, + SCX_ENQ_HEAD = ENQUEUE_HEAD, + SCX_ENQ_CPU_SELECTED = ENQUEUE_RQ_SELECTED, + + /* high 32bits are SCX specific */ + + /* + * Set the following to trigger preemption when calling + * scx_bpf_dsq_insert() with a local dsq as the target. The slice of the + * current task is cleared to zero and the CPU is kicked into the + * scheduling path. Implies %SCX_ENQ_HEAD. + */ + SCX_ENQ_PREEMPT = 1LLU << 32, + + /* + * Only allowed on local DSQs. Guarantees that the task either gets + * on the CPU immediately and stays on it, or gets reenqueued back + * to the BPF scheduler. It will never linger on a local DSQ or be + * silently put back after preemption. + * + * The protection persists until the next fresh enqueue - it + * survives SAVE/RESTORE cycles, slice extensions and preemption. + * If the task can't stay on the CPU for any reason, it gets + * reenqueued back to the BPF scheduler. + * + * Exiting and migration-disabled tasks bypass ops.enqueue() and + * are placed directly on a local DSQ without IMMED protection + * unless %SCX_OPS_ENQ_EXITING and %SCX_OPS_ENQ_MIGRATION_DISABLED + * are set respectively. + */ + SCX_ENQ_IMMED = 1LLU << 33, + + /* + * The task being enqueued was previously enqueued on a DSQ, but was + * removed and is being re-enqueued. See SCX_TASK_REENQ_* flags to find + * out why a given task is being reenqueued. + */ + SCX_ENQ_REENQ = 1LLU << 40, + + /* + * The task being enqueued is the only task available for the cpu. By + * default, ext core keeps executing such tasks but when + * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with the + * %SCX_ENQ_LAST flag set. + * + * The BPF scheduler is responsible for triggering a follow-up + * scheduling event. Otherwise, Execution may stall. + */ + SCX_ENQ_LAST = 1LLU << 41, + + /* high 8 bits are internal */ + __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, + + SCX_ENQ_CLEAR_OPSS = 1LLU << 56, + SCX_ENQ_DSQ_PRIQ = 1LLU << 57, + SCX_ENQ_NESTED = 1LLU << 58, + SCX_ENQ_GDSQ_FALLBACK = 1LLU << 59, /* fell back to global DSQ */ +}; + +enum scx_deq_flags { + /* expose select DEQUEUE_* flags as enums */ + SCX_DEQ_SLEEP = DEQUEUE_SLEEP, + + /* high 32bits are SCX specific */ + + /* + * The generic core-sched layer decided to execute the task even though + * it hasn't been dispatched yet. Dequeue from the BPF side. + */ + SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, + + /* + * The task is being dequeued due to a property change (e.g., + * sched_setaffinity(), sched_setscheduler(), set_user_nice(), + * etc.). + */ + SCX_DEQ_SCHED_CHANGE = 1LLU << 33, +}; + +enum scx_reenq_flags { + /* low 16bits determine which tasks should be reenqueued */ + SCX_REENQ_ANY = 1LLU << 0, /* all tasks */ + + __SCX_REENQ_FILTER_MASK = 0xffffLLU, + + __SCX_REENQ_USER_MASK = SCX_REENQ_ANY, + + /* bits 32-35 used by task_should_reenq() */ + SCX_REENQ_TSR_RQ_OPEN = 1LLU << 32, + SCX_REENQ_TSR_NOT_FIRST = 1LLU << 33, + + __SCX_REENQ_TSR_MASK = 0xfLLU << 32, +}; + +enum scx_pick_idle_cpu_flags { + SCX_PICK_IDLE_CORE = 1LLU << 0, /* pick a CPU whose SMT siblings are also idle */ + SCX_PICK_IDLE_IN_NODE = 1LLU << 1, /* pick a CPU in the same target NUMA node */ +}; + +enum scx_kick_flags { + /* + * Kick the target CPU if idle. Guarantees that the target CPU goes + * through at least one full scheduling cycle before going idle. If the + * target CPU can be determined to be currently not idle and going to go + * through a scheduling cycle before going idle, noop. + */ + SCX_KICK_IDLE = 1LLU << 0, + + /* + * Preempt the current task and execute the dispatch path. If the + * current task of the target CPU is an SCX task, its ->scx.slice is + * cleared to zero before the scheduling path is invoked so that the + * task expires and the dispatch path is invoked. + */ + SCX_KICK_PREEMPT = 1LLU << 1, + + /* + * The scx_bpf_kick_cpu() call will return after the current SCX task of + * the target CPU switches out. This can be used to implement e.g. core + * scheduling. This has no effect if the current task on the target CPU + * is not on SCX. + */ + SCX_KICK_WAIT = 1LLU << 2, +}; + +enum scx_tg_flags { + SCX_TG_ONLINE = 1U << 0, + SCX_TG_INITED = 1U << 1, +}; + +enum scx_enable_state { + SCX_ENABLING, + SCX_ENABLED, + SCX_DISABLING, + SCX_DISABLED, +}; + +static const char *scx_enable_state_str[] = { + [SCX_ENABLING] = "enabling", + [SCX_ENABLED] = "enabled", + [SCX_DISABLING] = "disabling", + [SCX_DISABLED] = "disabled", +}; + +/* + * Task Ownership State Machine (sched_ext_entity->ops_state) + * + * The sched_ext core uses this state machine to track task ownership + * between the SCX core and the BPF scheduler. This allows the BPF + * scheduler to dispatch tasks without strict ordering requirements, while + * the SCX core safely rejects invalid dispatches. + * + * State Transitions + * + * .------------> NONE (owned by SCX core) + * | | ^ + * | enqueue | | direct dispatch + * | v | + * | QUEUEING -------' + * | | + * | enqueue | + * | completes | + * | v + * | QUEUED (owned by BPF scheduler) + * | | + * | dispatch | + * | | + * | v + * | DISPATCHING + * | | + * | dispatch | + * | completes | + * `---------------' + * + * State Descriptions + * + * - %SCX_OPSS_NONE: + * Task is owned by the SCX core. It's either on a run queue, running, + * or being manipulated by the core scheduler. The BPF scheduler has no + * claim on this task. + * + * - %SCX_OPSS_QUEUEING: + * Transitional state while transferring a task from the SCX core to + * the BPF scheduler. The task's rq lock is held during this state. + * Since QUEUEING is both entered and exited under the rq lock, dequeue + * can never observe this state (it would be a BUG). When finishing a + * dispatch, if the task is still in %SCX_OPSS_QUEUEING the completion + * path busy-waits for it to leave this state (via wait_ops_state()) + * before retrying. + * + * - %SCX_OPSS_QUEUED: + * Task is owned by the BPF scheduler. It's on a DSQ (dispatch queue) + * and the BPF scheduler is responsible for dispatching it. A QSEQ + * (queue sequence number) is embedded in this state to detect + * dispatch/dequeue races: if a task is dequeued and re-enqueued, the + * QSEQ changes and any in-flight dispatch operations targeting the old + * QSEQ are safely ignored. + * + * - %SCX_OPSS_DISPATCHING: + * Transitional state while transferring a task from the BPF scheduler + * back to the SCX core. This state indicates the BPF scheduler has + * selected the task for execution. When dequeue needs to take the task + * off a DSQ and it is still in %SCX_OPSS_DISPATCHING, the dequeue path + * busy-waits for it to leave this state (via wait_ops_state()) before + * proceeding. Exits to %SCX_OPSS_NONE when dispatch completes. + * + * Memory Ordering + * + * Transitions out of %SCX_OPSS_QUEUEING and %SCX_OPSS_DISPATCHING into + * %SCX_OPSS_NONE or %SCX_OPSS_QUEUED must use atomic_long_set_release() + * and waiters must use atomic_long_read_acquire(). This ensures proper + * synchronization between concurrent operations. + * + * Cross-CPU Task Migration + * + * When moving a task in the %SCX_OPSS_DISPATCHING state, we can't simply + * grab the target CPU's rq lock because a concurrent dequeue might be + * waiting on %SCX_OPSS_DISPATCHING while holding the source rq lock + * (deadlock). + * + * The sched_ext core uses a "lock dancing" protocol coordinated by + * p->scx.holding_cpu. When moving a task to a different rq: + * + * 1. Verify task can be moved (CPU affinity, migration_disabled, etc.) + * 2. Set p->scx.holding_cpu to the current CPU + * 3. Set task state to %SCX_OPSS_NONE; dequeue waits while DISPATCHING + * is set, so clearing DISPATCHING first prevents the circular wait + * (safe to lock the rq we need) + * 4. Unlock the current CPU's rq + * 5. Lock src_rq (where the task currently lives) + * 6. Verify p->scx.holding_cpu == current CPU, if not, dequeue won the + * race (dequeue clears holding_cpu to -1 when it takes the task), in + * this case migration is aborted + * 7. If src_rq == dst_rq: clear holding_cpu and enqueue directly + * into dst_rq's local DSQ (no lock swap needed) + * 8. Otherwise: call move_remote_task_to_local_dsq(), which releases + * src_rq, locks dst_rq, and performs the deactivate/activate + * migration cycle (dst_rq is held on return) + * 9. Unlock dst_rq and re-lock the current CPU's rq to restore + * the lock state expected by the caller + * + * If any verification fails, abort the migration. + * + * This state tracking allows the BPF scheduler to try to dispatch any task + * at any time regardless of its state. The SCX core can safely + * reject/ignore invalid dispatches, simplifying the BPF scheduler + * implementation. + */ +enum scx_ops_state { + SCX_OPSS_NONE, /* owned by the SCX core */ + SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ + SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ + SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ + + /* + * QSEQ brands each QUEUED instance so that, when dispatch races + * dequeue/requeue, the dispatcher can tell whether it still has a claim + * on the task being dispatched. + * + * As some 32bit archs can't do 64bit store_release/load_acquire, + * p->scx.ops_state is atomic_long_t which leaves 30 bits for QSEQ on + * 32bit machines. The dispatch race window QSEQ protects is very narrow + * and runs with IRQ disabled. 30 bits should be sufficient. + */ + SCX_OPSS_QSEQ_SHIFT = 2, +}; + +/* Use macros to ensure that the type is unsigned long for the masks */ +#define SCX_OPSS_STATE_MASK ((1LU << SCX_OPSS_QSEQ_SHIFT) - 1) +#define SCX_OPSS_QSEQ_MASK (~SCX_OPSS_STATE_MASK) + +extern struct scx_sched __rcu *scx_root; +DECLARE_PER_CPU(struct rq *, scx_locked_rq_state); + +/* + * True when the currently loaded scheduler hierarchy is cid-form. All scheds + * in a hierarchy share one form, so this single key tells callsites which + * view to use without per-sch dereferences. Use scx_is_cid_type() to test. + */ +DECLARE_STATIC_KEY_FALSE(__scx_is_cid_type); + +int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id); + +bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where); + +__printf(5, 0) bool scx_vexit(struct scx_sched *sch, enum scx_exit_kind kind, + s64 exit_code, s32 exit_cpu, const char *fmt, + va_list args); +__printf(5, 6) bool __scx_exit(struct scx_sched *sch, enum scx_exit_kind kind, + s64 exit_code, s32 exit_cpu, const char *fmt, ...); + +#define scx_exit(sch, kind, exit_code, fmt, args...) \ + __scx_exit(sch, kind, exit_code, raw_smp_processor_id(), fmt, ##args) +#define scx_error(sch, fmt, args...) \ + scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args) +#define scx_verror(sch, fmt, args) \ + scx_vexit((sch), SCX_EXIT_ERROR, 0, raw_smp_processor_id(), fmt, args) + +/* + * Return the rq currently locked from an scx callback, or NULL if no rq is + * locked. + */ +static inline struct rq *scx_locked_rq(void) +{ + return __this_cpu_read(scx_locked_rq_state); +} + +static inline bool scx_bypassing(struct scx_sched *sch, s32 cpu) +{ + return unlikely(per_cpu_ptr(sch->pcpu, cpu)->flags & + SCX_SCHED_PCPU_BYPASSING); +} + +#ifdef CONFIG_EXT_SUB_SCHED +/** + * scx_task_sched - Find scx_sched scheduling a task + * @p: task of interest + * + * Return @p's scheduler instance. Must be called with @p's pi_lock or rq lock + * held. + */ +static inline struct scx_sched *scx_task_sched(const struct task_struct *p) +{ + return rcu_dereference_protected(p->scx.sched, + lockdep_is_held(&p->pi_lock) || + lockdep_is_held(__rq_lockp(task_rq(p)))); +} + +/** + * scx_task_sched_rcu - Find scx_sched scheduling a task + * @p: task of interest + * + * Return @p's scheduler instance. The returned scx_sched is RCU protected. + */ +static inline struct scx_sched *scx_task_sched_rcu(const struct task_struct *p) +{ + return rcu_dereference_all(p->scx.sched); +} + +/** + * scx_task_on_sched - Is a task on the specified sched? + * @sch: sched to test against + * @p: task of interest + * + * Returns %true if @p is on @sch, %false otherwise. + */ +static inline bool scx_task_on_sched(struct scx_sched *sch, + const struct task_struct *p) +{ + return rcu_access_pointer(p->scx.sched) == sch; +} + +/** + * scx_prog_sched - Find scx_sched associated with a BPF prog + * @aux: aux passed in from BPF to a kfunc + * + * To be called from kfuncs. Return the scheduler instance associated with the + * BPF program given the implicit kfunc argument aux. The returned scx_sched is + * RCU protected. + */ +static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux) +{ + struct sched_ext_ops *ops; + struct scx_sched *root; + + ops = bpf_prog_get_assoc_struct_ops(aux); + if (likely(ops)) + return rcu_dereference_all(ops->priv); + + root = rcu_dereference_all(scx_root); + if (root) { + /* + * COMPAT-v6.19: Schedulers built before sub-sched support was + * introduced may have unassociated non-struct_ops programs. + */ + if (!root->ops.sub_attach) + return root; + + if (!root->warned_unassoc_progs) { + printk_deferred(KERN_WARNING "sched_ext: Unassociated program %s (id %d)\n", + aux->name, aux->id); + root->warned_unassoc_progs = true; + } + } + + return NULL; +} +#else /* CONFIG_EXT_SUB_SCHED */ +static inline struct scx_sched *scx_task_sched(const struct task_struct *p) +{ + return rcu_dereference_protected(scx_root, + lockdep_is_held(&p->pi_lock) || + lockdep_is_held(__rq_lockp(task_rq(p)))); +} + +static inline struct scx_sched *scx_task_sched_rcu(const struct task_struct *p) +{ + return rcu_dereference_all(scx_root); +} + +static inline bool scx_task_on_sched(struct scx_sched *sch, + const struct task_struct *p) +{ + return true; +} + +static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux) +{ + return rcu_dereference_all(scx_root); +} +#endif /* CONFIG_EXT_SUB_SCHED */ diff --git a/kernel/sched/ext/types.h b/kernel/sched/ext/types.h new file mode 100644 index 000000000000..8b3527e21fca --- /dev/null +++ b/kernel/sched/ext/types.h @@ -0,0 +1,144 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Early sched_ext type definitions. + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2026 Tejun Heo + */ +#ifndef _KERNEL_SCHED_EXT_TYPES_H +#define _KERNEL_SCHED_EXT_TYPES_H + +enum scx_consts { + SCX_DSP_DFL_MAX_BATCH = 32, + SCX_DSP_MAX_LOOPS = 32, + SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, + + /* per-CPU chunk size for p->scx.tid allocation, see scx_alloc_tid() */ + SCX_TID_CHUNK = 1024, + + SCX_EXIT_BT_LEN = 64, + SCX_EXIT_MSG_LEN = 1024, + SCX_EXIT_DUMP_DFL_LEN = 32768, + + SCX_CPUPERF_ONE = SCHED_CAPACITY_SCALE, + + /* + * Iterating all tasks may take a while. Periodically drop + * scx_tasks_lock to avoid causing e.g. CSD and RCU stalls. + */ + SCX_TASK_ITER_BATCH = 32, + + SCX_BYPASS_HOST_NTH = 2, + + SCX_BYPASS_LB_DFL_INTV_US = 500 * USEC_PER_MSEC, + SCX_BYPASS_LB_DONOR_PCT = 125, + SCX_BYPASS_LB_MIN_DELTA_DIV = 4, + SCX_BYPASS_LB_BATCH = 256, + + SCX_REENQ_LOCAL_MAX_REPEAT = 256, + + SCX_SUB_MAX_DEPTH = 4, +}; + +/* + * Per-cid topology info. For each topology level (core, LLC, node), records + * the first cid in the unit and its global index. Global indices are + * consecutive integers assigned in cid-walk order, so e.g. core_idx ranges + * over [0, nr_cores_at_init) with no gaps. No-topo cids have all fields set + * to -1. + * + * @core_cid: first cid of this cid's core (smt-sibling group) + * @core_idx: global index of that core, in [0, nr_cores_at_init) + * @llc_cid: first cid of this cid's LLC + * @llc_idx: global index of that LLC, in [0, nr_llcs_at_init) + * @node_cid: first cid of this cid's NUMA node + * @node_idx: global index of that node, in [0, nr_nodes_at_init) + */ +struct scx_cid_topo { + s32 core_cid; + s32 core_idx; + s32 llc_cid; + s32 llc_idx; + s32 node_cid; + s32 node_idx; +}; + +/* + * cmask: variable-length, base-windowed bitmap over cid space + * ----------------------------------------------------------- + * + * A cmask covers the cid range [base, base + nr_cids). bits[] is aligned to the + * global 64-cid grid: bits[0] spans [base & ~63, (base & ~63) + 64), so the + * first (base & 63) bits of bits[0] are head padding and the trailing bits of + * the last active word past base + nr_cids are tail padding. Both stay zero; + * all mutating helpers preserve that. Words past the last active word are not + * read by any helper and have no constraint. + * + * Grid alignment means two cmasks always address bits[] against the same global + * 64-cid windows, so cross-cmask word ops (AND, OR, ...) reduce to + * + * dst->bits[i] OP= src->bits[i - delta] + * + * with no bit-shifting, regardless of how the two bases relate mod 64. + */ +struct scx_cmask { + u32 base; + u32 nr_cids; + u32 alloc_words; + u64 bits[] __counted_by(alloc_words); +}; + +/* + * Number of u64 words of bits[] storage that covers @nr_cids regardless of base + * alignment. The +1 absorbs up to 63 bits of head padding when base is not + * 64-aligned - always allocating one extra word beats branching on base or + * splitting the compute. The u64 cast keeps the +63 from wrapping when @nr_cids + * is near U32_MAX, so callers bounds-checking the result against @alloc_words + * catch the overflow instead of seeing a small value. + */ +#define SCX_CMASK_NR_WORDS(nr_cids) ((u32)(((u64)(nr_cids) + 63) / 64 + 1)) + +/** + * __SCX_CMASK_DEFINE - Define an on-stack cmask with explicit storage capacity + * @NAME: variable name to define + * @BASE: first cid of the active range + * @NR_CIDS: active range length + * @ALLOC_CIDS: storage capacity in cids, at least @NR_CIDS + * + * @NAME aliases zero-initialized storage with the active range set to + * [BASE, BASE + NR_CIDS). Use scx_cmask_reframe() to reshape later, up to + * @ALLOC_CIDS. + */ +#define __SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, ALLOC_CIDS) \ + _DEFINE_FLEX(struct scx_cmask, NAME, bits, SCX_CMASK_NR_WORDS(ALLOC_CIDS), \ + = { .base = (BASE), \ + .nr_cids = (NR_CIDS), \ + .alloc_words = SCX_CMASK_NR_WORDS(ALLOC_CIDS) }) + +/** + * SCX_CMASK_DEFINE - Define an on-stack cmask on tight storage + * @NAME: variable name to define + * @BASE: first cid of the active range + * @NR_CIDS: active range length, also storage capacity + * + * @NAME aliases zero-initialized storage with the active range and storage + * both [BASE, BASE + NR_CIDS). + */ +#define SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS) \ + __SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, NR_CIDS) + +/** + * SCX_CMASK_DEFINE_SHARD - Define an on-stack cmask sized to one shard + * @NAME: variable name to define + * @BASE: first cid of the active range + * @NR_CIDS: active range length, must be <= SCX_CID_SHARD_MAX_CPUS + * + * Storage is fixed at SCX_CID_SHARD_MAX_CPUS, active range framed by + * (BASE, NR_CIDS). Passing NR_CIDS > SCX_CID_SHARD_MAX_CPUS leaves the + * cmask claiming more bits than storage holds and subsequent cmask + * operations will overrun. + */ +#define SCX_CMASK_DEFINE_SHARD(NAME, BASE, NR_CIDS) \ + __SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, SCX_CID_SHARD_MAX_CPUS) + +#endif /* _KERNEL_SCHED_EXT_TYPES_H */ diff --git a/kernel/sched/ext_arena.c b/kernel/sched/ext_arena.c deleted file mode 100644 index 493c2424f842..000000000000 --- a/kernel/sched/ext_arena.c +++ /dev/null @@ -1,131 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst - * - * scx_arena_pool: kernel-side sub-allocator over BPF-arena pages. - * - * Each chunk added to @sch->arena_pool comes from one - * bpf_arena_alloc_pages_sleepable() call and is registered at the - * kernel-side mapping address. Callers translate to the BPF-arena form - * themselves if needed. - * - * Allocations grow the pool on demand. Underlying arena pages are released - * when the arena map itself is torn down. - * - * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. - * Copyright (c) 2026 Tejun Heo - */ - -enum scx_arena_consts { - SCX_ARENA_MIN_ORDER = 3, /* 8-byte minimum sub-allocation */ - SCX_ARENA_GROW_PAGES = 4, /* per growth */ -}; - -s32 scx_arena_pool_init(struct scx_sched *sch) -{ - if (!sch->arena_map) - return 0; - - sch->arena_pool = gen_pool_create(SCX_ARENA_MIN_ORDER, NUMA_NO_NODE); - if (!sch->arena_pool) - return -ENOMEM; - return 0; -} - -static void scx_arena_clear_chunk(struct gen_pool *pool, struct gen_pool_chunk *chunk, - void *data) -{ - int order = pool->min_alloc_order; - size_t chunk_sz = chunk->end_addr - chunk->start_addr + 1; - unsigned long end_bit = chunk_sz >> order; - unsigned long b, e; - - for_each_set_bitrange(b, e, chunk->bits, end_bit) - gen_pool_free(pool, chunk->start_addr + (b << order), - (e - b) << order); -} - -/* - * Tear down the pool. Outstanding gen_pool allocations are freed via - * scx_arena_clear_chunk() so gen_pool_destroy() doesn't BUG. The underlying - * arena pages are released when the arena map itself is torn down. - */ -void scx_arena_pool_destroy(struct scx_sched *sch) -{ - if (!sch->arena_pool) - return; - gen_pool_for_each_chunk(sch->arena_pool, scx_arena_clear_chunk, NULL); - gen_pool_destroy(sch->arena_pool); - sch->arena_pool = NULL; -} - -/* - * Grow the pool by @page_cnt pages. bpf_arena_alloc_pages_sleepable() and - * gen_pool_add() (which calls vzalloc(GFP_KERNEL)) require a sleepable - * context. - */ -static int scx_arena_grow(struct scx_sched *sch, u32 page_cnt) -{ - u64 kern_vm_start; - u32 uaddr32; - void *p; - int ret; - - if (!sch->arena_map || !sch->arena_pool) - return -EINVAL; - - p = bpf_arena_alloc_pages_sleepable(sch->arena_map, NULL, - page_cnt, NUMA_NO_NODE, 0); - if (!p) - return -ENOMEM; - - uaddr32 = (u32)(unsigned long)p; - /* arena.o, which defines these, is built only on MMU && 64BIT */ -#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) - kern_vm_start = bpf_arena_map_kern_vm_start(sch->arena_map); -#else - kern_vm_start = 0; -#endif - - ret = gen_pool_add(sch->arena_pool, kern_vm_start + uaddr32, - page_cnt * PAGE_SIZE, NUMA_NO_NODE); - if (ret) { - bpf_arena_free_pages_non_sleepable(sch->arena_map, p, page_cnt); - return ret; - } - return 0; -} - -/* - * Allocate @size bytes from the arena pool. Returns kernel VA on success, NULL - * on failure. May grow the pool via scx_arena_grow() which sleeps. Caller must - * be in a GFP_KERNEL context. - */ -void *scx_arena_alloc(struct scx_sched *sch, size_t size) -{ - unsigned long kern_va; - u32 page_cnt; - - might_sleep(); - - if (!sch->arena_pool) - return NULL; - - while (true) { - kern_va = gen_pool_alloc(sch->arena_pool, size); - if (kern_va) - break; - page_cnt = max_t(u32, SCX_ARENA_GROW_PAGES, - (size + PAGE_SIZE - 1) >> PAGE_SHIFT); - if (scx_arena_grow(sch, page_cnt)) - return NULL; - } - - return (void *)kern_va; -} - -void scx_arena_free(struct scx_sched *sch, void *kern_va, size_t size) -{ - if (sch->arena_pool && kern_va) - gen_pool_free(sch->arena_pool, (unsigned long)kern_va, size); -} diff --git a/kernel/sched/ext_arena.h b/kernel/sched/ext_arena.h deleted file mode 100644 index 4f3610160102..000000000000 --- a/kernel/sched/ext_arena.h +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst - * - * Copyright (c) 2025 Meta Platforms, Inc. and affiliates. - * Copyright (c) 2025 Tejun Heo - */ -#ifndef _KERNEL_SCHED_EXT_ARENA_H -#define _KERNEL_SCHED_EXT_ARENA_H - -struct scx_sched; - -s32 scx_arena_pool_init(struct scx_sched *sch); -void scx_arena_pool_destroy(struct scx_sched *sch); -void *scx_arena_alloc(struct scx_sched *sch, size_t size); -void scx_arena_free(struct scx_sched *sch, void *kern_va, size_t size); - -#endif /* _KERNEL_SCHED_EXT_ARENA_H */ diff --git a/kernel/sched/ext_cid.c b/kernel/sched/ext_cid.c deleted file mode 100644 index 66944a7ef79d..000000000000 --- a/kernel/sched/ext_cid.c +++ /dev/null @@ -1,707 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst - * - * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. - * Copyright (c) 2026 Tejun Heo - */ -#include - -/* - * cid tables. - * - * Pointers are published once on first enable and never revoked. The default - * mapping is populated before ops.init() runs; scx_bpf_cid_override() commits - * before it returns. As long as the BPF scheduler only uses the tables from - * those points onward, it sees a consistent view. - */ -s16 *scx_cid_to_cpu_tbl; -s16 *scx_cpu_to_cid_tbl; -struct scx_cid_topo *scx_cid_topo; - -#define SCX_CID_TOPO_NEG (struct scx_cid_topo) { \ - .core_cid = -1, .core_idx = -1, .llc_cid = -1, .llc_idx = -1, \ - .node_cid = -1, .node_idx = -1, \ -} - -/* - * Return @cpu's LLC shared_cpu_map. If cacheinfo isn't populated (offline or - * !present), record @cpu in @fallbacks and return its node mask instead - the - * worst that can happen is that the cpu's LLC becomes coarser than reality. - */ -static const struct cpumask *cpu_llc_mask(int cpu, struct cpumask *fallbacks) -{ - struct cpu_cacheinfo *ci = get_cpu_cacheinfo(cpu); - - if (!ci || !ci->info_list || !ci->num_leaves) { - cpumask_set_cpu(cpu, fallbacks); - return cpumask_of_node(cpu_to_node(cpu)); - } - return &ci->info_list[ci->num_leaves - 1].shared_cpu_map; -} - -/* Allocate the cid tables once on first enable; never freed. */ -static s32 scx_cid_arrays_alloc(void) -{ - u32 npossible = num_possible_cpus(); - s16 *cid_to_cpu, *cpu_to_cid; - struct scx_cid_topo *cid_topo; - - if (scx_cid_to_cpu_tbl) - return 0; - - cid_to_cpu = kzalloc_objs(*scx_cid_to_cpu_tbl, npossible, GFP_KERNEL); - cpu_to_cid = kzalloc_objs(*scx_cpu_to_cid_tbl, nr_cpu_ids, GFP_KERNEL); - cid_topo = kmalloc_objs(*scx_cid_topo, npossible, GFP_KERNEL); - - if (!cid_to_cpu || !cpu_to_cid || !cid_topo) { - kfree(cid_to_cpu); - kfree(cpu_to_cid); - kfree(cid_topo); - return -ENOMEM; - } - - WRITE_ONCE(scx_cid_to_cpu_tbl, cid_to_cpu); - WRITE_ONCE(scx_cpu_to_cid_tbl, cpu_to_cid); - WRITE_ONCE(scx_cid_topo, cid_topo); - return 0; -} - -/** - * scx_cid_init - build the cid mapping - * @sch: the scx_sched being initialized; used as the scx_error() target - * - * See "Topological CPU IDs" in ext_cid.h for the model. Walk online cpus by - * intersection at each level (parent_scratch & this_level_mask), which keeps - * containment correct by construction and naturally splits a physical LLC - * straddling two NUMA nodes into two LLC units. The caller must hold - * cpus_read_lock. - */ -s32 scx_cid_init(struct scx_sched *sch) -{ - cpumask_var_t to_walk __free(free_cpumask_var) = CPUMASK_VAR_NULL; - cpumask_var_t node_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; - cpumask_var_t llc_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; - cpumask_var_t core_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; - cpumask_var_t llc_fallback __free(free_cpumask_var) = CPUMASK_VAR_NULL; - cpumask_var_t online_no_topo __free(free_cpumask_var) = CPUMASK_VAR_NULL; - u32 next_cid = 0; - s32 next_node_idx = 0, next_llc_idx = 0, next_core_idx = 0; - s32 cpu, ret; - - /* CMASK_MAX_WORDS in cid.bpf.h covers NR_CPUS up to 8192 */ - BUILD_BUG_ON(NR_CPUS > 8192); - - lockdep_assert_cpus_held(); - - ret = scx_cid_arrays_alloc(); - if (ret) - return ret; - - if (!zalloc_cpumask_var(&to_walk, GFP_KERNEL) || - !zalloc_cpumask_var(&node_scratch, GFP_KERNEL) || - !zalloc_cpumask_var(&llc_scratch, GFP_KERNEL) || - !zalloc_cpumask_var(&core_scratch, GFP_KERNEL) || - !zalloc_cpumask_var(&llc_fallback, GFP_KERNEL) || - !zalloc_cpumask_var(&online_no_topo, GFP_KERNEL)) - return -ENOMEM; - - /* -1 sentinels for sparse-possible cpu id holes (0 is a valid cid) */ - for (cpu = 0; cpu < nr_cpu_ids; cpu++) - scx_cpu_to_cid_tbl[cpu] = -1; - - cpumask_copy(to_walk, cpu_online_mask); - - while (!cpumask_empty(to_walk)) { - s32 next_cpu = cpumask_first(to_walk); - s32 nid = cpu_to_node(next_cpu); - s32 node_cid = next_cid; - s32 node_idx; - - /* - * No NUMA info: skip and let the tail loop assign a no-topo - * cid. cpumask_of_node(-1) is undefined. - */ - if (nid < 0) { - cpumask_clear_cpu(next_cpu, to_walk); - continue; - } - - node_idx = next_node_idx++; - - /* node_scratch = to_walk & this node */ - cpumask_and(node_scratch, to_walk, cpumask_of_node(nid)); - if (WARN_ON_ONCE(!cpumask_test_cpu(next_cpu, node_scratch))) - return -EINVAL; - - while (!cpumask_empty(node_scratch)) { - s32 ncpu = cpumask_first(node_scratch); - const struct cpumask *llc_mask = cpu_llc_mask(ncpu, llc_fallback); - s32 llc_cid = next_cid; - s32 llc_idx = next_llc_idx++; - - /* llc_scratch = node_scratch & this llc */ - cpumask_and(llc_scratch, node_scratch, llc_mask); - if (WARN_ON_ONCE(!cpumask_test_cpu(ncpu, llc_scratch))) - return -EINVAL; - - while (!cpumask_empty(llc_scratch)) { - s32 lcpu = cpumask_first(llc_scratch); - const struct cpumask *sib = topology_sibling_cpumask(lcpu); - s32 core_cid = next_cid; - s32 core_idx = next_core_idx++; - s32 ccpu; - - /* core_scratch = llc_scratch & this core */ - cpumask_and(core_scratch, llc_scratch, sib); - if (WARN_ON_ONCE(!cpumask_test_cpu(lcpu, core_scratch))) - return -EINVAL; - - for_each_cpu(ccpu, core_scratch) { - s32 cid = next_cid++; - - scx_cid_to_cpu_tbl[cid] = ccpu; - scx_cpu_to_cid_tbl[ccpu] = cid; - scx_cid_topo[cid] = (struct scx_cid_topo){ - .core_cid = core_cid, - .core_idx = core_idx, - .llc_cid = llc_cid, - .llc_idx = llc_idx, - .node_cid = node_cid, - .node_idx = node_idx, - }; - - cpumask_clear_cpu(ccpu, llc_scratch); - cpumask_clear_cpu(ccpu, node_scratch); - cpumask_clear_cpu(ccpu, to_walk); - } - } - } - } - - /* - * No-topo section: any possible cpu without a cid - normally just the - * not-online ones. Collect any currently-online cpus that land here in - * @online_no_topo so we can warn about them at the end. - */ - for_each_cpu(cpu, cpu_possible_mask) { - s32 cid; - - if (__scx_cpu_to_cid(cpu) != -1) - continue; - if (cpu_online(cpu)) - cpumask_set_cpu(cpu, online_no_topo); - - cid = next_cid++; - scx_cid_to_cpu_tbl[cid] = cpu; - scx_cpu_to_cid_tbl[cpu] = cid; - scx_cid_topo[cid] = SCX_CID_TOPO_NEG; - } - - if (!cpumask_empty(llc_fallback)) - pr_warn("scx_cid: cpus without cacheinfo, using node mask as llc: %*pbl\n", - cpumask_pr_args(llc_fallback)); - if (!cpumask_empty(online_no_topo)) - pr_warn("scx_cid: online cpus with no usable topology: %*pbl\n", - cpumask_pr_args(online_no_topo)); - - return 0; -} - -/** - * scx_cmask_clear - Zero every bit in @m's active range - * @m: cmask to clear - * - * Storage past the active range is left as is. - */ -void scx_cmask_clear(struct scx_cmask *m) -{ - u32 nr_words; - - if (!m->nr_cids) - return; - nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1; - memset(m->bits, 0, nr_words * sizeof(u64)); -} - -/** - * scx_cmask_fill - Set every bit in @m's active range - * @m: cmask to fill - * - * Counterpart to scx_cmask_clear(). Storage past the active range is left as is. - */ -void scx_cmask_fill(struct scx_cmask *m) -{ - u32 nr_words, head_bits, tail_bits; - - if (!m->nr_cids) - return; - nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1; - memset(m->bits, 0xff, nr_words * sizeof(u64)); - - /* clear word-0 bits below base */ - head_bits = m->base & 63; - if (head_bits) - m->bits[0] &= ~((1ULL << head_bits) - 1); - - /* clear last-word bits at or past base + nr_cids */ - tail_bits = (m->base + m->nr_cids) & 63; - if (tail_bits) - m->bits[nr_words - 1] &= (1ULL << tail_bits) - 1; -} - -/** - * scx_cpumask_to_cmask - Translate a kernel cpumask into a cmask - * @src: source cpumask - * @dst: cmask to write - * - * Clear @dst's active range and set the bit for each cid whose cpu is in - * @src and lies within that range. Out-of-range cids are silently ignored. - */ -void scx_cpumask_to_cmask(const struct cpumask *src, struct scx_cmask *dst) -{ - s32 cpu; - - scx_cmask_clear(dst); - for_each_cpu(cpu, src) { - s32 cid = __scx_cpu_to_cid(cpu); - - if (cid >= 0) - __scx_cmask_set(cid, dst); - } -} - -__bpf_kfunc_start_defs(); - -/** - * scx_bpf_cid_override - Install an explicit cpu->cid mapping - * @cpu_to_cid: array of nr_cpu_ids s32 entries (cid for each cpu) - * @cpu_to_cid__sz: must be nr_cpu_ids * sizeof(s32) bytes - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * May only be called from ops.init() of the root scheduler. Replace the - * topology-probed cid mapping with the caller-provided one. Each possible cpu - * must map to a unique cid in [0, num_possible_cpus()). Topo info is cleared. - * On invalid input, trigger scx_error() to abort the scheduler. - */ -__bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid, u32 cpu_to_cid__sz, - const struct bpf_prog_aux *aux) -{ - cpumask_var_t seen __free(free_cpumask_var) = CPUMASK_VAR_NULL; - struct scx_sched *sch; - bool alloced; - s32 cpu, cid; - - /* GFP_KERNEL alloc must happen before the rcu read section */ - alloced = zalloc_cpumask_var(&seen, GFP_KERNEL); - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return; - - if (!alloced) { - scx_error(sch, "scx_bpf_cid_override: failed to allocate cpumask"); - return; - } - - if (scx_parent(sch)) { - scx_error(sch, "scx_bpf_cid_override() only allowed from root sched"); - return; - } - - if (cpu_to_cid__sz != nr_cpu_ids * sizeof(s32)) { - scx_error(sch, "scx_bpf_cid_override: expected %zu bytes, got %u", - nr_cpu_ids * sizeof(s32), cpu_to_cid__sz); - return; - } - - for_each_possible_cpu(cpu) { - s32 c = cpu_to_cid[cpu]; - - if (!cid_valid(sch, c)) - return; - if (cpumask_test_and_set_cpu(c, seen)) { - scx_error(sch, "cid %d assigned to multiple cpus", c); - return; - } - scx_cpu_to_cid_tbl[cpu] = c; - scx_cid_to_cpu_tbl[c] = cpu; - } - - /* Invalidate stale topo info - the override carries no topology. */ - for (cid = 0; cid < num_possible_cpus(); cid++) - scx_cid_topo[cid] = SCX_CID_TOPO_NEG; -} - -/** - * scx_bpf_cid_to_cpu - Return the raw CPU id for @cid - * @cid: cid to look up - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Return the raw CPU id for @cid. Trigger scx_error() and return -EINVAL if - * @cid is invalid. The cid<->cpu mapping is static for the lifetime of the - * loaded scheduler, so the BPF side can cache the result to avoid repeated - * kfunc invocations. - */ -__bpf_kfunc s32 scx_bpf_cid_to_cpu(s32 cid, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return -EINVAL; - return scx_cid_to_cpu(sch, cid); -} - -/** - * scx_bpf_cpu_to_cid - Return the cid for @cpu - * @cpu: cpu to look up - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Return the cid for @cpu. Trigger scx_error() and return -EINVAL if @cpu is - * invalid. The cid<->cpu mapping is static for the lifetime of the loaded - * scheduler, so the BPF side can cache the result to avoid repeated kfunc - * invocations. - */ -__bpf_kfunc s32 scx_bpf_cpu_to_cid(s32 cpu, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return -EINVAL; - return scx_cpu_to_cid(sch, cpu); -} - -/* - * Set ops on cmasks. cmask_walk_op2() shares one walk across mutating - * (and/or/copy/andnot) and predicate (subset/intersects) two-cmask forms; - * cmask_walk_op1() does the same shape over a single cmask range. Every public - * entry passes a compile-time-constant @op; cmask_walk_op{1,2}() and - * cmask_word_op{1,2}() are __always_inline so the inner switch collapses to the - * selected op and cmask_op2_is_pred() folds the predicate early-exit out of - * mutating ops. - * - * Two-cmask ops only touch @dst bits inside the intersection of the two ranges; - * bits outside stay untouched. In particular, scx_cmask_copy() does NOT zero - * @dst bits that lie outside @src's range. - * - * The _RACY variants are otherwise identical to their non-racy counterpart but - * read @src word-by-word via data_race(). Memory ordering with concurrent - * writers is the caller's responsibility. - */ -enum cmask_op2 { - /* mutating */ - CMASK_OP2_AND, - CMASK_OP2_OR, - CMASK_OP2_OR_RACY, - CMASK_OP2_COPY, - CMASK_OP2_COPY_RACY, - CMASK_OP2_ANDNOT, - /* predicates - short-circuit when the per-word result is true */ - CMASK_OP2_SUBSET, - CMASK_OP2_INTERSECTS, -}; - -static __always_inline bool cmask_op2_is_pred(const enum cmask_op2 op) -{ - return op == CMASK_OP2_SUBSET || op == CMASK_OP2_INTERSECTS; -} - -static __always_inline bool cmask_word_op2(u64 *av, const u64 *bp, u64 mask, - const enum cmask_op2 op) -{ - switch (op) { - case CMASK_OP2_AND: - *av &= ~mask | *bp; - return false; - case CMASK_OP2_OR: - *av |= *bp & mask; - return false; - case CMASK_OP2_OR_RACY: - *av |= data_race(*bp) & mask; - return false; - case CMASK_OP2_COPY: - *av = (*av & ~mask) | (*bp & mask); - return false; - case CMASK_OP2_COPY_RACY: - *av = (*av & ~mask) | (data_race(*bp) & mask); - return false; - case CMASK_OP2_ANDNOT: - *av &= ~(*bp & mask); - return false; - case CMASK_OP2_SUBSET: - /* stop on the first bit in @sub not set in @super */ - return (*bp & ~*av) & mask; - case CMASK_OP2_INTERSECTS: - return (*av & *bp) & mask; - } - unreachable(); -} - -/* - * Walk the intersection of [@a_base, @a_base + @a_nr_cids) with [@b_base, - * @b_base + @b_nr_cids) word by word, applying @op. Mutating ops walk all words - * and return false; predicates return true on the first word whose per-word - * test is true. Empty intersection returns false (matches "no bits to consider" - * for both mutate and predicate). - * - * Base/nr_cids are taken as parameters so callers with snapshotted bounds can - * drive the walk with values independent of the cmask's header. - */ -static __always_inline bool cmask_walk_op2(u64 *a_bits, u32 a_base, u32 a_nr_cids, - const u64 *b_bits, u32 b_base, u32 b_nr_cids, - const enum cmask_op2 op) -{ - u32 lo = max(a_base, b_base); - u32 hi = min(a_base + a_nr_cids, b_base + b_nr_cids); - u32 a_word_off = a_base / 64; - u32 b_word_off = b_base / 64; - u32 lo_word = lo / 64; - u32 hi_word = (hi - 1) / 64; - u64 head_mask = GENMASK_U64(63, lo & 63); - u64 tail_mask = GENMASK_U64((hi - 1) & 63, 0); - u32 w; - - if (lo >= hi) - return false; - - if (lo_word == hi_word) - return cmask_word_op2(&a_bits[lo_word - a_word_off], - &b_bits[lo_word - b_word_off], - head_mask & tail_mask, op); - - if (cmask_word_op2(&a_bits[lo_word - a_word_off], - &b_bits[lo_word - b_word_off], head_mask, op) && - cmask_op2_is_pred(op)) - return true; - - for (w = lo_word + 1; w < hi_word; w++) - if (cmask_word_op2(&a_bits[w - a_word_off], - &b_bits[w - b_word_off], ~0ULL, op) && - cmask_op2_is_pred(op)) - return true; - - return cmask_word_op2(&a_bits[hi_word - a_word_off], - &b_bits[hi_word - b_word_off], tail_mask, op); -} - -enum cmask_op1 { - CMASK_OP1_ANY_SET, -}; - -static __always_inline bool cmask_word_op1(const u64 *ap, u64 mask, - const enum cmask_op1 op) -{ - switch (op) { - case CMASK_OP1_ANY_SET: - return *ap & mask; - } - unreachable(); -} - -/* - * Walk [@a_base, @a_base + @a_nr_cids) of @a_bits word by word, applying @op. - * Returns true on the first word whose per-word test is true; returns false if - * no word matches or the range is empty. All current op1s short-circuit on - * per-word true; if a non-predicate op1 lands here, add a cmask_op1_is_pred() - * guard analogous to cmask_op2_is_pred(). - */ -static __always_inline bool cmask_walk_op1(const u64 *a_bits, u32 a_base, - u32 a_nr_cids, - const enum cmask_op1 op) -{ - u32 lo = a_base; - u32 hi = a_base + a_nr_cids; - u32 a_word_off = a_base / 64; - u32 lo_word = lo / 64; - u32 hi_word = (hi - 1) / 64; - u64 head_mask = GENMASK_U64(63, lo & 63); - u64 tail_mask = GENMASK_U64((hi - 1) & 63, 0); - u32 w; - - if (lo >= hi) - return false; - - if (lo_word == hi_word) - return cmask_word_op1(&a_bits[lo_word - a_word_off], - head_mask & tail_mask, op); - - if (cmask_word_op1(&a_bits[lo_word - a_word_off], head_mask, op)) - return true; - for (w = lo_word + 1; w < hi_word; w++) - if (cmask_word_op1(&a_bits[w - a_word_off], ~0ULL, op)) - return true; - return cmask_word_op1(&a_bits[hi_word - a_word_off], tail_mask, op); -} - -void scx_cmask_and(struct scx_cmask *dst, const struct scx_cmask *src) -{ - cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, - src->bits, src->base, src->nr_cids, CMASK_OP2_AND); -} - -void scx_cmask_or(struct scx_cmask *dst, const struct scx_cmask *src) -{ - cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, - src->bits, src->base, src->nr_cids, CMASK_OP2_OR); -} - -/** - * scx_cmask_or_racy - OR @src into @dst, reading @src without locking - * - * @src is read word-by-word through data_race(). Same per-bit independence - * rationale as scx_cmask_copy_racy(). Memory ordering with writers is the - * caller's responsibility. - */ -void scx_cmask_or_racy(struct scx_cmask *dst, const struct scx_cmask *src) -{ - cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, - src->bits, src->base, src->nr_cids, CMASK_OP2_OR_RACY); -} - -void scx_cmask_copy(struct scx_cmask *dst, const struct scx_cmask *src) -{ - cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, - src->bits, src->base, src->nr_cids, CMASK_OP2_COPY); -} - -/** - * scx_cmask_copy_racy - Snapshot @src into @dst without locking - * - * @src is read word-by-word through data_race(). Head/tail masking matches - * scx_cmask_copy(). Each bit in a cmask is independent, so partial updates - * just leave some bits fresher than others. Memory ordering with writers is - * the caller's responsibility. - */ -void scx_cmask_copy_racy(struct scx_cmask *dst, const struct scx_cmask *src) -{ - cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, - src->bits, src->base, src->nr_cids, CMASK_OP2_COPY_RACY); -} - -void scx_cmask_andnot(struct scx_cmask *dst, const struct scx_cmask *src) -{ - cmask_walk_op2(dst->bits, dst->base, dst->nr_cids, - src->bits, src->base, src->nr_cids, CMASK_OP2_ANDNOT); -} - -/* - * Return true if @cm has any bit set in [@lo, @hi). Caller must ensure - * [@lo, @hi) is contained in @cm's range. - */ -static bool cmask_any_set_in_range(const struct scx_cmask *cm, u32 lo, u32 hi) -{ - if (lo >= hi) - return false; - return cmask_walk_op1(&cm->bits[lo / 64 - cm->base / 64], lo, hi - lo, - CMASK_OP1_ANY_SET); -} - -/** - * scx_cmask_subset - test whether @sub is a subset of @super - * @sub: cmask to test - * @super: cmask to test against - * - * Return true iff every set bit of @sub is also set in @super. - */ -bool scx_cmask_subset(const struct scx_cmask *sub, const struct scx_cmask *super) -{ - u32 super_end = super->base + super->nr_cids; - u32 sub_end = sub->base + sub->nr_cids; - - /* - * Set bits in @sub outside @super's range can't be in @super, so any - * such bit means not a subset. The walk below only visits words - * common to both ranges, so these need a separate scan. - */ - if (sub->base < super->base && - cmask_any_set_in_range(sub, sub->base, min(super->base, sub_end))) - return false; - if (sub_end > super_end && - cmask_any_set_in_range(sub, max(sub->base, super_end), sub_end)) - return false; - - return !cmask_walk_op2((u64 *)super->bits, super->base, super->nr_cids, - sub->bits, sub->base, sub->nr_cids, CMASK_OP2_SUBSET); -} - -bool scx_cmask_intersects(const struct scx_cmask *a, const struct scx_cmask *b) -{ - return cmask_walk_op2((u64 *)a->bits, a->base, a->nr_cids, - b->bits, b->base, b->nr_cids, CMASK_OP2_INTERSECTS); -} - -/** - * scx_cmask_empty - Test whether @m has no bits set - * @m: cmask to test - * - * Return true iff @m's active range has no bits set. - */ -bool scx_cmask_empty(const struct scx_cmask *m) -{ - return !cmask_any_set_in_range(m, m->base, m->base + m->nr_cids); -} - -/** - * scx_bpf_cid_topo - Copy out per-cid topology info - * @cid: cid to look up - * @out__uninit: where to copy the topology info; fully written by this call - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Fill @out__uninit with the topology info for @cid. Trigger scx_error() if - * @cid is out of range. If @cid is valid but in the no-topo section, all fields - * are set to -1. - */ -__bpf_kfunc void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out__uninit, - const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch) || !cid_valid(sch, cid)) { - *out__uninit = SCX_CID_TOPO_NEG; - return; - } - - *out__uninit = READ_ONCE(scx_cid_topo)[cid]; -} - -__bpf_kfunc_end_defs(); - -BTF_KFUNCS_START(scx_kfunc_ids_init) -BTF_ID_FLAGS(func, scx_bpf_cid_override, KF_IMPLICIT_ARGS | KF_SLEEPABLE) -BTF_KFUNCS_END(scx_kfunc_ids_init) - -static const struct btf_kfunc_id_set scx_kfunc_set_init = { - .owner = THIS_MODULE, - .set = &scx_kfunc_ids_init, - .filter = scx_kfunc_context_filter, -}; - -BTF_KFUNCS_START(scx_kfunc_ids_cid) -BTF_ID_FLAGS(func, scx_bpf_cid_to_cpu, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_cpu_to_cid, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_cid_topo, KF_IMPLICIT_ARGS) -BTF_KFUNCS_END(scx_kfunc_ids_cid) - -static const struct btf_kfunc_id_set scx_kfunc_set_cid = { - .owner = THIS_MODULE, - .set = &scx_kfunc_ids_cid, -}; - -int scx_cid_kfunc_init(void) -{ - return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_init) ?: - register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_cid) ?: - register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &scx_kfunc_set_cid) ?: - register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_cid); -} diff --git a/kernel/sched/ext_cid.h b/kernel/sched/ext_cid.h deleted file mode 100644 index 5745e5785e89..000000000000 --- a/kernel/sched/ext_cid.h +++ /dev/null @@ -1,271 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Topological CPU IDs (cids) - * -------------------------- - * - * Raw cpu numbers are clumsy for sharding work and communication across - * topology units, especially from BPF: the space can be sparse, numerical - * closeness doesn't imply topological closeness (x86 hyperthreading often puts - * SMT siblings far apart), and a range of cpu ids doesn't mean anything. - * Sub-scheds make this acute - cpu allocation, revocation and other state are - * constantly communicated across sub-scheds, and passing whole cpumasks scales - * poorly with cpu count. cpumasks are also awkward in BPF: a variable-length - * kernel type sized for the maximum NR_CPUS (4k), with verbose helper sequences - * for every op. - * - * cids give every cpu a dense, topology-ordered id. CPUs sharing a core, LLC or - * NUMA node get contiguous cid ranges, so a topology unit becomes a (start, - * length) slice of cid space. Communication can pass a slice instead of a - * cpumask, and BPF code can process, for example, a u64 word's worth of cids at - * a time. - * - * The mapping is built once at root scheduler enable time by walking the - * topology of online cpus only. Going by online cpus is out of necessity: - * depending on the arch, topology info isn't reliably available for offline - * cpus. The expected usage model is restarting the scheduler on hotplug events - * so the mapping is rebuilt against the new online set. A scheduler that wants - * to handle hotplug without a restart can provide its own cid and shard mapping - * through the override interface. - * - * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. - * Copyright (c) 2026 Tejun Heo - */ -#ifndef _KERNEL_SCHED_EXT_CID_H -#define _KERNEL_SCHED_EXT_CID_H - -struct scx_sched; - -/* - * Cid space (total is always num_possible_cpus()) is laid out with - * topology-annotated cids first, then no-topo cids at the tail. The - * topology-annotated block covers the cpus that were online when scx_cid_init() - * ran and remains valid even after those cpus go offline. The tail block covers - * possible-but-not-online cpus and carries all-(-1) topo info (see - * scx_cid_topo); callers detect it via the -1 sentinels. - * - * See the comment above the table definitions in ext_cid.c for the - * memory-ordering and visibility contract. - */ -extern s16 *scx_cid_to_cpu_tbl; -extern s16 *scx_cpu_to_cid_tbl; -extern struct scx_cid_topo *scx_cid_topo; -extern struct btf_id_set8 scx_kfunc_ids_init; - -void scx_cmask_clear(struct scx_cmask *m); -void scx_cmask_fill(struct scx_cmask *m); -void scx_cmask_and(struct scx_cmask *dst, const struct scx_cmask *src); -void scx_cmask_or(struct scx_cmask *dst, const struct scx_cmask *src); -void scx_cmask_or_racy(struct scx_cmask *dst, const struct scx_cmask *src); -void scx_cmask_copy(struct scx_cmask *dst, const struct scx_cmask *src); -void scx_cmask_copy_racy(struct scx_cmask *dst, const struct scx_cmask *src); -void scx_cmask_andnot(struct scx_cmask *dst, const struct scx_cmask *src); -bool scx_cmask_subset(const struct scx_cmask *sub, const struct scx_cmask *super); -bool scx_cmask_intersects(const struct scx_cmask *a, const struct scx_cmask *b); -bool scx_cmask_empty(const struct scx_cmask *m); -s32 scx_cid_init(struct scx_sched *sch); -int scx_cid_kfunc_init(void); -void scx_cpumask_to_cmask(const struct cpumask *src, struct scx_cmask *dst); - -/** - * cid_valid - Verify a cid value, to be used on ops input args - * @sch: scx_sched to abort on error - * @cid: cid which came from a BPF ops - * - * Return true if @cid is in [0, num_possible_cpus()). On failure, trigger - * scx_error() and return false. - */ -static inline bool cid_valid(struct scx_sched *sch, s32 cid) -{ - if (likely(cid >= 0 && cid < num_possible_cpus())) - return true; - scx_error(sch, "invalid cid %d", cid); - return false; -} - -/** - * __scx_cid_to_cpu - Unchecked cid->cpu table lookup - * @cid: cid to look up. Must be in [0, num_possible_cpus()). - * - * Intended for callsites that have already validated @cid and that hold a - * non-NULL @sch from scx_prog_sched() - a live sched implies the table has - * been allocated, so no NULL check is needed here. - */ -static inline s32 __scx_cid_to_cpu(s32 cid) -{ - /* READ_ONCE pairs with WRITE_ONCE in scx_cid_arrays_alloc() */ - return READ_ONCE(scx_cid_to_cpu_tbl)[cid]; -} - -/** - * __scx_cpu_to_cid - Unchecked cpu->cid table lookup - * @cpu: cpu to look up. Must be a valid possible cpu id. - * - * Same usage constraints as __scx_cid_to_cpu(). - */ -static inline s32 __scx_cpu_to_cid(s32 cpu) -{ - return READ_ONCE(scx_cpu_to_cid_tbl)[cpu]; -} - -/** - * scx_cid_to_cpu - Translate @cid to its cpu - * @sch: scx_sched for error reporting - * @cid: cid to look up - * - * Return the cpu for @cid or a negative errno on failure. Invalid cid triggers - * scx_error() on @sch. The cid arrays are allocated on first scheduler enable - * and never freed, so the returned cpu is stable for the lifetime of the loaded - * scheduler. - */ -static inline s32 scx_cid_to_cpu(struct scx_sched *sch, s32 cid) -{ - if (!cid_valid(sch, cid)) - return -EINVAL; - return __scx_cid_to_cpu(cid); -} - -/** - * scx_cpu_to_cid - Translate @cpu to its cid - * @sch: scx_sched for error reporting - * @cpu: cpu to look up - * - * Return the cid for @cpu or a negative errno on failure. Invalid cpu triggers - * scx_error() on @sch. Same lifetime guarantee as scx_cid_to_cpu(). - */ -static inline s32 scx_cpu_to_cid(struct scx_sched *sch, s32 cpu) -{ - if (!scx_cpu_valid(sch, cpu, NULL)) - return -EINVAL; - return __scx_cpu_to_cid(cpu); -} - -/** - * scx_is_cid_type - Test whether the active scheduler hierarchy is cid-form - */ -static inline bool scx_is_cid_type(void) -{ - return static_branch_unlikely(&__scx_is_cid_type); -} - -static inline bool __scx_cmask_contains(u32 cid, const struct scx_cmask *m) -{ - return likely(cid >= m->base && cid < m->base + m->nr_cids); -} - -/* Word in bits[] covering @cid. @cid must satisfy __scx_cmask_contains(). */ -static inline u64 *__scx_cmask_word(u32 cid, const struct scx_cmask *m) -{ - return (u64 *)&m->bits[cid / 64 - m->base / 64]; -} - -/** - * __scx_cmask_init - Initialize @m with explicit storage capacity - * @m: cmask to initialize - * @base: first cid of the active range - * @nr_cids: number of cids in the active range - * @alloc_cids: storage capacity in cids, at least @nr_cids - * - * Use when storage is sized larger than the initial active range. All of - * bits[] is zeroed. - */ -static inline void __scx_cmask_init(struct scx_cmask *m, u32 base, u32 nr_cids, - u32 alloc_cids) -{ - if (WARN_ON_ONCE(alloc_cids < nr_cids)) - nr_cids = alloc_cids; - - m->base = base; - m->nr_cids = nr_cids; - m->alloc_words = SCX_CMASK_NR_WORDS(alloc_cids); - memset(m->bits, 0, m->alloc_words * sizeof(u64)); -} - -/** - * scx_cmask_init - Initialize @m on tight storage - * @m: cmask to initialize - * @base: first cid of the active range - * @nr_cids: number of cids in the active range - * - * All of bits[] is zeroed. - */ -static inline void scx_cmask_init(struct scx_cmask *m, u32 base, u32 nr_cids) -{ - __scx_cmask_init(m, base, nr_cids, nr_cids); -} - -/** - * scx_cmask_reframe - Reshape @m's active range without resizing storage - * @m: cmask to reframe - * @base: new active range base - * @nr_cids: new active range length, must fit within @m->alloc_words - * - * Body bits within the new range become garbage - only the head and tail - * words are zeroed to keep the padding invariant. - */ -static inline void scx_cmask_reframe(struct scx_cmask *m, u32 base, u32 nr_cids) -{ - if (WARN_ON_ONCE(SCX_CMASK_NR_WORDS(nr_cids) > m->alloc_words)) - return; - - if (nr_cids) { - u32 last_word = ((base & 63) + nr_cids - 1) / 64; - - m->bits[0] = 0; - m->bits[last_word] = 0; - } - - m->base = base; - m->nr_cids = nr_cids; -} - -static inline void __scx_cmask_set(u32 cid, struct scx_cmask *m) -{ - if (!__scx_cmask_contains(cid, m)) - return; - *__scx_cmask_word(cid, m) |= BIT_U64(cid & 63); -} - -/** - * scx_cmask_test - test whether @cid is set in @m - * @cid: cid to test - * @m: cmask to test - * - * Return %false if @cid is outside @m's active range. Otherwise return the - * bit's value. Read via READ_ONCE so callers can race set/clear writers. - */ -static inline bool scx_cmask_test(u32 cid, const struct scx_cmask *m) -{ - if (!__scx_cmask_contains(cid, m)) - return false; - return READ_ONCE(*__scx_cmask_word(cid, m)) & BIT_U64(cid & 63); -} - -/* - * Words of bits[] the active range spans, 0 if empty. Tighter than the storage - * SCX_CMASK_NR_WORDS() sizes for the worst-case base alignment. - */ -static inline u32 scx_cmask_nr_used_words(const struct scx_cmask *m) -{ - if (!m->nr_cids) - return 0; - return ((m->base & 63) + m->nr_cids - 1) / 64 + 1; -} - -/** - * scx_cmask_for_each_cid - iterate set cids in @m - * @cid: s32 loop var that receives each set cid in turn - * @m: cmask to iterate - * - * Visits set bits within @m's active range in ascending order. Scans only the - * words the active range spans, where head and tail padding is kept zero, so - * no per-cid range check is needed. - */ -#define scx_cmask_for_each_cid(cid, m) \ - for (u64 __bs = (m)->base & ~63u, __wi = 0, \ - __nw = scx_cmask_nr_used_words(m); \ - __wi < __nw; __wi++) \ - for (u64 __w = READ_ONCE((m)->bits[__wi]); \ - __w && ((cid) = __bs + __wi * 64 + __ffs64(__w), true); \ - __w &= __w - 1) - -#endif /* _KERNEL_SCHED_EXT_CID_H */ diff --git a/kernel/sched/ext_idle.c b/kernel/sched/ext_idle.c deleted file mode 100644 index 2077373d8da3..000000000000 --- a/kernel/sched/ext_idle.c +++ /dev/null @@ -1,1511 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst - * - * Built-in idle CPU tracking policy. - * - * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. - * Copyright (c) 2022 Tejun Heo - * Copyright (c) 2022 David Vernet - * Copyright (c) 2024 Andrea Righi - */ - -/* Enable/disable built-in idle CPU selection policy */ -static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); - -/* Enable/disable per-node idle cpumasks */ -static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_per_node); - -/* Enable/disable LLC aware optimizations */ -static DEFINE_STATIC_KEY_FALSE(scx_selcpu_topo_llc); - -/* Enable/disable NUMA aware optimizations */ -static DEFINE_STATIC_KEY_FALSE(scx_selcpu_topo_numa); - -/* - * cpumasks to track idle CPUs within each NUMA node. - * - * If SCX_OPS_BUILTIN_IDLE_PER_NODE is not enabled, a single global cpumask - * from is used to track all the idle CPUs in the system. - */ -struct scx_idle_cpus { - cpumask_var_t cpu; - cpumask_var_t smt; -}; - -/* - * Global host-wide idle cpumasks (used when SCX_OPS_BUILTIN_IDLE_PER_NODE - * is not enabled). - */ -static struct scx_idle_cpus scx_idle_global_masks; - -/* - * Per-node idle cpumasks. - */ -static struct scx_idle_cpus **scx_idle_node_masks; - -/* - * Local per-CPU cpumasks (used to generate temporary idle cpumasks). - */ -static DEFINE_PER_CPU(cpumask_var_t, local_idle_cpumask); -static DEFINE_PER_CPU(cpumask_var_t, local_llc_idle_cpumask); -static DEFINE_PER_CPU(cpumask_var_t, local_numa_idle_cpumask); - -/* - * Return the idle masks associated to a target @node. - * - * NUMA_NO_NODE identifies the global idle cpumask. - */ -static struct scx_idle_cpus *idle_cpumask(int node) -{ - return node == NUMA_NO_NODE ? &scx_idle_global_masks : scx_idle_node_masks[node]; -} - -/* - * Returns the NUMA node ID associated with a @cpu, or NUMA_NO_NODE if - * per-node idle cpumasks are disabled. - */ -static int scx_cpu_node_if_enabled(int cpu) -{ - if (!static_branch_maybe(CONFIG_NUMA, &scx_builtin_idle_per_node)) - return NUMA_NO_NODE; - - return cpu_to_node(cpu); -} - -static bool scx_idle_test_and_clear_cpu(int cpu) -{ - int node = scx_cpu_node_if_enabled(cpu); - struct cpumask *idle_cpus = idle_cpumask(node)->cpu; - - /* - * SMT mask should be cleared whether we can claim @cpu or not. The SMT - * cluster is not wholly idle either way. This also prevents - * scx_pick_idle_cpu() from getting caught in an infinite loop. - */ - if (sched_smt_active()) { - const struct cpumask *smt = cpu_smt_mask(cpu); - struct cpumask *idle_smts = idle_cpumask(node)->smt; - - /* - * If offline, @cpu is not its own sibling and - * scx_pick_idle_cpu() can get caught in an infinite loop as - * @cpu is never cleared from the idle SMT mask. Ensure that - * @cpu is eventually cleared. - * - * NOTE: Use cpumask_intersects() and cpumask_test_cpu() to - * reduce memory writes, which may help alleviate cache - * coherence pressure. - */ - if (cpumask_intersects(smt, idle_smts)) - cpumask_andnot(idle_smts, idle_smts, smt); - else if (cpumask_test_cpu(cpu, idle_smts)) - __cpumask_clear_cpu(cpu, idle_smts); - } - - return cpumask_test_and_clear_cpu(cpu, idle_cpus); -} - -/* - * Pick an idle CPU in a specific NUMA node. - */ -static s32 pick_idle_cpu_in_node(const struct cpumask *cpus_allowed, int node, u64 flags) -{ - int cpu; - -retry: - if (sched_smt_active()) { - cpu = cpumask_any_and_distribute(idle_cpumask(node)->smt, cpus_allowed); - if (cpu < nr_cpu_ids) - goto found; - - if (flags & SCX_PICK_IDLE_CORE) - return -EBUSY; - } - - cpu = cpumask_any_and_distribute(idle_cpumask(node)->cpu, cpus_allowed); - if (cpu >= nr_cpu_ids) - return -EBUSY; - -found: - if (scx_idle_test_and_clear_cpu(cpu)) - return cpu; - else - goto retry; -} - -#ifdef CONFIG_NUMA -/* - * Tracks nodes that have not yet been visited when searching for an idle - * CPU across all available nodes. - */ -static DEFINE_PER_CPU(nodemask_t, per_cpu_unvisited); - -/* - * Search for an idle CPU across all nodes, excluding @node. - */ -static s32 pick_idle_cpu_from_online_nodes(const struct cpumask *cpus_allowed, int node, u64 flags) -{ - nodemask_t *unvisited; - s32 cpu = -EBUSY; - - preempt_disable(); - unvisited = this_cpu_ptr(&per_cpu_unvisited); - - /* - * Restrict the search to the online nodes (excluding the current - * node that has been visited already). - */ - nodes_copy(*unvisited, node_states[N_ONLINE]); - node_clear(node, *unvisited); - - /* - * Traverse all nodes in order of increasing distance, starting - * from @node. - * - * This loop is O(N^2), with N being the amount of NUMA nodes, - * which might be quite expensive in large NUMA systems. However, - * this complexity comes into play only when a scheduler enables - * SCX_OPS_BUILTIN_IDLE_PER_NODE and it's requesting an idle CPU - * without specifying a target NUMA node, so it shouldn't be a - * bottleneck is most cases. - * - * As a future optimization we may want to cache the list of nodes - * in a per-node array, instead of actually traversing them every - * time. - */ - for_each_node_numadist(node, *unvisited) { - cpu = pick_idle_cpu_in_node(cpus_allowed, node, flags); - if (cpu >= 0) - break; - } - preempt_enable(); - - return cpu; -} -#else -static inline s32 -pick_idle_cpu_from_online_nodes(const struct cpumask *cpus_allowed, int node, u64 flags) -{ - return -EBUSY; -} -#endif - -/* - * Find an idle CPU in the system, starting from @node. - */ -static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed, int node, u64 flags) -{ - s32 cpu; - - /* - * Always search in the starting node first (this is an - * optimization that can save some cycles even when the search is - * not limited to a single node). - */ - cpu = pick_idle_cpu_in_node(cpus_allowed, node, flags); - if (cpu >= 0) - return cpu; - - /* - * Stop the search if we are using only a single global cpumask - * (NUMA_NO_NODE) or if the search is restricted to the first node - * only. - */ - if (node == NUMA_NO_NODE || flags & SCX_PICK_IDLE_IN_NODE) - return -EBUSY; - - /* - * Extend the search to the other online nodes. - */ - return pick_idle_cpu_from_online_nodes(cpus_allowed, node, flags); -} - -/* - * Return the amount of CPUs in the same LLC domain of @cpu (or zero if the LLC - * domain is not defined). - */ -static unsigned int llc_weight(s32 cpu) -{ - struct sched_domain *sd; - - sd = rcu_dereference(per_cpu(sd_llc, cpu)); - if (!sd) - return 0; - - return sd->span_weight; -} - -/* - * Return the cpumask representing the LLC domain of @cpu (or NULL if the LLC - * domain is not defined). - */ -static struct cpumask *llc_span(s32 cpu) -{ - struct sched_domain *sd; - - sd = rcu_dereference(per_cpu(sd_llc, cpu)); - if (!sd) - return NULL; - - return sched_domain_span(sd); -} - -/* - * Return the amount of CPUs in the same NUMA domain of @cpu (or zero if the - * NUMA domain is not defined). - */ -static unsigned int numa_weight(s32 cpu) -{ - struct sched_domain *sd; - struct sched_group *sg; - - sd = rcu_dereference(per_cpu(sd_numa, cpu)); - if (!sd) - return 0; - sg = sd->groups; - if (!sg) - return 0; - - return sg->group_weight; -} - -/* - * Return the cpumask representing the NUMA domain of @cpu (or NULL if the NUMA - * domain is not defined). - */ -static struct cpumask *numa_span(s32 cpu) -{ - struct sched_domain *sd; - struct sched_group *sg; - - sd = rcu_dereference(per_cpu(sd_numa, cpu)); - if (!sd) - return NULL; - sg = sd->groups; - if (!sg) - return NULL; - - return sched_group_span(sg); -} - -/* - * Return true if the LLC domains do not perfectly overlap with the NUMA - * domains, false otherwise. - */ -static bool llc_numa_mismatch(void) -{ - int cpu; - - /* - * We need to scan all online CPUs to verify whether their scheduling - * domains overlap. - * - * While it is rare to encounter architectures with asymmetric NUMA - * topologies, CPU hotplugging or virtualized environments can result - * in asymmetric configurations. - * - * For example: - * - * NUMA 0: - * - LLC 0: cpu0..cpu7 - * - LLC 1: cpu8..cpu15 [offline] - * - * NUMA 1: - * - LLC 0: cpu16..cpu23 - * - LLC 1: cpu24..cpu31 - * - * In this case, if we only check the first online CPU (cpu0), we might - * incorrectly assume that the LLC and NUMA domains are fully - * overlapping, which is incorrect (as NUMA 1 has two distinct LLC - * domains). - */ - for_each_online_cpu(cpu) - if (llc_weight(cpu) != numa_weight(cpu)) - return true; - - return false; -} - -/* - * Initialize topology-aware scheduling. - * - * Detect if the system has multiple LLC or multiple NUMA domains and enable - * cache-aware / NUMA-aware scheduling optimizations in the default CPU idle - * selection policy. - * - * Assumption: the kernel's internal topology representation assumes that each - * CPU belongs to a single LLC domain, and that each LLC domain is entirely - * contained within a single NUMA node. - */ -void scx_idle_update_selcpu_topology(struct sched_ext_ops *ops) -{ - bool enable_llc = false, enable_numa = false; - unsigned int nr_cpus; - s32 cpu = cpumask_first(cpu_online_mask); - - /* - * Enable LLC domain optimization only when there are multiple LLC - * domains among the online CPUs. If all online CPUs are part of a - * single LLC domain, the idle CPU selection logic can choose any - * online CPU without bias. - * - * Note that it is sufficient to check the LLC domain of the first - * online CPU to determine whether a single LLC domain includes all - * CPUs. - */ - rcu_read_lock(); - nr_cpus = llc_weight(cpu); - if (nr_cpus > 0) { - if (nr_cpus < num_online_cpus()) - enable_llc = true; - pr_debug("sched_ext: LLC=%*pb weight=%u\n", - cpumask_pr_args(llc_span(cpu)), llc_weight(cpu)); - } - - /* - * Enable NUMA optimization only when there are multiple NUMA domains - * among the online CPUs and the NUMA domains don't perfectly overlap - * with the LLC domains. - * - * If all CPUs belong to the same NUMA node and the same LLC domain, - * enabling both NUMA and LLC optimizations is unnecessary, as checking - * for an idle CPU in the same domain twice is redundant. - * - * If SCX_OPS_BUILTIN_IDLE_PER_NODE is enabled ignore the NUMA - * optimization, as we would naturally select idle CPUs within - * specific NUMA nodes querying the corresponding per-node cpumask. - */ - if (!(ops->flags & SCX_OPS_BUILTIN_IDLE_PER_NODE)) { - nr_cpus = numa_weight(cpu); - if (nr_cpus > 0) { - if (nr_cpus < num_online_cpus() && llc_numa_mismatch()) - enable_numa = true; - pr_debug("sched_ext: NUMA=%*pb weight=%u\n", - cpumask_pr_args(numa_span(cpu)), nr_cpus); - } - } - rcu_read_unlock(); - - pr_debug("sched_ext: LLC idle selection %s\n", - str_enabled_disabled(enable_llc)); - pr_debug("sched_ext: NUMA idle selection %s\n", - str_enabled_disabled(enable_numa)); - - if (enable_llc) - static_branch_enable_cpuslocked(&scx_selcpu_topo_llc); - else - static_branch_disable_cpuslocked(&scx_selcpu_topo_llc); - if (enable_numa) - static_branch_enable_cpuslocked(&scx_selcpu_topo_numa); - else - static_branch_disable_cpuslocked(&scx_selcpu_topo_numa); -} - -/* - * Return true if @p can run on all possible CPUs, false otherwise. - */ -static inline bool task_affinity_all(const struct task_struct *p) -{ - return p->nr_cpus_allowed >= num_possible_cpus(); -} - -/* - * Built-in CPU idle selection policy: - * - * 1. Prioritize full-idle cores: - * - always prioritize CPUs from fully idle cores (both logical CPUs are - * idle) to avoid interference caused by SMT. - * - * 2. Reuse the same CPU: - * - prefer the last used CPU to take advantage of cached data (L1, L2) and - * branch prediction optimizations. - * - * 3. Prefer @prev_cpu's SMT sibling: - * - if @prev_cpu is busy and no fully idle core is available, try to - * place the task on an idle SMT sibling of @prev_cpu; keeping the - * task on the same core makes migration cheaper, preserves L1 cache - * locality and reduces wakeup latency. - * - * 4. Pick a CPU within the same LLC (Last-Level Cache): - * - if the above conditions aren't met, pick a CPU that shares the same - * LLC, if the LLC domain is a subset of @cpus_allowed, to maintain - * cache locality. - * - * 5. Pick a CPU within the same NUMA node, if enabled: - * - choose a CPU from the same NUMA node, if the node cpumask is a - * subset of @cpus_allowed, to reduce memory access latency. - * - * 6. Pick any idle CPU within the @cpus_allowed domain. - * - * Step 4 and 5 are performed only if the system has, respectively, - * multiple LLCs / multiple NUMA nodes (see scx_selcpu_topo_llc and - * scx_selcpu_topo_numa) and they don't contain the same subset of CPUs. - * - * If %SCX_OPS_BUILTIN_IDLE_PER_NODE is enabled, the search will always - * begin in @prev_cpu's node and proceed to other nodes in order of - * increasing distance. - * - * Return the picked CPU if idle, or a negative value otherwise. - * - * NOTE: tasks that can only run on 1 CPU are excluded by this logic, because - * we never call ops.select_cpu() for them, see select_task_rq(). - */ -s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, - const struct cpumask *cpus_allowed, u64 flags) -{ - const struct cpumask *llc_cpus = NULL, *numa_cpus = NULL; - const struct cpumask *allowed = cpus_allowed ?: p->cpus_ptr; - int node = scx_cpu_node_if_enabled(prev_cpu); - bool is_prev_allowed; - s32 cpu; - - preempt_disable(); - - /* - * Determine the subset of CPUs usable by @p within @cpus_allowed. - */ - if (allowed != p->cpus_ptr) { - struct cpumask *local_cpus = this_cpu_cpumask_var_ptr(local_idle_cpumask); - - if (task_affinity_all(p)) { - allowed = cpus_allowed; - } else if (cpumask_and(local_cpus, cpus_allowed, p->cpus_ptr)) { - allowed = local_cpus; - } else { - cpu = -EBUSY; - goto out_enable; - } - } - - /* - * Check whether @prev_cpu is still within the allowed set. If not, - * we can still try selecting a nearby CPU. - */ - is_prev_allowed = cpumask_test_cpu(prev_cpu, allowed); - - /* - * This is necessary to protect llc_cpus. - */ - rcu_read_lock(); - - /* - * Determine the subset of CPUs that the task can use in its - * current LLC and node. - * - * If the task can run on all CPUs, use the node and LLC cpumasks - * directly. - */ - if (static_branch_maybe(CONFIG_NUMA, &scx_selcpu_topo_numa)) { - struct cpumask *local_cpus = this_cpu_cpumask_var_ptr(local_numa_idle_cpumask); - const struct cpumask *cpus = numa_span(prev_cpu); - - if (allowed == p->cpus_ptr && task_affinity_all(p)) - numa_cpus = cpus; - else if (cpus && cpumask_and(local_cpus, allowed, cpus)) - numa_cpus = local_cpus; - } - - if (static_branch_maybe(CONFIG_SCHED_MC, &scx_selcpu_topo_llc)) { - struct cpumask *local_cpus = this_cpu_cpumask_var_ptr(local_llc_idle_cpumask); - const struct cpumask *cpus = llc_span(prev_cpu); - - if (allowed == p->cpus_ptr && task_affinity_all(p)) - llc_cpus = cpus; - else if (cpus && cpumask_and(local_cpus, allowed, cpus)) - llc_cpus = local_cpus; - } - - /* - * If WAKE_SYNC, try to migrate the wakee to the waker's CPU. - */ - if (wake_flags & SCX_WAKE_SYNC) { - int waker_node; - - /* - * If the waker's CPU is cache affine and prev_cpu is idle, - * then avoid a migration. - */ - cpu = smp_processor_id(); - if (is_prev_allowed && cpus_share_cache(cpu, prev_cpu) && - scx_idle_test_and_clear_cpu(prev_cpu)) { - cpu = prev_cpu; - goto out_unlock; - } - - /* - * If the waker's local DSQ is empty, and the system is under - * utilized, try to wake up @p to the local DSQ of the waker. - * - * Checking only for an empty local DSQ is insufficient as it - * could give the wakee an unfair advantage when the system is - * oversaturated. - * - * Checking only for the presence of idle CPUs is also - * insufficient as the local DSQ of the waker could have tasks - * piled up on it even if there is an idle core elsewhere on - * the system. - */ - waker_node = scx_cpu_node_if_enabled(cpu); - if (!(current->flags & PF_EXITING) && - cpu_rq(cpu)->scx.local_dsq.nr == 0 && - (!(flags & SCX_PICK_IDLE_IN_NODE) || (waker_node == node)) && - !cpumask_empty(idle_cpumask(waker_node)->cpu)) { - if (cpumask_test_cpu(cpu, allowed)) - goto out_unlock; - } - } - - /* - * If CPU has SMT, any wholly idle CPU is likely a better pick than - * partially idle @prev_cpu. - */ - if (sched_smt_active()) { - /* - * Keep using @prev_cpu if it's part of a fully idle core. - */ - if (is_prev_allowed && - cpumask_test_cpu(prev_cpu, idle_cpumask(node)->smt) && - scx_idle_test_and_clear_cpu(prev_cpu)) { - cpu = prev_cpu; - goto out_unlock; - } - - /* - * Search for any fully idle core in the same LLC domain. - */ - if (llc_cpus) { - cpu = pick_idle_cpu_in_node(llc_cpus, node, SCX_PICK_IDLE_CORE); - if (cpu >= 0) - goto out_unlock; - } - - /* - * Search for any fully idle core in the same NUMA node. - */ - if (numa_cpus) { - cpu = pick_idle_cpu_in_node(numa_cpus, node, SCX_PICK_IDLE_CORE); - if (cpu >= 0) - goto out_unlock; - } - - /* - * Search for any full-idle core usable by the task. - * - * If the node-aware idle CPU selection policy is enabled - * (%SCX_OPS_BUILTIN_IDLE_PER_NODE), the search will always - * begin in prev_cpu's node and proceed to other nodes in - * order of increasing distance. - */ - cpu = scx_pick_idle_cpu(allowed, node, flags | SCX_PICK_IDLE_CORE); - if (cpu >= 0) - goto out_unlock; - - /* - * Give up if we're strictly looking for a full-idle SMT - * core. - */ - if (flags & SCX_PICK_IDLE_CORE) { - cpu = -EBUSY; - goto out_unlock; - } - } - - /* - * Use @prev_cpu if it's idle. - */ - if (is_prev_allowed && scx_idle_test_and_clear_cpu(prev_cpu)) { - cpu = prev_cpu; - goto out_unlock; - } - - /* - * Use @prev_cpu's sibling if it's idle. - */ - if (sched_smt_active()) { - for_each_cpu_and(cpu, cpu_smt_mask(prev_cpu), allowed) { - if (cpu == prev_cpu) - continue; - if (scx_idle_test_and_clear_cpu(cpu)) - goto out_unlock; - } - } - - /* - * Search for any idle CPU in the same LLC domain. - */ - if (llc_cpus) { - cpu = pick_idle_cpu_in_node(llc_cpus, node, 0); - if (cpu >= 0) - goto out_unlock; - } - - /* - * Search for any idle CPU in the same NUMA node. - */ - if (numa_cpus) { - cpu = pick_idle_cpu_in_node(numa_cpus, node, 0); - if (cpu >= 0) - goto out_unlock; - } - - /* - * Search for any idle CPU usable by the task. - * - * If the node-aware idle CPU selection policy is enabled - * (%SCX_OPS_BUILTIN_IDLE_PER_NODE), the search will always begin - * in prev_cpu's node and proceed to other nodes in order of - * increasing distance. - */ - cpu = scx_pick_idle_cpu(allowed, node, flags); - -out_unlock: - rcu_read_unlock(); -out_enable: - preempt_enable(); - - return cpu; -} - -/* - * Initialize global and per-node idle cpumasks. - */ -void scx_idle_init_masks(void) -{ - int i; - - /* Allocate global idle cpumasks */ - BUG_ON(!alloc_cpumask_var(&scx_idle_global_masks.cpu, GFP_KERNEL)); - BUG_ON(!alloc_cpumask_var(&scx_idle_global_masks.smt, GFP_KERNEL)); - - /* Allocate per-node idle cpumasks (use nr_node_ids for non-contiguous NUMA nodes) */ - scx_idle_node_masks = kzalloc_objs(*scx_idle_node_masks, nr_node_ids); - BUG_ON(!scx_idle_node_masks); - - for_each_node(i) { - scx_idle_node_masks[i] = kzalloc_node(sizeof(**scx_idle_node_masks), - GFP_KERNEL, i); - BUG_ON(!scx_idle_node_masks[i]); - - BUG_ON(!alloc_cpumask_var_node(&scx_idle_node_masks[i]->cpu, GFP_KERNEL, i)); - BUG_ON(!alloc_cpumask_var_node(&scx_idle_node_masks[i]->smt, GFP_KERNEL, i)); - } - - /* Allocate local per-cpu idle cpumasks */ - for_each_possible_cpu(i) { - BUG_ON(!alloc_cpumask_var_node(&per_cpu(local_idle_cpumask, i), - GFP_KERNEL, cpu_to_node(i))); - BUG_ON(!alloc_cpumask_var_node(&per_cpu(local_llc_idle_cpumask, i), - GFP_KERNEL, cpu_to_node(i))); - BUG_ON(!alloc_cpumask_var_node(&per_cpu(local_numa_idle_cpumask, i), - GFP_KERNEL, cpu_to_node(i))); - } -} - -static void update_builtin_idle(int cpu, bool idle) -{ - int node = scx_cpu_node_if_enabled(cpu); - struct cpumask *idle_cpus = idle_cpumask(node)->cpu; - - assign_cpu(cpu, idle_cpus, idle); - - if (sched_smt_active()) { - const struct cpumask *smt = cpu_smt_mask(cpu); - struct cpumask *idle_smts = idle_cpumask(node)->smt; - - if (idle) { - /* - * idle_smt handling is racy but that's fine as it's - * only for optimization and self-correcting. - */ - if (!cpumask_subset(smt, idle_cpus)) - return; - cpumask_or(idle_smts, idle_smts, smt); - } else { - cpumask_andnot(idle_smts, idle_smts, smt); - } - } -} - -/* - * Update the idle state of a CPU to @idle. - * - * If @do_notify is true, ops.update_idle() is invoked to notify the scx - * scheduler of an actual idle state transition (idle to busy or vice - * versa). If @do_notify is false, only the idle state in the idle masks is - * refreshed without invoking ops.update_idle(). - * - * This distinction is necessary, because an idle CPU can be "reserved" and - * awakened via scx_bpf_pick_idle_cpu() + scx_bpf_kick_cpu(), marking it as - * busy even if no tasks are dispatched. In this case, the CPU may return - * to idle without a true state transition. Refreshing the idle masks - * without invoking ops.update_idle() ensures accurate idle state tracking - * while avoiding unnecessary updates and maintaining balanced state - * transitions. - */ -void __scx_update_idle(struct rq *rq, bool idle, bool do_notify) -{ - struct scx_sched *sch = scx_root; - int cpu = cpu_of(rq); - - lockdep_assert_rq_held(rq); - - /* - * Update the idle masks: - * - for real idle transitions (do_notify == true) - * - for idle-to-idle transitions (indicated by the previous task - * being the idle thread, managed by pick_task_idle()) - * - * Skip updating idle masks if the previous task is not the idle - * thread, since set_next_task_idle() has already handled it when - * transitioning from a task to the idle thread (calling this - * function with do_notify == true). - * - * In this way we can avoid updating the idle masks twice, - * unnecessarily. - */ - if (static_branch_likely(&scx_builtin_idle_enabled)) - if (do_notify || is_idle_task(rq->curr)) - update_builtin_idle(cpu, idle); - - /* - * Trigger ops.update_idle() only when transitioning from a task to - * the idle thread and vice versa. - * - * Idle transitions are indicated by do_notify being set to true, - * managed by put_prev_task_idle()/set_next_task_idle(). - * - * This must come after builtin idle update so that BPF schedulers can - * create interlocking between ops.update_idle() and ops.enqueue() - - * either enqueue() sees the idle bit or update_idle() sees the task - * that enqueue() queued. - */ - if (SCX_HAS_OP(sch, update_idle) && do_notify && - !scx_bypassing(sch, cpu_of(rq))) - SCX_CALL_OP(sch, update_idle, rq, scx_cpu_arg(cpu_of(rq)), idle); -} - -static void reset_idle_masks(struct sched_ext_ops *ops) -{ - int node; - - /* - * Consider all online cpus idle. Should converge to the actual state - * quickly. - */ - if (!(ops->flags & SCX_OPS_BUILTIN_IDLE_PER_NODE)) { - cpumask_copy(idle_cpumask(NUMA_NO_NODE)->cpu, cpu_online_mask); - cpumask_copy(idle_cpumask(NUMA_NO_NODE)->smt, cpu_online_mask); - return; - } - - for_each_node(node) { - const struct cpumask *node_mask = cpumask_of_node(node); - - cpumask_and(idle_cpumask(node)->cpu, cpu_online_mask, node_mask); - cpumask_and(idle_cpumask(node)->smt, cpu_online_mask, node_mask); - } -} - -void scx_idle_enable(struct sched_ext_ops *ops) -{ - if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) - static_branch_enable_cpuslocked(&scx_builtin_idle_enabled); - else - static_branch_disable_cpuslocked(&scx_builtin_idle_enabled); - - if (ops->flags & SCX_OPS_BUILTIN_IDLE_PER_NODE) - static_branch_enable_cpuslocked(&scx_builtin_idle_per_node); - else - static_branch_disable_cpuslocked(&scx_builtin_idle_per_node); - - reset_idle_masks(ops); -} - -void scx_idle_disable(void) -{ - static_branch_disable(&scx_builtin_idle_enabled); - static_branch_disable(&scx_builtin_idle_per_node); -} - -/******************************************************************************** - * Helpers that can be called from the BPF scheduler. - */ - -static int validate_node(struct scx_sched *sch, int node) -{ - if (!static_branch_likely(&scx_builtin_idle_per_node)) { - scx_error(sch, "per-node idle tracking is disabled"); - return -EOPNOTSUPP; - } - - /* Return no entry for NUMA_NO_NODE (not a critical scx error) */ - if (node == NUMA_NO_NODE) - return -ENOENT; - - /* Make sure node is in a valid range */ - if (node < 0 || node >= nr_node_ids) { - scx_error(sch, "invalid node %d", node); - return -EINVAL; - } - - /* Make sure the node is part of the set of possible nodes */ - if (!node_possible(node)) { - scx_error(sch, "unavailable node %d", node); - return -EINVAL; - } - - return node; -} - -__bpf_kfunc_start_defs(); - -static bool check_builtin_idle_enabled(struct scx_sched *sch) -{ - if (static_branch_likely(&scx_builtin_idle_enabled)) - return true; - - scx_error(sch, "built-in idle tracking is disabled"); - return false; -} - -/* - * Determine whether @p is a migration-disabled task in the context of BPF - * code. - * - * We can't simply check whether @p->migration_disabled is set in a - * sched_ext callback, because the BPF prolog (__bpf_prog_enter) may disable - * migration for the current task while running BPF code. - * - * Since the BPF prolog calls migrate_disable() only when CONFIG_PREEMPT_RCU - * is enabled (via rcu_read_lock_dont_migrate()), migration_disabled == 1 for - * the current task is ambiguous only in that case: it could be from the BPF - * prolog rather than a real migrate_disable() call. - * - * Without CONFIG_PREEMPT_RCU, the BPF prolog never calls migrate_disable(), - * so migration_disabled == 1 always means the task is truly - * migration-disabled. - * - * Therefore, when migration_disabled == 1 and CONFIG_PREEMPT_RCU is enabled, - * check whether @p is the current task or not: if it is, then migration was - * not disabled before entering the callback, otherwise migration was disabled. - * - * Returns true if @p is migration-disabled, false otherwise. - */ -static bool is_bpf_migration_disabled(const struct task_struct *p) -{ - if (p->migration_disabled == 1) { - if (IS_ENABLED(CONFIG_PREEMPT_RCU)) - return p != current; - return true; - } - return p->migration_disabled; -} - -static s32 select_cpu_from_kfunc(struct scx_sched *sch, struct task_struct *p, - s32 prev_cpu, u64 wake_flags, - const struct cpumask *allowed, u64 flags) -{ - unsigned long irq_flags; - bool we_locked = false; - s32 cpu; - - if (!scx_cpu_valid(sch, prev_cpu, NULL)) - return -EINVAL; - - if (!check_builtin_idle_enabled(sch)) - return -EBUSY; - - /* - * Accessing p->cpus_ptr / p->nr_cpus_allowed needs either @p's rq - * lock or @p's pi_lock. Three cases: - * - * - inside ops.select_cpu(): try_to_wake_up() holds the wake-up - * task's pi_lock; the wake-up task is recorded in kf_tasks[0] - * by SCX_CALL_OP_TASK_RET(). - * - other rq-locked SCX op: scx_locked_rq() points at the held rq. - * - truly unlocked (UNLOCKED ops, SYSCALL, non-SCX struct_ops): - * nothing held, take pi_lock ourselves. - * - * In the first two cases, BPF schedulers may pass an arbitrary task - * that the held lock doesn't cover. Refuse those. - */ - if (this_rq()->scx.in_select_cpu) { - if (!scx_kf_arg_task_ok(sch, p)) - return -EINVAL; - lockdep_assert_held(&p->pi_lock); - } else if (scx_locked_rq()) { - if (task_rq(p) != scx_locked_rq()) - goto cross_task; - } else { - raw_spin_lock_irqsave(&p->pi_lock, irq_flags); - we_locked = true; - } - - /* - * This may also be called from ops.enqueue(), so we need to handle - * per-CPU tasks as well. For these tasks, we can skip all idle CPU - * selection optimizations and simply check whether the previously - * used CPU is idle and within the allowed cpumask. - */ - if (p->nr_cpus_allowed == 1 || is_bpf_migration_disabled(p)) { - if (cpumask_test_cpu(prev_cpu, allowed ?: p->cpus_ptr) && - scx_idle_test_and_clear_cpu(prev_cpu)) - cpu = prev_cpu; - else - cpu = -EBUSY; - } else { - cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, - allowed ?: p->cpus_ptr, flags); - } - - if (we_locked) - raw_spin_unlock_irqrestore(&p->pi_lock, irq_flags); - - return cpu; - -cross_task: - scx_error(sch, "select_cpu kfunc called cross-task on %s[%d]", - p->comm, p->pid); - return -EINVAL; -} - -/** - * scx_bpf_cpu_node - Return the NUMA node the given @cpu belongs to, or - * trigger an error if @cpu is invalid - * @cpu: target CPU - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - */ -__bpf_kfunc s32 scx_bpf_cpu_node(s32 cpu, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch) || !scx_cpu_valid(sch, cpu, NULL)) - return NUMA_NO_NODE; - return cpu_to_node(cpu); -} - -/** - * scx_bpf_select_cpu_dfl - The default implementation of ops.select_cpu() - * @p: task_struct to select a CPU for - * @prev_cpu: CPU @p was on previously - * @wake_flags: %SCX_WAKE_* flags - * @is_idle: out parameter indicating whether the returned CPU is idle - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Can be called from ops.select_cpu(), ops.enqueue(), or from an unlocked - * context such as a BPF test_run() call, as long as built-in CPU selection - * is enabled: ops.update_idle() is missing or %SCX_OPS_KEEP_BUILTIN_IDLE - * is set. - * - * Returns the picked CPU with *@is_idle indicating whether the picked CPU is - * currently idle and thus a good candidate for direct dispatching. - */ -__bpf_kfunc s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, - u64 wake_flags, bool *is_idle, - const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - s32 cpu; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return -ENODEV; - - cpu = select_cpu_from_kfunc(sch, p, prev_cpu, wake_flags, NULL, 0); - if (cpu >= 0) { - *is_idle = true; - return cpu; - } - *is_idle = false; - return prev_cpu; -} - -struct scx_bpf_select_cpu_and_args { - /* @p and @cpus_allowed can't be packed together as KF_RCU is not transitive */ - s32 prev_cpu; - u64 wake_flags; - u64 flags; -}; - -/** - * __scx_bpf_select_cpu_and - Arg-wrapped CPU selection with cpumask - * @p: task_struct to select a CPU for - * @cpus_allowed: cpumask of allowed CPUs - * @args: struct containing the rest of the arguments - * @args->prev_cpu: CPU @p was on previously - * @args->wake_flags: %SCX_WAKE_* flags - * @args->flags: %SCX_PICK_IDLE* flags - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Wrapper kfunc that takes arguments via struct to work around BPF's 5 argument - * limit. BPF programs should use scx_bpf_select_cpu_and() which is provided - * as an inline wrapper in common.bpf.h. - * - * Can be called from ops.select_cpu(), ops.enqueue(), or from an unlocked - * context such as a BPF test_run() call, as long as built-in CPU selection - * is enabled: ops.update_idle() is missing or %SCX_OPS_KEEP_BUILTIN_IDLE - * is set. - * - * @p, @args->prev_cpu and @args->wake_flags match ops.select_cpu(). - * - * Returns the selected idle CPU, which will be automatically awakened upon - * returning from ops.select_cpu() and can be used for direct dispatch, or - * a negative value if no idle CPU is available. - */ -__bpf_kfunc s32 -__scx_bpf_select_cpu_and(struct task_struct *p, const struct cpumask *cpus_allowed, - struct scx_bpf_select_cpu_and_args *args, - const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return -ENODEV; - - return select_cpu_from_kfunc(sch, p, args->prev_cpu, args->wake_flags, - cpus_allowed, args->flags); -} - -/* - * COMPAT: Will be removed in v6.22. - */ -__bpf_kfunc s32 scx_bpf_select_cpu_and(struct task_struct *p, s32 prev_cpu, u64 wake_flags, - const struct cpumask *cpus_allowed, u64 flags) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = rcu_dereference(scx_root); - if (unlikely(!sch)) - return -ENODEV; - -#ifdef CONFIG_EXT_SUB_SCHED - /* - * Disallow if any sub-scheds are attached. There is no way to tell - * which scheduler called us, just error out @p's scheduler. - */ - if (unlikely(!list_empty(&sch->children))) { - scx_error(scx_task_sched(p), "__scx_bpf_select_cpu_and() must be used"); - return -EINVAL; - } -#endif - - return select_cpu_from_kfunc(sch, p, prev_cpu, wake_flags, - cpus_allowed, flags); -} - -/** - * scx_bpf_get_idle_cpumask_node - Get a referenced kptr to the - * idle-tracking per-CPU cpumask of a target NUMA node. - * @node: target NUMA node - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Returns an empty cpumask if idle tracking is not enabled, if @node is - * not valid, or running on a UP kernel. In this case the actual error will - * be reported to the BPF scheduler via scx_error(). - */ -__bpf_kfunc const struct cpumask * -scx_bpf_get_idle_cpumask_node(s32 node, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return cpu_none_mask; - - node = validate_node(sch, node); - if (node < 0) - return cpu_none_mask; - - return idle_cpumask(node)->cpu; -} - -/** - * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking - * per-CPU cpumask. - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Returns an empty mask if idle tracking is not enabled, or running on a - * UP kernel. - */ -__bpf_kfunc const struct cpumask *scx_bpf_get_idle_cpumask(const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return cpu_none_mask; - - if (static_branch_unlikely(&scx_builtin_idle_per_node)) { - scx_error(sch, "SCX_OPS_BUILTIN_IDLE_PER_NODE enabled"); - return cpu_none_mask; - } - - if (!check_builtin_idle_enabled(sch)) - return cpu_none_mask; - - return idle_cpumask(NUMA_NO_NODE)->cpu; -} - -/** - * scx_bpf_get_idle_smtmask_node - Get a referenced kptr to the - * idle-tracking, per-physical-core cpumask of a target NUMA node. Can be - * used to determine if an entire physical core is free. - * @node: target NUMA node - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Returns an empty cpumask if idle tracking is not enabled, if @node is - * not valid, or running on a UP kernel. In this case the actual error will - * be reported to the BPF scheduler via scx_error(). - */ -__bpf_kfunc const struct cpumask * -scx_bpf_get_idle_smtmask_node(s32 node, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return cpu_none_mask; - - node = validate_node(sch, node); - if (node < 0) - return cpu_none_mask; - - if (sched_smt_active()) - return idle_cpumask(node)->smt; - else - return idle_cpumask(node)->cpu; -} - -/** - * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking, - * per-physical-core cpumask. Can be used to determine if an entire physical - * core is free. - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Returns an empty mask if idle tracking is not enabled, or running on a - * UP kernel. - */ -__bpf_kfunc const struct cpumask *scx_bpf_get_idle_smtmask(const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return cpu_none_mask; - - if (static_branch_unlikely(&scx_builtin_idle_per_node)) { - scx_error(sch, "SCX_OPS_BUILTIN_IDLE_PER_NODE enabled"); - return cpu_none_mask; - } - - if (!check_builtin_idle_enabled(sch)) - return cpu_none_mask; - - if (sched_smt_active()) - return idle_cpumask(NUMA_NO_NODE)->smt; - else - return idle_cpumask(NUMA_NO_NODE)->cpu; -} - -/** - * scx_bpf_put_idle_cpumask - Release a previously acquired referenced kptr to - * either the percpu, or SMT idle-tracking cpumask. - * @idle_mask: &cpumask to use - */ -__bpf_kfunc void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask) -{ - /* - * Empty function body because we aren't actually acquiring or releasing - * a reference to a global idle cpumask, which is read-only in the - * caller and is never released. The acquire / release semantics here - * are just used to make the cpumask a trusted pointer in the caller. - */ -} - -/** - * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state - * @cpu: cpu to test and clear idle for - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Returns %true if @cpu was idle and its idle state was successfully cleared. - * %false otherwise. - * - * Unavailable if ops.update_idle() is implemented and - * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. - */ -__bpf_kfunc bool scx_bpf_test_and_clear_cpu_idle(s32 cpu, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return false; - - if (!check_builtin_idle_enabled(sch)) - return false; - - if (!scx_cpu_valid(sch, cpu, NULL)) - return false; - - return scx_idle_test_and_clear_cpu(cpu); -} - -/** - * scx_bpf_pick_idle_cpu_node - Pick and claim an idle cpu from @node - * @cpus_allowed: Allowed cpumask - * @node: target NUMA node - * @flags: %SCX_PICK_IDLE_* flags - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Pick and claim an idle cpu in @cpus_allowed from the NUMA node @node. - * - * Returns the picked idle cpu number on success, or -%EBUSY if no matching - * cpu was found. - * - * The search starts from @node and proceeds to other online NUMA nodes in - * order of increasing distance (unless SCX_PICK_IDLE_IN_NODE is specified, - * in which case the search is limited to the target @node). - * - * Always returns an error if ops.update_idle() is implemented and - * %SCX_OPS_KEEP_BUILTIN_IDLE is not set, or if - * %SCX_OPS_BUILTIN_IDLE_PER_NODE is not set. - */ -__bpf_kfunc s32 scx_bpf_pick_idle_cpu_node(const struct cpumask *cpus_allowed, - s32 node, u64 flags, - const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return -ENODEV; - - node = validate_node(sch, node); - if (node < 0) - return node; - - return scx_pick_idle_cpu(cpus_allowed, node, flags); -} - -/** - * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu - * @cpus_allowed: Allowed cpumask - * @flags: %SCX_PICK_IDLE_CPU_* flags - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Pick and claim an idle cpu in @cpus_allowed. Returns the picked idle cpu - * number on success. -%EBUSY if no matching cpu was found. - * - * Idle CPU tracking may race against CPU scheduling state transitions. For - * example, this function may return -%EBUSY as CPUs are transitioning into the - * idle state. If the caller then assumes that there will be dispatch events on - * the CPUs as they were all busy, the scheduler may end up stalling with CPUs - * idling while there are pending tasks. Use scx_bpf_pick_any_cpu() and - * scx_bpf_kick_cpu() to guarantee that there will be at least one dispatch - * event in the near future. - * - * Unavailable if ops.update_idle() is implemented and - * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. - * - * Always returns an error if %SCX_OPS_BUILTIN_IDLE_PER_NODE is set, use - * scx_bpf_pick_idle_cpu_node() instead. - */ -__bpf_kfunc s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed, - u64 flags, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return -ENODEV; - - if (static_branch_maybe(CONFIG_NUMA, &scx_builtin_idle_per_node)) { - scx_error(sch, "per-node idle tracking is enabled"); - return -EBUSY; - } - - if (!check_builtin_idle_enabled(sch)) - return -EBUSY; - - return scx_pick_idle_cpu(cpus_allowed, NUMA_NO_NODE, flags); -} - -/** - * scx_bpf_pick_any_cpu_node - Pick and claim an idle cpu if available - * or pick any CPU from @node - * @cpus_allowed: Allowed cpumask - * @node: target NUMA node - * @flags: %SCX_PICK_IDLE_CPU_* flags - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Pick and claim an idle cpu in @cpus_allowed. If none is available, pick any - * CPU in @cpus_allowed. Guaranteed to succeed and returns the picked idle cpu - * number if @cpus_allowed is not empty. -%EBUSY is returned if @cpus_allowed is - * empty. - * - * The search starts from @node and proceeds to other online NUMA nodes in - * order of increasing distance (unless %SCX_PICK_IDLE_IN_NODE is specified, - * in which case the search is limited to the target @node, regardless of - * the CPU idle state). - * - * If ops.update_idle() is implemented and %SCX_OPS_KEEP_BUILTIN_IDLE is not - * set, this function can't tell which CPUs are idle and will always pick any - * CPU. - */ -__bpf_kfunc s32 scx_bpf_pick_any_cpu_node(const struct cpumask *cpus_allowed, - s32 node, u64 flags, - const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - s32 cpu; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return -ENODEV; - - node = validate_node(sch, node); - if (node < 0) - return node; - - cpu = scx_pick_idle_cpu(cpus_allowed, node, flags); - if (cpu >= 0) - return cpu; - - if (flags & SCX_PICK_IDLE_IN_NODE) - cpu = cpumask_any_and_distribute(cpumask_of_node(node), cpus_allowed); - else - cpu = cpumask_any_distribute(cpus_allowed); - if (cpu < nr_cpu_ids) - return cpu; - else - return -EBUSY; -} - -/** - * scx_bpf_pick_any_cpu - Pick and claim an idle cpu if available or pick any CPU - * @cpus_allowed: Allowed cpumask - * @flags: %SCX_PICK_IDLE_CPU_* flags - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Pick and claim an idle cpu in @cpus_allowed. If none is available, pick any - * CPU in @cpus_allowed. Guaranteed to succeed and returns the picked idle cpu - * number if @cpus_allowed is not empty. -%EBUSY is returned if @cpus_allowed is - * empty. - * - * If ops.update_idle() is implemented and %SCX_OPS_KEEP_BUILTIN_IDLE is not - * set, this function can't tell which CPUs are idle and will always pick any - * CPU. - * - * Always returns an error if %SCX_OPS_BUILTIN_IDLE_PER_NODE is set, use - * scx_bpf_pick_any_cpu_node() instead. - */ -__bpf_kfunc s32 scx_bpf_pick_any_cpu(const struct cpumask *cpus_allowed, - u64 flags, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - s32 cpu; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return -ENODEV; - - if (static_branch_maybe(CONFIG_NUMA, &scx_builtin_idle_per_node)) { - scx_error(sch, "per-node idle tracking is enabled"); - return -EBUSY; - } - - if (static_branch_likely(&scx_builtin_idle_enabled)) { - cpu = scx_pick_idle_cpu(cpus_allowed, NUMA_NO_NODE, flags); - if (cpu >= 0) - return cpu; - } - - cpu = cpumask_any_distribute(cpus_allowed); - if (cpu < nr_cpu_ids) - return cpu; - else - return -EBUSY; -} - -__bpf_kfunc_end_defs(); - -BTF_KFUNCS_START(scx_kfunc_ids_idle) -BTF_ID_FLAGS(func, scx_bpf_cpu_node, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE) -BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_IMPLICIT_ARGS | KF_ACQUIRE) -BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE) -BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_IMPLICIT_ARGS | KF_ACQUIRE) -BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) -BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu_node, KF_IMPLICIT_ARGS | KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu, KF_IMPLICIT_ARGS | KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu_node, KF_IMPLICIT_ARGS | KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu, KF_IMPLICIT_ARGS | KF_RCU) -BTF_KFUNCS_END(scx_kfunc_ids_idle) - -static const struct btf_kfunc_id_set scx_kfunc_set_idle = { - .owner = THIS_MODULE, - .set = &scx_kfunc_ids_idle, - .filter = scx_kfunc_context_filter, -}; - -/* - * The select_cpu kfuncs internally call task_rq_lock() when invoked from an - * rq-unlocked context, and thus cannot be safely called from arbitrary tracing - * contexts where @p's pi_lock state is unknown. Keep them out of - * BPF_PROG_TYPE_TRACING by registering them in their own set which is exposed - * only to STRUCT_OPS and SYSCALL programs. - * - * These kfuncs are also members of scx_kfunc_ids_unlocked (see ext.c) because - * they're callable from unlocked contexts in addition to ops.select_cpu() and - * ops.enqueue(). - */ -BTF_KFUNCS_START(scx_kfunc_ids_select_cpu) -BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS | KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_IMPLICIT_ARGS | KF_RCU) -BTF_KFUNCS_END(scx_kfunc_ids_select_cpu) - -static const struct btf_kfunc_id_set scx_kfunc_set_select_cpu = { - .owner = THIS_MODULE, - .set = &scx_kfunc_ids_select_cpu, - .filter = scx_kfunc_context_filter, -}; - -int scx_idle_init(void) -{ - return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_idle) ?: - register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &scx_kfunc_set_idle) ?: - register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_idle) ?: - register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_select_cpu) ?: - register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_select_cpu); -} diff --git a/kernel/sched/ext_idle.h b/kernel/sched/ext_idle.h deleted file mode 100644 index 8d169d3bbdf9..000000000000 --- a/kernel/sched/ext_idle.h +++ /dev/null @@ -1,27 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst - * - * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. - * Copyright (c) 2022 Tejun Heo - * Copyright (c) 2022 David Vernet - * Copyright (c) 2024 Andrea Righi - */ -#ifndef _KERNEL_SCHED_EXT_IDLE_H -#define _KERNEL_SCHED_EXT_IDLE_H - -struct sched_ext_ops; - -extern struct btf_id_set8 scx_kfunc_ids_idle; -extern struct btf_id_set8 scx_kfunc_ids_select_cpu; - -void scx_idle_update_selcpu_topology(struct sched_ext_ops *ops); -void scx_idle_init_masks(void); - -s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, - const struct cpumask *cpus_allowed, u64 flags); -void scx_idle_enable(struct sched_ext_ops *ops); -void scx_idle_disable(void); -int scx_idle_init(void); - -#endif /* _KERNEL_SCHED_EXT_IDLE_H */ diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext_internal.h deleted file mode 100644 index b04701190b23..000000000000 --- a/kernel/sched/ext_internal.h +++ /dev/null @@ -1,1653 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst - * - * Copyright (c) 2025 Meta Platforms, Inc. and affiliates. - * Copyright (c) 2025 Tejun Heo - */ -#define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) -#define SCX_MOFF_IDX(moff) ((moff) / sizeof(void (*)(void))) - -enum scx_exit_kind { - SCX_EXIT_NONE, - SCX_EXIT_DONE, - - SCX_EXIT_UNREG = 64, /* user-space initiated unregistration */ - SCX_EXIT_UNREG_BPF, /* BPF-initiated unregistration */ - SCX_EXIT_UNREG_KERN, /* kernel-initiated unregistration */ - SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ - SCX_EXIT_PARENT, /* parent exiting */ - - SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ - SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ - SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ -}; - -/* - * An exit code can be specified when exiting with scx_bpf_exit() or scx_exit(), - * corresponding to exit_kind UNREG_BPF and UNREG_KERN respectively. The codes - * are 64bit of the format: - * - * Bits: [63 .. 48 47 .. 32 31 .. 0] - * [ SYS ACT ] [ SYS RSN ] [ USR ] - * - * SYS ACT: System-defined exit actions - * SYS RSN: System-defined exit reasons - * USR : User-defined exit codes and reasons - * - * Using the above, users may communicate intention and context by ORing system - * actions and/or system reasons with a user-defined exit code. - */ -enum scx_exit_code { - /* Reasons */ - SCX_ECODE_RSN_HOTPLUG = 1LLU << 32, - SCX_ECODE_RSN_CGROUP_OFFLINE = 2LLU << 32, - - /* Actions */ - SCX_ECODE_ACT_RESTART = 1LLU << 48, -}; - -enum scx_exit_flags { - /* - * ops.exit() may be called even if the loading failed before ops.init() - * finishes successfully. This is because ops.exit() allows rich exit - * info communication. The following flag indicates whether ops.init() - * finished successfully. - */ - SCX_EFLAG_INITIALIZED = 1LLU << 0, -}; - -/* - * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is - * being disabled. - */ -struct scx_exit_info { - /* %SCX_EXIT_* - broad category of the exit reason */ - enum scx_exit_kind kind; - - /* - * CPU that initiated the exit, valid once @kind has been set. - * Negative if the exit path didn't identify a CPU. - */ - s32 exit_cpu; - - /* exit code if gracefully exiting */ - s64 exit_code; - - /* %SCX_EFLAG_* */ - u64 flags; - - /* textual representation of the above */ - const char *reason; - - /* backtrace if exiting due to an error */ - unsigned long *bt; - u32 bt_len; - - /* informational message */ - char *msg; - - /* debug dump */ - char *dump; -}; - -/* sched_ext_ops.flags */ -enum scx_ops_flags { - /* - * Keep built-in idle tracking even if ops.update_idle() is implemented. - */ - SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, - - /* - * By default, if there are no other task to run on the CPU, ext core - * keeps running the current task even after its slice expires. If this - * flag is specified, such tasks are passed to ops.enqueue() with - * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. - */ - SCX_OPS_ENQ_LAST = 1LLU << 1, - - /* - * An exiting task may schedule after PF_EXITING is set. In such cases, - * bpf_task_from_pid() may not be able to find the task and if the BPF - * scheduler depends on pid lookup for dispatching, the task will be - * lost leading to various issues including RCU grace period stalls. - * - * To mask this problem, by default, unhashed tasks are automatically - * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't - * depend on pid lookups and wants to handle these tasks directly, the - * following flag can be used. With %SCX_OPS_TID_TO_TASK, - * scx_bpf_tid_to_task() can find exiting tasks reliably. - */ - SCX_OPS_ENQ_EXITING = 1LLU << 2, - - /* - * If set, only tasks with policy set to SCHED_EXT are attached to - * sched_ext. If clear, SCHED_NORMAL tasks are also included. - */ - SCX_OPS_SWITCH_PARTIAL = 1LLU << 3, - - /* - * A migration disabled task can only execute on its current CPU. By - * default, such tasks are automatically put on the CPU's local DSQ with - * the default slice on enqueue. If this ops flag is set, they also go - * through ops.enqueue(). - * - * A migration disabled task never invokes ops.select_cpu() as it can - * only select the current CPU. Also, p->cpus_ptr will only contain its - * current CPU while p->nr_cpus_allowed keeps tracking p->user_cpus_ptr - * and thus may disagree with cpumask_weight(p->cpus_ptr). - */ - SCX_OPS_ENQ_MIGRATION_DISABLED = 1LLU << 4, - - /* - * Queued wakeup (ttwu_queue) is a wakeup optimization that invokes - * ops.enqueue() on the ops.select_cpu() selected or the wakee's - * previous CPU via IPI (inter-processor interrupt) to reduce cacheline - * transfers. When this optimization is enabled, ops.select_cpu() is - * skipped in some cases (when racing against the wakee switching out). - * As the BPF scheduler may depend on ops.select_cpu() being invoked - * during wakeups, queued wakeup is disabled by default. - * - * If this ops flag is set, queued wakeup optimization is enabled and - * the BPF scheduler must be able to handle ops.enqueue() invoked on the - * wakee's CPU without preceding ops.select_cpu() even for tasks which - * may be executed on multiple CPUs. - */ - SCX_OPS_ALLOW_QUEUED_WAKEUP = 1LLU << 5, - - /* - * If set, enable per-node idle cpumasks. If clear, use a single global - * flat idle cpumask. - */ - SCX_OPS_BUILTIN_IDLE_PER_NODE = 1LLU << 6, - - /* - * If set, %SCX_ENQ_IMMED is assumed to be set on all local DSQ - * enqueues. - */ - SCX_OPS_ALWAYS_ENQ_IMMED = 1LLU << 7, - - /* - * Maintain a mapping from p->scx.tid to task_struct so the BPF - * scheduler can recover task pointers from stored tids via - * scx_bpf_tid_to_task(). - * - * Only the root scheduler turns this on. A sub-sched may set the flag - * to declare a dependency on the lookup; if the root scheduler hasn't - * enabled it, attaching the sub-sched is rejected. - */ - SCX_OPS_TID_TO_TASK = 1LLU << 8, - - SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | - SCX_OPS_ENQ_LAST | - SCX_OPS_ENQ_EXITING | - SCX_OPS_ENQ_MIGRATION_DISABLED | - SCX_OPS_ALLOW_QUEUED_WAKEUP | - SCX_OPS_SWITCH_PARTIAL | - SCX_OPS_BUILTIN_IDLE_PER_NODE | - SCX_OPS_ALWAYS_ENQ_IMMED | - SCX_OPS_TID_TO_TASK, - - /* high 8 bits are internal, don't include in SCX_OPS_ALL_FLAGS */ - __SCX_OPS_INTERNAL_MASK = 0xffLLU << 56, - - SCX_OPS_HAS_CPU_PREEMPT = 1LLU << 56, -}; - -/* argument container for ops.init_task() */ -struct scx_init_task_args { - /* - * Set if ops.init_task() is being invoked on the fork path, as opposed - * to the scheduler transition path. - */ - bool fork; -#ifdef CONFIG_EXT_GROUP_SCHED - /* the cgroup the task is joining */ - struct cgroup *cgroup; -#endif -}; - -/* argument container for ops.exit_task() */ -struct scx_exit_task_args { - /* Whether the task exited before running on sched_ext. */ - bool cancelled; -}; - -/* argument container for ops.cgroup_init() */ -struct scx_cgroup_init_args { - /* the weight of the cgroup [1..10000] */ - u32 weight; - - /* bandwidth control parameters from cpu.max and cpu.max.burst */ - u64 bw_period_us; - u64 bw_quota_us; - u64 bw_burst_us; -}; - -enum scx_cpu_preempt_reason { - /* next task is being scheduled by &sched_class_rt */ - SCX_CPU_PREEMPT_RT, - /* next task is being scheduled by &sched_class_dl */ - SCX_CPU_PREEMPT_DL, - /* next task is being scheduled by &sched_class_stop */ - SCX_CPU_PREEMPT_STOP, - /* unknown reason for SCX being preempted */ - SCX_CPU_PREEMPT_UNKNOWN, -}; - -/* - * Argument container for ops.cpu_acquire(). Currently empty, but may be - * expanded in the future. - */ -struct scx_cpu_acquire_args {}; - -/* argument container for ops.cpu_release() */ -struct scx_cpu_release_args { - /* the reason the CPU was preempted */ - enum scx_cpu_preempt_reason reason; - - /* the task that's going to be scheduled on the CPU */ - struct task_struct *task; -}; - -/* informational context provided to dump operations */ -struct scx_dump_ctx { - enum scx_exit_kind kind; - s64 exit_code; - const char *reason; - u64 at_ns; - u64 at_jiffies; -}; - -/* argument container for ops.sub_attach() */ -struct scx_sub_attach_args { - struct sched_ext_ops *ops; - char *cgroup_path; -}; - -/* argument container for ops.sub_detach() */ -struct scx_sub_detach_args { - struct sched_ext_ops *ops; - char *cgroup_path; -}; - -/** - * struct sched_ext_ops - Operation table for BPF scheduler implementation - * - * A BPF scheduler can implement an arbitrary scheduling policy by - * implementing and loading operations in this table. Note that a userland - * scheduling policy can also be implemented using the BPF scheduler - * as a shim layer. - */ -struct sched_ext_ops { - /** - * @select_cpu: Pick the target CPU for a task which is being woken up - * @p: task being woken up - * @prev_cpu: the cpu @p was on before sleeping - * @wake_flags: SCX_WAKE_* - * - * Decision made here isn't final. @p may be moved to any CPU while it - * is getting dispatched for execution later. However, as @p is not on - * the rq at this point, getting the eventual execution CPU right here - * saves a small bit of overhead down the line. - * - * If an idle CPU is returned, the CPU is kicked and will try to - * dispatch. While an explicit custom mechanism can be added, - * select_cpu() serves as the default way to wake up idle CPUs. - * - * @p may be inserted into a DSQ directly by calling - * scx_bpf_dsq_insert(). If so, the ops.enqueue() will be skipped. - * Directly inserting into %SCX_DSQ_LOCAL will put @p in the local DSQ - * of the CPU returned by this operation. - * - * Note that select_cpu() is never called for tasks that can only run - * on a single CPU or tasks with migration disabled, as they don't have - * the option to select a different CPU. See select_task_rq() for - * details. - */ - s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); - - /** - * @enqueue: Enqueue a task on the BPF scheduler - * @p: task being enqueued - * @enq_flags: %SCX_ENQ_* - * - * @p is ready to run. Insert directly into a DSQ by calling - * scx_bpf_dsq_insert() or enqueue on the BPF scheduler. If not directly - * inserted, the bpf scheduler owns @p and if it fails to dispatch @p, - * the task will stall. - * - * If @p was inserted into a DSQ from ops.select_cpu(), this callback is - * skipped. - */ - void (*enqueue)(struct task_struct *p, u64 enq_flags); - - /** - * @dequeue: Remove a task from the BPF scheduler - * @p: task being dequeued - * @deq_flags: %SCX_DEQ_* - * - * Remove @p from the BPF scheduler. This is usually called to isolate - * the task while updating its scheduling properties (e.g. priority). - * - * The ext core keeps track of whether the BPF side owns a given task or - * not and can gracefully ignore spurious dispatches from BPF side, - * which makes it safe to not implement this method. However, depending - * on the scheduling logic, this can lead to confusing behaviors - e.g. - * scheduling position not being updated across a priority change. - */ - void (*dequeue)(struct task_struct *p, u64 deq_flags); - - /** - * @dispatch: Dispatch tasks from the BPF scheduler and/or user DSQs - * @cpu: CPU to dispatch tasks for - * @prev: previous task being switched out - * - * Called when a CPU's local dsq is empty. The operation should dispatch - * one or more tasks from the BPF scheduler into the DSQs using - * scx_bpf_dsq_insert() and/or move from user DSQs into the local DSQ - * using scx_bpf_dsq_move_to_local(). - * - * The maximum number of times scx_bpf_dsq_insert() can be called - * without an intervening scx_bpf_dsq_move_to_local() is specified by - * ops.dispatch_max_batch. See the comments on top of the two functions - * for more details. - * - * When not %NULL, @prev is an SCX task with its slice depleted. If - * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in - * @prev->scx.flags, it is not enqueued yet and will be enqueued after - * ops.dispatch() returns. To keep executing @prev, return without - * dispatching or moving any tasks. Also see %SCX_OPS_ENQ_LAST. - */ - void (*dispatch)(s32 cpu, struct task_struct *prev); - - /** - * @tick: Periodic tick - * @p: task running currently - * - * This operation is called every 1/HZ seconds on CPUs which are - * executing an SCX task. Setting @p->scx.slice to 0 will trigger an - * immediate dispatch cycle on the CPU. - */ - void (*tick)(struct task_struct *p); - - /** - * @runnable: A task is becoming runnable on its associated CPU - * @p: task becoming runnable - * @enq_flags: %SCX_ENQ_* - * - * This and the following three functions can be used to track a task's - * execution state transitions. A task becomes ->runnable() on a CPU, - * and then goes through one or more ->running() and ->stopping() pairs - * as it runs on the CPU, and eventually becomes ->quiescent() when it's - * done running on the CPU. - * - * @p is becoming runnable on the CPU because it's - * - * - waking up (%SCX_ENQ_WAKEUP) - * - being moved from another CPU - * - being restored after temporarily taken off the queue for an - * attribute change. - * - * This and ->enqueue() are related but not coupled. This operation - * notifies @p's state transition and may not be followed by ->enqueue() - * e.g. when @p is being dispatched to a remote CPU, or when @p is - * being enqueued on a CPU experiencing a hotplug event. Likewise, a - * task may be ->enqueue()'d without being preceded by this operation - * e.g. after exhausting its slice. - */ - void (*runnable)(struct task_struct *p, u64 enq_flags); - - /** - * @running: A task is starting to run on its associated CPU - * @p: task starting to run - * - * Note that this callback may be called from a CPU other than the - * one the task is going to run on. This can happen when a task - * property is changed (i.e., affinity), since scx_next_task_scx(), - * which triggers this callback, may run on a CPU different from - * the task's assigned CPU. - * - * Therefore, always use scx_bpf_task_cpu(@p) to determine the - * target CPU the task is going to use. - * - * See ->runnable() for explanation on the task state notifiers. - */ - void (*running)(struct task_struct *p); - - /** - * @stopping: A task is stopping execution - * @p: task stopping to run - * @runnable: is task @p still runnable? - * - * Note that this callback may be called from a CPU other than the - * one the task was running on. This can happen when a task - * property is changed (i.e., affinity), since dequeue_task_scx(), - * which triggers this callback, may run on a CPU different from - * the task's assigned CPU. - * - * Therefore, always use scx_bpf_task_cpu(@p) to retrieve the CPU - * the task was running on. - * - * See ->runnable() for explanation on the task state notifiers. If - * !@runnable, ->quiescent() will be invoked after this operation - * returns. - */ - void (*stopping)(struct task_struct *p, bool runnable); - - /** - * @quiescent: A task is becoming not runnable on its associated CPU - * @p: task becoming not runnable - * @deq_flags: %SCX_DEQ_* - * - * See ->runnable() for explanation on the task state notifiers. - * - * @p is becoming quiescent on the CPU because it's - * - * - sleeping (%SCX_DEQ_SLEEP) - * - being moved to another CPU - * - being temporarily taken off the queue for an attribute change - * (%SCX_DEQ_SAVE) - * - * This and ->dequeue() are related but not coupled. This operation - * notifies @p's state transition and may not be preceded by ->dequeue() - * e.g. when @p is being dispatched to a remote CPU. - */ - void (*quiescent)(struct task_struct *p, u64 deq_flags); - - /** - * @yield: Yield CPU - * @from: yielding task - * @to: optional yield target task - * - * If @to is NULL, @from is yielding the CPU to other runnable tasks. - * The BPF scheduler should ensure that other available tasks are - * dispatched before the yielding task. Return value is ignored in this - * case. - * - * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf - * scheduler can implement the request, return %true; otherwise, %false. - */ - bool (*yield)(struct task_struct *from, struct task_struct *to); - - /** - * @core_sched_before: Task ordering for core-sched - * @a: task A - * @b: task B - * - * Used by core-sched to determine the ordering between two tasks. See - * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on - * core-sched. - * - * Both @a and @b are runnable and may or may not currently be queued on - * the BPF scheduler. Should return %true if @a should run before @b. - * %false if there's no required ordering or @b should run before @a. - * - * If not specified, the default is ordering them according to when they - * became runnable. - */ - bool (*core_sched_before)(struct task_struct *a, struct task_struct *b); - - /** - * @set_weight: Set task weight - * @p: task to set weight for - * @weight: new weight [1..10000] - * - * Update @p's weight to @weight. - */ - void (*set_weight)(struct task_struct *p, u32 weight); - - /** - * @set_cpumask: Set CPU affinity - * @p: task to set CPU affinity for - * @cpumask: cpumask of cpus that @p can run on - * - * Update @p's CPU affinity to @cpumask. - */ - void (*set_cpumask)(struct task_struct *p, - const struct cpumask *cpumask); - - /** - * @update_idle: Update the idle state of a CPU - * @cpu: CPU to update the idle state for - * @idle: whether entering or exiting the idle state - * - * This operation is called when @rq's CPU goes or leaves the idle - * state. By default, implementing this operation disables the built-in - * idle CPU tracking and the following helpers become unavailable: - * - * - scx_bpf_select_cpu_dfl() - * - scx_bpf_select_cpu_and() - * - scx_bpf_test_and_clear_cpu_idle() - * - scx_bpf_pick_idle_cpu() - * - * The user also must implement ops.select_cpu() as the default - * implementation relies on scx_bpf_select_cpu_dfl(). - * - * Specify the %SCX_OPS_KEEP_BUILTIN_IDLE flag to keep the built-in idle - * tracking. - */ - void (*update_idle)(s32 cpu, bool idle); - - /** - * @init_task: Initialize a task to run in a BPF scheduler - * @p: task to initialize for BPF scheduling - * @args: init arguments, see the struct definition - * - * Either we're loading a BPF scheduler or a new task is being forked. - * Initialize @p for BPF scheduling. This operation may block and can - * be used for allocations, and is called exactly once for a task. - * - * Return 0 for success, -errno for failure. An error return while - * loading will abort loading of the BPF scheduler. During a fork, it - * will abort that specific fork. - */ - s32 (*init_task)(struct task_struct *p, struct scx_init_task_args *args); - - /** - * @exit_task: Exit a previously-running task from the system - * @p: task to exit - * @args: exit arguments, see the struct definition - * - * @p is exiting or the BPF scheduler is being unloaded. Perform any - * necessary cleanup for @p. - */ - void (*exit_task)(struct task_struct *p, struct scx_exit_task_args *args); - - /** - * @enable: Enable BPF scheduling for a task - * @p: task to enable BPF scheduling for - * - * Enable @p for BPF scheduling. enable() is called on @p any time it - * enters SCX, and is always paired with a matching disable(). - */ - void (*enable)(struct task_struct *p); - - /** - * @disable: Disable BPF scheduling for a task - * @p: task to disable BPF scheduling for - * - * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. - * Disable BPF scheduling for @p. A disable() call is always matched - * with a prior enable() call. - */ - void (*disable)(struct task_struct *p); - - /** - * @dump: Dump BPF scheduler state on error - * @ctx: debug dump context - * - * Use scx_bpf_dump() to generate BPF scheduler specific debug dump. - */ - void (*dump)(struct scx_dump_ctx *ctx); - - /** - * @dump_cpu: Dump BPF scheduler state for a CPU on error - * @ctx: debug dump context - * @cpu: CPU to generate debug dump for - * @idle: @cpu is currently idle without any runnable tasks - * - * Use scx_bpf_dump() to generate BPF scheduler specific debug dump for - * @cpu. If @idle is %true and this operation doesn't produce any - * output, @cpu is skipped for dump. - */ - void (*dump_cpu)(struct scx_dump_ctx *ctx, s32 cpu, bool idle); - - /** - * @dump_task: Dump BPF scheduler state for a runnable task on error - * @ctx: debug dump context - * @p: runnable task to generate debug dump for - * - * Use scx_bpf_dump() to generate BPF scheduler specific debug dump for - * @p. - */ - void (*dump_task)(struct scx_dump_ctx *ctx, struct task_struct *p); - -#ifdef CONFIG_EXT_GROUP_SCHED - /** - * @cgroup_init: Initialize a cgroup - * @cgrp: cgroup being initialized - * @args: init arguments, see the struct definition - * - * Either the BPF scheduler is being loaded or @cgrp created, initialize - * @cgrp for sched_ext. This operation may block. - * - * Return 0 for success, -errno for failure. An error return while - * loading will abort loading of the BPF scheduler. During cgroup - * creation, it will abort the specific cgroup creation. - */ - s32 (*cgroup_init)(struct cgroup *cgrp, - struct scx_cgroup_init_args *args); - - /** - * @cgroup_exit: Exit a cgroup - * @cgrp: cgroup being exited - * - * Either the BPF scheduler is being unloaded or @cgrp destroyed, exit - * @cgrp for sched_ext. This operation my block. - */ - void (*cgroup_exit)(struct cgroup *cgrp); - - /** - * @cgroup_prep_move: Prepare a task to be moved to a different cgroup - * @p: task being moved - * @from: cgroup @p is being moved from - * @to: cgroup @p is being moved to - * - * Prepare @p for move from cgroup @from to @to. This operation may - * block and can be used for allocations. - * - * Return 0 for success, -errno for failure. An error return aborts the - * migration. - */ - s32 (*cgroup_prep_move)(struct task_struct *p, - struct cgroup *from, struct cgroup *to); - - /** - * @cgroup_move: Commit cgroup move - * @p: task being moved - * @from: cgroup @p is being moved from - * @to: cgroup @p is being moved to - * - * Commit the move. @p is dequeued during this operation. - */ - void (*cgroup_move)(struct task_struct *p, - struct cgroup *from, struct cgroup *to); - - /** - * @cgroup_cancel_move: Cancel cgroup move - * @p: task whose cgroup move is being canceled - * @from: cgroup @p was being moved from - * @to: cgroup @p was being moved to - * - * @p was cgroup_prep_move()'d but failed before reaching cgroup_move(). - * Undo the preparation. - */ - void (*cgroup_cancel_move)(struct task_struct *p, - struct cgroup *from, struct cgroup *to); - - /** - * @cgroup_set_weight: A cgroup's weight is being changed - * @cgrp: cgroup whose weight is being updated - * @weight: new weight [1..10000] - * - * Update @cgrp's weight to @weight. - */ - void (*cgroup_set_weight)(struct cgroup *cgrp, u32 weight); - - /** - * @cgroup_set_bandwidth: A cgroup's bandwidth is being changed - * @cgrp: cgroup whose bandwidth is being updated - * @period_us: bandwidth control period - * @quota_us: bandwidth control quota - * @burst_us: bandwidth control burst - * - * Update @cgrp's bandwidth control parameters. This is from the cpu.max - * cgroup interface. - * - * @quota_us / @period_us determines the CPU bandwidth @cgrp is entitled - * to. For example, if @period_us is 1_000_000 and @quota_us is - * 2_500_000. @cgrp is entitled to 2.5 CPUs. @burst_us can be - * interpreted in the same fashion and specifies how much @cgrp can - * burst temporarily. The specific control mechanism and thus the - * interpretation of @period_us and burstiness is up to the BPF - * scheduler. - */ - void (*cgroup_set_bandwidth)(struct cgroup *cgrp, - u64 period_us, u64 quota_us, u64 burst_us); - - /** - * @cgroup_set_idle: A cgroup's idle state is being changed - * @cgrp: cgroup whose idle state is being updated - * @idle: whether the cgroup is entering or exiting idle state - * - * Update @cgrp's idle state to @idle. This callback is invoked when - * a cgroup transitions between idle and non-idle states, allowing the - * BPF scheduler to adjust its behavior accordingly. - */ - void (*cgroup_set_idle)(struct cgroup *cgrp, bool idle); - -#endif /* CONFIG_EXT_GROUP_SCHED */ - - /** - * @sub_attach: Attach a sub-scheduler - * @args: argument container, see the struct definition - * - * Return 0 to accept the sub-scheduler. -errno to reject. - */ - s32 (*sub_attach)(struct scx_sub_attach_args *args); - - /** - * @sub_detach: Detach a sub-scheduler - * @args: argument container, see the struct definition - */ - void (*sub_detach)(struct scx_sub_detach_args *args); - - /* - * All online ops must come before ops.cpu_online(). - */ - - /** - * @cpu_online: A CPU became online - * @cpu: CPU which just came up - * - * @cpu just came online. @cpu will not call ops.enqueue() or - * ops.dispatch(), nor run tasks associated with other CPUs beforehand. - */ - void (*cpu_online)(s32 cpu); - - /** - * @cpu_offline: A CPU is going offline - * @cpu: CPU which is going offline - * - * @cpu is going offline. @cpu will not call ops.enqueue() or - * ops.dispatch(), nor run tasks associated with other CPUs afterwards. - */ - void (*cpu_offline)(s32 cpu); - - /* - * All CPU hotplug ops must come before ops.init(). - */ - - /** - * @init: Initialize the BPF scheduler - */ - s32 (*init)(void); - - /** - * @exit: Clean up after the BPF scheduler - * @info: Exit info - * - * ops.exit() is also called on ops.init() failure, which is a bit - * unusual. This is to allow rich reporting through @info on how - * ops.init() failed. - */ - void (*exit)(struct scx_exit_info *info); - - /* - * Data fields must comes after all ops fields. - */ - - /** - * @dispatch_max_batch: Max nr of tasks that dispatch() can dispatch - */ - u32 dispatch_max_batch; - - /** - * @flags: %SCX_OPS_* flags - */ - u64 flags; - - /** - * @timeout_ms: The maximum amount of time, in milliseconds, that a - * runnable task should be able to wait before being scheduled. The - * maximum timeout may not exceed the default timeout of 30 seconds. - * - * Defaults to the maximum allowed timeout value of 30 seconds. - */ - u32 timeout_ms; - - /** - * @exit_dump_len: scx_exit_info.dump buffer length. If 0, the default - * value of 32768 is used. - */ - u32 exit_dump_len; - - /** - * @hotplug_seq: A sequence number that may be set by the scheduler to - * detect when a hotplug event has occurred during the loading process. - * If 0, no detection occurs. Otherwise, the scheduler will fail to - * load if the sequence number does not match @scx_hotplug_seq on the - * enable path. - */ - u64 hotplug_seq; - - /** - * @cgroup_id: When >1, attach the scheduler as a sub-scheduler on the - * specified cgroup. - */ - u64 sub_cgroup_id; - - /** - * @name: BPF scheduler's name - * - * Must be a non-zero valid BPF object name including only isalnum(), - * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the - * BPF scheduler is enabled. - */ - char name[SCX_OPS_NAME_LEN]; - - /* internal use only, must be NULL */ - void __rcu *priv; - - /* - * Deprecated callbacks. Kept at the end of the struct so the cid-form - * struct (sched_ext_ops_cid) can omit them without affecting the - * shared field offsets. Use SCX_ENQ_IMMED instead. Sitting past - * SCX_OPI_END means has_op doesn't cover them, so SCX_HAS_OP() cannot - * be used; callers must test sch->ops.cpu_acquire / cpu_release - * directly. - */ - - /** - * @cpu_acquire: A CPU is becoming available to the BPF scheduler - * @cpu: The CPU being acquired by the BPF scheduler. - * @args: Acquire arguments, see the struct definition. - * - * A CPU that was previously released from the BPF scheduler is now once - * again under its control. Deprecated; use SCX_ENQ_IMMED instead. - */ - void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); - - /** - * @cpu_release: A CPU is taken away from the BPF scheduler - * @cpu: The CPU being released by the BPF scheduler. - * @args: Release arguments, see the struct definition. - * - * The specified CPU is no longer under the control of the BPF - * scheduler. This could be because it was preempted by a higher - * priority sched_class, though there may be other reasons as well. The - * caller should consult @args->reason to determine the cause. - * Deprecated; use SCX_ENQ_IMMED instead. - */ - void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); -}; - -/** - * struct sched_ext_ops_cid - cid-form alternative to struct sched_ext_ops - * - * Mirrors struct sched_ext_ops with cpu/cpumask substituted with cid/cmask - * where applicable. Layout up to and including @priv matches sched_ext_ops - * byte-for-byte (verified by BUILD_BUG_ON checks at scx_init() time) so - * shared field offsets work for both struct types in bpf_scx_init_member() - * and bpf_scx_check_member(). The deprecated cpu_acquire/cpu_release - * callbacks at the tail of sched_ext_ops are omitted here entirely. - * - * Differences from sched_ext_ops: - * - select_cpu -> select_cid (returns cid) - * - dispatch -> dispatch (cpu arg is now cid) - * - update_idle -> update_idle (cpu arg is now cid) - * - set_cpumask -> set_cmask (cmask instead of cpumask) - * - cpu_online -> cid_online - * - cpu_offline -> cid_offline - * - dump_cpu -> dump_cid - * - cpu_acquire/cpu_release -> not present (deprecated in sched_ext_ops) - * - * BPF schedulers using this type cannot call cpu-form scx_bpf_* kfuncs; - * use the cid-form variants instead. Enforced at BPF verifier time via - * scx_kfunc_context_filter() branching on prog->aux->st_ops. - * - * See sched_ext_ops for callback documentation. - */ -struct sched_ext_ops_cid { - s32 (*select_cid)(struct task_struct *p, s32 prev_cid, u64 wake_flags); - void (*enqueue)(struct task_struct *p, u64 enq_flags); - void (*dequeue)(struct task_struct *p, u64 deq_flags); - void (*dispatch)(s32 cid, struct task_struct *prev); - void (*tick)(struct task_struct *p); - void (*runnable)(struct task_struct *p, u64 enq_flags); - void (*running)(struct task_struct *p); - void (*stopping)(struct task_struct *p, bool runnable); - void (*quiescent)(struct task_struct *p, u64 deq_flags); - bool (*yield)(struct task_struct *from, struct task_struct *to); - bool (*core_sched_before)(struct task_struct *a, - struct task_struct *b); - void (*set_weight)(struct task_struct *p, u32 weight); - void (*set_cmask)(struct task_struct *p, - const struct scx_cmask *cmask); - void (*update_idle)(s32 cid, bool idle); - s32 (*init_task)(struct task_struct *p, - struct scx_init_task_args *args); - void (*exit_task)(struct task_struct *p, - struct scx_exit_task_args *args); - void (*enable)(struct task_struct *p); - void (*disable)(struct task_struct *p); - void (*dump)(struct scx_dump_ctx *ctx); - void (*dump_cid)(struct scx_dump_ctx *ctx, s32 cid, bool idle); - void (*dump_task)(struct scx_dump_ctx *ctx, struct task_struct *p); -#ifdef CONFIG_EXT_GROUP_SCHED - s32 (*cgroup_init)(struct cgroup *cgrp, - struct scx_cgroup_init_args *args); - void (*cgroup_exit)(struct cgroup *cgrp); - s32 (*cgroup_prep_move)(struct task_struct *p, - struct cgroup *from, struct cgroup *to); - void (*cgroup_move)(struct task_struct *p, - struct cgroup *from, struct cgroup *to); - void (*cgroup_cancel_move)(struct task_struct *p, - struct cgroup *from, struct cgroup *to); - void (*cgroup_set_weight)(struct cgroup *cgrp, u32 weight); - void (*cgroup_set_bandwidth)(struct cgroup *cgrp, - u64 period_us, u64 quota_us, u64 burst_us); - void (*cgroup_set_idle)(struct cgroup *cgrp, bool idle); -#endif /* CONFIG_EXT_GROUP_SCHED */ - s32 (*sub_attach)(struct scx_sub_attach_args *args); - void (*sub_detach)(struct scx_sub_detach_args *args); - void (*cid_online)(s32 cid); - void (*cid_offline)(s32 cid); - s32 (*init)(void); - void (*exit)(struct scx_exit_info *info); - - /* Data fields - must match sched_ext_ops layout exactly */ - u32 dispatch_max_batch; - u64 flags; - u32 timeout_ms; - u32 exit_dump_len; - u64 hotplug_seq; - u64 sub_cgroup_id; - char name[SCX_OPS_NAME_LEN]; - - /* internal use only, must be NULL */ - void __rcu *priv; - - /* layout end anchor for the BUILD_BUG_ON in scx_init(); keep last */ - char __end[0]; -}; - -enum scx_opi { - SCX_OPI_BEGIN = 0, - SCX_OPI_NORMAL_BEGIN = 0, - SCX_OPI_NORMAL_END = SCX_OP_IDX(cpu_online), - SCX_OPI_CPU_HOTPLUG_BEGIN = SCX_OP_IDX(cpu_online), - SCX_OPI_CPU_HOTPLUG_END = SCX_OP_IDX(init), - SCX_OPI_END = SCX_OP_IDX(init), -}; - -/* - * Collection of event counters. Event types are placed in descending order. - */ -struct scx_event_stats { - /* - * If ops.select_cpu() returns a CPU which can't be used by the task, - * the core scheduler code silently picks a fallback CPU. - */ - s64 SCX_EV_SELECT_CPU_FALLBACK; - - /* - * When dispatching to a local DSQ, the CPU may have gone offline in - * the meantime. In this case, the task is bounced to the global DSQ. - */ - s64 SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE; - - /* - * If SCX_OPS_ENQ_LAST is not set, the number of times that a task - * continued to run because there were no other tasks on the CPU. - */ - s64 SCX_EV_DISPATCH_KEEP_LAST; - - /* - * If SCX_OPS_ENQ_EXITING is not set, the number of times that a task - * is dispatched to a local DSQ when exiting. - */ - s64 SCX_EV_ENQ_SKIP_EXITING; - - /* - * If SCX_OPS_ENQ_MIGRATION_DISABLED is not set, the number of times a - * migration disabled task skips ops.enqueue() and is dispatched to its - * local DSQ. - */ - s64 SCX_EV_ENQ_SKIP_MIGRATION_DISABLED; - - /* - * The number of times a task, enqueued on a local DSQ with - * SCX_ENQ_IMMED, was re-enqueued because the CPU was not available for - * immediate execution. - */ - s64 SCX_EV_REENQ_IMMED; - - /* - * The number of times a reenq of local DSQ caused another reenq of - * local DSQ. This can happen when %SCX_ENQ_IMMED races against a higher - * priority class task even if the BPF scheduler always satisfies the - * prerequisites for %SCX_ENQ_IMMED at the time of enqueue. However, - * that scenario is very unlikely and this count going up regularly - * indicates that the BPF scheduler is handling %SCX_ENQ_REENQ - * incorrectly causing recursive reenqueues. - */ - s64 SCX_EV_REENQ_LOCAL_REPEAT; - - /* - * Total number of times a task's time slice was refilled with the - * default value (SCX_SLICE_DFL). - */ - s64 SCX_EV_REFILL_SLICE_DFL; - - /* - * The total duration of bypass modes in nanoseconds. - */ - s64 SCX_EV_BYPASS_DURATION; - - /* - * The number of tasks dispatched in the bypassing mode. - */ - s64 SCX_EV_BYPASS_DISPATCH; - - /* - * The number of times the bypassing mode has been activated. - */ - s64 SCX_EV_BYPASS_ACTIVATE; - - /* - * The number of times the scheduler attempted to insert a task that it - * doesn't own into a DSQ. Such attempts are ignored. - * - * As BPF schedulers are allowed to ignore dequeues, it's difficult to - * tell whether such an attempt is from a scheduler malfunction or an - * ignored dequeue around sub-sched enabling. If this count keeps going - * up regardless of sub-sched enabling, it likely indicates a bug in the - * scheduler. - */ - s64 SCX_EV_INSERT_NOT_OWNED; - - /* - * The number of times tasks from bypassing descendants are scheduled - * from sub_bypass_dsq's. - */ - s64 SCX_EV_SUB_BYPASS_DISPATCH; -}; - -struct scx_sched; - -enum scx_sched_pcpu_flags { - SCX_SCHED_PCPU_BYPASSING = 1LLU << 0, -}; - -/* dispatch buf */ -struct scx_dsp_buf_ent { - struct task_struct *task; - unsigned long qseq; - u64 dsq_id; - u64 enq_flags; -}; - -struct scx_dsp_ctx { - struct rq *rq; - u32 cursor; - u32 nr_tasks; - struct scx_dsp_buf_ent buf[]; -}; - -struct scx_deferred_reenq_local { - struct list_head node; - u64 flags; - u64 seq; - u32 cnt; -}; - -struct scx_sched_pcpu { - struct scx_sched *sch; - u64 flags; /* protected by rq lock */ - - /* - * The event counters are in a per-CPU variable to minimize the - * accounting overhead. A system-wide view on the event counter is - * constructed when requested by scx_bpf_events(). - */ - struct scx_event_stats event_stats; - - struct scx_deferred_reenq_local deferred_reenq_local; - struct scx_dispatch_q bypass_dsq; -#ifdef CONFIG_EXT_SUB_SCHED - u32 bypass_host_seq; -#endif - - /* must be the last entry - contains flex array */ - struct scx_dsp_ctx dsp_ctx; -}; - -struct scx_sched_pnode { - struct scx_dispatch_q global_dsq; -}; - -struct scx_sched { - /* - * cpu-form and cid-form ops share field offsets up to .priv (verified - * by BUILD_BUG_ON in scx_init()). The anonymous union lets the kernel - * access either view of the same storage without function-pointer - * casts: use .ops for cpu-form and shared fields, .ops_cid for the - * cid-renamed callbacks (set_cmask, select_cid, cid_online, ...). - */ - union { - struct sched_ext_ops ops; - struct sched_ext_ops_cid ops_cid; - }; - bool is_cid_type; /* true if registered via bpf_sched_ext_ops_cid */ - - /* - * Arena map auto-discovered from member progs at struct_ops attach. - * cid-form schedulers must use exactly one arena across all member - * progs. NULL on cpu-form. - * - * @arena_pool sub-allocates @arena_map. Each gen_pool chunk is added - * at the kernel-side mapping address. @arena_kern_base is the start - * of the arena's kern_vm range. See scx_arena_to_kaddr() and - * scx_kaddr_to_arena(). - */ - struct bpf_map *arena_map; - struct gen_pool *arena_pool; - uintptr_t arena_kern_base; - - /* - * Per-CPU arena cmask used by scx_call_op_set_cpumask() to hand a cmask - * to ops_cid.set_cmask(). The kernel writes through the stored kern_va - * and hands BPF its arena pointer via scx_kaddr_to_arena(). - */ - struct scx_cmask * __percpu *set_cmask_scratch; - - DECLARE_BITMAP(has_op, SCX_OPI_END); - - /* - * Dispatch queues. - * - * The global DSQ (%SCX_DSQ_GLOBAL) is split per-node for scalability. - * This is to avoid live-locking in bypass mode where all tasks are - * dispatched to %SCX_DSQ_GLOBAL and all CPUs consume from it. If - * per-node split isn't sufficient, it can be further split. - */ - struct rhashtable dsq_hash; - struct scx_sched_pnode **pnode; - struct scx_sched_pcpu __percpu *pcpu; - - u64 slice_dfl; - u64 bypass_timestamp; - s32 bypass_depth; - - /* bypass dispatch path enable state, see bypass_dsp_enabled() */ - unsigned long bypass_dsp_claim; - atomic_t bypass_dsp_enable_depth; - - bool aborting; - bool dump_disabled; /* protected by scx_dump_lock */ - u32 dsp_max_batch; - s32 level; - - /* - * Updates to the following warned bitfields can race causing RMW issues - * but it doesn't really matter. - */ - bool warned_zero_slice:1; - bool warned_deprecated_rq:1; - bool warned_unassoc_progs:1; - - struct list_head all; - -#ifdef CONFIG_EXT_SUB_SCHED - struct rhash_head hash_node; - - struct list_head children; - struct list_head sibling; - struct cgroup *cgrp; - char *cgrp_path; - struct kset *sub_kset; - - bool sub_attached; -#endif /* CONFIG_EXT_SUB_SCHED */ - - /* - * The maximum amount of time in jiffies that a task may be runnable - * without being scheduled on a CPU. If this timeout is exceeded, it - * will trigger scx_error(). - */ - unsigned long watchdog_timeout; - - atomic_t exit_kind; - struct scx_exit_info *exit_info; - - struct kobject kobj; - - struct kthread_worker *helper; - struct irq_work disable_irq_work; - struct kthread_work disable_work; - struct timer_list bypass_lb_timer; - cpumask_var_t bypass_lb_donee_cpumask; - cpumask_var_t bypass_lb_resched_cpumask; - struct rcu_work rcu_work; - - /* all ancestors including self */ - struct scx_sched *ancestors[]; -}; - -/** - * scx_arena_to_kaddr - Translate a BPF-arena pointer to its kernel address - * @sch: scheduler whose arena hosts @bpf_ptr - * @bpf_ptr: BPF-arena pointer, only the low 32 bits are used - * - * The (u32) cast normalizes any input into the arena's 4 GiB kern_vm range, - * which combined with scratch-page fault recovery makes the returned pointer - * safe to dereference up to GUARD_SZ / 2 past the intended object. Accesses - * larger than GUARD_SZ / 2 must be explicitly bounds-checked. - */ -static inline void *scx_arena_to_kaddr(struct scx_sched *sch, const void *bpf_ptr) -{ - return (void *)(sch->arena_kern_base + (u32)(uintptr_t)bpf_ptr); -} - -/** - * scx_kaddr_to_arena - Translate a kernel arena address to its BPF form - * @sch: scheduler whose arena hosts @kaddr - * @kaddr: kernel-side arena address, supplied by trusted kernel code - */ -static inline void *scx_kaddr_to_arena(struct scx_sched *sch, const void *kaddr) -{ - return (void *)((uintptr_t)kaddr - sch->arena_kern_base); -} - -enum scx_wake_flags { - /* expose select WF_* flags as enums */ - SCX_WAKE_FORK = WF_FORK, - SCX_WAKE_TTWU = WF_TTWU, - SCX_WAKE_SYNC = WF_SYNC, -}; - -enum scx_enq_flags { - /* expose select ENQUEUE_* flags as enums */ - SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, - SCX_ENQ_HEAD = ENQUEUE_HEAD, - SCX_ENQ_CPU_SELECTED = ENQUEUE_RQ_SELECTED, - - /* high 32bits are SCX specific */ - - /* - * Set the following to trigger preemption when calling - * scx_bpf_dsq_insert() with a local dsq as the target. The slice of the - * current task is cleared to zero and the CPU is kicked into the - * scheduling path. Implies %SCX_ENQ_HEAD. - */ - SCX_ENQ_PREEMPT = 1LLU << 32, - - /* - * Only allowed on local DSQs. Guarantees that the task either gets - * on the CPU immediately and stays on it, or gets reenqueued back - * to the BPF scheduler. It will never linger on a local DSQ or be - * silently put back after preemption. - * - * The protection persists until the next fresh enqueue - it - * survives SAVE/RESTORE cycles, slice extensions and preemption. - * If the task can't stay on the CPU for any reason, it gets - * reenqueued back to the BPF scheduler. - * - * Exiting and migration-disabled tasks bypass ops.enqueue() and - * are placed directly on a local DSQ without IMMED protection - * unless %SCX_OPS_ENQ_EXITING and %SCX_OPS_ENQ_MIGRATION_DISABLED - * are set respectively. - */ - SCX_ENQ_IMMED = 1LLU << 33, - - /* - * The task being enqueued was previously enqueued on a DSQ, but was - * removed and is being re-enqueued. See SCX_TASK_REENQ_* flags to find - * out why a given task is being reenqueued. - */ - SCX_ENQ_REENQ = 1LLU << 40, - - /* - * The task being enqueued is the only task available for the cpu. By - * default, ext core keeps executing such tasks but when - * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with the - * %SCX_ENQ_LAST flag set. - * - * The BPF scheduler is responsible for triggering a follow-up - * scheduling event. Otherwise, Execution may stall. - */ - SCX_ENQ_LAST = 1LLU << 41, - - /* high 8 bits are internal */ - __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, - - SCX_ENQ_CLEAR_OPSS = 1LLU << 56, - SCX_ENQ_DSQ_PRIQ = 1LLU << 57, - SCX_ENQ_NESTED = 1LLU << 58, - SCX_ENQ_GDSQ_FALLBACK = 1LLU << 59, /* fell back to global DSQ */ -}; - -enum scx_deq_flags { - /* expose select DEQUEUE_* flags as enums */ - SCX_DEQ_SLEEP = DEQUEUE_SLEEP, - - /* high 32bits are SCX specific */ - - /* - * The generic core-sched layer decided to execute the task even though - * it hasn't been dispatched yet. Dequeue from the BPF side. - */ - SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, - - /* - * The task is being dequeued due to a property change (e.g., - * sched_setaffinity(), sched_setscheduler(), set_user_nice(), - * etc.). - */ - SCX_DEQ_SCHED_CHANGE = 1LLU << 33, -}; - -enum scx_reenq_flags { - /* low 16bits determine which tasks should be reenqueued */ - SCX_REENQ_ANY = 1LLU << 0, /* all tasks */ - - __SCX_REENQ_FILTER_MASK = 0xffffLLU, - - __SCX_REENQ_USER_MASK = SCX_REENQ_ANY, - - /* bits 32-35 used by task_should_reenq() */ - SCX_REENQ_TSR_RQ_OPEN = 1LLU << 32, - SCX_REENQ_TSR_NOT_FIRST = 1LLU << 33, - - __SCX_REENQ_TSR_MASK = 0xfLLU << 32, -}; - -enum scx_pick_idle_cpu_flags { - SCX_PICK_IDLE_CORE = 1LLU << 0, /* pick a CPU whose SMT siblings are also idle */ - SCX_PICK_IDLE_IN_NODE = 1LLU << 1, /* pick a CPU in the same target NUMA node */ -}; - -enum scx_kick_flags { - /* - * Kick the target CPU if idle. Guarantees that the target CPU goes - * through at least one full scheduling cycle before going idle. If the - * target CPU can be determined to be currently not idle and going to go - * through a scheduling cycle before going idle, noop. - */ - SCX_KICK_IDLE = 1LLU << 0, - - /* - * Preempt the current task and execute the dispatch path. If the - * current task of the target CPU is an SCX task, its ->scx.slice is - * cleared to zero before the scheduling path is invoked so that the - * task expires and the dispatch path is invoked. - */ - SCX_KICK_PREEMPT = 1LLU << 1, - - /* - * The scx_bpf_kick_cpu() call will return after the current SCX task of - * the target CPU switches out. This can be used to implement e.g. core - * scheduling. This has no effect if the current task on the target CPU - * is not on SCX. - */ - SCX_KICK_WAIT = 1LLU << 2, -}; - -enum scx_tg_flags { - SCX_TG_ONLINE = 1U << 0, - SCX_TG_INITED = 1U << 1, -}; - -enum scx_enable_state { - SCX_ENABLING, - SCX_ENABLED, - SCX_DISABLING, - SCX_DISABLED, -}; - -static const char *scx_enable_state_str[] = { - [SCX_ENABLING] = "enabling", - [SCX_ENABLED] = "enabled", - [SCX_DISABLING] = "disabling", - [SCX_DISABLED] = "disabled", -}; - -/* - * Task Ownership State Machine (sched_ext_entity->ops_state) - * - * The sched_ext core uses this state machine to track task ownership - * between the SCX core and the BPF scheduler. This allows the BPF - * scheduler to dispatch tasks without strict ordering requirements, while - * the SCX core safely rejects invalid dispatches. - * - * State Transitions - * - * .------------> NONE (owned by SCX core) - * | | ^ - * | enqueue | | direct dispatch - * | v | - * | QUEUEING -------' - * | | - * | enqueue | - * | completes | - * | v - * | QUEUED (owned by BPF scheduler) - * | | - * | dispatch | - * | | - * | v - * | DISPATCHING - * | | - * | dispatch | - * | completes | - * `---------------' - * - * State Descriptions - * - * - %SCX_OPSS_NONE: - * Task is owned by the SCX core. It's either on a run queue, running, - * or being manipulated by the core scheduler. The BPF scheduler has no - * claim on this task. - * - * - %SCX_OPSS_QUEUEING: - * Transitional state while transferring a task from the SCX core to - * the BPF scheduler. The task's rq lock is held during this state. - * Since QUEUEING is both entered and exited under the rq lock, dequeue - * can never observe this state (it would be a BUG). When finishing a - * dispatch, if the task is still in %SCX_OPSS_QUEUEING the completion - * path busy-waits for it to leave this state (via wait_ops_state()) - * before retrying. - * - * - %SCX_OPSS_QUEUED: - * Task is owned by the BPF scheduler. It's on a DSQ (dispatch queue) - * and the BPF scheduler is responsible for dispatching it. A QSEQ - * (queue sequence number) is embedded in this state to detect - * dispatch/dequeue races: if a task is dequeued and re-enqueued, the - * QSEQ changes and any in-flight dispatch operations targeting the old - * QSEQ are safely ignored. - * - * - %SCX_OPSS_DISPATCHING: - * Transitional state while transferring a task from the BPF scheduler - * back to the SCX core. This state indicates the BPF scheduler has - * selected the task for execution. When dequeue needs to take the task - * off a DSQ and it is still in %SCX_OPSS_DISPATCHING, the dequeue path - * busy-waits for it to leave this state (via wait_ops_state()) before - * proceeding. Exits to %SCX_OPSS_NONE when dispatch completes. - * - * Memory Ordering - * - * Transitions out of %SCX_OPSS_QUEUEING and %SCX_OPSS_DISPATCHING into - * %SCX_OPSS_NONE or %SCX_OPSS_QUEUED must use atomic_long_set_release() - * and waiters must use atomic_long_read_acquire(). This ensures proper - * synchronization between concurrent operations. - * - * Cross-CPU Task Migration - * - * When moving a task in the %SCX_OPSS_DISPATCHING state, we can't simply - * grab the target CPU's rq lock because a concurrent dequeue might be - * waiting on %SCX_OPSS_DISPATCHING while holding the source rq lock - * (deadlock). - * - * The sched_ext core uses a "lock dancing" protocol coordinated by - * p->scx.holding_cpu. When moving a task to a different rq: - * - * 1. Verify task can be moved (CPU affinity, migration_disabled, etc.) - * 2. Set p->scx.holding_cpu to the current CPU - * 3. Set task state to %SCX_OPSS_NONE; dequeue waits while DISPATCHING - * is set, so clearing DISPATCHING first prevents the circular wait - * (safe to lock the rq we need) - * 4. Unlock the current CPU's rq - * 5. Lock src_rq (where the task currently lives) - * 6. Verify p->scx.holding_cpu == current CPU, if not, dequeue won the - * race (dequeue clears holding_cpu to -1 when it takes the task), in - * this case migration is aborted - * 7. If src_rq == dst_rq: clear holding_cpu and enqueue directly - * into dst_rq's local DSQ (no lock swap needed) - * 8. Otherwise: call move_remote_task_to_local_dsq(), which releases - * src_rq, locks dst_rq, and performs the deactivate/activate - * migration cycle (dst_rq is held on return) - * 9. Unlock dst_rq and re-lock the current CPU's rq to restore - * the lock state expected by the caller - * - * If any verification fails, abort the migration. - * - * This state tracking allows the BPF scheduler to try to dispatch any task - * at any time regardless of its state. The SCX core can safely - * reject/ignore invalid dispatches, simplifying the BPF scheduler - * implementation. - */ -enum scx_ops_state { - SCX_OPSS_NONE, /* owned by the SCX core */ - SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ - SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ - SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ - - /* - * QSEQ brands each QUEUED instance so that, when dispatch races - * dequeue/requeue, the dispatcher can tell whether it still has a claim - * on the task being dispatched. - * - * As some 32bit archs can't do 64bit store_release/load_acquire, - * p->scx.ops_state is atomic_long_t which leaves 30 bits for QSEQ on - * 32bit machines. The dispatch race window QSEQ protects is very narrow - * and runs with IRQ disabled. 30 bits should be sufficient. - */ - SCX_OPSS_QSEQ_SHIFT = 2, -}; - -/* Use macros to ensure that the type is unsigned long for the masks */ -#define SCX_OPSS_STATE_MASK ((1LU << SCX_OPSS_QSEQ_SHIFT) - 1) -#define SCX_OPSS_QSEQ_MASK (~SCX_OPSS_STATE_MASK) - -extern struct scx_sched __rcu *scx_root; -DECLARE_PER_CPU(struct rq *, scx_locked_rq_state); - -/* - * True when the currently loaded scheduler hierarchy is cid-form. All scheds - * in a hierarchy share one form, so this single key tells callsites which - * view to use without per-sch dereferences. Use scx_is_cid_type() to test. - */ -DECLARE_STATIC_KEY_FALSE(__scx_is_cid_type); - -int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id); - -bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where); - -__printf(5, 0) bool scx_vexit(struct scx_sched *sch, enum scx_exit_kind kind, - s64 exit_code, s32 exit_cpu, const char *fmt, - va_list args); -__printf(5, 6) bool __scx_exit(struct scx_sched *sch, enum scx_exit_kind kind, - s64 exit_code, s32 exit_cpu, const char *fmt, ...); - -#define scx_exit(sch, kind, exit_code, fmt, args...) \ - __scx_exit(sch, kind, exit_code, raw_smp_processor_id(), fmt, ##args) -#define scx_error(sch, fmt, args...) \ - scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args) -#define scx_verror(sch, fmt, args) \ - scx_vexit((sch), SCX_EXIT_ERROR, 0, raw_smp_processor_id(), fmt, args) - -/* - * Return the rq currently locked from an scx callback, or NULL if no rq is - * locked. - */ -static inline struct rq *scx_locked_rq(void) -{ - return __this_cpu_read(scx_locked_rq_state); -} - -static inline bool scx_bypassing(struct scx_sched *sch, s32 cpu) -{ - return unlikely(per_cpu_ptr(sch->pcpu, cpu)->flags & - SCX_SCHED_PCPU_BYPASSING); -} - -#ifdef CONFIG_EXT_SUB_SCHED -/** - * scx_task_sched - Find scx_sched scheduling a task - * @p: task of interest - * - * Return @p's scheduler instance. Must be called with @p's pi_lock or rq lock - * held. - */ -static inline struct scx_sched *scx_task_sched(const struct task_struct *p) -{ - return rcu_dereference_protected(p->scx.sched, - lockdep_is_held(&p->pi_lock) || - lockdep_is_held(__rq_lockp(task_rq(p)))); -} - -/** - * scx_task_sched_rcu - Find scx_sched scheduling a task - * @p: task of interest - * - * Return @p's scheduler instance. The returned scx_sched is RCU protected. - */ -static inline struct scx_sched *scx_task_sched_rcu(const struct task_struct *p) -{ - return rcu_dereference_all(p->scx.sched); -} - -/** - * scx_task_on_sched - Is a task on the specified sched? - * @sch: sched to test against - * @p: task of interest - * - * Returns %true if @p is on @sch, %false otherwise. - */ -static inline bool scx_task_on_sched(struct scx_sched *sch, - const struct task_struct *p) -{ - return rcu_access_pointer(p->scx.sched) == sch; -} - -/** - * scx_prog_sched - Find scx_sched associated with a BPF prog - * @aux: aux passed in from BPF to a kfunc - * - * To be called from kfuncs. Return the scheduler instance associated with the - * BPF program given the implicit kfunc argument aux. The returned scx_sched is - * RCU protected. - */ -static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux) -{ - struct sched_ext_ops *ops; - struct scx_sched *root; - - ops = bpf_prog_get_assoc_struct_ops(aux); - if (likely(ops)) - return rcu_dereference_all(ops->priv); - - root = rcu_dereference_all(scx_root); - if (root) { - /* - * COMPAT-v6.19: Schedulers built before sub-sched support was - * introduced may have unassociated non-struct_ops programs. - */ - if (!root->ops.sub_attach) - return root; - - if (!root->warned_unassoc_progs) { - printk_deferred(KERN_WARNING "sched_ext: Unassociated program %s (id %d)\n", - aux->name, aux->id); - root->warned_unassoc_progs = true; - } - } - - return NULL; -} -#else /* CONFIG_EXT_SUB_SCHED */ -static inline struct scx_sched *scx_task_sched(const struct task_struct *p) -{ - return rcu_dereference_protected(scx_root, - lockdep_is_held(&p->pi_lock) || - lockdep_is_held(__rq_lockp(task_rq(p)))); -} - -static inline struct scx_sched *scx_task_sched_rcu(const struct task_struct *p) -{ - return rcu_dereference_all(scx_root); -} - -static inline bool scx_task_on_sched(struct scx_sched *sch, - const struct task_struct *p) -{ - return true; -} - -static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux) -{ - return rcu_dereference_all(scx_root); -} -#endif /* CONFIG_EXT_SUB_SCHED */ diff --git a/kernel/sched/ext_types.h b/kernel/sched/ext_types.h deleted file mode 100644 index 8b3527e21fca..000000000000 --- a/kernel/sched/ext_types.h +++ /dev/null @@ -1,144 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Early sched_ext type definitions. - * - * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. - * Copyright (c) 2026 Tejun Heo - */ -#ifndef _KERNEL_SCHED_EXT_TYPES_H -#define _KERNEL_SCHED_EXT_TYPES_H - -enum scx_consts { - SCX_DSP_DFL_MAX_BATCH = 32, - SCX_DSP_MAX_LOOPS = 32, - SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, - - /* per-CPU chunk size for p->scx.tid allocation, see scx_alloc_tid() */ - SCX_TID_CHUNK = 1024, - - SCX_EXIT_BT_LEN = 64, - SCX_EXIT_MSG_LEN = 1024, - SCX_EXIT_DUMP_DFL_LEN = 32768, - - SCX_CPUPERF_ONE = SCHED_CAPACITY_SCALE, - - /* - * Iterating all tasks may take a while. Periodically drop - * scx_tasks_lock to avoid causing e.g. CSD and RCU stalls. - */ - SCX_TASK_ITER_BATCH = 32, - - SCX_BYPASS_HOST_NTH = 2, - - SCX_BYPASS_LB_DFL_INTV_US = 500 * USEC_PER_MSEC, - SCX_BYPASS_LB_DONOR_PCT = 125, - SCX_BYPASS_LB_MIN_DELTA_DIV = 4, - SCX_BYPASS_LB_BATCH = 256, - - SCX_REENQ_LOCAL_MAX_REPEAT = 256, - - SCX_SUB_MAX_DEPTH = 4, -}; - -/* - * Per-cid topology info. For each topology level (core, LLC, node), records - * the first cid in the unit and its global index. Global indices are - * consecutive integers assigned in cid-walk order, so e.g. core_idx ranges - * over [0, nr_cores_at_init) with no gaps. No-topo cids have all fields set - * to -1. - * - * @core_cid: first cid of this cid's core (smt-sibling group) - * @core_idx: global index of that core, in [0, nr_cores_at_init) - * @llc_cid: first cid of this cid's LLC - * @llc_idx: global index of that LLC, in [0, nr_llcs_at_init) - * @node_cid: first cid of this cid's NUMA node - * @node_idx: global index of that node, in [0, nr_nodes_at_init) - */ -struct scx_cid_topo { - s32 core_cid; - s32 core_idx; - s32 llc_cid; - s32 llc_idx; - s32 node_cid; - s32 node_idx; -}; - -/* - * cmask: variable-length, base-windowed bitmap over cid space - * ----------------------------------------------------------- - * - * A cmask covers the cid range [base, base + nr_cids). bits[] is aligned to the - * global 64-cid grid: bits[0] spans [base & ~63, (base & ~63) + 64), so the - * first (base & 63) bits of bits[0] are head padding and the trailing bits of - * the last active word past base + nr_cids are tail padding. Both stay zero; - * all mutating helpers preserve that. Words past the last active word are not - * read by any helper and have no constraint. - * - * Grid alignment means two cmasks always address bits[] against the same global - * 64-cid windows, so cross-cmask word ops (AND, OR, ...) reduce to - * - * dst->bits[i] OP= src->bits[i - delta] - * - * with no bit-shifting, regardless of how the two bases relate mod 64. - */ -struct scx_cmask { - u32 base; - u32 nr_cids; - u32 alloc_words; - u64 bits[] __counted_by(alloc_words); -}; - -/* - * Number of u64 words of bits[] storage that covers @nr_cids regardless of base - * alignment. The +1 absorbs up to 63 bits of head padding when base is not - * 64-aligned - always allocating one extra word beats branching on base or - * splitting the compute. The u64 cast keeps the +63 from wrapping when @nr_cids - * is near U32_MAX, so callers bounds-checking the result against @alloc_words - * catch the overflow instead of seeing a small value. - */ -#define SCX_CMASK_NR_WORDS(nr_cids) ((u32)(((u64)(nr_cids) + 63) / 64 + 1)) - -/** - * __SCX_CMASK_DEFINE - Define an on-stack cmask with explicit storage capacity - * @NAME: variable name to define - * @BASE: first cid of the active range - * @NR_CIDS: active range length - * @ALLOC_CIDS: storage capacity in cids, at least @NR_CIDS - * - * @NAME aliases zero-initialized storage with the active range set to - * [BASE, BASE + NR_CIDS). Use scx_cmask_reframe() to reshape later, up to - * @ALLOC_CIDS. - */ -#define __SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, ALLOC_CIDS) \ - _DEFINE_FLEX(struct scx_cmask, NAME, bits, SCX_CMASK_NR_WORDS(ALLOC_CIDS), \ - = { .base = (BASE), \ - .nr_cids = (NR_CIDS), \ - .alloc_words = SCX_CMASK_NR_WORDS(ALLOC_CIDS) }) - -/** - * SCX_CMASK_DEFINE - Define an on-stack cmask on tight storage - * @NAME: variable name to define - * @BASE: first cid of the active range - * @NR_CIDS: active range length, also storage capacity - * - * @NAME aliases zero-initialized storage with the active range and storage - * both [BASE, BASE + NR_CIDS). - */ -#define SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS) \ - __SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, NR_CIDS) - -/** - * SCX_CMASK_DEFINE_SHARD - Define an on-stack cmask sized to one shard - * @NAME: variable name to define - * @BASE: first cid of the active range - * @NR_CIDS: active range length, must be <= SCX_CID_SHARD_MAX_CPUS - * - * Storage is fixed at SCX_CID_SHARD_MAX_CPUS, active range framed by - * (BASE, NR_CIDS). Passing NR_CIDS > SCX_CID_SHARD_MAX_CPUS leaves the - * cmask claiming more bits than storage holds and subsequent cmask - * operations will overrun. - */ -#define SCX_CMASK_DEFINE_SHARD(NAME, BASE, NR_CIDS) \ - __SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, SCX_CID_SHARD_MAX_CPUS) - -#endif /* _KERNEL_SCHED_EXT_TYPES_H */ diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index c7c2dea65edd..56acf502ba26 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -4211,6 +4211,6 @@ DEFINE_CLASS(sched_change, struct sched_change_ctx *, DEFINE_CLASS_IS_UNCONDITIONAL(sched_change) -#include "ext.h" +#include "ext/ext.h" #endif /* _KERNEL_SCHED_SCHED_H */ -- cgit v1.2.3 From 865730eec5435ce40a5dc3c615077d04c8f95098 Mon Sep 17 00:00:00 2001 From: Ethan Nelson-Moore Date: Sat, 13 Jun 2026 14:35:43 -0700 Subject: genirq/msi: Correct CONFIG_PCI_MSI_ARCH_FALLBACKS macro name in comment A comment in kernel/irq/msi.c incorrectly refers to CONFIG_PCI_MSI_ARCH_FALLBACK instead of CONFIG_PCI_MSI_ARCH_FALLBACKS. Correct it. Discovered while searching for CONFIG_* symbols referenced in code but not defined in any Kconfig file. Signed-off-by: Ethan Nelson-Moore Signed-off-by: Thomas Gleixner Link: https://patch.msgid.link/20260613213544.90613-1-enelsonmoore@gmail.com --- kernel/irq/msi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index 3cafa40e6ce3..903be7289c53 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -591,7 +591,7 @@ void msi_device_destroy_sysfs(struct device *dev) msi_for_each_desc(desc, dev, MSI_DESC_ALL) msi_sysfs_remove_desc(dev, desc); } -#endif /* CONFIG_PCI_MSI_ARCH_FALLBACK || CONFIG_PCI_XEN */ +#endif /* CONFIG_PCI_MSI_ARCH_FALLBACKS || CONFIG_PCI_XEN */ #else /* CONFIG_SYSFS */ static inline int msi_sysfs_create_group(struct device *dev) { return 0; } static inline int msi_sysfs_populate_desc(struct device *dev, struct msi_desc *desc) { return 0; } -- cgit v1.2.3 From 3a354149bceacadbcf7d7b4766f5ef26a85892ab Mon Sep 17 00:00:00 2001 From: Nuoqi Gui Date: Wed, 17 Jun 2026 23:20:20 +0800 Subject: bpf: Preserve pointer spill metadata during half-slot cleanup __clean_func_state() cleans dead stack slots in 4-byte halves. When the high half of a STACK_SPILL slot is dead and the low half remains live, cleanup converts the live low half to STACK_MISC or STACK_ZERO and clears the saved spilled_ptr metadata. That conversion is safe only for scalar spills. For a pointer spill, this metadata clear lets a later 32-bit fill from the still-live half avoid the normal non-scalar register-fill check and be treated as an ordinary scalar stack read. Leave non-scalar spill slots intact in this half-live shape. This is conservative for pruning and preserves the existing check_stack_read_fixed_off() rejection path for partial fills from pointer spills. Fixes: be23266b4a08 ("bpf: 4-byte precise clean_verifier_state") Acked-by: Eduard Zingerman Signed-off-by: Nuoqi Gui Link: https://lore.kernel.org/r/20260617-f01-06-half-slot-pointer-spill-v2-1-42b9cdc3cf64@mails.tsinghua.edu.cn Signed-off-by: Alexei Starovoitov --- kernel/bpf/states.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 32f346ce3ffc..ea2153cf28d0 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -436,12 +436,10 @@ static void __clean_func_state(struct bpf_verifier_env *env, continue; /* - * Only destroy spilled_ptr when hi half is dead. - * If hi half is still live with STACK_SPILL, the - * spilled_ptr metadata is needed for correct state - * comparison in stacksafe(). - * is_spilled_reg() is using slot_type[7], but - * is_spilled_scalar_after() check either slot_type[0] or [4] + * Only scalar spills can be degraded to raw stack bytes + * when their high half is dead. Pointer spills need the + * saved spilled_ptr metadata so partial fills keep + * rejecting as non-scalar register fills. */ if (!hi_live) { struct bpf_reg_state *spill = &st->stack[i].spilled_ptr; @@ -449,6 +447,9 @@ static void __clean_func_state(struct bpf_verifier_env *env, if (lo_live && stype == STACK_SPILL) { u8 val = STACK_MISC; + if (spill->type != SCALAR_VALUE) + continue; + /* * 8 byte spill of scalar 0 where half slot is dead * should become STACK_ZERO in lo 4 bytes. -- cgit v1.2.3 From 3cd1f76be638b7386201171e7bb4c88095774dd5 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 22 Jun 2026 07:29:39 -1000 Subject: sched_ext: Make kernel/sched/ext/ sources self-contained for clangd The sources under kernel/sched/ext/ build as a single translation unit: build_policy.c includes the source files and headers. An LSP/clangd editor parses each as a standalone unit, sees no types, and reports a flood of errors. Give each header its dependencies and include guard, and have each source include the headers it uses. ext.c, arena.c and the ext headers now parse clean standalone. idle.c and cid.c still reference a few macros and helpers defined in ext.c. The next patch moves those to shared headers. Suggested-by: Peter Zijlstra Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/arena.c | 4 ++++ kernel/sched/ext/arena.h | 2 ++ kernel/sched/ext/cid.c | 3 +++ kernel/sched/ext/cid.h | 2 ++ kernel/sched/ext/ext.c | 13 +++++++++++++ kernel/sched/ext/idle.c | 3 +++ kernel/sched/ext/idle.h | 4 ++++ kernel/sched/ext/internal.h | 8 ++++++++ kernel/sched/ext/types.h | 6 ++++++ 9 files changed, 45 insertions(+) (limited to 'kernel') diff --git a/kernel/sched/ext/arena.c b/kernel/sched/ext/arena.c index 493c2424f842..5783694ec21d 100644 --- a/kernel/sched/ext/arena.c +++ b/kernel/sched/ext/arena.c @@ -15,6 +15,10 @@ * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. * Copyright (c) 2026 Tejun Heo */ +#include + +#include "internal.h" +#include "arena.h" enum scx_arena_consts { SCX_ARENA_MIN_ORDER = 3, /* 8-byte minimum sub-allocation */ diff --git a/kernel/sched/ext/arena.h b/kernel/sched/ext/arena.h index 4f3610160102..c378ae5fbc02 100644 --- a/kernel/sched/ext/arena.h +++ b/kernel/sched/ext/arena.h @@ -8,6 +8,8 @@ #ifndef _KERNEL_SCHED_EXT_ARENA_H #define _KERNEL_SCHED_EXT_ARENA_H +#include + struct scx_sched; s32 scx_arena_pool_init(struct scx_sched *sch); diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c index aeaea88f34c5..af83084ec740 100644 --- a/kernel/sched/ext/cid.c +++ b/kernel/sched/ext/cid.c @@ -7,6 +7,9 @@ */ #include +#include "internal.h" +#include "cid.h" + /* * cid tables. * diff --git a/kernel/sched/ext/cid.h b/kernel/sched/ext/cid.h index 6e657fd147b0..41d0802c6af3 100644 --- a/kernel/sched/ext/cid.h +++ b/kernel/sched/ext/cid.h @@ -33,6 +33,8 @@ #ifndef _KERNEL_SCHED_EXT_CID_H #define _KERNEL_SCHED_EXT_CID_H +#include "internal.h" + struct scx_sched; /* diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 00fe6cc6d7e2..f3253c946764 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -6,6 +6,19 @@ * Copyright (c) 2022 Tejun Heo * Copyright (c) 2022 David Vernet */ +#include +#include +#include +#include +#include +#include +#include + +#include "../pelt.h" +#include "internal.h" +#include "cid.h" +#include "arena.h" +#include "idle.h" static DEFINE_RAW_SPINLOCK(scx_sched_lock); diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c index 2077373d8da3..8e8c6201b7df 100644 --- a/kernel/sched/ext/idle.c +++ b/kernel/sched/ext/idle.c @@ -9,6 +9,9 @@ * Copyright (c) 2022 David Vernet * Copyright (c) 2024 Andrea Righi */ +#include "internal.h" +#include "cid.h" +#include "idle.h" /* Enable/disable built-in idle CPU selection policy */ static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); diff --git a/kernel/sched/ext/idle.h b/kernel/sched/ext/idle.h index 8d169d3bbdf9..87a0e58f1eb7 100644 --- a/kernel/sched/ext/idle.h +++ b/kernel/sched/ext/idle.h @@ -10,7 +10,11 @@ #ifndef _KERNEL_SCHED_EXT_IDLE_H #define _KERNEL_SCHED_EXT_IDLE_H +#include + +struct cpumask; struct sched_ext_ops; +struct task_struct; extern struct btf_id_set8 scx_kfunc_ids_idle; extern struct btf_id_set8 scx_kfunc_ids_select_cpu; diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index b04701190b23..1f5312b3b387 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -5,6 +5,12 @@ * Copyright (c) 2025 Meta Platforms, Inc. and affiliates. * Copyright (c) 2025 Tejun Heo */ +#ifndef _KERNEL_SCHED_EXT_INTERNAL_H +#define _KERNEL_SCHED_EXT_INTERNAL_H + +#include "../sched.h" +#include "types.h" + #define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) #define SCX_MOFF_IDX(moff) ((moff) / sizeof(void (*)(void))) @@ -1651,3 +1657,5 @@ static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux) return rcu_dereference_all(scx_root); } #endif /* CONFIG_EXT_SUB_SCHED */ + +#endif /* _KERNEL_SCHED_EXT_INTERNAL_H */ diff --git a/kernel/sched/ext/types.h b/kernel/sched/ext/types.h index 8b3527e21fca..bc74eafd43f1 100644 --- a/kernel/sched/ext/types.h +++ b/kernel/sched/ext/types.h @@ -8,6 +8,12 @@ #ifndef _KERNEL_SCHED_EXT_TYPES_H #define _KERNEL_SCHED_EXT_TYPES_H +#include +#include +#include +#include +#include + enum scx_consts { SCX_DSP_DFL_MAX_BATCH = 32, SCX_DSP_MAX_LOOPS = 32, -- cgit v1.2.3 From 4437ad129cf5b37c00a5bc9fa5989d1da4d64d07 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 22 Jun 2026 07:29:39 -1000 Subject: sched_ext: Move shared helpers from ext.c into internal.h and cid.h idle.c and cid.c are included into build_policy.c together with ext.c and use helpers that ext.c defines. Because the helpers live in ext.c, the two files can not parse as standalone units and clangd reports errors in them. Move the helpers to the headers they belong to. The op-dispatch macros and helpers plus scx_parent() to internal.h, and scx_cpu_arg()/scx_cpu_ret() to cid.h. No functional change. idle.c and cid.c now parse clean standalone. Suggested-by: Peter Zijlstra Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/cid.h | 21 +++++++ kernel/sched/ext/ext.c | 141 -------------------------------------------- kernel/sched/ext/internal.h | 121 +++++++++++++++++++++++++++++++++++++ 3 files changed, 142 insertions(+), 141 deletions(-) (limited to 'kernel') diff --git a/kernel/sched/ext/cid.h b/kernel/sched/ext/cid.h index 41d0802c6af3..9c4f4b907f12 100644 --- a/kernel/sched/ext/cid.h +++ b/kernel/sched/ext/cid.h @@ -270,4 +270,25 @@ static inline u32 scx_cmask_nr_used_words(const struct scx_cmask *m) __w && ((cid) = __bs + __wi * 64 + __ffs64(__w), true); \ __w &= __w - 1) +/* + * scx_cpu_arg() wraps a cpu arg being handed to an SCX op. For cid-form + * schedulers it resolves to the matching cid; for cpu-form it passes @cpu + * through. scx_cpu_ret() is the inverse for a cpu/cid returned from an op + * (currently only ops.select_cpu); it validates the BPF-supplied cid and + * triggers scx_error() on @sch if invalid. + */ +static inline s32 scx_cpu_arg(s32 cpu) +{ + if (scx_is_cid_type()) + return __scx_cpu_to_cid(cpu); + return cpu; +} + +static inline s32 scx_cpu_ret(struct scx_sched *sch, s32 cpu_or_cid) +{ + if (cpu_or_cid < 0 || !scx_is_cid_type()) + return cpu_or_cid; + return scx_cid_to_cpu(sch, cpu_or_cid); +} + #endif /* _KERNEL_SCHED_EXT_CID_H */ diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index f3253c946764..691d53fe0f64 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -259,8 +259,6 @@ __printf(5, 6) bool __scx_exit(struct scx_sched *sch, return ret; } -#define SCX_HAS_OP(sch, op) test_bit(SCX_OP_IDX(op), (sch)->has_op) - static long jiffies_delta_msecs(unsigned long at, unsigned long now) { if (time_after(at, now)) @@ -275,20 +273,6 @@ static bool u32_before(u32 a, u32 b) } #ifdef CONFIG_EXT_SUB_SCHED -/** - * scx_parent - Find the parent sched - * @sch: sched to find the parent of - * - * Returns the parent scheduler or %NULL if @sch is root. - */ -static struct scx_sched *scx_parent(struct scx_sched *sch) -{ - if (sch->level) - return sch->ancestors[sch->level - 1]; - else - return NULL; -} - /** * scx_next_descendant_pre - find the next descendant for pre-order walk * @pos: the current position (%NULL to initiate traversal) @@ -336,7 +320,6 @@ static void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) rcu_assign_pointer(p->scx.sched, sch); } #else /* CONFIG_EXT_SUB_SCHED */ -static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; } static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; } static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {} #endif /* CONFIG_EXT_SUB_SCHED */ @@ -496,123 +479,12 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags) */ DEFINE_PER_CPU(struct rq *, scx_locked_rq_state); -static inline void update_locked_rq(struct rq *rq) -{ - /* - * Check whether @rq is actually locked. This can help expose bugs - * or incorrect assumptions about the context in which a kfunc or - * callback is executed. - */ - if (rq) - lockdep_assert_rq_held(rq); - __this_cpu_write(scx_locked_rq_state, rq); -} - -/* - * SCX ops can recurse via scx_bpf_sub_dispatch() - the inner call must not - * clobber the outer's scx_locked_rq_state. Save it on entry, restore on exit. - */ -#define SCX_CALL_OP(sch, op, locked_rq, args...) \ -do { \ - struct rq *__prev_locked_rq; \ - \ - if (locked_rq) { \ - __prev_locked_rq = scx_locked_rq(); \ - update_locked_rq(locked_rq); \ - } \ - (sch)->ops.op(args); \ - if (locked_rq) \ - update_locked_rq(__prev_locked_rq); \ -} while (0) - /* * Flipped on enable per sch->is_cid_type. Declared in internal.h so * subsystem inlines can read it. */ DEFINE_STATIC_KEY_FALSE(__scx_is_cid_type); -/* - * scx_cpu_arg() wraps a cpu arg being handed to an SCX op. For cid-form - * schedulers it resolves to the matching cid; for cpu-form it passes @cpu - * through. scx_cpu_ret() is the inverse for a cpu/cid returned from an op - * (currently only ops.select_cpu); it validates the BPF-supplied cid and - * triggers scx_error() on @sch if invalid. - */ -static s32 scx_cpu_arg(s32 cpu) -{ - if (scx_is_cid_type()) - return __scx_cpu_to_cid(cpu); - return cpu; -} - -static s32 scx_cpu_ret(struct scx_sched *sch, s32 cpu_or_cid) -{ - if (cpu_or_cid < 0 || !scx_is_cid_type()) - return cpu_or_cid; - return scx_cid_to_cpu(sch, cpu_or_cid); -} - -#define SCX_CALL_OP_RET(sch, op, locked_rq, args...) \ -({ \ - struct rq *__prev_locked_rq; \ - __typeof__((sch)->ops.op(args)) __ret; \ - \ - if (locked_rq) { \ - __prev_locked_rq = scx_locked_rq(); \ - update_locked_rq(locked_rq); \ - } \ - __ret = (sch)->ops.op(args); \ - if (locked_rq) \ - update_locked_rq(__prev_locked_rq); \ - __ret; \ -}) - -/* - * SCX_CALL_OP_TASK*() invokes an SCX op that takes one or two task arguments - * and records them in current->scx.kf_tasks[] for the duration of the call. A - * kfunc invoked from inside such an op can then use - * scx_kf_arg_task_ok() to verify that its task argument is one of - * those subject tasks. - * - * Every SCX_CALL_OP_TASK*() call site invokes its op with @p's rq lock held - - * either via the @locked_rq argument here, or (for ops.select_cpu()) via @p's - * pi_lock held by try_to_wake_up() with rq tracking via scx_rq.in_select_cpu. - * So if kf_tasks[] is set, @p's scheduler-protected fields are stable. - * - * kf_tasks[] can not stack, so task-based SCX ops must not nest. The - * WARN_ON_ONCE() in each macro catches a re-entry of any of the three variants - * while a previous one is still in progress. - */ -#define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...) \ -do { \ - WARN_ON_ONCE(current->scx.kf_tasks[0]); \ - current->scx.kf_tasks[0] = task; \ - SCX_CALL_OP((sch), op, locked_rq, task, ##args); \ - current->scx.kf_tasks[0] = NULL; \ -} while (0) - -#define SCX_CALL_OP_TASK_RET(sch, op, locked_rq, task, args...) \ -({ \ - __typeof__((sch)->ops.op(task, ##args)) __ret; \ - WARN_ON_ONCE(current->scx.kf_tasks[0]); \ - current->scx.kf_tasks[0] = task; \ - __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task, ##args); \ - current->scx.kf_tasks[0] = NULL; \ - __ret; \ -}) - -#define SCX_CALL_OP_2TASKS_RET(sch, op, locked_rq, task0, task1, args...) \ -({ \ - __typeof__((sch)->ops.op(task0, task1, ##args)) __ret; \ - WARN_ON_ONCE(current->scx.kf_tasks[0]); \ - current->scx.kf_tasks[0] = task0; \ - current->scx.kf_tasks[1] = task1; \ - __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task0, task1, ##args); \ - current->scx.kf_tasks[0] = NULL; \ - current->scx.kf_tasks[1] = NULL; \ - __ret; \ -}) - /** * scx_call_op_set_cpumask - invoke ops.set_cpumask / ops_cid.set_cmask for @task * @sch: scx_sched being invoked @@ -651,19 +523,6 @@ static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, current->scx.kf_tasks[0] = NULL; } -/* see SCX_CALL_OP_TASK() */ -static __always_inline bool scx_kf_arg_task_ok(struct scx_sched *sch, - struct task_struct *p) -{ - if (unlikely((p != current->scx.kf_tasks[0] && - p != current->scx.kf_tasks[1]))) { - scx_error(sch, "called on a task not being operated on"); - return false; - } - - return true; -} - enum scx_dsq_iter_flags { /* iterate in the reverse dispatch order */ SCX_DSQ_ITER_REV = 1U << 16, diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 1f5312b3b387..145272cb4d8a 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1553,6 +1553,111 @@ static inline struct rq *scx_locked_rq(void) return __this_cpu_read(scx_locked_rq_state); } +static inline void update_locked_rq(struct rq *rq) +{ + /* + * Check whether @rq is actually locked. This can help expose bugs + * or incorrect assumptions about the context in which a kfunc or + * callback is executed. + */ + if (rq) + lockdep_assert_rq_held(rq); + __this_cpu_write(scx_locked_rq_state, rq); +} + +#define SCX_HAS_OP(sch, op) test_bit(SCX_OP_IDX(op), (sch)->has_op) + +/* + * SCX ops can recurse via scx_bpf_sub_dispatch() - the inner call must not + * clobber the outer's scx_locked_rq_state. Save it on entry, restore on exit. + */ +#define SCX_CALL_OP(sch, op, locked_rq, args...) \ +do { \ + struct rq *__prev_locked_rq; \ + \ + if (locked_rq) { \ + __prev_locked_rq = scx_locked_rq(); \ + update_locked_rq(locked_rq); \ + } \ + (sch)->ops.op(args); \ + if (locked_rq) \ + update_locked_rq(__prev_locked_rq); \ +} while (0) + +#define SCX_CALL_OP_RET(sch, op, locked_rq, args...) \ +({ \ + struct rq *__prev_locked_rq; \ + __typeof__((sch)->ops.op(args)) __ret; \ + \ + if (locked_rq) { \ + __prev_locked_rq = scx_locked_rq(); \ + update_locked_rq(locked_rq); \ + } \ + __ret = (sch)->ops.op(args); \ + if (locked_rq) \ + update_locked_rq(__prev_locked_rq); \ + __ret; \ +}) + +/* + * SCX_CALL_OP_TASK*() invokes an SCX op that takes one or two task arguments + * and records them in current->scx.kf_tasks[] for the duration of the call. A + * kfunc invoked from inside such an op can then use + * scx_kf_arg_task_ok() to verify that its task argument is one of + * those subject tasks. + * + * Every SCX_CALL_OP_TASK*() call site invokes its op with @p's rq lock held - + * either via the @locked_rq argument here, or (for ops.select_cpu()) via @p's + * pi_lock held by try_to_wake_up() with rq tracking via scx_rq.in_select_cpu. + * So if kf_tasks[] is set, @p's scheduler-protected fields are stable. + * + * kf_tasks[] can not stack, so task-based SCX ops must not nest. The + * WARN_ON_ONCE() in each macro catches a re-entry of any of the three variants + * while a previous one is still in progress. + */ +#define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...) \ +do { \ + WARN_ON_ONCE(current->scx.kf_tasks[0]); \ + current->scx.kf_tasks[0] = task; \ + SCX_CALL_OP((sch), op, locked_rq, task, ##args); \ + current->scx.kf_tasks[0] = NULL; \ +} while (0) + +#define SCX_CALL_OP_TASK_RET(sch, op, locked_rq, task, args...) \ +({ \ + __typeof__((sch)->ops.op(task, ##args)) __ret; \ + WARN_ON_ONCE(current->scx.kf_tasks[0]); \ + current->scx.kf_tasks[0] = task; \ + __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task, ##args); \ + current->scx.kf_tasks[0] = NULL; \ + __ret; \ +}) + +#define SCX_CALL_OP_2TASKS_RET(sch, op, locked_rq, task0, task1, args...) \ +({ \ + __typeof__((sch)->ops.op(task0, task1, ##args)) __ret; \ + WARN_ON_ONCE(current->scx.kf_tasks[0]); \ + current->scx.kf_tasks[0] = task0; \ + current->scx.kf_tasks[1] = task1; \ + __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task0, task1, ##args); \ + current->scx.kf_tasks[0] = NULL; \ + current->scx.kf_tasks[1] = NULL; \ + __ret; \ +}) + +/* see SCX_CALL_OP_TASK() */ +static __always_inline bool scx_kf_arg_task_ok(struct scx_sched *sch, + struct task_struct *p) +{ + if (unlikely((p != current->scx.kf_tasks[0] && + p != current->scx.kf_tasks[1]))) { + scx_error(sch, "called on a task not being operated on"); + return false; + } + + return true; +} + static inline bool scx_bypassing(struct scx_sched *sch, s32 cpu) { return unlikely(per_cpu_ptr(sch->pcpu, cpu)->flags & @@ -1633,6 +1738,20 @@ static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux) return NULL; } + +/** + * scx_parent - Find the parent sched + * @sch: sched to find the parent of + * + * Returns the parent scheduler or %NULL if @sch is root. + */ +static inline struct scx_sched *scx_parent(struct scx_sched *sch) +{ + if (sch->level) + return sch->ancestors[sch->level - 1]; + else + return NULL; +} #else /* CONFIG_EXT_SUB_SCHED */ static inline struct scx_sched *scx_task_sched(const struct task_struct *p) { @@ -1656,6 +1775,8 @@ static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux) { return rcu_dereference_all(scx_root); } + +static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; } #endif /* CONFIG_EXT_SUB_SCHED */ #endif /* _KERNEL_SCHED_EXT_INTERNAL_H */ -- cgit v1.2.3 From 5e0b273e0a62cc04ec338c7b502797c66c2ed42a Mon Sep 17 00:00:00 2001 From: Tristan Madani Date: Mon, 22 Jun 2026 23:01:22 +0000 Subject: bpf: Reset register bounds before narrowing retval range in check_mem_access() When the BPF verifier processes a context load of an LSM hook return value, it calls __mark_reg_s32_range() to narrow the register to the hook's valid range. However, __mark_reg_s32_range() intersects the new range with the register's existing bounds using max_t()/min_t() rather than replacing them. If the destination register carries stale bounds from a prior instruction (e.g. BPF_MOV64_IMM), the intersection can produce a range narrower than reality. The verifier then believes it knows the register's exact value, while at runtime the actual hook return value is loaded, creating a verifier/runtime mismatch that can be used to bypass BPF memory safety checks. The else branch already calls mark_reg_unknown() to reset register state before any narrowing. Apply the same reset in the is_retval path so stale bounds are cleared before __mark_reg_s32_range() intersects. Fixes: 5d99e198be27 ("bpf, lsm: Add check for BPF LSM return value") Cc: stable@vger.kernel.org Signed-off-by: Tristan Madani Acked-by: Eduard Zingerman Link: https://lore.kernel.org/r/20260622230123.3695446-2-tristmd@gmail.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a2b348f98080..21a365d436a5 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6201,6 +6201,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b */ if (info.reg_type == SCALAR_VALUE) { if (info.is_retval && get_func_retval_range(env->prog, &range)) { + mark_reg_unknown(env, regs, value_regno); err = __mark_reg_s32_range(env, regs, value_regno, range.minval, range.maxval); if (err) -- cgit v1.2.3 From 12091470c6b4c1c14b2de12dcbae2ada6cb6d20b Mon Sep 17 00:00:00 2001 From: Bradley Morgan Date: Fri, 19 Jun 2026 13:03:03 +0000 Subject: bpf: Disable xfrm_decode_session hook attachment BPF LSM programs can currently attach to xfrm_decode_session(). That hook may return an error, but security_skb_classify_flow() calls it from a void path and triggers BUG_ON() if an error is returned. Disable BPF attachment to the hook to prevent a BPF LSM program from turning packet classification into a full panic. Fixes: 9e4e01dfd325 ("bpf: lsm: Implement attach, detach and execution") Signed-off-by: Bradley Morgan Link: https://lore.kernel.org/r/20260619130305.27779-1-include@grrlz.net Signed-off-by: Alexei Starovoitov --- kernel/bpf/bpf_lsm.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 564071a92d7d..1433809bb166 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -51,6 +51,9 @@ BTF_ID(func, bpf_lsm_key_getsecurity) #ifdef CONFIG_AUDIT BTF_ID(func, bpf_lsm_audit_rule_match) #endif +#ifdef CONFIG_SECURITY_NETWORK_XFRM +BTF_ID(func, bpf_lsm_xfrm_decode_session) +#endif BTF_ID(func, bpf_lsm_ismaclabel) BTF_ID(func, bpf_lsm_file_alloc_security) BTF_SET_END(bpf_lsm_disabled_hooks) -- cgit v1.2.3 From 71c6e7808ee99a6e1b29bc30486baf825f9ec728 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ilpo=20J=C3=A4rvinen?= Date: Wed, 29 Apr 2026 15:26:11 +0300 Subject: resource: Make resource_alignment() input const resource MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resource_alignment() does not need to change resource so it can be made const. Signed-off-by: Ilpo Järvinen Signed-off-by: Bjorn Helgaas Link: https://patch.msgid.link/20260429122617.7324-6-ilpo.jarvinen@linux.intel.com --- kernel/resource.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/resource.c b/kernel/resource.c index d02a53fb95d8..3d17e3196a3e 100644 --- a/kernel/resource.c +++ b/kernel/resource.c @@ -1238,7 +1238,7 @@ reserve_region_with_split(struct resource *root, resource_size_t start, * * Returns alignment on success, 0 (invalid alignment) on failure. */ -resource_size_t resource_alignment(struct resource *res) +resource_size_t resource_alignment(const struct resource *res) { switch (res->flags & (IORESOURCE_SIZEALIGN | IORESOURCE_STARTALIGN)) { case IORESOURCE_SIZEALIGN: -- cgit v1.2.3 From 72c8646956ffc8050bb8be5988a0f28fc37e1ac4 Mon Sep 17 00:00:00 2001 From: Martin Kaiser Date: Thu, 25 Jun 2026 08:34:45 +0900 Subject: tracing: probes: fix typo in a log message Fix a typo ("Invalid $-variable") in a log message. Link: https://lore.kernel.org/all/20260507081041.885781-4-martin@kaiser.cx/ Fixes: ab105a4fb894 ("tracing: Use tracing error_log with probe events") Signed-off-by: Martin Kaiser Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/trace_probe.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h index 15758cc11fc6..0f09f7aaf93f 100644 --- a/kernel/trace/trace_probe.h +++ b/kernel/trace/trace_probe.h @@ -511,7 +511,7 @@ extern int traceprobe_define_arg_fields(struct trace_event_call *event_call, C(NO_RETVAL, "This function returns 'void' type"), \ C(BAD_STACK_NUM, "Invalid stack number"), \ C(BAD_ARG_NUM, "Invalid argument number"), \ - C(BAD_VAR, "Invalid $-valiable specified"), \ + C(BAD_VAR, "Invalid $-variable specified"), \ C(BAD_REG_NAME, "Invalid register name"), \ C(BAD_MEM_ADDR, "Invalid memory address"), \ C(BAD_IMM, "Invalid immediate value"), \ -- cgit v1.2.3 From 931a577fc79ea6a169a33f5538f4c1433235c358 Mon Sep 17 00:00:00 2001 From: Yiyang Chen Date: Tue, 23 Jun 2026 06:11:09 +0000 Subject: bpf: Reject offset refcount acquire arguments bpf_refcount_acquire() increments the refcount at the caller-supplied pointer plus the refcount field offset, then returns the caller-supplied pointer unchanged. The verifier records the return value as a base pointer to the refcounted object. bpf_list_pop_front() and bpf_rbtree_remove() can return embedded graph-node pointers as PTR_TO_BTF_ID | MEM_ALLOC with a fixed offset equal to the node field offset. Passing such a pointer directly to bpf_refcount_acquire() currently passes the refcounted-kptr type check. That makes the runtime operation start from base + node_off while the verifier models the returned pointer as the object base. Require refcount-acquire arguments to have zero fixed offset by carrying the requirement through check_func_arg_reg_off() to __check_ptr_off_reg(). Programs can still acquire a refcount from a graph-node-derived pointer after normalizing it with container_of(). Fixes: 7c50b1cb76aca ("bpf: Add bpf_refcount_acquire kfunc") Signed-off-by: Yiyang Chen Acked-by: Eduard Zingerman Acked-by: Yonghong Song Link: https://lore.kernel.org/r/2f894647f56f71838fdddeb97a3e057ed35ea92e.1782192383.git.chenyy23@mails.tsinghua.edu.cn Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 21a365d436a5..3cdc2e90f643 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -7996,9 +7996,10 @@ reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields) return field; } -static int check_func_arg_reg_off(struct bpf_verifier_env *env, - const struct bpf_reg_state *reg, argno_t argno, - enum bpf_arg_type arg_type) +static int __check_func_arg_reg_off(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg, argno_t argno, + enum bpf_arg_type arg_type, + bool btf_id_fixed_off_ok) { u32 type = reg->type; @@ -8055,12 +8056,11 @@ static int check_func_arg_reg_off(struct bpf_verifier_env *env, case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU: /* When referenced PTR_TO_BTF_ID is passed to release function, * its fixed offset must be 0. In the other cases, fixed offset - * can be non-zero. This was already checked above. So pass - * fixed_off_ok as true to allow fixed offset for all other - * cases. var_off always must be 0 for PTR_TO_BTF_ID, hence we - * still need to do checks instead of returning. + * can be non-zero unless the caller requires otherwise. + * var_off always must be 0 for PTR_TO_BTF_ID, hence we still + * need to do checks instead of returning. */ - return __check_ptr_off_reg(env, reg, argno, true); + return __check_ptr_off_reg(env, reg, argno, btf_id_fixed_off_ok); case PTR_TO_CTX: /* * Allow fixed and variable offsets for syscall context, but @@ -8076,6 +8076,13 @@ static int check_func_arg_reg_off(struct bpf_verifier_env *env, } } +static int check_func_arg_reg_off(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg, argno_t argno, + enum bpf_arg_type arg_type) +{ + return __check_func_arg_reg_off(env, reg, argno, arg_type, true); +} + static int check_arg_const_str(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno) { @@ -11947,6 +11954,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ enum bpf_arg_type arg_type = ARG_DONTCARE; argno_t argno = argno_from_arg(i + 1); int regno = reg_from_argno(argno); + bool btf_id_fixed_off_ok = true; u32 ref_id, type_size; bool is_ret_buf_sz = false; int kf_arg_type; @@ -12120,7 +12128,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ case KF_ARG_PTR_TO_MEM: case KF_ARG_PTR_TO_MEM_SIZE: case KF_ARG_PTR_TO_CALLBACK: - case KF_ARG_PTR_TO_REFCOUNTED_KPTR: case KF_ARG_PTR_TO_CONST_STR: case KF_ARG_PTR_TO_WORKQUEUE: case KF_ARG_PTR_TO_TIMER: @@ -12134,6 +12141,10 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ case KF_ARG_PTR_TO_CTX: arg_type = ARG_PTR_TO_CTX; break; + case KF_ARG_PTR_TO_REFCOUNTED_KPTR: + arg_type = ARG_PTR_TO_BTF_ID; + btf_id_fixed_off_ok = false; + break; default: verifier_bug(env, "unknown kfunc arg type %d", kf_arg_type); return -EFAULT; @@ -12141,7 +12152,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ if (regno == meta->release_regno) arg_type |= OBJ_RELEASE; - ret = check_func_arg_reg_off(env, reg, argno, arg_type); + ret = __check_func_arg_reg_off(env, reg, argno, arg_type, + btf_id_fixed_off_ok); if (ret < 0) return ret; -- cgit v1.2.3 From 72a85e9464a5332fb2cd7efd26d9295275ceda2d Mon Sep 17 00:00:00 2001 From: Nuoqi Gui Date: Tue, 23 Jun 2026 18:43:38 +0800 Subject: bpf: Mask pseudo pointer values in verifier logs print_bpf_insn() masks ldimm64 immediates for pointer-bearing pseudo sources when pointer leaks are not allowed, but the mask only covers BPF_PSEUDO_MAP_FD and BPF_PSEUDO_MAP_VALUE. BPF_PSEUDO_MAP_IDX, BPF_PSEUDO_MAP_IDX_VALUE, and BPF_PSEUDO_BTF_ID can also be resolved to kernel pointer values before the verifier log prints the instruction. Include them in the existing pointer classification so the log prints 0x0 instead of the rewritten address. Fixes: 4976b718c355 ("bpf: Introduce pseudo_btf_id") Fixes: 387544bfa291 ("bpf: Introduce fd_idx") Signed-off-by: Nuoqi Gui Link: https://lore.kernel.org/r/20260623-f01-13-pseudo-btf-id-cap-bpf-v2-1-a190ebb8f3e2@mails.tsinghua.edu.cn Signed-off-by: Alexei Starovoitov Acked-by: Eduard Zingerman --- kernel/bpf/disasm.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c index f8a3c7eb451e..0391b3bc0073 100644 --- a/kernel/bpf/disasm.c +++ b/kernel/bpf/disasm.c @@ -323,7 +323,10 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, */ u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; bool is_ptr = insn->src_reg == BPF_PSEUDO_MAP_FD || - insn->src_reg == BPF_PSEUDO_MAP_VALUE; + insn->src_reg == BPF_PSEUDO_MAP_VALUE || + insn->src_reg == BPF_PSEUDO_MAP_IDX || + insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE || + insn->src_reg == BPF_PSEUDO_BTF_ID; char tmp[64]; if (is_ptr && !allow_ptr_leaks) -- cgit v1.2.3 From 26490a375cb9be9bac96b5171610fd85ca6c2305 Mon Sep 17 00:00:00 2001 From: KaFai Wan Date: Wed, 24 Jun 2026 20:35:35 +0800 Subject: bpf: Fix insn_aux_data leak on verifier err_free_env path When bpf_check() allocates env->insn_aux_data successfully but later fails to allocate env->succ, it jumps directly to err_free_env. The existing vfree(env->insn_aux_data) sits before the err_free_env label, so that direct jump bypasses it and leaks insn_aux_data. Move vfree(env->insn_aux_data) into err_free_env so all early and late exit paths release it consistently. Fixes: 2f69c5685427 ("bpf: make bpf_insn_successors to return a pointer") Signed-off-by: KaFai Wan Reviewed-by: Anton Protopopov Link: https://lore.kernel.org/r/20260624123536.114757-1-kafai.wan@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 3cdc2e90f643..6515d4d3c003 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -20006,13 +20006,13 @@ err_unlock: if (!is_priv) mutex_unlock(&bpf_verifier_lock); bpf_clear_insn_aux_data(env, 0, env->prog->len); - vfree(env->insn_aux_data); err_free_env: bpf_stack_liveness_free(env); kvfree(env->cfg.insn_postorder); kvfree(env->scc_info); kvfree(env->succ); kvfree(env->gotox_tmp_buf); + vfree(env->insn_aux_data); kvfree(env); return ret; } -- cgit v1.2.3 From 9b51a6155d14389876916726430da30eabb1d4ed Mon Sep 17 00:00:00 2001 From: Jann Horn Date: Fri, 26 Jun 2026 17:52:52 +0200 Subject: bpf,fork: wipe ->bpf_storage before bailouts that access it Currently, copy_process() can bail out to free_task() before p->bpf_storage has been initialized, with this call graph (shown here for the !CONFIG_MEMCG case): copy_process dup_task_struct arch_dup_task_struct [copies the entire task_struct, including ->bpf_storage member] [RLIMIT_NPROC check fails] delayed_free_task free_task bpf_task_storage_free rcu_dereference(task->bpf_storage) bpf_local_storage_destroy In this case, the nascent task's ->bpf_storage member that bpf_local_storage_destroy() operates on is a plain copy of the parent's ->bpf_storage pointer, not a real initialized pointer. This leads to badness (kernel hangs, UAF). This is reachable as long as the process calling fork() has been inserted into a task storage map. Cc: stable@kernel.org Fixes: a10787e6d58c ("bpf: Enable task local storage for tracing programs") Signed-off-by: Jann Horn Signed-off-by: Andrii Nakryiko --- kernel/fork.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/fork.c b/kernel/fork.c index 13e38e89a1f3..f0e2e131a9a5 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1009,6 +1009,11 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) tsk->mm_cid.active = 0; INIT_HLIST_NODE(&tsk->mm_cid.node); #endif + +#ifdef CONFIG_BPF_SYSCALL + RCU_INIT_POINTER(tsk->bpf_storage, NULL); + tsk->bpf_ctx = NULL; +#endif return tsk; free_stack: @@ -2247,10 +2252,6 @@ __latent_entropy struct task_struct *copy_process( p->sequential_io = 0; p->sequential_io_avg = 0; #endif -#ifdef CONFIG_BPF_SYSCALL - RCU_INIT_POINTER(p->bpf_storage, NULL); - p->bpf_ctx = NULL; -#endif unwind_task_init(p); -- cgit v1.2.3 From a6f0643e4f63cfaa0d5d4a69de4f132eac4b8fe4 Mon Sep 17 00:00:00 2001 From: Matt Bobrowski Date: Sun, 28 Jun 2026 20:11:03 +0000 Subject: bpf: Reject BPF_MAP_TYPE_INODE_STORAGE creation if BPF LSM is uninitialized When CONFIG_BPF_LSM=y is set, BPF inode storage maps (BPF_MAP_TYPE_INODE_STORAGE) are compiled into the kernel. However, if the BPF LSM is not explicitly enabled at boot time (e.g. omitted from the "lsm=" boot parameter), lsm_prepare() is never executed for the BPF LSM. Consequently, the BPF inode security blob offset (bpf_lsm_blob_sizes.lbs_inode) is never initialized and remains at its default compiled size of 8 bytes instead of being updated to a valid offset past the reserved struct rcu_head (typically 16 bytes or more). When a privileged user creates and updates a BPF_MAP_TYPE_INODE_STORAGE map, bpf_inode() evaluates inode->i_security + 8. This erroneously aliases the struct rcu_head.func callback pointer at the beginning of the inode->i_security blob. During subsequent map element cleanup or inode destruction, writing NULL to owner_storage clears the queued RCU callback pointer. When rcu_do_batch() later executes the queued callback, it attempts an instruction fetch at address 0x0, triggering an immediate kernel panic. Fix this by introducing a global bpf_lsm_initialized boolean flag marked with __ro_after_init. Set this flag to true inside bpf_lsm_init() when the LSM framework successfully registers the BPF LSM. Gate map allocation in inode_storage_map_alloc() on this flag, returning -EOPNOTSUPP if the BPF LSM is in turn uninitialized. This fail-fast approach prevents userspace from allocating inode storage maps when the supporting BPF LSM infrastructure is absent, avoiding zombie map states. Fixes: 8ea636848aca ("bpf: Implement bpf_local_storage for inodes") Reported-by: oxsignal Signed-off-by: Matt Bobrowski Signed-off-by: Daniel Borkmann Reviewed-by: Emil Tsalapatis Reviewed-by: Amery Hung Link: https://lore.kernel.org/bpf/20260628201103.3624525-1-mattbobrowski@google.com --- kernel/bpf/bpf_inode_storage.c | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/bpf_inode_storage.c b/kernel/bpf/bpf_inode_storage.c index 0da8d923e39d..f9e81060c1f4 100644 --- a/kernel/bpf/bpf_inode_storage.c +++ b/kernel/bpf/bpf_inode_storage.c @@ -178,6 +178,15 @@ static int notsupp_get_next_key(struct bpf_map *map, void *key, static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr) { + /* + * Do not allow allocation of BPF_MAP_TYPE_INODE_STORAGE if the BPF LSM + * was not initialized by the LSM framework at boot. Without proper + * initialization, the BPF inode security blob offset remains unprepared, + * causing bpf_inode() to calculate an invalid memory offset and corrupt + * inode->i_security. + */ + if (!bpf_lsm_initialized) + return ERR_PTR(-EOPNOTSUPP); return bpf_local_storage_map_alloc(attr, &inode_cache); } -- cgit v1.2.3 From 251a8fe1b9aedccd298b77bc28426d564c5a923f Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Thu, 25 Jun 2026 08:34:46 +0900 Subject: tracing/probes: Remove WARN_ON_ONCE from parse_btf_arg Sashiko found that user can cause this WARN_ON_ONCE() easily with adding a kprobe event based on a raw address with BTF parameter. Since this is not an unexpected condition, remove the WARN_ON_ONCE(). Link: https://lore.kernel.org/all/178177265367.2059927.13789953014706792126.stgit@mhiramat.tok.corp.google.com/ Link: https://sashiko.dev/#/patchset/178165816303.269421.7302603996990753309.stgit%40devnote2 Reported-by: Sashiko Fixes: b576e09701c7 ("tracing/probes: Support function parameters if BTF is available") Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/trace_probe.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index fd1caa1f9723..98532c503d02 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -678,7 +678,7 @@ static int parse_btf_arg(char *varname, int i, is_ptr, ret; u32 tid; - if (WARN_ON_ONCE(!ctx->funcname && !(ctx->flags & TPARG_FL_TEVENT))) + if (!ctx->funcname && !(ctx->flags & TPARG_FL_TEVENT)) return -EINVAL; is_ptr = split_next_field(varname, &field, ctx); -- cgit v1.2.3 From 206b25c09080cc20fd4c2bea12d59df4b7ba2121 Mon Sep 17 00:00:00 2001 From: Martin Kaiser Date: Thu, 25 Jun 2026 08:34:46 +0900 Subject: tracing: eprobe: read the complete FILTER_PTR_STRING pointer For a char * element in an event, the FILTER_PTR_STRING filter type is used. When the event occurs, a pointer is stored in the ringbuffer. If an eprobe references such a char * element of a "base event", the stored pointer is truncated when it's read from the ringbuffer. $ cd /sys/kernel/tracing $ echo 'e rcu.rcu_utilization $s:x64 $s:string' > dynamic_events $ echo 1 > tracing_on $ echo 1 > events/eprobes/enable $ sleep 1 $ echo 0 > events/eprobes/enable $ cat trace -0 ...: (rcu.rcu_utilization) arg1=0x4f arg2=(fault) -0 ...: (rcu.rcu_utilization) arg1=0x2 arg2=(fault) The problem is in get_event_field val = (unsigned long)(*(char *)addr); addr points to the position in the ringbuffer where the pointer was stored. The assignment reads only the lowest byte of the pointer. Fix the cast to read the whole pointer. The output of the test above is now -0 ... arg1=0xffffffff81c7d3f3 arg2="Start scheduler-tick" -0 ... arg1=0xffffffff81c57340 arg2="End scheduler-tick" Link: https://lore.kernel.org/all/20260620145339.3234726-1-martin@kaiser.cx/ Fixes: f04dec93466a ("tracing/eprobes: Fix reading of string fields") Signed-off-by: Martin Kaiser Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/trace_eprobe.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/trace/trace_eprobe.c b/kernel/trace/trace_eprobe.c index b66d6196338d..50518b071414 100644 --- a/kernel/trace/trace_eprobe.c +++ b/kernel/trace/trace_eprobe.c @@ -315,7 +315,7 @@ get_event_field(struct fetch_insn *code, void *rec) val = (unsigned long)addr; break; case FILTER_PTR_STRING: - val = (unsigned long)(*(char *)addr); + val = *(unsigned long *)addr; break; default: WARN_ON_ONCE(1); -- cgit v1.2.3 From 9a667b7750dda88cbf1cca96a53a2163b2ee71f7 Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Thu, 25 Jun 2026 08:34:47 +0900 Subject: tracing/probes: Fix double addition of offset for @+FOFFSET Since commit 533059281ee5 ("tracing: probeevent: Introduce new argument fetching code") wrongly use @offset local variable during the parsing, the offset value is added twice when dereferencing. Reset the @offset after setting it in FETCH_OP_FOFFS. Link: https://lore.kernel.org/all/178217905962.643090.1978577464942171332.stgit@devnote2/ Fixes: 533059281ee5 ("tracing: probeevent: Introduce new argument fetching code") Signed-off-by: Masami Hiramatsu (Google) Cc: stable@vger.kernel.org --- kernel/trace/trace_probe.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel') diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index 98532c503d02..502fa6da5949 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -1241,6 +1241,7 @@ parse_probe_arg(char *arg, const struct fetch_type *type, code->op = FETCH_OP_FOFFS; code->immediate = (unsigned long)offset; // imm64? + offset = 0; } else { /* uprobes don't support symbols */ if (!(ctx->flags & TPARG_FL_KERNEL)) { -- cgit v1.2.3 From 367c49d6e283c17b56a31e7a8d964a079244264c Mon Sep 17 00:00:00 2001 From: Sechang Lim Date: Thu, 25 Jun 2026 08:34:48 +0900 Subject: tracing/fprobe: Fix NULL pointer dereference in fprobe_fgraph_entry() fprobe_fgraph_entry() sizes a shadow-stack reservation in one walk of the per-ip fprobe list and fills it in a second walk, both under rcu_read_lock() only. A fprobe registered on an already-live ip can become visible between the two walks, so the fill walk processes an exit_handler the sizing walk did not count and used runs past reserved_words. If the sizing walk counted nothing, fgraph_data is NULL and the first write_fprobe_header() faults: Oops: general protection fault, probably for non-canonical address ... KASAN: null-ptr-deref in range [0x0000000000000000-0x0000000000000007] RIP: 0010:fprobe_fgraph_entry+0xa38/0xf10 kernel/trace/fprobe.c:167 Call Trace: function_graph_enter_regs+0x44c/0xa10 kernel/trace/fgraph.c:677 ftrace_graph_func+0xc5/0x140 arch/x86/kernel/ftrace.c:671 __kernel_text_address+0x9/0x40 kernel/extable.c:78 arch_stack_walk+0x117/0x170 arch/x86/kernel/stacktrace.c:26 kmem_cache_free+0x188/0x580 mm/slub.c:6378 tcp_data_queue+0x18d/0x6550 net/ipv4/tcp_input.c:5590 [...] The list cannot be frozen across the two walks, so skip a node that does not fit the reservation and count it as missed. Link: https://lore.kernel.org/all/20260619184425.3824774-1-rhkrqnwk98@gmail.com/ Fixes: 4346ba160409 ("fprobe: Rewrite fprobe on function-graph tracer") Signed-off-by: Sechang Lim Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/fprobe.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'kernel') diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c index f378613ad120..f215990b9061 100644 --- a/kernel/trace/fprobe.c +++ b/kernel/trace/fprobe.c @@ -613,6 +613,16 @@ static int fprobe_fgraph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops continue; data_size = fp->entry_data_size; + /* + * The list may have grown since it was sized, so this node + * may not fit. Skip it as missed rather than overrun the + * reservation. + */ + if (fp->exit_handler && + used + FPROBE_HEADER_SIZE_IN_LONG + SIZE_IN_LONG(data_size) > reserved_words) { + fp->nmissed++; + continue; + } if (data_size && fp->exit_handler) data = fgraph_data + used + FPROBE_HEADER_SIZE_IN_LONG; else -- cgit v1.2.3 From a369299c3f785cf556bbef2de2db0aa2d294c4c9 Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Thu, 25 Jun 2026 08:34:48 +0900 Subject: tracing/probes: Make the $ prefix mandatory for comm access Since $comm or $COMM are not event field but special fetcharg variables to access current->comm, It should not be accessed without '$' prefix even with typecast. Link: https://lore.kernel.org/all/178231209724.732967.12049805699091810641.stgit@devnote2/ Fixes: 69efd863a785 ("tracing/eprobes: Allow use of BTF names to dereference pointers") Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/trace_probe.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index 502fa6da5949..d17cfee77d9c 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -342,10 +342,6 @@ static int parse_trace_event(char *arg, struct fetch_insn *code, ret = parse_trace_event_arg(arg, code, ctx); if (!ret) return 0; - if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) { - code->op = FETCH_OP_COMM; - return 0; - } return -EINVAL; } @@ -1068,8 +1064,14 @@ static int parse_probe_vars(char *orig_arg, const struct fetch_type *t, int len; if (ctx->flags & TPARG_FL_TEVENT) { - if (parse_trace_event(arg, code, ctx) < 0) + if (parse_trace_event(arg, code, ctx) < 0) { + /* 'comm' should be checked after field parsing. */ + if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) { + code->op = FETCH_OP_COMM; + return 0; + } goto inval; + } return 0; } -- cgit v1.2.3 From 96cce16e26dd02a8678f1e87f88a4b5cdb63b995 Mon Sep 17 00:00:00 2001 From: Pawan Gupta Date: Mon, 29 Jun 2026 22:37:52 -0700 Subject: bpf: Support for hardening against JIT spraying The BPF JIT allocator packs many small programs into larger executable allocations and reuses space within those allocations as programs are loaded and freed. When fresh code is written into space that a previous program occupied, an indirect jump into the new program can reuse a branch prediction left behind by the old one. Flush the indirect branch predictors before reusing JIT memory so that indirect jumps into a newly written program don't reuse predictions from an old program that occupied the same space. Introduce bpf_arch_pred_flush_enabled static key and bpf_arch_pred_flush static call for flushing the branch predictors on JIT memory reuse. Architectures that need a flush, can update it to a predictor flush function. By default, its a NOP and does not emit any CALL. Allocations larger than a pack are not covered by this flush. That is safe because cBPF programs (the unprivileged attack surface) are bounded well below a pack size. Issue a warning if this assumption is ever violated while the flush is active. Signed-off-by: Pawan Gupta Acked-by: Daniel Borkmann Signed-off-by: Daniel Borkmann --- kernel/bpf/core.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 649cce41e13f..7f0a17f128d4 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -883,6 +884,15 @@ void bpf_jit_fill_hole_with_zero(void *area, unsigned int size) memset(area, 0, size); } +DEFINE_STATIC_CALL_NULL(bpf_arch_pred_flush, bpf_arch_pred_flush); + +/* + * Enabled once bpf_arch_pred_flush points at a real flush routine. Lets the + * pack allocator test "is a predictor flush wired up at all" with a cheap + * static branch instead of repeatedly querying the static call target. + */ +DEFINE_STATIC_KEY_FALSE(bpf_pred_flush_enabled); + #define BPF_PROG_SIZE_TO_NBITS(size) (round_up(size, BPF_PROG_CHUNK_SIZE) / BPF_PROG_CHUNK_SIZE) static DEFINE_MUTEX(pack_mutex); @@ -941,6 +951,14 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) mutex_lock(&pack_mutex); if (size > BPF_PROG_PACK_SIZE) { + /* + * Allocations larger than a pack get their own pages, and + * predictors are not flushed for such allocation. This is only + * safe because cBPF programs (the unprivileged attack surface) + * are bounded well below a pack size. + */ + if (static_branch_unlikely(&bpf_pred_flush_enabled)) + pr_warn_once("BPF: Predictors not flushed for allocations greater than BPF_PROG_PACK_SIZE\n"); size = round_up(size, PAGE_SIZE); ptr = bpf_jit_alloc_exec(size); if (ptr) { @@ -971,6 +989,7 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) pos = 0; found_free_area: + static_call_cond(bpf_arch_pred_flush)(); bitmap_set(pack->bitmap, pos, nbits); ptr = (void *)(pack->ptr) + (pos << BPF_PROG_CHUNK_SHIFT); -- cgit v1.2.3 From 0bb99f2cfaae6822d734d69722de30af823efdf3 Mon Sep 17 00:00:00 2001 From: Pawan Gupta Date: Mon, 29 Jun 2026 22:38:23 -0700 Subject: bpf: Restrict JIT predictor flush to cBPF Currently predictor flush on memory reuse is done for all BPF JIT allocations, but only cBPF programs can be loaded by an unprivileged user. eBPF is privileged by default, and flushing predictors for all CPUs on every eBPF reuse penalizes the common case for no security benefit. eBPF allocations can be frequent on busy systems, only flush predictors for cBPF programs. Trampoline and dispatcher allocations also skip the flush as they are eBPF-only. Signed-off-by: Pawan Gupta Acked-by: Daniel Borkmann Signed-off-by: Daniel Borkmann --- kernel/bpf/core.c | 13 ++++++++----- kernel/bpf/dispatcher.c | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 7f0a17f128d4..1614ccc3f111 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -942,7 +942,7 @@ out: return NULL; } -void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) +void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool was_classic) { unsigned int nbits = BPF_PROG_SIZE_TO_NBITS(size); struct bpf_prog_pack *pack; @@ -957,7 +957,7 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) * safe because cBPF programs (the unprivileged attack surface) * are bounded well below a pack size. */ - if (static_branch_unlikely(&bpf_pred_flush_enabled)) + if (was_classic && static_branch_unlikely(&bpf_pred_flush_enabled)) pr_warn_once("BPF: Predictors not flushed for allocations greater than BPF_PROG_PACK_SIZE\n"); size = round_up(size, PAGE_SIZE); ptr = bpf_jit_alloc_exec(size); @@ -989,7 +989,9 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) pos = 0; found_free_area: - static_call_cond(bpf_arch_pred_flush)(); + /* Flush only for cBPF as it may contain a crafted gadget */ + if (static_branch_unlikely(&bpf_pred_flush_enabled) && was_classic) + static_call_cond(bpf_arch_pred_flush)(); bitmap_set(pack->bitmap, pos, nbits); ptr = (void *)(pack->ptr) + (pos << BPF_PROG_CHUNK_SHIFT); @@ -1149,7 +1151,8 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr, unsigned int alignment, struct bpf_binary_header **rw_header, u8 **rw_image, - bpf_jit_fill_hole_t bpf_fill_ill_insns) + bpf_jit_fill_hole_t bpf_fill_ill_insns, + bool was_classic) { struct bpf_binary_header *ro_header; u32 size, hole, start; @@ -1162,7 +1165,7 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr, if (bpf_jit_charge_modmem(size)) return NULL; - ro_header = bpf_prog_pack_alloc(size, bpf_fill_ill_insns); + ro_header = bpf_prog_pack_alloc(size, bpf_fill_ill_insns, was_classic); if (!ro_header) { bpf_jit_uncharge_modmem(size); return NULL; diff --git a/kernel/bpf/dispatcher.c b/kernel/bpf/dispatcher.c index b77db7413f8c..ea2d60dc1fee 100644 --- a/kernel/bpf/dispatcher.c +++ b/kernel/bpf/dispatcher.c @@ -145,7 +145,7 @@ void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, mutex_lock(&d->mutex); if (!d->image) { - d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero); + d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero, false); if (!d->image) goto out; d->rw_image = bpf_jit_alloc_exec(PAGE_SIZE); -- cgit v1.2.3 From a23c1c5396a91680703360d1ee28a44657c503c4 Mon Sep 17 00:00:00 2001 From: Pawan Gupta Date: Mon, 29 Jun 2026 22:38:38 -0700 Subject: bpf: Skip redundant IBPB in pack allocator bpf_prog_pack_alloc() issues IBPB on all CPUs on every cBPF allocation, even when reusing chunks from an existing pack where no new memory was touched since the last IBPB. Since IBPB on all CPUs is heavy, Dave Hansen suggested to track allocation since last IBPB, and only issue IBPB at reuse for the chunks that have not seen an IBPB since they were last freed. Track per-pack whether an IBPB is needed via arch_flush_needed. Set it when allocating a chunk, reset on IBPB flush. On reuse, conditionally issue the flush. Since IBPB invalidates all BTB entries, clear the flag on all packs after flushing. Signed-off-by: Pawan Gupta Acked-by: Daniel Borkmann Signed-off-by: Daniel Borkmann --- kernel/bpf/core.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 1614ccc3f111..50aba113ef9d 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -876,6 +876,7 @@ int bpf_jit_add_poke_descriptor(struct bpf_prog *prog, struct bpf_prog_pack { struct list_head list; void *ptr; + bool arch_flush_needed; unsigned long bitmap[]; }; @@ -928,6 +929,8 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_ins bpf_fill_ill_insns(pack->ptr, BPF_PROG_PACK_SIZE); bitmap_zero(pack->bitmap, BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE); + if (static_branch_unlikely(&bpf_pred_flush_enabled)) + pack->arch_flush_needed = true; set_vm_flush_reset_perms(pack->ptr); err = set_memory_rox((unsigned long)pack->ptr, BPF_PROG_PACK_SIZE / PAGE_SIZE); @@ -990,8 +993,15 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool found_free_area: /* Flush only for cBPF as it may contain a crafted gadget */ - if (static_branch_unlikely(&bpf_pred_flush_enabled) && was_classic) + if (static_branch_unlikely(&bpf_pred_flush_enabled) && + pack->arch_flush_needed && + was_classic) { + struct bpf_prog_pack *p; + static_call_cond(bpf_arch_pred_flush)(); + list_for_each_entry(p, &pack_list, list) + p->arch_flush_needed = false; + } bitmap_set(pack->bitmap, pos, nbits); ptr = (void *)(pack->ptr) + (pos << BPF_PROG_CHUNK_SHIFT); @@ -1029,6 +1039,9 @@ void bpf_prog_pack_free(void *ptr, u32 size) "bpf_prog_pack bug: missing bpf_arch_text_invalidate?\n"); bitmap_clear(pack->bitmap, pos, nbits); + + if (static_branch_unlikely(&bpf_pred_flush_enabled)) + pack->arch_flush_needed = true; if (bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0, BPF_PROG_CHUNK_COUNT, 0) == 0) { list_del(&pack->list); -- cgit v1.2.3 From a9b1f19a6a673ba06820898d0f1ad02883ea1639 Mon Sep 17 00:00:00 2001 From: Pawan Gupta Date: Mon, 29 Jun 2026 22:38:54 -0700 Subject: bpf: Prefer packs that won't trigger an IBPB flush on allocation Currently BPF pack allocator picks the chunks from the first available pack. While this is okay, it naturally leads to more frequent flushes when there are multiple packs in the system that weren't used since the last flush. As an optimization prefer allocating the new programs from packs that are unused since last flush. When all packs are dirty, allocation forces a flush and marks all packs clean. Below are some future optimizations ideas: 1. Currently, the "dirty" tracking is only done at the pack-level. Flush frequency can further be reduced with chunk-level tracking. This requires a new bitmap per-pack to track the dirty state. 2. IBPB flush is done on all CPUs, even if only a single CPU ran the BPF program. On a system with hundreds of CPUs this could be a major bottleneck forcing hundreds of IPIs to deliver the flush. The solution is to track the CPUs where a BPF program ran, and issue IBPB only on those CPUs. 3. Avoid IBPB when flush is already done at other sources (e.g. context switch). Signed-off-by: Pawan Gupta Acked-by: Daniel Borkmann Signed-off-by: Daniel Borkmann --- kernel/bpf/core.c | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 50aba113ef9d..1b32b9f2491f 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -948,8 +948,8 @@ out: void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool was_classic) { unsigned int nbits = BPF_PROG_SIZE_TO_NBITS(size); - struct bpf_prog_pack *pack; - unsigned long pos; + struct bpf_prog_pack *pack, *fallback_pack = NULL; + unsigned long pos, fallback_pos = 0; void *ptr = NULL; mutex_lock(&pack_mutex); @@ -981,8 +981,29 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool list_for_each_entry(pack, &pack_list, list) { pos = bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0, nbits, 0); - if (pos < BPF_PROG_CHUNK_COUNT) + if (pos >= BPF_PROG_CHUNK_COUNT) + continue; + /* Flush not enabled, use any pack */ + if (!static_branch_unlikely(&bpf_pred_flush_enabled)) goto found_free_area; + /* + * cBPF reuse of a dirty pack triggers a flush, so prefer a + * clean pack for cBPF. eBPF never flushes, so pick the first + * free pack, dirty or clean. + */ + if (!was_classic || !pack->arch_flush_needed) + goto found_free_area; + if (!fallback_pack) { + fallback_pack = pack; + fallback_pos = pos; + } + } + + /* No preferred pack found */ + if (fallback_pack) { + pack = fallback_pack; + pos = fallback_pos; + goto found_free_area; } pack = alloc_new_pack(bpf_fill_ill_insns); -- cgit v1.2.3 From b72e29e0f7ee329d89f86db8700c8ea99b4a370a Mon Sep 17 00:00:00 2001 From: Pawan Gupta Date: Mon, 29 Jun 2026 22:39:29 -0700 Subject: bpf: Prefer dirty packs for eBPF allocations The pack allocator only flushes predictors when reusing a dirty pack for cBPF, eBPF allocations never trigger a flush. Currently, eBPF picks the first free pack, which could be a clean pack. As an optimization, leaving a clean pack for cBPF can avoid flushes. Prefer dirty packs for eBPF and keep clean packs free for cBPF. This mirrors the existing cBPF preference for clean packs: each program kind prefers the pack that avoids an extra flush, and falls back to the other kind only when no preferred pack has room. eBPF reuse of a dirty pack is harmless since eBPF being privileged does not flush. Signed-off-by: Pawan Gupta Acked-by: Daniel Borkmann Signed-off-by: Daniel Borkmann --- kernel/bpf/core.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 1b32b9f2491f..6e19a030da6f 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -988,10 +988,10 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool goto found_free_area; /* * cBPF reuse of a dirty pack triggers a flush, so prefer a - * clean pack for cBPF. eBPF never flushes, so pick the first - * free pack, dirty or clean. + * clean pack for cBPF. eBPF never flushes, so steer it to a + * dirty pack and keep clean packs free for cBPF. */ - if (!was_classic || !pack->arch_flush_needed) + if (was_classic ^ pack->arch_flush_needed) goto found_free_area; if (!fallback_pack) { fallback_pack = pack; -- cgit v1.2.3 From 037a3c43edfb597665dd34457cd22b14692f2ba3 Mon Sep 17 00:00:00 2001 From: Taeyang Lee <0wn@theori.io> Date: Sun, 14 Jun 2026 23:22:18 +0900 Subject: perf/core: Detach event groups during remove_on_exec perf_event_remove_on_exec() removes events by calling perf_event_exit_event(). For top-level events, this removes the event from the context with DETACH_EXIT only. This can leave inconsistent group state when a removed event is a group leader and the group contains siblings without remove_on_exec. If the group was active, the surviving siblings can remain active and attached to the removed leader's sibling list, but are no longer represented by a valid group leader on the PMU context active lists. A later close of the removed leader uses DETACH_GROUP and can promote the still-active siblings from this stale group state. The next schedule-in can then add an already-linked active_list entry again, corrupting the PMU context active list. With DEBUG_LIST enabled, this is caught as a list_add double-add in merge_sched_in(). Fix this by detaching group relationships when remove_on_exec removes an event. This preserves the existing task-exit and revoke behavior, while ensuring surviving siblings are ungrouped before the removed event leaves the context. Fixes: 2e498d0a74e5 ("perf: Add support for event removal on exec") Signed-off-by: Taeyang Lee <0wn@theori.io> Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/ai65GgZcC0LAlWLG@Taeyangs-MacBook-Pro.local --- kernel/events/core.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/events/core.c b/kernel/events/core.c index 954c36e28101..d7f3e2c2ecb1 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -4729,7 +4729,7 @@ static void perf_remove_from_owner(struct perf_event *event); static void perf_event_exit_event(struct perf_event *event, struct perf_event_context *ctx, struct task_struct *task, - bool revoke); + unsigned long detach_flags); /* * Removes all events from the current task that have been marked @@ -4756,7 +4756,7 @@ static void perf_event_remove_on_exec(struct perf_event_context *ctx) modified = true; - perf_event_exit_event(event, ctx, ctx->task, false); + perf_event_exit_event(event, ctx, ctx->task, DETACH_GROUP); } raw_spin_lock_irqsave(&ctx->lock, flags); @@ -12937,7 +12937,7 @@ static void __pmu_detach_event(struct pmu *pmu, struct perf_event *event, /* * De-schedule the event and mark it REVOKED. */ - perf_event_exit_event(event, ctx, ctx->task, true); + perf_event_exit_event(event, ctx, ctx->task, DETACH_REVOKE); /* * All _free_event() bits that rely on event->pmu: @@ -14525,12 +14525,13 @@ static void perf_event_exit_event(struct perf_event *event, struct perf_event_context *ctx, struct task_struct *task, - bool revoke) + unsigned long detach_flags) { struct perf_event *parent_event = event->parent; - unsigned long detach_flags = DETACH_EXIT; unsigned int attach_state; + detach_flags |= DETACH_EXIT; + if (parent_event) { /* * Do not destroy the 'original' grouping; because of the @@ -14553,8 +14554,8 @@ perf_event_exit_event(struct perf_event *event, sync_child_event(event, task); } - if (revoke) - detach_flags |= DETACH_GROUP | DETACH_REVOKE; + if (detach_flags & DETACH_REVOKE) + detach_flags |= DETACH_GROUP; perf_remove_from_context(event, detach_flags); /* @@ -14642,7 +14643,7 @@ static void perf_event_exit_task_context(struct task_struct *task, bool exit) perf_event_task(task, ctx, 0); list_for_each_entry_safe(child_event, next, &ctx->event_list, event_entry) - perf_event_exit_event(child_event, ctx, exit ? task : NULL, false); + perf_event_exit_event(child_event, ctx, exit ? task : NULL, 0); mutex_unlock(&ctx->mutex); -- cgit v1.2.3 From 39def6d250d370298f86c116f4ac60093cefadaa Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Wed, 1 Jul 2026 15:11:50 +0200 Subject: futex/requeue: Revert "Prevent NULL pointer dereference in remove_waiter() on self-deadlock"" The commit cited below should not have been merged. It attemted to fix an existing problem ansd thereby introduced new problems by keeping the pi_state in state Q_REQUEUE_PI_IN_PROGRESS and leaking it. Based on the commit description the intention was to handle the case when task_blocks_on_rt_mutex() returns -EDEADLK and the following remove_waiter() dereferences the NULL pointer in waiter->task. That is already handled by Davidlohr in commit 40a25d59e85b3 ("locking/rtmutex: Skip remove_waiter() when waiter is not enqueued") and requires no further acting. Revert the commit breaking the "waiter == owner" case again. Fixes: 74e144274af39 ("futex/requeue: Prevent NULL pointer dereference in remove_waiter() on self-deadlock") Reported-by: Michael Bommarito Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260701131150.0Ijhq4Dw@linutronix.de Closes: https://lore.kernel.org/all/20260629020049.2082397-1-michael.bommarito@gmail.com --- kernel/futex/requeue.c | 6 ------ 1 file changed, 6 deletions(-) (limited to 'kernel') diff --git a/kernel/futex/requeue.c b/kernel/futex/requeue.c index 7384672916fb..79823ad13683 100644 --- a/kernel/futex/requeue.c +++ b/kernel/futex/requeue.c @@ -645,12 +645,6 @@ retry_private: continue; } - /* Self-deadlock: non-top waiter already owns the PI futex. */ - if (rt_mutex_owner(&pi_state->pi_mutex) == this->task) { - ret = -EDEADLK; - break; - } - ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex, this->rt_waiter, this->task); -- cgit v1.2.3