summaryrefslogtreecommitdiff
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/capability.c19
-rw-r--r--kernel/cgroup/cgroup.c6
-rw-r--r--kernel/cgroup/cpuset.c39
-rw-r--r--kernel/dma/contiguous.c16
-rw-r--r--kernel/events/uprobes.c2
-rw-r--r--kernel/locking/lockdep.c58
-rw-r--r--kernel/sched/cpufreq_schedutil.c15
-rw-r--r--kernel/sched/ext/ext.c47
-rw-r--r--kernel/sched/ext/internal.h25
-rw-r--r--kernel/sched/fair.c12
-rw-r--r--kernel/trace/ring_buffer.c7
-rw-r--r--kernel/trace/simple_ring_buffer.c4
-rw-r--r--kernel/trace/trace.c6
-rw-r--r--kernel/trace/trace_events.c2
-rw-r--r--kernel/trace/trace_events_hist.c4
-rw-r--r--kernel/trace/trace_events_user.c3
-rw-r--r--kernel/workqueue.c54
17 files changed, 251 insertions, 68 deletions
diff --git a/kernel/capability.c b/kernel/capability.c
index 829f49ae07b9..90e6ab62f6db 100644
--- a/kernel/capability.c
+++ b/kernel/capability.c
@@ -326,7 +326,6 @@ bool has_capability_noaudit(struct task_struct *t, int cap)
{
return has_ns_capability_noaudit(t, &init_user_ns, cap);
}
-EXPORT_SYMBOL(has_capability_noaudit);
static bool ns_capable_common(struct user_namespace *ns,
int cap,
@@ -416,6 +415,24 @@ bool capable(int cap)
return ns_capable(&init_user_ns, cap);
}
EXPORT_SYMBOL(capable);
+
+/**
+ * capable_noaudit - Determine if the current task has a superior
+ * capability in effect by checking the process's effective
+ * capabilities (unaudited).
+ * @cap: The capability to be tested for
+ *
+ * This is the same as capable(), except it uses CAP_OPT_NOAUDIT as to prevent
+ * issuing spurious audit messages.
+ *
+ * This sets PF_SUPERPRIV on the task if the capability is available on the
+ * assumption that it's about to be used.
+ */
+bool capable_noaudit(int cap)
+{
+ return ns_capable_noaudit(&init_user_ns, cap);
+}
+EXPORT_SYMBOL(capable_noaudit);
#endif /* CONFIG_MULTIUSER */
/**
diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index c3a12fee7528..2d532bf2c0c7 100644
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -6873,10 +6873,7 @@ static int cgroup_css_set_fork(struct kernel_clone_args *kargs)
spin_lock_irq(&css_set_lock);
cset = task_css_set(current);
get_css_set(cset);
- if (kargs->cgrp)
- kargs->kill_seq = kargs->cgrp->kill_seq;
- else
- kargs->kill_seq = cset->dfl_cgrp->kill_seq;
+ kargs->kill_seq = cset->dfl_cgrp->kill_seq;
spin_unlock_irq(&css_set_lock);
if (!(kargs->flags & CLONE_INTO_CGROUP)) {
@@ -6940,6 +6937,7 @@ static int cgroup_css_set_fork(struct kernel_clone_args *kargs)
put_css_set(cset);
kargs->cgrp = dst_cgrp;
+ kargs->kill_seq = dst_cgrp->kill_seq;
return ret;
err:
diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c
index d100634fa12b..2538faac9aba 100644
--- a/kernel/cgroup/cpuset.c
+++ b/kernel/cgroup/cpuset.c
@@ -1259,6 +1259,28 @@ static void reset_partition_data(struct cpuset *cs)
cpumask_copy(cs->effective_cpus, parent->effective_cpus);
}
+/* Return true if isolated_cpus changes. */
+static bool isolated_cpu_update(int new_prs, int cpu)
+{
+ lockdep_assert_held(&callback_lock);
+ lockdep_assert_held(&cpuset_mutex);
+
+ if (new_prs == PRS_ISOLATED) {
+ if (cpumask_test_cpu(cpu, isolated_cpus))
+ return false;
+ cpumask_set_cpu(cpu, isolated_cpus);
+ return true;
+ }
+
+ /* CPUs isolated at boot must remain isolated. */
+ if (!cpumask_test_cpu(cpu,
+ housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT)) ||
+ !cpumask_test_cpu(cpu, isolated_cpus))
+ return false;
+ cpumask_clear_cpu(cpu, isolated_cpus);
+ return true;
+}
+
/*
* isolated_cpus_update - Update the isolated_cpus mask
* @old_prs: old partition_root_state
@@ -1267,19 +1289,16 @@ static void reset_partition_data(struct cpuset *cs)
*/
static void isolated_cpus_update(int old_prs, int new_prs, struct cpumask *xcpus)
{
+ bool updated = false;
+ int cpu;
+
WARN_ON_ONCE(old_prs == new_prs);
lockdep_assert_held(&callback_lock);
lockdep_assert_held(&cpuset_mutex);
- if (new_prs == PRS_ISOLATED) {
- if (cpumask_subset(xcpus, isolated_cpus))
- return;
- cpumask_or(isolated_cpus, isolated_cpus, xcpus);
- } else {
- if (!cpumask_intersects(xcpus, isolated_cpus))
- return;
- cpumask_andnot(isolated_cpus, isolated_cpus, xcpus);
- }
- update_housekeeping = true;
+ for_each_cpu(cpu, xcpus)
+ updated |= isolated_cpu_update(new_prs, cpu);
+ if (updated)
+ update_housekeeping = true;
}
/*
diff --git a/kernel/dma/contiguous.c b/kernel/dma/contiguous.c
index f754079a287d..66093460584e 100644
--- a/kernel/dma/contiguous.c
+++ b/kernel/dma/contiguous.c
@@ -140,21 +140,22 @@ static bool numa_cma_configured __initdata;
static int __init early_numa_cma(char *p)
{
int nid, count = 0;
- unsigned long tmp;
+ unsigned long node;
+ phys_addr_t size;
char *s = p;
while (*s) {
- if (sscanf(s, "%lu%n", &tmp, &count) != 1)
+ if (sscanf(s, "%lu%n", &node, &count) != 1)
break;
if (s[count] == ':') {
- if (tmp >= MAX_NUMNODES)
+ if (node >= MAX_NUMNODES)
break;
- nid = array_index_nospec(tmp, MAX_NUMNODES);
+ nid = array_index_nospec(node, MAX_NUMNODES);
s += count + 1;
- tmp = memparse(s, &s);
- numa_cma_size[nid] = tmp;
+ size = memparse(s, &s);
+ numa_cma_size[nid] = size;
if (*s == ',')
s++;
@@ -207,9 +208,10 @@ static void __init dma_numa_cma_reserve(void)
pernuma_size_bytes = cma_get_size(dma_contiguous_default_area);
for_each_node(nid) {
- int size, ret;
+ phys_addr_t size;
char name[CMA_MAX_NAME];
struct cma **cma;
+ int ret;
if (!node_online(nid)) {
if (pernuma_size_bytes || numa_cma_size[nid])
diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index b25531331902..7709ea882477 100644
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -513,7 +513,7 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma,
uprobe = container_of(auprobe, struct uprobe, arch);
- if (WARN_ON_ONCE(!is_cow_mapping(vma->vm_flags)))
+ if (WARN_ON_ONCE(!vma_is_cow_mapping(vma)))
return -EINVAL;
/*
diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c
index 25d77d4a1061..c56a7f91d72e 100644
--- a/kernel/locking/lockdep.c
+++ b/kernel/locking/lockdep.c
@@ -5093,7 +5093,7 @@ static int __lock_is_held(const struct lockdep_map *lock, int read);
static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
int trylock, int read, int check, int hardirqs_off,
struct lockdep_map *nest_lock, unsigned long ip,
- int references, int pin_count, int sync)
+ int references, int pin_count, int sync, int seq)
{
struct task_struct *curr = current;
struct lock_class *class = NULL;
@@ -5199,6 +5199,7 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
hlock->holdtime_stamp = lockstat_clock();
#endif
hlock->pin_count = pin_count;
+ hlock->seq_count = seq;
if (check_wait_context(curr, hlock))
return 0;
@@ -5404,7 +5405,7 @@ static int reacquire_held_locks(struct task_struct *curr, unsigned int depth,
hlock->read, hlock->check,
hlock->hardirqs_off,
hlock->nest_lock, hlock->acquire_ip,
- hlock->references, hlock->pin_count, 0)) {
+ hlock->references, hlock->pin_count, 0, hlock->seq_count)) {
case 0:
return 1;
case 1:
@@ -5687,14 +5688,17 @@ static void __lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie
struct held_lock *hlock = curr->held_locks + i;
if (match_held_lock(hlock, lock)) {
+ int pin_count;
+
if (WARN(!hlock->pin_count, "unpinning an unpinned lock\n"))
return;
- hlock->pin_count -= cookie.val;
+ pin_count = hlock->pin_count - cookie.val;
- if (WARN((int)hlock->pin_count < 0, "pin count corrupted\n"))
- hlock->pin_count = 0;
+ if (WARN(pin_count < 0, "pin count corrupted\n"))
+ pin_count = 0;
+ hlock->pin_count = pin_count;
return;
}
}
@@ -5702,6 +5706,24 @@ static void __lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie
WARN(1, "unpinning an unheld lock\n");
}
+static u32 __lock_sequence(struct lockdep_map *lock)
+{
+ struct task_struct *curr = current;
+ int i;
+
+ if (unlikely(!debug_locks))
+ return ~0;
+
+ for (i = 0; i < curr->lockdep_depth; i++) {
+ struct held_lock *hlock = curr->held_locks + i;
+
+ if (match_held_lock(hlock, lock))
+ return hlock->seq_count;
+ }
+
+ return ~0;
+}
+
/*
* Check whether we follow the irq-flags state precisely:
*/
@@ -5884,7 +5906,8 @@ void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
lockdep_recursion_inc();
__lock_acquire(lock, subclass, trylock, read, check,
- irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 0);
+ irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 0,
+ ++current->lockdep_seq);
lockdep_recursion_finish();
raw_local_irq_restore(flags);
}
@@ -5932,7 +5955,8 @@ void lock_sync(struct lockdep_map *lock, unsigned subclass, int read,
lockdep_recursion_inc();
__lock_acquire(lock, subclass, 0, read, check,
- irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 1);
+ irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 1,
+ ++current->lockdep_seq);
check_chain_key(current);
lockdep_recursion_finish();
raw_local_irq_restore(flags);
@@ -6018,6 +6042,26 @@ void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie)
}
EXPORT_SYMBOL_GPL(lock_unpin_lock);
+u32 lock_sequence(struct lockdep_map *lock)
+{
+ unsigned long flags;
+ u32 seq = ~0;
+
+ if (unlikely(!lockdep_enabled()))
+ return seq;
+
+ raw_local_irq_save(flags);
+ check_flags(flags);
+
+ lockdep_recursion_inc();
+ seq = __lock_sequence(lock);
+ lockdep_recursion_finish();
+ raw_local_irq_restore(flags);
+
+ return seq;
+}
+EXPORT_SYMBOL_GPL(lock_sequence);
+
#ifdef CONFIG_LOCK_STAT
static void print_lock_contention_bug(struct task_struct *curr,
struct lockdep_map *lock,
diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c
index a1782755efcc..49ccd6f1c185 100644
--- a/kernel/sched/cpufreq_schedutil.c
+++ b/kernel/sched/cpufreq_schedutil.c
@@ -62,6 +62,17 @@ static DEFINE_PER_CPU(struct sugov_cpu, sugov_cpu);
/************************ Governor internals ***********************/
+static void sugov_update_rate_limit_us(struct sugov_policy *sg_policy)
+{
+ /*
+ * Cast rate_limit_us before multiplication to force 64-bit arithmetic.
+ * Otherwise, on 32-bit platforms, both operands are converted to
+ * 32-bit unsigned long and the multiplication may overflow.
+ */
+ sg_policy->freq_update_delay_ns =
+ (s64)sg_policy->tunables->rate_limit_us * NSEC_PER_USEC;
+}
+
static bool sugov_should_update_freq(struct sugov_policy *sg_policy, u64 time)
{
s64 delta_ns;
@@ -608,7 +619,7 @@ rate_limit_us_store(struct gov_attr_set *attr_set, const char *buf, size_t count
tunables->rate_limit_us = rate_limit_us;
list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook)
- sg_policy->freq_update_delay_ns = rate_limit_us * NSEC_PER_USEC;
+ sugov_update_rate_limit_us(sg_policy);
return count;
}
@@ -850,7 +861,7 @@ static int sugov_start(struct cpufreq_policy *policy)
void (*uu)(struct update_util_data *data, u64 time, unsigned int flags);
unsigned int cpu;
- sg_policy->freq_update_delay_ns = sg_policy->tunables->rate_limit_us * NSEC_PER_USEC;
+ sugov_update_rate_limit_us(sg_policy);
sg_policy->last_freq_update_time = 0;
sg_policy->next_freq = 0;
sg_policy->work_in_progress = false;
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index 10af28a9f2c0..713aa26b2828 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -876,9 +876,9 @@ struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter)
* unloading. The init_tasks ("swappers") should be excluded
* from the iteration because:
*
- * - It's unsafe to use __setschduler_prio() on an init_task to
- * determine the sched_class to use as it won't preserve its
- * idle_sched_class.
+ * - It's unsafe to use __setscheduler_class() on an init_task
+ * to determine the sched_class to use as it won't preserve
+ * its idle_sched_class.
*
* - ops.init/exit_task() can easily be confused if called with
* init_tasks as they, e.g., share PID 0.
@@ -2806,6 +2806,8 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
* @p: task to finish dispatching
* @qseq_at_dispatch: qseq when @p started getting dispatched
* @dsq_id: destination DSQ ID
+ * @slice: slice carried by the insert verdict, 0 keeps the current value
+ * @vtime: vtime carried by the insert verdict, committed on PRIQ inserts
* @enq_flags: %SCX_ENQ_*
*
* Dispatching to local DSQs may need to wait for queueing to complete or
@@ -5514,7 +5516,7 @@ static const struct kset_uevent_ops scx_uevent_ops = {
};
/*
- * Used by sched_fork() and __setscheduler_prio() to pick the matching
+ * Used by sched_fork() and __setscheduler_class() to pick the matching
* sched_class. dl/rt are already handled.
*/
bool task_should_scx(int policy)
@@ -7694,7 +7696,7 @@ static void scx_root_enable_workfn(struct kthread_work *work)
/*
* Enable ops for every task. Fork is excluded by scx_fork_rwsem
* preventing new tasks from being added. No need to exclude tasks
- * leaving as sched_ext_free() can handle both prepped and enabled
+ * leaving as sched_ext_dead() can handle both prepped and enabled
* tasks. Prep all tasks first and then enable them with preemption
* disabled.
*
@@ -7786,7 +7788,7 @@ static void scx_root_enable_workfn(struct kthread_work *work)
/*
* We're fully committed and can't fail. The task READY -> ENABLED
- * transitions here are synchronized against sched_ext_free() through
+ * transitions here are synchronized against sched_ext_dead() through
* scx_tasks_lock.
*/
percpu_down_write(&scx_fork_rwsem);
@@ -8079,6 +8081,7 @@ static int bpf_scx_check_member(const struct btf_type *t,
case offsetof(struct sched_ext_ops, cgroup_init):
case offsetof(struct sched_ext_ops, cgroup_exit):
case offsetof(struct sched_ext_ops, cgroup_prep_move):
+ case offsetof(struct sched_ext_ops, cgroup_set_bandwidth):
#endif
case offsetof(struct sched_ext_ops, cpu_online):
case offsetof(struct sched_ext_ops, cpu_offline):
@@ -9003,12 +9006,6 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
if (unlikely(READ_ONCE(sch->aborting)))
return false;
- if (unlikely(!scx_task_on_sched(sch, p))) {
- scx_error(sch, "scx_bpf_dsq_move[_vtime]() on %s[%d] but the task belongs to a different scheduler",
- p->comm, p->pid);
- return false;
- }
-
/*
* Can be called from either ops.dispatch() holding the dispatched rq's
* lock or any context where no rq lock is held. If latter, lock @p's
@@ -9040,6 +9037,17 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
goto out;
}
+ /*
+ * @p has been on $src_dsq and can't move anymore. If @p is not on @sch,
+ * the caller didn't have authority over @p at the time of the call.
+ */
+ if (unlikely(!scx_task_on_sched(sch, p))) {
+ scx_error(sch, "scx_bpf_dsq_move[_vtime]() on %s[%d] but the task belongs to a different scheduler",
+ p->comm, p->pid);
+ raw_spin_unlock(&src_dsq->lock);
+ goto out;
+ }
+
/* @p is still on $src_dsq and stable, determine the destination */
dst_dsq = find_dsq_for_dispatch(sch, locked_rq ?: this_rq(), dsq_id, task_cpu(p));
@@ -9765,7 +9773,7 @@ __bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *i
* bpf_iter_scx_dsq_destroy - Destroy a DSQ iterator
* @it: iterator to destroy
*
- * Undo scx_iter_scx_dsq_new().
+ * Undo bpf_iter_scx_dsq_new().
*/
__bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it)
{
@@ -11041,3 +11049,16 @@ static int __init scx_init(void)
return 0;
}
__initcall(scx_init);
+
+/*
+ * Compatibility markers for userspace. Existence of a marker function
+ * represents that the kernel supports that sched-ext feature.
+ */
+
+/*
+ * scx_compat_marker_cgroup_set_bandwidth_may_sleep: advertises that
+ * ops.cgroup_set_bandwidth() may be implemented as a sleepable callback.
+ */
+#ifdef CONFIG_EXT_GROUP_SCHED
+DEFINE_SCX_COMPAT_MARKER(cgroup_set_bandwidth_may_sleep);
+#endif /* CONFIG_EXT_GROUP_SCHED */
diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h
index 27bbf5e04d90..0967b99a4948 100644
--- a/kernel/sched/ext/internal.h
+++ b/kernel/sched/ext/internal.h
@@ -442,7 +442,7 @@ struct sched_ext_ops {
*
* Note that this callback may be called from a CPU other than the
* one the task is going to run on. This can happen when a task
- * property is changed (i.e., affinity), since scx_next_task_scx(),
+ * property is changed (i.e., affinity), since set_next_task_scx(),
* which triggers this callback, may run on a CPU different from
* the task's assigned CPU.
*
@@ -753,7 +753,7 @@ struct sched_ext_ops {
* @burst_us: bandwidth control burst
*
* Update @cgrp's bandwidth control parameters. This is from the cpu.max
- * cgroup interface.
+ * cgroup interface. This operation may block.
*
* @quota_us / @period_us determines the CPU bandwidth @cgrp is entitled
* to. For example, if @period_us is 1_000_000 and @quota_us is
@@ -2001,6 +2001,27 @@ struct scx_bstr_buf {
char line[SCX_EXIT_MSG_LEN];
};
+/* Internal helper for DEFINE_SCX_COMPAT_MARKER(). */
+#define DECLARE_SCX_COMPAT_MARKER(func) \
+ extern void scx_compat_marker_##func(void)
+
+/**
+ * DEFINE_SCX_COMPAT_MARKER() - define a userspace capability marker
+ * @func: marker suffix; the defined symbol is scx_compat_marker_@func
+ *
+ * Emit an empty, callerless function that is retained in the kernel's BTF.
+ * Its presence is part of the kernel<->userspace contract: userspace probes
+ * scx_compat_marker_@func (e.g. via BTF) to detect that this kernel supports
+ * the corresponding feature.
+ *
+ * The leading declaration suppresses the missing-prototype warning; the
+ * trailing declaration consumes the semicolon at the use site.
+ */
+#define DEFINE_SCX_COMPAT_MARKER(func) \
+ DECLARE_SCX_COMPAT_MARKER(func); \
+ __used __retain void scx_compat_marker_##func(void) {} \
+ DECLARE_SCX_COMPAT_MARKER(func)
+
extern struct scx_sched __rcu *scx_root;
DECLARE_PER_CPU(struct rq *, scx_locked_rq_state);
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 6d881e530f89..8dff37059faf 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -7253,7 +7253,7 @@ static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b)
* period the timer is deactivated until scheduling resumes; cfs_b->idle is
* used to track this state.
*/
-static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun)
+static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags)
__must_hold(&cfs_b->lock)
{
int throttled;
@@ -7288,10 +7288,10 @@ static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun)
* This check is repeated as we release cfs_b->lock while we unthrottle.
*/
while (throttled && cfs_b->runtime > 0) {
- raw_spin_unlock_irq_enable(&cfs_b->lock);
+ raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
/* we can't nest cfs_b->lock while distributing bandwidth */
throttled = distribute_cfs_runtime(cfs_b);
- raw_spin_lock_irq_disable(&cfs_b->lock);
+ raw_spin_lock_irqsave(&cfs_b->lock, flags);
}
/*
@@ -7399,7 +7399,7 @@ static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)
{
/* confirm we're still not at a refresh boundary */
- scoped_guard(raw_spinlock_irq, &cfs_b->lock) {
+ scoped_guard(raw_spinlock_irqsave, &cfs_b->lock) {
u64 runtime = 0, slice = sched_cfs_bandwidth_slice();
cfs_b->slack_started = false;
@@ -7484,14 +7484,14 @@ static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer)
int idle = 0;
int count = 0;
- guard(raw_spinlock_irq)(&cfs_b->lock);
+ CLASS(raw_spinlock_irqsave, cfsb_guard)(&cfs_b->lock);
for (;;) {
overrun = hrtimer_forward_now(timer, cfs_b->period);
if (!overrun)
break;
- idle = do_sched_cfs_period_timer(cfs_b, overrun);
+ idle = do_sched_cfs_period_timer(cfs_b, overrun, cfsb_guard.flags);
if (++count > 3) {
u64 new, old = ktime_to_ns(cfs_b->period);
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index 3c3ed639923d..b0963ac6fd16 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -5805,8 +5805,11 @@ __rb_get_reader_page_from_remote(struct ring_buffer_per_cpu *cpu_buffer)
prev_reader = cpu_buffer->subbuf_ids[cpu_buffer->meta_page->reader.id];
- WARN_ON_ONCE(cpu_buffer->remote->swap_reader_page(cpu_buffer->cpu,
- cpu_buffer->remote->priv));
+ if (cpu_buffer->remote->swap_reader_page(cpu_buffer->cpu,
+ cpu_buffer->remote->priv)) {
+ pr_warn_ratelimited("Remote reader page swap failed\n");
+ return NULL;
+ }
/* nr_pages doesn't include the reader page */
if (WARN_ON_ONCE(cpu_buffer->meta_page->reader.id > cpu_buffer->nr_pages))
return NULL;
diff --git a/kernel/trace/simple_ring_buffer.c b/kernel/trace/simple_ring_buffer.c
index f4642f5adda3..49913bb0057a 100644
--- a/kernel/trace/simple_ring_buffer.c
+++ b/kernel/trace/simple_ring_buffer.c
@@ -160,8 +160,8 @@ int simple_ring_buffer_swap_reader_page(struct simple_rb_per_cpu *cpu_buffer)
overrun = cpu_buffer->meta->overrun;
} while (!simple_bpage_unset_head_link(last, reader, SIMPLE_RB_LINK_NORMAL) && retry--);
- if (!retry)
- return -EINVAL;
+ if (retry < 0)
+ return -EBUSY;
cpu_buffer->head_page = simple_bpage_from_link(reader->link.next);
cpu_buffer->head_page->link.prev = &reader->link;
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 3e0907aef172..a946e0183fd1 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -8216,6 +8216,8 @@ buffer_subbuf_size_write(struct file *filp, const char __user *ubuf,
/* Do not allow tracing while changing the order of the ring buffer */
tracing_stop_tr(tr);
+ trace_access_lock(RING_BUFFER_ALL_CPUS);
+
old_order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer);
if (old_order == order)
goto out;
@@ -8255,6 +8257,7 @@ buffer_subbuf_size_write(struct file *filp, const char __user *ubuf,
#endif
(*ppos)++;
out:
+ trace_access_unlock(RING_BUFFER_ALL_CPUS);
if (ret)
cnt = ret;
tracing_start_tr(tr);
@@ -9728,7 +9731,8 @@ __init static void enable_instances(void)
tr = trace_array_create_systems(name, NULL, addr, size);
if (IS_ERR(tr)) {
- pr_warn("Tracing: Failed to create instance buffer %s\n", curr_str);
+ pr_warn("Tracing: Failed to create instance buffer '%s' (%ld)\n", name,
+ PTR_ERR(tr));
continue;
}
diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c
index 9f8f2d02276c..1d39eaf6a0f7 100644
--- a/kernel/trace/trace_events.c
+++ b/kernel/trace/trace_events.c
@@ -5019,6 +5019,8 @@ static __init void event_test_stuff(void)
struct task_struct *test_thread;
test_thread = kthread_run(event_test_thread, NULL, "test-events");
+ if (WARN_ON(IS_ERR(test_thread)))
+ return;
msleep(1);
kthread_stop(test_thread);
}
diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c
index 893bd8b0e48a..963e0d6b61fd 100644
--- a/kernel/trace/trace_events_hist.c
+++ b/kernel/trace/trace_events_hist.c
@@ -6661,8 +6661,10 @@ static int hist_register_trigger(char *glob,
tracing_set_filter_buffering(file->tr, true);
}
- if (named_data)
+ if (named_data) {
+ remove_hist_vars(hist_data);
destroy_hist_data(hist_data);
+ }
out:
return ret;
}
diff --git a/kernel/trace/trace_events_user.c b/kernel/trace/trace_events_user.c
index 2bbc89d4a266..93cda2f6f269 100644
--- a/kernel/trace/trace_events_user.c
+++ b/kernel/trace/trace_events_user.c
@@ -868,6 +868,9 @@ void user_event_mm_dup(struct task_struct *t, struct user_event_mm *old_mm)
struct user_event_mm *mm = user_event_mm_alloc(t);
struct user_event_enabler *enabler;
+ /* On failure, do not free parent's copy */
+ t->user_event_mm = NULL;
+
if (!mm)
return;
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index bfeef512f6dd..1ae3732a2c51 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -3197,7 +3197,16 @@ restart:
#ifdef CONFIG_PREEMPT_RT
static void worker_lock_callback(struct worker_pool *pool)
{
- spin_lock(&pool->cb_lock);
+ /*
+ * SINGLE_DEPTH_NESTING is for a dead pool's bh_worker() running from
+ * drain_dead_softirq_workfn() inside a live pool's bh_worker(). The
+ * unlocked read is stable: the flag is only set while @pool's CPU is
+ * dead, inside a serialized hotplug operation. data_race() as the value
+ * only affects the lockdep annotation and the read can be elided when
+ * lockdep is disabled.
+ */
+ spin_lock_nested(&pool->cb_lock,
+ data_race(pool->flags) & POOL_BH_DRAINING ? SINGLE_DEPTH_NESTING : 0);
}
static void worker_unlock_callback(struct worker_pool *pool)
@@ -5285,12 +5294,6 @@ static void pwq_release_workfn(struct kthread_work *work)
mutex_unlock(&wq->mutex);
}
- if (!is_percpu_pool(pool)) {
- mutex_lock(&wq_pool_mutex);
- put_unbound_pool(pool);
- mutex_unlock(&wq_pool_mutex);
- }
-
if (!list_empty(&pwq->pending_node)) {
struct wq_node_nr_active *nna =
wq_node_nr_active(pwq->wq, pwq->pool->node);
@@ -5300,6 +5303,12 @@ static void pwq_release_workfn(struct kthread_work *work)
raw_spin_unlock_irq(&nna->lock);
}
+ if (!is_percpu_pool(pool)) {
+ mutex_lock(&wq_pool_mutex);
+ put_unbound_pool(pool);
+ mutex_unlock(&wq_pool_mutex);
+ }
+
kfree_rcu(pwq, rcu);
/*
@@ -6294,6 +6303,30 @@ bool current_is_workqueue_rescuer(void)
}
/**
+ * current_is_workqueue_mem_reclaim - is %current a %WQ_MEM_RECLAIM worker?
+ *
+ * Determine whether %current is a workqueue worker executing on a workqueue
+ * created with %WQ_MEM_RECLAIM. This mirrors the condition that
+ * check_flush_dependency() warns on: flushing (or otherwise waiting on) a
+ * !WQ_MEM_RECLAIM workqueue from such a context breaks the forward-progress
+ * guarantee and can deadlock. Callers that may recurse into such a flush --
+ * e.g. NFS LOCALIO submitting into a stacked filesystem that flushes its own
+ * !WQ_MEM_RECLAIM workqueue -- can use this to decide whether they must defer
+ * the work to a !WQ_MEM_RECLAIM workqueue rather than run it inline.
+ *
+ * Return: %true if %current is a %WQ_MEM_RECLAIM worker. %false otherwise.
+ */
+bool current_is_workqueue_mem_reclaim(void)
+{
+ struct worker *worker = current_wq_worker();
+
+ return worker &&
+ ((worker->current_pwq->wq->flags &
+ (WQ_MEM_RECLAIM | __WQ_LEGACY)) == WQ_MEM_RECLAIM);
+}
+EXPORT_SYMBOL_GPL(current_is_workqueue_mem_reclaim);
+
+/**
* workqueue_congested - test whether a workqueue is congested
* @cpu: CPU in question
* @wq: target workqueue
@@ -8026,6 +8059,9 @@ static int wq_watchdog_param_set_thresh(const char *val,
if (ret)
return ret;
+ if (thresh > MAX_JIFFY_OFFSET / HZ)
+ return -ERANGE;
+
if (system_percpu_wq)
wq_watchdog_set_thresh(thresh);
else
@@ -8056,12 +8092,12 @@ static inline void wq_watchdog_init(void) { }
static void bh_pool_kick_normal(struct irq_work *irq_work)
{
- raise_softirq_irqoff(TASKLET_SOFTIRQ);
+ raise_softirq(TASKLET_SOFTIRQ);
}
static void bh_pool_kick_highpri(struct irq_work *irq_work)
{
- raise_softirq_irqoff(HI_SOFTIRQ);
+ raise_softirq(HI_SOFTIRQ);
}
static void __init restrict_unbound_cpumask(const char *name, const struct cpumask *mask)
generated by cgit v1.2.3 (git 2.25.1) at 2026-09-02 18:27:33 +0000