summaryrefslogtreecommitdiff
path: root/kernel
diff options
context:
space:
mode:
authorRafael J. Wysocki <rafael.j.wysocki@intel.com>2026-07-23 13:25:55 +0200
committerRafael J. Wysocki <rafael.j.wysocki@intel.com>2026-07-23 13:25:55 +0200
commit7f464fb22975abfe56a1e10eb5af3f37c200de10 (patch)
tree61d72ed8ce1acb7a3917c4a6a1c3e1c16ad80f6c /kernel
parenta343c6f15cc94a93aa2d51674d8ca10b36e750bc (diff)
parent39c0cf62fc7851a17782e7efe8dfb2948739c681 (diff)
Merge tag 'amd-pstate-v7.3-2026-07-22' of ssh://gitolite.kernel.org/pub/scm/linux/kernel/git/superm1/linux
Merge amd-pstate content for 7.3 (07/22/26) from Mario Limonciello: "* Avoid running unit tests without amd-pstate * Fixes for EPP on shared memory systems * Fixes for dynamic EPP callbacks * Avoid loading on guests * Allow lowest nonlinear == minimum freq" * tag 'amd-pstate-v7.3-2026-07-22' of ssh://gitolite.kernel.org/pub/scm/linux/kernel/git/superm1/linux: (923 commits) cpufreq/amd-pstate: handle missing policy in dynamic EPP callbacks cpufreq/amd-pstate: Cache the firmware programmed EPP value cpufreq/amd-pstate: Toggle auto_sel in active mode on shared memory systems cpufreq/amd-pstate: Fix EPP return type and handle errors during initialization cpufreq: amd-pstate-ut: Skip tests when amd-pstate driver is not active cpufreq/amd-pstate: Prevent the driver from loading on unsupported hardware cpufreq/amd-pstate: Loosen requirement on lowest nonlinear frequency != min freq Linux 7.2-rc4 Revert "drm/amd/display: Restore 5s vbl offdelay for NV3x+ DGPUs" drm/amd/display: check GRPH_FLIP status before sending event drm/amd/display: consolidate DCN vblank/flip handling onto vupdate_no_lock drm/amd: Create a device link between APU display and XHCI devices drm/amd/display: wire DCN42B mcache programming callback drm/amd/display: set new_stream to NULL after release drm/amd/display: Force PWM backlight on Lenovo Legion 5 15ARH05 drm/amdkfd: free MQD managers on DQM init failures drm/amdgpu/ttm: Consider concurrent VM flushes for buffer entities drm/amd/pm/smu7: Fix AC/DC switch notification drm/amdgpu: Disable PCIe dynamic speed switching on Ryzen Pinnacle Ridge drm/amdgpu: always emit the job vm fence ...
Diffstat (limited to 'kernel')
-rw-r--r--kernel/audit.c21
-rw-r--r--kernel/bpf/btf.c29
-rw-r--r--kernel/bpf/verifier.c97
-rw-r--r--kernel/cgroup/cpuset.c7
-rw-r--r--kernel/events/core.c2
-rw-r--r--kernel/exit.c7
-rw-r--r--kernel/sched/ext/ext.c177
-rw-r--r--kernel/sched/ext/internal.h23
-rw-r--r--kernel/signal.c10
-rw-r--r--kernel/time/posix-cpu-timers.c173
-rw-r--r--kernel/trace/ring_buffer.c10
-rw-r--r--kernel/trace/trace.c3
-rw-r--r--kernel/trace/trace_events_filter.c6
-rw-r--r--kernel/trace/trace_events_synth.c10
-rw-r--r--kernel/trace/trace_events_user.c39
-rw-r--r--kernel/trace/trace_functions.c8
-rw-r--r--kernel/trace/trace_osnoise.c4
-rw-r--r--kernel/trace/trace_preemptirq.c2
-rw-r--r--kernel/trace/trace_remote.c18
19 files changed, 454 insertions, 192 deletions
diff --git a/kernel/audit.c b/kernel/audit.c
index dcc657d35776..562476937fa7 100644
--- a/kernel/audit.c
+++ b/kernel/audit.c
@@ -62,6 +62,7 @@
#include <net/ip.h>
#include <net/ipv6.h>
#include <linux/sctp.h>
+#include <linux/overflow.h>
#include "audit.h"
@@ -950,7 +951,7 @@ main_queue:
* do the multicast send and rotate records from the
* main queue to the retry/hold queues */
wait_event_freezable(kauditd_wait,
- (skb_queue_len(&audit_queue) ? 1 : 0));
+ (skb_queue_len_lockless(&audit_queue) ? 1 : 0));
}
return 0;
@@ -1283,7 +1284,7 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh,
s.rate_limit = audit_rate_limit;
s.backlog_limit = audit_backlog_limit;
s.lost = atomic_read(&audit_lost);
- s.backlog = skb_queue_len(&audit_queue);
+ s.backlog = skb_queue_len_lockless(&audit_queue);
s.feature_bitmap = AUDIT_FEATURE_BITMAP_ALL;
s.backlog_wait_time = audit_backlog_wait_time;
s.backlog_wait_time_actual = atomic_read(&audit_backlog_wait_time_actual);
@@ -1627,7 +1628,7 @@ static void audit_receive(struct sk_buff *skb)
/* can't block with the ctrl lock, so penalize the sender now */
if (audit_backlog_limit &&
- (skb_queue_len(&audit_queue) > audit_backlog_limit)) {
+ (skb_queue_len_lockless(&audit_queue) > audit_backlog_limit)) {
DECLARE_WAITQUEUE(wait, current);
/* wake kauditd to try and flush the queue */
@@ -1933,7 +1934,7 @@ struct audit_buffer *audit_log_start(struct audit_context *ctx, gfp_t gfp_mask,
long stime = audit_backlog_wait_time;
while (audit_backlog_limit &&
- (skb_queue_len(&audit_queue) > audit_backlog_limit)) {
+ (skb_queue_len_lockless(&audit_queue) > audit_backlog_limit)) {
/* wake kauditd to try and flush the queue */
wake_up_interruptible(&kauditd_wait);
@@ -1953,7 +1954,7 @@ struct audit_buffer *audit_log_start(struct audit_context *ctx, gfp_t gfp_mask,
} else {
if (audit_rate_check() && printk_ratelimit())
pr_warn("audit_backlog=%d > audit_backlog_limit=%d\n",
- skb_queue_len(&audit_queue),
+ skb_queue_len_lockless(&audit_queue),
audit_backlog_limit);
audit_log_lost("backlog limit exceeded");
return NULL;
@@ -2080,7 +2081,8 @@ void audit_log_format(struct audit_buffer *ab, const char *fmt, ...)
void audit_log_n_hex(struct audit_buffer *ab, const unsigned char *buf,
size_t len)
{
- int i, avail, new_len;
+ int avail;
+ size_t i, new_len;
unsigned char *ptr;
struct sk_buff *skb;
@@ -2090,7 +2092,12 @@ void audit_log_n_hex(struct audit_buffer *ab, const unsigned char *buf,
BUG_ON(!ab->skb);
skb = ab->skb;
avail = skb_tailroom(skb);
- new_len = len<<1;
+
+ if (check_shl_overflow(len, 1, &new_len)) {
+ audit_log_format(ab, "?");
+ return;
+ }
+
if (new_len >= avail) {
/* Round the buffer request up to the next multiple */
new_len = AUDIT_BUFSIZ*(((new_len-avail)/AUDIT_BUFSIZ) + 1);
diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index 64572f85edc8..c4673a54c4ba 100644
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -9114,6 +9114,35 @@ u32 *btf_kfunc_flags(const struct btf *btf, u32 kfunc_btf_id, const struct bpf_p
return btf_kfunc_id_set_contains(btf, hook, kfunc_btf_id);
}
+/*
+ * Check a single KF_* @flag on a kfunc across all of its hook sets.
+ * Returns:
+ * * 1 if @flag is set
+ * * 0 if @flag is not set
+ * * -EINVAL if @flag is set inconsistently across the sets
+ * * -ENOENT if kfunc_btf_id is not a registered kfunc
+ */
+int btf_kfunc_check_flag(const struct btf *btf, u32 kfunc_btf_id, u32 flag)
+{
+ enum btf_kfunc_hook hook;
+ int res = -ENOENT;
+ bool is_set;
+ u32 *flags;
+
+ for (hook = 0; hook < BTF_KFUNC_HOOK_MAX; hook++) {
+ flags = btf_kfunc_id_set_contains(btf, hook, kfunc_btf_id);
+ if (!flags)
+ continue;
+ is_set = *flags & flag;
+ if (res < 0)
+ res = is_set;
+ else if (res != is_set)
+ return -EINVAL;
+ }
+
+ return res;
+}
+
u32 *btf_kfunc_is_modify_return(const struct btf *btf, u32 kfunc_btf_id,
const struct bpf_prog *prog)
{
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 6515d4d3c003..99444eae917e 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -2584,24 +2584,25 @@ static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset)
#define KF_IMPL_SUFFIX "_impl"
-static const struct btf_type *find_kfunc_impl_proto(struct bpf_verifier_env *env,
+static const struct btf_type *find_kfunc_impl_proto(struct bpf_verifier_log *log,
struct btf *btf,
const char *func_name)
{
- char *buf = env->tmp_str_buf;
const struct btf_type *func;
+ char buf[KSYM_NAME_LEN];
s32 impl_id;
int len;
- len = snprintf(buf, TMP_STR_BUF_LEN, "%s%s", func_name, KF_IMPL_SUFFIX);
- if (len < 0 || len >= TMP_STR_BUF_LEN) {
- verbose(env, "function name %s%s is too long\n", func_name, KF_IMPL_SUFFIX);
+ len = snprintf(buf, sizeof(buf), "%s%s", func_name, KF_IMPL_SUFFIX);
+ if (len < 0 || len >= sizeof(buf)) {
+ bpf_log(log, "function name %s%s is too long\n",
+ func_name, KF_IMPL_SUFFIX);
return NULL;
}
impl_id = btf_find_by_name_kind(btf, buf, BTF_KIND_FUNC);
if (impl_id <= 0) {
- verbose(env, "cannot find function %s in BTF\n", buf);
+ bpf_log(log, "cannot find function %s in BTF\n", buf);
return NULL;
}
@@ -2653,7 +2654,7 @@ static int fetch_kfunc_meta(struct bpf_verifier_env *env,
* can be found through the counterpart _impl kfunc.
*/
if (kfunc_flags && (*kfunc_flags & KF_IMPLICIT_ARGS))
- func_proto = find_kfunc_impl_proto(env, btf, func_name);
+ func_proto = find_kfunc_impl_proto(&env->log, btf, func_name);
else
func_proto = btf_type_by_id(btf, func->type);
@@ -5326,14 +5327,11 @@ static int check_max_stack_depth(struct bpf_verifier_env *env)
static int __check_buffer_access(struct bpf_verifier_env *env,
const char *buf_info,
const struct bpf_reg_state *reg,
- argno_t argno, int off, int size)
+ argno_t argno, int off, int size,
+ u32 *access_end)
{
- if (off < 0) {
- verbose(env,
- "%s invalid %s buffer access: off=%d, size=%d\n",
- reg_arg_name(env, argno), buf_info, off, size);
- return -EACCES;
- }
+ s64 start;
+
if (!tnum_is_const(reg->var_off)) {
char tn_buf[48];
@@ -5344,6 +5342,15 @@ static int __check_buffer_access(struct bpf_verifier_env *env,
return -EACCES;
}
+ start = (s64)reg->var_off.value + off;
+ if (start < 0) {
+ verbose(env,
+ "%s invalid negative %s buffer offset: off=%d, var_off=%lld\n",
+ reg_arg_name(env, argno), buf_info, off, (s64)reg->var_off.value);
+ return -EACCES;
+ }
+
+ *access_end = start + size;
return 0;
}
@@ -5351,14 +5358,14 @@ static int check_tp_buffer_access(struct bpf_verifier_env *env,
const struct bpf_reg_state *reg,
argno_t argno, int off, int size)
{
+ u32 access_end;
int err;
- err = __check_buffer_access(env, "tracepoint", reg, argno, off, size);
+ err = __check_buffer_access(env, "tracepoint", reg, argno, off, size, &access_end);
if (err)
return err;
- env->prog->aux->max_tp_access = max(reg->var_off.value + off + size,
- env->prog->aux->max_tp_access);
+ env->prog->aux->max_tp_access = max(access_end, env->prog->aux->max_tp_access);
return 0;
}
@@ -5370,13 +5377,14 @@ static int check_buffer_access(struct bpf_verifier_env *env,
u32 *max_access)
{
const char *buf_info = type_is_rdonly_mem(reg->type) ? "rdonly" : "rdwr";
+ u32 access_end;
int err;
- err = __check_buffer_access(env, buf_info, reg, argno, off, size);
+ err = __check_buffer_access(env, buf_info, reg, argno, off, size, &access_end);
if (err)
return err;
- *max_access = max(reg->var_off.value + off + size, *max_access);
+ *max_access = max(access_end, *max_access);
return 0;
}
@@ -18873,6 +18881,47 @@ static int btf_id_allow_sleepable(u32 btf_id, unsigned long addr, const struct b
return -EINVAL;
}
+/*
+ * Resolve the prototype describing a trace target's real ABI. A
+ * KF_IMPLICIT_ARGS kfunc has its injected args stripped from the public
+ * prototype, so use the _impl prototype; other targets use their own.
+ */
+static const struct btf_type *
+btf_attach_func_proto(struct bpf_verifier_log *log, struct btf *btf, u32 func_id)
+{
+ const struct btf_type *func;
+ struct module *mod = NULL;
+ const char *name;
+ int implicit;
+
+ func = btf_type_by_id(btf, func_id);
+ if (!func || !btf_type_is_func(func))
+ return NULL;
+ name = btf_name_by_offset(btf, func->name_off);
+
+ /*
+ * btf_kfunc_check_flag() reads kfunc_set_tab, which for a module is
+ * stable only once it is live; hold a module ref across the read to
+ * exclude a concurrent module load.
+ */
+ if (btf_is_module(btf)) {
+ mod = btf_try_get_module(btf);
+ if (!mod)
+ return NULL;
+ }
+ implicit = btf_kfunc_check_flag(btf, func_id, KF_IMPLICIT_ARGS);
+ module_put(mod);
+
+ if (implicit == -EINVAL) {
+ bpf_log(log, "kfunc %s has inconsistent KF_IMPLICIT_ARGS\n", name);
+ return NULL;
+ }
+ if (implicit > 0)
+ return find_kfunc_impl_proto(log, btf, name);
+
+ return btf_type_by_id(btf, func->type);
+}
+
int bpf_check_attach_target(struct bpf_verifier_log *log,
const struct bpf_prog *prog,
const struct bpf_prog *tgt_prog,
@@ -19121,8 +19170,8 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
if (prog_extension &&
btf_check_type_match(log, prog, btf, t))
return -EINVAL;
- t = btf_type_by_id(btf, t->type);
- if (!btf_type_is_func_proto(t))
+ t = btf_attach_func_proto(log, btf, btf_id);
+ if (!t || !btf_type_is_func_proto(t))
return -EINVAL;
if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) &&
@@ -19405,10 +19454,8 @@ int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 bt
tname = btf_name_by_offset(btf, t->name_off);
if (!tname)
return -EINVAL;
- if (!btf_type_is_func(t))
- return -EINVAL;
- t = btf_type_by_id(btf, t->type);
- if (!btf_type_is_func_proto(t))
+ t = btf_attach_func_proto(NULL, btf, btf_id);
+ if (!t || !btf_type_is_func_proto(t))
return -EINVAL;
err = btf_distill_func_proto(NULL, btf, t, tname, &tgt_info->fmodel);
if (err < 0)
diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c
index 591e3aa487fc..45944b3e31ca 100644
--- a/kernel/cgroup/cpuset.c
+++ b/kernel/cgroup/cpuset.c
@@ -2653,7 +2653,12 @@ void cpuset_update_tasks_nodemask(struct cpuset *cs)
migrate = is_memory_migrate(cs);
- mpol_rebind_mm(mm, &cs->mems_allowed);
+ /*
+ * For v1 we can have empty effective_mems, but we cannot
+ * attach any tasks (see cpuset_can_attach_check()). For v2,
+ * effective_mems is guaranteed to not be empty.
+ */
+ mpol_rebind_mm(mm, &cs->effective_mems);
if (migrate)
cpuset_migrate_mm(mm, &cs->old_mems_allowed, &newmems);
else
diff --git a/kernel/events/core.c b/kernel/events/core.c
index d7f3e2c2ecb1..ba5bd6a78fe7 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -7150,6 +7150,8 @@ static int map_range(struct perf_buffer *rb, struct vm_area_struct *vma)
int err = 0;
unsigned long pagenum;
+ guard(mutex)(&rb->aux_mutex);
+
/*
* We map this as a VM_PFNMAP VMA.
*
diff --git a/kernel/exit.c b/kernel/exit.c
index 1056422bc101..2c0b1c02920f 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -212,7 +212,12 @@ static void __exit_signal(struct release_task_post *post, struct task_struct *ts
__unhash_process(post, tsk, group_dead);
write_sequnlock(&sig->stats_lock);
- tsk->sighand = NULL;
+ /*
+ * Ensure that all preceeding state is visible. Pairs with
+ * the smp_acquire__after_ctrl_dep() in the sighand == NULL
+ * path of lock_task_sighand().
+ */
+ smp_store_release(&tsk->sighand, NULL);
spin_unlock(&sighand->siglock);
__cleanup_sighand(sighand);
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index 691d53fe0f64..e3fa7b2fac9d 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -479,6 +479,18 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags)
*/
DEFINE_PER_CPU(struct rq *, scx_locked_rq_state);
+static void switch_rq_lock(struct rq *from, struct rq *to)
+{
+ bool tracked = scx_locked_rq() == from;
+
+ if (tracked)
+ update_locked_rq(NULL);
+ raw_spin_rq_unlock(from);
+ raw_spin_rq_lock(to);
+ if (tracked)
+ update_locked_rq(to);
+}
+
/*
* Flipped on enable per sch->is_cid_type. Declared in internal.h so
* subsystem inlines can read it.
@@ -2274,8 +2286,7 @@ static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags,
deactivate_task(src_rq, p, 0);
set_task_cpu(p, cpu_of(dst_rq));
- raw_spin_rq_unlock(src_rq);
- raw_spin_rq_lock(dst_rq);
+ switch_rq_lock(src_rq, dst_rq);
/*
* We want to pass scx-specific enq_flags but activate_task() will
@@ -2307,6 +2318,7 @@ static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags,
* no to the BPF scheduler initiated migrations while offline.
*
* The caller must ensure that @p and @rq are on different CPUs.
+ * If enforce == true, caller must hold @p's rq lock.
*/
static bool task_can_run_on_remote_rq(struct scx_sched *sch,
struct task_struct *p, struct rq *rq,
@@ -2314,6 +2326,14 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch,
{
s32 cpu = cpu_of(rq);
+ /*
+ * To prevent races with @p still running on its old CPU while switching
+ * out, make sure we're holding @p's rq lock so as not to risk
+ * erroneously killing the BPF scheduler.
+ */
+ if (enforce)
+ lockdep_assert_rq_held(task_rq(p));
+
WARN_ON_ONCE(task_cpu(p) == cpu);
/*
@@ -2581,13 +2601,6 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
return;
}
- if (src_rq != dst_rq &&
- unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) {
- dispatch_enqueue(sch, rq, find_global_dsq(sch, task_cpu(p)), p,
- enq_flags | SCX_ENQ_CLEAR_OPSS | SCX_ENQ_GDSQ_FALLBACK);
- return;
- }
-
/*
* @p is on a possibly remote @src_rq which we need to lock to move the
* task. If dequeue is in progress, it'd be locking @src_rq and waiting
@@ -2606,14 +2619,14 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
/* switch to @src_rq lock */
if (locked_rq != src_rq) {
- raw_spin_rq_unlock(locked_rq);
+ switch_rq_lock(locked_rq, src_rq);
locked_rq = src_rq;
- raw_spin_rq_lock(src_rq);
}
/* task_rq couldn't have changed if we're still the holding cpu */
if (likely(p->scx.holding_cpu == raw_smp_processor_id()) &&
!WARN_ON_ONCE(src_rq != task_rq(p))) {
+ bool fallback = false;
/*
* If @p is staying on the same rq, there's no need to go
* through the full deactivate/activate cycle. Optimize by
@@ -2623,6 +2636,11 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
p->scx.holding_cpu = -1;
dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p,
enq_flags);
+ } else if (unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) {
+ p->scx.holding_cpu = -1;
+ fallback = true;
+ dispatch_enqueue(sch, src_rq, find_global_dsq(sch, task_cpu(p)),
+ p, enq_flags | SCX_ENQ_GDSQ_FALLBACK);
} else {
move_remote_task_to_local_dsq(p, enq_flags,
src_rq, dst_rq);
@@ -2631,15 +2649,13 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
}
/* if the destination CPU is idle, wake it up */
- if (sched_class_above(p->sched_class, dst_rq->curr->sched_class))
+ if (!fallback && sched_class_above(p->sched_class, dst_rq->curr->sched_class))
resched_curr(dst_rq);
}
/* switch back to @rq lock */
- if (locked_rq != rq) {
- raw_spin_rq_unlock(locked_rq);
- raw_spin_rq_lock(rq);
- }
+ if (locked_rq != rq)
+ switch_rq_lock(locked_rq, rq);
}
/**
@@ -2970,24 +2986,38 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first)
/*
* @p is getting newly scheduled or got kicked after someone updated its
- * slice. Refresh whether tick can be stopped. See scx_can_stop_tick().
+ * slice. Update SCX_RQ_CAN_STOP_TICK to reflect whether the tick can be
+ * stopped. See scx_can_stop_tick().
+ *
+ * Moreover, refresh the load_avgs just when transitioning in and out of
+ * nohz. In the future, we might want to add a mechanism to update
+ * load_avgs periodically on tick-stopped CPUs.
*/
- if ((p->scx.slice == SCX_SLICE_INF) !=
- (bool)(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) {
- if (p->scx.slice == SCX_SLICE_INF)
+ if (p->scx.slice == SCX_SLICE_INF) {
+ if (!(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) {
+ /*
+ * Bypass mode always assigns finite slices, so @p
+ * can't have an infinite slice while bypassing.
+ * Therefore, sched_update_tick_dependency() can safely
+ * evaluate the outgoing task.
+ */
rq->scx.flags |= SCX_RQ_CAN_STOP_TICK;
- else
- rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK;
+ sched_update_tick_dependency(rq);
- sched_update_tick_dependency(rq);
+ update_other_load_avgs(rq);
+ }
+ } else {
+ if (rq->scx.flags & SCX_RQ_CAN_STOP_TICK) {
+ rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK;
+ update_other_load_avgs(rq);
+ }
/*
- * For now, let's refresh the load_avgs just when transitioning
- * in and out of nohz. In the future, we might want to add a
- * mechanism which calls the following periodically on
- * tick-stopped CPUs.
+ * @rq still references the outgoing scheduling context. A finite
+ * slice is sufficient by itself to require the tick.
*/
- update_other_load_avgs(rq);
+ if (tick_nohz_full_cpu(cpu_of(rq)))
+ tick_nohz_dep_set_cpu(cpu_of(rq), TICK_DEP_BIT_SCHED);
}
}
@@ -3082,9 +3112,14 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p,
* sched_class, %SCX_OPS_ENQ_LAST must be set. Tell
* ops.enqueue() that @p is the only one available for this cpu,
* which should trigger an explicit follow-up scheduling event.
+ *
+ * Core scheduling can force this CPU idle while @p stays
+ * runnable. @p's cookie then won't match the core's, so skip
+ * the warning in that case.
*/
if (next && sched_class_above(&ext_sched_class, next->sched_class)) {
- WARN_ON_ONCE(!(sch->ops.flags & SCX_OPS_ENQ_LAST));
+ WARN_ON_ONCE(sched_cpu_cookie_match(rq, p) &&
+ !(sch->ops.flags & SCX_OPS_ENQ_LAST));
do_enqueue_task(rq, p, SCX_ENQ_LAST, -1);
} else {
do_enqueue_task(rq, p, 0, -1);
@@ -3648,6 +3683,13 @@ static void scx_disable_task(struct scx_sched *sch, struct task_struct *p)
scx_set_task_state(p, SCX_TASK_READY);
/*
+ * Reset the SCX-managed fields when @p leaves the BPF scheduler's
+ * control, after ops.disable() has observed their final values.
+ */
+ p->scx.dsq_vtime = 0;
+ p->scx.slice = 0;
+
+ /*
* Verify the task is not in BPF scheduler's custody. If flag
* transitions are consistent, the flag should always be clear
* here.
@@ -3925,6 +3967,17 @@ static void reweight_task_scx(struct rq *rq, struct task_struct *p,
if (task_dead_and_done(p))
return;
+ /*
+ * When switching sched_class away from SCX, reweight_task_scx()
+ * is called _after_ scx_disable_task(). Skip calling ops.set_weight()
+ * since the BPF scheduler may have already forgotten the task in
+ * ops.disable().
+ * p->scx.weight will be recalculated in scx_enable_task() if the task
+ * ever returns to SCX class.
+ */
+ if (scx_get_task_state(p) != SCX_TASK_ENABLED)
+ return;
+
p->scx.weight = sched_weight_to_cgroup(scale_load_down(lw->weight));
if (SCX_HAS_OP(sch, set_weight))
SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight);
@@ -4301,6 +4354,15 @@ bool scx_can_stop_tick(struct rq *rq)
if (p->sched_class != &ext_sched_class)
return true;
+ /*
+ * @rq->curr may still reference an outgoing EXT task after it has been
+ * dequeued. If no EXT tasks are accounted on @rq, ignore its stale
+ * slice state. If another task is dispatched from a DSQ,
+ * set_next_task_scx() will update the dependency for the incoming task.
+ */
+ if (!rq->scx.nr_running)
+ return true;
+
if (scx_bypassing(sch, cpu_of(rq)))
return false;
@@ -4901,6 +4963,8 @@ static void scx_sched_free_rcu_work(struct work_struct *work)
cgroup_put(sch_cgroup(sch));
if (sch->sub_kset)
kobject_put(&sch->sub_kset->kobj);
+ if (scx_parent(sch))
+ kobject_put(&scx_parent(sch)->kobj);
#endif /* CONFIG_EXT_SUB_SCHED */
for_each_possible_cpu(cpu) {
@@ -5672,7 +5736,7 @@ static void free_kick_syncs(void)
int cpu;
for_each_possible_cpu(cpu) {
- struct scx_kick_syncs **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu);
+ struct scx_kick_syncs __rcu **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu);
struct scx_kick_syncs *to_free;
to_free = rcu_replace_pointer(*ksyncs, NULL, true);
@@ -6653,7 +6717,7 @@ static int alloc_kick_syncs(void)
* can exceed percpu allocator limits on large machines.
*/
for_each_possible_cpu(cpu) {
- struct scx_kick_syncs **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu);
+ struct scx_kick_syncs __rcu **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu);
struct scx_kick_syncs *new_ksyncs;
WARN_ON_ONCE(rcu_access_pointer(*ksyncs));
@@ -6825,11 +6889,6 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
sch->ops = *cmd->ops;
}
- rcu_assign_pointer(ops->priv, sch);
-
- sch->kobj.kset = scx_kset;
- INIT_LIST_HEAD(&sch->all);
-
#ifdef CONFIG_EXT_SUB_SCHED
char *buf = kzalloc(PATH_MAX, GFP_KERNEL);
if (!buf) {
@@ -6847,13 +6906,32 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
sch->cgrp = cgrp;
INIT_LIST_HEAD(&sch->children);
INIT_LIST_HEAD(&sch->sibling);
+#endif /* CONFIG_EXT_SUB_SCHED */
- if (parent)
+ /*
+ * Publishing makes @sch visible to scx_prog_sched() readers. Failure
+ * paths after this point must free @sch through kobject_put() whose
+ * release path defers the actual freeing by an RCU grace period.
+ */
+ rcu_assign_pointer(ops->priv, sch);
+
+ sch->kobj.kset = scx_kset;
+ INIT_LIST_HEAD(&sch->all);
+
+#ifdef CONFIG_EXT_SUB_SCHED
+ if (parent) {
+ /*
+ * Pin @parent for @sch's lifetime. The kobject hierarchy pins
+ * it only via @parent->sub_kset, which is dropped during
+ * disable. Released in scx_sched_free_rcu_work().
+ */
+ kobject_get(&parent->kobj);
ret = kobject_init_and_add(&sch->kobj, &scx_ktype,
&parent->sub_kset->kobj,
"sub-%llu", cgroup_id(cgrp));
- else
+ } else {
ret = kobject_init_and_add(&sch->kobj, &scx_ktype, NULL, "root");
+ }
if (ret < 0) {
RCU_INIT_POINTER(ops->priv, NULL);
@@ -6895,7 +6973,6 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
#ifdef CONFIG_EXT_SUB_SCHED
err_free_lb_resched:
- RCU_INIT_POINTER(ops->priv, NULL);
free_cpumask_var(sch->bypass_lb_resched_cpumask);
#endif
err_free_lb_cpumask:
@@ -6988,7 +7065,7 @@ static int validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops)
* run past the BPF allocation. Skip for cid-form.
*/
if (!sch->is_cid_type && (ops->cpu_acquire || ops->cpu_release))
- pr_warn("ops->cpu_acquire/release() are deprecated, use sched_switch TP instead\n");
+ pr_warn_ratelimited("ops->cpu_acquire/release() are deprecated, use sched_switch TP instead\n");
/*
* Sub-scheduler support is tied to the cid-form struct_ops. A sub-sched
@@ -7686,6 +7763,12 @@ err_unlock_and_disable:
percpu_up_write(&scx_fork_rwsem);
err_disable:
mutex_unlock(&scx_enable_mutex);
+ /*
+ * Some enable failures only return an errno (e.g. -ENOMEM from an
+ * allocation) without calling scx_error(). Record it so
+ * scx_flush_disable_work() runs the disable and ops.exit() fires.
+ */
+ scx_error(sch, "scx_sub_enable() failed (%d)", ret);
scx_flush_disable_work(sch);
cmd->ret = 0;
}
@@ -7806,7 +7889,7 @@ static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log,
off + size <= offsetofend(struct task_struct, scx.slice)) ||
(off >= offsetof(struct task_struct, scx.dsq_vtime) &&
off + size <= offsetofend(struct task_struct, scx.dsq_vtime))) {
- pr_warn("sched_ext: Writing directly to p->scx.slice/dsq_vtime is deprecated, use scx_bpf_task_set_slice/dsq_vtime()");
+ pr_warn_ratelimited("sched_ext: Writing directly to p->scx.slice/dsq_vtime is deprecated, use scx_bpf_task_set_slice/dsq_vtime()\n");
return SCALAR_VALUE;
}
@@ -8796,10 +8879,8 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
in_balance = this_rq->scx.flags & SCX_RQ_IN_BALANCE;
if (in_balance) {
- if (this_rq != src_rq) {
- raw_spin_rq_unlock(this_rq);
- raw_spin_rq_lock(src_rq);
- }
+ if (this_rq != src_rq)
+ switch_rq_lock(this_rq, src_rq);
} else {
raw_spin_rq_lock(src_rq);
}
@@ -8831,10 +8912,8 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
dispatched = true;
out:
if (in_balance) {
- if (this_rq != locked_rq) {
- raw_spin_rq_unlock(locked_rq);
- raw_spin_rq_lock(this_rq);
- }
+ if (this_rq != locked_rq)
+ switch_rq_lock(locked_rq, this_rq);
} else {
raw_spin_rq_unlock_irqrestore(locked_rq, flags);
}
diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h
index 145272cb4d8a..673059fa9d72 100644
--- a/kernel/sched/ext/internal.h
+++ b/kernel/sched/ext/internal.h
@@ -1469,21 +1469,24 @@ static const char *scx_enable_state_str[] = {
* The sched_ext core uses a "lock dancing" protocol coordinated by
* p->scx.holding_cpu. When moving a task to a different rq:
*
- * 1. Verify task can be moved (CPU affinity, migration_disabled, etc.)
- * 2. Set p->scx.holding_cpu to the current CPU
- * 3. Set task state to %SCX_OPSS_NONE; dequeue waits while DISPATCHING
+ * 1. Set p->scx.holding_cpu to the current CPU
+ * 2. Set task state to %SCX_OPSS_NONE; dequeue waits while DISPATCHING
* is set, so clearing DISPATCHING first prevents the circular wait
* (safe to lock the rq we need)
- * 4. Unlock the current CPU's rq
- * 5. Lock src_rq (where the task currently lives)
- * 6. Verify p->scx.holding_cpu == current CPU, if not, dequeue won the
+ * 3. Unlock the current CPU's rq
+ * 4. Lock src_rq (where the task currently lives)
+ * 5. Verify p->scx.holding_cpu == current CPU, if not, dequeue won the
* race (dequeue clears holding_cpu to -1 when it takes the task), in
* this case migration is aborted
- * 7. If src_rq == dst_rq: clear holding_cpu and enqueue directly
+ * 6. If src_rq == dst_rq: clear holding_cpu and enqueue directly
* into dst_rq's local DSQ (no lock swap needed)
- * 8. Otherwise: call move_remote_task_to_local_dsq(), which releases
- * src_rq, locks dst_rq, and performs the deactivate/activate
- * migration cycle (dst_rq is held on return)
+ * 7. Otherwise, verify under src_rq lock that the task can be moved to dst_rq
+ * (CPU affinity, migration_disabled, etc.). If not, clear holding_cpu,
+ * leave the task on src_rq, and enqueue it on the fallback DSQ.
+ * 8. Otherwise (i.e. if the task can be moved to dst_rq), call
+ * move_remote_task_to_local_dsq(), which releases src_rq, locks dst_rq,
+ * and performs the deactivate/activate migration cycle
+ * (dst_rq is held on return)
* 9. Unlock dst_rq and re-lock the current CPU's rq to restore
* the lock state expected by the caller
*
diff --git a/kernel/signal.c b/kernel/signal.c
index 9c2b32c4d755..bbc0fd4cc4d7 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -1362,8 +1362,16 @@ struct sighand_struct *lock_task_sighand(struct task_struct *tsk,
rcu_read_lock();
for (;;) {
sighand = rcu_dereference(tsk->sighand);
- if (unlikely(sighand == NULL))
+ if (unlikely(sighand == NULL)) {
+ /*
+ * Pairs with the smp_store_release() in
+ * __exit_signal(). It ensures that all state
+ * modifications to the task preceeding the store are
+ * visible to the callers of lock_task_sighand().
+ */
+ smp_acquire__after_ctrl_dep();
break;
+ }
/*
* This sighand can be already freed and even reused, but
diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c
index 5e633d8750d1..a7d3e8229c4b 100644
--- a/kernel/time/posix-cpu-timers.c
+++ b/kernel/time/posix-cpu-timers.c
@@ -461,6 +461,109 @@ static void disarm_timer(struct k_itimer *timer, struct task_struct *p)
trigger_base_recalc_expires(timer, p);
}
+/*
+ * Lookup the task via timer->it.cpu.pid and attempt to lock the task's sighand.
+ *
+ * This can race with the reaping of the task:
+ *
+ * CPU0 CPU1
+ *
+ * // Finds task
+ * p = pid_task(pid, pid_type); __exit_signal(p)
+ * lock(p, sighand);
+ * posix_cpu_timers*_exit();
+ * sighand = lock_task_sighand(p); unhash_task(p);
+ * p->sighand = NULL;
+ * unlock(sighand);
+ *
+ * In this case sighand is NULL, which means the task and the associated timer
+ * queue cannot be longer accessed safely.
+ *
+ * __exit_signal() invokes posix_cpu_timers_exit() and if the thread group is
+ * dead it also invokes posix_cpu_timers_group_exit(). These functions delete
+ * all pending timers from the related timer queues. The POSIX timers (k_itimer)
+ * themself are still accessible, but not longer connected to the task.
+ *
+ * exec() works slightly differently. The task which exec()'s terminates all
+ * other threads in the thread group and runs __exit_signal() on them. As the
+ * thread group is not dead they only clean up the per task timers via
+ * posix_cpu_timers_exit().
+ *
+ * As the TGID on exec() stays the same per process timers stay queued, if they
+ * are armed. This works without a problem when exec() is done by the thread
+ * group leader. If a non-leader thread exec()'s this can end up in the
+ * following scenario:
+ *
+ * CPU0 CPU1
+ * // Returns old leader
+ * p = pid_task(pid, pid_type); de_thread()
+ * switch_leader()
+ * release_task(old leader)
+ * __exit_signal()
+ * old_leader->sighand = NULL;
+ * // Returns NULL
+ * sighand = lock_task_sighand(p)
+ *
+ * That's problematic for several functions:
+ *
+ * - posix_cpu_timer_del(): If the timer is still enqueued on the task the
+ * underlying k_itimer will be freed which results in a UAF in
+ * run_posix_cpu_timers() or on timerqueue related add/delete operations.
+ * If the timer is not enqueued, the failure is harmless
+ *
+ * - posix_cpu_timer_set(): Independent of the enqueued state that results in a
+ * transient failure which is user space visible (-ESRCH) for regular posix
+ * timers. But for the use case in do_cpu_nanosleep() it's the same UAF
+ * problem just that the timer is allocated on the stack.
+ *
+ * - posix_cpu_timer_rearm(): Timer is not enqueued at that point, but this
+ * silently ignores the rearm request, which is a functional problem as the
+ * timer wont expire anymore.
+ */
+static struct task_struct *timer_lock_sighand(struct k_itimer *timer, unsigned long *flags)
+{
+ enum pid_type type = clock_pid_type(timer->it_clock);
+ struct cpu_timer *ctmr = &timer->it.cpu;
+
+ guard(rcu)();
+
+ for (;;) {
+ struct task_struct *t = pid_task(timer->it.cpu.pid, type);
+
+ /* Fail if the task cannot be found. */
+ if (!t)
+ break;
+
+ /* Try to lock the task's sighand */
+ if (lock_task_sighand(t, flags))
+ return t;
+
+ /*
+ * The next PID lookup might either fail or return the new
+ * leader. This is correct for both exit() and exec().
+ */
+ }
+
+ /*
+ * If the timer is still enqueued, warn. There is nothing safe to do
+ * here as there might be two timers in there which are removed in
+ * parallel and that will cause more damage than good. This should never
+ * happen!
+ *
+ * Ensure that the stores to the timer and timerqueue are visible:
+ *
+ * __exit_signal()
+ * posix_cpu_timers*_exit()
+ * write_seqlock(seqlock)
+ * smp_wmb(); <-------
+ * __unhash_process() | !pid_task()
+ * ----> smp_rmb();
+ * WARN_ON_ONCE(...)
+ */
+ smp_rmb();
+ WARN_ON_ONCE(ctmr->head || timerqueue_node_queued(&ctmr->node));
+ return NULL;
+}
/*
* Clean up a CPU-clock timer that is about to be destroyed.
@@ -470,29 +573,13 @@ static void disarm_timer(struct k_itimer *timer, struct task_struct *p)
*/
static int posix_cpu_timer_del(struct k_itimer *timer)
{
- struct cpu_timer *ctmr = &timer->it.cpu;
- struct sighand_struct *sighand;
struct task_struct *p;
unsigned long flags;
int ret = 0;
- rcu_read_lock();
- p = cpu_timer_task_rcu(timer);
- if (!p)
- goto out;
+ p = timer_lock_sighand(timer, &flags);
- /*
- * Protect against sighand release/switch in exit/exec and process/
- * thread timer list entry concurrent read/writes.
- */
- sighand = lock_task_sighand(p, &flags);
- if (unlikely(sighand == NULL)) {
- /*
- * This raced with the reaping of the task. The exit cleanup
- * should have removed this timer from the timer queue.
- */
- WARN_ON_ONCE(ctmr->head || timerqueue_node_queued(&ctmr->node));
- } else {
+ if (likely(p)) {
if (timer->it.cpu.firing) {
/*
* Prevent signal delivery. The timer cannot be dequeued
@@ -508,11 +595,8 @@ static int posix_cpu_timer_del(struct k_itimer *timer)
unlock_task_sighand(p, &flags);
}
-out:
- rcu_read_unlock();
-
if (!ret) {
- put_pid(ctmr->pid);
+ put_pid(timer->it.cpu.pid);
timer->it_status = POSIX_TIMER_DISARMED;
}
return ret;
@@ -626,21 +710,17 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags,
clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);
struct cpu_timer *ctmr = &timer->it.cpu;
u64 old_expires, new_expires, now;
- struct sighand_struct *sighand;
struct task_struct *p;
unsigned long flags;
int ret = 0;
- rcu_read_lock();
- p = cpu_timer_task_rcu(timer);
- if (!p) {
- /*
- * If p has just been reaped, we can no
- * longer get any information about it at all.
- */
- rcu_read_unlock();
+ p = timer_lock_sighand(timer, &flags);
+ /*
+ * If p has just been reaped, we can no longer get any information about
+ * it at all.
+ */
+ if (!p)
return -ESRCH;
- }
/*
* Use the to_ktime conversion because that clamps the maximum
@@ -648,20 +728,6 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags,
*/
new_expires = ktime_to_ns(timespec64_to_ktime(new->it_value));
- /*
- * Protect against sighand release/switch in exit/exec and p->cpu_timers
- * and p->signal->cpu_timers read/write in arm_timer()
- */
- sighand = lock_task_sighand(p, &flags);
- /*
- * If p has just been reaped, we can no
- * longer get any information about it at all.
- */
- if (unlikely(sighand == NULL)) {
- rcu_read_unlock();
- return -ESRCH;
- }
-
/* Retrieve the current expiry time before disarming the timer */
old_expires = cpu_timer_getexpires(ctmr);
@@ -698,7 +764,7 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags,
/* Retry if the timer expiry is running concurrently */
if (unlikely(ret)) {
unlock_task_sighand(p, &flags);
- goto out;
+ return ret;
}
/* Convert relative expiry time to absolute */
@@ -733,8 +799,6 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags,
*/
if (!sigev_none && new_expires && now >= new_expires)
cpu_timer_fire(timer);
-out:
- rcu_read_unlock();
return ret;
}
@@ -1018,19 +1082,12 @@ static void check_process_timers(struct task_struct *tsk,
static bool posix_cpu_timer_rearm(struct k_itimer *timer)
{
clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);
- struct sighand_struct *sighand;
struct task_struct *p;
unsigned long flags;
u64 now;
- guard(rcu)();
- p = cpu_timer_task_rcu(timer);
- if (!p)
- return true;
-
- /* Protect timer list r/w in arm_timer() */
- sighand = lock_task_sighand(p, &flags);
- if (unlikely(sighand == NULL))
+ p = timer_lock_sighand(timer, &flags);
+ if (unlikely(!p))
return true;
/*
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index 56a328e94395..804ccae694d2 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -270,7 +270,8 @@ unsigned ring_buffer_event_length(struct ring_buffer_event *event)
if (event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX)
return length;
length -= RB_EVNT_HDR_SIZE;
- if (length > RB_MAX_SMALL_DATA + sizeof(event->array[0]))
+ if (length > RB_MAX_SMALL_DATA + sizeof(event->array[0]) ||
+ RB_FORCE_8BYTE_ALIGNMENT)
length -= sizeof(event->array[0]);
return length;
}
@@ -2329,10 +2330,7 @@ static struct ring_buffer_desc *ring_buffer_desc(struct trace_buffer_desc *trace
size_t len;
int i;
- if (!trace_desc)
- return NULL;
-
- if (cpu >= trace_desc->nr_cpus)
+ if (!trace_desc || !trace_desc->nr_cpus)
return NULL;
end = (struct ring_buffer_desc *)((void *)trace_desc + trace_desc->struct_len);
@@ -7174,7 +7172,7 @@ int ring_buffer_read_page(struct trace_buffer *buffer,
rpos = reader->read;
pos += event_size;
- if (rpos >= event_size)
+ if (rpos >= size)
break;
event = rb_reader_event(cpu_buffer);
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 1146b83b711a..18710c190c92 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -87,7 +87,7 @@ void __init disable_tracing_selftest(const char *reason)
/* Pipe tracepoints to printk */
static struct trace_iterator *tracepoint_print_iter;
-int tracepoint_printk;
+static int tracepoint_printk;
static bool tracepoint_printk_stop_on_boot __initdata;
static bool traceoff_after_boot __initdata;
static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key);
@@ -5015,7 +5015,6 @@ int tracing_set_tracer(struct trace_array *tr, const char *buf)
RING_BUFFER_ALL_CPUS);
if (ret < 0)
return ret;
- ret = 0;
}
list_for_each_entry(t, &tr->tracers, list) {
diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c
index 609325f57942..6385cd662d8d 100644
--- a/kernel/trace/trace_events_filter.c
+++ b/kernel/trace/trace_events_filter.c
@@ -1056,11 +1056,9 @@ static int regex_match_end(char *str, struct regex *r, int len)
return 0;
}
-static int regex_match_glob(char *str, struct regex *r, int len __maybe_unused)
+static int regex_match_glob(char *str, struct regex *r, int len)
{
- if (glob_match(r->pattern, str))
- return 1;
- return 0;
+ return glob_match_len(r->pattern, str, len) ? 1 : 0;
}
/**
diff --git a/kernel/trace/trace_events_synth.c b/kernel/trace/trace_events_synth.c
index e6871230bde9..dc15658a887c 100644
--- a/kernel/trace/trace_events_synth.c
+++ b/kernel/trace/trace_events_synth.c
@@ -839,8 +839,10 @@ static struct synth_field *parse_synth_field(int argc, char **argv,
seq_buf_puts(&s, "__data_loc ");
seq_buf_puts(&s, field->type);
- if (WARN_ON_ONCE(!seq_buf_buffer_left(&s)))
+ if (WARN_ON_ONCE(!seq_buf_buffer_left(&s))) {
+ kfree(type);
goto free;
+ }
s.buffer[s.len] = '\0';
kfree(field->type);
@@ -1446,13 +1448,13 @@ static int __create_synth_event(const char *name, const char *raw_fields)
if (cmd_version > 1 && n_fields_this_loop >= 1) {
synth_err(SYNTH_ERR_INVALID_CMD, errpos(field_str));
ret = -EINVAL;
- goto err_free_arg;
+ goto err_free_field;
}
if (n_fields == SYNTH_FIELDS_MAX) {
synth_err(SYNTH_ERR_TOO_MANY_FIELDS, 0);
ret = -EINVAL;
- goto err_free_arg;
+ goto err_free_field;
}
fields[n_fields++] = field;
@@ -1491,6 +1493,8 @@ static int __create_synth_event(const char *name, const char *raw_fields)
kfree(saved_fields);
return ret;
+ err_free_field:
+ free_synth_field(field);
err_free_arg:
argv_free(argv);
err:
diff --git a/kernel/trace/trace_events_user.c b/kernel/trace/trace_events_user.c
index c4ba484f7b38..8c82ecb735f4 100644
--- a/kernel/trace/trace_events_user.c
+++ b/kernel/trace/trace_events_user.c
@@ -109,6 +109,9 @@ struct user_event_enabler {
/* Track enable bit, flags, etc. Aligned for bitops. */
unsigned long values;
+
+ /* Defer the event put and enabler free past an RCU grace period. */
+ struct rcu_work put_rwork;
};
/* Bits 0-5 are for the bit to update upon enable/disable (0-63 allowed) */
@@ -396,17 +399,39 @@ error:
return NULL;
};
-static void user_event_enabler_destroy(struct user_event_enabler *enabler,
- bool locked)
+static void delayed_user_event_enabler_put(struct work_struct *work)
{
- list_del_rcu(&enabler->mm_enablers_link);
+ struct user_event_enabler *enabler = container_of(to_rcu_work(work),
+ struct user_event_enabler, put_rwork);
/* No longer tracking the event via the enabler */
- user_event_put(enabler->event, locked);
+ user_event_put(enabler->event, false);
+ /* Run from queue_rcu_work(), the RCU grace period has elapsed */
kfree(enabler);
}
+static void user_event_enabler_destroy(struct user_event_enabler *enabler)
+{
+ list_del_rcu(&enabler->mm_enablers_link);
+
+ /*
+ * The enabler is removed from an RCU-traversed list
+ * (user_event_mm_dup() walks mm->enablers under rcu_read_lock() only),
+ * and readers there dereference enabler->event and take a new ref on
+ * it. Both the put of that event reference and the free of the enabler
+ * therefore have to wait for a grace period so no reader can be looking
+ * at the enabler or racing the last put of its event.
+ *
+ * The put itself must not run in RCU context: when it drops the last
+ * reference user_event_put() takes event_mutex, which cannot be taken
+ * from a softirq/RCU callback. Defer both to a work item scheduled
+ * after a grace period via queue_rcu_work().
+ */
+ INIT_RCU_WORK(&enabler->put_rwork, delayed_user_event_enabler_put);
+ queue_rcu_work(system_percpu_wq, &enabler->put_rwork);
+}
+
static int user_event_mm_fault_in(struct user_event_mm *mm, unsigned long uaddr,
int attempt)
{
@@ -464,7 +489,7 @@ static void user_event_enabler_fault_fixup(struct work_struct *work)
/* User asked for enabler to be removed during fault */
if (test_bit(ENABLE_VAL_FREEING_BIT, ENABLE_BITOPS(enabler))) {
- user_event_enabler_destroy(enabler, true);
+ user_event_enabler_destroy(enabler);
goto out;
}
@@ -764,7 +789,7 @@ static void user_event_mm_destroy(struct user_event_mm *mm)
struct user_event_enabler *enabler, *next;
list_for_each_entry_safe(enabler, next, &mm->enablers, mm_enablers_link)
- user_event_enabler_destroy(enabler, false);
+ user_event_enabler_destroy(enabler);
mmdrop(mm->mm);
kfree(mm);
@@ -2645,7 +2670,7 @@ static long user_events_ioctl_unreg(unsigned long uarg)
flags |= enabler->values & ENABLE_VAL_COMPAT_MASK;
if (!test_bit(ENABLE_VAL_FAULTING_BIT, ENABLE_BITOPS(enabler)))
- user_event_enabler_destroy(enabler, true);
+ user_event_enabler_destroy(enabler);
/* Removed at least one */
ret = 0;
diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c
index f283391a4dc8..cd37f2013758 100644
--- a/kernel/trace/trace_functions.c
+++ b/kernel/trace/trace_functions.c
@@ -458,12 +458,12 @@ func_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
ftrace_func_t func;
u32 new_flags;
- /* Do nothing if already set. */
- if (!!set == !!(tr->current_trace_flags->val & bit))
+ /* We can change this flag only when current tracer is function. */
+ if (tr->current_trace != &function_trace)
return 0;
- /* We can change this flag only when not running. */
- if (tr->current_trace != &function_trace)
+ /* Do nothing if already set. */
+ if (!!set == !!(tr->current_trace_flags->val & bit))
return 0;
new_flags = (tr->current_trace_flags->val & ~bit) | (set ? bit : 0);
diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c
index 5e83c4f6f2b4..0e1265acd1cc 100644
--- a/kernel/trace/trace_osnoise.c
+++ b/kernel/trace/trace_osnoise.c
@@ -179,7 +179,9 @@ static void osnoise_unregister_instance(struct trace_array *tr)
if (!found)
return;
- kvfree_rcu_mightsleep(inst);
+ /* Do a full sync to ensure that tr remains valid, not just inst */
+ synchronize_rcu();
+ kvfree(inst);
}
/*
diff --git a/kernel/trace/trace_preemptirq.c b/kernel/trace/trace_preemptirq.c
index 0c42b15c3800..b63e3558948f 100644
--- a/kernel/trace/trace_preemptirq.c
+++ b/kernel/trace/trace_preemptirq.c
@@ -30,7 +30,7 @@
#else
#define trace(point, args) \
do { \
- if (trace_##point##_enabled()) { \
+ if (__trace_##point##_enabled()) { \
bool exit_rcu = false; \
if (in_nmi()) \
break; \
diff --git a/kernel/trace/trace_remote.c b/kernel/trace/trace_remote.c
index 2a6cc000ec98..0f6ef5c36d84 100644
--- a/kernel/trace/trace_remote.c
+++ b/kernel/trace/trace_remote.c
@@ -979,33 +979,30 @@ EXPORT_SYMBOL_GPL(trace_remote_free_buffer);
int trace_remote_alloc_buffer(struct trace_buffer_desc *desc, size_t desc_size, size_t buffer_size,
const struct cpumask *cpumask)
{
+ size_t min_desc_size = trace_buffer_desc_size(buffer_size, cpumask_weight(cpumask));
unsigned int nr_pages = max(DIV_ROUND_UP(buffer_size, PAGE_SIZE), 2UL) + 1;
- void *desc_end = desc + desc_size;
struct ring_buffer_desc *rb_desc;
int cpu, ret = -ENOMEM;
- if (desc_size < struct_size(desc, __data, 0))
+ if (desc_size < min_desc_size)
return -EINVAL;
desc->nr_cpus = 0;
- desc->struct_len = struct_size(desc, __data, 0);
+ desc->struct_len = min_desc_size;
- rb_desc = (struct ring_buffer_desc *)&desc->__data[0];
+ rb_desc = __first_ring_buffer_desc(desc);
for_each_cpu(cpu, cpumask) {
unsigned int id;
- if ((void *)rb_desc + struct_size(rb_desc, page_va, nr_pages) > desc_end) {
- ret = -EINVAL;
- goto err;
- }
-
rb_desc->cpu = cpu;
rb_desc->nr_page_va = 0;
rb_desc->meta_va = (unsigned long)__get_free_page(GFP_KERNEL);
if (!rb_desc->meta_va)
goto err;
+ desc->nr_cpus++;
+
for (id = 0; id < nr_pages; id++) {
rb_desc->page_va[id] = (unsigned long)__get_free_page(GFP_KERNEL);
if (!rb_desc->page_va[id])
@@ -1013,9 +1010,6 @@ int trace_remote_alloc_buffer(struct trace_buffer_desc *desc, size_t desc_size,
rb_desc->nr_page_va++;
}
- desc->nr_cpus++;
- desc->struct_len += offsetof(struct ring_buffer_desc, page_va);
- desc->struct_len += struct_size(rb_desc, page_va, rb_desc->nr_page_va);
rb_desc = __next_ring_buffer_desc(rb_desc);
}