summaryrefslogtreecommitdiff
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/audit.c11
-rw-r--r--kernel/auditfilter.c6
-rw-r--r--kernel/bpf/rqspinlock.c5
-rw-r--r--kernel/bpf/verifier.c45
-rw-r--r--kernel/cgroup/cgroup.c23
-rw-r--r--kernel/events/uprobes.c2
-rw-r--r--kernel/futex/core.c92
-rw-r--r--kernel/liveupdate/kexec_handover.c22
-rw-r--r--kernel/sched/cpufreq_schedutil.c11
-rw-r--r--kernel/sched/deadline.c3
-rw-r--r--kernel/sched/ext/ext.c47
-rw-r--r--kernel/sched/ext/idle.c4
-rw-r--r--kernel/sched/psi.c75
-rw-r--r--kernel/trace/fprobe.c6
-rw-r--r--kernel/trace/ftrace.c33
-rw-r--r--kernel/trace/ring_buffer.c17
-rw-r--r--kernel/trace/trace.c2
-rw-r--r--kernel/trace/trace_events.c4
-rw-r--r--kernel/trace/trace_events_filter.c3
-rw-r--r--kernel/trace/trace_fprobe.c12
-rw-r--r--kernel/trace/trace_mmiotrace.c13
-rw-r--r--kernel/trace/trace_probe.c6
22 files changed, 321 insertions, 121 deletions
diff --git a/kernel/audit.c b/kernel/audit.c
index 562476937fa7..9412af9144bc 100644
--- a/kernel/audit.c
+++ b/kernel/audit.c
@@ -2120,7 +2120,8 @@ void audit_log_n_hex(struct audit_buffer *ab, const unsigned char *buf,
void audit_log_n_string(struct audit_buffer *ab, const char *string,
size_t slen)
{
- int avail, new_len;
+ int avail;
+ size_t new_len;
unsigned char *ptr;
struct sk_buff *skb;
@@ -2130,7 +2131,13 @@ void audit_log_n_string(struct audit_buffer *ab, const char *string,
BUG_ON(!ab->skb);
skb = ab->skb;
avail = skb_tailroom(skb);
- new_len = slen + 3; /* enclosing quotes + null terminator */
+
+ /* enclosing quotes + null terminator */
+ if (check_add_overflow(slen, 3, &new_len)) {
+ audit_log_format(ab, "?");
+ return;
+ }
+
if (new_len > avail) {
avail = audit_expand(ab, new_len);
if (!avail)
diff --git a/kernel/auditfilter.c b/kernel/auditfilter.c
index 4401119b5275..7f791afe5791 100644
--- a/kernel/auditfilter.c
+++ b/kernel/auditfilter.c
@@ -1045,6 +1045,10 @@ int audit_del_rule(struct audit_entry *entry)
goto out;
}
+ list_del_rcu(&e->list);
+ list_del(&e->rule.list);
+ synchronize_rcu();
+
if (e->rule.watch)
audit_remove_watch_rule(&e->rule);
@@ -1062,8 +1066,6 @@ int audit_del_rule(struct audit_entry *entry)
audit_signals--;
#endif
- list_del_rcu(&e->list);
- list_del(&e->rule.list);
call_rcu(&e->rcu, audit_free_rule_rcu);
out:
diff --git a/kernel/bpf/rqspinlock.c b/kernel/bpf/rqspinlock.c
index e4e338cdb437..2129defc4a9a 100644
--- a/kernel/bpf/rqspinlock.c
+++ b/kernel/bpf/rqspinlock.c
@@ -572,9 +572,10 @@ queue:
/* Disable queue destruction when we detect deadlocks. */
if (ret == -EDEADLK) {
- if (!next)
+ if (!try_cmpxchg_tail(lock, tail, 0)) {
next = smp_cond_load_relaxed(&node->next, (VAL));
- arch_mcs_spin_unlock_contended(&next->locked);
+ arch_mcs_spin_unlock_contended(&next->locked);
+ }
goto err_release_node;
}
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 7aa47342dc65..fdc5fbb1f78c 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -13557,23 +13557,21 @@ static void sanitize_mark_insn_seen(struct bpf_verifier_env *env)
env->insn_aux_data[env->insn_idx].seen = env->pass_cnt;
}
-static int sanitize_err(struct bpf_verifier_env *env,
- const struct bpf_insn *insn, int reason,
- const struct bpf_reg_state *off_reg,
- const struct bpf_reg_state *dst_reg)
+static int sanitize_err(struct bpf_verifier_env *env, const struct bpf_insn *insn, int reason)
{
static const char *err = "pointer arithmetic with it prohibited for !root";
const char *op = BPF_OP(insn->code) == BPF_ADD ? "add" : "sub";
u32 dst = insn->dst_reg, src = insn->src_reg;
+ struct bpf_reg_state *regs = cur_regs(env);
switch (reason) {
case REASON_BOUNDS:
verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n",
- off_reg == dst_reg ? dst : src, err);
+ regs[src].type == SCALAR_VALUE ? src : dst, err);
break;
case REASON_TYPE:
verbose(env, "R%d has pointer with unsupported alu operation, %s\n",
- off_reg == dst_reg ? src : dst, err);
+ regs[src].type == SCALAR_VALUE ? dst : src, err);
break;
case REASON_PATHS:
verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n",
@@ -13709,13 +13707,6 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
return -EACCES;
}
- /*
- * Accesses to untrusted PTR_TO_MEM are done through probe
- * instructions, hence no need to track offsets.
- */
- if (base_type(ptr_reg->type) == PTR_TO_MEM && (ptr_reg->type & PTR_UNTRUSTED))
- return 0;
-
switch (base_type(ptr_reg->type)) {
case PTR_TO_CTX:
case PTR_TO_MAP_VALUE:
@@ -13745,11 +13736,19 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
return -EACCES;
}
- /* In case of 'scalar += pointer', dst_reg inherits pointer type and id.
- * The id may be overwritten later if we create a new variable offset.
+ /* For 'scalar += pointer', dst_reg inherits the complete pointer
+ * register state. Individual fields may be adjusted later by pointer
+ * arithmetic. Callers guarantee that below does not overwrite off_reg.
*/
- dst_reg->type = ptr_reg->type;
- dst_reg->id = ptr_reg->id;
+ if (dst_reg != ptr_reg)
+ *dst_reg = *ptr_reg;
+
+ /*
+ * Accesses to untrusted PTR_TO_MEM are done through probe
+ * instructions, hence no need to track offsets.
+ */
+ if (base_type(ptr_reg->type) == PTR_TO_MEM && (ptr_reg->type & PTR_UNTRUSTED))
+ return 0;
if (!check_reg_sane_offset_scalar(env, off_reg, ptr_reg->type) ||
!check_reg_sane_offset_ptr(env, ptr_reg, ptr_reg->type))
@@ -13762,7 +13761,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg,
&info, false);
if (ret < 0)
- return sanitize_err(env, insn, ret, off_reg, dst_reg);
+ return sanitize_err(env, insn, ret);
}
switch (opcode) {
@@ -13792,7 +13791,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
}
break;
case BPF_SUB:
- if (dst_reg == off_reg) {
+ if (dst_reg != ptr_reg) {
/* scalar -= pointer. Creates an unknown scalar */
verbose(env, "R%d tried to subtract pointer from scalar\n",
dst);
@@ -13855,7 +13854,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
return -EFAULT;
}
if (ret < 0)
- return sanitize_err(env, insn, ret, off_reg, dst_reg);
+ return sanitize_err(env, insn, ret);
}
return 0;
@@ -14607,7 +14606,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env,
if (sanitize_needed(opcode)) {
ret = sanitize_val_alu(env, insn);
if (ret < 0)
- return sanitize_err(env, insn, ret, NULL, NULL);
+ return sanitize_err(env, insn, ret);
}
/* Calculate sign/unsigned bounds and tnum for alu32 and alu64 bit ops.
@@ -14810,8 +14809,8 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,
err = mark_chain_precision(env, insn->dst_reg);
if (err)
return err;
- return adjust_ptr_min_max_vals(env, insn,
- src_reg, dst_reg);
+ off_reg = *dst_reg;
+ return adjust_ptr_min_max_vals(env, insn, src_reg, &off_reg);
}
} else if (ptr_reg) {
/* pointer += scalar */
diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index 38f8d9df8fbc..b5b461d4418b 100644
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -3996,6 +3996,7 @@ static ssize_t pressure_write(struct kernfs_open_file *of, char *buf,
struct psi_trigger *new;
struct cgroup *cgrp;
struct psi_group *psi;
+ bool need_rtpoll_worker;
ssize_t ret = 0;
cgrp = cgroup_kn_lock_live(of->kn, false);
@@ -4015,12 +4016,32 @@ static ssize_t pressure_write(struct kernfs_open_file *of, char *buf,
}
psi = cgroup_psi(cgrp);
- new = psi_trigger_create(psi, buf, res, of->file, of);
+ new = psi_trigger_create(psi, buf, res, of->file, of,
+ &need_rtpoll_worker);
if (IS_ERR(new)) {
ret = PTR_ERR(new);
goto out_unlock;
}
+ /*
+ * The worker fork must run with neither cgroup_mutex nor the file's
+ * kernfs active reference held. The latter is broken since
+ * cgroup_kn_lock_live(). @of->priv may be released while unlocked, so
+ * recheck before publishing @new.
+ */
+ if (need_rtpoll_worker) {
+ cgroup_unlock();
+ ret = psi_trigger_create_rtpoll_worker(psi);
+ cgroup_lock();
+
+ if (!ret && !of->priv)
+ ret = -ENODEV;
+ if (ret) {
+ psi_trigger_destroy(new);
+ goto out_unlock;
+ }
+ }
+
smp_store_release(&ctx->psi.trigger, new);
out_unlock:
diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index 4084e926e284..6300b216012c 100644
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -830,7 +830,7 @@ static struct uprobe *hprobe_expire(struct hprobe *hprobe, bool get)
if (try_cmpxchg(&hprobe->state, &hstate, uprobe ? HPROBE_STABLE : HPROBE_GONE)) {
/* We won the race, we are the ones to unlock SRCU */
__srcu_read_unlock(&uretprobes_srcu, hprobe->srcu_idx);
- return get ? get_uprobe(uprobe) : uprobe;
+ return get && uprobe ? get_uprobe(uprobe) : uprobe;
}
/*
diff --git a/kernel/futex/core.c b/kernel/futex/core.c
index 179b26e9c934..128c5752f225 100644
--- a/kernel/futex/core.c
+++ b/kernel/futex/core.c
@@ -982,8 +982,11 @@ retry:
return -1;
/*
- * Special case for regular (non PI) futexes. The unlock path in
- * user space has two race scenarios:
+ * Special case for regular (non PI) futexes. Ordinarily, we do
+ * not perform any processing here unless the current thread was
+ * the owner of the futex (by the TID check below).
+ *
+ * However, the unlock path has three race scenarios:
*
* 1. The unlock path releases the user space futex value and
* before it can execute the futex() syscall to wake up
@@ -992,42 +995,70 @@ retry:
* 2. A woken up waiter is killed before it can acquire the
* futex in user space.
*
- * In the second case, the wake up notification could be generated
- * by the unlock path in user space after setting the futex value
- * to zero or by the kernel after setting the OWNER_DIED bit below.
+ * 3. A woken up waiter is killed in user space after another
+ * thread has acquired the futex, but before it can set
+ * FUTEX_WAITERS.
+ *
+ * Note that, if userspace uses the FUTEX_ROBUST_UNLOCK flag, we
+ * will not see case 1 here.
+ *
+ * In the second and third case, the wake up notification could
+ * be generated from any of:
+ *
+ * i. An ordinary futex wakeup after unlock (with or
+ * without FUTEX_ROBUST_UNLOCK)
+ * ii. A robust wakeup from another thread's death
+ * iii. A previous round through this special case
+ *
+ * As a result, the futex world will be in one of four states:
+ *
+ * A. The futex word is 0 (unlocked)
+ * B. The futex word is owned by another thread
+ * (FUTEX_WAITERS is not set)
+ * C. The futex word is owned by another thread
+ * (FUTEX_WAITERS set)
+ * D. The futex's owner died and OWNER_DIED is set
+ * (the owner part of the word is 0)
+ *
+ * The key issue is that the kernel usually (at least from
+ * sources ii. and iii. or when so requested by userspace from
+ * source i.) only ever wakes *one* waiter at a time. If this
+ * waiter dies before acquiring the futex (or setting the
+ * FUTEX_WAITERS bit), the kernel *must* still wake the next
+ * waiter down the line to uphold the futex invariants and
+ * avoid lost wakeups. Note we do not need to handle state C,
+ * as it does not matter to us whether *we* successfully set
+ * the bit or a third thread did so in the meantime.
*
- * In both cases the TID validation below prevents a wakeup of
- * potential waiters which can cause these waiters to block
- * forever.
+ * Therefore, in these cases we must issue an additional
+ * futex_wake(). Note however that we *must not* set OWNER_DIED
+ * here. Our thread is *not* the owner of the futex.
*
- * In both cases the following conditions are met:
+ * Thus to summarize, the conditions for needing the additional
+ * futex_wake() are:
*
- * 1) task->futex.robust_list->list_op_pending != NULL
- * @pending_op == true
- * 2) The owner part of user space futex value == 0
+ * 1) @pending_op == true (the thread has not finished the
+ * mutex operation)
+ * 2) The futex word is in one of the states A, B or D
* 3) Regular futex: @pi == false
*
- * If these conditions are met, it is safe to attempt waking up a
- * potential waiter without touching the user space futex value and
- * trying to set the OWNER_DIED bit. If the futex value is zero,
- * the rest of the user space mutex state is consistent, so a woken
- * waiter will just take over the uncontended futex. Setting the
- * OWNER_DIED bit would create inconsistent state and malfunction
- * of the user space owner died handling. Otherwise, the OWNER_DIED
- * bit is already set, and the woken waiter is expected to deal with
- * this.
+ * Note in particular that in all of the states A-D the owner
+ * portion of the futex word differs from our thread's TID
+ * (unless the actual owner has the same TID in another PID
+ * namespace, but we cannot currently distinguish that
+ * scenario), so this can be a special-case wakeup in the bail
+ * path of the ordinary TID check.
*/
owner = uval & FUTEX_TID_MASK;
- if (pending_op && !pi && !owner) {
- futex_wake(uaddr, FLAGS_SIZE_32 | FLAGS_SHARED, NULL, 1,
- FUTEX_BITSET_MATCH_ANY);
+ if (owner != task_pid_vnr(curr)) {
+ if (pending_op && !pi && (!owner || !(uval & FUTEX_WAITERS))) {
+ futex_wake(uaddr, FLAGS_SIZE_32 | FLAGS_SHARED, NULL, 1,
+ FUTEX_BITSET_MATCH_ANY);
+ }
return 0;
}
- if (owner != task_pid_vnr(curr))
- return 0;
-
/*
* Ok, this dying thread is truly holding a futex
* of interest. Set the OWNER_DIED bit atomically
@@ -1752,14 +1783,15 @@ void futex_hash_free(struct mm_struct *mm)
static bool futex_pivot_pending(struct mm_struct *mm)
{
+ struct futex_mm_phash *mmph = &mm->futex.phash;
struct futex_private_hash *fph;
- guard(rcu)();
+ guard(mutex)(&mmph->lock);
- if (!mm->futex.phash.hash_new)
+ if (!mmph->hash_new)
return true;
- fph = rcu_dereference(mm->futex.phash.hash);
+ fph = rcu_dereference_raw(mmph->hash);
return futex_ref_is_dead(fph);
}
diff --git a/kernel/liveupdate/kexec_handover.c b/kernel/liveupdate/kexec_handover.c
index 4834a809985a..175c08a6e41e 100644
--- a/kernel/liveupdate/kexec_handover.c
+++ b/kernel/liveupdate/kexec_handover.c
@@ -38,6 +38,16 @@
#include "../kexec_internal.h"
#include "kexec_handover_internal.h"
+/*
+ * This is the minimal alignment required by deferred struct page init.
+ * deferred_init_memmap_chunk frees memory to the buddy allocator, which looks
+ * at the neighboring pages (up to MAX_PAGE_ORDER) to merge them.
+ * If KHO scratch is not aligned to that value, buddy can access uninitialized
+ * struct pages, which can cause a crash.
+ */
+#define SCRATCH_ALIGNMENT_BYTES (PAGE_SIZE * MAX_ORDER_NR_PAGES)
+static_assert(SCRATCH_ALIGNMENT_BYTES >= CMA_MIN_ALIGNMENT_BYTES);
+
/* The magic token for preserved pages */
#define KHO_PAGE_MAGIC 0x4b484f50U /* ASCII for 'KHOP' */
@@ -640,8 +650,8 @@ static void __init scratch_size_update(void)
* Scratch areas are released as MIGRATE_CMA. Round them up to the right
* size.
*/
- scratch_size_lowmem = round_up(scratch_size_lowmem, CMA_MIN_ALIGNMENT_BYTES);
- scratch_size_global = round_up(scratch_size_global, CMA_MIN_ALIGNMENT_BYTES);
+ scratch_size_lowmem = round_up(scratch_size_lowmem, SCRATCH_ALIGNMENT_BYTES);
+ scratch_size_global = round_up(scratch_size_global, SCRATCH_ALIGNMENT_BYTES);
}
static phys_addr_t __init scratch_size_node(int nid)
@@ -656,7 +666,7 @@ static phys_addr_t __init scratch_size_node(int nid)
size = scratch_size_pernode;
}
- return round_up(size, CMA_MIN_ALIGNMENT_BYTES);
+ return round_up(size, SCRATCH_ALIGNMENT_BYTES);
}
/**
@@ -692,7 +702,7 @@ static void __init kho_reserve_scratch(void)
* next kernel
*/
size = scratch_size_lowmem;
- addr = memblock_phys_alloc_range(size, CMA_MIN_ALIGNMENT_BYTES, 0,
+ addr = memblock_phys_alloc_range(size, SCRATCH_ALIGNMENT_BYTES, 0,
ARCH_LOW_ADDRESS_LIMIT);
if (!addr) {
pr_err("Failed to reserve lowmem scratch buffer\n");
@@ -705,7 +715,7 @@ static void __init kho_reserve_scratch(void)
/* reserve large contiguous area for allocations without nid */
size = scratch_size_global;
- addr = memblock_phys_alloc(size, CMA_MIN_ALIGNMENT_BYTES);
+ addr = memblock_phys_alloc(size, SCRATCH_ALIGNMENT_BYTES);
if (!addr) {
pr_err("Failed to reserve global scratch buffer\n");
goto err_free_scratch_areas;
@@ -721,7 +731,7 @@ static void __init kho_reserve_scratch(void)
*/
for_each_node_state(nid, N_MEMORY) {
size = scratch_size_node(nid);
- addr = memblock_alloc_range_nid(size, CMA_MIN_ALIGNMENT_BYTES,
+ addr = memblock_alloc_range_nid(size, SCRATCH_ALIGNMENT_BYTES,
0, MEMBLOCK_ALLOC_ACCESSIBLE,
nid, true);
if (!addr) {
diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c
index a4e689eefdfb..dff4ee04694c 100644
--- a/kernel/sched/cpufreq_schedutil.c
+++ b/kernel/sched/cpufreq_schedutil.c
@@ -870,8 +870,19 @@ static int sugov_start(struct cpufreq_policy *policy)
memset(sg_cpu, 0, sizeof(*sg_cpu));
sg_cpu->cpu = cpu;
sg_cpu->sg_policy = sg_policy;
+ }
+
+ /*
+ * Publish the hooks only after all per-CPU data is initialized, so a
+ * shared policy's sugov_update_shared() never reads an uninitialized
+ * sibling sugov_cpu.
+ */
+ for_each_cpu(cpu, policy->cpus) {
+ struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu);
+
cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util, uu);
}
+
return 0;
}
diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index 0f858b98c9aa..200300043fa5 100644
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -1017,7 +1017,8 @@ static void update_dl_entity(struct sched_dl_entity *dl_se)
if (dl_time_before(dl_se->deadline, rq_clock(rq)) ||
dl_entity_overflow(dl_se, rq_clock(rq))) {
- if (unlikely((!dl_is_implicit(dl_se) || dl_se->dl_defer) &&
+ if (unlikely((!dl_is_implicit(dl_se) ||
+ (dl_se->dl_defer && dl_se->dl_defer_running)) &&
!dl_time_before(dl_se->deadline, rq_clock(rq)) &&
!is_dl_boosted(dl_se))) {
update_dl_revised_wakeup(dl_se, rq);
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index e3fa7b2fac9d..18183062f751 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -3606,6 +3606,9 @@ static int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fo
} else if (unlikely(fork)) {
scx_error(sch, "ops.init_task() set task->scx.disallow for %s[%d] during fork",
p->comm, p->pid);
+ } else if (unlikely(scx_enable_state() != SCX_ENABLING)) {
+ scx_error(sch, "ops.init_task() set task->scx.disallow for %s[%d] outside the enable path",
+ p->comm, p->pid);
} else {
struct rq *rq;
struct rq_flags rf;
@@ -4577,20 +4580,25 @@ static struct cgroup *root_cgroup(void)
return &cgrp_dfl_root.cgrp;
}
+/*
+ * cgroup_lock() must nest outside the rwsem write side: a writer waiting
+ * for cgroup_mutex deadlocks with cgroup teardown, which holds it while
+ * draining a set_* file write blocked on the rwsem behind the writer.
+ */
static void scx_cgroup_lock(void)
{
+ cgroup_lock();
#ifdef CONFIG_EXT_GROUP_SCHED
percpu_down_write(&scx_cgroup_ops_rwsem);
#endif
- cgroup_lock();
}
static void scx_cgroup_unlock(void)
{
- cgroup_unlock();
#ifdef CONFIG_EXT_GROUP_SCHED
percpu_up_write(&scx_cgroup_ops_rwsem);
#endif
+ cgroup_unlock();
}
#else /* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */
static inline struct cgroup *root_cgroup(void) { return NULL; }
@@ -5929,6 +5937,15 @@ static void scx_sub_disable(struct scx_sched *sch)
percpu_down_write(&scx_fork_rwsem);
scx_cgroup_lock();
+ /*
+ * An enable that failed before scx_link_sched() never owned a cgroup or
+ * task and won't be waited on by an ancestor's drain_descendants().
+ * Nothing to reparent and walking the tasks can misbehave as the task
+ * ownership invariant (either owned by self or parent) does not hold.
+ */
+ if (list_empty(&sch->sibling))
+ goto dump;
+
set_cgroup_sched(sch_cgroup(sch), parent);
scx_task_iter_start(&sti, sch->cgrp);
@@ -5941,8 +5958,8 @@ static void scx_sub_disable(struct scx_sched *sch)
continue;
/*
- * By the time control reaches here, all descendant schedulers
- * should already have been disabled.
+ * By the time control reaches here, all linked descendant
+ * schedulers should have been disabled.
*/
WARN_ON_ONCE(!scx_task_on_sched(sch, p));
@@ -5993,15 +6010,22 @@ static void scx_sub_disable(struct scx_sched *sch)
/*
* $p is initialized for $parent and still attached to
* @sch. Disable and exit for @sch, switch over to
- * $parent, override the state to READY to account for
- * $p having already been initialized, and then enable.
+ * $parent and override the state to READY to account
+ * for $p having already been initialized.
*/
scx_disable_and_exit_task(sch, p);
scx_set_task_state(p, SCX_TASK_INIT_BEGIN);
scx_set_task_state(p, SCX_TASK_INIT);
scx_set_task_sched(p, parent);
scx_set_task_state(p, SCX_TASK_READY);
- scx_enable_task(parent, p);
+
+ /*
+ * A task on a non-ext class, possible under an
+ * %SCX_OPS_SWITCH_PARTIAL root, stays READY and is
+ * enabled by switching_to_scx() if it switches over.
+ */
+ if (p->sched_class == &ext_sched_class)
+ scx_enable_task(parent, p);
}
task_rq_unlock(rq, p, &rf);
@@ -6009,6 +6033,7 @@ static void scx_sub_disable(struct scx_sched *sch)
}
scx_task_iter_stop(&sti);
+dump:
scx_disable_dump(sch);
scx_cgroup_unlock();
@@ -7708,10 +7733,14 @@ static void scx_sub_enable_workfn(struct kthread_work *work)
/*
* $p is now only initialized for @sch and READY, which
- * is what we want. Assign it to @sch and enable.
+ * is what we want. Assign it to @sch and, if it's on
+ * the ext class, enable. A non-ext task, possible under
+ * an %SCX_OPS_SWITCH_PARTIAL root, stays READY and is
+ * enabled by switching_to_scx() if it switches over.
*/
scx_set_task_sched(p, sch);
- scx_enable_task(sch, p);
+ if (p->sched_class == &ext_sched_class)
+ scx_enable_task(sch, p);
p->scx.flags &= ~SCX_TASK_SUB_INIT;
}
diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c
index 8e8c6201b7df..6f93cc32b650 100644
--- a/kernel/sched/ext/idle.c
+++ b/kernel/sched/ext/idle.c
@@ -554,8 +554,10 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags,
cpu_rq(cpu)->scx.local_dsq.nr == 0 &&
(!(flags & SCX_PICK_IDLE_IN_NODE) || (waker_node == node)) &&
!cpumask_empty(idle_cpumask(waker_node)->cpu)) {
- if (cpumask_test_cpu(cpu, allowed))
+ if (cpumask_test_cpu(cpu, allowed)) {
+ scx_idle_test_and_clear_cpu(cpu);
goto out_unlock;
+ }
}
}
diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c
index d9c9d9480a45..e2e825dcd088 100644
--- a/kernel/sched/psi.c
+++ b/kernel/sched/psi.c
@@ -1134,6 +1134,12 @@ void psi_cgroup_free(struct cgroup *cgroup)
return;
cancel_delayed_work_sync(&cgroup->psi->avgs_work);
+ /*
+ * A psi_schedule_rtpoll_work() call racing the last trigger's
+ * destruction may have re-armed the timer after psi_trigger_destroy()
+ * deleted it. Spurious firing while the group is alive is harmless.
+ */
+ timer_shutdown_sync(&cgroup->psi->rtpoll_timer);
free_percpu(cgroup->psi->pcpu);
/* All triggers must be removed by now */
WARN_ONCE(cgroup->psi->rtpoll_states, "psi: trigger leak\n");
@@ -1292,9 +1298,44 @@ int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res)
return 0;
}
+/*
+ * Create @group's rtpoll worker after psi_trigger_create() reported the need
+ * for one. kthread creation depends on the whole fork path and we don't want
+ * all of that nested inside cgroup_mutex, so the caller must drop it and any
+ * other lock that forks can wait behind. If two callers race, the loser stops
+ * its never-woken kthread.
+ */
+int psi_trigger_create_rtpoll_worker(struct psi_group *group)
+{
+ struct task_struct *task;
+
+ task = kthread_create(psi_rtpoll_worker, group, "psimon");
+ if (IS_ERR(task))
+ return PTR_ERR(task);
+
+ scoped_guard(mutex, &group->rtpoll_trigger_lock) {
+ if (!rcu_access_pointer(group->rtpoll_task)) {
+ atomic_set(&group->rtpoll_wakeup, 0);
+ wake_up_process(task);
+ rcu_assign_pointer(group->rtpoll_task, task);
+
+ /*
+ * Poll once to catch up on scheduling attempts dropped
+ * while there was no rtpoll worker.
+ */
+ psi_schedule_rtpoll_work(group, 1, true);
+ return 0;
+ }
+ }
+
+ kthread_stop(task);
+ return 0;
+}
+
struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf,
enum psi_res res, struct file *file,
- struct kernfs_open_file *of)
+ struct kernfs_open_file *of,
+ bool *need_rtpoll_worker)
{
struct psi_trigger *t;
enum psi_states state;
@@ -1302,6 +1343,8 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf,
bool privileged;
u32 window_us;
+ *need_rtpoll_worker = false;
+
if (static_branch_likely(&psi_disabled))
return ERR_PTR(-EOPNOTSUPP);
@@ -1362,26 +1405,14 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf,
if (privileged) {
mutex_lock(&group->rtpoll_trigger_lock);
- if (!rcu_access_pointer(group->rtpoll_task)) {
- struct task_struct *task;
-
- task = kthread_create(psi_rtpoll_worker, group, "psimon");
- if (IS_ERR(task)) {
- kfree(t);
- mutex_unlock(&group->rtpoll_trigger_lock);
- return ERR_CAST(task);
- }
- atomic_set(&group->rtpoll_wakeup, 0);
- wake_up_process(task);
- rcu_assign_pointer(group->rtpoll_task, task);
- }
-
list_add(&t->node, &group->rtpoll_triggers);
group->rtpoll_min_period = min(group->rtpoll_min_period,
div_u64(t->win.size, UPDATES_PER_WINDOW));
group->rtpoll_nr_triggers[t->state]++;
group->rtpoll_states |= (1 << t->state);
+ *need_rtpoll_worker = !rcu_access_pointer(group->rtpoll_task);
+
mutex_unlock(&group->rtpoll_trigger_lock);
} else {
mutex_lock(&group->avgs_lock);
@@ -1541,6 +1572,8 @@ static ssize_t psi_write(struct file *file, const char __user *user_buf,
size_t buf_size;
struct seq_file *seq;
struct psi_trigger *new;
+ bool need_rtpoll_worker;
+ int ret;
if (static_branch_likely(&psi_disabled))
return -EOPNOTSUPP;
@@ -1565,12 +1598,22 @@ static ssize_t psi_write(struct file *file, const char __user *user_buf,
return -EBUSY;
}
- new = psi_trigger_create(&psi_system, buf, res, file, NULL);
+ new = psi_trigger_create(&psi_system, buf, res, file, NULL,
+ &need_rtpoll_worker);
if (IS_ERR(new)) {
mutex_unlock(&seq->lock);
return PTR_ERR(new);
}
+ if (need_rtpoll_worker) {
+ ret = psi_trigger_create_rtpoll_worker(&psi_system);
+ if (ret) {
+ psi_trigger_destroy(new);
+ mutex_unlock(&seq->lock);
+ return ret;
+ }
+ }
+
smp_store_release(&seq->private, new);
mutex_unlock(&seq->lock);
diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c
index f215990b9061..f681015413b8 100644
--- a/kernel/trace/fprobe.c
+++ b/kernel/trace/fprobe.c
@@ -961,10 +961,8 @@ int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter
return -ENOMEM;
ret = get_ips_from_filter(filter, notfilter, addrs, mods, num);
- if (ret < 0)
- return ret;
-
- ret = register_fprobe_ips(fp, addrs, ret);
+ if (ret >= 0)
+ ret = register_fprobe_ips(fp, addrs, ret);
for (int i = 0; i < num; i++) {
if (mods[i])
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 6c47a94f5924..7c50f8ae5a0c 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -2645,7 +2645,8 @@ unsigned long ftrace_find_rec_direct(unsigned long ip)
{
struct ftrace_func_entry *entry;
- entry = __ftrace_lookup_ip(direct_functions, ip);
+ guard(preempt_notrace)();
+ entry = __ftrace_lookup_ip(rcu_dereference_sched(direct_functions), ip);
if (!entry)
return 0;
@@ -6511,6 +6512,7 @@ int update_ftrace_direct_del(struct ftrace_ops *ops, struct ftrace_hash *hash)
struct ftrace_hash *new_direct_functions;
struct ftrace_hash *new_filter_hash = NULL;
struct ftrace_hash *old_filter_hash;
+ struct ftrace_hash *direct_hash;
struct ftrace_func_entry *entry;
struct ftrace_func_entry *del;
unsigned long size;
@@ -6522,11 +6524,13 @@ int update_ftrace_direct_del(struct ftrace_ops *ops, struct ftrace_hash *hash)
return -EINVAL;
if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
return -EINVAL;
- if (direct_functions == EMPTY_HASH)
- return -EINVAL;
mutex_lock(&direct_mutex);
+ direct_hash = rcu_dereference_protected(direct_functions, lockdep_is_held(&direct_mutex));
+ if (direct_hash == EMPTY_HASH)
+ goto out_unlock;
+
old_filter_hash = ops->func_hash ? ops->func_hash->filter_hash : NULL;
if (!hash_count(old_filter_hash))
@@ -6536,7 +6540,7 @@ int update_ftrace_direct_del(struct ftrace_ops *ops, struct ftrace_hash *hash)
size = 1 << hash->size_bits;
for (int i = 0; i < size; i++) {
hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
- del = __ftrace_lookup_ip(direct_functions, entry->ip);
+ del = __ftrace_lookup_ip(direct_hash, entry->ip);
if (!del || del->direct != entry->direct)
goto out_unlock;
}
@@ -6547,7 +6551,7 @@ int update_ftrace_direct_del(struct ftrace_ops *ops, struct ftrace_hash *hash)
if (!new_filter_hash)
goto out_unlock;
- new_direct_functions = hash_sub(direct_functions, hash);
+ new_direct_functions = hash_sub(direct_hash, hash);
if (!new_direct_functions)
goto out_unlock;
@@ -6574,7 +6578,7 @@ int update_ftrace_direct_del(struct ftrace_ops *ops, struct ftrace_hash *hash)
/* free the new_direct_functions */
old_direct_functions = new_direct_functions;
} else {
- old_direct_functions = direct_functions;
+ old_direct_functions = direct_hash;
rcu_assign_pointer(direct_functions, new_direct_functions);
}
@@ -6613,6 +6617,7 @@ int update_ftrace_direct_mod(struct ftrace_ops *ops, struct ftrace_hash *hash, b
.func = ftrace_stub,
.flags = FTRACE_OPS_FL_STUB,
};
+ struct ftrace_hash *direct_hash;
struct ftrace_hash *orig_hash;
unsigned long size, i;
int err = -EINVAL;
@@ -6623,8 +6628,6 @@ int update_ftrace_direct_mod(struct ftrace_ops *ops, struct ftrace_hash *hash, b
return -EINVAL;
if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
return -EINVAL;
- if (direct_functions == EMPTY_HASH)
- return -EINVAL;
/*
* We can be called from within ops_func callback with direct_mutex
@@ -6632,6 +6635,12 @@ int update_ftrace_direct_mod(struct ftrace_ops *ops, struct ftrace_hash *hash, b
*/
if (do_direct_lock)
mutex_lock(&direct_mutex);
+ else
+ lockdep_assert_held_once(&direct_mutex);
+
+ direct_hash = rcu_dereference_protected(direct_functions, lockdep_is_held(&direct_mutex));
+ if (direct_hash == EMPTY_HASH)
+ goto unlock;
orig_hash = ops->func_hash ? ops->func_hash->filter_hash : NULL;
if (!orig_hash)
@@ -6663,7 +6672,7 @@ int update_ftrace_direct_mod(struct ftrace_ops *ops, struct ftrace_hash *hash, b
size = 1 << hash->size_bits;
for (i = 0; i < size; i++) {
hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
- tmp = __ftrace_lookup_ip(direct_functions, entry->ip);
+ tmp = __ftrace_lookup_ip(direct_hash, entry->ip);
if (!tmp)
continue;
tmp->direct = entry->direct;
@@ -8296,7 +8305,8 @@ static void add_to_clear_hash_list(struct list_head *clear_list,
void ftrace_free_mem(struct module *mod, void *start_ptr, void *end_ptr)
{
unsigned long start = (unsigned long)(start_ptr);
- unsigned long end = (unsigned long)(end_ptr);
+ /* end is inclusive and end_ptr is exclusive */
+ unsigned long end = (unsigned long)(end_ptr) - 1;
struct ftrace_page **last_pg = &ftrace_pages_start;
struct ftrace_page *tmp_page = NULL;
struct ftrace_page *pg;
@@ -8306,6 +8316,9 @@ void ftrace_free_mem(struct module *mod, void *start_ptr, void *end_ptr)
struct ftrace_init_func *func, *func_next;
LIST_HEAD(clear_hash);
+ if (start_ptr >= end_ptr)
+ return;
+
key.ip = start;
key.flags = end; /* overload flags, as it is unsigned long */
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index 804ccae694d2..2667992f0aa2 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -2510,6 +2510,7 @@ rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu)
bpage = alloc_cpu_page(cpu);
if (!bpage)
return NULL;
+ bpage->order = cpu_buffer->buffer->subbuf_order;
rb_check_bpage(cpu_buffer, bpage);
@@ -2528,6 +2529,8 @@ rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu)
if (cpu_buffer->ring_meta->head_buffer)
rb_meta_buffer_update(cpu_buffer, bpage);
bpage->range = 1;
+
+ atomic_inc(&cpu_buffer->resize_disabled);
} else if (buffer->remote) {
struct ring_buffer_desc *desc = ring_buffer_desc(buffer->remote->desc, cpu);
@@ -2599,6 +2602,7 @@ rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu)
return_ptr(cpu_buffer);
fail_free_reader:
+ kfree(cpu_buffer->subbuf_ids);
free_buffer_page(cpu_buffer->reader_page);
return NULL;
@@ -5783,6 +5787,7 @@ __rb_get_reader_page_from_remote(struct ring_buffer_per_cpu *cpu_buffer)
cpu_buffer->head_page = new_head;
cpu_buffer->reader_page = new_reader;
+ cpu_buffer->reader_page->read = 0;
cpu_buffer->pages = &new_head->list;
cpu_buffer->read_stamp = new_reader->page->time_stamp;
cpu_buffer->lost_events = cpu_buffer->meta_page->reader.lost_events;
@@ -6850,7 +6855,7 @@ int ring_buffer_swap_cpu(struct trace_buffer *buffer_a,
{
struct ring_buffer_per_cpu *cpu_buffer_a;
struct ring_buffer_per_cpu *cpu_buffer_b;
- int ret = -EINVAL;
+ int ret = -EBUSY;
if (!cpumask_test_cpu(cpu, buffer_a->cpumask) ||
!cpumask_test_cpu(cpu, buffer_b->cpumask))
@@ -6891,10 +6896,10 @@ int ring_buffer_swap_cpu(struct trace_buffer *buffer_a,
atomic_inc(&cpu_buffer_a->record_disabled);
atomic_inc(&cpu_buffer_b->record_disabled);
- ret = -EBUSY;
- if (local_read(&cpu_buffer_a->committing))
+ /* Do not swap if either buffer is in the process of writing */
+ if (cpu_buffer_a->current_context)
goto out_dec;
- if (local_read(&cpu_buffer_b->committing))
+ if (cpu_buffer_b->current_context)
goto out_dec;
/*
@@ -7356,7 +7361,7 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order)
cpu_buffer = buffer->buffers[cpu];
- if (cpu_buffer->mapped) {
+ if (atomic_read(&cpu_buffer->resize_disabled)) {
err = -EBUSY;
goto error;
}
@@ -8212,7 +8217,7 @@ static __init int test_ringbuffer(void)
out_free:
for_each_online_cpu(cpu) {
- if (!rb_threads[cpu])
+ if (IS_ERR_OR_NULL(rb_threads[cpu]))
break;
kthread_stop(rb_threads[cpu]);
}
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 01a5e87af299..395238b2b715 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -1788,7 +1788,7 @@ void trace_buffered_event_enable(void)
per_cpu(trace_buffered_event, cpu) = event;
- scoped_guard(preempt,) {
+ scoped_guard(preempt) {
if (cpu == smp_processor_id() &&
__this_cpu_read(trace_buffered_event) !=
per_cpu(trace_buffered_event, cpu))
diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c
index 956692856fa8..c01b10b99f67 100644
--- a/kernel/trace/trace_events.c
+++ b/kernel/trace/trace_events.c
@@ -3933,8 +3933,8 @@ static void trace_module_add_events(struct module *mod)
end = mod->trace_events + mod->num_trace_events;
for_each_event(call, start, end) {
- __register_event(*call, mod);
- __add_event_to_tracers(*call);
+ if (!__register_event(*call, mod))
+ __add_event_to_tracers(*call);
}
update_cache_events(mod);
diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c
index 6385cd662d8d..2b46ca536045 100644
--- a/kernel/trace/trace_events_filter.c
+++ b/kernel/trace/trace_events_filter.c
@@ -1027,6 +1027,9 @@ static int regex_match_full(char *str, struct regex *r, int len)
if (!len)
return strcmp(str, r->pattern) == 0;
+ if (len < r->len)
+ return 0;
+
return strncmp(str, r->pattern, len) == 0;
}
diff --git a/kernel/trace/trace_fprobe.c b/kernel/trace/trace_fprobe.c
index 9f5f08c0e7c2..3120403a5b60 100644
--- a/kernel/trace/trace_fprobe.c
+++ b/kernel/trace/trace_fprobe.c
@@ -1481,11 +1481,21 @@ static int enable_trace_fprobe(struct trace_event_call *call,
list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) {
ret = __register_trace_fprobe(tf);
if (ret < 0)
- return ret;
+ goto err;
}
}
return 0;
+
+err:
+ /* Failed to enable one of them. Roll back all */
+ list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list)
+ __unregister_trace_fprobe(tf);
+ if (file)
+ trace_probe_remove_file(tp, file);
+ else
+ trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
+ return ret;
}
/*
diff --git a/kernel/trace/trace_mmiotrace.c b/kernel/trace/trace_mmiotrace.c
index b88b8d9923ad..df8692c2dea8 100644
--- a/kernel/trace/trace_mmiotrace.c
+++ b/kernel/trace/trace_mmiotrace.c
@@ -29,6 +29,7 @@ static void mmio_reset_data(struct trace_array *tr)
{
overrun_detected = false;
prev_overruns = 0;
+ atomic_set(&dropped_count, 0);
tracing_reset_online_cpus(&tr->array_buffer);
}
@@ -293,11 +294,15 @@ device_initcall(init_mmio_trace);
static void __trace_mmiotrace_rw(struct trace_array *tr,
struct mmiotrace_rw *rw)
{
- struct trace_buffer *buffer = tr->array_buffer.buffer;
+ struct trace_buffer *buffer;
struct ring_buffer_event *event;
struct trace_mmiotrace_rw *entry;
unsigned int trace_ctx;
+ if (!tr)
+ return;
+
+ buffer = tr->array_buffer.buffer;
trace_ctx = tracing_gen_ctx_flags(0);
event = trace_buffer_lock_reserve(buffer, TRACE_MMIO_RW,
sizeof(*entry), trace_ctx);
@@ -320,11 +325,15 @@ void mmio_trace_rw(struct mmiotrace_rw *rw)
static void __trace_mmiotrace_map(struct trace_array *tr,
struct mmiotrace_map *map)
{
- struct trace_buffer *buffer = tr->array_buffer.buffer;
+ struct trace_buffer *buffer;
struct ring_buffer_event *event;
struct trace_mmiotrace_map *entry;
unsigned int trace_ctx;
+ if (!tr)
+ return;
+
+ buffer = tr->array_buffer.buffer;
trace_ctx = tracing_gen_ctx_flags(0);
event = trace_buffer_lock_reserve(buffer, TRACE_MMIO_MAP,
sizeof(*entry), trace_ctx);
diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c
index 506e6037e163..c8fd9b946f44 100644
--- a/kernel/trace/trace_probe.c
+++ b/kernel/trace/trace_probe.c
@@ -1901,7 +1901,11 @@ const char **traceprobe_expand_meta_args(int argc, const char *argv[],
trace_probe_log_err(0, BAD_VAR);
return ERR_PTR(-ENOENT);
}
- /* Note: $argN starts from $arg1 */
+ /* Note: $argN starts from $arg1, so $arg0 is invalid. */
+ if (n == 0) {
+ trace_probe_log_err(0, BAD_ARG_NUM);
+ return ERR_PTR(-EINVAL);
+ }
ret = sprint_nth_btf_arg(n - 1, type, buf + used,
bufsize - used, ctx);
if (ret < 0)
generated by cgit v1.2.3 (git 2.25.1) at 2026-09-01 10:58:55 +0000