From 31f61ac33032ee87ea404d6d996ba2c386502a36 Mon Sep 17 00:00:00 2001 From: Amery Hung Date: Tue, 14 Apr 2026 12:10:14 -0700 Subject: bpf: Refactor dynptr mutability tracking Redefine dynptr mutability and fix inconsistency in the verifier and kfunc signatures. Dynptr mutability is at two levels. The first is the bpf_dynptr structure and the second is the memory the dynptr points to. The verifer currently tracks the mutability of the bpf_dynptr struct through helper and kfunc prototypes, where "const struct bpf_dynptr *" means the structure itself is immutable. The second level is tracked in upper bit of bpf_dynptr->size in runtime and is not changed in this patch. There are two type of inconsistency in the verfier regarding the mutability of the bpf_dynptr struct. First, there are many existing kfuncs whose prototypes are wrong. For example, bpf_dynptr_adjust() mutates a dynptr's start and offset but marks the argument as a const pointer. At the same time many other kfuncs that does not mutate the dynptr but mark themselves as mutable. Second, the verifier currently does not honor the const qualifier in kfunc prototypes as it determines whether tagging the arg_type with MEM_RDONLY or not based on the register state. Since all the verifier care is to prevent CONST_PTR_TO_DYNPTR from being destroyed in callback and global subprogram, redefine the mutability at the bpf_dynptr level to just bpf_dynptr_kern->data. Then, explicitly prohibit passing CONST_PTR_TO_DYNPTR to an argument tagged with MEM_UNINIT or OBJ_RELEASE. The mutability of a dynptr's view is not really interesting so drop MEM_RDONLY annotation for dynptr from the helpers and kfuncs. Plus, if the mutability of the entire bpf_dynptr were to be done correctly, it would kill the bpf_dynptr_adjust() usage in callback and global subporgram. Implementation wise - First, make sure all kfunc arg are correctly tagged: Tag the dynptr argument of bpf_dynptr_file_discard() with OBJ_RELEASE. - Then, in process_dynptr_func(), make sure CONST_PTR_TO_DYNPTR cannot be passed to argument tagged with MEM_UNINIT or OBJ_RELEASE. For MEM_UNINIT, it is already checked by is_dynptr_reg_valid_uninit(). For OBJ_RELEASE, check against OBJ_RELEASE instead of MEM_RDONLY and drop a now identical check in unmark_stack_slots_dynptr(). - Remove the mutual exclusive check between MEM_UNINIT and MEM_RDONLY, but don't add a MEM_UNINIT and OBJ_RELEASE version as it is obviously wrong. Note that while this patch stops following the C semantic for the mutability of bpf_dynptr, the prototype of kfuncs are still fixed to maintain the correct C semantics in the implementation. Adding or removing the const qualifier does not break backward compatibility. In addition, fix kfuncs dropping the const qualifier when casting the opaque bpf_dynptr to bpf_dynptr_kern. In test_kfunc_dynptr_param.c, initialize dynptr to 0 to avoid -Wuninitialized-const-pointer warning. Signed-off-by: Amery Hung Acked-by: Kumar Kartikeya Dwivedi Link: https://lore.kernel.org/bpf/20260414191014.1218567-1-ameryhung@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/trace/bpf_trace.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index af7079aa0f36..e916f0ccbed9 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -3397,12 +3397,12 @@ typedef int (*copy_fn_t)(void *dst, const void *src, u32 size, struct task_struc * direct calls into all the specific callback implementations * (copy_user_data_sleepable, copy_user_data_nofault, and so on) */ -static __always_inline int __bpf_dynptr_copy_str(struct bpf_dynptr *dptr, u64 doff, u64 size, +static __always_inline int __bpf_dynptr_copy_str(const struct bpf_dynptr *dptr, u64 doff, u64 size, const void *unsafe_src, copy_fn_t str_copy_fn, struct task_struct *tsk) { - struct bpf_dynptr_kern *dst; + const struct bpf_dynptr_kern *dst; u64 chunk_sz, off; void *dst_slice; int cnt, err; @@ -3438,7 +3438,7 @@ static __always_inline int __bpf_dynptr_copy(const struct bpf_dynptr *dptr, u64 u64 size, const void *unsafe_src, copy_fn_t copy_fn, struct task_struct *tsk) { - struct bpf_dynptr_kern *dst; + const struct bpf_dynptr_kern *dst; void *dst_slice; char buf[256]; u64 off, chunk_sz; @@ -3539,49 +3539,49 @@ __bpf_kfunc int bpf_send_signal_task(struct task_struct *task, int sig, enum pid return bpf_send_signal_common(sig, type, task, value); } -__bpf_kfunc int bpf_probe_read_user_dynptr(struct bpf_dynptr *dptr, u64 off, +__bpf_kfunc int bpf_probe_read_user_dynptr(const struct bpf_dynptr *dptr, u64 off, u64 size, const void __user *unsafe_ptr__ign) { return __bpf_dynptr_copy(dptr, off, size, (const void __force *)unsafe_ptr__ign, copy_user_data_nofault, NULL); } -__bpf_kfunc int bpf_probe_read_kernel_dynptr(struct bpf_dynptr *dptr, u64 off, +__bpf_kfunc int bpf_probe_read_kernel_dynptr(const struct bpf_dynptr *dptr, u64 off, u64 size, const void *unsafe_ptr__ign) { return __bpf_dynptr_copy(dptr, off, size, unsafe_ptr__ign, copy_kernel_data_nofault, NULL); } -__bpf_kfunc int bpf_probe_read_user_str_dynptr(struct bpf_dynptr *dptr, u64 off, +__bpf_kfunc int bpf_probe_read_user_str_dynptr(const struct bpf_dynptr *dptr, u64 off, u64 size, const void __user *unsafe_ptr__ign) { return __bpf_dynptr_copy_str(dptr, off, size, (const void __force *)unsafe_ptr__ign, copy_user_str_nofault, NULL); } -__bpf_kfunc int bpf_probe_read_kernel_str_dynptr(struct bpf_dynptr *dptr, u64 off, +__bpf_kfunc int bpf_probe_read_kernel_str_dynptr(const struct bpf_dynptr *dptr, u64 off, u64 size, const void *unsafe_ptr__ign) { return __bpf_dynptr_copy_str(dptr, off, size, unsafe_ptr__ign, copy_kernel_str_nofault, NULL); } -__bpf_kfunc int bpf_copy_from_user_dynptr(struct bpf_dynptr *dptr, u64 off, +__bpf_kfunc int bpf_copy_from_user_dynptr(const struct bpf_dynptr *dptr, u64 off, u64 size, const void __user *unsafe_ptr__ign) { return __bpf_dynptr_copy(dptr, off, size, (const void __force *)unsafe_ptr__ign, copy_user_data_sleepable, NULL); } -__bpf_kfunc int bpf_copy_from_user_str_dynptr(struct bpf_dynptr *dptr, u64 off, +__bpf_kfunc int bpf_copy_from_user_str_dynptr(const struct bpf_dynptr *dptr, u64 off, u64 size, const void __user *unsafe_ptr__ign) { return __bpf_dynptr_copy_str(dptr, off, size, (const void __force *)unsafe_ptr__ign, copy_user_str_sleepable, NULL); } -__bpf_kfunc int bpf_copy_from_user_task_dynptr(struct bpf_dynptr *dptr, u64 off, +__bpf_kfunc int bpf_copy_from_user_task_dynptr(const struct bpf_dynptr *dptr, u64 off, u64 size, const void __user *unsafe_ptr__ign, struct task_struct *tsk) { @@ -3589,7 +3589,7 @@ __bpf_kfunc int bpf_copy_from_user_task_dynptr(struct bpf_dynptr *dptr, u64 off, copy_user_data_sleepable, tsk); } -__bpf_kfunc int bpf_copy_from_user_task_str_dynptr(struct bpf_dynptr *dptr, u64 off, +__bpf_kfunc int bpf_copy_from_user_task_str_dynptr(const struct bpf_dynptr *dptr, u64 off, u64 size, const void __user *unsafe_ptr__ign, struct task_struct *tsk) { -- cgit v1.2.3 From 439ebd5b5708f236f7a4a9784194f7ecb77cd814 Mon Sep 17 00:00:00 2001 From: Mykyta Yatsenko Date: Wed, 22 Apr 2026 12:41:06 -0700 Subject: bpf: Add sleepable support for raw tracepoint programs Rework __bpf_trace_run() to support sleepable BPF programs by using explicit RCU flavor selection, following the uprobe_prog_run() pattern. For sleepable programs, use rcu_read_lock_tasks_trace() for lifetime protection with migrate_disable(). For non-sleepable programs, use the regular rcu_read_lock_dont_migrate(). Remove the preempt_disable_notrace/preempt_enable_notrace pair from the faultable tracepoint BPF probe wrapper in bpf_probe.h, since migration protection and RCU locking are now handled per-program inside __bpf_trace_run(). Adapt bpf_prog_test_run_raw_tp() for sleepable programs: reject BPF_F_TEST_RUN_ON_CPU since sleepable programs cannot run in hardirq or preempt-disabled context, and call __bpf_prog_test_run_raw_tp() directly instead of via smp_call_function_single(). Rework __bpf_prog_test_run_raw_tp() to select RCU flavor per-program and add per-program recursion context guard for private stack safety. Signed-off-by: Mykyta Yatsenko Acked-by: Kumar Kartikeya Dwivedi Link: https://lore.kernel.org/bpf/20260422-sleepable_tracepoints-v13-1-99005dff21ef@meta.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/trace/bpf_trace.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index e916f0ccbed9..7276c72c1d31 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -2072,11 +2072,19 @@ void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp) static __always_inline void __bpf_trace_run(struct bpf_raw_tp_link *link, u64 *args) { + struct srcu_ctr __percpu *scp = NULL; struct bpf_prog *prog = link->link.prog; + bool sleepable = prog->sleepable; struct bpf_run_ctx *old_run_ctx; struct bpf_trace_run_ctx run_ctx; - rcu_read_lock_dont_migrate(); + if (sleepable) { + scp = rcu_read_lock_tasks_trace(); + migrate_disable(); + } else { + rcu_read_lock_dont_migrate(); + } + if (unlikely(!bpf_prog_get_recursion_context(prog))) { bpf_prog_inc_misses_counter(prog); goto out; @@ -2085,12 +2093,18 @@ void __bpf_trace_run(struct bpf_raw_tp_link *link, u64 *args) run_ctx.bpf_cookie = link->cookie; old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx); - (void) bpf_prog_run(prog, args); + (void)bpf_prog_run(prog, args); bpf_reset_run_ctx(old_run_ctx); out: bpf_prog_put_recursion_context(prog); - rcu_read_unlock_migrate(); + + if (sleepable) { + migrate_enable(); + rcu_read_unlock_tasks_trace(scp); + } else { + rcu_read_unlock_migrate(); + } } #define UNPACK(...) __VA_ARGS__ -- cgit v1.2.3 From 57918341dd19e5ca8a77622ffae3db19e5ba4cc7 Mon Sep 17 00:00:00 2001 From: Mykyta Yatsenko Date: Wed, 22 Apr 2026 12:41:08 -0700 Subject: bpf: Add sleepable support for classic tracepoint programs Add trace_call_bpf_faultable(), a variant of trace_call_bpf() for faultable tracepoints that supports sleepable BPF programs. It uses rcu_tasks_trace for lifetime protection and bpf_prog_run_array_sleepable() for per-program RCU flavor selection, following the uprobe_prog_run() pattern. Restructure perf_syscall_enter() and perf_syscall_exit() to run BPF programs before perf event processing. Previously, BPF ran after the per-cpu perf trace buffer was allocated under preempt_disable, requiring cleanup via perf_swevent_put_recursion_context() on filter. Now BPF runs in faultable context before preempt_disable, reading syscall arguments from local variables instead of the per-cpu trace record, removing the dependency on buffer allocation. This allows sleepable BPF programs to execute and avoids unnecessary buffer allocation when BPF filters the event. The perf event submission path (buffer allocation, fill, submit) remains under preempt_disable as before. Since BPF no longer runs within the buffer allocation context, the fake_regs output parameter to perf_trace_buf_alloc() is no longer needed and is replaced with NULL. Add an attach-time check in __perf_event_set_bpf_prog() to reject sleepable BPF_PROG_TYPE_TRACEPOINT programs on non-syscall tracepoints, since only syscall tracepoints run in faultable context. This prepares the classic tracepoint runtime and attach paths for sleepable programs. The verifier changes to allow loading sleepable BPF_PROG_TYPE_TRACEPOINT programs are in a subsequent patch. To: Peter Zijlstra To: Steven Rostedt Signed-off-by: Mykyta Yatsenko Acked-by: Kumar Kartikeya Dwivedi # for BPF bits Acked-by: Steven Rostedt Link: https://lore.kernel.org/bpf/20260422-sleepable_tracepoints-v13-3-99005dff21ef@meta.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/trace/bpf_trace.c | 28 +++++++++++ kernel/trace/trace_syscalls.c | 110 ++++++++++++++++++++++-------------------- 2 files changed, 86 insertions(+), 52 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 7276c72c1d31..a822c589c9bd 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -152,6 +152,34 @@ unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx) return ret; } +/** + * trace_call_bpf_faultable - invoke BPF program in faultable context + * @call: tracepoint event + * @ctx: opaque context pointer + * + * Variant of trace_call_bpf() for faultable tracepoints (syscall + * tracepoints). Supports sleepable BPF programs by using rcu_tasks_trace + * for lifetime protection and bpf_prog_run_array_sleepable() for per-program + * RCU flavor selection, following the uprobe pattern. + * + * Per-program recursion protection is provided by + * bpf_prog_run_array_sleepable(). Global bpf_prog_active is not + * needed because syscall tracepoints cannot self-recurse. + * + * Must be called from a faultable/preemptible context. + */ +unsigned int trace_call_bpf_faultable(struct trace_event_call *call, void *ctx) +{ + struct bpf_prog_array *prog_array; + + might_fault(); + guard(rcu_tasks_trace)(); + + prog_array = rcu_dereference_check(call->prog_array, + rcu_read_lock_trace_held()); + return bpf_prog_run_array_sleepable(prog_array, ctx, bpf_prog_run); +} + #ifdef CONFIG_BPF_KPROBE_OVERRIDE BPF_CALL_2(bpf_override_return, struct pt_regs *, regs, unsigned long, rc) { diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c index 8ad72e17d8eb..e98ee7e1e66f 100644 --- a/kernel/trace/trace_syscalls.c +++ b/kernel/trace/trace_syscalls.c @@ -1371,33 +1371,33 @@ static DECLARE_BITMAP(enabled_perf_exit_syscalls, NR_syscalls); static int sys_perf_refcount_enter; static int sys_perf_refcount_exit; -static int perf_call_bpf_enter(struct trace_event_call *call, struct pt_regs *regs, +static int perf_call_bpf_enter(struct trace_event_call *call, struct syscall_metadata *sys_data, - struct syscall_trace_enter *rec) + int syscall_nr, unsigned long *args) { struct syscall_tp_t { struct trace_entry ent; int syscall_nr; unsigned long args[SYSCALL_DEFINE_MAXARGS]; } __aligned(8) param; + struct pt_regs regs = {}; int i; BUILD_BUG_ON(sizeof(param.ent) < sizeof(void *)); - /* bpf prog requires 'regs' to be the first member in the ctx (a.k.a. ¶m) */ - perf_fetch_caller_regs(regs); - *(struct pt_regs **)¶m = regs; - param.syscall_nr = rec->nr; + /* bpf prog requires 'regs' to be the first member in the ctx */ + perf_fetch_caller_regs(®s); + *(struct pt_regs **)¶m = ®s; + param.syscall_nr = syscall_nr; for (i = 0; i < sys_data->nb_args; i++) - param.args[i] = rec->args[i]; - return trace_call_bpf(call, ¶m); + param.args[i] = args[i]; + return trace_call_bpf_faultable(call, ¶m); } static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) { struct syscall_metadata *sys_data; struct syscall_trace_enter *rec; - struct pt_regs *fake_regs; struct hlist_head *head; unsigned long args[6]; bool valid_prog_array; @@ -1410,12 +1410,7 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) int size = 0; int uargs = 0; - /* - * Syscall probe called with preemption enabled, but the ring - * buffer and per-cpu data require preemption to be disabled. - */ might_fault(); - guard(preempt_notrace)(); syscall_nr = trace_get_syscall_nr(current, regs); if (syscall_nr < 0 || syscall_nr >= NR_syscalls) @@ -1429,6 +1424,26 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) syscall_get_arguments(current, regs, args); + /* + * Run BPF program in faultable context before per-cpu buffer + * allocation, allowing sleepable BPF programs to execute. + */ + valid_prog_array = bpf_prog_array_valid(sys_data->enter_event); + if (valid_prog_array && + !perf_call_bpf_enter(sys_data->enter_event, sys_data, + syscall_nr, args)) + return; + + /* + * Per-cpu ring buffer and perf event list operations require + * preemption to be disabled. + */ + guard(preempt_notrace)(); + + head = this_cpu_ptr(sys_data->enter_event->perf_events); + if (hlist_empty(head)) + return; + /* Check if this syscall event faults in user space memory */ mayfault = sys_data->user_mask != 0; @@ -1438,17 +1453,12 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) return; } - head = this_cpu_ptr(sys_data->enter_event->perf_events); - valid_prog_array = bpf_prog_array_valid(sys_data->enter_event); - if (!valid_prog_array && hlist_empty(head)) - return; - /* get the size after alignment with the u32 buffer size field */ size += sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec); size = ALIGN(size + sizeof(u32), sizeof(u64)); size -= sizeof(u32); - rec = perf_trace_buf_alloc(size, &fake_regs, &rctx); + rec = perf_trace_buf_alloc(size, NULL, &rctx); if (!rec) return; @@ -1458,13 +1468,6 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) if (mayfault) syscall_put_data(sys_data, rec, user_ptr, size, user_sizes, uargs); - if ((valid_prog_array && - !perf_call_bpf_enter(sys_data->enter_event, fake_regs, sys_data, rec)) || - hlist_empty(head)) { - perf_swevent_put_recursion_context(rctx); - return; - } - perf_trace_buf_submit(rec, size, rctx, sys_data->enter_event->event.type, 1, regs, head, NULL); @@ -1514,40 +1517,35 @@ static void perf_sysenter_disable(struct trace_event_call *call) syscall_fault_buffer_disable(); } -static int perf_call_bpf_exit(struct trace_event_call *call, struct pt_regs *regs, - struct syscall_trace_exit *rec) +static int perf_call_bpf_exit(struct trace_event_call *call, + int syscall_nr, long ret_val) { struct syscall_tp_t { struct trace_entry ent; int syscall_nr; unsigned long ret; } __aligned(8) param; - - /* bpf prog requires 'regs' to be the first member in the ctx (a.k.a. ¶m) */ - perf_fetch_caller_regs(regs); - *(struct pt_regs **)¶m = regs; - param.syscall_nr = rec->nr; - param.ret = rec->ret; - return trace_call_bpf(call, ¶m); + struct pt_regs regs = {}; + + /* bpf prog requires 'regs' to be the first member in the ctx */ + perf_fetch_caller_regs(®s); + *(struct pt_regs **)¶m = ®s; + param.syscall_nr = syscall_nr; + param.ret = ret_val; + return trace_call_bpf_faultable(call, ¶m); } static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret) { struct syscall_metadata *sys_data; struct syscall_trace_exit *rec; - struct pt_regs *fake_regs; struct hlist_head *head; bool valid_prog_array; int syscall_nr; int rctx; int size; - /* - * Syscall probe called with preemption enabled, but the ring - * buffer and per-cpu data require preemption to be disabled. - */ might_fault(); - guard(preempt_notrace)(); syscall_nr = trace_get_syscall_nr(current, regs); if (syscall_nr < 0 || syscall_nr >= NR_syscalls) @@ -1559,29 +1557,37 @@ static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret) if (!sys_data) return; - head = this_cpu_ptr(sys_data->exit_event->perf_events); + /* + * Run BPF program in faultable context before per-cpu buffer + * allocation, allowing sleepable BPF programs to execute. + */ valid_prog_array = bpf_prog_array_valid(sys_data->exit_event); - if (!valid_prog_array && hlist_empty(head)) + if (valid_prog_array && + !perf_call_bpf_exit(sys_data->exit_event, syscall_nr, + syscall_get_return_value(current, regs))) + return; + + /* + * Per-cpu ring buffer and perf event list operations require + * preemption to be disabled. + */ + guard(preempt_notrace)(); + + head = this_cpu_ptr(sys_data->exit_event->perf_events); + if (hlist_empty(head)) return; /* We can probably do that at build time */ size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64)); size -= sizeof(u32); - rec = perf_trace_buf_alloc(size, &fake_regs, &rctx); + rec = perf_trace_buf_alloc(size, NULL, &rctx); if (!rec) return; rec->nr = syscall_nr; rec->ret = syscall_get_return_value(current, regs); - if ((valid_prog_array && - !perf_call_bpf_exit(sys_data->exit_event, fake_regs, rec)) || - hlist_empty(head)) { - perf_swevent_put_recursion_context(rctx); - return; - } - perf_trace_buf_submit(rec, size, rctx, sys_data->exit_event->event.type, 1, regs, head, NULL); } -- cgit v1.2.3 From ea19506013ad13685573e4674fbeddb790e27906 Mon Sep 17 00:00:00 2001 From: Yiyang Chen Date: Fri, 15 May 2026 00:05:05 +0800 Subject: sched/clock: Provide !HAVE_UNSTABLE_SCHED_CLOCK stub for sched_clock_stable() When CONFIG_HAVE_UNSTABLE_SCHED_CLOCK is disabled, sched_clock() is already assumed to provide stable semantics, but the public header doesn't provide a sched_clock_stable() stub for that case. Add a header stub that always returns true and clean up the duplicate local stub in ring_buffer.c, so callers can use sched_clock_stable() unconditionally. Signed-off-by: Yiyang Chen Signed-off-by: Peter Zijlstra (Intel) Acked-by: Steven Rostedt Link: https://patch.msgid.link/56e45338858946cd9581b75c8bd45dd37dba52c5.1778773587.git.cyyzero16@gmail.com --- kernel/trace/ring_buffer.c | 7 ------- 1 file changed, 7 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 5326924615a4..02691c3c6dd6 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -3769,13 +3769,6 @@ rb_add_time_stamp(struct ring_buffer_per_cpu *cpu_buffer, return skip_time_extend(event); } -#ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK -static inline bool sched_clock_stable(void) -{ - return true; -} -#endif - static void rb_check_timestamp(struct ring_buffer_per_cpu *cpu_buffer, struct rb_event_info *info) -- cgit v1.2.3 From 576ec047d20b368b43c4d5db98c4f2e0f3c101ec Mon Sep 17 00:00:00 2001 From: David Carlier Date: Fri, 8 May 2026 20:57:47 +0100 Subject: tracing: Avoid NULL return from hist_field_name() on truncation hist_field_name() returns "" everywhere except the fully-qualified VAR_REF/EXPR case, where snprintf() truncation returns NULL early and bypasses the bottom NULL->"" guard. Callers don't expect NULL: strcat(expr, hist_field_name(field, 0)) at trace_events_hist.c:1758 and the strcmp() in the sort-key match loop at :4804 both deref it. system and event_name are bounded by MAX_EVENT_NAME_LEN, but the field name on a VAR_REF is kstrdup'd from a histogram variable name parsed out of the trigger string and has no length cap, so a long enough var name in a fully qualified reference can reach the truncation path. Keep the length check but leave field_name as "" on overflow. Link: https://patch.msgid.link/20260508195747.25492-1-devnexen@gmail.com Fixes: 5ec1d1e97de1 ("tracing: Rebuild full_name on each hist_field_name() call") Signed-off-by: David Carlier Signed-off-by: Steven Rostedt --- kernel/trace/trace_events_hist.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index 0dbbf6cca9bc..eb2c2bc8bc3d 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -1369,10 +1369,8 @@ static const char *hist_field_name(struct hist_field *field, len = snprintf(full_name, sizeof(full_name), fmt, field->system, field->event_name, field->name); - if (len >= sizeof(full_name)) - return NULL; - - field_name = full_name; + if (len < sizeof(full_name)) + field_name = full_name; } else field_name = field->name; } else if (field->flags & HIST_FIELD_FL_TIMESTAMP) -- cgit v1.2.3 From e11c9c8365cd7655e81c4e6611492f1b15850aa5 Mon Sep 17 00:00:00 2001 From: Crystal Wood Date: Mon, 11 May 2026 17:31:43 -0500 Subject: tracing/osnoise: Dump stack on timerlat uret threshold event Dump the saved IRQ stack trace regardless of whether the event was THREAD_CONTEXT or THREAD_URET. In the uret case, the latency presumably had not yet crossed the threshold at IRQ time (or else it would have dumped the stack at thread wakeup time, unless we're racing with a change to the threshold), but it may have at least contributed -- and this is possible with THREAD_CONTEXT as well. In any case, it helps with writing reliable rtla tests if we always get a stack trace on a threshold event. Cc: John Kacur Cc: Tomas Glozar Cc: Costa Shulyupin Cc: Wander Lairson Costa Link: https://patch.msgid.link/20260511223143.1477332-1-crwood@redhat.com Signed-off-by: Crystal Wood Signed-off-by: Steven Rostedt --- kernel/trace/trace_osnoise.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c index 75678053b21c..62c2667d97fa 100644 --- a/kernel/trace/trace_osnoise.c +++ b/kernel/trace/trace_osnoise.c @@ -2544,9 +2544,12 @@ timerlat_fd_read(struct file *file, char __user *ubuf, size_t count, notify_new_max_latency(diff); tlat->tracing_thread = false; - if (osnoise_data.stop_tracing_total) - if (time_to_us(diff) >= osnoise_data.stop_tracing_total) + if (osnoise_data.stop_tracing_total) { + if (time_to_us(diff) >= osnoise_data.stop_tracing_total) { + timerlat_dump_stack(time_to_us(diff)); osnoise_stop_tracing(); + } + } } else { tlat->tracing_thread = false; tlat->kthread = current; -- cgit v1.2.3 From a254b6d13b0edd6272926674d2afc46d46e496b7 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Wed, 20 May 2026 22:08:01 -0400 Subject: ring-buffer: Fix reporting of missed events in iterator When tracing is active while reading the trace file, if the iterator reading the buffer detects that the writer has passed the iterator head, it will reset and set a "missed events" flag. This flag is passed to the output processing to show the user that events were missed: CPU:4 [LOST EVENTS] The problem is that the flag is reset after it is checked in ring_buffer_iter_dropped(). But the "trace" file iterates over all the CPU ring buffers and it will check if they are dropped when figuring out which buffer to print next. This prematurely clears the missed_events flag if the CPU buffer with the missed events is not the one that is printed next. On the iteration where the CPU buffer with the missed events is printed, the check if it had missed events would return false and the output does not show that events were missed. Do not reset the missed_events flag when checking if there were missed events, but instead clear it when moving the iterator head to the next event. Cc: stable@vger.kernel.org Cc: Mathieu Desnoyers Link: https://patch.msgid.link/20260520220801.4fd09d13@fedora Fixes: c9b7a4a72ff64 ("ring-buffer/tracing: Have iterator acknowledge dropped events") Acked-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 5326924615a4..fcd93d49851e 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -5407,6 +5407,7 @@ static void rb_iter_reset(struct ring_buffer_iter *iter) iter->head_page = cpu_buffer->reader_page; iter->head = cpu_buffer->reader_page->read; iter->next_event = iter->head; + iter->missed_events = 0; iter->cache_reader_page = iter->head_page; iter->cache_read = cpu_buffer->read; @@ -6086,10 +6087,7 @@ ring_buffer_peek(struct trace_buffer *buffer, int cpu, u64 *ts, */ bool ring_buffer_iter_dropped(struct ring_buffer_iter *iter) { - bool ret = iter->missed_events != 0; - - iter->missed_events = 0; - return ret; + return iter->missed_events != 0; } EXPORT_SYMBOL_GPL(ring_buffer_iter_dropped); @@ -6251,7 +6249,7 @@ void ring_buffer_iter_advance(struct ring_buffer_iter *iter) unsigned long flags; raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); - + iter->missed_events = 0; rb_advance_iter(iter); raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); -- cgit v1.2.3 From a494d3c8d5392bcdff83c2a593df0c160ff9f322 Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Thu, 30 Apr 2026 12:28:16 +0900 Subject: ring-buffer: Flush and stop persistent ring buffer on panic On real hardware, panic and machine reboot may not flush hardware cache to memory. This means the persistent ring buffer, which relies on a coherent state of memory, may not have its events written to the buffer and they may be lost. Moreover, there may be inconsistency with the counters which are used for validation of the integrity of the persistent ring buffer which may cause all data to be discarded. To avoid this issue, stop recording of the ring buffer on panic and flush the cache of the ring buffer's memory. Fixes: e645535a954a ("tracing: Add option to use memmapped memory for trace boot instance") Cc: stable@vger.kernel.org Cc: Will Deacon Cc: Mathieu Desnoyers Cc: Ian Rogers Link: https://patch.msgid.link/177751969602.2136606.12031934362587643488.stgit@mhiramat.tok.corp.google.com Signed-off-by: Masami Hiramatsu (Google) Acked-by: Catalin Marinas Acked-by: Geert Uytterhoeven Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index fcd93d49851e..7b07d2004cc6 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,7 @@ #include #include +#include #include #include #include @@ -559,6 +561,7 @@ struct trace_buffer { unsigned long range_addr_start; unsigned long range_addr_end; + struct notifier_block flush_nb; struct ring_buffer_meta *meta; @@ -2521,6 +2524,16 @@ static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer) kfree(cpu_buffer); } +/* Stop recording on a persistent buffer and flush cache if needed. */ +static int rb_flush_buffer_cb(struct notifier_block *nb, unsigned long event, void *data) +{ + struct trace_buffer *buffer = container_of(nb, struct trace_buffer, flush_nb); + + ring_buffer_record_off(buffer); + arch_ring_buffer_flush_range(buffer->range_addr_start, buffer->range_addr_end); + return NOTIFY_DONE; +} + static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, int order, unsigned long start, unsigned long end, @@ -2651,6 +2664,12 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, mutex_init(&buffer->mutex); + /* Persistent ring buffer needs to flush cache before reboot. */ + if (start && end) { + buffer->flush_nb.notifier_call = rb_flush_buffer_cb; + atomic_notifier_chain_register(&panic_notifier_list, &buffer->flush_nb); + } + return_ptr(buffer); fail_free_buffers: @@ -2749,6 +2768,9 @@ ring_buffer_free(struct trace_buffer *buffer) { int cpu; + if (buffer->range_addr_start && buffer->range_addr_end) + atomic_notifier_chain_unregister(&panic_notifier_list, &buffer->flush_nb); + cpuhp_state_remove_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node); irq_work_sync(&buffer->irq_work.work); -- cgit v1.2.3 From c2d2856cf6c9efccdf5e0d2564162ec616ce58cf Mon Sep 17 00:00:00 2001 From: David Carlier Date: Tue, 12 May 2026 14:54:20 +0100 Subject: tracing: Fix nr_subbufs initialization in simple_ring_buffer_init_mm() nr_subbufs in the ring buffer metadata is always initialized to zero because it is assigned from cpu_buffer->nr_pages before the page initialization loop has run. While nr_subbufs is not currently read by the kernel, it should reflect the actual buffer geometry in the meta page for correctness. Move the assignment after the page loop so that cpu_buffer->nr_pages holds the final count. Link: https://patch.msgid.link/20260512135420.99194-1-devnexen@gmail.com Fixes: 34e5b958bdad ("tracing: Introduce simple_ring_buffer") Reviewed-by: Vincent Donnefort Assisted-by: Claude:claude-opus-4-7 Signed-off-by: David Carlier Signed-off-by: Steven Rostedt --- kernel/trace/simple_ring_buffer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel/trace') diff --git a/kernel/trace/simple_ring_buffer.c b/kernel/trace/simple_ring_buffer.c index 02af2297ae5a..f731f14d0ff7 100644 --- a/kernel/trace/simple_ring_buffer.c +++ b/kernel/trace/simple_ring_buffer.c @@ -395,7 +395,6 @@ int simple_ring_buffer_init_mm(struct simple_rb_per_cpu *cpu_buffer, memset(cpu_buffer->meta, 0, sizeof(*cpu_buffer->meta)); cpu_buffer->meta->meta_page_size = PAGE_SIZE; - cpu_buffer->meta->nr_subbufs = cpu_buffer->nr_pages; /* The reader page is not part of the ring initially */ page = load_page(desc->page_va[0]); @@ -437,6 +436,7 @@ int simple_ring_buffer_init_mm(struct simple_rb_per_cpu *cpu_buffer, return ret; } + cpu_buffer->meta->nr_subbufs = cpu_buffer->nr_pages; /* Close the ring */ bpage->link.next = &cpu_buffer->tail_page->link; cpu_buffer->tail_page->link.prev = &bpage->link; -- cgit v1.2.3 From a0a2f42a37f90b29d8c43374dd9c8bd2f3e7bdcc Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Tue, 12 May 2026 15:16:14 +0100 Subject: tracing: Fix unload_page for simple_ring_buffer init rollback The unload_page callback expects the return value of load_page() as its argument: ret = load_page(va); unload(ret). Fix the rollback code in simple_ring_buffer_init_mm() where the descriptor's VA is used instead of the loaded page address. Link: https://patch.msgid.link/20260512141614.1759430-1-vdonnefort@google.com Fixes: 635923081c79 ("tracing: load/unload page callbacks for simple_ring_buffer") Signed-off-by: Vincent Donnefort Signed-off-by: Steven Rostedt --- kernel/trace/simple_ring_buffer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel/trace') diff --git a/kernel/trace/simple_ring_buffer.c b/kernel/trace/simple_ring_buffer.c index f731f14d0ff7..f4642f5adda3 100644 --- a/kernel/trace/simple_ring_buffer.c +++ b/kernel/trace/simple_ring_buffer.c @@ -430,7 +430,7 @@ int simple_ring_buffer_init_mm(struct simple_rb_per_cpu *cpu_buffer, if (ret) { for (i--; i >= 0; i--) - unload_page((void *)desc->page_va[i]); + unload_page(bpages[i].page); unload_page(cpu_buffer->meta); return ret; -- cgit v1.2.3 From 057caace5214da3b457bbd295e1a2ad34d3685ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Wed, 20 May 2026 20:01:55 +0200 Subject: tracing: Create output file from cmd_check_undefined MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As the output file is currently never created, the check will run every time, even if the inputs have not changed. Create an empty output file which allows make to skip the execution when it is not necessary. Cc: Masami Hiramatsu Cc: Mathieu Desnoyers Cc: Vincent Donnefort Cc: Marc Zyngier Cc: Arnd Bergmann Link: https://patch.msgid.link/20260520-tracing-ringbuffer-check-v1-1-d979cfab1338@weissschuh.net Fixes: 1211907ac0b5 ("tracing: Generate undef symbols allowlist for simple_ring_buffer") Fixes: 58b4bd18390e ("tracing: Adjust cmd_check_undefined to show unexpected undefined symbols") Reviewed-by: Nathan Chancellor Tested-by: Nathan Chancellor Signed-off-by: Thomas Weißschuh Signed-off-by: Steven Rostedt --- kernel/trace/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'kernel/trace') diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile index 9b0834134cae..8d3d96e847d8 100644 --- a/kernel/trace/Makefile +++ b/kernel/trace/Makefile @@ -154,7 +154,8 @@ quiet_cmd_check_undefined = NM $< echo "Unexpected symbols in $<:" >&2; \ echo "$$undefsyms" >&2; \ false; \ - fi + fi; \ + touch $@ $(obj)/%.o.checked: $(obj)/%.o $(obj)/undefsyms_base.o FORCE $(call if_changed,check_undefined) -- cgit v1.2.3 From 8f0f5c4fb9df0e19a341e0c6ed8dc4fda9124f03 Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Thu, 21 May 2026 13:49:14 +0900 Subject: tracing: Do not call map->ops->elt_free() if elt_alloc() fails In paths where tracing_map_elt_alloc() failed to allocate objects, the map->ops->elt_alloc() call was never successful. In this case, map->ops->elt_free() should not be called. Link: https://sashiko.dev/#/patchset/20260520223101.34710-1-rosenp%40gmail.com Cc: stable@vger.kernel.org Cc: Tom Zanussi Cc: Mathieu Desnoyers Cc: Rosen Penev Reported-by: Sashiko Fixes: 2734b629525a ("tracing: Add per-element variable support to tracing_map") Link: https://patch.msgid.link/177933895460.108746.5396070821443932634.stgit@devnote2 Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt --- kernel/trace/tracing_map.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/tracing_map.c b/kernel/trace/tracing_map.c index bf1a507695b6..0dd7927df22a 100644 --- a/kernel/trace/tracing_map.c +++ b/kernel/trace/tracing_map.c @@ -386,13 +386,11 @@ static void tracing_map_elt_init_fields(struct tracing_map_elt *elt) } } -static void tracing_map_elt_free(struct tracing_map_elt *elt) +static void __tracing_map_elt_free(struct tracing_map_elt *elt) { if (!elt) return; - if (elt->map->ops && elt->map->ops->elt_free) - elt->map->ops->elt_free(elt); kfree(elt->fields); kfree(elt->vars); kfree(elt->var_set); @@ -400,6 +398,17 @@ static void tracing_map_elt_free(struct tracing_map_elt *elt) kfree(elt); } +static void tracing_map_elt_free(struct tracing_map_elt *elt) +{ + if (!elt) + return; + + /* Only objects initialized with alloc_elt() should be passed to free_elt().*/ + if (elt->map->ops && elt->map->ops->elt_free) + elt->map->ops->elt_free(elt); + __tracing_map_elt_free(elt); +} + static struct tracing_map_elt *tracing_map_elt_alloc(struct tracing_map *map) { struct tracing_map_elt *elt; @@ -444,7 +453,7 @@ static struct tracing_map_elt *tracing_map_elt_alloc(struct tracing_map *map) } return elt; free: - tracing_map_elt_free(elt); + __tracing_map_elt_free(elt); return ERR_PTR(err); } -- cgit v1.2.3 From 3839a8eedde57a9f5b869025ce2626a7a37f5ccb Mon Sep 17 00:00:00 2001 From: Yash Suthar Date: Mon, 20 Apr 2026 15:42:36 +0530 Subject: tracing: Remove redundant IS_ERR() check in trace_pipe_open() in trace_pipe_open() already check the IS_ERR(iter) and return early on error,so iter after will be valid and it is safe to return 0 at end. Link: https://patch.msgid.link/20260420101236.223919-1-yashsuthar983@gmail.com Signed-off-by: Yash Suthar Signed-off-by: Steven Rostedt --- kernel/trace/trace_remote.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_remote.c b/kernel/trace/trace_remote.c index d6c3f94d67cd..2a6cc000ec98 100644 --- a/kernel/trace/trace_remote.c +++ b/kernel/trace/trace_remote.c @@ -602,7 +602,7 @@ static int trace_pipe_open(struct inode *inode, struct file *filp) filp->private_data = iter; - return IS_ERR(iter) ? PTR_ERR(iter) : 0; + return 0; } static int trace_pipe_release(struct inode *inode, struct file *filp) -- cgit v1.2.3 From f07883450eb14d1cf020b55d9f3a7ec5683bcd26 Mon Sep 17 00:00:00 2001 From: Pengpeng Hou Date: Thu, 30 Apr 2026 12:33:50 +0800 Subject: tracing: Bound synthetic-field strings with seq_buf The synthetic field helpers build a prefixed synthetic variable name and a generated hist command in fixed MAX_FILTER_STR_VAL buffers. The current code appends those strings with raw strcat(), so long key lists, field names, or saved filters can run past the end of the staging buffers. Build both strings with seq_buf and propagate -E2BIG if either the synthetic variable name or the generated command exceeds MAX_FILTER_STR_VAL. This keeps the existing tracing-side limit while using the helper intended for bounded command construction. Cc: Mathieu Desnoyers Cc: Tom Zanussi Link: https://patch.msgid.link/20260430043350.57928-1-pengpeng@iscas.ac.cn Fixes: 02205a6752f2 ("tracing: Add support for 'field variables'") Acked-by: Masami Hiramatsu (Google) Reviewed-by: Tom Zanussi Signed-off-by: Pengpeng Hou [ sdr: Moved struct seq_buf *s for upside-down x-mas tree formatting ] Signed-off-by: Steven Rostedt --- kernel/trace/trace_events_hist.c | 41 ++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index eb2c2bc8bc3d..9701650c89b2 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -2967,13 +2968,22 @@ find_synthetic_field_var(struct hist_trigger_data *target_hist_data, { struct hist_field *event_var; char *synthetic_name; + struct seq_buf s; synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); if (!synthetic_name) return ERR_PTR(-ENOMEM); - strcpy(synthetic_name, "synthetic_"); - strcat(synthetic_name, field_name); + seq_buf_init(&s, synthetic_name, MAX_FILTER_STR_VAL); + seq_buf_printf(&s, "synthetic_%s", field_name); + + /* Terminate synthetic_name with a NUL. */ + seq_buf_str(&s); + + if (seq_buf_has_overflowed(&s)) { + kfree(synthetic_name); + return ERR_PTR(-E2BIG); + } event_var = find_event_var(target_hist_data, system, event_name, synthetic_name); @@ -3019,6 +3029,7 @@ create_field_var_hist(struct hist_trigger_data *target_hist_data, struct hist_field *key_field; struct hist_field *event_var; char *saved_filter; + struct seq_buf s; char *cmd; int ret; @@ -3063,28 +3074,34 @@ create_field_var_hist(struct hist_trigger_data *target_hist_data, return ERR_PTR(-ENOMEM); } + seq_buf_init(&s, cmd, MAX_FILTER_STR_VAL); + /* Use the same keys as the compatible histogram */ - strcat(cmd, "keys="); + seq_buf_puts(&s, "keys="); for_each_hist_key_field(i, hist_data) { key_field = hist_data->fields[i]; if (!first) - strcat(cmd, ","); - strcat(cmd, key_field->field->name); + seq_buf_putc(&s, ','); + seq_buf_puts(&s, key_field->field->name); first = false; } /* Create the synthetic field variable specification */ - strcat(cmd, ":synthetic_"); - strcat(cmd, field_name); - strcat(cmd, "="); - strcat(cmd, field_name); + seq_buf_printf(&s, ":synthetic_%s=%s", field_name, field_name); /* Use the same filter as the compatible histogram */ saved_filter = find_trigger_filter(hist_data, file); - if (saved_filter) { - strcat(cmd, " if "); - strcat(cmd, saved_filter); + if (saved_filter) + seq_buf_printf(&s, " if %s", saved_filter); + + /* Terminate cmd with a NUL. */ + seq_buf_str(&s); + + if (seq_buf_has_overflowed(&s)) { + kfree(cmd); + kfree(var_hist); + return ERR_PTR(-E2BIG); } var_hist->cmd = kstrdup(cmd, GFP_KERNEL); -- cgit v1.2.3 From 8a6e9af2fac8cf212f86cff282ae96f6c7d7ca18 Mon Sep 17 00:00:00 2001 From: Yash Suthar Date: Sat, 2 May 2026 23:17:41 +0530 Subject: tracing: Switch trace_recursion_record.c code over to use guard() Switch mutex_lock()/mutex_unlock() to guard(). also drop the ret local variable and return directly. Link: https://patch.msgid.link/20260502174741.39636-1-yashsuthar983@gmail.com Signed-off-by: Yash Suthar Signed-off-by: Steven Rostedt --- kernel/trace/trace_recursion_record.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_recursion_record.c b/kernel/trace/trace_recursion_record.c index 784fe1fbb866..bac4bc844ccd 100644 --- a/kernel/trace/trace_recursion_record.c +++ b/kernel/trace/trace_recursion_record.c @@ -180,9 +180,8 @@ static const struct seq_operations recursed_function_seq_ops = { static int recursed_function_open(struct inode *inode, struct file *file) { - int ret = 0; + guard(mutex)(&recursed_function_lock); - mutex_lock(&recursed_function_lock); /* If this file was opened for write, then erase contents */ if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { /* disable updating records */ @@ -194,10 +193,9 @@ static int recursed_function_open(struct inode *inode, struct file *file) atomic_set(&nr_records, 0); } if (file->f_mode & FMODE_READ) - ret = seq_open(file, &recursed_function_seq_ops); - mutex_unlock(&recursed_function_lock); + return seq_open(file, &recursed_function_seq_ops); - return ret; + return 0; } static ssize_t recursed_function_write(struct file *file, -- cgit v1.2.3 From bb4613842e8033a714bacf3b62cc70638926cdcf Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Wed, 13 May 2026 15:00:07 -0400 Subject: tracing: Allow perf to read synthetic events Currently, perf can not enable synthetic events. When it does, it either causes a warning in the kernel or errors with "no such device". Add the necessary code to allow perf to also attach to synthetic events. Cc: Masami Hiramatsu Cc: Mathieu Desnoyers Cc: Arnaldo Carvalho de Melo Cc: Jiri Olsa Cc: Peter Zijlstra Link: https://patch.msgid.link/20260513150007.3b280e87@gandalf.local.home Reported-by: Ian Rogers Acked-by: Namhyung Kim Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace_events_synth.c | 121 +++++++++++++++++++++++++++++--------- 1 file changed, 94 insertions(+), 27 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_events_synth.c b/kernel/trace/trace_events_synth.c index 39ac4eba0702..e6871230bde9 100644 --- a/kernel/trace/trace_events_synth.c +++ b/kernel/trace/trace_events_synth.c @@ -499,28 +499,19 @@ static unsigned int trace_stack(struct synth_trace_event *entry, return len; } -static void trace_event_raw_event_synth(void *__data, - u64 *var_ref_vals, - unsigned int *var_ref_idx) +static __always_inline int get_field_size(struct synth_event *event, + u64 *var_ref_vals, + unsigned int *var_ref_idx) { - unsigned int i, n_u64, val_idx, len, data_size = 0; - struct trace_event_file *trace_file = __data; - struct synth_trace_event *entry; - struct trace_event_buffer fbuffer; - struct trace_buffer *buffer; - struct synth_event *event; - int fields_size = 0; - - event = trace_file->event_call->data; - - if (trace_trigger_soft_disabled(trace_file)) - return; + int fields_size; fields_size = event->n_u64 * sizeof(u64); - for (i = 0; i < event->n_dynamic_fields; i++) { + for (int i = 0; i < event->n_dynamic_fields; i++) { unsigned int field_pos = event->dynamic_fields[i]->field_pos; char *str_val; + int val_idx; + int len; val_idx = var_ref_idx[field_pos]; str_val = (char *)(long)var_ref_vals[val_idx]; @@ -535,18 +526,18 @@ static void trace_event_raw_event_synth(void *__data, fields_size += len; } + return fields_size; +} - /* - * Avoid ring buffer recursion detection, as this event - * is being performed within another event. - */ - buffer = trace_file->tr->array_buffer.buffer; - guard(ring_buffer_nest)(buffer); - - entry = trace_event_buffer_reserve(&fbuffer, trace_file, - sizeof(*entry) + fields_size); - if (!entry) - return; +static __always_inline void write_synth_entry(struct synth_event *event, + struct synth_trace_event *entry, + u64 *var_ref_vals, + unsigned int *var_ref_idx) +{ + int data_size = 0; + int i, n_u64; + int val_idx; + int len; for (i = 0, n_u64 = 0; i < event->n_fields; i++) { val_idx = var_ref_idx[i]; @@ -587,10 +578,83 @@ static void trace_event_raw_event_synth(void *__data, n_u64++; } } +} + +static void trace_event_raw_event_synth(void *__data, + u64 *var_ref_vals, + unsigned int *var_ref_idx) +{ + struct trace_event_file *trace_file = __data; + struct synth_trace_event *entry; + struct trace_event_buffer fbuffer; + struct trace_buffer *buffer; + struct synth_event *event; + int fields_size; + + event = trace_file->event_call->data; + + if (trace_trigger_soft_disabled(trace_file)) + return; + + fields_size = get_field_size(event, var_ref_vals, var_ref_idx); + + /* + * Avoid ring buffer recursion detection, as this event + * is being performed within another event. + */ + buffer = trace_file->tr->array_buffer.buffer; + guard(ring_buffer_nest)(buffer); + + entry = trace_event_buffer_reserve(&fbuffer, trace_file, + sizeof(*entry) + fields_size); + if (!entry) + return; + + write_synth_entry(event, entry, var_ref_vals, var_ref_idx); trace_event_buffer_commit(&fbuffer); } +#ifdef CONFIG_PERF_EVENTS +static void perf_event_raw_event_synth(void *__data, + u64 *var_ref_vals, + unsigned int *var_ref_idx) +{ + struct trace_event_call *call = __data; + struct synth_trace_event *entry; + struct hlist_head *perf_head; + struct synth_event *event; + struct pt_regs *regs; + int fields_size; + size_t size; + int context; + + event = call->data; + + perf_head = this_cpu_ptr(call->perf_events); + + if (!perf_head || hlist_empty(perf_head)) + return; + + fields_size = get_field_size(event, var_ref_vals, var_ref_idx); + + size = ALIGN(sizeof(*entry) + fields_size, 8); + + entry = perf_trace_buf_alloc(size, ®s, &context); + + if (unlikely(!entry)) + return; + + write_synth_entry(event, entry, var_ref_vals, var_ref_idx); + + perf_fetch_caller_regs(regs); + + perf_trace_buf_submit(entry, size, context, + call->event.type, 1, regs, + perf_head, NULL); +} +#endif + static void free_synth_event_print_fmt(struct trace_event_call *call) { if (call) { @@ -917,6 +981,9 @@ static int register_synth_event(struct synth_event *event) call->flags = TRACE_EVENT_FL_TRACEPOINT; call->class->reg = synth_event_reg; call->class->probe = trace_event_raw_event_synth; +#ifdef CONFIG_PERF_EVENTS + call->class->perf_probe = perf_event_raw_event_synth; +#endif call->data = event; call->tp = event->tp; -- cgit v1.2.3 From 5ec75d6a1b5b8beebea3bcaa85baad295ee09dcb Mon Sep 17 00:00:00 2001 From: Yu Peng Date: Tue, 19 May 2026 16:16:20 +0800 Subject: tracing/branch: Use pr_warn() instead of printk(KERN_WARNING) Use pr_warn() instead of printk(KERN_WARNING ...) for the branch tracer warning messages. Keep the message text unchanged. The change only removes the open-coded log level from these warnings. Cc: Masami Hiramatsu Cc: Mathieu Desnoyers Link: https://patch.msgid.link/20260519081620.3874441-1-pengyu@kylinos.cn Signed-off-by: Yu Peng Signed-off-by: Steven Rostedt --- kernel/trace/trace_branch.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_branch.c b/kernel/trace/trace_branch.c index d1564db95a8f..d8e97ad798f0 100644 --- a/kernel/trace/trace_branch.c +++ b/kernel/trace/trace_branch.c @@ -181,8 +181,7 @@ __init static int init_branch_tracer(void) ret = register_trace_event(&trace_branch_event); if (!ret) { - printk(KERN_WARNING "Warning: could not register " - "branch events\n"); + pr_warn("Warning: could not register branch events\n"); return 1; } return register_tracer(&branch_trace); @@ -374,8 +373,7 @@ __init static int init_annotated_branch_stats(void) ret = register_stat_tracer(&annotated_branch_stats); if (ret) { - printk(KERN_WARNING "Warning: could not register " - "annotated branches stats\n"); + pr_warn("Warning: could not register annotated branches stats\n"); return ret; } return 0; @@ -439,8 +437,7 @@ __init static int all_annotated_branch_stats(void) ret = register_stat_tracer(&all_branch_stats); if (ret) { - printk(KERN_WARNING "Warning: could not register " - "all branches stats\n"); + pr_warn("Warning: could not register all branches stats\n"); return ret; } return 0; -- cgit v1.2.3 From 153498f200d295f3efa6bd37fc6e7ff105ab344f Mon Sep 17 00:00:00 2001 From: Yu Peng Date: Tue, 19 May 2026 16:34:09 +0800 Subject: tracing: Use krealloc_array() for trace option array growth Use krealloc_array() when growing tr->topts instead of open-coding the size calculation in krealloc(). This makes the resize path use the helper intended for array allocations and avoids manual multiplication of the element count and element size. Cc: Masami Hiramatsu Cc: Mathieu Desnoyers Link: https://patch.msgid.link/20260519083409.3885032-1-pengyu@kylinos.cn Signed-off-by: Yu Peng Signed-off-by: Steven Rostedt --- kernel/trace/trace.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 6eb4d3097a4d..aec3f31ed027 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -7928,8 +7928,8 @@ create_trace_option_files(struct trace_array *tr, struct tracer *tracer, if (!topts) return 0; - tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1), - GFP_KERNEL); + tr_topts = krealloc_array(tr->topts, tr->nr_topts + 1, sizeof(*tr->topts), + GFP_KERNEL); if (!tr_topts) { kfree(topts); return -ENOMEM; -- cgit v1.2.3 From 3a5b29657593fe7695337dfcf40c49eb6fa544d4 Mon Sep 17 00:00:00 2001 From: Ao Sun Date: Thu, 21 May 2026 01:52:34 +0000 Subject: tracing: Fix README path for synthetic_events The events/ prefix should be removed, since synthetic_events is now directly under the tracing root directory. Cc: Cc: Link: https://patch.msgid.link/20260521015211.111-1-ao.sun@transsion.com Signed-off-by: Ao Sun Signed-off-by: Steven Rostedt --- kernel/trace/trace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index aec3f31ed027..4218c174460b 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -4474,7 +4474,7 @@ static const char readme_msg[] = "\t snapshot() - snapshot the trace buffer\n\n" #endif #ifdef CONFIG_SYNTH_EVENTS - " events/synthetic_events\t- Create/append/remove/show synthetic events\n" + " synthetic_events\t- Create/append/remove/show synthetic events\n" "\t Write into this file to define/undefine new synthetic events.\n" "\t example: echo 'myevent u64 lat; char name[]; long[] stack' >> synthetic_events\n" #endif -- cgit v1.2.3 From 817e148935dd76ce39bf468f51308ec77de87033 Mon Sep 17 00:00:00 2001 From: Rosen Penev Date: Wed, 20 May 2026 14:50:06 -0700 Subject: tracing: Simplify pages allocation for tracing_map logic Change to a flexible array member to allocate together with the array struct. Simplifies code slightly by removing no longer correct null checks for pages and removing kfrees. Cc: Mathieu Desnoyers Cc: Kees Cook Cc: "Gustavo A. R. Silva" Link: https://patch.msgid.link/20260520215006.12008-1-rosenp@gmail.com Signed-off-by: Rosen Penev Acked-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt --- kernel/trace/tracing_map.c | 30 +++++++++++------------------- kernel/trace/tracing_map.h | 2 +- 2 files changed, 12 insertions(+), 20 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/tracing_map.c b/kernel/trace/tracing_map.c index 0dd7927df22a..d7922f40dbe2 100644 --- a/kernel/trace/tracing_map.c +++ b/kernel/trace/tracing_map.c @@ -288,9 +288,6 @@ static void tracing_map_array_clear(struct tracing_map_array *a) { unsigned int i; - if (!a->pages) - return; - for (i = 0; i < a->n_pages; i++) memset(a->pages[i], 0, PAGE_SIZE); } @@ -302,9 +299,6 @@ static void tracing_map_array_free(struct tracing_map_array *a) if (!a) return; - if (!a->pages) - goto free; - for (i = 0; i < a->n_pages; i++) { if (!a->pages[i]) break; @@ -312,9 +306,6 @@ static void tracing_map_array_free(struct tracing_map_array *a) free_page((unsigned long)a->pages[i]); } - kfree(a->pages); - - free: kfree(a); } @@ -322,24 +313,25 @@ static struct tracing_map_array *tracing_map_array_alloc(unsigned int n_elts, unsigned int entry_size) { struct tracing_map_array *a; + unsigned int entry_size_shift; + unsigned int entries_per_page; + unsigned int n_pages; unsigned int i; - a = kzalloc_obj(*a); + entry_size_shift = fls(roundup_pow_of_two(entry_size) - 1); + entries_per_page = PAGE_SIZE / (1 << entry_size_shift); + n_pages = max(1, n_elts / entries_per_page); + + a = kzalloc_flex(*a, pages, n_pages); if (!a) return NULL; - a->entry_size_shift = fls(roundup_pow_of_two(entry_size) - 1); - a->entries_per_page = PAGE_SIZE / (1 << a->entry_size_shift); - a->n_pages = n_elts / a->entries_per_page; - if (!a->n_pages) - a->n_pages = 1; + a->entry_size_shift = entry_size_shift; + a->entries_per_page = entries_per_page; + a->n_pages = n_pages; a->entry_shift = fls(a->entries_per_page) - 1; a->entry_mask = (1 << a->entry_shift) - 1; - a->pages = kcalloc(a->n_pages, sizeof(void *), GFP_KERNEL); - if (!a->pages) - goto free; - for (i = 0; i < a->n_pages; i++) { a->pages[i] = (void *)get_zeroed_page(GFP_KERNEL); if (!a->pages[i]) diff --git a/kernel/trace/tracing_map.h b/kernel/trace/tracing_map.h index 99c37eeebc16..18a02959d77b 100644 --- a/kernel/trace/tracing_map.h +++ b/kernel/trace/tracing_map.h @@ -167,7 +167,7 @@ struct tracing_map_array { unsigned int entry_shift; unsigned int entry_mask; unsigned int n_pages; - void **pages; + void *pages[] __counted_by(n_pages); }; #define TRACING_MAP_ARRAY_ELT(array, idx) \ -- cgit v1.2.3 From 292c8197e3685aa7ea5a3803ca240b2210fcd021 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Thu, 21 May 2026 09:50:26 -0400 Subject: tracing: Move trace_iterator_increment() into trace_find_next_entry_inc() trace_iterator_increment() is only called from trace_find_next_entry_inc(). It's a small enough function that really doesn't need to be separated. Move the code from trace_iterator_increment() into trace_find_next_entry_inc() and remove trace_iterator_increment(). Cc: Masami Hiramatsu Cc: Mathieu Desnoyers Link: https://patch.msgid.link/20260521095026.20c9799d@gandalf.local.home Signed-off-by: Steven Rostedt --- kernel/trace/trace.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 4218c174460b..ae527c419508 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -2338,15 +2338,6 @@ void trace_last_func_repeats(struct trace_array *tr, __buffer_unlock_commit(buffer, event); } -static void trace_iterator_increment(struct trace_iterator *iter) -{ - struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu); - - iter->idx++; - if (buf_iter) - ring_buffer_iter_advance(buf_iter); -} - static struct trace_entry * peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts, unsigned long *lost_events) @@ -2676,11 +2667,17 @@ struct trace_entry *trace_find_next_entry(struct trace_iterator *iter, /* Find the next real entry, and increment the iterator to the next entry */ void *trace_find_next_entry_inc(struct trace_iterator *iter) { + struct ring_buffer_iter *buf_iter; + iter->ent = __find_next_entry(iter, &iter->cpu, &iter->lost_events, &iter->ts); - if (iter->ent) - trace_iterator_increment(iter); + if (iter->ent) { + iter->idx++; + buf_iter = trace_buffer_iter(iter, iter->cpu); + if (buf_iter) + ring_buffer_iter_advance(buf_iter); + } return iter->ent ? iter : NULL; } -- cgit v1.2.3 From eb3bd277b37cd435d26a44a40d8f7c87ff16feb6 Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Fri, 22 May 2026 13:08:58 -0400 Subject: ring-buffer: Skip invalid sub-buffers when validating persistent ring buffer Skip invalid sub-buffers when validating the persistent ring buffer instead of discarding the entire ring buffer. Only skipped buffers are invalidated (cleared). If the cache data in memory fails to be synchronized during a reboot, the persistent ring buffer may become partially corrupted, but other sub-buffers may still contain readable event data. Only discard the subbuffers that are found to be corrupted. Link: https://lore.kernel.org/all/20260520185018.051228084@kernel.org/ Link: https://patch.msgid.link/20260522171050.914418536@kernel.org Signed-off-by: Masami Hiramatsu (Google) [SDR: Fixed max_loops in rb_iter_peek() as well ] Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 120 +++++++++++++++++++++++++++------------------ 1 file changed, 73 insertions(+), 47 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 7b07d2004cc6..6afd8ea5081a 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -370,6 +370,12 @@ static __always_inline unsigned int rb_page_commit(struct buffer_page *bpage) return local_read(&bpage->page->commit); } +/* Size is determined by what has been committed */ +static __always_inline unsigned int rb_page_size(struct buffer_page *bpage) +{ + return rb_page_commit(bpage) & ~RB_MISSED_MASK; +} + static void free_buffer_page(struct buffer_page *bpage) { /* Range pages are not to be freed */ @@ -1762,7 +1768,6 @@ static bool rb_cpu_meta_valid(struct ring_buffer_cpu_meta *meta, int cpu, unsigned long *subbuf_mask) { int subbuf_size = PAGE_SIZE; - struct buffer_data_page *subbuf; unsigned long buffers_start; unsigned long buffers_end; int i; @@ -1770,6 +1775,11 @@ static bool rb_cpu_meta_valid(struct ring_buffer_cpu_meta *meta, int cpu, if (!subbuf_mask) return false; + if (meta->subbuf_size != PAGE_SIZE) { + pr_info("Ring buffer boot meta [%d] invalid subbuf_size\n", cpu); + return false; + } + buffers_start = meta->first_buffer; buffers_end = meta->first_buffer + (subbuf_size * meta->nr_subbufs); @@ -1786,11 +1796,12 @@ static bool rb_cpu_meta_valid(struct ring_buffer_cpu_meta *meta, int cpu, return false; } - subbuf = rb_subbufs_from_meta(meta); - bitmap_clear(subbuf_mask, 0, meta->nr_subbufs); - /* Is the meta buffers and the subbufs themselves have correct data? */ + /* + * Ensure the meta::buffers array has correct data. The data in each subbufs + * are checked later in rb_meta_validate_events(). + */ for (i = 0; i < meta->nr_subbufs; i++) { if (meta->buffers[i] < 0 || meta->buffers[i] >= meta->nr_subbufs) { @@ -1798,18 +1809,12 @@ static bool rb_cpu_meta_valid(struct ring_buffer_cpu_meta *meta, int cpu, return false; } - if ((unsigned)local_read(&subbuf->commit) > subbuf_size) { - pr_info("Ring buffer boot meta [%d] buffer invalid commit\n", cpu); - return false; - } - if (test_bit(meta->buffers[i], subbuf_mask)) { pr_info("Ring buffer boot meta [%d] array has duplicates\n", cpu); return false; } set_bit(meta->buffers[i], subbuf_mask); - subbuf = (void *)subbuf + subbuf_size; } return true; @@ -1873,13 +1878,22 @@ static int rb_read_data_buffer(struct buffer_data_page *dpage, int tail, int cpu return events; } -static int rb_validate_buffer(struct buffer_data_page *dpage, int cpu) +static int rb_validate_buffer(struct buffer_data_page *dpage, int cpu, + struct ring_buffer_cpu_meta *meta) { unsigned long long ts; + unsigned long tail; u64 delta; - int tail; - tail = local_read(&dpage->commit); + /* + * When a sub-buffer is recovered from a read, the commit value may + * have RB_MISSED_* bits set, as these bits are reset on reuse. + * Even after clearing these bits, a commit value greater than the + * subbuf_size is considered invalid. + */ + tail = local_read(&dpage->commit) & ~RB_MISSED_MASK; + if (tail > meta->subbuf_size - BUF_PAGE_HDR_SIZE) + return -1; return rb_read_data_buffer(dpage, tail, cpu, &ts, &delta); } @@ -1890,6 +1904,7 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) struct buffer_page *head_page, *orig_head, *orig_reader; unsigned long entry_bytes = 0; unsigned long entries = 0; + int discarded = 0; int ret; u64 ts; int i; @@ -1901,14 +1916,19 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) orig_reader = cpu_buffer->reader_page; /* Do the reader page first */ - ret = rb_validate_buffer(orig_reader->page, cpu_buffer->cpu); + ret = rb_validate_buffer(orig_reader->page, cpu_buffer->cpu, meta); if (ret < 0) { - pr_info("Ring buffer reader page is invalid\n"); - goto invalid; + pr_info("Ring buffer meta [%d] invalid reader page detected\n", + cpu_buffer->cpu); + discarded++; + /* Instead of discard whole ring buffer, discard only this sub-buffer. */ + local_set(&orig_reader->entries, 0); + local_set(&orig_reader->page->commit, 0); + } else { + entries += ret; + entry_bytes += rb_page_size(orig_reader); + local_set(&orig_reader->entries, ret); } - entries += ret; - entry_bytes += local_read(&orig_reader->page->commit); - local_set(&orig_reader->entries, ret); ts = head_page->page->time_stamp; @@ -1936,7 +1956,7 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) break; /* Stop rewind if the page is invalid. */ - ret = rb_validate_buffer(head_page->page, cpu_buffer->cpu); + ret = rb_validate_buffer(head_page->page, cpu_buffer->cpu, meta); if (ret < 0) break; @@ -1945,7 +1965,7 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) if (ret) local_inc(&cpu_buffer->pages_touched); entries += ret; - entry_bytes += rb_page_commit(head_page); + entry_bytes += rb_page_size(head_page); } if (i) pr_info("Ring buffer [%d] rewound %d pages\n", cpu_buffer->cpu, i); @@ -2015,21 +2035,24 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) if (head_page == orig_reader) continue; - ret = rb_validate_buffer(head_page->page, cpu_buffer->cpu); + ret = rb_validate_buffer(head_page->page, cpu_buffer->cpu, meta); if (ret < 0) { - pr_info("Ring buffer meta [%d] invalid buffer page\n", - cpu_buffer->cpu); - goto invalid; - } - - /* If the buffer has content, update pages_touched */ - if (ret) - local_inc(&cpu_buffer->pages_touched); - - entries += ret; - entry_bytes += local_read(&head_page->page->commit); - local_set(&head_page->entries, ret); + if (!discarded) + pr_info("Ring buffer meta [%d] invalid buffer page detected\n", + cpu_buffer->cpu); + discarded++; + /* Instead of discard whole ring buffer, discard only this sub-buffer. */ + local_set(&head_page->entries, 0); + local_set(&head_page->page->commit, 0); + } else { + /* If the buffer has content, update pages_touched */ + if (ret) + local_inc(&cpu_buffer->pages_touched); + entries += ret; + entry_bytes += rb_page_size(head_page); + local_set(&head_page->entries, ret); + } if (head_page == cpu_buffer->commit_page) break; } @@ -2043,7 +2066,10 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) local_set(&cpu_buffer->entries, entries); local_set(&cpu_buffer->entries_bytes, entry_bytes); - pr_info("Ring buffer meta [%d] is from previous boot!\n", cpu_buffer->cpu); + pr_info("Ring buffer meta [%d] is from previous boot!", cpu_buffer->cpu); + if (discarded) + pr_cont(" (%d pages discarded)", discarded); + pr_cont("\n"); return; invalid: @@ -3330,12 +3356,6 @@ rb_iter_head_event(struct ring_buffer_iter *iter) return NULL; } -/* Size is determined by what has been committed */ -static __always_inline unsigned rb_page_size(struct buffer_page *bpage) -{ - return rb_page_commit(bpage) & ~RB_MISSED_MASK; -} - static __always_inline unsigned rb_commit_index(struct ring_buffer_per_cpu *cpu_buffer) { @@ -5636,8 +5656,9 @@ __rb_get_reader_page_from_remote(struct ring_buffer_per_cpu *cpu_buffer) static struct buffer_page * __rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) { - struct buffer_page *reader = NULL; + int max_loops = cpu_buffer->ring_meta ? cpu_buffer->nr_pages : 3; unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size); + struct buffer_page *reader = NULL; unsigned long overwrite; unsigned long flags; int nr_loops = 0; @@ -5649,11 +5670,14 @@ __rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) again: /* * This should normally only loop twice. But because the - * start of the reader inserts an empty page, it causes - * a case where we will loop three times. There should be no - * reason to loop four times (that I know of). + * start of the reader inserts an empty page, it causes a + * case where we will loop three times. There should be no + * reason to loop four times unless the ring buffer is a + * recovered persistent ring buffer. For persistent ring buffers, + * invalid pages are reset during recovery, so there may be more + * than 3 contiguous pages can be empty, but less than nr_pages. */ - if (RB_WARN_ON(cpu_buffer, ++nr_loops > 3)) { + if (RB_WARN_ON(cpu_buffer, ++nr_loops > max_loops)) { reader = NULL; goto out; } @@ -5950,12 +5974,14 @@ rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts) struct ring_buffer_per_cpu *cpu_buffer; struct ring_buffer_event *event; int nr_loops = 0; + int max_loops; if (ts) *ts = 0; cpu_buffer = iter->cpu_buffer; buffer = cpu_buffer->buffer; + max_loops = cpu_buffer->ring_meta ? cpu_buffer->nr_pages : 3; /* * Check if someone performed a consuming read to the buffer @@ -5978,7 +6004,7 @@ rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts) * the ring buffer with an active write as the consumer is. * Do not warn if the three failures is reached. */ - if (++nr_loops > 3) + if (++nr_loops > max_loops) return NULL; if (rb_per_cpu_empty(cpu_buffer)) -- cgit v1.2.3 From c8a7d4b4723a21e7464efe86dcf80627e0b4df33 Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Fri, 22 May 2026 13:08:59 -0400 Subject: ring-buffer: Skip invalid sub-buffers when rewinding persistent ring buffer Skip invalid sub-buffers when rewinding the persistent ring buffer instead of stopping the rewinding the ring buffer. The skipped buffers are cleared. To ensure the rewinding stops at the unused page, this also clears buffer_data_page::time_stamp when tracing resets the buffer. This allows us to identify unused pages and empty pages. Link: https://patch.msgid.link/20260522171051.091265852@kernel.org Signed-off-by: Masami Hiramatsu (Google) [ SDR: Have reader_page still get evaluated if header_page fails ] Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 107 +++++++++++++++++++++++++++++---------------- 1 file changed, 69 insertions(+), 38 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 6afd8ea5081a..c10cf4ba91d6 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -363,6 +363,7 @@ struct buffer_page { static void rb_init_page(struct buffer_data_page *bpage) { local_set(&bpage->commit, 0); + bpage->time_stamp = 0; } static __always_inline unsigned int rb_page_commit(struct buffer_page *bpage) @@ -1878,12 +1879,14 @@ static int rb_read_data_buffer(struct buffer_data_page *dpage, int tail, int cpu return events; } -static int rb_validate_buffer(struct buffer_data_page *dpage, int cpu, - struct ring_buffer_cpu_meta *meta) +static int rb_validate_buffer(struct buffer_page *bpage, int cpu, + struct ring_buffer_cpu_meta *meta, u64 prev_ts, u64 next_ts) { + struct buffer_data_page *dpage = bpage->page; unsigned long long ts; unsigned long tail; u64 delta; + int ret; /* * When a sub-buffer is recovered from a read, the commit value may @@ -1892,9 +1895,27 @@ static int rb_validate_buffer(struct buffer_data_page *dpage, int cpu, * subbuf_size is considered invalid. */ tail = local_read(&dpage->commit) & ~RB_MISSED_MASK; - if (tail > meta->subbuf_size - BUF_PAGE_HDR_SIZE) - return -1; - return rb_read_data_buffer(dpage, tail, cpu, &ts, &delta); + if (tail <= meta->subbuf_size - BUF_PAGE_HDR_SIZE) + ret = rb_read_data_buffer(dpage, tail, cpu, &ts, &delta); + else + ret = -1; + + /* + * The timestamp must be greater than @prev_ts and smaller than @next_ts. + * Since this function works in both forward (verify) and reverse (unwind) + * loop, we don't know both @prev_ts and @next_ts at the same time. + * So use the known boundary as the boundary. + */ + if (ret < 0 || (prev_ts && prev_ts > ts) || (next_ts && ts > next_ts)) { + local_set(&bpage->entries, 0); + local_set(&dpage->commit, 0); + dpage->time_stamp = prev_ts ? prev_ts : next_ts; + ret = -1; + } else { + local_set(&bpage->entries, ret); + } + + return ret; } /* If the meta data has been validated, now validate the events */ @@ -1905,6 +1926,7 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) unsigned long entry_bytes = 0; unsigned long entries = 0; int discarded = 0; + bool skip = false; int ret; u64 ts; int i; @@ -1915,25 +1937,35 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) orig_head = head_page = cpu_buffer->head_page; orig_reader = cpu_buffer->reader_page; - /* Do the reader page first */ - ret = rb_validate_buffer(orig_reader->page, cpu_buffer->cpu, meta); + /* Do the head page first */ + ret = rb_validate_buffer(head_page, cpu_buffer->cpu, meta, 0, 0); + if (ret < 0) { + pr_info("Ring buffer meta [%d] invalid head page detected\n", + cpu_buffer->cpu); + /* Don't bother rewinding */ + skip = true; + ts = 0; + } else { + ts = head_page->page->time_stamp; + } + + /* Do the reader page - reader must be previous to head. */ + ret = rb_validate_buffer(orig_reader, cpu_buffer->cpu, meta, 0, ts); if (ret < 0) { pr_info("Ring buffer meta [%d] invalid reader page detected\n", cpu_buffer->cpu); discarded++; - /* Instead of discard whole ring buffer, discard only this sub-buffer. */ - local_set(&orig_reader->entries, 0); - local_set(&orig_reader->page->commit, 0); } else { entries += ret; entry_bytes += rb_page_size(orig_reader); - local_set(&orig_reader->entries, ret); + ts = orig_reader->page->time_stamp; } - ts = head_page->page->time_stamp; + if (skip) + goto skip_rewind; /* - * Try to rewind the head so that we can read the pages which already + * Try to rewind the head so that we can read the pages which are already * read in the previous boot. */ if (head_page == cpu_buffer->tail_page) @@ -1946,26 +1978,27 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) if (head_page == cpu_buffer->tail_page) break; - /* Ensure the page has older data than head. */ - if (ts < head_page->page->time_stamp) + /* Rewind until unused page (no timestamp, no commit). */ + if (!head_page->page->time_stamp && rb_page_commit(head_page) == 0) break; - ts = head_page->page->time_stamp; - /* Ensure the page has correct timestamp and some data. */ - if (!ts || rb_page_commit(head_page) == 0) - break; - - /* Stop rewind if the page is invalid. */ - ret = rb_validate_buffer(head_page->page, cpu_buffer->cpu, meta); - if (ret < 0) - break; - - /* Recover the number of entries and update stats. */ - local_set(&head_page->entries, ret); - if (ret) - local_inc(&cpu_buffer->pages_touched); - entries += ret; - entry_bytes += rb_page_size(head_page); + /* + * Skip if the page is invalid, or its timestamp is newer than the + * previous valid page. + */ + ret = rb_validate_buffer(head_page, cpu_buffer->cpu, meta, 0, ts); + if (ret < 0) { + if (!discarded) + pr_info("Ring buffer meta [%d] invalid buffer page detected\n", + cpu_buffer->cpu); + discarded++; + } else { + entries += ret; + entry_bytes += rb_page_size(head_page); + if (ret > 0) + local_inc(&cpu_buffer->pages_touched); + ts = head_page->page->time_stamp; + } } if (i) pr_info("Ring buffer [%d] rewound %d pages\n", cpu_buffer->cpu, i); @@ -2027,6 +2060,7 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) /* Nothing more to do, the only page is the reader page */ goto done; } + ts = head_page->page->time_stamp; /* Iterate until finding the commit page */ for (i = 0; i < meta->nr_subbufs + 1; i++, rb_inc_page(&head_page)) { @@ -2035,15 +2069,12 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) if (head_page == orig_reader) continue; - ret = rb_validate_buffer(head_page->page, cpu_buffer->cpu, meta); + ret = rb_validate_buffer(head_page, cpu_buffer->cpu, meta, ts, 0); if (ret < 0) { if (!discarded) pr_info("Ring buffer meta [%d] invalid buffer page detected\n", cpu_buffer->cpu); discarded++; - /* Instead of discard whole ring buffer, discard only this sub-buffer. */ - local_set(&head_page->entries, 0); - local_set(&head_page->page->commit, 0); } else { /* If the buffer has content, update pages_touched */ if (ret) @@ -2051,7 +2082,7 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) entries += ret; entry_bytes += rb_page_size(head_page); - local_set(&head_page->entries, ret); + ts = head_page->page->time_stamp; } if (head_page == cpu_buffer->commit_page) break; @@ -2079,12 +2110,12 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) /* Reset the reader page */ local_set(&cpu_buffer->reader_page->entries, 0); - local_set(&cpu_buffer->reader_page->page->commit, 0); + rb_init_page(cpu_buffer->reader_page->page); /* Reset all the subbuffers */ for (i = 0; i < meta->nr_subbufs - 1; i++, rb_inc_page(&head_page)) { local_set(&head_page->entries, 0); - local_set(&head_page->page->commit, 0); + rb_init_page(head_page->page); } } -- cgit v1.2.3 From 68413a36f0051bd7ba7ea37ae2167951aeae5bd2 Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Fri, 22 May 2026 13:09:00 -0400 Subject: ring-buffer: Add persistent ring buffer invalid-page inject test Add a self-corrupting test for the persistent ring buffer. This will inject an erroneous value to some sub-buffer pages (where the index is even or multiples of 5) in the persistent ring buffer when the kernel panics, and checks whether the number of detected invalid pages and the total entry_bytes are the same as the recorded values after reboot. This ensures that the kernel can correctly recover a partially corrupted persistent ring buffer after a reboot or panic. The test only runs on the persistent ring buffer whose name is "ptracingtest". The user has to fill it with events before a kernel panic. To run the test, enable CONFIG_RING_BUFFER_PERSISTENT_INJECT and add the following kernel cmdline: reserve_mem=20M:2M:trace trace_instance=ptracingtest^traceoff@trace panic=1 Run the following commands after the 1st boot: cd /sys/kernel/tracing/instances/ptracingtest echo 1 > tracing_on echo 1 > events/enable sleep 3 echo c > /proc/sysrq-trigger After panic message, the kernel will reboot and run the verification on the persistent ring buffer, e.g. Ring buffer meta [2] invalid buffer page detected Ring buffer meta [2] is from previous boot! (318 pages discarded) Ring buffer testing [2] invalid pages: PASSED (318/318) Ring buffer testing [2] entry_bytes: PASSED (1300476/1300476) Link: https://patch.msgid.link/20260522171051.260140328@kernel.org Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt --- kernel/trace/Kconfig | 34 ++++++++++++++++++++ kernel/trace/ring_buffer.c | 79 ++++++++++++++++++++++++++++++++++++++++++++++ kernel/trace/trace.c | 4 +++ 3 files changed, 117 insertions(+) (limited to 'kernel/trace') diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index e130da35808f..084f34dc6c9f 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig @@ -1202,6 +1202,40 @@ config RING_BUFFER_VALIDATE_TIME_DELTAS Only say Y if you understand what this does, and you still want it enabled. Otherwise say N +config RING_BUFFER_PERSISTENT_INJECT + bool "Enable persistent ring buffer error injection test" + depends on RING_BUFFER + help + This option will have the kernel check if the persistent ring + buffer is named "ptracingtest". and if so, it will corrupt some + of its pages on a kernel panic. This is used to test if the + persistent ring buffer can recover from some of its sub-buffers + being corrupted. + To use this, boot a kernel with a "ptracingtest" persistent + ring buffer, e.g. + + reserve_mem=20M:2M:trace trace_instance=ptracingtest@trace panic=1 + + And after the 1st boot, run the following commands: + + cd /sys/kernel/tracing/instances/ptracingtest + echo 1 > events/enable + echo 1 > tracing_on + sleep 3 + echo c > /proc/sysrq-trigger + + After the panic message, the kernel will reboot and will show + the test results in the console output. + + Note that events for the test ring buffer needs to be enabled + prior to crashing the kernel so that the ring buffer has content + that the test will corrupt. + As the test will corrupt events in the "ptracingtest" persistent + ring buffer, it should not be used for any other purpose other + than this test. + + If unsure, say N + config MMIOTRACE_TEST tristate "Test module for mmiotrace" depends on MMIOTRACE && m diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index c10cf4ba91d6..dc603d9c9414 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -64,6 +64,10 @@ struct ring_buffer_cpu_meta { unsigned long commit_buffer; __u32 subbuf_size; __u32 nr_subbufs; +#ifdef CONFIG_RING_BUFFER_PERSISTENT_INJECT + __u32 nr_invalid; + __u32 entry_bytes; +#endif int buffers[]; }; @@ -2101,6 +2105,21 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) if (discarded) pr_cont(" (%d pages discarded)", discarded); pr_cont("\n"); + +#ifdef CONFIG_RING_BUFFER_PERSISTENT_INJECT + if (meta->nr_invalid) + pr_warn("Ring buffer testing [%d] invalid pages: %s (%d/%d)\n", + cpu_buffer->cpu, + (discarded == meta->nr_invalid) ? "PASSED" : "FAILED", + discarded, meta->nr_invalid); + if (meta->entry_bytes) + pr_warn("Ring buffer testing [%d] entry_bytes: %s (%ld/%ld)\n", + cpu_buffer->cpu, + (entry_bytes == meta->entry_bytes) ? "PASSED" : "FAILED", + (long)entry_bytes, (long)meta->entry_bytes); + meta->nr_invalid = 0; + meta->entry_bytes = 0; +#endif return; invalid: @@ -2581,12 +2600,72 @@ static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer) kfree(cpu_buffer); } +#ifdef CONFIG_RING_BUFFER_PERSISTENT_INJECT +static void rb_test_inject_invalid_pages(struct trace_buffer *buffer) +{ + struct ring_buffer_per_cpu *cpu_buffer; + struct ring_buffer_cpu_meta *meta; + struct buffer_data_page *dpage; + unsigned long entry_bytes = 0; + unsigned long ptr; + int subbuf_size; + int invalid = 0; + int cpu; + int i; + + if (!(buffer->flags & RB_FL_TESTING)) + return; + + guard(preempt)(); + cpu = smp_processor_id(); + + cpu_buffer = buffer->buffers[cpu]; + if (!cpu_buffer) + return; + meta = cpu_buffer->ring_meta; + if (!meta) + return; + + ptr = (unsigned long)rb_subbufs_from_meta(meta); + subbuf_size = meta->subbuf_size; + + for (i = 0; i < meta->nr_subbufs; i++) { + unsigned long idx = meta->buffers[i]; + + dpage = (void *)(ptr + idx * subbuf_size); + /* Skip unused pages */ + if (!local_read(&dpage->commit)) + continue; + + /* + * Invalidate even pages or multiples of 5. This will cause 3 + * contiguous invalidated(empty) pages. + */ + if (!(i & 0x1) || !(i % 5)) { + local_add(subbuf_size + 1, &dpage->commit); + invalid++; + } else { + /* Count total commit bytes. */ + entry_bytes += local_read(&dpage->commit) & ~RB_MISSED_MASK; + } + } + + pr_info("Inject invalidated %d pages on CPU%d, total size: %ld\n", + invalid, cpu, (long)entry_bytes); + meta->nr_invalid = invalid; + meta->entry_bytes = entry_bytes; +} +#else /* !CONFIG_RING_BUFFER_PERSISTENT_INJECT */ +#define rb_test_inject_invalid_pages(buffer) do { } while (0) +#endif + /* Stop recording on a persistent buffer and flush cache if needed. */ static int rb_flush_buffer_cb(struct notifier_block *nb, unsigned long event, void *data) { struct trace_buffer *buffer = container_of(nb, struct trace_buffer, flush_nb); ring_buffer_record_off(buffer); + rb_test_inject_invalid_pages(buffer); arch_ring_buffer_flush_range(buffer->range_addr_start, buffer->range_addr_end); return NOTIFY_DONE; } diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 6eb4d3097a4d..4573f65d68ce 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -8383,6 +8383,8 @@ static void setup_trace_scratch(struct trace_array *tr, memset(tscratch, 0, size); } +#define TRACE_TEST_PTRACING_NAME "ptracingtest" + int allocate_trace_buffer(struct trace_array *tr, struct array_buffer *buf, int size) { enum ring_buffer_flags rb_flags; @@ -8394,6 +8396,8 @@ int allocate_trace_buffer(struct trace_array *tr, struct array_buffer *buf, int buf->tr = tr; if (tr->range_addr_start && tr->range_addr_size) { + if (tr->name && !strcmp(tr->name, TRACE_TEST_PTRACING_NAME)) + rb_flags |= RB_FL_TESTING; /* Add scratch buffer to handle 128 modules */ buf->buffer = ring_buffer_alloc_range(size, rb_flags, 0, tr->range_addr_start, -- cgit v1.2.3 From e500acc34b556df7774213d95ff130801b9d6512 Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Fri, 22 May 2026 13:09:01 -0400 Subject: ring-buffer: Show commit numbers in buffer_meta file In addition to the index number, show the commit numbers of each data page in the per_cpu buffer_meta file. This is useful for understanding the current status of the persistent ring buffer. (Note that this file is shown only for persistent ring buffer and its backup instance) Link: https://patch.msgid.link/20260522171051.424411323@kernel.org Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index dc603d9c9414..88e613e78632 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -2231,6 +2231,7 @@ static int rbm_show(struct seq_file *m, void *v) struct ring_buffer_per_cpu *cpu_buffer = m->private; struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta; unsigned long val = (unsigned long)v; + struct buffer_data_page *dpage; if (val == 1) { seq_printf(m, "head_buffer: %d\n", @@ -2243,7 +2244,9 @@ static int rbm_show(struct seq_file *m, void *v) } val -= 2; - seq_printf(m, "buffer[%ld]: %d\n", val, meta->buffers[val]); + dpage = rb_range_buffer(cpu_buffer, val); + seq_printf(m, "buffer[%ld]: %d (commit: %ld)\n", + val, meta->buffers[val], dpage ? local_read(&dpage->commit) : -1); return 0; } -- cgit v1.2.3 From ea250b7ef9ff2af4241b4d7f078594e51d155b81 Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Fri, 22 May 2026 13:09:02 -0400 Subject: ring-buffer: Cleanup persistent ring buffer validation Cleanup rb_meta_validate_events() function to make it easier to read. This includes the following cleanups: - Introduce rb_validatation_state to hold working variables in validation. - Move repleated validation state updates into rb_validate_buffer(). - Move reader_page injection code outside of rb_meta_validate_events(). Link: https://patch.msgid.link/20260522171051.577231395@kernel.org Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 200 ++++++++++++++++++++++++--------------------- 1 file changed, 108 insertions(+), 92 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 88e613e78632..73f453ba12ce 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -1883,8 +1883,16 @@ static int rb_read_data_buffer(struct buffer_data_page *dpage, int tail, int cpu return events; } -static int rb_validate_buffer(struct buffer_page *bpage, int cpu, - struct ring_buffer_cpu_meta *meta, u64 prev_ts, u64 next_ts) +struct rb_validation_state { + unsigned long entries; + unsigned long entry_bytes; + int discarded; + u64 ts; +}; + +static int __rb_validate_buffer(struct buffer_page *bpage, int cpu, + struct ring_buffer_cpu_meta *meta, + u64 prev_ts, u64 next_ts) { struct buffer_data_page *dpage = bpage->page; unsigned long long ts; @@ -1922,17 +1930,95 @@ static int rb_validate_buffer(struct buffer_page *bpage, int cpu, return ret; } +/** + * rb_validate_buffer - validates a single buffer page and updates the state. + * @bpage: buffer page to validate + * @cpu_buffer: cpu_buffer this page belongs to + * @meta: meta of the cpu_buffer + * @state: validation state + * @prev_ts: previous buffer's timestamp (optional) + * @next_ts: next buffer's timestamp (optional) + * + * If the page is invalid (wrong event length or timestamp), it increments the + * discarded counter and warns it. Otherwise, it updates the validation state. + */ +static void rb_validate_buffer(struct buffer_page *bpage, + struct ring_buffer_per_cpu *cpu_buffer, + struct ring_buffer_cpu_meta *meta, + struct rb_validation_state *state, + u64 prev_ts, u64 next_ts) +{ + int ret; + + ret = __rb_validate_buffer(bpage, cpu_buffer->cpu, meta, prev_ts, next_ts); + if (ret < 0) { + if (!state->discarded) + pr_info("Ring buffer meta [%d] invalid buffer page detected\n", + cpu_buffer->cpu); + state->discarded++; + } else { + /* If the buffer has content, update pages_touched */ + if (ret) + local_inc(&cpu_buffer->pages_touched); + + state->entries += ret; + state->entry_bytes += rb_page_size(bpage); + state->ts = bpage->page->time_stamp; + } +} + +static void rb_meta_inject_reader_page(struct ring_buffer_per_cpu *cpu_buffer, + struct ring_buffer_cpu_meta *meta, + struct buffer_page *orig_head, + struct buffer_page *head_page) +{ + struct buffer_page *bpage = orig_head; + int i; + + rb_dec_page(&bpage); + /* + * Insert the reader_page before the original head page. + * Since the list encode RB_PAGE flags, general list + * operations should be avoided. + */ + cpu_buffer->reader_page->list.next = &orig_head->list; + cpu_buffer->reader_page->list.prev = orig_head->list.prev; + orig_head->list.prev = &cpu_buffer->reader_page->list; + bpage->list.next = &cpu_buffer->reader_page->list; + + /* Make the head_page the reader page */ + cpu_buffer->reader_page = head_page; + bpage = head_page; + rb_inc_page(&head_page); + head_page->list.prev = bpage->list.prev; + rb_dec_page(&bpage); + bpage->list.next = &head_page->list; + rb_set_list_to_head(&bpage->list); + cpu_buffer->pages = &head_page->list; + + cpu_buffer->head_page = head_page; + meta->head_buffer = (unsigned long)head_page->page; + + /* Reset all the indexes */ + bpage = cpu_buffer->reader_page; + meta->buffers[0] = rb_meta_subbuf_idx(meta, bpage->page); + bpage->id = 0; + + for (i = 1, bpage = head_page; i < meta->nr_subbufs; + i++, rb_inc_page(&bpage)) { + meta->buffers[i] = rb_meta_subbuf_idx(meta, bpage->page); + bpage->id = i; + } +} + /* If the meta data has been validated, now validate the events */ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) { struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta; struct buffer_page *head_page, *orig_head, *orig_reader; - unsigned long entry_bytes = 0; - unsigned long entries = 0; - int discarded = 0; + struct rb_validation_state state = { 0 }; bool skip = false; int ret; - u64 ts; int i; if (!meta || !meta->head_buffer) @@ -1942,28 +2028,19 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) orig_reader = cpu_buffer->reader_page; /* Do the head page first */ - ret = rb_validate_buffer(head_page, cpu_buffer->cpu, meta, 0, 0); + ret = __rb_validate_buffer(head_page, cpu_buffer->cpu, meta, 0, 0); if (ret < 0) { pr_info("Ring buffer meta [%d] invalid head page detected\n", cpu_buffer->cpu); /* Don't bother rewinding */ skip = true; - ts = 0; + state.ts = 0; } else { - ts = head_page->page->time_stamp; + state.ts = head_page->page->time_stamp; } /* Do the reader page - reader must be previous to head. */ - ret = rb_validate_buffer(orig_reader, cpu_buffer->cpu, meta, 0, ts); - if (ret < 0) { - pr_info("Ring buffer meta [%d] invalid reader page detected\n", - cpu_buffer->cpu); - discarded++; - } else { - entries += ret; - entry_bytes += rb_page_size(orig_reader); - ts = orig_reader->page->time_stamp; - } + rb_validate_buffer(orig_reader, cpu_buffer, meta, &state, 0, state.ts); if (skip) goto skip_rewind; @@ -1990,19 +2067,7 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) * Skip if the page is invalid, or its timestamp is newer than the * previous valid page. */ - ret = rb_validate_buffer(head_page, cpu_buffer->cpu, meta, 0, ts); - if (ret < 0) { - if (!discarded) - pr_info("Ring buffer meta [%d] invalid buffer page detected\n", - cpu_buffer->cpu); - discarded++; - } else { - entries += ret; - entry_bytes += rb_page_size(head_page); - if (ret > 0) - local_inc(&cpu_buffer->pages_touched); - ts = head_page->page->time_stamp; - } + rb_validate_buffer(head_page, cpu_buffer, meta, &state, 0, state.ts); } if (i) pr_info("Ring buffer [%d] rewound %d pages\n", cpu_buffer->cpu, i); @@ -2016,43 +2081,7 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) * into the location just before the original head page. */ if (head_page != orig_head) { - struct buffer_page *bpage = orig_head; - - rb_dec_page(&bpage); - /* - * Insert the reader_page before the original head page. - * Since the list encode RB_PAGE flags, general list - * operations should be avoided. - */ - cpu_buffer->reader_page->list.next = &orig_head->list; - cpu_buffer->reader_page->list.prev = orig_head->list.prev; - orig_head->list.prev = &cpu_buffer->reader_page->list; - bpage->list.next = &cpu_buffer->reader_page->list; - - /* Make the head_page the reader page */ - cpu_buffer->reader_page = head_page; - bpage = head_page; - rb_inc_page(&head_page); - head_page->list.prev = bpage->list.prev; - rb_dec_page(&bpage); - bpage->list.next = &head_page->list; - rb_set_list_to_head(&bpage->list); - cpu_buffer->pages = &head_page->list; - - cpu_buffer->head_page = head_page; - meta->head_buffer = (unsigned long)head_page->page; - - /* Reset all the indexes */ - bpage = cpu_buffer->reader_page; - meta->buffers[0] = rb_meta_subbuf_idx(meta, bpage->page); - bpage->id = 0; - - for (i = 1, bpage = head_page; i < meta->nr_subbufs; - i++, rb_inc_page(&bpage)) { - meta->buffers[i] = rb_meta_subbuf_idx(meta, bpage->page); - bpage->id = i; - } - + rb_meta_inject_reader_page(cpu_buffer, meta, orig_head, head_page); /* We'll restart verifying from orig_head */ head_page = orig_head; } @@ -2064,7 +2093,7 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) /* Nothing more to do, the only page is the reader page */ goto done; } - ts = head_page->page->time_stamp; + state.ts = head_page->page->time_stamp; /* Iterate until finding the commit page */ for (i = 0; i < meta->nr_subbufs + 1; i++, rb_inc_page(&head_page)) { @@ -2073,21 +2102,8 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) if (head_page == orig_reader) continue; - ret = rb_validate_buffer(head_page, cpu_buffer->cpu, meta, ts, 0); - if (ret < 0) { - if (!discarded) - pr_info("Ring buffer meta [%d] invalid buffer page detected\n", - cpu_buffer->cpu); - discarded++; - } else { - /* If the buffer has content, update pages_touched */ - if (ret) - local_inc(&cpu_buffer->pages_touched); + rb_validate_buffer(head_page, cpu_buffer, meta, &state, state.ts, 0); - entries += ret; - entry_bytes += rb_page_size(head_page); - ts = head_page->page->time_stamp; - } if (head_page == cpu_buffer->commit_page) break; } @@ -2098,25 +2114,25 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) goto invalid; } done: - local_set(&cpu_buffer->entries, entries); - local_set(&cpu_buffer->entries_bytes, entry_bytes); + local_set(&cpu_buffer->entries, state.entries); + local_set(&cpu_buffer->entries_bytes, state.entry_bytes); pr_info("Ring buffer meta [%d] is from previous boot!", cpu_buffer->cpu); - if (discarded) - pr_cont(" (%d pages discarded)", discarded); + if (state.discarded) + pr_cont(" (%d pages discarded)", state.discarded); pr_cont("\n"); #ifdef CONFIG_RING_BUFFER_PERSISTENT_INJECT if (meta->nr_invalid) pr_warn("Ring buffer testing [%d] invalid pages: %s (%d/%d)\n", cpu_buffer->cpu, - (discarded == meta->nr_invalid) ? "PASSED" : "FAILED", - discarded, meta->nr_invalid); + (state.discarded == meta->nr_invalid) ? "PASSED" : "FAILED", + state.discarded, meta->nr_invalid); if (meta->entry_bytes) pr_warn("Ring buffer testing [%d] entry_bytes: %s (%ld/%ld)\n", cpu_buffer->cpu, - (entry_bytes == meta->entry_bytes) ? "PASSED" : "FAILED", - (long)entry_bytes, (long)meta->entry_bytes); + (state.entry_bytes == meta->entry_bytes) ? "PASSED" : "FAILED", + (long)state.entry_bytes, (long)meta->entry_bytes); meta->nr_invalid = 0; meta->entry_bytes = 0; #endif -- cgit v1.2.3 From 7cf02d0aa6bd4a36f784f6e5dabf225e10b31f3a Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Fri, 22 May 2026 13:09:03 -0400 Subject: ring-buffer: Cleanup buffer_data_page related code Code cleanup related to buffer_data_page for readability, which includes: - Introduce rb_data_page_commit() and rb_data_page_size() - Use 'dpage' for buffer_data_page, instead of 'bpage' because 'bpage' is used for buffer_page. Link: https://patch.msgid.link/20260522171051.722645963@kernel.org Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 112 ++++++++++++++++++++++++--------------------- 1 file changed, 60 insertions(+), 52 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 73f453ba12ce..fd4a8dc5484e 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -364,21 +364,30 @@ struct buffer_page { #define RB_WRITE_MASK 0xfffff #define RB_WRITE_INTCNT (1 << 20) -static void rb_init_page(struct buffer_data_page *bpage) +static void rb_init_data_page(struct buffer_data_page *bpage) { local_set(&bpage->commit, 0); bpage->time_stamp = 0; } +static __always_inline long rb_data_page_commit(struct buffer_data_page *dpage) +{ + return local_read(&dpage->commit); +} + +static __always_inline long rb_data_page_size(struct buffer_data_page *dpage) +{ + return rb_data_page_commit(dpage) & ~RB_MISSED_MASK; +} + static __always_inline unsigned int rb_page_commit(struct buffer_page *bpage) { - return local_read(&bpage->page->commit); + return rb_data_page_commit(bpage->page); } -/* Size is determined by what has been committed */ static __always_inline unsigned int rb_page_size(struct buffer_page *bpage) { - return rb_page_commit(bpage) & ~RB_MISSED_MASK; + return rb_data_page_size(bpage->page); } static void free_buffer_page(struct buffer_page *bpage) @@ -419,7 +428,7 @@ static struct buffer_data_page *alloc_cpu_data(int cpu, int order) return NULL; dpage = page_address(page); - rb_init_page(dpage); + rb_init_data_page(dpage); return dpage; } @@ -659,7 +668,7 @@ static void verify_event(struct ring_buffer_per_cpu *cpu_buffer, do { if (page == tail_page || WARN_ON_ONCE(stop++ > 100)) done = true; - commit = local_read(&page->page->commit); + commit = rb_page_commit(page); write = local_read(&page->write); if (addr >= (unsigned long)&page->page->data[commit] && addr < (unsigned long)&page->page->data[write]) @@ -1906,7 +1915,7 @@ static int __rb_validate_buffer(struct buffer_page *bpage, int cpu, * Even after clearing these bits, a commit value greater than the * subbuf_size is considered invalid. */ - tail = local_read(&dpage->commit) & ~RB_MISSED_MASK; + tail = rb_data_page_size(dpage); if (tail <= meta->subbuf_size - BUF_PAGE_HDR_SIZE) ret = rb_read_data_buffer(dpage, tail, cpu, &ts, &delta); else @@ -2145,12 +2154,12 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) /* Reset the reader page */ local_set(&cpu_buffer->reader_page->entries, 0); - rb_init_page(cpu_buffer->reader_page->page); + rb_init_data_page(cpu_buffer->reader_page->page); /* Reset all the subbuffers */ for (i = 0; i < meta->nr_subbufs - 1; i++, rb_inc_page(&head_page)) { local_set(&head_page->entries, 0); - rb_init_page(head_page->page); + rb_init_data_page(head_page->page); } } @@ -2210,7 +2219,7 @@ static void rb_range_meta_init(struct trace_buffer *buffer, int nr_pages, int sc */ for (i = 0; i < meta->nr_subbufs; i++) { meta->buffers[i] = i; - rb_init_page(subbuf); + rb_init_data_page(subbuf); subbuf += meta->subbuf_size; } } @@ -2262,7 +2271,7 @@ static int rbm_show(struct seq_file *m, void *v) val -= 2; dpage = rb_range_buffer(cpu_buffer, val); seq_printf(m, "buffer[%ld]: %d (commit: %ld)\n", - val, meta->buffers[val], dpage ? local_read(&dpage->commit) : -1); + val, meta->buffers[val], dpage ? rb_data_page_commit(dpage) : -1); return 0; } @@ -2653,7 +2662,7 @@ static void rb_test_inject_invalid_pages(struct trace_buffer *buffer) dpage = (void *)(ptr + idx * subbuf_size); /* Skip unused pages */ - if (!local_read(&dpage->commit)) + if (!rb_data_page_commit(dpage)) continue; /* @@ -2665,7 +2674,7 @@ static void rb_test_inject_invalid_pages(struct trace_buffer *buffer) invalid++; } else { /* Count total commit bytes. */ - entry_bytes += local_read(&dpage->commit) & ~RB_MISSED_MASK; + entry_bytes += rb_data_page_size(dpage); } } @@ -4194,8 +4203,7 @@ rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer) local_set(&cpu_buffer->commit_page->page->commit, rb_page_write(cpu_buffer->commit_page)); RB_WARN_ON(cpu_buffer, - local_read(&cpu_buffer->commit_page->page->commit) & - ~RB_WRITE_MASK); + rb_page_commit(cpu_buffer->commit_page) & ~RB_WRITE_MASK); barrier(); } @@ -4567,7 +4575,7 @@ static const char *show_interrupt_level(void) return show_irq_str(level); } -static void dump_buffer_page(struct buffer_data_page *bpage, +static void dump_buffer_page(struct buffer_data_page *dpage, struct rb_event_info *info, unsigned long tail) { @@ -4575,12 +4583,12 @@ static void dump_buffer_page(struct buffer_data_page *bpage, u64 ts, delta; int e; - ts = bpage->time_stamp; + ts = dpage->time_stamp; pr_warn(" [%lld] PAGE TIME STAMP\n", ts); for (e = 0; e < tail; e += rb_event_length(event)) { - event = (struct ring_buffer_event *)(bpage->data + e); + event = (struct ring_buffer_event *)(dpage->data + e); switch (event->type_len) { @@ -4630,7 +4638,7 @@ static atomic_t ts_dump; } \ atomic_inc(&cpu_buffer->record_disabled); \ pr_warn(fmt, ##__VA_ARGS__); \ - dump_buffer_page(bpage, info, tail); \ + dump_buffer_page(dpage, info, tail); \ atomic_dec(&ts_dump); \ /* There's some cases in boot up that this can happen */ \ if (WARN_ON_ONCE(system_state != SYSTEM_BOOTING)) \ @@ -4646,16 +4654,16 @@ static void check_buffer(struct ring_buffer_per_cpu *cpu_buffer, struct rb_event_info *info, unsigned long tail) { - struct buffer_data_page *bpage; + struct buffer_data_page *dpage; u64 ts, delta; bool full = false; int ret; - bpage = info->tail_page->page; + dpage = info->tail_page->page; if (tail == CHECK_FULL_PAGE) { full = true; - tail = local_read(&bpage->commit); + tail = rb_data_page_commit(dpage); } else if (info->add_timestamp & (RB_ADD_STAMP_FORCE | RB_ADD_STAMP_ABSOLUTE)) { /* Ignore events with absolute time stamps */ @@ -4666,7 +4674,7 @@ static void check_buffer(struct ring_buffer_per_cpu *cpu_buffer, * Do not check the first event (skip possible extends too). * Also do not check if previous events have not been committed. */ - if (tail <= 8 || tail > local_read(&bpage->commit)) + if (tail <= 8 || tail > rb_data_page_commit(dpage)) return; /* @@ -4675,7 +4683,7 @@ static void check_buffer(struct ring_buffer_per_cpu *cpu_buffer, if (atomic_inc_return(this_cpu_ptr(&checking)) != 1) goto out; - ret = rb_read_data_buffer(bpage, tail, cpu_buffer->cpu, &ts, &delta); + ret = rb_read_data_buffer(dpage, tail, cpu_buffer->cpu, &ts, &delta); if (ret < 0) { if (delta < ts) { buffer_warn_return("[CPU: %d]ABSOLUTE TIME WENT BACKWARDS: last ts: %lld absolute ts: %lld clock:%pS\n", @@ -6466,7 +6474,7 @@ static void rb_clear_buffer_page(struct buffer_page *page) { local_set(&page->write, 0); local_set(&page->entries, 0); - rb_init_page(page->page); + rb_init_data_page(page->page); page->read = 0; } @@ -6951,7 +6959,7 @@ ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu) local_irq_restore(flags); if (bpage->data) { - rb_init_page(bpage->data); + rb_init_data_page(bpage->data); } else { bpage->data = alloc_cpu_data(cpu, cpu_buffer->buffer->subbuf_order); if (!bpage->data) { @@ -6976,8 +6984,8 @@ void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, struct buffer_data_read_page *data_page) { struct ring_buffer_per_cpu *cpu_buffer; - struct buffer_data_page *bpage = data_page->data; - struct page *page = virt_to_page(bpage); + struct buffer_data_page *dpage = data_page->data; + struct page *page = virt_to_page(dpage); unsigned long flags; if (!buffer || !buffer->buffers || !buffer->buffers[cpu]) @@ -6997,15 +7005,15 @@ void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, arch_spin_lock(&cpu_buffer->lock); if (!cpu_buffer->free_page) { - cpu_buffer->free_page = bpage; - bpage = NULL; + cpu_buffer->free_page = dpage; + dpage = NULL; } arch_spin_unlock(&cpu_buffer->lock); local_irq_restore(flags); out: - free_pages((unsigned long)bpage, data_page->order); + free_pages((unsigned long)dpage, data_page->order); kfree(data_page); } EXPORT_SYMBOL_GPL(ring_buffer_free_read_page); @@ -7050,7 +7058,7 @@ int ring_buffer_read_page(struct trace_buffer *buffer, { struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu]; struct ring_buffer_event *event; - struct buffer_data_page *bpage; + struct buffer_data_page *dpage; struct buffer_page *reader; unsigned long missed_events; unsigned int commit; @@ -7076,8 +7084,8 @@ int ring_buffer_read_page(struct trace_buffer *buffer, if (data_page->order != buffer->subbuf_order) return -1; - bpage = data_page->data; - if (!bpage) + dpage = data_page->data; + if (!dpage) return -1; guard(raw_spinlock_irqsave)(&cpu_buffer->reader_lock); @@ -7143,7 +7151,7 @@ int ring_buffer_read_page(struct trace_buffer *buffer, * We have already ensured there's enough space if this * is a time extend. */ size = rb_event_length(event); - memcpy(bpage->data + pos, rpage->data + rpos, size); + memcpy(dpage->data + pos, rpage->data + rpos, size); len -= size; @@ -7159,9 +7167,9 @@ int ring_buffer_read_page(struct trace_buffer *buffer, size = rb_event_ts_length(event); } while (len >= size); - /* update bpage */ - local_set(&bpage->commit, pos); - bpage->time_stamp = save_timestamp; + /* update dpage */ + local_set(&dpage->commit, pos); + dpage->time_stamp = save_timestamp; /* we copied everything to the beginning */ read = 0; @@ -7171,13 +7179,13 @@ int ring_buffer_read_page(struct trace_buffer *buffer, cpu_buffer->read_bytes += rb_page_size(reader); /* swap the pages */ - rb_init_page(bpage); - bpage = reader->page; + rb_init_data_page(dpage); + dpage = reader->page; reader->page = data_page->data; local_set(&reader->write, 0); local_set(&reader->entries, 0); reader->read = 0; - data_page->data = bpage; + data_page->data = dpage; /* * Use the real_end for the data size, @@ -7185,12 +7193,12 @@ int ring_buffer_read_page(struct trace_buffer *buffer, * on the page. */ if (reader->real_end) - local_set(&bpage->commit, reader->real_end); + local_set(&dpage->commit, reader->real_end); } cpu_buffer->lost_events = 0; - commit = local_read(&bpage->commit); + commit = rb_data_page_commit(dpage); /* * Set a flag in the commit field if we lost events */ @@ -7199,19 +7207,19 @@ int ring_buffer_read_page(struct trace_buffer *buffer, * missed events, then record it there. */ if (buffer->subbuf_size - commit >= sizeof(missed_events)) { - memcpy(&bpage->data[commit], &missed_events, + memcpy(&dpage->data[commit], &missed_events, sizeof(missed_events)); - local_add(RB_MISSED_STORED, &bpage->commit); + local_add(RB_MISSED_STORED, &dpage->commit); commit += sizeof(missed_events); } - local_add(RB_MISSED_EVENTS, &bpage->commit); + local_add(RB_MISSED_EVENTS, &dpage->commit); } /* * This page may be off to user land. Zero it out here. */ if (commit < buffer->subbuf_size) - memset(&bpage->data[commit], 0, buffer->subbuf_size - commit); + memset(&dpage->data[commit], 0, buffer->subbuf_size - commit); return read; } @@ -7842,7 +7850,7 @@ consume: if (missed_events) { if (cpu_buffer->reader_page != cpu_buffer->commit_page) { - struct buffer_data_page *bpage = reader->page; + struct buffer_data_page *dpage = reader->page; unsigned int commit; /* * Use the real_end for the data size, @@ -7850,18 +7858,18 @@ consume: * on the page. */ if (reader->real_end) - local_set(&bpage->commit, reader->real_end); + local_set(&dpage->commit, reader->real_end); /* * If there is room at the end of the page to save the * missed events, then record it there. */ commit = rb_page_size(reader); if (buffer->subbuf_size - commit >= sizeof(missed_events)) { - memcpy(&bpage->data[commit], &missed_events, + memcpy(&dpage->data[commit], &missed_events, sizeof(missed_events)); - local_add(RB_MISSED_STORED, &bpage->commit); + local_add(RB_MISSED_STORED, &dpage->commit); } - local_add(RB_MISSED_EVENTS, &bpage->commit); + local_add(RB_MISSED_EVENTS, &dpage->commit); } else if (!WARN_ONCE(cpu_buffer->reader_page == cpu_buffer->tail_page, "Reader on commit with %ld missed events", missed_events)) { -- cgit v1.2.3 From 8913e2a48b8d3c1ef9c99481e5725afca841762e Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Fri, 22 May 2026 13:09:04 -0400 Subject: ring-buffer: Have dropped subbuffers be persistent across reboots When the persistent ring buffer detects a corrupted subbuffer, it will zero its size and report dropped pages in the dmesg, then it continues normally. But if a reboot happens without clearing or restarting tracing on the persistent ring buffer, the next boot will show no pages are dropped. If the persistent ring buffer is still the same, then it should still report dropped pages so the user knows that the buffer has missing events. Add the RB_MISSED_EVENTS flag to the commit value of the subbuffer so that the next boot will still show that pages were dropped. Link: https://patch.msgid.link/20260522171051.860780286@kernel.org Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index fd4a8dc5484e..db72969aefa9 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -1915,7 +1915,7 @@ static int __rb_validate_buffer(struct buffer_page *bpage, int cpu, * Even after clearing these bits, a commit value greater than the * subbuf_size is considered invalid. */ - tail = rb_data_page_size(dpage); + tail = rb_data_page_commit(dpage); if (tail <= meta->subbuf_size - BUF_PAGE_HDR_SIZE) ret = rb_read_data_buffer(dpage, tail, cpu, &ts, &delta); else @@ -1929,7 +1929,7 @@ static int __rb_validate_buffer(struct buffer_page *bpage, int cpu, */ if (ret < 0 || (prev_ts && prev_ts > ts) || (next_ts && ts > next_ts)) { local_set(&bpage->entries, 0); - local_set(&dpage->commit, 0); + local_set(&dpage->commit, RB_MISSED_EVENTS); dpage->time_stamp = prev_ts ? prev_ts : next_ts; ret = -1; } else { @@ -3451,7 +3451,7 @@ rb_iter_head_event(struct ring_buffer_iter *iter) * is a mb(), which will synchronize with the rmb here. * (see rb_tail_page_update() and __rb_reserve_next()) */ - commit = rb_page_commit(iter_head_page); + commit = rb_page_size(iter_head_page); smp_rmb(); /* An event needs to be at least 8 bytes in size */ @@ -3480,7 +3480,7 @@ rb_iter_head_event(struct ring_buffer_iter *iter) /* Make sure the page didn't change since we read this */ if (iter->page_stamp != iter_head_page->page->time_stamp || - commit > rb_page_commit(iter_head_page)) + commit > rb_page_size(iter_head_page)) goto reset; iter->next_event = iter->head + length; @@ -5651,7 +5651,7 @@ int ring_buffer_iter_empty(struct ring_buffer_iter *iter) * (see rb_tail_page_update()) */ smp_rmb(); - commit = rb_page_commit(commit_page); + commit = rb_page_size(commit_page); /* We want to make sure that the commit page doesn't change */ smp_rmb(); @@ -5844,6 +5844,7 @@ __rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) */ local_set(&cpu_buffer->reader_page->write, 0); local_set(&cpu_buffer->reader_page->entries, 0); + rb_init_data_page(cpu_buffer->reader_page->page); cpu_buffer->reader_page->real_end = 0; spin: -- cgit v1.2.3 From 97628ff073a8235b401b99e8619c121040f39251 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Fri, 22 May 2026 13:09:05 -0400 Subject: ring-buffer: Show persistent buffer dropped events in trace file When the persistent ring buffer is validated on boot up, if a subbuffer is deemed invalid, it resets the buffer and continues. Currently, these lost events are not shown in the trace file output. Have the trace iterator look for subbuffers that have the RB_MISSED_EVENTS set and set the iter->missed_events flag when it is detected. This will then have the trace file shows "LOST EVENTS" when it reads across a subbuffer that was corrupted and invalidated. For example: <...>-1016 [005] ...1. 6230.660403: preempt_disable: caller=__mod_memcg_state+0x1c8/0x200 parent=__mod_memcg_state+0x1c8/0x200 CPU:5 [LOST EVENTS] <...>-1016 [005] ..... 6230.660673: kmem_cache_alloc: call_site=__anon_vma_prepare+0x1ad/0x1e0 ptr=000000006e40294c name=anon_vma bytes_req=200 bytes_alloc=208 gfp_flags=GFP_KERNEL node=-1 accounted=true Link: https://patch.msgid.link/20260522171052.006276604@kernel.org Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index db72969aefa9..988915f035c7 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -3525,6 +3525,9 @@ static void rb_inc_iter(struct ring_buffer_iter *iter) else rb_inc_page(&iter->head_page); + if (rb_page_commit(iter->head_page) & RB_MISSED_EVENTS) + iter->missed_events = -1; + iter->page_stamp = iter->read_stamp = iter->head_page->page->time_stamp; iter->head = 0; iter->next_event = 0; @@ -7061,7 +7064,7 @@ int ring_buffer_read_page(struct trace_buffer *buffer, struct ring_buffer_event *event; struct buffer_data_page *dpage; struct buffer_page *reader; - unsigned long missed_events; + long missed_events; unsigned int commit; unsigned int read; u64 save_timestamp; @@ -7187,6 +7190,8 @@ int ring_buffer_read_page(struct trace_buffer *buffer, local_set(&reader->entries, 0); reader->read = 0; data_page->data = dpage; + if (!missed_events && rb_data_page_commit(dpage) & RB_MISSED_EVENTS) + missed_events = -1; /* * Use the real_end for the data size, @@ -7204,10 +7209,12 @@ int ring_buffer_read_page(struct trace_buffer *buffer, * Set a flag in the commit field if we lost events */ if (missed_events) { - /* If there is room at the end of the page to save the + /* + * If there is room at the end of the page to save the * missed events, then record it there. */ - if (buffer->subbuf_size - commit >= sizeof(missed_events)) { + if (missed_events > 0 && + buffer->subbuf_size - commit >= sizeof(missed_events)) { memcpy(&dpage->data[commit], &missed_events, sizeof(missed_events)); local_add(RB_MISSED_STORED, &dpage->commit); -- cgit v1.2.3 From 8928e4a3be34bf053f9ef1cad67263604bf4f05e Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Fri, 22 May 2026 13:09:06 -0400 Subject: ring-buffer: Show persistent buffer dropped events in trace_pipe file When the persistent ring buffer is validated on boot up, if a subbuffer is deemed invalid, it resets the buffer and continues. Have the code preserve the RB_MISSED_EVENTS flag in the commit portion of the subbuffer header and pass that back so that the trace_pipe file can show the missed events like the trace file does. For example: <...>-1242 [005] d.... 4429.120116: page_fault_user: address=0x7ffaebb6e728 ip=0x7ffaeb9d4960 error_code=0x7 <...>-1242 [005] ..... 4429.120124: mm_page_alloc: page=00000000055254f3 pfn=0x1373bd order=0 migratetype=1 gfp_flags=GFP_HIGHUSER_MOVABLE|__GFP_COMP <...>-1242 [005] d..2. 4429.120132: tlb_flush: pages:1 reason:local MM shootdown (3) CPU:5 [LOST EVENTS] <...>-1242 [005] d.... 4429.120661: page_fault_user: address=0x55ba7c2d0944 ip=0x55ba7c20cd02 error_code=0x7 <...>-1242 [005] ..... 4429.120669: mm_page_alloc: page=0000000005a02500 pfn=0x12b6e4 order=0 migratetype=1 gfp_flags=GFP_HIGHUSER_MOVABLE|__GFP_COMP <...>-1242 [005] d..2. 4429.120680: tlb_flush: pages:1 reason:local MM shootdown (3) Link: https://patch.msgid.link/20260522171052.156419479@kernel.org Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 56 ++++++++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 22 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 988915f035c7..910f6b3adf74 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -5801,6 +5801,7 @@ __rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) struct buffer_page *reader = NULL; unsigned long overwrite; unsigned long flags; + int missed_events = 0; int nr_loops = 0; bool ret; @@ -5901,6 +5902,9 @@ __rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) if (!ret) goto spin; + if (rb_page_commit(reader) & RB_MISSED_EVENTS) + missed_events = -1; + if (cpu_buffer->ring_meta) rb_update_meta_reader(cpu_buffer, reader); @@ -5965,6 +5969,8 @@ __rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) */ smp_rmb(); + if (!cpu_buffer->lost_events) + cpu_buffer->lost_events = missed_events; return reader; } @@ -7066,6 +7072,7 @@ int ring_buffer_read_page(struct trace_buffer *buffer, struct buffer_page *reader; long missed_events; unsigned int commit; + unsigned int size; unsigned int read; u64 save_timestamp; bool force_memcpy; @@ -7101,7 +7108,8 @@ int ring_buffer_read_page(struct trace_buffer *buffer, event = rb_reader_event(cpu_buffer); read = reader->read; - commit = rb_page_size(reader); + commit = rb_page_commit(reader); + size = rb_page_size(reader); /* Check if any events were dropped */ missed_events = cpu_buffer->lost_events; @@ -7115,13 +7123,14 @@ int ring_buffer_read_page(struct trace_buffer *buffer, * we must copy the data from the page to the buffer. * Otherwise, we can simply swap the page with the one passed in. */ - if (read || (len < (commit - read)) || + if (read || (len < (size - read)) || cpu_buffer->reader_page == cpu_buffer->commit_page || force_memcpy) { struct buffer_data_page *rpage = cpu_buffer->reader_page->page; unsigned int rpos = read; unsigned int pos = 0; - unsigned int size; + unsigned int event_size; + unsigned int flags = 0; /* * If a full page is expected, this can still be returned @@ -7130,19 +7139,22 @@ int ring_buffer_read_page(struct trace_buffer *buffer, * the reader page. */ if (full && - (!read || (len < (commit - read)) || + (!read || (len < (size - read)) || cpu_buffer->reader_page == cpu_buffer->commit_page)) return -1; - if (len > (commit - read)) - len = (commit - read); + if (len > (size - read)) + len = (size - read); /* Always keep the time extend and data together */ - size = rb_event_ts_length(event); + event_size = rb_event_ts_length(event); - if (len < size) + if (len < event_size) return -1; + if (commit & RB_MISSED_EVENTS) + flags = RB_MISSED_EVENTS; + /* save the current timestamp, since the user will need it */ save_timestamp = cpu_buffer->read_stamp; @@ -7154,25 +7166,25 @@ int ring_buffer_read_page(struct trace_buffer *buffer, * one or two events. * We have already ensured there's enough space if this * is a time extend. */ - size = rb_event_length(event); - memcpy(dpage->data + pos, rpage->data + rpos, size); + event_size = rb_event_length(event); + memcpy(dpage->data + pos, rpage->data + rpos, event_size); - len -= size; + len -= event_size; rb_advance_reader(cpu_buffer); rpos = reader->read; - pos += size; + pos += event_size; - if (rpos >= commit) + if (rpos >= event_size) break; event = rb_reader_event(cpu_buffer); /* Always keep the time extend and data together */ - size = rb_event_ts_length(event); - } while (len >= size); + event_size = rb_event_ts_length(event); + } while (len >= event_size); /* update dpage */ - local_set(&dpage->commit, pos); + local_set(&dpage->commit, pos | flags); dpage->time_stamp = save_timestamp; /* we copied everything to the beginning */ @@ -7204,7 +7216,7 @@ int ring_buffer_read_page(struct trace_buffer *buffer, cpu_buffer->lost_events = 0; - commit = rb_data_page_commit(dpage); + size = rb_data_page_size(dpage); /* * Set a flag in the commit field if we lost events */ @@ -7214,11 +7226,11 @@ int ring_buffer_read_page(struct trace_buffer *buffer, * missed events, then record it there. */ if (missed_events > 0 && - buffer->subbuf_size - commit >= sizeof(missed_events)) { - memcpy(&dpage->data[commit], &missed_events, + buffer->subbuf_size - size >= sizeof(missed_events)) { + memcpy(&dpage->data[size], &missed_events, sizeof(missed_events)); local_add(RB_MISSED_STORED, &dpage->commit); - commit += sizeof(missed_events); + size += sizeof(missed_events); } local_add(RB_MISSED_EVENTS, &dpage->commit); } @@ -7226,8 +7238,8 @@ int ring_buffer_read_page(struct trace_buffer *buffer, /* * This page may be off to user land. Zero it out here. */ - if (commit < buffer->subbuf_size) - memset(&dpage->data[commit], 0, buffer->subbuf_size - commit); + if (size < buffer->subbuf_size) + memset(&dpage->data[size], 0, buffer->subbuf_size - size); return read; } -- cgit v1.2.3 From 9cb99c598643ba78638dfd668cf020544159cf70 Mon Sep 17 00:00:00 2001 From: Crystal Wood Date: Mon, 11 May 2026 17:30:35 -0500 Subject: tracing/osnoise: Array printk init and cleanup None of the calls to trace_array_printk_buf() will do anything if we don't initialize the buffer on instance creation (unless some other tracer called it), so do that. Add an osnoise_print() function to facilitate adding debug prints (without tainting). Use trace_array_printk() instead of trace_array_printk_buf(), as we're only writing to the main buffer (of a non-main instance) anyway -- and trace_array_printk_buf() skips the check to make sure we're not printing to the global instance. Link: https://patch.msgid.link/20260511223035.1475676-1-crwood@redhat.com Signed-off-by: Crystal Wood Signed-off-by: Steven Rostedt --- kernel/trace/trace_osnoise.c | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c index 62c2667d97fa..5e83c4f6f2b4 100644 --- a/kernel/trace/trace_osnoise.c +++ b/kernel/trace/trace_osnoise.c @@ -83,6 +83,22 @@ struct osnoise_instance { static struct list_head osnoise_instances; +static void osnoise_print(const char *fmt, ...) +{ + struct osnoise_instance *inst; + struct trace_array *tr; + va_list ap; + + rcu_read_lock(); + list_for_each_entry_rcu(inst, &osnoise_instances, list) { + tr = inst->tr; + va_start(ap, fmt); + trace_array_vprintk(tr, _RET_IP_, fmt, ap); + va_end(ap); + } + rcu_read_unlock(); +} + static bool osnoise_has_registered_instances(void) { return !!list_first_or_null_rcu(&osnoise_instances, @@ -123,6 +139,7 @@ static int osnoise_register_instance(struct trace_array *tr) * trace_types_lock. */ lockdep_assert_held(&trace_types_lock); + trace_array_init_printk(tr); inst = kmalloc_obj(*inst); if (!inst) @@ -471,15 +488,7 @@ static void print_osnoise_headers(struct seq_file *s) * osnoise_taint - report an osnoise error. */ #define osnoise_taint(msg) ({ \ - struct osnoise_instance *inst; \ - struct trace_buffer *buffer; \ - \ - rcu_read_lock(); \ - list_for_each_entry_rcu(inst, &osnoise_instances, list) { \ - buffer = inst->tr->array_buffer.buffer; \ - trace_array_printk_buf(buffer, _THIS_IP_, msg); \ - } \ - rcu_read_unlock(); \ + osnoise_print(msg); \ osnoise_data.tainted = true; \ }) @@ -1189,10 +1198,10 @@ static __always_inline void osnoise_stop_exception(char *msg, int cpu) rcu_read_lock(); list_for_each_entry_rcu(inst, &osnoise_instances, list) { tr = inst->tr; - trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_, - "stop tracing hit on cpu %d due to exception: %s\n", - smp_processor_id(), - msg); + trace_array_printk(tr, _THIS_IP_, + "stop tracing hit on cpu %d due to exception: %s\n", + smp_processor_id(), + msg); if (test_bit(OSN_PANIC_ON_STOP, &osnoise_options)) panic("tracer hit on cpu %d due to exception: %s\n", @@ -1362,8 +1371,8 @@ static __always_inline void osnoise_stop_tracing(void) rcu_read_lock(); list_for_each_entry_rcu(inst, &osnoise_instances, list) { tr = inst->tr; - trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_, - "stop tracing hit on cpu %d\n", smp_processor_id()); + trace_array_printk(tr, _THIS_IP_, + "stop tracing hit on cpu %d\n", smp_processor_id()); if (test_bit(OSN_PANIC_ON_STOP, &osnoise_options)) panic("tracer hit stop condition on CPU %d\n", smp_processor_id()); -- cgit v1.2.3 From bfda3b9424e02c6b2afac74410457c9c1743ba4a Mon Sep 17 00:00:00 2001 From: Rosen Penev Date: Fri, 22 May 2026 14:44:07 -0700 Subject: tracing: Turn hist_elt_data field_var_str into a flexible array The field_var_str array was allocated separately via kcalloc() with its length already known at elt_data allocation time. Convert it to a flexible array member and fold the two allocations into a single kzalloc_flex(), reordering hist_trigger_elt_data_alloc() so n_str is computed and bounds-checked before the struct allocation. hist_elt_data is only reached through tracing_map_elt::private_data (a void *), never embedded, so adding a FAM imposes no tail-position constraint on any enclosing struct. Added __counted_by for extra runtime analysis. Link: https://patch.msgid.link/20260522214407.18120-1-rosenp@gmail.com Assisted-by: Claude:Opus-4.7 Signed-off-by: Rosen Penev Signed-off-by: Steven Rostedt --- kernel/trace/trace_events_hist.c | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index 9701650c89b2..82ce492ab268 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -683,8 +683,8 @@ struct track_data { struct hist_elt_data { char *comm; u64 *var_ref_vals; - char **field_var_str; int n_field_var_str; + char *field_var_str[] __counted_by(n_field_var_str); }; struct snapshot_context { @@ -1629,8 +1629,6 @@ static void hist_elt_data_free(struct hist_elt_data *elt_data) for (i = 0; i < elt_data->n_field_var_str; i++) kfree(elt_data->field_var_str[i]); - kfree(elt_data->field_var_str); - kfree(elt_data->comm); kfree(elt_data); } @@ -1650,10 +1648,19 @@ static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt) struct hist_field *hist_field; unsigned int i, n_str; - elt_data = kzalloc_obj(*elt_data); + BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1)); + + n_str = hist_data->n_field_var_str + hist_data->n_save_var_str + + hist_data->n_var_str; + if (n_str > SYNTH_FIELDS_MAX) + return -EINVAL; + + elt_data = kzalloc_flex(*elt_data, field_var_str, n_str); if (!elt_data) return -ENOMEM; + elt_data->n_field_var_str = n_str; + for_each_hist_field(i, hist_data) { hist_field = hist_data->fields[i]; @@ -1667,24 +1674,8 @@ static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt) } } - n_str = hist_data->n_field_var_str + hist_data->n_save_var_str + - hist_data->n_var_str; - if (n_str > SYNTH_FIELDS_MAX) { - hist_elt_data_free(elt_data); - return -EINVAL; - } - - BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1)); - size = STR_VAR_LEN_MAX; - elt_data->field_var_str = kcalloc(n_str, sizeof(char *), GFP_KERNEL); - if (!elt_data->field_var_str) { - hist_elt_data_free(elt_data); - return -EINVAL; - } - elt_data->n_field_var_str = n_str; - for (i = 0; i < n_str; i++) { elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL); if (!elt_data->field_var_str[i]) { -- cgit v1.2.3 From 01046072880b654dbadf71be2f645aad4a7b5d87 Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Mon, 25 May 2026 19:04:28 +0200 Subject: tracing: Disable KCOV instrumentation for trace_irqsoff.o When KCOV runs its boot selftest with whole-kernel instrumentation enabled, it sets current->kcov_mode to KCOV_MODE_TRACE_PC without installing a coverage area. Any instrumented code accepted as task-context coverage in that window dereferences current->kcov_area and crashes. On ARMv5 Versatile PB with CONFIG_KCOV_SELFTEST=y, CONFIG_KCOV_INSTRUMENT_ALL=y and CONFIG_IRQSOFF_TRACER=y, boot hits a NULL pointer fault during the selftest: kcov: running self test Internal error: Oops: 5 [#1] ARM PC is at __sanitizer_cov_trace_pc+0x4c/0x90 Kernel panic - not syncing: Fatal exception A diagnostic run showed the unwanted coverage comes from the IRQs-off tracer callbacks reached from ARM IRQ entry before hardirq context is visible to KCOV: __sanitizer_cov_trace_pc from tracer_hardirqs_off+0x18/0x1cc tracer_hardirqs_off from trace_hardirqs_off+0x34/0x54 trace_hardirqs_off from __irq_svc+0x58/0xb0 __irq_svc from kcov_init+0x7c/0xdc and similarly through tracer_hardirqs_on(). trace_preemptirq.o is already excluded because this tracing path can run from early interrupt code and produce coverage unrelated to syscall inputs. Exclude trace_irqsoff.o as well, instead of requiring users to turn off CONFIG_KCOV_INSTRUMENT_ALL=y, which is the default whole-kernel KCOV mode. With the exclusion in place, the same ARMv5 Versatile PB QEMU test boots through the KCOV selftest and reaches userspace. Tested on ARMv5 Versatile PB QEMU with CONFIG_KCOV_SELFTEST=y, CONFIG_KCOV_INSTRUMENT_ALL=y and CONFIG_IRQSOFF_TRACER=y. Link: https://patch.msgid.link/20260525170428.67211-1-kmehltretter@gmail.com Assisted-by: Codex:gpt-5 Signed-off-by: Karl Mehltretter Signed-off-by: Steven Rostedt --- kernel/trace/Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile index 9b0834134cae..660675e1d426 100644 --- a/kernel/trace/Makefile +++ b/kernel/trace/Makefile @@ -48,9 +48,10 @@ ifdef CONFIG_GCOV_PROFILE_FTRACE GCOV_PROFILE := y endif -# Functions in this file could be invoked from early interrupt -# code and produce random code coverage. +# Functions in these files can run from IRQ entry before hardirq context +# is visible to KCOV, and produce coverage unrelated to syscall inputs. KCOV_INSTRUMENT_trace_preemptirq.o := n +KCOV_INSTRUMENT_trace_irqsoff.o := n CFLAGS_bpf_trace.o := -I$(src) -- cgit v1.2.3 From 9581123304b23049437324038698af9fb56ee663 Mon Sep 17 00:00:00 2001 From: Rik van Riel Date: Wed, 27 May 2026 11:13:01 -0400 Subject: perf/ftrace: Fix WARNING in __unregister_ftrace_function perf_ftrace_function_unregister() unconditionally calls unregister_ftrace_function() without checking whether the ftrace_ops was ever successfully registered. This triggers a WARN_ON in __unregister_ftrace_function() when the ops doesn't have FTRACE_OPS_FL_ENABLED set. This can happen during perf_event_alloc() error cleanup when perf_trace_destroy() is called via __free_event() on an event whose ftrace_ops registration failed or was already torn down by perf_try_init_event()'s err_destroy path. The call path is: perf_event_alloc() error cleanup -> __free_event() -> event->destroy() [tp_perf_event_destroy] -> perf_trace_destroy() -> perf_trace_event_close() -> TRACE_REG_PERF_CLOSE -> perf_ftrace_function_unregister() -> unregister_ftrace_function() -> __unregister_ftrace_function() -> WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)) Fix this by checking FTRACE_OPS_FL_ENABLED before attempting to unregister. If the ops is not enabled, just free the filter and return success. Link: https://patch.msgid.link/20260527111301.2d0d8256@fangorn Signed-off-by: Rik van Riel Signed-off-by: Steven Rostedt --- kernel/trace/trace_event_perf.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_event_perf.c b/kernel/trace/trace_event_perf.c index a6bb7577e8c5..5b272856e5ab 100644 --- a/kernel/trace/trace_event_perf.c +++ b/kernel/trace/trace_event_perf.c @@ -497,7 +497,17 @@ static int perf_ftrace_function_register(struct perf_event *event) static int perf_ftrace_function_unregister(struct perf_event *event) { struct ftrace_ops *ops = &event->ftrace_ops; - int ret = unregister_ftrace_function(ops); + int ret = 0; + + /* + * Perf will call this unconditionally even if the ops is not + * enabled. The unregister_ftrace_function() will warn if called + * when not enabled. Just bypass the unregistering if ops isn't + * enabled here. + */ + if (ops->flags & FTRACE_OPS_FL_ENABLED) + ret = unregister_ftrace_function(ops); + ftrace_free_filter(ops); return ret; } -- cgit v1.2.3 From 85e0f27dd1396307913ffc5745b0c05137e9beac Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Mon, 25 May 2026 11:21:14 +0900 Subject: tracing/probes: Point the error offset correctly for eprobe argument error Fix to point the error offset correctly for eprobe argument error. In the cleanup commit 1b8b0cd754cd ("tracing/probes: Move event parameter fetching code to common parser"), due to incorrect backward compatibility aimed at conforming to the test specifications, the error location was set to 0 when a non-existent formal parameter was specified for Eprobe. However, this should be corrected in both the test and the implementation to point correct error position. Link: https://lore.kernel.org/all/177967567399.209006.1451571244515632097.stgit@devnote2/ Fixes: 1b8b0cd754cd ("tracing/probes: Move event parameter fetching code to common parser") Cc: stable@vger.kernel.org Signed-off-by: Masami Hiramatsu (Google) Reviewed-by: Steven Rostedt --- kernel/trace/trace_probe.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index e0d3a0da26af..44c22d4e7881 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -962,8 +962,6 @@ static int parse_probe_vars(char *orig_arg, const struct fetch_type *t, code->op = FETCH_OP_COMM; return 0; } - /* backward compatibility */ - ctx->offset = 0; goto inval; } -- cgit v1.2.3 From 4e8729e6598ba0d10021dbf48b308cd53a06bbc4 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Thu, 28 May 2026 22:37:38 -0400 Subject: ring-buffer: Better comment the use of RB_MISSED_EVENTS If the persistent ring buffer is detected on boot up to have a corrupted sub-buffer, that sub-buffer is cleared to zero and its commit value has the RB_MISSED_EVENTS bit set. That bit is to allow the "trace", "trace_pipe" and "trace_pipe_raw" files know that events were dropped by outputting "[LOST EVENTS]". Only in this case does that bit get set in the writeable portion of the ring buffer. When events are dropped in the normal ring buffer, that information is stored in the cpu_buffer descriptor and the RB_MISSED_EVENTS is set in the buffer page at the time the page is consumed. It is never set in the writeable portion of the buffer. Add comments to describe this better as it can be confusing to know when the RB_MISSED_EVENTS are set in the commit portion of the buffer page. Link: https://lore.kernel.org/all/20260529001500.14178455a046a5cbc6180861@kernel.org/ Acked-by: Masami Hiramatsu (Google) Link: https://patch.msgid.link/20260528223738.41276c0e@fedora Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 910f6b3adf74..06fb365bb86e 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -1929,6 +1929,12 @@ static int __rb_validate_buffer(struct buffer_page *bpage, int cpu, */ if (ret < 0 || (prev_ts && prev_ts > ts) || (next_ts && ts > next_ts)) { local_set(&bpage->entries, 0); + /* + * Note, the RB_MISSED_EVENTS is only set inside the main write + * buffer by this verification logic. The normal ring buffer + * has this bit set when the page is read and passed to the + * consumers. + */ local_set(&dpage->commit, RB_MISSED_EVENTS); dpage->time_stamp = prev_ts ? prev_ts : next_ts; ret = -1; @@ -7232,6 +7238,14 @@ int ring_buffer_read_page(struct trace_buffer *buffer, local_add(RB_MISSED_STORED, &dpage->commit); size += sizeof(missed_events); } + /* + * Note, for the persistent ring buffer, the RB_MISSED_EVENTS + * may have been set in the main buffer via the verification code. + * But here, dpage is a copy of that page and has not yet had + * the RB_MISSED_EVENTS set. As for the normal buffers, + * the main write buffer does not set these bits and it needs + * to be set here. + */ local_add(RB_MISSED_EVENTS, &dpage->commit); } -- cgit v1.2.3 From 4304c8165281fe730b6861d37a30b567d3c57832 Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Mon, 1 Jun 2026 23:35:21 +0900 Subject: tracing/probes: Ensure the uprobe buffer size is bigger than event size Add BUILD_BUG_ON() to ensure the uprobe per-CPU working buffer size is bigger than the event size. Link: https://lore.kernel.org/all/177849383209.8038.1902170479780501237.stgit@devnote2/ Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/trace_uprobe.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c index 2cabf8a23ec5..c5ee7920dec6 100644 --- a/kernel/trace/trace_uprobe.c +++ b/kernel/trace/trace_uprobe.c @@ -979,6 +979,7 @@ static struct uprobe_cpu_buffer *prepare_uprobe_buffer(struct trace_uprobe *tu, ucb = uprobe_buffer_get(); ucb->dsize = tu->tp.size + dsize; + BUILD_BUG_ON(MAX_UCB_BUFFER_SIZE < MAX_PROBE_EVENT_SIZE); if (WARN_ON_ONCE(ucb->dsize > MAX_UCB_BUFFER_SIZE)) { ucb->dsize = MAX_UCB_BUFFER_SIZE; dsize = MAX_UCB_BUFFER_SIZE - tu->tp.size; -- cgit v1.2.3 From cf24cbb4e5861caacfdb5bface90b80eaa26e649 Mon Sep 17 00:00:00 2001 From: Rosen Penev Date: Mon, 1 Jun 2026 23:35:21 +0900 Subject: tracing: Use flexible array for entry fetch code Store probe entry fetch instructions in the probe_entry_arg allocation instead of allocating a separate instruction array. This keeps the entry fetch code tied to the entry argument lifetime while leaving regular probe_arg instruction arrays separately allocated and freed. Assisted-by: Codex:GPT-5.5 Link: https://lore.kernel.org/all/20260520215817.16560-1-rosenp@gmail.com/ Signed-off-by: Rosen Penev Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/trace_probe.c | 8 +------- kernel/trace/trace_probe.h | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index 44c22d4e7881..695310571b08 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -838,15 +838,10 @@ static int __store_entry_arg(struct trace_probe *tp, int argnum) int i, offset, last_offset = 0; if (!earg) { - earg = kzalloc_obj(*tp->entry_arg); + earg = kzalloc_flex(*earg, code, 2 * tp->nr_args + 1); if (!earg) return -ENOMEM; earg->size = 2 * tp->nr_args + 1; - earg->code = kzalloc_objs(struct fetch_insn, earg->size); - if (!earg->code) { - kfree(earg); - return -ENOMEM; - } /* Fill the code buffer with 'end' to simplify it */ for (i = 0; i < earg->size; i++) earg->code[i].op = FETCH_OP_END; @@ -2049,7 +2044,6 @@ void trace_probe_cleanup(struct trace_probe *tp) traceprobe_free_probe_arg(&tp->args[i]); if (tp->entry_arg) { - kfree(tp->entry_arg->code); kfree(tp->entry_arg); tp->entry_arg = NULL; } diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h index 262d8707a3df..1076f1df347b 100644 --- a/kernel/trace/trace_probe.h +++ b/kernel/trace/trace_probe.h @@ -238,8 +238,8 @@ struct probe_arg { }; struct probe_entry_arg { - struct fetch_insn *code; unsigned int size; /* The entry data size */ + struct fetch_insn code[] __counted_by(size); }; struct trace_uprobe_filter { -- cgit v1.2.3 From 585abc02be3d3ab82fbcc4dbcbbf0ceb61a02129 Mon Sep 17 00:00:00 2001 From: Yash Suthar Date: Mon, 1 Jun 2026 23:35:21 +0900 Subject: tracing: Replace BUG_ON with lockdep_assert_held in uprobe_buffer functions Replace BUG_ON(!mutex_is_locked(&event_mutex)) with lockdep_assert_held(&event_mutex) in uprobe_buffer_enable() and uprobe_buffer_disable(). BUG_ON() will crash the kernel. mutex_is_locked() only checks if any task holds lock,but not the caller task. lockdep_assert_held() also check current task for lock and no crash on true condition. Link: https://lore.kernel.org/all/20260521192846.8306-1-yashsuthar983@gmail.com/ Signed-off-by: Yash Suthar Acked-by: Steven Rostedt Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/trace_uprobe.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c index c5ee7920dec6..c274346853d1 100644 --- a/kernel/trace/trace_uprobe.c +++ b/kernel/trace/trace_uprobe.c @@ -912,7 +912,7 @@ static int uprobe_buffer_enable(void) { int ret = 0; - BUG_ON(!mutex_is_locked(&event_mutex)); + lockdep_assert_held(&event_mutex); if (uprobe_buffer_refcnt++ == 0) { ret = uprobe_buffer_init(); @@ -927,7 +927,7 @@ static void uprobe_buffer_disable(void) { int cpu; - BUG_ON(!mutex_is_locked(&event_mutex)); + lockdep_assert_held(&event_mutex); if (--uprobe_buffer_refcnt == 0) { for_each_possible_cpu(cpu) -- cgit v1.2.3 From 69efd863a78584b9416ed6be0e1e7349124b4a00 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Mon, 1 Jun 2026 13:07:46 -0400 Subject: tracing/eprobes: Allow use of BTF names to dereference pointers Add syntax to the parsing of eprobes to be able to typecast a trace event field that is a pointer to a structure. Currently, a dereference must be a number, where the user has to figure out manually the offset of a member of a structure that they want to dereference. But for event probes that records a field that happens to be a pointer to a structure, it cannot dereference these values with BTF naming, but must use numerical offsets. For example, to find out what device a sk_buff is pointing to in the net_dev_xmit trace event, one must first use gdb to find the offsets of the members of the structures: (gdb) p &((struct sk_buff *)0)->dev $1 = (struct net_device **) 0x10 (gdb) p &((struct net_device *)0)->name $2 = (char (*)[16]) 0x118 And then use the raw numbers to dereference: # echo 'e:xmit net.net_dev_xmit +0x118(+0x10($skbaddr)):string' >> dynamic_events If BTF is in the kernel, then instead, the skbaddr can be typecast to sk_buff and use the normal dereference logic. # echo 'e:xmit net.net_dev_xmit (sk_buff)skbaddr->dev->name:string' >> dynamic_events # echo 1 > events/eprobes/xmit/enable # cat trace [..] sshd-session-1022 [000] b..2. 860.249343: xmit: (net.net_dev_xmit) arg1="enp7s0" sshd-session-1022 [000] b..2. 860.250061: xmit: (net.net_dev_xmit) arg1="enp7s0" sshd-session-1022 [000] b..2. 860.250142: xmit: (net.net_dev_xmit) arg1="enp7s0" sshd-session-1022 [000] b..2. 860.263553: xmit: (net.net_dev_xmit) arg1="enp7s0" sshd-session-1022 [000] b..2. 860.283820: xmit: (net.net_dev_xmit) arg1="enp7s0" sshd-session-1022 [000] b..2. 860.302716: xmit: (net.net_dev_xmit) arg1="enp7s0" sshd-session-1022 [000] b..2. 860.322905: xmit: (net.net_dev_xmit) arg1="enp7s0" sshd-session-1022 [000] b..2. 860.342828: xmit: (net.net_dev_xmit) arg1="enp7s0" sshd-session-1022 [000] b..2. 860.362268: xmit: (net.net_dev_xmit) arg1="enp7s0" sshd-session-1022 [000] b..2. 860.382335: xmit: (net.net_dev_xmit) arg1="enp7s0" sshd-session-1022 [000] b..2. 860.400856: xmit: (net.net_dev_xmit) arg1="enp7s0" sshd-session-1022 [000] b..2. 860.419893: xmit: (net.net_dev_xmit) arg1="enp7s0" The syntax is simply: (STRUCT)(FIELD)->MEMBER[->MEMBER..] Also add comments around the #else and #endif of #ifdef CONFIG_PROBE_EVENTS_BTF_ARGS to know what they are for. Link: https://lore.kernel.org/all/20260601130746.2139d926@gandalf.local.home/ Signed-off-by: Steven Rostedt (Google) Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/trace_probe.c | 173 ++++++++++++++++++++++++++++++++++++++------- kernel/trace/trace_probe.h | 5 +- 2 files changed, 150 insertions(+), 28 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index 695310571b08..fd1caa1f9723 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -332,6 +332,23 @@ static int parse_trace_event_arg(char *arg, struct fetch_insn *code, return -ENOENT; } +static int parse_trace_event(char *arg, struct fetch_insn *code, + struct traceprobe_parse_context *ctx) +{ + int ret; + + if (code->data) + return -EFAULT; + ret = parse_trace_event_arg(arg, code, ctx); + if (!ret) + return 0; + if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) { + code->op = FETCH_OP_COMM; + return 0; + } + return -EINVAL; +} + #ifdef CONFIG_PROBE_EVENTS_BTF_ARGS static u32 btf_type_int(const struct btf_type *t) @@ -376,11 +393,16 @@ static bool btf_type_is_char_array(struct btf *btf, const struct btf_type *type) && BTF_INT_BITS(intdata) == 8; } +static struct btf *ctx_btf(struct traceprobe_parse_context *ctx) +{ + return ctx->struct_btf ? : ctx->btf; +} + static int check_prepare_btf_string_fetch(char *typename, struct fetch_insn **pcode, struct traceprobe_parse_context *ctx) { - struct btf *btf = ctx->btf; + struct btf *btf = ctx_btf(ctx); if (!btf || !ctx->last_type) return 0; @@ -506,6 +528,15 @@ static int query_btf_context(struct traceprobe_parse_context *ctx) return 0; } +static void clear_struct_btf(struct traceprobe_parse_context *ctx) +{ + if (ctx->struct_btf) { + btf_put(ctx->struct_btf); + ctx->struct_btf = NULL; + ctx->last_struct = NULL; + } +} + static void clear_btf_context(struct traceprobe_parse_context *ctx) { if (ctx->btf) { @@ -554,22 +585,29 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type, struct fetch_insn *code = *pcode; const struct btf_member *field; u32 bitoffs, anon_offs; + bool is_struct = ctx->struct_btf != NULL; + struct btf *btf = ctx_btf(ctx); char *next; int is_ptr; s32 tid; do { - /* Outer loop for solving arrow operator ('->') */ - if (BTF_INFO_KIND(type->info) != BTF_KIND_PTR) { - trace_probe_log_err(ctx->offset, NO_PTR_STRCT); - return -EINVAL; - } - /* Convert a struct pointer type to a struct type */ - type = btf_type_skip_modifiers(ctx->btf, type->type, &tid); - if (!type) { - trace_probe_log_err(ctx->offset, BAD_BTF_TID); - return -EINVAL; + if (!is_struct) { + /* Outer loop for solving arrow operator ('->') */ + if (BTF_INFO_KIND(type->info) != BTF_KIND_PTR) { + trace_probe_log_err(ctx->offset, NO_PTR_STRCT); + return -EINVAL; + } + + /* Convert a struct pointer type to a struct type */ + type = btf_type_skip_modifiers(btf, type->type, &tid); + if (!type) { + trace_probe_log_err(ctx->offset, BAD_BTF_TID); + return -EINVAL; + } } + /* Only the first type can skip being a pointer */ + is_struct = false; bitoffs = 0; do { @@ -580,7 +618,7 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type, return is_ptr; anon_offs = 0; - field = btf_find_struct_member(ctx->btf, type, fieldname, + field = btf_find_struct_member(btf, type, fieldname, &anon_offs); if (IS_ERR(field)) { trace_probe_log_err(ctx->offset, BAD_BTF_TID); @@ -602,7 +640,7 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type, ctx->last_bitsize = 0; } - type = btf_type_skip_modifiers(ctx->btf, field->type, &tid); + type = btf_type_skip_modifiers(btf, field->type, &tid); if (!type) { trace_probe_log_err(ctx->offset, BAD_BTF_TID); return -EINVAL; @@ -640,7 +678,7 @@ static int parse_btf_arg(char *varname, int i, is_ptr, ret; u32 tid; - if (WARN_ON_ONCE(!ctx->funcname)) + if (WARN_ON_ONCE(!ctx->funcname && !(ctx->flags & TPARG_FL_TEVENT))) return -EINVAL; is_ptr = split_next_field(varname, &field, ctx); @@ -653,6 +691,19 @@ static int parse_btf_arg(char *varname, return -EOPNOTSUPP; } + if (ctx->flags & TPARG_FL_TEVENT) { + ret = parse_trace_event(varname, code, ctx); + if (ret < 0) { + trace_probe_log_err(ctx->offset, BAD_ATTACH_ARG); + return ret; + } + /* TEVENT is only here via a typecast */ + if (WARN_ON_ONCE(ctx->struct_btf == NULL)) + return -EINVAL; + type = ctx->last_struct; + goto found_type; + } + if (ctx->flags & TPARG_FL_RETURN && !strcmp(varname, "$retval")) { code->op = FETCH_OP_RETVAL; /* Check whether the function return type is not void */ @@ -709,6 +760,7 @@ static int parse_btf_arg(char *varname, found: type = btf_type_skip_modifiers(ctx->btf, tid, &tid); +found_type: if (!type) { trace_probe_log_err(ctx->offset, BAD_BTF_TID); return -EINVAL; @@ -727,7 +779,7 @@ found: static const struct fetch_type *find_fetch_type_from_btf_type( struct traceprobe_parse_context *ctx) { - struct btf *btf = ctx->btf; + struct btf *btf = ctx_btf(ctx); const char *typestr = NULL; if (btf && ctx->last_type) @@ -758,7 +810,67 @@ static int parse_btf_bitfield(struct fetch_insn **pcode, return 0; } -#else +static int query_btf_struct(const char *sname, struct traceprobe_parse_context *ctx) +{ + struct btf *btf = NULL; + int id; + + /* A struct_btf should only be used by a single argument */ + if (WARN_ON_ONCE(ctx->struct_btf)) { + btf_put(ctx->struct_btf); + ctx->struct_btf = NULL; + } + + id = bpf_find_btf_id(sname, BTF_KIND_STRUCT, &btf); + if (id < 0) + return id; + ctx->struct_btf = btf; + ctx->last_struct = btf_type_by_id(ctx->struct_btf, id); + return 0; +} + +static int handle_typecast(char *arg, struct fetch_insn **pcode, + struct fetch_insn *end, + struct traceprobe_parse_context *ctx) +{ + char *tmp; + int ret; + + /* Currently this only works for eprobes */ + if (!(ctx->flags & TPARG_FL_TEVENT)) { + trace_probe_log_err(ctx->offset, TYPECAST_NOT_EVENT); + return -EINVAL; + } + + tmp = strchr(arg, ')'); + if (!tmp) { + trace_probe_log_err(ctx->offset + strlen(arg), + DEREF_OPEN_BRACE); + return -EINVAL; + } + *tmp = '\0'; + ret = query_btf_struct(arg + 1, ctx); + *tmp = ')'; + + if (ret < 0) { + trace_probe_log_err(ctx->offset + 1, NO_PTR_STRCT); + return -EINVAL; + } + + tmp++; + + ctx->offset += tmp - arg; + ret = parse_btf_arg(tmp, pcode, end, ctx); + return ret; +} + +#else /* !CONFIG_PROBE_EVENTS_BTF_ARGS */ + +static void clear_struct_btf(struct traceprobe_parse_context *ctx) +{ + ctx->struct_btf = NULL; +} + static void clear_btf_context(struct traceprobe_parse_context *ctx) { ctx->btf = NULL; @@ -794,7 +906,15 @@ static int check_prepare_btf_string_fetch(char *typename, return 0; } -#endif +static int handle_typecast(char *arg, struct fetch_insn **pcode, + struct fetch_insn *end, + struct traceprobe_parse_context *ctx) +{ + trace_probe_log_err(ctx->offset, NOSUP_BTFARG); + return -EOPNOTSUPP; +} + +#endif /* CONFIG_PROBE_EVENTS_BTF_ARGS */ #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API @@ -948,16 +1068,9 @@ static int parse_probe_vars(char *orig_arg, const struct fetch_type *t, int len; if (ctx->flags & TPARG_FL_TEVENT) { - if (code->data) - return -EFAULT; - ret = parse_trace_event_arg(arg, code, ctx); - if (!ret) - return 0; - if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) { - code->op = FETCH_OP_COMM; - return 0; - } - goto inval; + if (parse_trace_event(arg, code, ctx) < 0) + goto inval; + return 0; } if (str_has_prefix(arg, "retval")) { @@ -1224,6 +1337,9 @@ parse_probe_arg(char *arg, const struct fetch_type *type, code->op = FETCH_OP_IMM; } break; + case '(': + ret = handle_typecast(arg, pcode, end, ctx); + break; default: if (isalpha(arg[0]) || arg[0] == '_') { /* BTF variable */ if (!tparg_is_function_entry(ctx->flags) && @@ -1556,6 +1672,9 @@ fail: } kfree(tmp); + /* struct_btf should not be passed to other arguments */ + clear_struct_btf(ctx); + return ret; } diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h index 1076f1df347b..15758cc11fc6 100644 --- a/kernel/trace/trace_probe.h +++ b/kernel/trace/trace_probe.h @@ -422,7 +422,9 @@ struct traceprobe_parse_context { const struct btf_param *params; /* Parameter of the function */ s32 nr_params; /* The number of the parameters */ struct btf *btf; /* The BTF to be used */ + struct btf *struct_btf; /* The BTF to be used for structs */ const struct btf_type *last_type; /* Saved type */ + const struct btf_type *last_struct; /* Saved structure */ u32 last_bitoffs; /* Saved bitoffs */ u32 last_bitsize; /* Saved bitsize */ struct trace_probe *tp; @@ -563,7 +565,8 @@ extern int traceprobe_define_arg_fields(struct trace_event_call *event_call, C(NEED_STRING_TYPE, "$comm and immediate-string only accepts string type"),\ C(TOO_MANY_ARGS, "Too many arguments are specified"), \ C(TOO_MANY_EARGS, "Too many entry arguments specified"), \ - C(EVENT_TOO_BIG, "Event too big (too many fields?)"), + C(EVENT_TOO_BIG, "Event too big (too many fields?)"), \ + C(TYPECAST_NOT_EVENT, "Typecasts are only for eprobe fields"), #undef C #define C(a, b) TP_ERR_##a -- cgit v1.2.3 From 4793e8a6e20e4c17ed4e755ea40e5912cc3539af Mon Sep 17 00:00:00 2001 From: Gabriele Monaco Date: Mon, 1 Jun 2026 17:38:28 +0200 Subject: rv: Fix __user specifier usage in extract_params() The attributes variables extracted from syscalls in the helper are both defined with the __user specifier although only the actual pointer to user data should be marked. Remove the __user specifier from attr. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202604150820.Ny143u6X-lkp@intel.com Fixes: b133207deb72 ("rv: Add nomiss deadline monitor") Reviewed-by: Wen Yang Reviewed-by: Nam Cao Link: https://lore.kernel.org/r/20260601153840.124372-2-gmonaco@redhat.com Signed-off-by: Gabriele Monaco --- kernel/trace/rv/monitors/deadline/deadline.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'kernel/trace') diff --git a/kernel/trace/rv/monitors/deadline/deadline.h b/kernel/trace/rv/monitors/deadline/deadline.h index 0bbfd2543329..78fca873d61e 100644 --- a/kernel/trace/rv/monitors/deadline/deadline.h +++ b/kernel/trace/rv/monitors/deadline/deadline.h @@ -95,7 +95,8 @@ static inline u8 get_server_type(struct task_struct *tsk) static inline int extract_params(struct pt_regs *regs, long id, pid_t *pid_out) { size_t size = offsetofend(struct sched_attr, sched_flags); - struct sched_attr __user *uattr, attr; + struct sched_attr __user *uattr; + struct sched_attr attr; int new_policy = -1, ret; unsigned long args[6]; -- cgit v1.2.3 From 700782ec8f6589c5792b323efd6e004dd183328b Mon Sep 17 00:00:00 2001 From: Gabriele Monaco Date: Mon, 1 Jun 2026 17:38:34 +0200 Subject: rv: Add automatic cleanup handlers for per-task HA monitors Hybrid automata monitors may start timers, depending on the model, these may remain active on an exiting task and cause false positives or even access freed memory. Add an enable/disable hook in the HA code, currently only populated by the per-task handler for registration and deregistration. This hooks to the sched_process_exit event and ensures the timer is stopped for every exiting task. The handler is enabled automatically but may be disabled, for instance if the monitor uses the event for another purpose (but should still manually ensure timers are stopped). Fixes: f5587d1b6ec9 ("rv: Add Hybrid Automata monitor type") Reviewed-by: Nam Cao Link: https://lore.kernel.org/r/20260601153840.124372-8-gmonaco@redhat.com Signed-off-by: Gabriele Monaco --- kernel/trace/rv/monitors/nomiss/nomiss.c | 4 ++-- kernel/trace/rv/monitors/opid/opid.c | 4 ++-- kernel/trace/rv/monitors/stall/stall.c | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/rv/monitors/nomiss/nomiss.c b/kernel/trace/rv/monitors/nomiss/nomiss.c index 31f90f3638d8..8ead8783c29f 100644 --- a/kernel/trace/rv/monitors/nomiss/nomiss.c +++ b/kernel/trace/rv/monitors/nomiss/nomiss.c @@ -227,7 +227,7 @@ static int enable_nomiss(void) { int retval; - retval = da_monitor_init(); + retval = ha_monitor_init(); if (retval) return retval; @@ -263,7 +263,7 @@ static void disable_nomiss(void) rv_detach_trace_probe("nomiss", sched_switch, handle_sched_switch); rv_detach_trace_probe("nomiss", sched_wakeup, handle_sched_wakeup); - da_monitor_destroy(); + ha_monitor_destroy(); } static struct rv_monitor rv_this = { diff --git a/kernel/trace/rv/monitors/opid/opid.c b/kernel/trace/rv/monitors/opid/opid.c index 4594c7c46601..2922318c6112 100644 --- a/kernel/trace/rv/monitors/opid/opid.c +++ b/kernel/trace/rv/monitors/opid/opid.c @@ -73,7 +73,7 @@ static int enable_opid(void) { int retval; - retval = da_monitor_init(); + retval = ha_monitor_init(); if (retval) return retval; @@ -90,7 +90,7 @@ static void disable_opid(void) rv_detach_trace_probe("opid", sched_set_need_resched_tp, handle_sched_need_resched); rv_detach_trace_probe("opid", sched_waking, handle_sched_waking); - da_monitor_destroy(); + ha_monitor_destroy(); } /* diff --git a/kernel/trace/rv/monitors/stall/stall.c b/kernel/trace/rv/monitors/stall/stall.c index 9ccfda6b0e73..3c38fb1a0159 100644 --- a/kernel/trace/rv/monitors/stall/stall.c +++ b/kernel/trace/rv/monitors/stall/stall.c @@ -103,7 +103,7 @@ static int enable_stall(void) { int retval; - retval = da_monitor_init(); + retval = ha_monitor_init(); if (retval) return retval; @@ -120,7 +120,7 @@ static void disable_stall(void) rv_detach_trace_probe("stall", sched_switch, handle_sched_switch); rv_detach_trace_probe("stall", sched_wakeup, handle_sched_wakeup); - da_monitor_destroy(); + ha_monitor_destroy(); } static struct rv_monitor rv_this = { -- cgit v1.2.3 From d9022172c1abea9e220d13ee453813dafec5b9e4 Mon Sep 17 00:00:00 2001 From: Gabriele Monaco Date: Mon, 1 Jun 2026 17:38:37 +0200 Subject: rv: Use 0 to check preemption enabled in opid Tracepoint handlers no longer run with preemption disabled by default since a46023d5616 ("tracing: Guard __DECLARE_TRACE() use of __DO_TRACE_CALL() with SRCU-fast"), the opid monitor should now count 1 in the preemption count as preemption disabled. Change the rule for preempt_off to preempt > 0. Reviewed-by: Nam Cao Link: https://lore.kernel.org/r/20260601153840.124372-11-gmonaco@redhat.com Signed-off-by: Gabriele Monaco --- kernel/trace/rv/monitors/opid/opid.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/rv/monitors/opid/opid.c b/kernel/trace/rv/monitors/opid/opid.c index 2922318c6112..3b6a85e815b8 100644 --- a/kernel/trace/rv/monitors/opid/opid.c +++ b/kernel/trace/rv/monitors/opid/opid.c @@ -22,14 +22,8 @@ static u64 ha_get_env(struct ha_monitor *ha_mon, enum envs_opid env, u64 time_ns if (env == irq_off_opid) return irqs_disabled(); else if (env == preempt_off_opid) { - /* - * If CONFIG_PREEMPTION is enabled, then the tracepoint itself disables - * preemption (adding one to the preempt_count). Since we are - * interested in the preempt_count at the time the tracepoint was - * hit, we consider 1 as still enabled. - */ if (IS_ENABLED(CONFIG_PREEMPTION)) - return (preempt_count() & PREEMPT_MASK) > 1; + return (preempt_count() & PREEMPT_MASK) > 0; return true; } return ENV_INVALID_VALUE; -- cgit v1.2.3 From e57f13eaab259ece7c9e8d81ba2c40c4f057ca2c Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Sat, 6 Jun 2026 14:39:26 +0200 Subject: ftrace: Add ftrace_hash_count function Adding external ftrace_hash_count function so we could get hash count outside of ftrace object. Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260606123955.345967-2-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/trace/ftrace.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index b2611de3f594..57ab01fd00bd 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -6288,11 +6288,16 @@ int modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr) } EXPORT_SYMBOL_GPL(modify_ftrace_direct); -static unsigned long hash_count(struct ftrace_hash *hash) +static inline unsigned long hash_count(struct ftrace_hash *hash) { return hash ? hash->count : 0; } +unsigned long ftrace_hash_count(struct ftrace_hash *hash) +{ + return hash_count(hash); +} + /** * hash_add - adds two struct ftrace_hash and returns the result * @a: struct ftrace_hash object -- cgit v1.2.3 From af7c32365090a1a8ff981f85d7c24b344a2eaa75 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Sat, 6 Jun 2026 14:39:27 +0200 Subject: ftrace: Add ftrace_hash_remove function Adding ftrace_hash_remove function that removes all entries from struct ftrace_hash object without freeing them. It will be used in following changes where entries are allocated as part of another structure and are free-ed separately. Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260606123955.345967-3-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/trace/ftrace.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'kernel/trace') diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 57ab01fd00bd..45548b0200eb 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -1249,6 +1249,25 @@ remove_hash_entry(struct ftrace_hash *hash, hash->count--; } +void ftrace_hash_remove(struct ftrace_hash *hash) +{ + struct ftrace_func_entry *entry; + struct hlist_head *hhd; + struct hlist_node *tn; + int size; + int i; + + if (!hash || !hash->count) + return; + size = 1 << hash->size_bits; + for (i = 0; i < size; i++) { + hhd = &hash->buckets[i]; + hlist_for_each_entry_safe(entry, tn, hhd, hlist) + remove_hash_entry(hash, entry); + } + FTRACE_WARN_ON(hash->count); +} + static void ftrace_hash_clear(struct ftrace_hash *hash) { struct hlist_head *hhd; -- cgit v1.2.3 From 2cd298c106e00ba1d8799b022594f131703f32fa Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Sat, 6 Jun 2026 14:39:28 +0200 Subject: ftrace: Add add_ftrace_hash_entry function Renaming __add_hash_entry to add_ftrace_hash_entry and making it global, it will be used in following changes outside ftrace.c object. Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260606123955.345967-4-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/trace/ftrace.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 45548b0200eb..f93e34dd2328 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -1198,8 +1198,7 @@ ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip) return __ftrace_lookup_ip(hash, ip); } -static void __add_hash_entry(struct ftrace_hash *hash, - struct ftrace_func_entry *entry) +void add_ftrace_hash_entry(struct ftrace_hash *hash, struct ftrace_func_entry *entry) { struct hlist_head *hhd; unsigned long key; @@ -1221,7 +1220,7 @@ add_ftrace_hash_entry_direct(struct ftrace_hash *hash, unsigned long ip, unsigne entry->ip = ip; entry->direct = direct; - __add_hash_entry(hash, entry); + add_ftrace_hash_entry(hash, entry); return entry; } @@ -1477,7 +1476,7 @@ static struct ftrace_hash *__move_hash(struct ftrace_hash *src, int size) hhd = &src->buckets[i]; hlist_for_each_entry_safe(entry, tn, hhd, hlist) { remove_hash_entry(src, entry); - __add_hash_entry(new_hash, entry); + add_ftrace_hash_entry(new_hash, entry); } } return new_hash; @@ -5360,7 +5359,7 @@ int ftrace_func_mapper_add_ip(struct ftrace_func_mapper *mapper, map->entry.ip = ip; map->data = data; - __add_hash_entry(&mapper->hash, &map->entry); + add_ftrace_hash_entry(&mapper->hash, &map->entry); return 0; } -- cgit v1.2.3 From c1d32dea5d4694c1a6c14d1d1c3192d0e18ffc7b Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Sat, 6 Jun 2026 14:39:38 +0200 Subject: bpf: Add support for tracing multi link Adding new link to allow to attach program to multiple function BTF IDs. The link is represented by struct bpf_tracing_multi_link. To configure the link, new fields are added to bpf_attr::link_create to pass array of BTF IDs; struct { __aligned_u64 ids; __u32 cnt; } tracing_multi; Each BTF ID represents function (BTF_KIND_FUNC) that the link will attach bpf program to. We use previously added bpf_trampoline_multi_attach/detach functions to attach/detach the link. The linkinfo/fdinfo callbacks will be implemented in following changes. Note this is supported only for archs (x86_64) with ftrace direct and have single ops support. CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS && CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS Note using sort_r (instead of plain sort) in check_dup_ids, because we will use the swap callback in following changes. Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260606123955.345967-14-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/trace/bpf_trace.c | 130 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) (limited to 'kernel/trace') diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index d853f97bd154..9e3cb547651e 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -42,6 +42,7 @@ #define MAX_UPROBE_MULTI_CNT (1U << 20) #define MAX_KPROBE_MULTI_CNT (1U << 20) +#define MAX_TRACING_MULTI_CNT (1U << 20) #ifdef CONFIG_MODULES struct bpf_trace_module { @@ -3641,3 +3642,132 @@ __bpf_kfunc int bpf_copy_from_user_task_str_dynptr(const struct bpf_dynptr *dptr } __bpf_kfunc_end_defs(); + +#if defined(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS) && \ + defined(CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS) + +static void bpf_tracing_multi_link_release(struct bpf_link *link) +{ + struct bpf_tracing_multi_link *tr_link = + container_of(link, struct bpf_tracing_multi_link, link); + + WARN_ON_ONCE(bpf_trampoline_multi_detach(link->prog, tr_link)); +} + +static void bpf_tracing_multi_link_dealloc(struct bpf_link *link) +{ + struct bpf_tracing_multi_link *tr_link = + container_of(link, struct bpf_tracing_multi_link, link); + + kvfree(tr_link); +} + +static const struct bpf_link_ops bpf_tracing_multi_link_lops = { + .release = bpf_tracing_multi_link_release, + .dealloc_deferred = bpf_tracing_multi_link_dealloc, +}; + +static int ids_cmp_r(const void *pa, const void *pb, const void *priv __maybe_unused) +{ + u32 a = *(u32 *) pa; + u32 b = *(u32 *) pb; + + return (a > b) - (a < b); +} + +static void ids_swap_r(void *a, void *b, int size __maybe_unused, + const void *priv __maybe_unused) +{ + u32 *id_a = a, *id_b = b; + + swap(*id_a, *id_b); +} + +static int check_dup_ids(u32 *ids, u32 cnt) +{ + int err = 0; + + /* + * Sort ids array (together with cookies array if defined) + * and check it for duplicates. The ids and cookies arrays + * are left sorted. + */ + sort_r_nonatomic(ids, cnt, sizeof(ids[0]), ids_cmp_r, ids_swap_r, NULL); + + for (int i = 1; i < cnt; i++) { + if (ids[i] == ids[i - 1]) { + err = -EINVAL; + break; + } + } + return err; +} + +int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) +{ + struct bpf_tracing_multi_link *link = NULL; + struct bpf_link_primer link_primer; + u32 cnt, *ids = NULL; + u32 __user *uids; + int err; + + uids = u64_to_user_ptr(attr->link_create.tracing_multi.ids); + cnt = attr->link_create.tracing_multi.cnt; + + if (!cnt || !uids) + return -EINVAL; + if (cnt > MAX_TRACING_MULTI_CNT) + return -E2BIG; + if (attr->link_create.flags || attr->link_create.target_fd) + return -EINVAL; + + ids = kvmalloc_objs(*ids, cnt); + if (!ids) + return -ENOMEM; + + if (copy_from_user(ids, uids, cnt * sizeof(*ids))) { + err = -EFAULT; + goto error; + } + + err = check_dup_ids(ids, cnt); + if (err) + goto error; + + link = kvzalloc_flex(*link, nodes, cnt); + if (!link) { + err = -ENOMEM; + goto error; + } + + bpf_link_init(&link->link, BPF_LINK_TYPE_TRACING_MULTI, + &bpf_tracing_multi_link_lops, prog, prog->expected_attach_type); + + err = bpf_link_prime(&link->link, &link_primer); + if (err) + goto error; + + link->nodes_cnt = cnt; + + err = bpf_trampoline_multi_attach(prog, ids, link); + kvfree(ids); + if (err) { + bpf_link_cleanup(&link_primer); + return err; + } + return bpf_link_settle(&link_primer); + +error: + kvfree(ids); + kvfree(link); + return err; +} + +#else + +int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) +{ + return -EOPNOTSUPP; +} + +#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS && CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS */ -- cgit v1.2.3 From 46b42af27d40021a97c147d23de8cb29eb5020df Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Sat, 6 Jun 2026 14:39:39 +0200 Subject: bpf: Add support for tracing_multi link cookies Add support to specify cookies for tracing_multi link. Cookies are provided in array where each value is paired with provided BTF ID value with the same array index. Such cookie can be retrieved by bpf program with bpf_get_attach_cookie helper call. We need to sort cookies array together with ids array in check_dup_ids, to keep the id->cookie relation. Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260606123955.345967-15-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/trace/bpf_trace.c | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 9e3cb547651e..e33492739ed1 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -3659,6 +3659,7 @@ static void bpf_tracing_multi_link_dealloc(struct bpf_link *link) struct bpf_tracing_multi_link *tr_link = container_of(link, struct bpf_tracing_multi_link, link); + kvfree(tr_link->cookies); kvfree(tr_link); } @@ -3678,13 +3679,24 @@ static int ids_cmp_r(const void *pa, const void *pb, const void *priv __maybe_un static void ids_swap_r(void *a, void *b, int size __maybe_unused, const void *priv __maybe_unused) { - u32 *id_a = a, *id_b = b; + u64 *cookie_a, *cookie_b, *cookies; + u32 *id_a = a, *id_b = b, *ids; + void **data = (void **) priv; + ids = data[0]; + cookies = data[1]; + + if (cookies) { + cookie_a = cookies + (id_a - ids); + cookie_b = cookies + (id_b - ids); + swap(*cookie_a, *cookie_b); + } swap(*id_a, *id_b); } -static int check_dup_ids(u32 *ids, u32 cnt) +static int check_dup_ids(u32 *ids, u64 *cookies, u32 cnt) { + void *data[2] = { ids, cookies }; int err = 0; /* @@ -3692,7 +3704,7 @@ static int check_dup_ids(u32 *ids, u32 cnt) * and check it for duplicates. The ids and cookies arrays * are left sorted. */ - sort_r_nonatomic(ids, cnt, sizeof(ids[0]), ids_cmp_r, ids_swap_r, NULL); + sort_r_nonatomic(ids, cnt, sizeof(ids[0]), ids_cmp_r, ids_swap_r, data); for (int i = 1; i < cnt; i++) { if (ids[i] == ids[i - 1]) { @@ -3708,6 +3720,8 @@ int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) struct bpf_tracing_multi_link *link = NULL; struct bpf_link_primer link_primer; u32 cnt, *ids = NULL; + u64 __user *ucookies; + u64 *cookies = NULL; u32 __user *uids; int err; @@ -3730,7 +3744,20 @@ int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) goto error; } - err = check_dup_ids(ids, cnt); + ucookies = u64_to_user_ptr(attr->link_create.tracing_multi.cookies); + if (ucookies) { + cookies = kvmalloc_objs(*cookies, cnt); + if (!cookies) { + err = -ENOMEM; + goto error; + } + if (copy_from_user(cookies, ucookies, cnt * sizeof(*cookies))) { + err = -EFAULT; + goto error; + } + } + + err = check_dup_ids(ids, cookies, cnt); if (err) goto error; @@ -3748,6 +3775,7 @@ int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) goto error; link->nodes_cnt = cnt; + link->cookies = cookies; err = bpf_trampoline_multi_attach(prog, ids, link); kvfree(ids); @@ -3758,6 +3786,7 @@ int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) return bpf_link_settle(&link_primer); error: + kvfree(cookies); kvfree(ids); kvfree(link); return err; -- cgit v1.2.3 From ba042ed6446fc524c1d804227765b45616f9cba3 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Sat, 6 Jun 2026 14:39:40 +0200 Subject: bpf: Add support for tracing_multi link session Adding support to use session attachment with tracing_multi link. Adding new BPF_TRACE_FSESSION_MULTI program attach type, that follows the BPF_TRACE_FSESSION behaviour but on the tracing_multi link. Such program is called on entry and exit of the attached function and allows to pass cookie value from entry to exit execution. Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260606123955.345967-16-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/trace/bpf_trace.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'kernel/trace') diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index e33492739ed1..a0d688fffc5a 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -1334,7 +1334,8 @@ static inline bool is_uprobe_session(const struct bpf_prog *prog) static inline bool is_trace_fsession(const struct bpf_prog *prog) { return prog->type == BPF_PROG_TYPE_TRACING && - prog->expected_attach_type == BPF_TRACE_FSESSION; + (prog->expected_attach_type == BPF_TRACE_FSESSION || + prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI); } static const struct bpf_func_proto * @@ -3659,6 +3660,7 @@ static void bpf_tracing_multi_link_dealloc(struct bpf_link *link) struct bpf_tracing_multi_link *tr_link = container_of(link, struct bpf_tracing_multi_link, link); + kvfree(tr_link->fexits); kvfree(tr_link->cookies); kvfree(tr_link); } @@ -3718,6 +3720,7 @@ static int check_dup_ids(u32 *ids, u64 *cookies, u32 cnt) int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) { struct bpf_tracing_multi_link *link = NULL; + struct bpf_tramp_node *fexits = NULL; struct bpf_link_primer link_primer; u32 cnt, *ids = NULL; u64 __user *ucookies; @@ -3761,6 +3764,14 @@ int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) if (err) goto error; + if (prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI) { + fexits = kvmalloc_objs(*fexits, cnt); + if (!fexits) { + err = -ENOMEM; + goto error; + } + } + link = kvzalloc_flex(*link, nodes, cnt); if (!link) { err = -ENOMEM; @@ -3776,6 +3787,7 @@ int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) link->nodes_cnt = cnt; link->cookies = cookies; + link->fexits = fexits; err = bpf_trampoline_multi_attach(prog, ids, link); kvfree(ids); @@ -3786,6 +3798,7 @@ int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) return bpf_link_settle(&link_primer); error: + kvfree(fexits); kvfree(cookies); kvfree(ids); kvfree(link); -- cgit v1.2.3 From 8abecdafd57553c053bb68db47ed32a54972d5f4 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Sat, 6 Jun 2026 14:39:41 +0200 Subject: bpf: Add support for tracing_multi link fdinfo Adding tracing_multi link fdinfo support with following output: pos: 0 flags: 02000000 mnt_id: 19 ino: 3087 link_type: tracing_multi link_id: 9 prog_tag: 599ba0e317244f86 prog_id: 94 attach_type: 59 cnt: 10 obj-id btf-id cookie func 1 91593 8 bpf_fentry_test1+0x4/0x10 1 91595 9 bpf_fentry_test2+0x4/0x10 1 91596 7 bpf_fentry_test3+0x4/0x20 1 91597 5 bpf_fentry_test4+0x4/0x20 1 91598 4 bpf_fentry_test5+0x4/0x20 1 91599 2 bpf_fentry_test6+0x4/0x20 1 91600 3 bpf_fentry_test7+0x4/0x10 1 91601 1 bpf_fentry_test8+0x4/0x10 1 91602 10 bpf_fentry_test9+0x4/0x10 1 91594 6 bpf_fentry_test10+0x4/0x10 Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260606123955.345967-17-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/trace/bpf_trace.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) (limited to 'kernel/trace') diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index a0d688fffc5a..90432f0fc2a8 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -3665,9 +3665,39 @@ static void bpf_tracing_multi_link_dealloc(struct bpf_link *link) kvfree(tr_link); } +#ifdef CONFIG_PROC_FS +static void bpf_tracing_multi_show_fdinfo(const struct bpf_link *link, + struct seq_file *seq) +{ + struct bpf_tracing_multi_link *tr_link = + container_of(link, struct bpf_tracing_multi_link, link); + bool has_cookies = !!tr_link->cookies; + + seq_printf(seq, "attach_type:\t%u\n", tr_link->link.attach_type); + seq_printf(seq, "cnt:\t%u\n", tr_link->nodes_cnt); + + seq_printf(seq, "%s\t %s\t %s\t %s\n", "obj-id", "btf-id", "cookie", "func"); + for (int i = 0; i < tr_link->nodes_cnt; i++) { + struct bpf_tracing_multi_node *mnode = &tr_link->nodes[i]; + u32 btf_id, obj_id; + + bpf_trampoline_unpack_key(mnode->trampoline->key, &obj_id, &btf_id); + seq_printf(seq, "%u\t %u\t %llu\t %pS\n", + obj_id, btf_id, + has_cookies ? tr_link->cookies[i] : 0, + (void *) mnode->trampoline->ip); + + cond_resched(); + } +} +#endif + static const struct bpf_link_ops bpf_tracing_multi_link_lops = { .release = bpf_tracing_multi_link_release, .dealloc_deferred = bpf_tracing_multi_link_dealloc, +#ifdef CONFIG_PROC_FS + .show_fdinfo = bpf_tracing_multi_show_fdinfo, +#endif }; static int ids_cmp_r(const void *pa, const void *pb, const void *priv __maybe_unused) -- cgit v1.2.3 From 4d87a251d45b4a95eb4c0abcfab809c9f231258a Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Thu, 11 Jun 2026 13:42:24 +0200 Subject: bpf: Guard __get_user acesss with access_ok for uprobe_multi data As reported by sashiko [1] we need to use access_ok to check the user space data bounds before we use __get-user to get it. [1] https://lore.kernel.org/bpf/20260610145235.CB1441F00893@smtp.kernel.org/ Fixes: 0b779b61f651 ("bpf: Add cookies support for uprobe_multi link") Fixes: 89ae89f53d20 ("bpf: Add multi uprobe link") Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260611114230.950379-2-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/trace/bpf_trace.c | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'kernel/trace') diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 90432f0fc2a8..b5a12af2d3f8 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -3224,6 +3224,7 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr unsigned long __user *uoffsets; u64 __user *ucookies; void __user *upath; + unsigned long size; u32 flags, cnt, i; struct path path; char *name; @@ -3261,6 +3262,16 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr uref_ctr_offsets = u64_to_user_ptr(attr->link_create.uprobe_multi.ref_ctr_offsets); ucookies = u64_to_user_ptr(attr->link_create.uprobe_multi.cookies); + /* + * All uoffsets/uref_ctr_offsets/ucookies arrays have the same value + * size, we need to check their address range is safe for __get_user + * calls. + */ + size = sizeof(*uoffsets) * cnt; + if (!access_ok(uoffsets, size) || !access_ok(uref_ctr_offsets, size) || + !access_ok(ucookies, size)) + return -EFAULT; + name = strndup_user(upath, PATH_MAX); if (IS_ERR(name)) { err = PTR_ERR(name); -- cgit v1.2.3 From 65d81609e93140d8dd745fd41eb8a195f83ba7cd Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Thu, 11 Jun 2026 13:42:25 +0200 Subject: bpf: Use user_path_at for path resolution in uprobe_multi Resolve the uprobe_multi user path with user_path_at() instead of copying the string with strndup_user() and passing it to kern_path(). This removes the temporary allocation and keeps the lookup logic in one helper. Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260611114230.950379-3-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/trace/bpf_trace.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index b5a12af2d3f8..f8990bc6b64c 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -3227,7 +3227,6 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr unsigned long size; u32 flags, cnt, i; struct path path; - char *name; pid_t pid; int err; @@ -3272,14 +3271,7 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr !access_ok(ucookies, size)) return -EFAULT; - name = strndup_user(upath, PATH_MAX); - if (IS_ERR(name)) { - err = PTR_ERR(name); - return err; - } - - err = kern_path(name, LOOKUP_FOLLOW, &path); - kfree(name); + err = user_path_at(AT_FDCWD, upath, LOOKUP_FOLLOW, &path); if (err) return err; -- cgit v1.2.3 From 26330a9226417c9a3395db9fdb403f7d7371e6b7 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Thu, 11 Jun 2026 13:42:26 +0200 Subject: bpf: Add support to specify uprobe_multi target via file descriptor Allow uprobe_multi link to identify the target binary by an already opened file descriptor. Adding new BPF_F_UPROBE_MULTI_PATH_FD flag and the path_fd field for the attr.link_create.uprobe_multi struct. When the flag is set, we resolve the target from path_fd, without the flag, we keep the existing string path behavior. I don't see a use case for supporting O_PATH file descriptors, because we need to read the binary first to get probes offsets, so I'm using the CLASS(fd, f), which fails for O_PATH fds. Assisted-by: Codex:GPT-5.4 Signed-off-by: Jiri Olsa Link: https://lore.kernel.org/r/20260611114230.950379-4-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/trace/bpf_trace.c | 43 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 6 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index f8990bc6b64c..82f8feea6931 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -23,6 +23,7 @@ #include #include #include +#include #include @@ -3214,6 +3215,38 @@ static u64 bpf_uprobe_multi_cookie(struct bpf_run_ctx *ctx) return run_ctx->uprobe->cookie; } +static int bpf_uprobe_multi_get_path(const union bpf_attr *attr, struct path *path) +{ + void __user *upath = u64_to_user_ptr(attr->link_create.uprobe_multi.path); + u32 path_fd = attr->link_create.uprobe_multi.path_fd; + u32 flags = attr->link_create.uprobe_multi.flags; + + if (flags & BPF_F_UPROBE_MULTI_PATH_FD) { + /* + * When BPF_F_UPROBE_MULTI_PATH_FD is set, the executable is + * identified by path_fd, upath must be NULL. + */ + if (upath) + return -EINVAL; + + CLASS(fd, f)(path_fd); + if (fd_empty(f)) + return -EBADF; + *path = fd_file(f)->f_path; + path_get(path); + return 0; + } + + /* + * When BPF_F_UPROBE_MULTI_PATH_FD is not set, the path is resolved + * relative to the cwd (AT_FDCWD) or absolute using the upath string. + */ + if (!upath || path_fd) + return -EINVAL; + + return user_path_at(AT_FDCWD, upath, LOOKUP_FOLLOW, path); +} + int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) { struct bpf_uprobe_multi_link *link = NULL; @@ -3223,7 +3256,6 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr struct task_struct *task = NULL; unsigned long __user *uoffsets; u64 __user *ucookies; - void __user *upath; unsigned long size; u32 flags, cnt, i; struct path path; @@ -3241,19 +3273,18 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr return -EINVAL; flags = attr->link_create.uprobe_multi.flags; - if (flags & ~BPF_F_UPROBE_MULTI_RETURN) + if (flags & ~(BPF_F_UPROBE_MULTI_RETURN | BPF_F_UPROBE_MULTI_PATH_FD)) return -EINVAL; /* - * path, offsets and cnt are mandatory, + * offsets and cnt are mandatory, * ref_ctr_offsets and cookies are optional */ - upath = u64_to_user_ptr(attr->link_create.uprobe_multi.path); uoffsets = u64_to_user_ptr(attr->link_create.uprobe_multi.offsets); cnt = attr->link_create.uprobe_multi.cnt; pid = attr->link_create.uprobe_multi.pid; - if (!upath || !uoffsets || !cnt || pid < 0) + if (!uoffsets || !cnt || pid < 0) return -EINVAL; if (cnt > MAX_UPROBE_MULTI_CNT) return -E2BIG; @@ -3271,7 +3302,7 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr !access_ok(ucookies, size)) return -EFAULT; - err = user_path_at(AT_FDCWD, upath, LOOKUP_FOLLOW, &path); + err = bpf_uprobe_multi_get_path(attr, &path); if (err) return err; -- cgit v1.2.3 From d5dc200c3a3f217de072af269dd90adddf90e48d Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Tue, 16 Jun 2026 10:30:56 +0200 Subject: bpf: Add missing access_ok call to copy_user_syms As reported by sashiko we use __get_user without prior access_ok call on the user space pointer. Adding the missing call for the whole pointer array. Plus removing the err check in the error path, because it's not needed and also we can return -ENOMEM directly from the first kvmalloc_array fail path. Cc: stable@vger.kernel.org [1] https://lore.kernel.org/bpf/20260611115503.AC16D1F00893@smtp.kernel.org/ Fixes: 0236fec57a15 ("bpf: Resolve symbols with ftrace_lookup_symbols for kprobe multi link") Reported-by: Sashiko Closes: https://lore.kernel.org/bpf/20260611115503.AC16D1F00893@smtp.kernel.org/ Signed-off-by: Jiri Olsa Reviewed-by: Emil Tsalapatis Link: https://lore.kernel.org/r/20260616083056.405652-1-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- kernel/trace/bpf_trace.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 82f8feea6931..75495a5c3507 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -2376,9 +2376,12 @@ static int copy_user_syms(struct user_syms *us, unsigned long __user *usyms, u32 int err = -ENOMEM; unsigned int i; + if (!access_ok(usyms, cnt * sizeof(*usyms))) + return -EFAULT; + syms = kvmalloc_array(cnt, sizeof(*syms), GFP_KERNEL); if (!syms) - goto error; + return -ENOMEM; buf = kvmalloc_array(cnt, KSYM_NAME_LEN, GFP_KERNEL); if (!buf) @@ -2403,10 +2406,8 @@ static int copy_user_syms(struct user_syms *us, unsigned long __user *usyms, u32 return 0; error: - if (err) { - kvfree(syms); - kvfree(buf); - } + kvfree(syms); + kvfree(buf); return err; } -- cgit v1.2.3 From 72c8646956ffc8050bb8be5988a0f28fc37e1ac4 Mon Sep 17 00:00:00 2001 From: Martin Kaiser Date: Thu, 25 Jun 2026 08:34:45 +0900 Subject: tracing: probes: fix typo in a log message Fix a typo ("Invalid $-variable") in a log message. Link: https://lore.kernel.org/all/20260507081041.885781-4-martin@kaiser.cx/ Fixes: ab105a4fb894 ("tracing: Use tracing error_log with probe events") Signed-off-by: Martin Kaiser Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/trace_probe.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h index 15758cc11fc6..0f09f7aaf93f 100644 --- a/kernel/trace/trace_probe.h +++ b/kernel/trace/trace_probe.h @@ -511,7 +511,7 @@ extern int traceprobe_define_arg_fields(struct trace_event_call *event_call, C(NO_RETVAL, "This function returns 'void' type"), \ C(BAD_STACK_NUM, "Invalid stack number"), \ C(BAD_ARG_NUM, "Invalid argument number"), \ - C(BAD_VAR, "Invalid $-valiable specified"), \ + C(BAD_VAR, "Invalid $-variable specified"), \ C(BAD_REG_NAME, "Invalid register name"), \ C(BAD_MEM_ADDR, "Invalid memory address"), \ C(BAD_IMM, "Invalid immediate value"), \ -- cgit v1.2.3 From 251a8fe1b9aedccd298b77bc28426d564c5a923f Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Thu, 25 Jun 2026 08:34:46 +0900 Subject: tracing/probes: Remove WARN_ON_ONCE from parse_btf_arg Sashiko found that user can cause this WARN_ON_ONCE() easily with adding a kprobe event based on a raw address with BTF parameter. Since this is not an unexpected condition, remove the WARN_ON_ONCE(). Link: https://lore.kernel.org/all/178177265367.2059927.13789953014706792126.stgit@mhiramat.tok.corp.google.com/ Link: https://sashiko.dev/#/patchset/178165816303.269421.7302603996990753309.stgit%40devnote2 Reported-by: Sashiko Fixes: b576e09701c7 ("tracing/probes: Support function parameters if BTF is available") Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/trace_probe.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index fd1caa1f9723..98532c503d02 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -678,7 +678,7 @@ static int parse_btf_arg(char *varname, int i, is_ptr, ret; u32 tid; - if (WARN_ON_ONCE(!ctx->funcname && !(ctx->flags & TPARG_FL_TEVENT))) + if (!ctx->funcname && !(ctx->flags & TPARG_FL_TEVENT)) return -EINVAL; is_ptr = split_next_field(varname, &field, ctx); -- cgit v1.2.3 From 206b25c09080cc20fd4c2bea12d59df4b7ba2121 Mon Sep 17 00:00:00 2001 From: Martin Kaiser Date: Thu, 25 Jun 2026 08:34:46 +0900 Subject: tracing: eprobe: read the complete FILTER_PTR_STRING pointer For a char * element in an event, the FILTER_PTR_STRING filter type is used. When the event occurs, a pointer is stored in the ringbuffer. If an eprobe references such a char * element of a "base event", the stored pointer is truncated when it's read from the ringbuffer. $ cd /sys/kernel/tracing $ echo 'e rcu.rcu_utilization $s:x64 $s:string' > dynamic_events $ echo 1 > tracing_on $ echo 1 > events/eprobes/enable $ sleep 1 $ echo 0 > events/eprobes/enable $ cat trace -0 ...: (rcu.rcu_utilization) arg1=0x4f arg2=(fault) -0 ...: (rcu.rcu_utilization) arg1=0x2 arg2=(fault) The problem is in get_event_field val = (unsigned long)(*(char *)addr); addr points to the position in the ringbuffer where the pointer was stored. The assignment reads only the lowest byte of the pointer. Fix the cast to read the whole pointer. The output of the test above is now -0 ... arg1=0xffffffff81c7d3f3 arg2="Start scheduler-tick" -0 ... arg1=0xffffffff81c57340 arg2="End scheduler-tick" Link: https://lore.kernel.org/all/20260620145339.3234726-1-martin@kaiser.cx/ Fixes: f04dec93466a ("tracing/eprobes: Fix reading of string fields") Signed-off-by: Martin Kaiser Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/trace_eprobe.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_eprobe.c b/kernel/trace/trace_eprobe.c index b66d6196338d..50518b071414 100644 --- a/kernel/trace/trace_eprobe.c +++ b/kernel/trace/trace_eprobe.c @@ -315,7 +315,7 @@ get_event_field(struct fetch_insn *code, void *rec) val = (unsigned long)addr; break; case FILTER_PTR_STRING: - val = (unsigned long)(*(char *)addr); + val = *(unsigned long *)addr; break; default: WARN_ON_ONCE(1); -- cgit v1.2.3 From 9a667b7750dda88cbf1cca96a53a2163b2ee71f7 Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Thu, 25 Jun 2026 08:34:47 +0900 Subject: tracing/probes: Fix double addition of offset for @+FOFFSET Since commit 533059281ee5 ("tracing: probeevent: Introduce new argument fetching code") wrongly use @offset local variable during the parsing, the offset value is added twice when dereferencing. Reset the @offset after setting it in FETCH_OP_FOFFS. Link: https://lore.kernel.org/all/178217905962.643090.1978577464942171332.stgit@devnote2/ Fixes: 533059281ee5 ("tracing: probeevent: Introduce new argument fetching code") Signed-off-by: Masami Hiramatsu (Google) Cc: stable@vger.kernel.org --- kernel/trace/trace_probe.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index 98532c503d02..502fa6da5949 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -1241,6 +1241,7 @@ parse_probe_arg(char *arg, const struct fetch_type *type, code->op = FETCH_OP_FOFFS; code->immediate = (unsigned long)offset; // imm64? + offset = 0; } else { /* uprobes don't support symbols */ if (!(ctx->flags & TPARG_FL_KERNEL)) { -- cgit v1.2.3 From 367c49d6e283c17b56a31e7a8d964a079244264c Mon Sep 17 00:00:00 2001 From: Sechang Lim Date: Thu, 25 Jun 2026 08:34:48 +0900 Subject: tracing/fprobe: Fix NULL pointer dereference in fprobe_fgraph_entry() fprobe_fgraph_entry() sizes a shadow-stack reservation in one walk of the per-ip fprobe list and fills it in a second walk, both under rcu_read_lock() only. A fprobe registered on an already-live ip can become visible between the two walks, so the fill walk processes an exit_handler the sizing walk did not count and used runs past reserved_words. If the sizing walk counted nothing, fgraph_data is NULL and the first write_fprobe_header() faults: Oops: general protection fault, probably for non-canonical address ... KASAN: null-ptr-deref in range [0x0000000000000000-0x0000000000000007] RIP: 0010:fprobe_fgraph_entry+0xa38/0xf10 kernel/trace/fprobe.c:167 Call Trace: function_graph_enter_regs+0x44c/0xa10 kernel/trace/fgraph.c:677 ftrace_graph_func+0xc5/0x140 arch/x86/kernel/ftrace.c:671 __kernel_text_address+0x9/0x40 kernel/extable.c:78 arch_stack_walk+0x117/0x170 arch/x86/kernel/stacktrace.c:26 kmem_cache_free+0x188/0x580 mm/slub.c:6378 tcp_data_queue+0x18d/0x6550 net/ipv4/tcp_input.c:5590 [...] The list cannot be frozen across the two walks, so skip a node that does not fit the reservation and count it as missed. Link: https://lore.kernel.org/all/20260619184425.3824774-1-rhkrqnwk98@gmail.com/ Fixes: 4346ba160409 ("fprobe: Rewrite fprobe on function-graph tracer") Signed-off-by: Sechang Lim Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/fprobe.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'kernel/trace') diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c index f378613ad120..f215990b9061 100644 --- a/kernel/trace/fprobe.c +++ b/kernel/trace/fprobe.c @@ -613,6 +613,16 @@ static int fprobe_fgraph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops continue; data_size = fp->entry_data_size; + /* + * The list may have grown since it was sized, so this node + * may not fit. Skip it as missed rather than overrun the + * reservation. + */ + if (fp->exit_handler && + used + FPROBE_HEADER_SIZE_IN_LONG + SIZE_IN_LONG(data_size) > reserved_words) { + fp->nmissed++; + continue; + } if (data_size && fp->exit_handler) data = fgraph_data + used + FPROBE_HEADER_SIZE_IN_LONG; else -- cgit v1.2.3 From a369299c3f785cf556bbef2de2db0aa2d294c4c9 Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Thu, 25 Jun 2026 08:34:48 +0900 Subject: tracing/probes: Make the $ prefix mandatory for comm access Since $comm or $COMM are not event field but special fetcharg variables to access current->comm, It should not be accessed without '$' prefix even with typecast. Link: https://lore.kernel.org/all/178231209724.732967.12049805699091810641.stgit@devnote2/ Fixes: 69efd863a785 ("tracing/eprobes: Allow use of BTF names to dereference pointers") Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/trace_probe.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index 502fa6da5949..d17cfee77d9c 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -342,10 +342,6 @@ static int parse_trace_event(char *arg, struct fetch_insn *code, ret = parse_trace_event_arg(arg, code, ctx); if (!ret) return 0; - if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) { - code->op = FETCH_OP_COMM; - return 0; - } return -EINVAL; } @@ -1068,8 +1064,14 @@ static int parse_probe_vars(char *orig_arg, const struct fetch_type *t, int len; if (ctx->flags & TPARG_FL_TEVENT) { - if (parse_trace_event(arg, code, ctx) < 0) + if (parse_trace_event(arg, code, ctx) < 0) { + /* 'comm' should be checked after field parsing. */ + if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) { + code->op = FETCH_OP_COMM; + return 0; + } goto inval; + } return 0; } -- cgit v1.2.3