diff options
Diffstat (limited to 'kernel')
60 files changed, 1117 insertions, 497 deletions
diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index ef315b168b29..0ce26b538075 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -436,7 +436,7 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, void __percpu *pptr; void *ptr, *val; u32 size; - int cpu; + int cpu, off = 0; if (unlikely((map_flags & BPF_F_LOCK) || (u32)map_flags > BPF_F_ALL_CPUS)) /* unknown flags */ @@ -468,9 +468,10 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, } for_each_possible_cpu(cpu) { ptr = per_cpu_ptr(pptr, cpu); - val = (map_flags & BPF_F_ALL_CPUS) ? value : value + size * cpu; + val = (map_flags & BPF_F_ALL_CPUS) ? value : value + off; copy_map_value(map, ptr, val); bpf_obj_cancel_fields(map, ptr); + off += size; } unlock: rcu_read_unlock(); diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c index a2b18a9f1694..47282ffeeaf9 100644 --- a/kernel/bpf/backtrack.c +++ b/kernel/bpf/backtrack.c @@ -520,37 +520,34 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, return -EFAULT; } } else if (opcode == BPF_EXIT) { - bool r0_precise; + bool from_subprog_call, r0_precise; + + /* BPF_EXIT in subprog or callback always returns + * right after the call instruction, so by checking + * whether the instruction at subseq_idx-1 is subprog + * call or not we can distinguish actual exit from + * *subprog* from exit from *callback*. In the former + * case, we need to propagate r0 precision, if + * necessary. In the former we never do that. + */ + from_subprog_call = subseq_idx - 1 >= 0 && + bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]); + + r0_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_0); /* Backtracking to a nested function call, 'idx' is a part of * the inner frame 'subseq_idx' is a part of the outer frame. * In case of a regular function call, instructions giving * precision to registers R1-R5 should have been found already. - * In case of a callback, it is ok to have R1-R5 marked for - * backtracking, as these registers are set by the function - * invoking callback. + * In case of a callback from bpf_loop(), R{1,4} in the calling + * frame would be set as precise and that is correct. */ - if (subseq_idx >= 0 && bpf_calls_callback(env, subseq_idx)) - for (i = BPF_REG_1; i <= BPF_REG_5; i++) - bt_clear_reg(bt, i); - if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { + if (from_subprog_call && (bt_reg_mask(bt) & BPF_REGMASK_ARGS)) { verifier_bug(env, "backtracking exit unexpected regs %x", bt_reg_mask(bt)); return -EFAULT; } - /* BPF_EXIT in subprog or callback always returns - * right after the call instruction, so by checking - * whether the instruction at subseq_idx-1 is subprog - * call or not we can distinguish actual exit from - * *subprog* from exit from *callback*. In the former - * case, we need to propagate r0 precision, if - * necessary. In the former we never do that. - */ - r0_precise = subseq_idx - 1 >= 0 && - bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) && - bt_is_reg_set(bt, BPF_REG_0); - bt_clear_reg(bt, BPF_REG_0); if (bt_subprog_enter(bt)) return -EFAULT; @@ -582,16 +579,29 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, */ } } else if (class == BPF_LD) { - if (!bt_is_reg_set(bt, dreg)) - return 0; - bt_clear_reg(bt, dreg); /* It's ld_imm64 or ld_abs or ld_ind. * For ld_imm64 no further tracking of precision * into parent is necessary */ - if (mode == BPF_IND || mode == BPF_ABS) - /* to be analyzed */ - return -ENOTSUPP; + if (mode == BPF_IMM) { + bt_clear_reg(bt, dreg); + return 0; + } + /* + * BPF_{IND,ABS} are modelled as two branches: + * - fallthrough; + * - implicit subprogram exit. + * It is necessary to switch current frame if + * implicit subprogram exit branch is backtracked. + */ + if (mode == BPF_IND || mode == BPF_ABS) { + if (bt_is_reg_set(bt, dreg)) + return -ENOTSUPP; + if (subseq_idx != idx + 1) + if (bt_subprog_enter(bt)) + return -EFAULT; + return 0; + } } /* Propagate precision marks to linked registers, to account for * registers marked as precise in this function. diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c index 14a5fdfa0421..b40eb404adab 100644 --- a/kernel/bpf/bpf_iter.c +++ b/kernel/bpf/bpf_iter.c @@ -754,7 +754,7 @@ const struct bpf_func_proto bpf_loop_proto = { .func = bpf_loop, .gpl_only = false, .ret_type = RET_INTEGER, - .arg1_type = ARG_ANYTHING, + .arg1_type = ARG_SCALAR, .arg2_type = ARG_PTR_TO_FUNC, .arg3_type = ARG_PTR_TO_STACK_OR_NULL, .arg4_type = ARG_ANYTHING, diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index da36d4b9d31a..9f33e95d5741 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -2911,14 +2911,29 @@ static void btf_modifier_show(const struct btf *btf, else t = btf_type_skip_modifiers(btf, type_id, NULL); - btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show); + /* + * A modifier can resolve to void, which has no show op; print a + * placeholder rather than dereferencing NULL. + */ + if (!btf_type_ops(t)) + btf_df_show(btf, t, type_id, data, bits_offset, show); + else + btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show); } static void btf_var_show(const struct btf *btf, const struct btf_type *t, u32 type_id, void *data, u8 bits_offset, struct btf_show *show) { - t = btf_type_id_resolve(btf, &type_id); + /* + * btf_type_id_resolve() dereferences btf->resolved_ids, which is NULL + * for a base BTF (e.g. the vmlinux BTF that bpf_snprintf_btf() uses). + * Resolve the var's type directly in that case. + */ + if (btf->resolved_ids) + t = btf_type_id_resolve(btf, &type_id); + else + t = btf_type_skip_modifiers(btf, t->type, &type_id); btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show); } @@ -6657,6 +6672,10 @@ struct bpf_raw_tp_null_args { static const struct bpf_raw_tp_null_args raw_tp_null_args[] = { /* sched */ { "sched_pi_setprio", 0x10 }, + /* + * do_wait() passes NULL for wait4(-1) and waitid(P_ALL). + */ + { "sched_process_wait", 0x1 }, /* ... from sched_numa_pair_template event class */ { "sched_stick_numa", 0x100 }, { "sched_swap_numa", 0x100 }, @@ -6717,6 +6736,9 @@ static const struct bpf_raw_tp_null_args raw_tp_null_args[] = { { "rxrpc_resend", 0x10 }, { "rxrpc_tq", 0x10 }, { "rxrpc_client", 0x1 }, + /* signal */ + { "signal_generate", 0x20 }, + { "signal_deliver", 0x20 }, /* skb */ {"kfree_skb", 0x1000}, /* sunrpc */ @@ -8727,6 +8749,7 @@ BPF_CALL_4(bpf_btf_find_by_name_kind, char *, name, int, name_sz, u32, kind, int const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = { .func = bpf_btf_find_by_name_kind, .gpl_only = false, + .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg2_type = ARG_MEM_SIZE, diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c index 0f13c13f4133..842c7d1eabcc 100644 --- a/kernel/bpf/cfg.c +++ b/kernel/bpf/cfg.c @@ -125,6 +125,7 @@ static int push_insn(int t, int w, int e, struct bpf_verifier_env *env) /* mark branch target for state pruning */ mark_prune_point(env, w); mark_jmp_point(env, w); + mark_jump_target(env, w); } if (insn_state[w] == 0) { @@ -403,6 +404,7 @@ static int visit_gotox_insn(int t, struct bpf_verifier_env *env) } mark_jmp_point(env, w); + mark_jump_target(env, w); /* EXPLORED || DISCOVERED */ if (insn_state[w]) @@ -564,6 +566,7 @@ static int visit_insn(int t, struct bpf_verifier_env *env) mark_prune_point(env, t + off + 1); mark_jmp_point(env, t + off + 1); + mark_jump_target(env, t + off + 1); return ret; diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index d55e737ed75a..8b294dfc1ad4 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -1128,11 +1128,6 @@ void *bpf_jit_alloc_exec(unsigned long size) return execmem_alloc(EXECMEM_BPF, size); } -void *bpf_jit_alloc_exec_rw(unsigned long size) -{ - return execmem_alloc_rw(EXECMEM_BPF, size); -} - void bpf_jit_free_exec(void *addr) { execmem_free(addr); diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c index b682fd2be443..0abbbe177e31 100644 --- a/kernel/bpf/diagnostics.c +++ b/kernel/bpf/diagnostics.c @@ -266,7 +266,7 @@ static char *diag_fmt_alloc(struct bpf_verifier_env *env, size_t size) } capacity = max_t(size_t, BPF_DIAG_FMT_CHUNK_SIZE, size); - chunk = kmalloc(struct_size(chunk, data, capacity), GFP_KERNEL_ACCOUNT); + chunk = kmalloc_flex(*chunk, data, capacity, GFP_KERNEL_ACCOUNT); if (!chunk) return NULL; diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c index 50b3ca5149a0..3ce8d74b0e40 100644 --- a/kernel/bpf/disasm.c +++ b/kernel/bpf/disasm.c @@ -7,6 +7,9 @@ #include "disasm.h" +/* Only defined by the non-UAPI linux/filter.h, which this file cannot use. */ +#define BPF_PROBE_ATOMIC 0xe0 + #define __BPF_FUNC_STR_FN(x) [BPF_FUNC_ ## x] = __stringify(bpf_ ## x) static const char * const func_id_str[] = { __BPF_FUNC_MAPPER(__BPF_FUNC_STR_FN) @@ -226,57 +229,57 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, insn->imm); } } else if (class == BPF_STX) { + const char *probe_pfx = BPF_MODE(insn->code) == BPF_PROBE_ATOMIC ? "probe " : ""; + bool atomic = BPF_MODE(insn->code) == BPF_ATOMIC || + BPF_MODE(insn->code) == BPF_PROBE_ATOMIC; + if (BPF_MODE(insn->code) == BPF_MEM) verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = r%d", insn->code, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, insn->src_reg); - else if (BPF_MODE(insn->code) == BPF_ATOMIC && + else if (atomic && (insn->imm == BPF_ADD || insn->imm == BPF_AND || insn->imm == BPF_OR || insn->imm == BPF_XOR)) { - verbose(cbs->private_data, "(%02x) lock *(%s *)(r%d %+d) %s r%d", - insn->code, + verbose(cbs->private_data, "(%02x) %slock *(%s *)(r%d %+d) %s r%d", + insn->code, probe_pfx, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, bpf_alu_string[BPF_OP(insn->imm) >> 4], insn->src_reg); - } else if (BPF_MODE(insn->code) == BPF_ATOMIC && + } else if (atomic && (insn->imm == (BPF_ADD | BPF_FETCH) || insn->imm == (BPF_AND | BPF_FETCH) || insn->imm == (BPF_OR | BPF_FETCH) || insn->imm == (BPF_XOR | BPF_FETCH))) { - verbose(cbs->private_data, "(%02x) r%d = atomic%s_fetch_%s((%s *)(r%d %+d), r%d)", - insn->code, insn->src_reg, + verbose(cbs->private_data, "(%02x) %sr%d = atomic%s_fetch_%s((%s *)(r%d %+d), r%d)", + insn->code, probe_pfx, insn->src_reg, BPF_SIZE(insn->code) == BPF_DW ? "64" : "", bpf_atomic_alu_string[BPF_OP(insn->imm) >> 4], bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, insn->src_reg); - } else if (BPF_MODE(insn->code) == BPF_ATOMIC && - insn->imm == BPF_CMPXCHG) { - verbose(cbs->private_data, "(%02x) r0 = atomic%s_cmpxchg((%s *)(r%d %+d), r0, r%d)", - insn->code, + } else if (atomic && insn->imm == BPF_CMPXCHG) { + verbose(cbs->private_data, "(%02x) %sr0 = atomic%s_cmpxchg((%s *)(r%d %+d), r0, r%d)", + insn->code, probe_pfx, BPF_SIZE(insn->code) == BPF_DW ? "64" : "", bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, insn->src_reg); - } else if (BPF_MODE(insn->code) == BPF_ATOMIC && - insn->imm == BPF_XCHG) { - verbose(cbs->private_data, "(%02x) r%d = atomic%s_xchg((%s *)(r%d %+d), r%d)", - insn->code, insn->src_reg, + } else if (atomic && insn->imm == BPF_XCHG) { + verbose(cbs->private_data, "(%02x) %sr%d = atomic%s_xchg((%s *)(r%d %+d), r%d)", + insn->code, probe_pfx, insn->src_reg, BPF_SIZE(insn->code) == BPF_DW ? "64" : "", bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, insn->src_reg); - } else if (BPF_MODE(insn->code) == BPF_ATOMIC && - insn->imm == BPF_LOAD_ACQ) { - verbose(cbs->private_data, "(%02x) r%d = load_acquire((%s *)(r%d %+d))", - insn->code, insn->dst_reg, + } else if (atomic && insn->imm == BPF_LOAD_ACQ) { + verbose(cbs->private_data, "(%02x) %sr%d = load_acquire((%s *)(r%d %+d))", + insn->code, probe_pfx, insn->dst_reg, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->src_reg, insn->off); - } else if (BPF_MODE(insn->code) == BPF_ATOMIC && - insn->imm == BPF_STORE_REL) { - verbose(cbs->private_data, "(%02x) store_release((%s *)(r%d %+d), r%d)", - insn->code, + } else if (atomic && insn->imm == BPF_STORE_REL) { + verbose(cbs->private_data, "(%02x) %sstore_release((%s *)(r%d %+d), r%d)", + insn->code, probe_pfx, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, insn->src_reg); } else { @@ -295,7 +298,8 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, verbose(cbs->private_data, "BUG_st_%02x", insn->code); } } else if (class == BPF_LDX) { - if (BPF_MODE(insn->code) != BPF_MEM && BPF_MODE(insn->code) != BPF_MEMSX) { + if ((BPF_MODE(insn->code) != BPF_MEM && BPF_MODE(insn->code) != BPF_MEMSX) || + (BPF_MODE(insn->code) == BPF_MEMSX && BPF_SIZE(insn->code) == BPF_DW)) { verbose(cbs->private_data, "BUG_ldx_%02x", insn->code); return; } diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 65b441e4a351..52d3cec33672 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -13,10 +13,15 @@ #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) +/* + * Matches BPF_PROBE_ATOMIC too: bpf_convert_ctx_accesses() rewrites arena + * atomics before bpf_opt_subreg_zext_lo32_rnd_hi32() runs. + */ static bool is_cmpxchg_insn(const struct bpf_insn *insn) { return BPF_CLASS(insn->code) == BPF_STX && - BPF_MODE(insn->code) == BPF_ATOMIC && + (BPF_MODE(insn->code) == BPF_ATOMIC || + BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) && insn->imm == BPF_CMPXCHG; } diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index d40cb5dd446c..6f331c80130d 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -511,7 +511,7 @@ static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma, if (IS_ERR_OR_NULL(map->record)) return 0; - hrec = kzalloc(sizeof(*hrec), GFP_KERNEL); + hrec = kzalloc_obj(*hrec); if (!hrec) return -ENOMEM; hrec->key_size = map->key_size; @@ -530,6 +530,9 @@ static int htab_map_check_btf(struct bpf_map *map, const struct btf *btf, { struct bpf_htab *htab = container_of(map, struct bpf_htab, map); + if (btf_type_is_void(key_type)) + return -EINVAL; + if (htab_is_prealloc(htab)) return 0; /* @@ -1025,7 +1028,7 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr, } else { u32 size = round_up(htab->map.value_size, 8); void *val; - int cpu; + int cpu, off = 0; if (map_flags & BPF_F_CPU) { cpu = map_flags >> 32; @@ -1037,9 +1040,10 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr, for_each_possible_cpu(cpu) { ptr = per_cpu_ptr(pptr, cpu); - val = (map_flags & BPF_F_ALL_CPUS) ? value : value + size * cpu; + val = (map_flags & BPF_F_ALL_CPUS) ? value : value + off; copy_map_value(&htab->map, ptr, val); bpf_obj_cancel_fields(&htab->map, ptr); + off += size; } } } @@ -2864,16 +2868,6 @@ static int rhtab_map_alloc_check(union bpf_attr *attr) return htab_map_alloc_check(attr); } -static void rhtab_check_and_free_fields(struct bpf_rhtab *rhtab, - struct rhtab_elem *elem) -{ - if (IS_ERR_OR_NULL(rhtab->map.record)) - return; - - bpf_obj_free_fields(rhtab->map.record, - rhtab_elem_value(elem, rhtab->map.key_size)); -} - static void rhtab_mem_dtor(void *obj, void *ctx) { struct htab_btf_record *hrec = ctx; @@ -2963,8 +2957,8 @@ static int rhtab_delete_elem(struct bpf_rhtab *rhtab, struct rhtab_elem *elem, v rhtab_read_elem_value(&rhtab->map, copy, elem, flags); check_and_init_map_value(&rhtab->map, copy); } - /* Release internal structs: kptr, bpf_timer, task_work, wq */ - rhtab_check_and_free_fields(rhtab, elem); + bpf_obj_cancel_fields(&rhtab->map, + rhtab_elem_value(elem, rhtab->map.key_size)); bpf_mem_cache_free_rcu(&rhtab->ma, elem); return 0; } @@ -3005,7 +2999,6 @@ static int rhtab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, void static long rhtab_map_update_existing(struct bpf_map *map, struct rhtab_elem *elem, void *value, u64 map_flags) { - struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map); void *old_val = rhtab_elem_value(elem, map->key_size); if (map_flags & BPF_NOEXIST) @@ -3025,7 +3018,7 @@ static long rhtab_map_update_existing(struct bpf_map *map, struct rhtab_elem *el * kptrs/etc. still sit in the slot. Cancel them after the copy * to match arraymap's update semantics. */ - rhtab_check_and_free_fields(rhtab, elem); + bpf_obj_cancel_fields(map, old_val); return 0; } @@ -3066,7 +3059,6 @@ static long rhtab_map_update_elem(struct bpf_map *map, void *key, void *value, u memcpy(elem->data, key, map->key_size); copy_map_value(map, rhtab_elem_value(elem, map->key_size), value); - check_and_init_map_value(map, rhtab_elem_value(elem, map->key_size)); /* Prevent deadlock for NMI programs attempting to take bucket lock */ bpf_disable_instrumentation(); @@ -3110,6 +3102,9 @@ static int rhtab_map_check_btf(struct bpf_map *map, const struct btf *btf, { struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map); + if (btf_type_is_void(key_type)) + return -EINVAL; + return bpf_ma_set_dtor(map, &rhtab->ma, rhtab_mem_dtor); } diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 74fc4b3f80d6..301fc60bddc4 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -85,7 +85,7 @@ static struct func_instance *call_instance(struct bpf_verifier_env *env, if (f) return f; - f = kvzalloc(sizeof(*f), GFP_KERNEL_ACCOUNT); + f = kvzalloc_obj(*f, GFP_KERNEL_ACCOUNT); if (!f) return ERR_PTR(-ENOMEM); f->callsite = lookup_key; diff --git a/kernel/bpf/local_storage.c b/kernel/bpf/local_storage.c index 23267213a17f..83cd527a2542 100644 --- a/kernel/bpf/local_storage.c +++ b/kernel/bpf/local_storage.c @@ -220,7 +220,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *_map, void *key, struct bpf_cgroup_storage *storage; void *val; u32 size; - int cpu; + int cpu, off = 0; if ((u32)map_flags & ~(BPF_ANY | BPF_EXIST | BPF_F_CPU | BPF_F_ALL_CPUS)) return -EINVAL; @@ -245,8 +245,9 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *_map, void *key, } size = round_up(_map->value_size, 8); for_each_possible_cpu(cpu) { - val = (map_flags & BPF_F_ALL_CPUS) ? value : value + size * cpu; + val = (map_flags & BPF_F_ALL_CPUS) ? value : value + off; copy_map_value(_map, per_cpu_ptr(storage->percpu_buf, cpu), val); + off += size; } unlock: rcu_read_unlock(); diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index 589770ca3d3a..fb032dfdc0de 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c @@ -862,7 +862,7 @@ struct bpf_verifier_log *bpf_log_attr_create_vlog(struct bpf_log_attr *attr_log, if (!size) return NULL; - log = kzalloc_obj(*log, GFP_KERNEL); + log = kzalloc_obj(*log); if (!log) return ERR_PTR(-ENOMEM); diff --git a/kernel/bpf/percpu_freelist.c b/kernel/bpf/percpu_freelist.c index 632762b57299..06ce588d13a3 100644 --- a/kernel/bpf/percpu_freelist.c +++ b/kernel/bpf/percpu_freelist.c @@ -17,6 +17,8 @@ int pcpu_freelist_init(struct pcpu_freelist *s) raw_res_spin_lock_init(&head->lock); head->first = NULL; } + raw_res_spin_lock_init(&s->extralist.lock); + s->extralist.first = NULL; return 0; } @@ -46,22 +48,28 @@ void __pcpu_freelist_push(struct pcpu_freelist *s, struct pcpu_freelist_node *node) { struct pcpu_freelist_head *head; - int cpu; + int cpu, this_cpu; if (___pcpu_freelist_push(this_cpu_ptr(s->freelist), node)) return; + this_cpu = raw_smp_processor_id(); while (true) { - for_each_cpu_wrap(cpu, cpu_possible_mask, raw_smp_processor_id()) { - if (cpu == raw_smp_processor_id()) + for_each_cpu_wrap(cpu, cpu_possible_mask, this_cpu) { + if (cpu == this_cpu) continue; + head = per_cpu_ptr(s->freelist, cpu); - if (raw_res_spin_lock(&head->lock)) - continue; - pcpu_freelist_push_node(head, node); - raw_res_spin_unlock(&head->lock); - return; + if (___pcpu_freelist_push(head, node)) + return; } + + /* + * Push cannot fail. Use the extra list when none of the + * per-CPU freelists can accept the node. + */ + if (___pcpu_freelist_push(&s->extralist, node)) + return; } } @@ -117,6 +125,17 @@ static struct pcpu_freelist_node *___pcpu_freelist_pop(struct pcpu_freelist *s) } raw_res_spin_unlock(&head->lock); } + + /* Per-CPU lists are empty or unavailable, try the extra list. */ + head = &s->extralist; + if (!READ_ONCE(head->first)) + return NULL; + if (raw_res_spin_lock(&head->lock)) + return NULL; + node = head->first; + if (node) + WRITE_ONCE(head->first, node->next); + raw_res_spin_unlock(&head->lock); return node; } diff --git a/kernel/bpf/percpu_freelist.h b/kernel/bpf/percpu_freelist.h index 914798b74967..980cf2884fd2 100644 --- a/kernel/bpf/percpu_freelist.h +++ b/kernel/bpf/percpu_freelist.h @@ -14,6 +14,7 @@ struct pcpu_freelist_head { struct pcpu_freelist { struct pcpu_freelist_head __percpu *freelist; + struct pcpu_freelist_head extralist; }; struct pcpu_freelist_node { diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index a839041e0d00..d09d4c3fe547 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -875,6 +875,7 @@ BPF_CALL_4(bpf_get_stack_sleepable, struct pt_regs *, regs, void *, buf, u32, si const struct bpf_func_proto bpf_get_stack_sleepable_proto = { .func = bpf_get_stack_sleepable, .gpl_only = true, + .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, @@ -928,6 +929,7 @@ BPF_CALL_4(bpf_get_task_stack_sleepable, struct task_struct *, task, void *, buf const struct bpf_func_proto bpf_get_task_stack_sleepable_proto = { .func = bpf_get_task_stack_sleepable, .gpl_only = false, + .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 4e6aafad33bd..66fb11b6c6a7 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -445,22 +445,19 @@ static void __clean_func_state(struct bpf_verifier_env *env, struct bpf_reg_state *spill = &st->stack[i].spilled_ptr; if (lo_live && stype == STACK_SPILL) { - u8 val = STACK_MISC; - if (spill->type != SCALAR_VALUE) continue; - /* - * 8 byte spill of scalar 0 where half slot is dead - * should become STACK_ZERO in lo 4 bytes. + * Can't replace with STACK_ZERO, because + * that requires bpf_mark_chain_precision(). */ if (bpf_register_is_null(spill)) - val = STACK_ZERO; + continue; for (j = 0; j < 4; j++) { u8 *t = &st->stack[i].slot_type[j]; if (*t == STACK_SPILL) - *t = val; + *t = STACK_MISC; } } bpf_mark_reg_not_init(env, spill); diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 6874ba1424af..c7bc9ba9b331 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -6568,6 +6568,7 @@ EXPORT_SYMBOL_NS(kern_sys_bpf, "BPF_INTERNAL"); static const struct bpf_func_proto bpf_sys_bpf_proto = { .func = bpf_sys_bpf, .gpl_only = false, + .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_ANYTHING, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, @@ -6593,6 +6594,7 @@ BPF_CALL_1(bpf_sys_close, u32, fd) static const struct bpf_func_proto bpf_sys_close_proto = { .func = bpf_sys_close, .gpl_only = false, + .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_ANYTHING, }; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index e421ea2b80c3..72a3f5998dd2 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -352,9 +352,18 @@ static bool reg_not_null(struct bpf_verifier_env *env, const struct bpf_reg_stat if (type_may_be_null(type)) return false; + /* + * The types below guarantee a non-NULL base, an unbounded offset can + * still wrap base + offset to zero. + */ + if (reg_smin(reg) <= -BPF_MAX_VAR_OFF || reg_smax(reg) >= BPF_MAX_VAR_OFF) + return false; + type = base_type(type); return type == PTR_TO_SOCKET || type == PTR_TO_TCP_SOCK || + type == PTR_TO_XDP_SOCK || + type == PTR_TO_BUF || type == PTR_TO_MAP_VALUE || type == PTR_TO_MAP_KEY || type == PTR_TO_SOCK_COMMON || @@ -4237,6 +4246,15 @@ static int mark_stack_arg_precision(struct bpf_verifier_env *env, int arg_idx) return mark_chain_precision_batch(env, env->cur_state); } +static int mark_arg_precision(struct bpf_verifier_env *env, argno_t argno) +{ + int regno = reg_from_argno(argno); + + if (regno >= 0) + return mark_chain_precision(env, regno); + return mark_stack_arg_precision(env, arg_idx_from_argno(argno)); +} + static int check_outgoing_stack_args(struct bpf_verifier_env *env, struct bpf_func_state *caller, int nargs, const char *callee_name, const struct btf *btf, const struct btf_param *args) @@ -4486,6 +4504,13 @@ static int map_kptr_match_type(struct bpf_verifier_env *env, if (type_flag(reg->type) & ~perm_flags) goto bad_type; + /* + * A BPF_KPTR_PERCPU field is read back as MEM_PERCPU, so the value + * stored in it must carry the same flag. + */ + if ((kptr_field->type == BPF_KPTR_PERCPU) != !!(reg->type & MEM_PERCPU)) + goto bad_type; + /* We need to verify reg->type and reg->btf, before accessing reg->btf */ reg_name = btf_type_name(reg->btf, reg->btf_id); @@ -4692,8 +4717,15 @@ static int check_map_kptr_access(struct bpf_verifier_env *env, return ret; } else if (class == BPF_STX) { val_reg = reg_state(env, value_regno); - if (!bpf_register_is_null(val_reg) && - map_kptr_match_type(env, kptr_field, val_reg, value_regno)) + if (bpf_register_is_null(val_reg)) { + /* + * This store is valid only because the scalar is known to be + * zero. Mark it precise so another scalar cannot be pruned + * against this state. + */ + return mark_chain_precision(env, value_regno); + } + if (map_kptr_match_type(env, kptr_field, val_reg, value_regno)) return -EACCES; } else if (class == BPF_ST) { if (insn->imm) { @@ -5300,6 +5332,15 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, if (!priv_stack_supported) subprog[idx].priv_stack_mode = NO_PRIV_STACK; process_func: + if (subprog[idx].has_ld_abs) { + for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) { + if (subprog[tmp].is_cb) { + verbose(env, "cannot use BPF_LD_[ABS|IND] within callback\n"); + return -EINVAL; + } + } + } + /* protect against potential stack overflow that might happen when * bpf2bpf calls get combined with tailcalls. Limit the caller's stack * depth for such case down to 256 so that the worst case scenario @@ -5483,7 +5524,7 @@ static int check_max_stack_depth(struct bpf_verifier_env *env) bool priv_stack_supported; int ret; - dinfo = kvcalloc(env->subprog_cnt, sizeof(*dinfo), GFP_KERNEL_ACCOUNT); + dinfo = kvzalloc_objs(*dinfo, env->subprog_cnt, GFP_KERNEL_ACCOUNT); if (!dinfo) return -ENOMEM; @@ -6020,7 +6061,13 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, return -EACCES; } - if (type_is_alloc(reg->type) && !type_is_non_owning_ref(reg->type) && + /* + * A fault-prone allocated object may still be read through a + * BPF_PROBE_MEM load after its lifetime protection ends. Writes + * through such pointers were rejected above. + */ + if (type_is_alloc(reg->type) && !bpf_may_fault_on_deref(reg->type) && + !type_is_non_owning_ref(reg->type) && !(reg->type & MEM_RCU) && !reg_is_referenced(env, reg)) { verifier_bug(env, "allocated object must have a referenced id"); return -EFAULT; @@ -7113,14 +7160,8 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, if (err && failure) *failure = BPF_MEM_SIZE_FAIL_MEMORY; - if (!err) { - int regno = reg_from_argno(size_argno); - - if (regno >= 0) - err = mark_chain_precision(env, regno); - else - err = mark_stack_arg_precision(env, arg_idx_from_argno(size_argno)); - } + if (!err) + err = mark_arg_precision(env, size_argno); return err; @@ -7137,7 +7178,7 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg int size, err = 0; if (bpf_register_is_null(reg)) - return 0; + return mark_arg_precision(env, argno); if (known_memory) *known_memory = true; @@ -7398,10 +7439,14 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state lock); return -EINVAL; } + /* + * Invalidate non-owning refs before RCU demotion clears their + * NON_OWN_REF flag. + */ + invalidate_non_owning_refs(env); + if (!in_rcu_cs(env)) invalidate_rcu_protected_refs(env); - - invalidate_non_owning_refs(env); } return 0; } @@ -8166,6 +8211,7 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { [ARG_MEM_SIZE] = &scalar_types, [ARG_MEM_SIZE_OR_ZERO] = &scalar_types, [ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types, + [ARG_SCALAR] = &scalar_types, [ARG_CONST_MAP_PTR] = &const_map_ptr_types, [ARG_PTR_TO_CTX] = &context_types, [ARG_PTR_TO_SOCK_COMMON] = &sock_types, @@ -8717,11 +8763,15 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, return err; } - if (bpf_register_is_null(reg) && type_may_be_null(arg_type)) + if (bpf_register_is_null(reg) && type_may_be_null(arg_type)) { /* A NULL register has a SCALAR_VALUE type, so skip * type checking. */ + err = mark_chain_precision(env, regno); + if (err) + return err; goto skip_type_check; + } /* arg_btf_id and arg_size are in a union. */ if (base_type(arg_type) == ARG_PTR_TO_BTF_ID || @@ -9501,7 +9551,7 @@ static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env) bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, stack, clear_mask, ({ if (reg->type & MEM_RCU) { bpf_diag_mod_begin(env, reg, NULL, BPF_DIAG_MOD_WRITE); - reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL); + reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL | NON_OWN_REF); reg->type |= PTR_UNTRUSTED; bpf_diag_mod_end(env); } @@ -9719,8 +9769,12 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, struct bpf_call_arg_meta meta; int err; - if (bpf_register_is_null(reg) && type_may_be_null(arg->arg_type)) + if (bpf_register_is_null(reg) && type_may_be_null(arg->arg_type)) { + err = mark_arg_precision(env, argno); + if (err) + return err; continue; + } memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */ err = check_reg_type(env, reg, argno, arg->arg_type, &arg->btf_id, &meta, @@ -9976,10 +10030,12 @@ int map_set_for_each_callback_args(struct bpf_verifier_env *env, callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; __mark_reg_known_zero(&callee->regs[BPF_REG_2]); callee->regs[BPF_REG_2].map_ptr = caller->regs[BPF_REG_1].map_ptr; + callee->regs[BPF_REG_2].map_uid = caller->regs[BPF_REG_1].map_uid; callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; __mark_reg_known_zero(&callee->regs[BPF_REG_3]); callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr; + callee->regs[BPF_REG_3].map_uid = caller->regs[BPF_REG_1].map_uid; /* pointer to stack or null */ callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3]; @@ -10057,6 +10113,7 @@ static int set_timer_callback_state(struct bpf_verifier_env *env, int insn_idx) { struct bpf_map *map_ptr = caller->regs[BPF_REG_1].map_ptr; + u32 map_uid = caller->regs[BPF_REG_1].map_uid; /* bpf_timer_set_callback(struct bpf_timer *timer, void *callback_fn); * callback_fn(struct bpf_map *map, void *key, void *value); @@ -10064,14 +10121,17 @@ static int set_timer_callback_state(struct bpf_verifier_env *env, callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP; __mark_reg_known_zero(&callee->regs[BPF_REG_1]); callee->regs[BPF_REG_1].map_ptr = map_ptr; + callee->regs[BPF_REG_1].map_uid = map_uid; callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; __mark_reg_known_zero(&callee->regs[BPF_REG_2]); callee->regs[BPF_REG_2].map_ptr = map_ptr; + callee->regs[BPF_REG_2].map_uid = map_uid; callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; __mark_reg_known_zero(&callee->regs[BPF_REG_3]); callee->regs[BPF_REG_3].map_ptr = map_ptr; + callee->regs[BPF_REG_3].map_uid = map_uid; /* unused */ bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); @@ -10171,6 +10231,7 @@ static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env, int insn_idx) { struct bpf_map *map_ptr = caller->regs[BPF_REG_3].map_ptr; + u32 map_uid = caller->regs[BPF_REG_3].map_uid; /* * callback_fn(struct bpf_map *map, void *key, void *value); @@ -10178,14 +10239,17 @@ static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env, callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP; __mark_reg_known_zero(&callee->regs[BPF_REG_1]); callee->regs[BPF_REG_1].map_ptr = map_ptr; + callee->regs[BPF_REG_1].map_uid = map_uid; callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; __mark_reg_known_zero(&callee->regs[BPF_REG_2]); callee->regs[BPF_REG_2].map_ptr = map_ptr; + callee->regs[BPF_REG_2].map_uid = map_uid; callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; __mark_reg_known_zero(&callee->regs[BPF_REG_3]); callee->regs[BPF_REG_3].map_ptr = map_ptr; + callee->regs[BPF_REG_3].map_uid = map_uid; /* unused */ bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); @@ -10233,9 +10297,10 @@ static void account_current_path(struct bpf_verifier_env *env) frame ? state->frame[frame - 1] : NULL); } -/* Are we currently verifying the callback for a rbtree helper that must - * be called with lock held? If so, no need to complain about unreleased - * lock +/* + * Are we currently verifying the callback for an rbtree kfunc that must + * be called with a lock held, or one of that callback's subprogs? If so, + * no need to complain about an unreleased lock. */ static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env) { @@ -10243,17 +10308,19 @@ static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env) struct bpf_insn *insn = env->prog->insnsi; struct bpf_func_state *callee; int kfunc_btf_id; + u32 frame; - if (!state->curframe) - return false; - - callee = state->frame[state->curframe]; + for (frame = state->curframe; frame; frame--) { + callee = state->frame[frame]; + if (!callee->in_callback_fn) + continue; - if (!callee->in_callback_fn) - return false; + kfunc_btf_id = insn[callee->callsite].imm; + if (is_rbtree_lock_required_kfunc(kfunc_btf_id)) + return true; + } - kfunc_btf_id = insn[callee->callsite].imm; - return is_rbtree_lock_required_kfunc(kfunc_btf_id); + return false; } static bool retval_range_within(struct bpf_retval_range range, const struct bpf_reg_state *reg) @@ -10630,33 +10697,45 @@ static struct bpf_insn_aux_data *cur_aux(const struct bpf_verifier_env *env) return &env->insn_aux_data[env->insn_idx]; } -static bool loop_flag_is_zero(struct bpf_verifier_env *env) +/* Returns 1 if R4 is a known zero, 0 if it is not, a negative errno on error. */ +static int loop_flag_is_zero(struct bpf_verifier_env *env) { struct bpf_reg_state *reg = reg_state(env, BPF_REG_4); - bool reg_is_null = bpf_register_is_null(reg); + int err; - if (reg_is_null) - mark_chain_precision(env, BPF_REG_4); + if (!bpf_register_is_null(reg)) + return 0; - return reg_is_null; + err = mark_chain_precision(env, BPF_REG_4); + if (err) + return err; + return 1; } -static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno) +static int update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno) { struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state; + int flag_is_zero; if (!state->initialized) { + flag_is_zero = loop_flag_is_zero(env); + if (flag_is_zero < 0) + return flag_is_zero; state->initialized = 1; - state->fit_for_inline = loop_flag_is_zero(env); + state->fit_for_inline = flag_is_zero; state->callback_subprogno = subprogno; - return; + return 0; } if (!state->fit_for_inline) - return; + return 0; - state->fit_for_inline = (loop_flag_is_zero(env) && + flag_is_zero = loop_flag_is_zero(env); + if (flag_is_zero < 0) + return flag_is_zero; + state->fit_for_inline = (flag_is_zero && state->callback_subprogno == subprogno); + return 0; } /* Returns whether or not the given map can potentially elide @@ -10868,6 +10947,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn verbose(env, "get_local_storage() doesn't support non-zero flags\n"); return -EINVAL; } + err = mark_chain_precision(env, BPF_REG_2); + if (err) + return err; break; case BPF_FUNC_for_each_map_elem: err = push_callback_call(env, insn, insn_idx, meta.subprogno, @@ -10885,7 +10967,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn err = check_bpf_snprintf_call(env, regs); break; case BPF_FUNC_loop: - update_loop_inline_state(env, meta.subprogno); + err = update_loop_inline_state(env, meta.subprogno); + if (err) + return err; /* Verifier relies on R1 value to determine if bpf_loop() iteration * is finished, thus mark it precise. */ @@ -11226,6 +11310,17 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (env->cur_state->curframe) { struct bpf_verifier_state *branch; + /* + * A taken tail call is modeled as a return from the current + * frame. A callback frame cannot be left that way because + * prepare_func_exit() would apply its return contract to the + * unknown R0 synthesized below. Stack-depth validation rejects + * this construct anyway. + */ + if (cur_func(env)->in_callback_fn) { + verbose(env, "cannot tail call within callback\n"); + return -EINVAL; + } mark_reg_scratched(env, BPF_REG_0); branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false); if (IS_ERR(branch)) @@ -12651,8 +12746,12 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me if (reg_is_referenced(env, reg)) update_ref_obj(&meta->ref_obj, reg); - if (bpf_register_is_null(reg) && type_may_be_null(kf_arg_type)) + if (bpf_register_is_null(reg) && type_may_be_null(kf_arg_type)) { + ret = mark_arg_precision(env, argno); + if (ret) + return ret; continue; + } if (is_kfunc_arg_map(btf, &args[i])) { ref_id = *reg2btf_ids[CONST_PTR_TO_MAP]; @@ -13146,7 +13245,7 @@ check_ok: bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } - if (!type_is_non_owning_ref(reg->type)) + if (!type_is_non_owning_ref(reg->type) && reg_is_referenced(env, reg)) meta->arg_owning_ref = true; rec = reg_btf_record(reg); @@ -13228,6 +13327,11 @@ check_ok: { int flags = PROCESS_RES_LOCK; + if (in_rbtree_lock_required_cb(env)) { + verbose(env, "can't res_spin_{lock,unlock} in rbtree cb\n"); + return -EACCES; + } + if (reg->type != PTR_TO_MAP_VALUE && reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { verbose(env, "%s doesn't point to map value or allocated object\n", reg_arg_name(env, argno)); @@ -14560,9 +14664,6 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn return -EINVAL; } - /* pointer types do not carry 32-bit bounds at the moment. */ - __mark_reg32_unbounded(dst_reg); - if (sanitize_needed(opcode)) { ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, &info, false); @@ -14570,6 +14671,14 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn return sanitize_err(env, insn, ret); } + /* + * Pointer types do not carry 32-bit bounds at the moment. Blank r32 + * only after sanitize_ptr_alu() may have snapshotted dst_reg into a + * speculative path: otherwise reg_bounds_sanity_check() might hit some + * constraints violations. + */ + __mark_reg32_unbounded(dst_reg); + switch (opcode) { case BPF_ADD: /* @@ -16295,6 +16404,13 @@ static int is_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_state *r if (__is_pointer_value(false, reg1) || __is_pointer_value(false, reg2)) { u64 val; + /* + * The low 32 bits of a valid pointer may well be zero, hence + * nothing below applies to a 32-bit comparison. + */ + if (is_jmp32) + return -1; + /* arrange that reg2 is a scalar, and reg1 is a pointer */ if (!is_reg_const(reg2, is_jmp32)) { opcode = flip_opcode(opcode); @@ -16856,6 +16972,16 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, return err; } + /* + * Collect the linked registers before env->{true,false}_reg{1,2} setup, + * otherwise ids dropped by collect_linked_regs() would be resurrected + * when env->{true,false}_reg{1,2} are copied back. + */ + if (BPF_SRC(insn->code) == BPF_X && src_reg->type == SCALAR_VALUE && src_reg->id) + collect_linked_regs(env, this_branch, src_reg->id, &linked_regs); + if (dst_reg->type == SCALAR_VALUE && dst_reg->id) + collect_linked_regs(env, this_branch, dst_reg->id, &linked_regs); + is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; env->false_reg1 = *dst_reg; env->false_reg2 = *src_reg; @@ -16910,10 +17036,6 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, * 'this_branch' and 'other_branch' share this history * if parent state is created. */ - if (BPF_SRC(insn->code) == BPF_X && src_reg->type == SCALAR_VALUE && src_reg->id) - collect_linked_regs(env, this_branch, src_reg->id, &linked_regs); - if (dst_reg->type == SCALAR_VALUE && dst_reg->id) - collect_linked_regs(env, this_branch, dst_reg->id, &linked_regs); if (linked_regs.cnt > 1) { err = bpf_push_jmp_history(env, this_branch, 0, 0, 0, linked_regs_pack(&linked_regs)); if (err) @@ -16963,7 +17085,6 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, */ if (!is_jmp32 && BPF_SRC(insn->code) == BPF_X && __is_pointer_value(false, src_reg) && __is_pointer_value(false, dst_reg) && - type_may_be_null(src_reg->type) != type_may_be_null(dst_reg->type) && base_type(src_reg->type) != PTR_TO_BTF_ID && base_type(dst_reg->type) != PTR_TO_BTF_ID) { eq_branch_regs = NULL; @@ -16979,9 +17100,11 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, break; } if (eq_branch_regs) { - if (type_may_be_null(src_reg->type)) + /* src == dst && dst != NULL => src != NULL */ + if (reg_not_null(env, dst_reg) && type_may_be_null(src_reg->type)) mark_ptr_not_null_reg(&eq_branch_regs[insn->src_reg]); - else + /* src == dst && src != NULL => dst != NULL */ + if (reg_not_null(env, src_reg) && type_may_be_null(dst_reg->type)) mark_ptr_not_null_reg(&eq_branch_regs[insn->dst_reg]); } } @@ -16996,6 +17119,15 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, type_may_be_null(dst_reg->type) && ((BPF_SRC(insn->code) == BPF_K && insn->imm == 0) || (BPF_SRC(insn->code) == BPF_X && bpf_register_is_null(src_reg)))) { + /* + * For BPF_X the zero is a property of this execution path, + * hence src_reg has to be precise. + */ + if (BPF_SRC(insn->code) == BPF_X) { + err = mark_chain_precision(env, insn->src_reg); + if (err) + return err; + } /* Mark all identical registers in each branch as either * safe or unknown depending R == 0 or R != 0 conditional. */ @@ -17081,6 +17213,15 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) verbose(env, "callback function not static\n"); return -EINVAL; } + /* + * When env->subprog_cnt == 1 this instruction won't be rewritten + * to hold a real function address. Assume that no usable program + * combines e.g. main and timer callback and just reject here. + */ + if (subprogno == 0) { + verbose(env, "callback function cannot be the main program\n"); + return -EINVAL; + } dst_reg->type = PTR_TO_FUNC; dst_reg->subprogno = subprogno; @@ -17146,6 +17287,7 @@ static bool may_access_skb(enum bpf_prog_type type) */ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) { + struct bpf_verifier_state *state = env->cur_state; struct bpf_reg_state *regs = cur_regs(env); static const int ctx_reg = BPF_REG_6; u8 mode = BPF_MODE(insn->code); @@ -17156,6 +17298,13 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) return -EINVAL; } + for (i = state->curframe; i; i--) { + if (state->frame[i]->in_callback_fn) { + verbose(env, "cannot use BPF_LD_[ABS|IND] within callback\n"); + return -EINVAL; + } + } + if (!env->ops->gen_ld_abs) { verifier_bug(env, "gen_ld_abs is null"); return -EFAULT; @@ -17623,6 +17772,10 @@ bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, * r0 = *(u64 *)(r10 - 8); r0 += r1; * r0 += r1; exit; * exit; + * + * Both uses of the marks assume that a pattern is entered at its first + * spill and thus executes as a unit, hence a pattern is not grown past + * an instruction targeted by a jump. */ static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env, struct bpf_subprog_info *subprog, @@ -17661,6 +17814,10 @@ static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env, for (i = 1, off = lowest_off; i <= ARRAY_SIZE(caller_saved); ++i, off += BPF_REG_SIZE) { if (insn_idx - i < 0 || insn_idx + i >= env->prog->len) break; + /* stx/ldx/call must not be a jump targets, a jump to the first stx is fine */ + if (bpf_is_jump_target(env, insn_idx - i + 1) || + bpf_is_jump_target(env, insn_idx + i)) + break; stx = &insns[insn_idx - i]; ldx = &insns[insn_idx + i]; /* must be a stack spill/fill pair */ @@ -20536,8 +20693,7 @@ static int process_fd_array_continuous(struct bpf_verifier_env *env, return -E2BIG; } - env->fd_array = kvcalloc(cnt, sizeof(*env->fd_array), - GFP_KERNEL_ACCOUNT); + env->fd_array = kvzalloc_objs(*env->fd_array, cnt, GFP_KERNEL_ACCOUNT); if (!env->fd_array) return -ENOMEM; env->fd_array_cnt = cnt; diff --git a/kernel/capability.c b/kernel/capability.c index 829f49ae07b9..90e6ab62f6db 100644 --- a/kernel/capability.c +++ b/kernel/capability.c @@ -326,7 +326,6 @@ bool has_capability_noaudit(struct task_struct *t, int cap) { return has_ns_capability_noaudit(t, &init_user_ns, cap); } -EXPORT_SYMBOL(has_capability_noaudit); static bool ns_capable_common(struct user_namespace *ns, int cap, @@ -416,6 +415,24 @@ bool capable(int cap) return ns_capable(&init_user_ns, cap); } EXPORT_SYMBOL(capable); + +/** + * capable_noaudit - Determine if the current task has a superior + * capability in effect by checking the process's effective + * capabilities (unaudited). + * @cap: The capability to be tested for + * + * This is the same as capable(), except it uses CAP_OPT_NOAUDIT as to prevent + * issuing spurious audit messages. + * + * This sets PF_SUPERPRIV on the task if the capability is available on the + * assumption that it's about to be used. + */ +bool capable_noaudit(int cap) +{ + return ns_capable_noaudit(&init_user_ns, cap); +} +EXPORT_SYMBOL(capable_noaudit); #endif /* CONFIG_MULTIUSER */ /** diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index c3a12fee7528..2d532bf2c0c7 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -6873,10 +6873,7 @@ static int cgroup_css_set_fork(struct kernel_clone_args *kargs) spin_lock_irq(&css_set_lock); cset = task_css_set(current); get_css_set(cset); - if (kargs->cgrp) - kargs->kill_seq = kargs->cgrp->kill_seq; - else - kargs->kill_seq = cset->dfl_cgrp->kill_seq; + kargs->kill_seq = cset->dfl_cgrp->kill_seq; spin_unlock_irq(&css_set_lock); if (!(kargs->flags & CLONE_INTO_CGROUP)) { @@ -6940,6 +6937,7 @@ static int cgroup_css_set_fork(struct kernel_clone_args *kargs) put_css_set(cset); kargs->cgrp = dst_cgrp; + kargs->kill_seq = dst_cgrp->kill_seq; return ret; err: diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index d100634fa12b..2538faac9aba 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -1259,6 +1259,28 @@ static void reset_partition_data(struct cpuset *cs) cpumask_copy(cs->effective_cpus, parent->effective_cpus); } +/* Return true if isolated_cpus changes. */ +static bool isolated_cpu_update(int new_prs, int cpu) +{ + lockdep_assert_held(&callback_lock); + lockdep_assert_held(&cpuset_mutex); + + if (new_prs == PRS_ISOLATED) { + if (cpumask_test_cpu(cpu, isolated_cpus)) + return false; + cpumask_set_cpu(cpu, isolated_cpus); + return true; + } + + /* CPUs isolated at boot must remain isolated. */ + if (!cpumask_test_cpu(cpu, + housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT)) || + !cpumask_test_cpu(cpu, isolated_cpus)) + return false; + cpumask_clear_cpu(cpu, isolated_cpus); + return true; +} + /* * isolated_cpus_update - Update the isolated_cpus mask * @old_prs: old partition_root_state @@ -1267,19 +1289,16 @@ static void reset_partition_data(struct cpuset *cs) */ static void isolated_cpus_update(int old_prs, int new_prs, struct cpumask *xcpus) { + bool updated = false; + int cpu; + WARN_ON_ONCE(old_prs == new_prs); lockdep_assert_held(&callback_lock); lockdep_assert_held(&cpuset_mutex); - if (new_prs == PRS_ISOLATED) { - if (cpumask_subset(xcpus, isolated_cpus)) - return; - cpumask_or(isolated_cpus, isolated_cpus, xcpus); - } else { - if (!cpumask_intersects(xcpus, isolated_cpus)) - return; - cpumask_andnot(isolated_cpus, isolated_cpus, xcpus); - } - update_housekeeping = true; + for_each_cpu(cpu, xcpus) + updated |= isolated_cpu_update(new_prs, cpu); + if (updated) + update_housekeeping = true; } /* diff --git a/kernel/dma/map_benchmark.c b/kernel/dma/map_benchmark.c index fdc070f419f6..957707158ff6 100644 --- a/kernel/dma/map_benchmark.c +++ b/kernel/dma/map_benchmark.c @@ -51,8 +51,7 @@ struct dma_single_map_param { static void *dma_single_map_benchmark_prepare(struct map_benchmark_data *map) { - struct dma_single_map_param *params __free(kfree) = kzalloc(sizeof(*params), - GFP_KERNEL); + struct dma_single_map_param *params __free(kfree) = kzalloc_obj(*params); if (!params) return NULL; diff --git a/kernel/events/core.c b/kernel/events/core.c index a6c8e38a3110..33210aff3ee6 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7029,7 +7029,6 @@ static void perf_mmap_close(struct vm_area_struct *vma) mapped_f unmapped = get_mapped(event, event_unmapped); struct perf_buffer *rb = ring_buffer_get(event); struct user_struct *mmap_user = rb->mmap_user; - bool detach_rest = false; /* FIXIES vs perf_pmu_unregister() */ if (unmapped) @@ -7060,17 +7059,18 @@ static void perf_mmap_close(struct vm_area_struct *vma) mutex_unlock(&rb->aux_mutex); } - if (refcount_dec_and_test(&rb->mmap_count)) - detach_rest = true; - - if (!refcount_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) - goto out_put; - - ring_buffer_attach(event, NULL); - mutex_unlock(&event->mmap_mutex); + /* + * Drop references in reverse order of perf_mmap() to prevent + * rb revival after rb->mmap_count reaches zero. + */ + if (refcount_dec_and_mutex_lock(&event->mmap_count, + &event->mmap_mutex)) { + ring_buffer_attach(event, NULL); + mutex_unlock(&event->mmap_mutex); + } /* If there's still other mmap()s of this buffer, we're done. */ - if (!detach_rest) + if (!refcount_dec_and_test(&rb->mmap_count)) goto out_put; /* @@ -13558,9 +13558,8 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu, return ERR_PTR(err); if (has_addr_filter(event)) { - event->addr_filter_ranges = kcalloc(pmu->nr_addr_filters, - sizeof(struct perf_addr_filter_range), - GFP_KERNEL); + event->addr_filter_ranges = kzalloc_objs(struct perf_addr_filter_range, + pmu->nr_addr_filters); if (!event->addr_filter_ranges) return ERR_PTR(-ENOMEM); diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c index 9fe92161715e..1b1ffe0533e5 100644 --- a/kernel/events/ring_buffer.c +++ b/kernel/events/ring_buffer.c @@ -509,7 +509,10 @@ void perf_aux_output_end(struct perf_output_handle *handle, unsigned long size) /* * Only send RECORD_AUX if we have something useful to communicate * - * Note: the OVERWRITE records by themselves are not considered + * PMU_FORMAT bits identify the PMU type rather than an AUX event + * has occurred, so ignore them for zero-sized records. + * + * The OVERWRITE records by themselves are not considered * useful, as they don't communicate any *new* information, * aside from the short-lived offset, that becomes history at * the next event sched-in and therefore isn't useful. @@ -518,7 +521,9 @@ void perf_aux_output_end(struct perf_output_handle *handle, unsigned long size) * offset. So, from now on we don't output AUX records that * have *only* OVERWRITE flag set. */ - if (size || (handle->aux_flags & ~(u64)PERF_AUX_FLAG_OVERWRITE)) + if (size || + (handle->aux_flags & ~(u64)(PERF_AUX_FLAG_PMU_FORMAT_TYPE_MASK | + PERF_AUX_FLAG_OVERWRITE))) perf_event_aux_event(handle->event, aux_head, size, handle->aux_flags); diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 51ba5e1257c0..a061f54b606d 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -1874,8 +1874,8 @@ static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags) free_percpu(ref); } - fph = kvzalloc(struct_size(fph, queues, hash_slots), - GFP_KERNEL_ACCOUNT | __GFP_NOWARN); + fph = kvzalloc_flex(*fph, queues, hash_slots, + GFP_KERNEL_ACCOUNT | __GFP_NOWARN); if (!fph) return -ENOMEM; @@ -2103,7 +2103,7 @@ static int __init futex_init(void) size = sizeof(struct futex_hash_bucket) * hashsize; order = get_order(size); - __futex_queues = kcalloc(nr_node_ids, sizeof(*__futex_queues), GFP_KERNEL); + __futex_queues = kzalloc_objs(*__futex_queues, nr_node_ids); kmemleak_not_leak(__futex_queues); runtime_const_init(shift, __futex_shift); diff --git a/kernel/futex/pi.c b/kernel/futex/pi.c index 88788e584ec8..98f1b962e59a 100644 --- a/kernel/futex/pi.c +++ b/kernel/futex/pi.c @@ -1070,17 +1070,11 @@ retry_private: * Caution; releasing @hb in-scope. The hb->lock is still locked * while the reference is dropped. The reference can not be dropped * after the unlock because if a user initiated resize is in progress - * then we might need to wake him. This can not be done after the - * rt_mutex_pre_schedule() invocation. The hb will remain valid because - * the thread, performing resize, will block on hb->lock during - * the requeue. + * then we might need to wake him. The hb will remain valid + * because the thread, performing resize, will block on + * hb->lock during the requeue. */ futex_private_hash_put(no_free_ptr(hbr.fph)); - /* - * Must be done before we enqueue the waiter, here is unfortunately - * under the hb lock, but that *should* work because it does nothing. - */ - rt_mutex_pre_schedule(); rt_mutex_init_waiter(&rt_waiter); @@ -1146,10 +1140,6 @@ cleanup: * the */ futex_q_lockptr_lock(&q); - /* - * Waiter is unqueued. - */ - rt_mutex_post_schedule(); no_block: /* * Fixup the pi_state owner and possibly acquire the lock if we diff --git a/kernel/futex/requeue.c b/kernel/futex/requeue.c index 79823ad13683..b3f4a4bccb12 100644 --- a/kernel/futex/requeue.c +++ b/kernel/futex/requeue.c @@ -154,8 +154,16 @@ static inline void futex_requeue_pi_complete(struct futex_q *q, int locked) } while (!atomic_try_cmpxchg(&q->requeue_state, &old, new)); #ifdef CONFIG_PREEMPT_RT - /* If the waiter interleaved with the requeue let it know */ - if (unlikely(old == Q_REQUEUE_PI_WAIT)) + /* + * The waiter in futex_requeue_pi_wakeup_sync() can interleave with the + * wake below: It will assign Q_REQUEUE_PI_IN_PROGRESS and here it will + * be updated to Q_REQUEUE_PI_LOCKED (locked = 1). The rcuwait_wait_event() + * will already read Q_REQUEUE_PI_LOCKED and skip the schedule() invocation, + * leading to an access of futex_q::requeue_wait after the waiter returned. + * In this case only we skip the wake here and rely on following wake in + * requeue_pi_wake_futex() to perform the wake if needed. + */ + if (unlikely(old == Q_REQUEUE_PI_WAIT) && new != Q_REQUEUE_PI_LOCKED) rcuwait_wake_up(&q->requeue_wait); #endif } diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index 2fbff2618a1e..57eff26fa646 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -2306,7 +2306,7 @@ int request_nmi(unsigned int irq, irq_handler_t handler, !irq_supports_nmi(desc)) return -EINVAL; - action = kzalloc(sizeof(struct irqaction), GFP_KERNEL); + action = kzalloc_obj(struct irqaction); if (!action) return -ENOMEM; diff --git a/kernel/jump_label.c b/kernel/jump_label.c index e851e4b37d0e..ab6b774bcfd7 100644 --- a/kernel/jump_label.c +++ b/kernel/jump_label.c @@ -726,12 +726,11 @@ static int jump_label_add_module(struct module *mod) if (static_key_sealed(key)) goto do_poke; - jlm = kzalloc(sizeof(struct static_key_mod), GFP_KERNEL); + jlm = kzalloc_obj(struct static_key_mod); if (!jlm) return -ENOMEM; if (!static_key_linked(key)) { - jlm2 = kzalloc(sizeof(struct static_key_mod), - GFP_KERNEL); + jlm2 = kzalloc_obj(struct static_key_mod); if (!jlm2) { kfree(jlm); return -ENOMEM; diff --git a/kernel/kprobes.c b/kernel/kprobes.c index bfc89083daa9..6337da5cab9e 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c @@ -1447,8 +1447,14 @@ static bool __within_kprobe_blacklist(unsigned long addr) /* * If 'kprobe_blacklist' is defined, check the address and * reject any probe registration in the prohibited area. + * Note: this can return true during transition period where + * (start_addr, end_addr) in the black list is shrinking + * but old entry has not been removed yet. This is acceptable + * because the worst case is that we reject more probes than + * we should. */ - list_for_each_entry(ent, &kprobe_blacklist, list) { + guard(rcu)(); + list_for_each_entry_rcu(ent, &kprobe_blacklist, list) { if (addr >= ent->start_addr && addr < ent->end_addr) return true; } @@ -2509,7 +2515,7 @@ int kprobe_add_ksym_blacklist(unsigned long entry) ent->start_addr = entry; ent->end_addr = entry + size; INIT_LIST_HEAD(&ent->list); - list_add_tail(&ent->list, &kprobe_blacklist); + list_add_tail_rcu(&ent->list, &kprobe_blacklist); return (int)size; } @@ -2603,8 +2609,8 @@ static void kprobe_remove_area_blacklist(unsigned long start, unsigned long end) list_for_each_entry_safe(ent, n, &kprobe_blacklist, list) { if (ent->start_addr < start || ent->start_addr >= end) continue; - list_del(&ent->list); - kfree(ent); + list_del_rcu(&ent->list); + kfree_rcu(ent, rcu); } } diff --git a/kernel/kthread.c b/kernel/kthread.c index 63beb59b7a3d..a3f95c90456b 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -852,7 +852,7 @@ int kthread_affine_preferred(struct task_struct *p, const struct cpumask *mask) if (!zalloc_cpumask_var(&affinity, GFP_KERNEL)) return -ENOMEM; - kthread->preferred_affinity = kzalloc(sizeof(struct cpumask), GFP_KERNEL); + kthread->preferred_affinity = kzalloc_obj(struct cpumask); if (!kthread->preferred_affinity) { ret = -ENOMEM; goto out; diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index c56a7f91d72e..c3dc84a7cef2 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -963,6 +963,34 @@ look_up_lock_class(const struct lockdep_map *lock, unsigned int subclass) return NULL; } +static __always_inline bool lock_class_cache_is_valid(const struct lockdep_map *lock, + const struct lock_class *class, + unsigned int subclass) +{ + unsigned int class_subclass; + + if (!class) + return false; + + if (unlikely(class < lock_classes || class >= lock_classes + MAX_LOCKDEP_KEYS)) + return false; + + if (unlikely(!arch_test_bit(class - lock_classes, lock_classes_in_use))) + return false; + + if (unlikely(!lock->key)) + return false; + + class_subclass = subclass ? subclass : class->subclass; + if (unlikely(class_subclass >= MAX_LOCKDEP_SUBCLASSES)) + return false; + + if (unlikely(READ_ONCE(class->key) != lock->key->subkeys + class_subclass)) + return false; + + return true; +} + /* * Static locks do not have their class-keys yet - for them the key is * the lock object itself. If the lock is in the per cpu area, the @@ -1395,9 +1423,9 @@ out_unlock_set: out_set_class_cache: if (!subclass || force) - lock->class_cache[0] = class; + WRITE_ONCE(lock->class_cache[0], class); else if (subclass < NR_LOCKDEP_CACHING_CLASSES) - lock->class_cache[subclass] = class; + WRITE_ONCE(lock->class_cache[subclass], class); /* * Hash collision, did we smoke some? We found a class with a matching @@ -4957,7 +4985,7 @@ void lockdep_init_map_type(struct lockdep_map *lock, const char *name, int i; for (i = 0; i < NR_LOCKDEP_CACHING_CLASSES; i++) - lock->class_cache[i] = NULL; + WRITE_ONCE(lock->class_cache[i], NULL); #ifdef CONFIG_LOCK_STAT lock->cpu = raw_smp_processor_id(); @@ -5022,12 +5050,15 @@ EXPORT_SYMBOL_GPL(__lockdep_no_track__); void lockdep_set_lock_cmp_fn(struct lockdep_map *lock, lock_cmp_fn cmp_fn, lock_print_fn print_fn) { - struct lock_class *class = lock->class_cache[0]; + struct lock_class *class = READ_ONCE(lock->class_cache[0]); unsigned long flags; raw_local_irq_save(flags); lockdep_recursion_inc(); + if (!lock_class_cache_is_valid(lock, class, 0)) + class = NULL; + if (!class) class = register_lock_class(lock, 0, 0); @@ -5119,8 +5150,11 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, if (DEBUG_LOCKS_WARN_ON(subclass >= MAX_LOCKDEP_SUBCLASSES)) return 0; - if (subclass < NR_LOCKDEP_CACHING_CLASSES) - class = lock->class_cache[subclass]; + if (subclass < NR_LOCKDEP_CACHING_CLASSES) { + class = READ_ONCE(lock->class_cache[subclass]); + if (!lock_class_cache_is_valid(lock, class, subclass)) + class = NULL; + } /* * Not cached? */ @@ -5324,9 +5358,9 @@ static noinstr int match_held_lock(const struct held_lock *hlock, return 1; if (hlock->references) { - const struct lock_class *class = lock->class_cache[0]; + const struct lock_class *class = READ_ONCE(lock->class_cache[0]); - if (!class) + if (!lock_class_cache_is_valid(lock, class, 0)) class = look_up_lock_class(lock, 0); /* diff --git a/kernel/locking/rtmutex_api.c b/kernel/locking/rtmutex_api.c index 5d48d64725b1..eb18b094473c 100644 --- a/kernel/locking/rtmutex_api.c +++ b/kernel/locking/rtmutex_api.c @@ -423,6 +423,7 @@ int __sched rt_mutex_wait_proxy_lock(struct rt_mutex_base *lock, { int ret; + rt_mutex_futex_pre_schedule(); raw_spin_lock_irq(&lock->wait_lock); /* sleep on the mutex */ set_current_state(TASK_INTERRUPTIBLE); @@ -433,6 +434,7 @@ int __sched rt_mutex_wait_proxy_lock(struct rt_mutex_base *lock, */ fixup_rt_mutex_waiters(lock, true); raw_spin_unlock_irq(&lock->wait_lock); + rt_mutex_futex_post_schedule(); return ret; } diff --git a/kernel/sched/core.c b/kernel/sched/core.c index f78275192036..b998ef6b87af 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3742,11 +3742,17 @@ static inline void ttwu_do_wakeup(struct task_struct *p) void update_rq_avg_idle(struct rq *rq) { - u64 delta = rq_clock(rq) - rq->idle_stamp; - u64 max = 2*rq->max_idle_balance_cost; + u64 idle_stamp = rq->idle_stamp; + u64 delta, max; + + if (!idle_stamp) + return; + + delta = rq_clock(rq) - idle_stamp; update_avg(&rq->avg_idle, delta); + max = 2 * rq->max_idle_balance_cost; if (rq->avg_idle > max) rq->avg_idle = max; rq->idle_stamp = 0; @@ -7637,6 +7643,17 @@ void rt_mutex_pre_schedule(void) sched_submit_work(current); } +/* + * Used within the futex syscall context, skips sched_submit_work() because none + * its work will be done. Asserts ensure that it is indeed the case. + */ +void rt_mutex_futex_pre_schedule(void) +{ + lockdep_assert(!(current->flags & (PF_WQ_WORKER | PF_IO_WORKER))); + lockdep_assert(!current->plug); + lockdep_assert(!fetch_and_set(current->sched_rt_mutex, 1)); +} + void rt_mutex_schedule(void) { lockdep_assert(current->sched_rt_mutex); @@ -7649,6 +7666,11 @@ void rt_mutex_post_schedule(void) lockdep_assert(fetch_and_set(current->sched_rt_mutex, 0)); } +void rt_mutex_futex_post_schedule(void) +{ + lockdep_assert(fetch_and_set(current->sched_rt_mutex, 0)); +} + /* * rt_mutex_setprio - set the current priority of a task * @p: task to boost diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 857dbe3519a8..0663c00c41c0 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -3028,8 +3028,8 @@ static struct task_struct *pick_next_pushable_dl_task(struct rq *rq) next_node = rb_first_cached(&rq->dl.pushable_dl_tasks_root); while (next_node) { i = __node_2_pdl(next_node); - /* make sure task isn't on_cpu (possible with proxy-exec) */ - if (!task_on_cpu(rq, i)) { + /* skip tasks that cannot be migrated */ + if (!task_on_cpu(rq, i) && !is_migration_disabled(i)) { p = i; break; } diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c index 39f88deb94bc..bc4eee5bb4cb 100644 --- a/kernel/sched/ext/cid.c +++ b/kernel/sched/ext/cid.c @@ -98,16 +98,16 @@ static struct scx_cid_tables *scx_cid_alloc_tables(void) u32 npossible = num_possible_cpus(); struct scx_cid_tables *tbls; - tbls = kzalloc_obj(*tbls, GFP_KERNEL); + tbls = kzalloc_obj(*tbls); if (!tbls) return NULL; - tbls->cid_to_cpu = kvcalloc(npossible, sizeof(*tbls->cid_to_cpu), GFP_KERNEL); - tbls->cpu_to_cid = kvcalloc(nr_cpu_ids, sizeof(*tbls->cpu_to_cid), GFP_KERNEL); - tbls->cid_to_shard = kvcalloc(npossible, sizeof(*tbls->cid_to_shard), GFP_KERNEL); - tbls->shard_node = kvcalloc(npossible, sizeof(*tbls->shard_node), GFP_KERNEL); - tbls->shard_ranges = kvcalloc(npossible, sizeof(*tbls->shard_ranges), GFP_KERNEL); - tbls->topo = kvcalloc(npossible, sizeof(*tbls->topo), GFP_KERNEL); + tbls->cid_to_cpu = kvzalloc_objs(*tbls->cid_to_cpu, npossible); + tbls->cpu_to_cid = kvzalloc_objs(*tbls->cpu_to_cid, nr_cpu_ids); + tbls->cid_to_shard = kvzalloc_objs(*tbls->cid_to_shard, npossible); + tbls->shard_node = kvzalloc_objs(*tbls->shard_node, npossible); + tbls->shard_ranges = kvzalloc_objs(*tbls->shard_ranges, npossible); + tbls->topo = kvzalloc_objs(*tbls->topo, npossible); if (!tbls->cid_to_cpu || !tbls->cpu_to_cid || !tbls->cid_to_shard || !tbls->shard_node || !tbls->shard_ranges || !tbls->topo) { @@ -490,7 +490,7 @@ __bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid__arena, u32 cpu_to_c * region that arena fault recovery covers. */ alloced = zalloc_cpumask_var(&seen, GFP_KERNEL); - node_counts = kcalloc(nr_node_ids, sizeof(*node_counts), GFP_KERNEL); + node_counts = kzalloc_objs(*node_counts, nr_node_ids); if (cpu_to_cid_cnt == nr_cpu_ids) cpu_to_cid = kmemdup(cpu_to_cid__arena, cpu_to_cid_cnt * sizeof(s32), GFP_KERNEL); diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 10af28a9f2c0..51de1d8b72a1 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -876,9 +876,9 @@ struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter) * unloading. The init_tasks ("swappers") should be excluded * from the iteration because: * - * - It's unsafe to use __setschduler_prio() on an init_task to - * determine the sched_class to use as it won't preserve its - * idle_sched_class. + * - It's unsafe to use __setscheduler_class() on an init_task + * to determine the sched_class to use as it won't preserve + * its idle_sched_class. * * - ops.init/exit_task() can easily be confused if called with * init_tasks as they, e.g., share PID 0. @@ -2806,6 +2806,8 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, * @p: task to finish dispatching * @qseq_at_dispatch: qseq when @p started getting dispatched * @dsq_id: destination DSQ ID + * @slice: slice carried by the insert verdict, 0 keeps the current value + * @vtime: vtime carried by the insert verdict, committed on PRIQ inserts * @enq_flags: %SCX_ENQ_* * * Dispatching to local DSQs may need to wait for queueing to complete or @@ -5447,7 +5449,7 @@ static ssize_t scx_attr_caps_show(struct kobject *kobj, struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); u32 npossible = num_possible_cpus(); struct scx_cmask *agg __free(kfree) = - kzalloc(struct_size(agg, bits, SCX_CMASK_NR_WORDS(npossible)), GFP_KERNEL); + kzalloc_flex(*agg, bits, SCX_CMASK_NR_WORDS(npossible)); unsigned long *agg_bm __free(bitmap) = bitmap_zalloc(npossible, GFP_KERNEL); ssize_t count = 0; s32 cap, si; @@ -5514,7 +5516,7 @@ static const struct kset_uevent_ops scx_uevent_ops = { }; /* - * Used by sched_fork() and __setscheduler_prio() to pick the matching + * Used by sched_fork() and __setscheduler_class() to pick the matching * sched_class. dl/rt are already handled. */ bool task_should_scx(int policy) @@ -7694,7 +7696,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) /* * Enable ops for every task. Fork is excluded by scx_fork_rwsem * preventing new tasks from being added. No need to exclude tasks - * leaving as sched_ext_free() can handle both prepped and enabled + * leaving as sched_ext_dead() can handle both prepped and enabled * tasks. Prep all tasks first and then enable them with preemption * disabled. * @@ -7786,7 +7788,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) /* * We're fully committed and can't fail. The task READY -> ENABLED - * transitions here are synchronized against sched_ext_free() through + * transitions here are synchronized against sched_ext_dead() through * scx_tasks_lock. */ percpu_down_write(&scx_fork_rwsem); @@ -8079,6 +8081,7 @@ static int bpf_scx_check_member(const struct btf_type *t, case offsetof(struct sched_ext_ops, cgroup_init): case offsetof(struct sched_ext_ops, cgroup_exit): case offsetof(struct sched_ext_ops, cgroup_prep_move): + case offsetof(struct sched_ext_ops, cgroup_set_bandwidth): #endif case offsetof(struct sched_ext_ops, cpu_online): case offsetof(struct sched_ext_ops, cpu_offline): @@ -9003,12 +9006,6 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, if (unlikely(READ_ONCE(sch->aborting))) return false; - if (unlikely(!scx_task_on_sched(sch, p))) { - scx_error(sch, "scx_bpf_dsq_move[_vtime]() on %s[%d] but the task belongs to a different scheduler", - p->comm, p->pid); - return false; - } - /* * Can be called from either ops.dispatch() holding the dispatched rq's * lock or any context where no rq lock is held. If latter, lock @p's @@ -9040,6 +9037,17 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, goto out; } + /* + * @p has been on $src_dsq and can't move anymore. If @p is not on @sch, + * the caller didn't have authority over @p at the time of the call. + */ + if (unlikely(!scx_task_on_sched(sch, p))) { + scx_error(sch, "scx_bpf_dsq_move[_vtime]() on %s[%d] but the task belongs to a different scheduler", + p->comm, p->pid); + raw_spin_unlock(&src_dsq->lock); + goto out; + } + /* @p is still on $src_dsq and stable, determine the destination */ dst_dsq = find_dsq_for_dispatch(sch, locked_rq ?: this_rq(), dsq_id, task_cpu(p)); @@ -9765,7 +9773,7 @@ __bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *i * bpf_iter_scx_dsq_destroy - Destroy a DSQ iterator * @it: iterator to destroy * - * Undo scx_iter_scx_dsq_new(). + * Undo bpf_iter_scx_dsq_new(). */ __bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) { @@ -11041,3 +11049,16 @@ static int __init scx_init(void) return 0; } __initcall(scx_init); + +/* + * Compatibility markers for userspace. Existence of a marker function + * represents that the kernel supports that sched-ext feature. + */ + +/* + * scx_compat_marker_cgroup_set_bandwidth_may_sleep: advertises that + * ops.cgroup_set_bandwidth() may be implemented as a sleepable callback. + */ +#ifdef CONFIG_EXT_GROUP_SCHED +DEFINE_SCX_COMPAT_MARKER(cgroup_set_bandwidth_may_sleep); +#endif /* CONFIG_EXT_GROUP_SCHED */ diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 27bbf5e04d90..0967b99a4948 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -442,7 +442,7 @@ struct sched_ext_ops { * * Note that this callback may be called from a CPU other than the * one the task is going to run on. This can happen when a task - * property is changed (i.e., affinity), since scx_next_task_scx(), + * property is changed (i.e., affinity), since set_next_task_scx(), * which triggers this callback, may run on a CPU different from * the task's assigned CPU. * @@ -753,7 +753,7 @@ struct sched_ext_ops { * @burst_us: bandwidth control burst * * Update @cgrp's bandwidth control parameters. This is from the cpu.max - * cgroup interface. + * cgroup interface. This operation may block. * * @quota_us / @period_us determines the CPU bandwidth @cgrp is entitled * to. For example, if @period_us is 1_000_000 and @quota_us is @@ -2001,6 +2001,27 @@ struct scx_bstr_buf { char line[SCX_EXIT_MSG_LEN]; }; +/* Internal helper for DEFINE_SCX_COMPAT_MARKER(). */ +#define DECLARE_SCX_COMPAT_MARKER(func) \ + extern void scx_compat_marker_##func(void) + +/** + * DEFINE_SCX_COMPAT_MARKER() - define a userspace capability marker + * @func: marker suffix; the defined symbol is scx_compat_marker_@func + * + * Emit an empty, callerless function that is retained in the kernel's BTF. + * Its presence is part of the kernel<->userspace contract: userspace probes + * scx_compat_marker_@func (e.g. via BTF) to detect that this kernel supports + * the corresponding feature. + * + * The leading declaration suppresses the missing-prototype warning; the + * trailing declaration consumes the semicolon at the use site. + */ +#define DEFINE_SCX_COMPAT_MARKER(func) \ + DECLARE_SCX_COMPAT_MARKER(func); \ + __used __retain void scx_compat_marker_##func(void) {} \ + DECLARE_SCX_COMPAT_MARKER(func) + extern struct scx_sched __rcu *scx_root; DECLARE_PER_CPU(struct rq *, scx_locked_rq_state); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 0554448835bd..9e7040482bde 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -194,7 +194,7 @@ s32 scx_alloc_pshards(struct scx_sched *sch) shard_node = rcu_dereference_protected(scx_shard_node, lockdep_is_held(&scx_enable_mutex)); - pshard = kzalloc_objs(pshard[0], scx_nr_cid_shards, GFP_KERNEL); + pshard = kzalloc_objs(pshard[0], scx_nr_cid_shards); if (!pshard) return -ENOMEM; diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 8dff37059faf..ade1eceb39b8 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6978,14 +6978,14 @@ static int tg_throttle_down(struct task_group *tg, void *data) static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) { struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); - struct sched_entity *curr = cfs_rq->curr; + struct sched_entity *curr = cfs_rq->h_curr; struct rq *rq = rq_of(cfs_rq); scoped_guard(raw_spinlock, &cfs_b->lock) { u64 target_runtime = 1; /* - * If cfs_rq->curr is still runnable, we are here from an + * If cfs_rq->h_curr is still runnable, we are here from an * update_curr(). Request sysctl_sched_cfs_bandwidth_slice * worth of bandwidth to continue running. * @@ -7192,7 +7192,7 @@ static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b) if (!list_empty(&cfs_rq->throttled_csd_list)) continue; - if (cfs_rq->curr) { + if (cfs_rq->h_curr) { update_rq_clock(rq); update_curr(cfs_rq); } @@ -10057,7 +10057,7 @@ again: /* Might not have done put_prev_entity() */ if (cfs_rq->curr && cfs_rq->curr->on_rq) - update_curr(cfs_rq); + update_curr_eevdf(cfs_rq); se = pick_next_entity(rq, true); if (!se) @@ -10160,7 +10160,7 @@ static void yield_task_fair(struct rq *rq) /* * Update run-time statistics of the 'current'. */ - update_curr(cfs_rq); + update_curr_eevdf(cfs_rq); /* * Tell update_rq_clock() that we've just updated, * so we don't do microscopic update in schedule() @@ -10691,17 +10691,40 @@ static enum llc_mig can_migrate_llc(int src_cpu, int dst_cpu, return mig_llc; } +static inline bool task_misfits_asym_cpu(struct lb_env *env, struct task_struct *p) +{ + /* + * On asymmetric CPU capacity domains, do not let cache-aware + * balancing pull the task onto a destination CPU that cannot + * accommodate it. Doing so would turn the task into a misfit on + * the destination, trading a cache-locality gain for a capacity + * loss. If the task already does not fit its source CPU, the move + * cannot make things worse, so let the LLC preference decide. + */ + if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && p && + !task_fits_cpu(p, env->dst_cpu) && + task_fits_cpu(p, env->src_cpu)) + return true; + + return false; +} + /* * Check if task p can migrate from source LLC to * destination LLC in terms of cache aware load balance. */ -static enum llc_mig can_migrate_llc_task(int src_cpu, int dst_cpu, +static enum llc_mig can_migrate_llc_task(struct lb_env *env, struct task_struct *p) { struct mm_struct *mm; bool to_pref; - int cpu; + int cpu, src_cpu, dst_cpu; + + if (task_misfits_asym_cpu(env, p)) + return mig_forbid; + src_cpu = env->src_cpu; + dst_cpu = env->dst_cpu; mm = p->mm; if (!mm) return mig_unrestricted; @@ -10758,6 +10781,14 @@ alb_break_llc(struct lb_env *env) unsigned long util = 0; struct task_struct *cur; + /* + * Migrating misfit tasks from current CPU + * to CPU with a better fit. + * Prioritize that over LLC preference. + */ + if (env->migration_type == migrate_misfit) + return false; + if (env->src_rq->nr_running <= 1) return true; @@ -10765,7 +10796,8 @@ alb_break_llc(struct lb_env *env) if (cur && cur->sched_class == &fair_sched_class) util = task_util(cur); - if (can_migrate_llc(env->src_cpu, env->dst_cpu, + if (task_misfits_asym_cpu(env, cur) || + can_migrate_llc(env->src_cpu, env->dst_cpu, util, false) == mig_forbid) return true; } @@ -10805,8 +10837,7 @@ static bool migrate_degrades_llc(struct task_struct *p, struct lb_env *env) READ_ONCE(p->preferred_llc) != llc_id(env->dst_cpu)) return true; - if (can_migrate_llc_task(env->src_cpu, - env->dst_cpu, p) != mig_forbid) + if (can_migrate_llc_task(env, p) != mig_forbid) return false; return true; @@ -11870,6 +11901,15 @@ static inline bool llc_balance(struct lb_env *env, struct sg_lb_stats *sgs, return false; /* + * On asymmetric domains, group_misfit_task_load + * should be prioritized to move tasks to CPU that fit them + * over aggregating tasks to their preferred LLC. + */ + if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && + sgs->group_misfit_task_load) + return false; + + /* * Skip cache aware tagging if nr_balanced_failed is sufficiently high. * Threshold of cache_nice_tries is set to 1 higher than nr_balance_failed * to avoid excessive task migration at the same time. diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index e6e5f8a2caaf..85303add726d 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -1872,8 +1872,8 @@ static struct task_struct *pick_next_pushable_task(struct rq *rq) return NULL; plist_for_each_entry(i, head, pushable_tasks) { - /* make sure task isn't on_cpu (possible with proxy-exec) */ - if (!task_on_cpu(rq, i)) { + /* skip tasks that cannot be migrated */ + if (!task_on_cpu(rq, i) && !is_migration_disabled(i)) { p = i; break; } diff --git a/kernel/softirq.c b/kernel/softirq.c index 7980a4a232f9..5d02c36c40e3 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -91,11 +91,11 @@ EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context); DEFINE_PER_CPU(unsigned long, local_interrupt_disable_state); -void _local_interrupt_disable(void) +void _local_interrupt_save_state(unsigned long flags) { - __local_interrupt_disable(); + __local_interrupt_save_state(flags); } -EXPORT_SYMBOL(_local_interrupt_disable); +EXPORT_SYMBOL(_local_interrupt_save_state); void _local_interrupt_enable(void) { @@ -749,16 +749,7 @@ static inline void __irq_exit_rcu(void) #endif account_hardirq_exit(current); preempt_count_sub(HARDIRQ_OFFSET); - /* - * Interrupts may happen between hardirq_disable_enter() and - * local_irq_save() in local_interrupt_disable(), if irq_exit() invokes - * softirq here, we may have a softirq handler calling - * local_interrupt_disable() but it won't disable the IRQ because - * hardirq disabling count is already 1, hence we need to prevent - * invoking softirq when a local_interrupt_disable() is ongoing. - */ - if (!in_interrupt() && !hardirq_disable_count() && - local_softirq_pending()) { + if (!in_interrupt() && local_softirq_pending()) { /* * If we left hrtimers unarmed, make sure to arm them now, * before enabling interrupts to run softirq. diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c index ddb0b12a5c4a..1e9b00997ff2 100644 --- a/kernel/trace/fprobe.c +++ b/kernel/trace/fprobe.c @@ -945,7 +945,7 @@ int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter if (num < 0) return num; - addrs = kcalloc(num, sizeof(*addrs), GFP_KERNEL); + addrs = kzalloc_objs(*addrs, num); if (!addrs) return -ENOMEM; diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index f9d80c7bd9f1..53d5db60bfa5 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -75,6 +75,8 @@ .func_hash = &opsname.local_hash, \ .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock), \ .subop_list = LIST_HEAD_INIT(opsname.subop_list), +/* Used only to synchronize the initialization of ftrace_ops */ +static DEFINE_MUTEX(ops_mutex); #else #define INIT_OPS_HASH(opsname) #endif @@ -159,11 +161,18 @@ const struct ftrace_ops ftrace_nop_ops = { static inline void ftrace_ops_init(struct ftrace_ops *ops) { #ifdef CONFIG_DYNAMIC_FTRACE - if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) { + unsigned long flags = smp_load_acquire(&ops->flags); + + if (!(flags & FTRACE_OPS_FL_INITIALIZED)) { + guard(mutex)(&ops_mutex); + /* Could have been initialized before lock taken */ + if (unlikely(ops->flags & FTRACE_OPS_FL_INITIALIZED)) + return; mutex_init(&ops->local_hash.regex_lock); INIT_LIST_HEAD(&ops->subop_list); ops->func_hash = &ops->local_hash; - ops->flags |= FTRACE_OPS_FL_INITIALIZED; + flags = ops->flags | FTRACE_OPS_FL_INITIALIZED; + smp_store_release(&ops->flags, flags); } #endif } @@ -4677,7 +4686,8 @@ ftrace_avail_addrs_open(struct inode *inode, struct file *file) /** * ftrace_regex_open - initialize function tracer filter files - * @ops: The ftrace_ops that hold the hash filters + * @tr: The trace_array that holds the ftrace_ops [optional] + * @ops: The ftrace_ops that hold the hash filters [optional] * @flag: The type of filter to process * @inode: The inode, usually passed in to your open routine * @file: The file, usually passed in to your open routine @@ -4691,26 +4701,45 @@ ftrace_avail_addrs_open(struct inode *inode, struct file *file) * tracing_lseek() should be used as the lseek routine, and * release must call ftrace_regex_release(). * + * Note, If @tr is not NULL, its reference has to be taken before + * @ops may be referenced. + * If @ops is NULL and @tr is not, then @tr->ops is used. + * If @tr is NULL and @ops is not then @ops->private is uesd for @tr. + * If both @tr and @ops are NULL, then the &global_ops is + * to be used, and @tr will be the global_ops.private pointer. + * * Returns: 0 on success or a negative errno value on failure */ int -ftrace_regex_open(struct ftrace_ops *ops, int flag, +ftrace_regex_open(struct trace_array *tr, struct ftrace_ops *ops, int flag, struct inode *inode, struct file *file) { - struct ftrace_iterator *iter; + struct ftrace_iterator *iter = NULL; struct ftrace_hash *hash; struct list_head *mod_head; - struct trace_array *tr = ops->private; - int ret = -ENOMEM; - - ftrace_ops_init(ops); + int ret = -ENODEV; if (unlikely(ftrace_disabled)) return -ENODEV; + if (!tr) { + if (!ops) + ops = &global_ops; + tr = ops->private; + } + if (tracing_check_open_get_tr(tr)) return -ENODEV; + if (!ops) + ops = tr->ops; + + if (WARN_ON_ONCE(!ops)) + goto out; + + ftrace_ops_init(ops); + + ret = -ENOMEM; iter = kzalloc_obj(*iter); if (!iter) goto out; @@ -4788,21 +4817,19 @@ ftrace_regex_open(struct ftrace_ops *ops, int flag, static int ftrace_filter_open(struct inode *inode, struct file *file) { - struct ftrace_ops *ops = inode->i_private; + struct trace_array *tr = inode->i_private; - /* Checks for tracefs lockdown */ - return ftrace_regex_open(ops, - FTRACE_ITER_FILTER | FTRACE_ITER_DO_PROBES, - inode, file); + return ftrace_regex_open(tr, NULL, + FTRACE_ITER_FILTER | FTRACE_ITER_DO_PROBES, + inode, file); } static int ftrace_notrace_open(struct inode *inode, struct file *file) { - struct ftrace_ops *ops = inode->i_private; + struct trace_array *tr = inode->i_private; - /* Checks for tracefs lockdown */ - return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE, + return ftrace_regex_open(tr, NULL, FTRACE_ITER_NOTRACE, inode, file); } @@ -7492,15 +7519,15 @@ static const struct file_operations ftrace_graph_notrace_fops = { }; #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ -void ftrace_create_filter_files(struct ftrace_ops *ops, +void ftrace_create_filter_files(struct trace_array *tr, struct dentry *parent) { trace_create_file("set_ftrace_filter", TRACE_MODE_WRITE, parent, - ops, &ftrace_filter_fops); + tr, &ftrace_filter_fops); trace_create_file("set_ftrace_notrace", TRACE_MODE_WRITE, parent, - ops, &ftrace_notrace_fops); + tr, &ftrace_notrace_fops); } /* @@ -7525,7 +7552,6 @@ void ftrace_destroy_filter_files(struct ftrace_ops *ops) static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { - trace_create_file("available_filter_functions", TRACE_MODE_READ, d_tracer, NULL, &ftrace_avail_fops); @@ -7538,7 +7564,7 @@ static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer) trace_create_file("touched_functions", TRACE_MODE_READ, d_tracer, NULL, &ftrace_touched_fops); - ftrace_create_filter_files(&global_ops, d_tracer); + ftrace_create_filter_files(NULL, d_tracer); #ifdef CONFIG_FUNCTION_GRAPH_TRACER trace_create_file("set_graph_function", TRACE_MODE_WRITE, d_tracer, diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index b0963ac6fd16..9c03a555a6ba 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -330,6 +330,14 @@ struct buffer_data_read_page { struct buffer_data_page *data; /* actual data, stored in this page */ }; +static __always_inline unsigned int rb_read_page_capacity(struct buffer_data_read_page *rpage) +{ + return (PAGE_SIZE << rpage->order) - BUF_PAGE_HDR_SIZE; +} + +/* The number of bits for static buffer ids */ +#define RB_STATIC_BITS 30 + /* * Note, the buffer_page list must be first. The buffer pages * are allocated in cache lines, which means that each buffer @@ -345,7 +353,7 @@ struct buffer_page { local_t entries; /* entries on this page */ unsigned long real_end; /* real end of data */ unsigned order; /* order of the page */ - u32 id:30; /* ID for external mapping */ + u32 id:RB_STATIC_BITS; /* ID for external mapping */ u32 range:1; /* Mapped via a range */ struct buffer_data_page *page; /* Actual data page */ }; @@ -652,6 +660,15 @@ static bool rb_is_static(struct ring_buffer_per_cpu *cpu_buffer) return cpu_buffer->user_mapped || cpu_buffer->remote || cpu_buffer->ring_meta; } +static unsigned long rb_static_max_pages(void) +{ + /* + * Static ring buffers are using bpage::id and must account for the + * reader page. + */ + return (1UL << RB_STATIC_BITS) - 1; +} + struct ring_buffer_iter { struct ring_buffer_per_cpu *cpu_buffer; unsigned long head; @@ -1669,7 +1686,7 @@ out_locked: * This is used to help find the next per cpu subbuffer within a mapped range. */ static unsigned long -rb_range_align_subbuf(unsigned long addr, int subbuf_size, int nr_subbufs) +rb_range_align_subbuf(unsigned long addr, unsigned int subbuf_size, unsigned long nr_subbufs) { addr += sizeof(struct ring_buffer_cpu_meta) + sizeof(int) * nr_subbufs; @@ -1679,13 +1696,12 @@ rb_range_align_subbuf(unsigned long addr, int subbuf_size, int nr_subbufs) /* * Return the ring_buffer_meta for a given @cpu. */ -static void *rb_range_meta(struct trace_buffer *buffer, int nr_pages, int cpu) +static void *rb_range_meta(struct trace_buffer *buffer, unsigned long nr_pages, int cpu) { - int subbuf_size = rb_subbuf_size(buffer); + unsigned int subbuf_size = rb_subbuf_size(buffer); struct ring_buffer_cpu_meta *meta; struct ring_buffer_meta *bmeta; - unsigned long ptr; - int nr_subbufs; + unsigned long ptr, nr_subbufs; bmeta = buffer->meta; if (!bmeta) @@ -1731,7 +1747,7 @@ static void *rb_range_meta(struct trace_buffer *buffer, int nr_pages, int cpu) /* Return the start of subbufs given the meta pointer */ static void *rb_subbufs_from_meta(struct ring_buffer_cpu_meta *meta) { - int subbuf_size = meta->subbuf_size; + unsigned int subbuf_size = meta->subbuf_size; unsigned long ptr; ptr = (unsigned long)meta; @@ -1743,11 +1759,11 @@ static void *rb_subbufs_from_meta(struct ring_buffer_cpu_meta *meta) /* * Return a specific sub-buffer for a given @cpu defined by @idx. */ -static void *rb_range_buffer(struct ring_buffer_per_cpu *cpu_buffer, int idx) +static void *rb_range_buffer(struct ring_buffer_per_cpu *cpu_buffer, unsigned long idx) { struct ring_buffer_cpu_meta *meta; + unsigned int subbuf_size; unsigned long ptr; - int subbuf_size; meta = rb_range_meta(cpu_buffer->buffer, 0, cpu_buffer->cpu); if (!meta) @@ -1763,7 +1779,7 @@ static void *rb_range_buffer(struct ring_buffer_per_cpu *cpu_buffer, int idx) ptr = (unsigned long)rb_subbufs_from_meta(meta); - ptr += subbuf_size * idx; + ptr += (unsigned long)subbuf_size * idx; if (ptr + subbuf_size > cpu_buffer->buffer->range_addr_end) return NULL; @@ -1840,13 +1856,12 @@ static bool rb_meta_init(struct trace_buffer *buffer, int scratch_size) * must be the same. */ static bool rb_cpu_meta_valid(struct ring_buffer_cpu_meta *meta, int cpu, - struct trace_buffer *buffer, int nr_pages, + struct trace_buffer *buffer, unsigned long nr_pages, unsigned long *subbuf_mask) { - int subbuf_size = PAGE_SIZE; unsigned long buffers_start; unsigned long buffers_end; - int i; + unsigned long i; if (!subbuf_mask) return false; @@ -1856,8 +1871,13 @@ static bool rb_cpu_meta_valid(struct ring_buffer_cpu_meta *meta, int cpu, return false; } + if (meta->nr_subbufs != nr_pages + 1) { + pr_info("Ring buffer boot meta [%d] invalid nr_subbufs\n", cpu); + return false; + } + buffers_start = meta->first_buffer; - buffers_end = meta->first_buffer + (subbuf_size * meta->nr_subbufs); + buffers_end = meta->first_buffer + (meta->nr_subbufs * PAGE_SIZE); /* Is the head and commit buffers within the range of buffers? */ if (meta->head_buffer < buffers_start || @@ -2095,8 +2115,8 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) struct buffer_page *head_page, *orig_head, *orig_reader; struct rb_validation_state state = { 0 }; bool skip = false; + unsigned long i; int ret; - int i; if (!meta || !meta->head_buffer) return; @@ -2147,7 +2167,7 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) rb_validate_buffer(head_page, cpu_buffer, meta, &state, 0, state.ts); } if (i) - pr_info("Ring buffer [%d] rewound %d pages\n", cpu_buffer->cpu, i); + pr_info("Ring buffer [%d] rewound %lu pages\n", cpu_buffer->cpu, i); /* The last rewound page must be skipped. */ if (head_page != orig_head) @@ -2231,7 +2251,8 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) } } -static void rb_range_meta_init(struct trace_buffer *buffer, int nr_pages, int scratch_size) +static void rb_range_meta_init(struct trace_buffer *buffer, unsigned long nr_pages, + int scratch_size) { struct ring_buffer_cpu_meta *meta; unsigned long *subbuf_mask; @@ -2331,8 +2352,8 @@ static int rbm_show(struct seq_file *m, void *v) rb_meta_subbuf_idx(meta, (void *)meta->head_buffer)); seq_printf(m, "commit_buffer: %d\n", rb_meta_subbuf_idx(meta, (void *)meta->commit_buffer)); - seq_printf(m, "subbuf_size: %d\n", meta->subbuf_size); - seq_printf(m, "nr_subbufs: %d\n", meta->nr_subbufs); + seq_printf(m, "subbuf_size: %u\n", meta->subbuf_size); + seq_printf(m, "nr_subbufs: %u\n", meta->nr_subbufs); return 0; } @@ -2417,7 +2438,7 @@ static void *ring_buffer_desc_page(struct ring_buffer_desc *desc, unsigned int p } static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, - long nr_pages, struct list_head *pages) + unsigned long nr_pages, struct list_head *pages) { struct trace_buffer *buffer = cpu_buffer->buffer; struct ring_buffer_cpu_meta *meta = NULL; @@ -2545,7 +2566,7 @@ static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, } static struct ring_buffer_per_cpu * -rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) +rb_allocate_cpu_buffer(struct trace_buffer *buffer, unsigned long nr_pages, int cpu) { struct ring_buffer_per_cpu *cpu_buffer __free(kfree) = alloc_cpu_buffer(cpu); @@ -2600,8 +2621,8 @@ rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) cpu_buffer->remote = buffer->remote; cpu_buffer->meta_page = (struct trace_buffer_meta *)(void *)desc->meta_va; cpu_buffer->nr_pages = nr_pages; - cpu_buffer->subbuf_ids = kcalloc(cpu_buffer->nr_pages + 1, - sizeof(*cpu_buffer->subbuf_ids), GFP_KERNEL); + cpu_buffer->subbuf_ids = kzalloc_objs(*cpu_buffer->subbuf_ids, + cpu_buffer->nr_pages + 1); if (!cpu_buffer->subbuf_ids) goto fail_free_reader; @@ -2702,8 +2723,8 @@ static void rb_test_inject_invalid_pages(struct trace_buffer *buffer) struct ring_buffer_cpu_meta *meta; struct buffer_data_page *dpage; unsigned long entry_bytes = 0; + unsigned int subbuf_size; unsigned long ptr; - int subbuf_size; int invalid = 0; int cpu; int i; @@ -2773,8 +2794,8 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, struct ring_buffer_remote *remote) { struct trace_buffer *buffer __free(kfree) = NULL; - long nr_pages; - int subbuf_size; + unsigned int subbuf_size; + unsigned long nr_pages; int bsize; int cpu; int ret; @@ -2828,6 +2849,8 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, size = end - buffers_start; size = size / nr_cpu_ids; + if (size < sizeof(struct ring_buffer_cpu_meta)) + goto fail_free_buffers; /* * The number of sub-buffers (nr_pages) is determined by the * total size allocated minus the meta data size. @@ -2837,6 +2860,10 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, */ nr_pages = (size - sizeof(struct ring_buffer_cpu_meta)) / (subbuf_size + sizeof(int)); + + if (nr_pages > rb_static_max_pages()) + goto fail_free_buffers; + /* Need at least two pages plus the reader page */ if (nr_pages < 3) goto fail_free_buffers; @@ -2869,6 +2896,10 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, /* The writer is remote. This ring-buffer is read-only */ atomic_inc(&buffer->record_disabled); nr_pages = desc->nr_page_va - 1; + + if (nr_pages > rb_static_max_pages()) + goto fail_free_buffers; + if (nr_pages < 2) goto fail_free_buffers; } else { @@ -5853,12 +5884,12 @@ __rb_get_reader_page_from_remote(struct ring_buffer_per_cpu *cpu_buffer) static struct buffer_page * __rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) { - int max_loops = cpu_buffer->ring_meta ? cpu_buffer->nr_pages : 3; + unsigned long max_loops = cpu_buffer->ring_meta ? cpu_buffer->nr_pages : 3; struct buffer_page *reader = NULL; + unsigned long nr_loops = 0; unsigned long overwrite; unsigned long flags; int missed_events = 0; - int nr_loops = 0; bool ret; local_irq_save(flags); @@ -6176,8 +6207,8 @@ rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts) struct trace_buffer *buffer; struct ring_buffer_per_cpu *cpu_buffer; struct ring_buffer_event *event; - int nr_loops = 0; - int max_loops; + unsigned long nr_loops = 0; + unsigned long max_loops; if (ts) *ts = 0; @@ -6993,56 +7024,78 @@ EXPORT_SYMBOL_GPL(ring_buffer_swap_cpu); * ring_buffer_alloc_read_page - allocate a page to read from buffer * @buffer: the buffer to allocate for. * @cpu: the cpu buffer to allocate. + * @rpage: pointer to pass in an already allocated page (can be NULL) + * and returns the allocated page. * - * This function is used in conjunction with ring_buffer_read_page. + * This function is used in conjunction with ring_buffer_read_page(). * When reading a full page from the ring buffer, these functions * can be used to speed up the process. The calling function should * allocate a few pages first with this function. Then when it * needs to get pages from the ring buffer, it passes the result - * of this function into ring_buffer_read_page, which will swap + * of this function into ring_buffer_read_page(), which will swap * the page that was allocated, with the read page of the buffer. * + * If @rpage is provided, and it has a different order than the current + * subbuffer order, its payload will be freed and re-allocated. If it + * already matches the order, it is simply returned. + * * Returns: - * The page allocated, or ERR_PTR + * 0 on success, < 0 on error */ -struct buffer_data_read_page * -ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu) +int ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu, + struct buffer_data_read_page **rpage) { struct ring_buffer_per_cpu *cpu_buffer; - struct buffer_data_read_page *bpage = NULL; unsigned long flags; + unsigned int order; if (!cpumask_test_cpu(cpu, buffer->cpumask)) - return ERR_PTR(-ENODEV); + return -ENODEV; - bpage = kzalloc_obj(*bpage); - if (!bpage) - return ERR_PTR(-ENOMEM); + if (!rpage) + return -EINVAL; - bpage->order = buffer->subbuf_order; + order = READ_ONCE(buffer->subbuf_order); + + if (*rpage) { + if ((*rpage)->order == order) + return 0; + + /* We can reuse rpage, but we discard the payload */ + free_pages((unsigned long)(*rpage)->data, (*rpage)->order); + (*rpage)->data = NULL; + } else { + *rpage = kzalloc_obj(**rpage); + if (!*rpage) + return -ENOMEM; + } + + (*rpage)->order = order; cpu_buffer = buffer->buffers[cpu]; + local_irq_save(flags); arch_spin_lock(&cpu_buffer->lock); if (cpu_buffer->free_page.data) { - *bpage = cpu_buffer->free_page; + **rpage = cpu_buffer->free_page; cpu_buffer->free_page.data = NULL; } arch_spin_unlock(&cpu_buffer->lock); local_irq_restore(flags); - if (bpage->data) { - rb_init_data_page(bpage->data); + if ((*rpage)->data) { + rb_init_data_page((*rpage)->data); } else { - bpage->data = alloc_cpu_data(cpu, bpage->order); - if (!bpage->data) { - kfree(bpage); - return ERR_PTR(-ENOMEM); + (*rpage)->data = alloc_cpu_data(cpu, (*rpage)->order); + if (!(*rpage)->data) { + kfree(*rpage); + *rpage = NULL; + return -ENOMEM; } } - return bpage; + return 0; } EXPORT_SYMBOL_GPL(ring_buffer_alloc_read_page); @@ -7050,21 +7103,30 @@ EXPORT_SYMBOL_GPL(ring_buffer_alloc_read_page); * ring_buffer_free_read_page - free an allocated read page * @buffer: the buffer the page was allocate for * @cpu: the cpu buffer the page came from - * @data_page: the page to free + * @rpage: the buffer_data_read_page to free * * Free a page allocated from ring_buffer_alloc_read_page. */ void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, - struct buffer_data_read_page *data_page) + struct buffer_data_read_page *rpage) { struct ring_buffer_per_cpu *cpu_buffer; - struct buffer_data_page *dpage = data_page->data; - struct page *page = virt_to_page(dpage); + struct buffer_data_page *dpage; unsigned long flags; + struct page *page; if (!buffer || !buffer->buffers || !buffer->buffers[cpu]) return; + if (!rpage) + return; + + dpage = rpage->data; + if (!dpage) + goto out; + + page = virt_to_page(dpage); + cpu_buffer = buffer->buffers[cpu]; /* @@ -7072,14 +7134,14 @@ void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, * is different from the subbuffer order of the buffer - * we can't reuse it */ - if (page_ref_count(page) > 1 || data_page->order != buffer->subbuf_order) + if (page_ref_count(page) > 1 || rpage->order != READ_ONCE(buffer->subbuf_order)) goto out; local_irq_save(flags); arch_spin_lock(&cpu_buffer->lock); if (!cpu_buffer->free_page.data) { - cpu_buffer->free_page = *data_page; + cpu_buffer->free_page = *rpage; dpage = NULL; } @@ -7087,8 +7149,8 @@ void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, local_irq_restore(flags); out: - free_pages((unsigned long)dpage, data_page->order); - kfree(data_page); + free_pages((unsigned long)dpage, rpage->order); + kfree(rpage); } EXPORT_SYMBOL_GPL(ring_buffer_free_read_page); @@ -7159,10 +7221,9 @@ int ring_buffer_read_page(struct trace_buffer *buffer, if (!dpage) return -1; - guard(raw_spinlock_irqsave)(&cpu_buffer->reader_lock); + len = min_t(size_t, len, rb_read_page_capacity(data_page)); - if (data_page->order != cpu_buffer->reader_page->order) - return -1; + guard(raw_spinlock_irqsave)(&cpu_buffer->reader_lock); reader = rb_get_reader_page(cpu_buffer); if (!reader) @@ -7177,31 +7238,26 @@ int ring_buffer_read_page(struct trace_buffer *buffer, /* Check if any events were dropped */ missed_events = cpu_buffer->lost_events; - /* - * If this page has been partially read or - * if len is not big enough to read the rest of the page or - * a writer is still on the page, then - * we must copy the data from the page to the buffer. - * Otherwise, we can simply swap the page with the one passed in. - */ + /* + * It is not possible to swap the reader page if: + * - It has been partially read + * - len is not big enough to read it entirely + * - A writer is still on it + * - The ring buffer is static + * - The order doesn't match + */ if (read || (len < (size - read)) || cpu_buffer->reader_page == cpu_buffer->commit_page || - rb_is_static(cpu_buffer)) { + rb_is_static(cpu_buffer) || + data_page->order != reader->order) { struct buffer_data_page *rpage = cpu_buffer->reader_page->page; unsigned int rpos = read; unsigned int pos = 0; unsigned int event_size; unsigned int flags = 0; - /* - * If a full page is expected, this can still be returned - * if there's been a previous partial read and the - * rest of the page can be read and the commit page is off - * the reader page. - */ - if (full && - (!read || (len < (size - read)) || - cpu_buffer->reader_page == cpu_buffer->commit_page)) + /* If a full page is requested, it cannot be the commit page */ + if (full && cpu_buffer->reader_page == cpu_buffer->commit_page) return -1; if (len > (size - read)) @@ -7287,7 +7343,7 @@ int ring_buffer_read_page(struct trace_buffer *buffer, * missed events, then record it there. */ if (missed_events > 0 && - rb_page_capacity(reader) - size >= sizeof(missed_events)) { + rb_read_page_capacity(data_page) - size >= sizeof(missed_events)) { memcpy(&dpage->data[size], &missed_events, sizeof(missed_events)); local_add(RB_MISSED_STORED, &dpage->commit); @@ -7307,8 +7363,8 @@ int ring_buffer_read_page(struct trace_buffer *buffer, /* * This page may be off to user land. Zero it out here. */ - if (size < rb_page_capacity(reader)) - memset(&dpage->data[size], 0, rb_page_capacity(reader) - size); + if (size < rb_read_page_capacity(data_page)) + memset(&dpage->data[size], 0, rb_read_page_capacity(data_page) - size); return read; } @@ -7327,6 +7383,18 @@ void *ring_buffer_read_page_data(struct buffer_data_read_page *page) EXPORT_SYMBOL_GPL(ring_buffer_read_page_data); /** + * ring_buffer_read_page_size - get size of the read page. + * @page: the page to get the size from + * + * Returns size of the page in bytes. + */ +unsigned int ring_buffer_read_page_size(struct buffer_data_read_page *rpage) +{ + return rpage ? PAGE_SIZE << rpage->order : 0; +} +EXPORT_SYMBOL_GPL(ring_buffer_read_page_size); + +/** * ring_buffer_subbuf_size_get - get size of the sub buffer. * @buffer: the buffer to get the sub buffer size from * @@ -7380,8 +7448,8 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) struct ring_buffer_per_cpu *cpu_buffer; struct buffer_page *bpage, *tmp; unsigned int old_capacity; + unsigned long nr_pages; int old_order; - int nr_pages; int psize; int err; int cpu; @@ -7411,7 +7479,7 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) /* Make sure all commits have finished */ synchronize_rcu(); - buffer->subbuf_order = order; + WRITE_ONCE(buffer->subbuf_order, order); /* Make sure all new buffers are allocated, before deleting the old ones */ for_each_buffer_cpu(buffer, cpu) { @@ -7515,7 +7583,7 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) return 0; error: - buffer->subbuf_order = old_order; + WRITE_ONCE(buffer->subbuf_order, old_order); atomic_dec(&buffer->record_disabled); @@ -7563,10 +7631,10 @@ static void rb_setup_ids_meta_page(struct ring_buffer_per_cpu *cpu_buffer, struct buffer_page **subbuf_ids) { struct trace_buffer_meta *meta = cpu_buffer->meta_page; - unsigned int nr_subbufs = cpu_buffer->nr_pages + 1; + unsigned long nr_subbufs = cpu_buffer->nr_pages + 1; struct buffer_page *first_subbuf, *subbuf; - int cnt = 0; - int id = 0; + unsigned int cnt = 0; + unsigned int id = 0; id = rb_page_id(cpu_buffer, cpu_buffer->reader_page, id); subbuf_ids[id++] = cpu_buffer->reader_page; @@ -7794,6 +7862,9 @@ int ring_buffer_map(struct trace_buffer *buffer, int cpu, /* prevent another thread from changing buffer/sub-buffer sizes */ guard(mutex)(&buffer->mutex); + if (cpu_buffer->nr_pages > rb_static_max_pages()) + return -E2BIG; + err = rb_alloc_meta_page(cpu_buffer); if (err) return err; diff --git a/kernel/trace/ring_buffer_benchmark.c b/kernel/trace/ring_buffer_benchmark.c index 593e3b59e42e..c3d34c0e64e2 100644 --- a/kernel/trace/ring_buffer_benchmark.c +++ b/kernel/trace/ring_buffer_benchmark.c @@ -104,7 +104,7 @@ static enum event_status read_event(int cpu) static enum event_status read_page(int cpu) { - struct buffer_data_read_page *bpage; + struct buffer_data_read_page *bpage = NULL; struct ring_buffer_event *event; struct rb_page *rpage; unsigned long commit; @@ -114,8 +114,8 @@ static enum event_status read_page(int cpu) int inc; int i; - bpage = ring_buffer_alloc_read_page(buffer, cpu); - if (IS_ERR(bpage)) + ret = ring_buffer_alloc_read_page(buffer, cpu, &bpage); + if (ret < 0) return EVENT_DROPPED; page_size = ring_buffer_subbuf_size_get(buffer); diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index a946e0183fd1..8658cad53cb5 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -7082,8 +7082,8 @@ ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf, { struct ftrace_buffer_info *info = filp->private_data; struct trace_iterator *iter = &info->iter; + unsigned int spare_size; void *trace_data; - int page_size; ssize_t ret = 0; ssize_t size; @@ -7093,36 +7093,22 @@ ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf, if (iter->snapshot && tracer_uses_snapshot(iter->tr->current_trace)) return -EBUSY; - page_size = ring_buffer_subbuf_size_get(iter->array_buffer->buffer); + spare_size = ring_buffer_read_page_size(info->spare); - /* Make sure the spare matches the current sub buffer size */ - if (info->spare) { - if (page_size != info->spare_size) { - ring_buffer_free_read_page(iter->array_buffer->buffer, - info->spare_cpu, info->spare); - info->spare = NULL; - } - } +again: + /* Do we have previous read data to read? */ + if (info->read < spare_size) + goto read; - if (!info->spare) { - info->spare = ring_buffer_alloc_read_page(iter->array_buffer->buffer, - iter->cpu_file); - if (IS_ERR(info->spare)) { - ret = PTR_ERR(info->spare); - info->spare = NULL; - } else { - info->spare_cpu = iter->cpu_file; - info->spare_size = page_size; - } - } - if (!info->spare) + ret = ring_buffer_alloc_read_page(iter->array_buffer->buffer, iter->cpu_file, + &info->spare); + if (ret) return ret; - /* Do we have previous read data to read? */ - if (info->read < page_size) - goto read; + spare_size = ring_buffer_read_page_size(info->spare); + info->read = spare_size; + info->spare_cpu = iter->cpu_file; - again: trace_access_lock(iter->cpu_file); ret = ring_buffer_read_page(iter->array_buffer->buffer, info->spare, @@ -7148,8 +7134,9 @@ ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf, } info->read = 0; + read: - size = page_size - info->read; + size = spare_size - info->read; if (size > count) size = count; trace_data = ring_buffer_read_page_data(info->spare); @@ -7190,26 +7177,24 @@ int tracing_buffers_release(struct inode *inode, struct file *file) __trace_array_put(iter->tr); - if (info->spare) - ring_buffer_free_read_page(iter->array_buffer->buffer, - info->spare_cpu, info->spare); + ring_buffer_free_read_page(iter->array_buffer->buffer, info->spare_cpu, info->spare); kvfree(info); return 0; } struct buffer_ref { - struct trace_buffer *buffer; - void *page; - int cpu; - refcount_t refcount; + struct trace_buffer *buffer; + struct buffer_data_read_page *rpage; + int cpu; + refcount_t refcount; }; static void buffer_ref_release(struct buffer_ref *ref) { if (!refcount_dec_and_test(&ref->refcount)) return; - ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page); + ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->rpage); kfree(ref); } @@ -7268,25 +7253,15 @@ ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos, .ops = &buffer_pipe_buf_ops, .spd_release = buffer_spd_release, }; + unsigned int page_size = 0; struct buffer_ref *ref; bool woken = false; - int page_size; int entries, i; ssize_t ret = 0; if (iter->snapshot && tracer_uses_snapshot(iter->tr->current_trace)) return -EBUSY; - page_size = ring_buffer_subbuf_size_get(iter->array_buffer->buffer); - if (*ppos & (page_size - 1)) - return -EINVAL; - - if (len & (page_size - 1)) { - if (len < page_size) - return -EINVAL; - len &= (~(page_size - 1)); - } - if (splice_grow_spd(pipe, &spd)) return -ENOMEM; @@ -7306,25 +7281,39 @@ ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos, refcount_set(&ref->refcount, 1); ref->buffer = iter->array_buffer->buffer; - ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file); - if (IS_ERR(ref->page)) { - ret = PTR_ERR(ref->page); - ref->page = NULL; + + ret = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file, &ref->rpage); + if (ret) { kfree(ref); break; } ref->cpu = iter->cpu_file; - r = ring_buffer_read_page(ref->buffer, ref->page, - len, iter->cpu_file, 1); + page_size = ring_buffer_read_page_size(ref->rpage); + + r = -EINVAL; + if (IS_ALIGNED(*ppos, page_size) && len >= page_size) { + r = ring_buffer_read_page(ref->buffer, ref->rpage, len, iter->cpu_file, 1); + } else if (!i) { + /* + * If this fails to read on the first iteration, it + * means the length was too small and an error should + * be returned to user space. Otherwise, at least + * one sub-buffer was successfully read but this failed + * due to either the length was unaligned or the + * subbuf order changed. Either case, do not report + * an error. + */ + ret = -EINVAL; + } + if (r < 0) { - ring_buffer_free_read_page(ref->buffer, ref->cpu, - ref->page); + ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->rpage); kfree(ref); break; } - page = virt_to_page(ring_buffer_read_page_data(ref->page)); + page = virt_to_page(ring_buffer_read_page_data(ref->rpage)); spd.pages[i] = page; spd.partial[i].len = page_size; @@ -7842,11 +7831,70 @@ trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt, return cnt; } +/* + * The tr_index is the address of a trace_array->trace_flags_index[] + * element that holds the index of the trace flag. But since the + * trace_array reference has not been taken yet, it cannot be referenced + * as it could have been freed by a rmdir of the instance the trace_array + * represents. + * + * Search the list of trace_arrays and compare the tr_index to the + * address of the entire trace_array trace_flags_index array for each + * trace_array in the list. If one is matched, then take the reference + * and return it. If not, the trace_array no longer exits. + */ +static int trace_array_options_get(void *tr_index) +{ + struct trace_array *tr; + int ret; + + ret = security_locked_down(LOCKDOWN_TRACEFS); + if (ret) + return ret; + + if (tracing_disabled) + return -ENODEV; + + guard(mutex)(&trace_types_lock); + list_for_each_entry(tr, &ftrace_trace_arrays, list) { + if (tr_index >= (void *)&tr->trace_flags_index[0] && + tr_index < (void *)&tr->trace_flags_index[TRACE_FLAGS_MAX_SIZE]) + return __trace_array_get(tr); + } + return -ENODEV; +} + +static int trace_options_open(struct inode *inode, struct file *filp) +{ + void *tr_index = inode->i_private; + + if (trace_array_options_get(tr_index) < 0) + return -ENODEV; + + filp->private_data = tr_index; + + return 0; +} + +static int trace_options_release(struct inode *inode, struct file *filp) +{ + void *tr_index = filp->private_data; + struct trace_array *tr; + unsigned int index; + + get_tr_index(tr_index, &tr, &index); + + trace_array_put(tr); + + return 0; +} + static const struct file_operations trace_options_core_fops = { - .open = tracing_open_generic, - .read = trace_options_core_read, - .write = trace_options_core_write, - .llseek = generic_file_llseek, + .open = trace_options_open, + .read = trace_options_core_read, + .write = trace_options_core_write, + .llseek = generic_file_llseek, + .release = trace_options_release, }; struct dentry *trace_create_file(const char *name, @@ -9651,6 +9699,11 @@ __init static void enable_instances(void) if (flag_delim) *flag_delim++ = '\0'; + if (trace_array_find(name)) { + pr_warn("Tracing: Instance %s already exists\n", name); + continue; + } + if (backup) { if (backup_instance_area(backup, &addr, &size) < 0) continue; diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 74a7a50d1e78..5e76f94e7a80 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -745,11 +745,10 @@ static inline int tracing_get_cpu(struct inode *inode) void tracing_reset_cpu(struct array_buffer *buf, int cpu); struct ftrace_buffer_info { - struct trace_iterator iter; - void *spare; - unsigned int spare_cpu; - unsigned int spare_size; - unsigned int read; + struct trace_iterator iter; + struct buffer_data_read_page *spare; + unsigned int spare_cpu; + unsigned int read; }; /** @@ -1340,7 +1339,7 @@ extern void clear_ftrace_function_probes(struct trace_array *tr); int register_ftrace_command(struct ftrace_func_command *cmd); int unregister_ftrace_command(struct ftrace_func_command *cmd); -void ftrace_create_filter_files(struct ftrace_ops *ops, +void ftrace_create_filter_files(struct trace_array *tr, struct dentry *parent); void ftrace_destroy_filter_files(struct ftrace_ops *ops); @@ -1363,11 +1362,12 @@ static inline void clear_ftrace_function_probes(struct trace_array *tr) { } +static inline void ftrace_create_filter_files(struct trace_array *tr, + struct dentry *parent) { } /* * The ops parameter passed in is usually undefined. * This must be a macro. */ -#define ftrace_create_filter_files(ops, parent) do { } while (0) #define ftrace_destroy_filter_files(ops) do { } while (0) #endif /* CONFIG_FUNCTION_TRACER && CONFIG_DYNAMIC_FTRACE */ diff --git a/kernel/trace/trace_btf.c b/kernel/trace/trace_btf.c index 00172f301f25..ee7a04886bf6 100644 --- a/kernel/trace/trace_btf.c +++ b/kernel/trace/trace_btf.c @@ -61,47 +61,50 @@ struct btf_anon_stack { /* * Find a member of data structure/union by name and return it. - * Return NULL if not found, or -EINVAL if parameter is invalid. - * If the member is an member of anonymous union/structure, the offset - * of that anonymous union/structure is stored into @anon_offset. Caller - * can calculate the correct offset from the root data structure by - * adding anon_offset to the member's offset. + * Return NULL if not found, or ERR_PTR(-EINVAL) if parameter is invalid. + * If the member is a member of an anonymous union/structure, the bit offset + * of that anonymous union/structure is stored into @anon_offset. + * If @member_type is non-NULL, the actual containing structure/union type + * of the found member is stored into @member_type. */ const struct btf_member *btf_find_struct_member(struct btf *btf, const struct btf_type *type, const char *member_name, - u32 *anon_offset) + u32 *anon_offset, + const struct btf_type **member_type) { struct btf_anon_stack *anon_stack; const struct btf_member *member; + const struct btf_type *mtype; u32 tid, cur_offset = 0; const char *name; int i, top = 0; + if (!btf_type_is_struct(type)) + return ERR_PTR(-EINVAL); + anon_stack = kzalloc_objs(*anon_stack, BTF_ANON_STACK_MAX); if (!anon_stack) return ERR_PTR(-ENOMEM); retry: - if (!btf_type_is_struct(type)) { - member = ERR_PTR(-EINVAL); - goto out; - } - for_each_member(i, type, member) { if (!member->name_off) { /* Anonymous union/struct: push it for later use */ - if (btf_type_skip_modifiers(btf, member->type, &tid) && + mtype = btf_type_skip_modifiers(btf, member->type, &tid); + if (mtype && btf_type_is_struct(mtype) && top < BTF_ANON_STACK_MAX) { anon_stack[top].tid = tid; - anon_stack[top++].offset = - cur_offset + member->offset; + anon_stack[top++].offset = cur_offset + + __btf_member_bit_offset(type, member); } } else { name = btf_name_by_offset(btf, member->name_off); if (name && !strcmp(member_name, name)) { if (anon_offset) *anon_offset = cur_offset; + if (member_type) + *member_type = type; goto out; } } diff --git a/kernel/trace/trace_btf.h b/kernel/trace/trace_btf.h index 4bc44bc261e6..4bd26bceae23 100644 --- a/kernel/trace/trace_btf.h +++ b/kernel/trace/trace_btf.h @@ -8,4 +8,5 @@ const struct btf_param *btf_get_func_param(const struct btf_type *func_proto, const struct btf_member *btf_find_struct_member(struct btf *btf, const struct btf_type *type, const char *member_name, - u32 *anon_offset); + u32 *anon_offset, + const struct btf_type **member_type); diff --git a/kernel/trace/trace_eprobe.c b/kernel/trace/trace_eprobe.c index 78fa1cbda9ac..998e6390937a 100644 --- a/kernel/trace/trace_eprobe.c +++ b/kernel/trace/trace_eprobe.c @@ -930,7 +930,7 @@ static int __trace_eprobe_create(int argc, const char *argv[]) } else ep->filter_str = NULL; - ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + ctx = kzalloc_obj(*ctx); if (!ctx) return -ENOMEM; ctx->event = ep->event; diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 1d39eaf6a0f7..9dbc2441763b 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -2736,14 +2736,14 @@ static const struct file_operations ftrace_show_event_filters_fops = { .open = ftrace_event_show_filters_open, .read = seq_read, .llseek = seq_lseek, - .release = seq_release, + .release = ftrace_event_release, }; static const struct file_operations ftrace_show_event_triggers_fops = { .open = ftrace_event_show_triggers_open, .read = seq_read, .llseek = seq_lseek, - .release = seq_release, + .release = ftrace_event_release, }; static const struct file_operations ftrace_set_event_pid_fops = { @@ -2908,7 +2908,17 @@ ftrace_event_set_open(struct inode *inode, struct file *file) static int ftrace_event_show_filters_open(struct inode *inode, struct file *file) { - return ftrace_event_open(inode, file, &show_show_event_filters_seq_ops); + struct trace_array *tr = inode->i_private; + int ret; + + ret = tracing_check_open_get_tr(tr); + if (ret) + return ret; + + ret = ftrace_event_open(inode, file, &show_show_event_filters_seq_ops); + if (ret < 0) + trace_array_put(tr); + return ret; } /** @@ -2922,7 +2932,17 @@ ftrace_event_show_filters_open(struct inode *inode, struct file *file) static int ftrace_event_show_triggers_open(struct inode *inode, struct file *file) { - return ftrace_event_open(inode, file, &show_show_event_triggers_seq_ops); + struct trace_array *tr = inode->i_private; + int ret; + + ret = tracing_check_open_get_tr(tr); + if (ret) + return ret; + + ret = ftrace_event_open(inode, file, &show_show_event_triggers_seq_ops); + if (ret < 0) + trace_array_put(tr); + return ret; } static int diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c index cd37f2013758..c879d43a5fbb 100644 --- a/kernel/trace/trace_functions.c +++ b/kernel/trace/trace_functions.c @@ -101,7 +101,7 @@ int ftrace_create_function_files(struct trace_array *tr, return ret; } - ftrace_create_filter_files(tr->ops, parent); + ftrace_create_filter_files(tr, parent); return 0; } diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index c4163904ba74..804442b2f7d2 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -625,6 +625,7 @@ static int get_bitoffset_of_field(char **pfieldname, const struct btf_type **pty { const struct btf_type *type = *ptype; const struct btf_member *field; + const struct btf_type *mtype; struct btf *btf = ctx_btf(ctx); char *fieldname = *pfieldname; int bitoffs = 0; @@ -640,7 +641,7 @@ static int get_bitoffset_of_field(char **pfieldname, const struct btf_type **pty anon_offs = 0; field = btf_find_struct_member(btf, type, fieldname, - &anon_offs); + &anon_offs, &mtype); if (IS_ERR(field)) { trace_probe_log_err(ctx->offset, BAD_BTF_TID); return PTR_ERR(field); @@ -653,7 +654,7 @@ static int get_bitoffset_of_field(char **pfieldname, const struct btf_type **pty bitoffs += anon_offs; /* Accumulate the bit-offsets of the dot-connected fields */ - if (btf_type_kflag(type)) { + if (btf_type_kflag(mtype)) { bitoffs += BTF_MEMBER_BIT_OFFSET(field->offset); ctx->last_bitsize = BTF_MEMBER_BITFIELD_SIZE(field->offset); } else { @@ -661,11 +662,11 @@ static int get_bitoffset_of_field(char **pfieldname, const struct btf_type **pty ctx->last_bitsize = 0; } - type = btf_type_skip_modifiers(btf, field->type, NULL); - if (!type) { - trace_probe_log_err(ctx->offset, BAD_BTF_TID); - return -EINVAL; - } + type = btf_type_skip_modifiers(btf, field->type, NULL); + if (!type) { + trace_probe_log_err(ctx->offset, BAD_BTF_TID); + return -EINVAL; + } if (next) ctx->offset += next - fieldname; @@ -2552,19 +2553,60 @@ int traceprobe_set_print_fmt(struct trace_probe *tp, enum probe_print_type ptype int traceprobe_define_arg_fields(struct trace_event_call *event_call, size_t offset, struct trace_probe *tp) { + struct trace_probe_event *tpe = trace_probe_event_from_call(event_call); int ret, i; + /* + * A field created by trace_define_field() only stores the name and + * type pointers, it does not copy the strings. Here they point into + * the probe_arg of @tp, which is freed when @tp is removed. For an + * event with multiple probes attached, the field list is defined + * once by the first probe but kept alive by the surviving siblings, + * so removing that first probe would leave the fields referencing + * freed memory. Duplicate the strings and anchor the copies on the + * trace_probe_event, which lives as long as the field list itself. + * + * event_define_fields() ignores the return value of this hook, so + * if a previous attempt failed before creating any field, it may + * call here again. Release duplicates left behind by such an + * attempt before starting over. + */ + for (i = 0; i < tpe->nr_field_strings; i++) + kfree(tpe->field_strings[i]); + kfree(tpe->field_strings); + tpe->field_strings = NULL; + tpe->nr_field_strings = 0; + + if (tp->nr_args) { + tpe->field_strings = kcalloc(tp->nr_args * 2, sizeof(char *), + GFP_KERNEL); + if (!tpe->field_strings) + return -ENOMEM; + } + /* Set argument names as fields */ for (i = 0; i < tp->nr_args; i++) { struct probe_arg *parg = &tp->args[i]; const char *fmt = parg->type->fmttype; int size = parg->type->size; + char *name, *type; if (parg->fmt) fmt = parg->fmt; if (parg->count) size *= parg->count; - ret = trace_define_field(event_call, fmt, parg->name, + + name = kstrdup(parg->name, GFP_KERNEL); + type = kstrdup(fmt, GFP_KERNEL); + if (!name || !type) { + kfree(name); + kfree(type); + return -ENOMEM; + } + tpe->field_strings[tpe->nr_field_strings++] = name; + tpe->field_strings[tpe->nr_field_strings++] = type; + + ret = trace_define_field(event_call, type, name, offset + parg->offset, size, parg->type->is_signed, FILTER_OTHER); @@ -2576,6 +2618,11 @@ int traceprobe_define_arg_fields(struct trace_event_call *event_call, static void trace_probe_event_free(struct trace_probe_event *tpe) { + int i; + + for (i = 0; i < tpe->nr_field_strings; i++) + kfree(tpe->field_strings[i]); + kfree(tpe->field_strings); kfree(tpe->class.system); kfree(tpe->call.name); kfree(tpe->call.print_fmt); diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h index fba1af092a9b..d1fb3520700f 100644 --- a/kernel/trace/trace_probe.h +++ b/kernel/trace/trace_probe.h @@ -264,6 +264,8 @@ struct trace_probe_event { struct trace_event_call call; struct list_head files; struct list_head probes; + char **field_strings; + int nr_field_strings; struct trace_uprobe_filter filter[]; }; diff --git a/kernel/trace/trace_remote.c b/kernel/trace/trace_remote.c index e6724f947170..75fa1ffc4c96 100644 --- a/kernel/trace/trace_remote.c +++ b/kernel/trace/trace_remote.c @@ -251,8 +251,8 @@ static int trace_remote_get(struct trace_remote *remote, int cpu) if (cpu != RING_BUFFER_ALL_CPUS && !remote->pcpu_reader_locks) { int lock_cpu; - remote->pcpu_reader_locks = kcalloc(nr_cpu_ids, sizeof(*remote->pcpu_reader_locks), - GFP_KERNEL); + remote->pcpu_reader_locks = kzalloc_objs(*remote->pcpu_reader_locks, + nr_cpu_ids); if (!remote->pcpu_reader_locks) { trace_remote_try_unload(remote); return -ENOMEM; @@ -324,7 +324,7 @@ static int __alloc_ring_buffer_iter(struct trace_remote_iterator *iter, int cpu) return iter->rb_iter ? 0 : -ENOMEM; } - iter->rb_iters = kcalloc(nr_cpu_ids, sizeof(*iter->rb_iters), GFP_KERNEL); + iter->rb_iters = kzalloc_objs(*iter->rb_iters, nr_cpu_ids); if (!iter->rb_iters) return -ENOMEM; @@ -1204,7 +1204,7 @@ remote_events_dir_header_page_read(struct file *filp, char __user *ubuf, size_t struct trace_seq *s; int ret; - s = kmalloc(sizeof(*s), GFP_KERNEL); + s = kmalloc_obj(*s); if (!s) return -ENOMEM; @@ -1227,7 +1227,7 @@ remote_events_dir_header_event_read(struct file *filp, char __user *ubuf, size_t struct trace_seq *s; int ret; - s = kmalloc(sizeof(*s), GFP_KERNEL); + s = kmalloc_obj(*s); if (!s) return -ENOMEM; diff --git a/kernel/trace/trace_stack.c b/kernel/trace/trace_stack.c index 0aa2514a6593..e7f4e523587d 100644 --- a/kernel/trace/trace_stack.c +++ b/kernel/trace/trace_stack.c @@ -499,7 +499,7 @@ stack_trace_filter_open(struct inode *inode, struct file *file) struct ftrace_ops *ops = inode->i_private; /* Checks for tracefs lockdown */ - return ftrace_regex_open(ops, FTRACE_ITER_FILTER, + return ftrace_regex_open(NULL, ops, FTRACE_ITER_FILTER, inode, file); } diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c index 861d857adadb..22cc3c8181b8 100644 --- a/kernel/trace/trace_uprobe.c +++ b/kernel/trace/trace_uprobe.c @@ -368,7 +368,7 @@ error: static void free_trace_uprobe(struct trace_uprobe *tu) { - if (!tu) + if (IS_ERR_OR_NULL(tu)) return; path_put(&tu->path); @@ -533,7 +533,7 @@ static int register_trace_uprobe(struct trace_uprobe *tu) return ret; } -DEFINE_FREE(free_trace_uprobe, struct trace_uprobe *, if (_T) free_trace_uprobe(_T)) +DEFINE_FREE(free_trace_uprobe, struct trace_uprobe *, free_trace_uprobe(_T)) /* * Argument syntax: diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 3c034cbc5bb3..1ae3732a2c51 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -3197,7 +3197,16 @@ restart: #ifdef CONFIG_PREEMPT_RT static void worker_lock_callback(struct worker_pool *pool) { - spin_lock(&pool->cb_lock); + /* + * SINGLE_DEPTH_NESTING is for a dead pool's bh_worker() running from + * drain_dead_softirq_workfn() inside a live pool's bh_worker(). The + * unlocked read is stable: the flag is only set while @pool's CPU is + * dead, inside a serialized hotplug operation. data_race() as the value + * only affects the lockdep annotation and the read can be elided when + * lockdep is disabled. + */ + spin_lock_nested(&pool->cb_lock, + data_race(pool->flags) & POOL_BH_DRAINING ? SINGLE_DEPTH_NESTING : 0); } static void worker_unlock_callback(struct worker_pool *pool) @@ -5285,12 +5294,6 @@ static void pwq_release_workfn(struct kthread_work *work) mutex_unlock(&wq->mutex); } - if (!is_percpu_pool(pool)) { - mutex_lock(&wq_pool_mutex); - put_unbound_pool(pool); - mutex_unlock(&wq_pool_mutex); - } - if (!list_empty(&pwq->pending_node)) { struct wq_node_nr_active *nna = wq_node_nr_active(pwq->wq, pwq->pool->node); @@ -5300,6 +5303,12 @@ static void pwq_release_workfn(struct kthread_work *work) raw_spin_unlock_irq(&nna->lock); } + if (!is_percpu_pool(pool)) { + mutex_lock(&wq_pool_mutex); + put_unbound_pool(pool); + mutex_unlock(&wq_pool_mutex); + } + kfree_rcu(pwq, rcu); /* @@ -8050,6 +8059,9 @@ static int wq_watchdog_param_set_thresh(const char *val, if (ret) return ret; + if (thresh > MAX_JIFFY_OFFSET / HZ) + return -ERANGE; + if (system_percpu_wq) wq_watchdog_set_thresh(thresh); else @@ -8080,12 +8092,12 @@ static inline void wq_watchdog_init(void) { } static void bh_pool_kick_normal(struct irq_work *irq_work) { - raise_softirq_irqoff(TASKLET_SOFTIRQ); + raise_softirq(TASKLET_SOFTIRQ); } static void bh_pool_kick_highpri(struct irq_work *irq_work) { - raise_softirq_irqoff(HI_SOFTIRQ); + raise_softirq(HI_SOFTIRQ); } static void __init restrict_unbound_cpumask(const char *name, const struct cpumask *mask) |