From 931a577fc79ea6a169a33f5538f4c1433235c358 Mon Sep 17 00:00:00 2001 From: Yiyang Chen Date: Tue, 23 Jun 2026 06:11:09 +0000 Subject: bpf: Reject offset refcount acquire arguments bpf_refcount_acquire() increments the refcount at the caller-supplied pointer plus the refcount field offset, then returns the caller-supplied pointer unchanged. The verifier records the return value as a base pointer to the refcounted object. bpf_list_pop_front() and bpf_rbtree_remove() can return embedded graph-node pointers as PTR_TO_BTF_ID | MEM_ALLOC with a fixed offset equal to the node field offset. Passing such a pointer directly to bpf_refcount_acquire() currently passes the refcounted-kptr type check. That makes the runtime operation start from base + node_off while the verifier models the returned pointer as the object base. Require refcount-acquire arguments to have zero fixed offset by carrying the requirement through check_func_arg_reg_off() to __check_ptr_off_reg(). Programs can still acquire a refcount from a graph-node-derived pointer after normalizing it with container_of(). Fixes: 7c50b1cb76aca ("bpf: Add bpf_refcount_acquire kfunc") Signed-off-by: Yiyang Chen Acked-by: Eduard Zingerman Acked-by: Yonghong Song Link: https://lore.kernel.org/r/2f894647f56f71838fdddeb97a3e057ed35ea92e.1782192383.git.chenyy23@mails.tsinghua.edu.cn Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 21a365d436a5..3cdc2e90f643 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -7996,9 +7996,10 @@ reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields) return field; } -static int check_func_arg_reg_off(struct bpf_verifier_env *env, - const struct bpf_reg_state *reg, argno_t argno, - enum bpf_arg_type arg_type) +static int __check_func_arg_reg_off(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg, argno_t argno, + enum bpf_arg_type arg_type, + bool btf_id_fixed_off_ok) { u32 type = reg->type; @@ -8055,12 +8056,11 @@ static int check_func_arg_reg_off(struct bpf_verifier_env *env, case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU: /* When referenced PTR_TO_BTF_ID is passed to release function, * its fixed offset must be 0. In the other cases, fixed offset - * can be non-zero. This was already checked above. So pass - * fixed_off_ok as true to allow fixed offset for all other - * cases. var_off always must be 0 for PTR_TO_BTF_ID, hence we - * still need to do checks instead of returning. + * can be non-zero unless the caller requires otherwise. + * var_off always must be 0 for PTR_TO_BTF_ID, hence we still + * need to do checks instead of returning. */ - return __check_ptr_off_reg(env, reg, argno, true); + return __check_ptr_off_reg(env, reg, argno, btf_id_fixed_off_ok); case PTR_TO_CTX: /* * Allow fixed and variable offsets for syscall context, but @@ -8076,6 +8076,13 @@ static int check_func_arg_reg_off(struct bpf_verifier_env *env, } } +static int check_func_arg_reg_off(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg, argno_t argno, + enum bpf_arg_type arg_type) +{ + return __check_func_arg_reg_off(env, reg, argno, arg_type, true); +} + static int check_arg_const_str(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno) { @@ -11947,6 +11954,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ enum bpf_arg_type arg_type = ARG_DONTCARE; argno_t argno = argno_from_arg(i + 1); int regno = reg_from_argno(argno); + bool btf_id_fixed_off_ok = true; u32 ref_id, type_size; bool is_ret_buf_sz = false; int kf_arg_type; @@ -12120,7 +12128,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ case KF_ARG_PTR_TO_MEM: case KF_ARG_PTR_TO_MEM_SIZE: case KF_ARG_PTR_TO_CALLBACK: - case KF_ARG_PTR_TO_REFCOUNTED_KPTR: case KF_ARG_PTR_TO_CONST_STR: case KF_ARG_PTR_TO_WORKQUEUE: case KF_ARG_PTR_TO_TIMER: @@ -12134,6 +12141,10 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ case KF_ARG_PTR_TO_CTX: arg_type = ARG_PTR_TO_CTX; break; + case KF_ARG_PTR_TO_REFCOUNTED_KPTR: + arg_type = ARG_PTR_TO_BTF_ID; + btf_id_fixed_off_ok = false; + break; default: verifier_bug(env, "unknown kfunc arg type %d", kf_arg_type); return -EFAULT; @@ -12141,7 +12152,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ if (regno == meta->release_regno) arg_type |= OBJ_RELEASE; - ret = check_func_arg_reg_off(env, reg, argno, arg_type); + ret = __check_func_arg_reg_off(env, reg, argno, arg_type, + btf_id_fixed_off_ok); if (ret < 0) return ret; -- cgit v1.2.3 From 72a85e9464a5332fb2cd7efd26d9295275ceda2d Mon Sep 17 00:00:00 2001 From: Nuoqi Gui Date: Tue, 23 Jun 2026 18:43:38 +0800 Subject: bpf: Mask pseudo pointer values in verifier logs print_bpf_insn() masks ldimm64 immediates for pointer-bearing pseudo sources when pointer leaks are not allowed, but the mask only covers BPF_PSEUDO_MAP_FD and BPF_PSEUDO_MAP_VALUE. BPF_PSEUDO_MAP_IDX, BPF_PSEUDO_MAP_IDX_VALUE, and BPF_PSEUDO_BTF_ID can also be resolved to kernel pointer values before the verifier log prints the instruction. Include them in the existing pointer classification so the log prints 0x0 instead of the rewritten address. Fixes: 4976b718c355 ("bpf: Introduce pseudo_btf_id") Fixes: 387544bfa291 ("bpf: Introduce fd_idx") Signed-off-by: Nuoqi Gui Link: https://lore.kernel.org/r/20260623-f01-13-pseudo-btf-id-cap-bpf-v2-1-a190ebb8f3e2@mails.tsinghua.edu.cn Signed-off-by: Alexei Starovoitov Acked-by: Eduard Zingerman --- kernel/bpf/disasm.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c index f8a3c7eb451e..0391b3bc0073 100644 --- a/kernel/bpf/disasm.c +++ b/kernel/bpf/disasm.c @@ -323,7 +323,10 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, */ u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; bool is_ptr = insn->src_reg == BPF_PSEUDO_MAP_FD || - insn->src_reg == BPF_PSEUDO_MAP_VALUE; + insn->src_reg == BPF_PSEUDO_MAP_VALUE || + insn->src_reg == BPF_PSEUDO_MAP_IDX || + insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE || + insn->src_reg == BPF_PSEUDO_BTF_ID; char tmp[64]; if (is_ptr && !allow_ptr_leaks) -- cgit v1.2.3 From 26490a375cb9be9bac96b5171610fd85ca6c2305 Mon Sep 17 00:00:00 2001 From: KaFai Wan Date: Wed, 24 Jun 2026 20:35:35 +0800 Subject: bpf: Fix insn_aux_data leak on verifier err_free_env path When bpf_check() allocates env->insn_aux_data successfully but later fails to allocate env->succ, it jumps directly to err_free_env. The existing vfree(env->insn_aux_data) sits before the err_free_env label, so that direct jump bypasses it and leaks insn_aux_data. Move vfree(env->insn_aux_data) into err_free_env so all early and late exit paths release it consistently. Fixes: 2f69c5685427 ("bpf: make bpf_insn_successors to return a pointer") Signed-off-by: KaFai Wan Reviewed-by: Anton Protopopov Link: https://lore.kernel.org/r/20260624123536.114757-1-kafai.wan@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 3cdc2e90f643..6515d4d3c003 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -20006,13 +20006,13 @@ err_unlock: if (!is_priv) mutex_unlock(&bpf_verifier_lock); bpf_clear_insn_aux_data(env, 0, env->prog->len); - vfree(env->insn_aux_data); err_free_env: bpf_stack_liveness_free(env); kvfree(env->cfg.insn_postorder); kvfree(env->scc_info); kvfree(env->succ); kvfree(env->gotox_tmp_buf); + vfree(env->insn_aux_data); kvfree(env); return ret; } -- cgit v1.2.3 From 9b51a6155d14389876916726430da30eabb1d4ed Mon Sep 17 00:00:00 2001 From: Jann Horn Date: Fri, 26 Jun 2026 17:52:52 +0200 Subject: bpf,fork: wipe ->bpf_storage before bailouts that access it Currently, copy_process() can bail out to free_task() before p->bpf_storage has been initialized, with this call graph (shown here for the !CONFIG_MEMCG case): copy_process dup_task_struct arch_dup_task_struct [copies the entire task_struct, including ->bpf_storage member] [RLIMIT_NPROC check fails] delayed_free_task free_task bpf_task_storage_free rcu_dereference(task->bpf_storage) bpf_local_storage_destroy In this case, the nascent task's ->bpf_storage member that bpf_local_storage_destroy() operates on is a plain copy of the parent's ->bpf_storage pointer, not a real initialized pointer. This leads to badness (kernel hangs, UAF). This is reachable as long as the process calling fork() has been inserted into a task storage map. Cc: stable@kernel.org Fixes: a10787e6d58c ("bpf: Enable task local storage for tracing programs") Signed-off-by: Jann Horn Signed-off-by: Andrii Nakryiko --- kernel/fork.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/fork.c b/kernel/fork.c index 13e38e89a1f3..f0e2e131a9a5 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1009,6 +1009,11 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) tsk->mm_cid.active = 0; INIT_HLIST_NODE(&tsk->mm_cid.node); #endif + +#ifdef CONFIG_BPF_SYSCALL + RCU_INIT_POINTER(tsk->bpf_storage, NULL); + tsk->bpf_ctx = NULL; +#endif return tsk; free_stack: @@ -2247,10 +2252,6 @@ __latent_entropy struct task_struct *copy_process( p->sequential_io = 0; p->sequential_io_avg = 0; #endif -#ifdef CONFIG_BPF_SYSCALL - RCU_INIT_POINTER(p->bpf_storage, NULL); - p->bpf_ctx = NULL; -#endif unwind_task_init(p); -- cgit v1.2.3 From a6f0643e4f63cfaa0d5d4a69de4f132eac4b8fe4 Mon Sep 17 00:00:00 2001 From: Matt Bobrowski Date: Sun, 28 Jun 2026 20:11:03 +0000 Subject: bpf: Reject BPF_MAP_TYPE_INODE_STORAGE creation if BPF LSM is uninitialized When CONFIG_BPF_LSM=y is set, BPF inode storage maps (BPF_MAP_TYPE_INODE_STORAGE) are compiled into the kernel. However, if the BPF LSM is not explicitly enabled at boot time (e.g. omitted from the "lsm=" boot parameter), lsm_prepare() is never executed for the BPF LSM. Consequently, the BPF inode security blob offset (bpf_lsm_blob_sizes.lbs_inode) is never initialized and remains at its default compiled size of 8 bytes instead of being updated to a valid offset past the reserved struct rcu_head (typically 16 bytes or more). When a privileged user creates and updates a BPF_MAP_TYPE_INODE_STORAGE map, bpf_inode() evaluates inode->i_security + 8. This erroneously aliases the struct rcu_head.func callback pointer at the beginning of the inode->i_security blob. During subsequent map element cleanup or inode destruction, writing NULL to owner_storage clears the queued RCU callback pointer. When rcu_do_batch() later executes the queued callback, it attempts an instruction fetch at address 0x0, triggering an immediate kernel panic. Fix this by introducing a global bpf_lsm_initialized boolean flag marked with __ro_after_init. Set this flag to true inside bpf_lsm_init() when the LSM framework successfully registers the BPF LSM. Gate map allocation in inode_storage_map_alloc() on this flag, returning -EOPNOTSUPP if the BPF LSM is in turn uninitialized. This fail-fast approach prevents userspace from allocating inode storage maps when the supporting BPF LSM infrastructure is absent, avoiding zombie map states. Fixes: 8ea636848aca ("bpf: Implement bpf_local_storage for inodes") Reported-by: oxsignal Signed-off-by: Matt Bobrowski Signed-off-by: Daniel Borkmann Reviewed-by: Emil Tsalapatis Reviewed-by: Amery Hung Link: https://lore.kernel.org/bpf/20260628201103.3624525-1-mattbobrowski@google.com --- kernel/bpf/bpf_inode_storage.c | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/bpf_inode_storage.c b/kernel/bpf/bpf_inode_storage.c index 0da8d923e39d..f9e81060c1f4 100644 --- a/kernel/bpf/bpf_inode_storage.c +++ b/kernel/bpf/bpf_inode_storage.c @@ -178,6 +178,15 @@ static int notsupp_get_next_key(struct bpf_map *map, void *key, static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr) { + /* + * Do not allow allocation of BPF_MAP_TYPE_INODE_STORAGE if the BPF LSM + * was not initialized by the LSM framework at boot. Without proper + * initialization, the BPF inode security blob offset remains unprepared, + * causing bpf_inode() to calculate an invalid memory offset and corrupt + * inode->i_security. + */ + if (!bpf_lsm_initialized) + return ERR_PTR(-EOPNOTSUPP); return bpf_local_storage_map_alloc(attr, &inode_cache); } -- cgit v1.2.3 From 96cce16e26dd02a8678f1e87f88a4b5cdb63b995 Mon Sep 17 00:00:00 2001 From: Pawan Gupta Date: Mon, 29 Jun 2026 22:37:52 -0700 Subject: bpf: Support for hardening against JIT spraying The BPF JIT allocator packs many small programs into larger executable allocations and reuses space within those allocations as programs are loaded and freed. When fresh code is written into space that a previous program occupied, an indirect jump into the new program can reuse a branch prediction left behind by the old one. Flush the indirect branch predictors before reusing JIT memory so that indirect jumps into a newly written program don't reuse predictions from an old program that occupied the same space. Introduce bpf_arch_pred_flush_enabled static key and bpf_arch_pred_flush static call for flushing the branch predictors on JIT memory reuse. Architectures that need a flush, can update it to a predictor flush function. By default, its a NOP and does not emit any CALL. Allocations larger than a pack are not covered by this flush. That is safe because cBPF programs (the unprivileged attack surface) are bounded well below a pack size. Issue a warning if this assumption is ever violated while the flush is active. Signed-off-by: Pawan Gupta Acked-by: Daniel Borkmann Signed-off-by: Daniel Borkmann --- kernel/bpf/core.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 649cce41e13f..7f0a17f128d4 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -883,6 +884,15 @@ void bpf_jit_fill_hole_with_zero(void *area, unsigned int size) memset(area, 0, size); } +DEFINE_STATIC_CALL_NULL(bpf_arch_pred_flush, bpf_arch_pred_flush); + +/* + * Enabled once bpf_arch_pred_flush points at a real flush routine. Lets the + * pack allocator test "is a predictor flush wired up at all" with a cheap + * static branch instead of repeatedly querying the static call target. + */ +DEFINE_STATIC_KEY_FALSE(bpf_pred_flush_enabled); + #define BPF_PROG_SIZE_TO_NBITS(size) (round_up(size, BPF_PROG_CHUNK_SIZE) / BPF_PROG_CHUNK_SIZE) static DEFINE_MUTEX(pack_mutex); @@ -941,6 +951,14 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) mutex_lock(&pack_mutex); if (size > BPF_PROG_PACK_SIZE) { + /* + * Allocations larger than a pack get their own pages, and + * predictors are not flushed for such allocation. This is only + * safe because cBPF programs (the unprivileged attack surface) + * are bounded well below a pack size. + */ + if (static_branch_unlikely(&bpf_pred_flush_enabled)) + pr_warn_once("BPF: Predictors not flushed for allocations greater than BPF_PROG_PACK_SIZE\n"); size = round_up(size, PAGE_SIZE); ptr = bpf_jit_alloc_exec(size); if (ptr) { @@ -971,6 +989,7 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) pos = 0; found_free_area: + static_call_cond(bpf_arch_pred_flush)(); bitmap_set(pack->bitmap, pos, nbits); ptr = (void *)(pack->ptr) + (pos << BPF_PROG_CHUNK_SHIFT); -- cgit v1.2.3 From 0bb99f2cfaae6822d734d69722de30af823efdf3 Mon Sep 17 00:00:00 2001 From: Pawan Gupta Date: Mon, 29 Jun 2026 22:38:23 -0700 Subject: bpf: Restrict JIT predictor flush to cBPF Currently predictor flush on memory reuse is done for all BPF JIT allocations, but only cBPF programs can be loaded by an unprivileged user. eBPF is privileged by default, and flushing predictors for all CPUs on every eBPF reuse penalizes the common case for no security benefit. eBPF allocations can be frequent on busy systems, only flush predictors for cBPF programs. Trampoline and dispatcher allocations also skip the flush as they are eBPF-only. Signed-off-by: Pawan Gupta Acked-by: Daniel Borkmann Signed-off-by: Daniel Borkmann --- kernel/bpf/core.c | 13 ++++++++----- kernel/bpf/dispatcher.c | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 7f0a17f128d4..1614ccc3f111 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -942,7 +942,7 @@ out: return NULL; } -void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) +void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool was_classic) { unsigned int nbits = BPF_PROG_SIZE_TO_NBITS(size); struct bpf_prog_pack *pack; @@ -957,7 +957,7 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) * safe because cBPF programs (the unprivileged attack surface) * are bounded well below a pack size. */ - if (static_branch_unlikely(&bpf_pred_flush_enabled)) + if (was_classic && static_branch_unlikely(&bpf_pred_flush_enabled)) pr_warn_once("BPF: Predictors not flushed for allocations greater than BPF_PROG_PACK_SIZE\n"); size = round_up(size, PAGE_SIZE); ptr = bpf_jit_alloc_exec(size); @@ -989,7 +989,9 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) pos = 0; found_free_area: - static_call_cond(bpf_arch_pred_flush)(); + /* Flush only for cBPF as it may contain a crafted gadget */ + if (static_branch_unlikely(&bpf_pred_flush_enabled) && was_classic) + static_call_cond(bpf_arch_pred_flush)(); bitmap_set(pack->bitmap, pos, nbits); ptr = (void *)(pack->ptr) + (pos << BPF_PROG_CHUNK_SHIFT); @@ -1149,7 +1151,8 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr, unsigned int alignment, struct bpf_binary_header **rw_header, u8 **rw_image, - bpf_jit_fill_hole_t bpf_fill_ill_insns) + bpf_jit_fill_hole_t bpf_fill_ill_insns, + bool was_classic) { struct bpf_binary_header *ro_header; u32 size, hole, start; @@ -1162,7 +1165,7 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr, if (bpf_jit_charge_modmem(size)) return NULL; - ro_header = bpf_prog_pack_alloc(size, bpf_fill_ill_insns); + ro_header = bpf_prog_pack_alloc(size, bpf_fill_ill_insns, was_classic); if (!ro_header) { bpf_jit_uncharge_modmem(size); return NULL; diff --git a/kernel/bpf/dispatcher.c b/kernel/bpf/dispatcher.c index b77db7413f8c..ea2d60dc1fee 100644 --- a/kernel/bpf/dispatcher.c +++ b/kernel/bpf/dispatcher.c @@ -145,7 +145,7 @@ void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, mutex_lock(&d->mutex); if (!d->image) { - d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero); + d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero, false); if (!d->image) goto out; d->rw_image = bpf_jit_alloc_exec(PAGE_SIZE); -- cgit v1.2.3 From a23c1c5396a91680703360d1ee28a44657c503c4 Mon Sep 17 00:00:00 2001 From: Pawan Gupta Date: Mon, 29 Jun 2026 22:38:38 -0700 Subject: bpf: Skip redundant IBPB in pack allocator bpf_prog_pack_alloc() issues IBPB on all CPUs on every cBPF allocation, even when reusing chunks from an existing pack where no new memory was touched since the last IBPB. Since IBPB on all CPUs is heavy, Dave Hansen suggested to track allocation since last IBPB, and only issue IBPB at reuse for the chunks that have not seen an IBPB since they were last freed. Track per-pack whether an IBPB is needed via arch_flush_needed. Set it when allocating a chunk, reset on IBPB flush. On reuse, conditionally issue the flush. Since IBPB invalidates all BTB entries, clear the flag on all packs after flushing. Signed-off-by: Pawan Gupta Acked-by: Daniel Borkmann Signed-off-by: Daniel Borkmann --- kernel/bpf/core.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 1614ccc3f111..50aba113ef9d 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -876,6 +876,7 @@ int bpf_jit_add_poke_descriptor(struct bpf_prog *prog, struct bpf_prog_pack { struct list_head list; void *ptr; + bool arch_flush_needed; unsigned long bitmap[]; }; @@ -928,6 +929,8 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_ins bpf_fill_ill_insns(pack->ptr, BPF_PROG_PACK_SIZE); bitmap_zero(pack->bitmap, BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE); + if (static_branch_unlikely(&bpf_pred_flush_enabled)) + pack->arch_flush_needed = true; set_vm_flush_reset_perms(pack->ptr); err = set_memory_rox((unsigned long)pack->ptr, BPF_PROG_PACK_SIZE / PAGE_SIZE); @@ -990,8 +993,15 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool found_free_area: /* Flush only for cBPF as it may contain a crafted gadget */ - if (static_branch_unlikely(&bpf_pred_flush_enabled) && was_classic) + if (static_branch_unlikely(&bpf_pred_flush_enabled) && + pack->arch_flush_needed && + was_classic) { + struct bpf_prog_pack *p; + static_call_cond(bpf_arch_pred_flush)(); + list_for_each_entry(p, &pack_list, list) + p->arch_flush_needed = false; + } bitmap_set(pack->bitmap, pos, nbits); ptr = (void *)(pack->ptr) + (pos << BPF_PROG_CHUNK_SHIFT); @@ -1029,6 +1039,9 @@ void bpf_prog_pack_free(void *ptr, u32 size) "bpf_prog_pack bug: missing bpf_arch_text_invalidate?\n"); bitmap_clear(pack->bitmap, pos, nbits); + + if (static_branch_unlikely(&bpf_pred_flush_enabled)) + pack->arch_flush_needed = true; if (bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0, BPF_PROG_CHUNK_COUNT, 0) == 0) { list_del(&pack->list); -- cgit v1.2.3 From a9b1f19a6a673ba06820898d0f1ad02883ea1639 Mon Sep 17 00:00:00 2001 From: Pawan Gupta Date: Mon, 29 Jun 2026 22:38:54 -0700 Subject: bpf: Prefer packs that won't trigger an IBPB flush on allocation Currently BPF pack allocator picks the chunks from the first available pack. While this is okay, it naturally leads to more frequent flushes when there are multiple packs in the system that weren't used since the last flush. As an optimization prefer allocating the new programs from packs that are unused since last flush. When all packs are dirty, allocation forces a flush and marks all packs clean. Below are some future optimizations ideas: 1. Currently, the "dirty" tracking is only done at the pack-level. Flush frequency can further be reduced with chunk-level tracking. This requires a new bitmap per-pack to track the dirty state. 2. IBPB flush is done on all CPUs, even if only a single CPU ran the BPF program. On a system with hundreds of CPUs this could be a major bottleneck forcing hundreds of IPIs to deliver the flush. The solution is to track the CPUs where a BPF program ran, and issue IBPB only on those CPUs. 3. Avoid IBPB when flush is already done at other sources (e.g. context switch). Signed-off-by: Pawan Gupta Acked-by: Daniel Borkmann Signed-off-by: Daniel Borkmann --- kernel/bpf/core.c | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 50aba113ef9d..1b32b9f2491f 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -948,8 +948,8 @@ out: void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool was_classic) { unsigned int nbits = BPF_PROG_SIZE_TO_NBITS(size); - struct bpf_prog_pack *pack; - unsigned long pos; + struct bpf_prog_pack *pack, *fallback_pack = NULL; + unsigned long pos, fallback_pos = 0; void *ptr = NULL; mutex_lock(&pack_mutex); @@ -981,8 +981,29 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool list_for_each_entry(pack, &pack_list, list) { pos = bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0, nbits, 0); - if (pos < BPF_PROG_CHUNK_COUNT) + if (pos >= BPF_PROG_CHUNK_COUNT) + continue; + /* Flush not enabled, use any pack */ + if (!static_branch_unlikely(&bpf_pred_flush_enabled)) goto found_free_area; + /* + * cBPF reuse of a dirty pack triggers a flush, so prefer a + * clean pack for cBPF. eBPF never flushes, so pick the first + * free pack, dirty or clean. + */ + if (!was_classic || !pack->arch_flush_needed) + goto found_free_area; + if (!fallback_pack) { + fallback_pack = pack; + fallback_pos = pos; + } + } + + /* No preferred pack found */ + if (fallback_pack) { + pack = fallback_pack; + pos = fallback_pos; + goto found_free_area; } pack = alloc_new_pack(bpf_fill_ill_insns); -- cgit v1.2.3 From b72e29e0f7ee329d89f86db8700c8ea99b4a370a Mon Sep 17 00:00:00 2001 From: Pawan Gupta Date: Mon, 29 Jun 2026 22:39:29 -0700 Subject: bpf: Prefer dirty packs for eBPF allocations The pack allocator only flushes predictors when reusing a dirty pack for cBPF, eBPF allocations never trigger a flush. Currently, eBPF picks the first free pack, which could be a clean pack. As an optimization, leaving a clean pack for cBPF can avoid flushes. Prefer dirty packs for eBPF and keep clean packs free for cBPF. This mirrors the existing cBPF preference for clean packs: each program kind prefers the pack that avoids an extra flush, and falls back to the other kind only when no preferred pack has room. eBPF reuse of a dirty pack is harmless since eBPF being privileged does not flush. Signed-off-by: Pawan Gupta Acked-by: Daniel Borkmann Signed-off-by: Daniel Borkmann --- kernel/bpf/core.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 1b32b9f2491f..6e19a030da6f 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -988,10 +988,10 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool goto found_free_area; /* * cBPF reuse of a dirty pack triggers a flush, so prefer a - * clean pack for cBPF. eBPF never flushes, so pick the first - * free pack, dirty or clean. + * clean pack for cBPF. eBPF never flushes, so steer it to a + * dirty pack and keep clean packs free for cBPF. */ - if (!was_classic || !pack->arch_flush_needed) + if (was_classic ^ pack->arch_flush_needed) goto found_free_area; if (!fallback_pack) { fallback_pack = pack; -- cgit v1.2.3