summaryrefslogtreecommitdiff
path: root/kernel/bpf
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/bpf')
-rw-r--r--kernel/bpf/arena.c61
-rw-r--r--kernel/bpf/arraymap.c8
-rw-r--r--kernel/bpf/bpf_lru_list.c165
-rw-r--r--kernel/bpf/bpf_lru_list.h25
-rw-r--r--kernel/bpf/bpf_lsm.c23
-rw-r--r--kernel/bpf/cgroup.c168
-rw-r--r--kernel/bpf/core.c2
-rw-r--r--kernel/bpf/devmap.c12
-rw-r--r--kernel/bpf/disasm.c5
-rw-r--r--kernel/bpf/hashtab.c32
-rw-r--r--kernel/bpf/helpers.c17
-rw-r--r--kernel/bpf/inode.c13
-rw-r--r--kernel/bpf/map_in_map.c5
-rw-r--r--kernel/bpf/map_iter.c4
-rw-r--r--kernel/bpf/states.c13
-rw-r--r--kernel/bpf/syscall.c35
-rw-r--r--kernel/bpf/verifier.c56
17 files changed, 445 insertions, 199 deletions
diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c
index 49a8f7b1beef..40c42e28a616 100644
--- a/kernel/bpf/arena.c
+++ b/kernel/bpf/arena.c
@@ -59,6 +59,8 @@ struct bpf_arena {
struct list_head vma_list;
/* protects vma_list */
struct mutex lock;
+ u64 zap_gen;
+ struct mutex zap_mutex;
struct irq_work free_irq;
struct work_struct free_work;
struct llist_head free_spans;
@@ -228,6 +230,7 @@ static struct bpf_map *arena_map_alloc(union bpf_attr *attr)
goto err;
}
mutex_init(&arena->lock);
+ mutex_init(&arena->zap_mutex);
raw_res_spin_lock_init(&arena->spinlock);
err = populate_pgtable_except_pte(arena);
if (err) {
@@ -318,6 +321,7 @@ struct vma_list {
struct vm_area_struct *vma;
struct list_head head;
refcount_t mmap_count;
+ u64 zap_gen;
};
static int remember_vma(struct bpf_arena *arena, struct vm_area_struct *vma)
@@ -330,6 +334,7 @@ static int remember_vma(struct bpf_arena *arena, struct vm_area_struct *vma)
refcount_set(&vml->mmap_count, 1);
vma->vm_private_data = vml;
vml->vma = vma;
+ vml->zap_gen = 0;
list_add(&vml->head, &arena->vma_list);
return 0;
}
@@ -668,12 +673,60 @@ out_free_pages:
*/
static void zap_pages(struct bpf_arena *arena, long uaddr, long page_cnt)
{
+ unsigned long size = (unsigned long)page_cnt << PAGE_SHIFT;
+ struct vm_area_struct *vma;
+ struct mm_struct *mm;
struct vma_list *vml;
+ unsigned long vm_start;
+ u64 my_gen;
- guard(mutex)(&arena->lock);
- /* iterate link list under lock */
- list_for_each_entry(vml, &arena->vma_list, head)
- zap_vma_range(vml->vma, uaddr, PAGE_SIZE * page_cnt);
+ /*
+ * Taking mmap_read_lock() under arena->lock would deadlock against
+ * arena_vm_close(), which runs with mmap_write_lock held and then
+ * acquires arena->lock. Drop arena->lock for mmap_read_lock().
+ *
+ * Use per-call my_gen, recorded in vml->zap_gen, to remember which
+ * vmls this invocation has already processed across the lock drop.
+ * Hold zap_mutex around the whole walk so concurrent zap_pages()
+ * callers cannot overwrite each other's marks on shared vmls --
+ * otherwise call B's mark would make call A skip a vml that A has
+ * not yet zapped for A's uaddr range.
+ */
+ mutex_lock(&arena->zap_mutex);
+ mutex_lock(&arena->lock);
+ my_gen = ++arena->zap_gen;
+ for (;;) {
+ mm = NULL;
+ list_for_each_entry(vml, &arena->vma_list, head) {
+ if (vml->zap_gen >= my_gen)
+ continue;
+ vml->zap_gen = my_gen;
+ if (!mmget_not_zero(vml->vma->vm_mm))
+ continue;
+ mm = vml->vma->vm_mm;
+ vm_start = vml->vma->vm_start;
+ break;
+ }
+ if (!mm)
+ break;
+ mutex_unlock(&arena->lock);
+
+ mmap_read_lock(mm);
+ /*
+ * Re-resolve: while we waited the VMA could have been unmapped
+ * and a different mapping installed at the same address.
+ */
+ vma = find_vma(mm, vm_start);
+ if (vma && vma->vm_start == vm_start &&
+ vma->vm_file && vma->vm_file->private_data == &arena->map)
+ zap_vma_range(vma, uaddr, size);
+ mmap_read_unlock(mm);
+ mmput(mm);
+
+ mutex_lock(&arena->lock);
+ }
+ mutex_unlock(&arena->lock);
+ mutex_unlock(&arena->zap_mutex);
}
static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, bool sleepable)
diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c
index dfb2110ab733..4b68d7a2b90e 100644
--- a/kernel/bpf/arraymap.c
+++ b/kernel/bpf/arraymap.c
@@ -386,7 +386,7 @@ static long array_map_update_elem(struct bpf_map *map, void *key, void *value,
if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
val = this_cpu_ptr(array->pptrs[index & array->index_mask]);
copy_map_value(map, val, value);
- bpf_obj_free_fields(array->map.record, val);
+ bpf_obj_cancel_fields(map, val);
} else {
val = array->value +
(u64)array->elem_size * (index & array->index_mask);
@@ -394,7 +394,7 @@ static long array_map_update_elem(struct bpf_map *map, void *key, void *value,
copy_map_value_locked(map, val, value, false);
else
copy_map_value(map, val, value);
- bpf_obj_free_fields(array->map.record, val);
+ bpf_obj_cancel_fields(map, val);
}
return 0;
}
@@ -434,14 +434,14 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
cpu = map_flags >> 32;
ptr = per_cpu_ptr(pptr, cpu);
copy_map_value(map, ptr, value);
- bpf_obj_free_fields(array->map.record, ptr);
+ bpf_obj_cancel_fields(map, ptr);
goto unlock;
}
for_each_possible_cpu(cpu) {
ptr = per_cpu_ptr(pptr, cpu);
val = (map_flags & BPF_F_ALL_CPUS) ? value : value + size * cpu;
copy_map_value(map, ptr, val);
- bpf_obj_free_fields(array->map.record, ptr);
+ bpf_obj_cancel_fields(map, ptr);
}
unlock:
rcu_read_unlock();
diff --git a/kernel/bpf/bpf_lru_list.c b/kernel/bpf/bpf_lru_list.c
index e7a2fc60523f..5ed7cb4b98c0 100644
--- a/kernel/bpf/bpf_lru_list.c
+++ b/kernel/bpf/bpf_lru_list.c
@@ -13,23 +13,8 @@
#define PERCPU_FREE_TARGET (4)
#define PERCPU_NR_SCANS PERCPU_FREE_TARGET
-/* Helpers to get the local list index */
-#define LOCAL_LIST_IDX(t) ((t) - BPF_LOCAL_LIST_T_OFFSET)
-#define LOCAL_FREE_LIST_IDX LOCAL_LIST_IDX(BPF_LRU_LOCAL_LIST_T_FREE)
-#define LOCAL_PENDING_LIST_IDX LOCAL_LIST_IDX(BPF_LRU_LOCAL_LIST_T_PENDING)
#define IS_LOCAL_LIST_TYPE(t) ((t) >= BPF_LOCAL_LIST_T_OFFSET)
-/* Local list helpers */
-static struct list_head *local_free_list(struct bpf_lru_locallist *loc_l)
-{
- return &loc_l->lists[LOCAL_FREE_LIST_IDX];
-}
-
-static struct list_head *local_pending_list(struct bpf_lru_locallist *loc_l)
-{
- return &loc_l->lists[LOCAL_PENDING_LIST_IDX];
-}
-
/* bpf_lru_node helpers */
static bool bpf_lru_node_is_ref(const struct bpf_lru_node *node)
{
@@ -72,6 +57,7 @@ static void __bpf_lru_node_move_to_free(struct bpf_lru_list *l,
bpf_lru_list_count_dec(l, node->type);
node->type = tgt_free_type;
+ WRITE_ONCE(node->pending_free, 0);
list_move(&node->list, free_list);
}
@@ -87,6 +73,9 @@ static void __bpf_lru_node_move_in(struct bpf_lru_list *l,
bpf_lru_list_count_inc(l, tgt_type);
node->type = tgt_type;
bpf_lru_node_clear_ref(node);
+ /* Reset pending_free only when moving to the free list */
+ if (tgt_type == BPF_LRU_LIST_T_FREE)
+ WRITE_ONCE(node->pending_free, 0);
list_move(&node->list, &l->lists[tgt_type]);
}
@@ -212,9 +201,11 @@ __bpf_lru_list_shrink_inactive(struct bpf_lru *lru,
unsigned int i = 0;
list_for_each_entry_safe_reverse(node, tmp_node, inactive, list) {
- if (bpf_lru_node_is_ref(node)) {
+ if (bpf_lru_node_is_ref(node) &&
+ !READ_ONCE(node->pending_free)) {
__bpf_lru_node_move(l, node, BPF_LRU_LIST_T_ACTIVE);
- } else if (lru->del_from_htab(lru->del_arg, node)) {
+ } else if (READ_ONCE(node->pending_free) ||
+ lru->del_from_htab(lru->del_arg, node)) {
__bpf_lru_node_move_to_free(l, node, free_list,
tgt_free_type);
if (++nshrinked == tgt_nshrink)
@@ -273,7 +264,8 @@ static unsigned int __bpf_lru_list_shrink(struct bpf_lru *lru,
list_for_each_entry_safe_reverse(node, tmp_node, force_shrink_list,
list) {
- if (lru->del_from_htab(lru->del_arg, node)) {
+ if (READ_ONCE(node->pending_free) ||
+ lru->del_from_htab(lru->del_arg, node)) {
__bpf_lru_node_move_to_free(l, node, free_list,
tgt_free_type);
return 1;
@@ -290,8 +282,10 @@ static void __local_list_flush(struct bpf_lru_list *l,
struct bpf_lru_node *node, *tmp_node;
list_for_each_entry_safe_reverse(node, tmp_node,
- local_pending_list(loc_l), list) {
- if (bpf_lru_node_is_ref(node))
+ &loc_l->pending_list, list) {
+ if (READ_ONCE(node->pending_free))
+ __bpf_lru_node_move_in(l, node, BPF_LRU_LIST_T_FREE);
+ else if (bpf_lru_node_is_ref(node))
__bpf_lru_node_move_in(l, node, BPF_LRU_LIST_T_ACTIVE);
else
__bpf_lru_node_move_in(l, node,
@@ -307,9 +301,12 @@ static void bpf_lru_list_push_free(struct bpf_lru_list *l,
if (WARN_ON_ONCE(IS_LOCAL_LIST_TYPE(node->type)))
return;
- raw_spin_lock_irqsave(&l->lock, flags);
+ if (raw_res_spin_lock_irqsave(&l->lock, flags)) {
+ WRITE_ONCE(node->pending_free, 1);
+ return;
+ }
__bpf_lru_node_move(l, node, BPF_LRU_LIST_T_FREE);
- raw_spin_unlock_irqrestore(&l->lock, flags);
+ raw_res_spin_unlock_irqrestore(&l->lock, flags);
}
static void bpf_lru_list_pop_free_to_local(struct bpf_lru *lru,
@@ -318,8 +315,10 @@ static void bpf_lru_list_pop_free_to_local(struct bpf_lru *lru,
struct bpf_lru_list *l = &lru->common_lru.lru_list;
struct bpf_lru_node *node, *tmp_node;
unsigned int nfree = 0;
+ LIST_HEAD(tmp_free);
- raw_spin_lock(&l->lock);
+ if (raw_res_spin_lock(&l->lock))
+ return;
__local_list_flush(l, loc_l);
@@ -327,7 +326,7 @@ static void bpf_lru_list_pop_free_to_local(struct bpf_lru *lru,
list_for_each_entry_safe(node, tmp_node, &l->lists[BPF_LRU_LIST_T_FREE],
list) {
- __bpf_lru_node_move_to_free(l, node, local_free_list(loc_l),
+ __bpf_lru_node_move_to_free(l, node, &tmp_free,
BPF_LRU_LOCAL_LIST_T_FREE);
if (++nfree == lru->target_free)
break;
@@ -335,10 +334,19 @@ static void bpf_lru_list_pop_free_to_local(struct bpf_lru *lru,
if (nfree < lru->target_free)
__bpf_lru_list_shrink(lru, l, lru->target_free - nfree,
- local_free_list(loc_l),
+ &tmp_free,
BPF_LRU_LOCAL_LIST_T_FREE);
- raw_spin_unlock(&l->lock);
+ raw_res_spin_unlock(&l->lock);
+
+ /*
+ * Transfer the harvested nodes from the temporary list_head into
+ * the lockless per-CPU free llist.
+ */
+ list_for_each_entry_safe(node, tmp_node, &tmp_free, list) {
+ list_del(&node->list);
+ llist_add(&node->llist, &loc_l->free_llist);
+ }
}
static void __local_list_add_pending(struct bpf_lru *lru,
@@ -350,22 +358,21 @@ static void __local_list_add_pending(struct bpf_lru *lru,
*(u32 *)((void *)node + lru->hash_offset) = hash;
node->cpu = cpu;
node->type = BPF_LRU_LOCAL_LIST_T_PENDING;
+ WRITE_ONCE(node->pending_free, 0);
bpf_lru_node_clear_ref(node);
- list_add(&node->list, local_pending_list(loc_l));
+ list_add(&node->list, &loc_l->pending_list);
}
static struct bpf_lru_node *
__local_list_pop_free(struct bpf_lru_locallist *loc_l)
{
- struct bpf_lru_node *node;
+ struct llist_node *llnode;
- node = list_first_entry_or_null(local_free_list(loc_l),
- struct bpf_lru_node,
- list);
- if (node)
- list_del(&node->list);
+ llnode = llist_del_first(&loc_l->free_llist);
+ if (!llnode)
+ return NULL;
- return node;
+ return container_of(llnode, struct bpf_lru_node, llist);
}
static struct bpf_lru_node *
@@ -376,10 +383,10 @@ __local_list_pop_pending(struct bpf_lru *lru, struct bpf_lru_locallist *loc_l)
ignore_ref:
/* Get from the tail (i.e. older element) of the pending list. */
- list_for_each_entry_reverse(node, local_pending_list(loc_l),
- list) {
+ list_for_each_entry_reverse(node, &loc_l->pending_list, list) {
if ((!bpf_lru_node_is_ref(node) || force) &&
- lru->del_from_htab(lru->del_arg, node)) {
+ (READ_ONCE(node->pending_free) ||
+ lru->del_from_htab(lru->del_arg, node))) {
list_del(&node->list);
return node;
}
@@ -404,7 +411,8 @@ static struct bpf_lru_node *bpf_percpu_lru_pop_free(struct bpf_lru *lru,
l = per_cpu_ptr(lru->percpu_lru, cpu);
- raw_spin_lock_irqsave(&l->lock, flags);
+ if (raw_res_spin_lock_irqsave(&l->lock, flags))
+ return NULL;
__bpf_lru_list_rotate(lru, l);
@@ -420,7 +428,7 @@ static struct bpf_lru_node *bpf_percpu_lru_pop_free(struct bpf_lru *lru,
__bpf_lru_node_move(l, node, BPF_LRU_LIST_T_INACTIVE);
}
- raw_spin_unlock_irqrestore(&l->lock, flags);
+ raw_res_spin_unlock_irqrestore(&l->lock, flags);
return node;
}
@@ -437,7 +445,8 @@ static struct bpf_lru_node *bpf_common_lru_pop_free(struct bpf_lru *lru,
loc_l = per_cpu_ptr(clru->local_list, cpu);
- raw_spin_lock_irqsave(&loc_l->lock, flags);
+ if (raw_res_spin_lock_irqsave(&loc_l->lock, flags))
+ return NULL;
node = __local_list_pop_free(loc_l);
if (!node) {
@@ -448,17 +457,22 @@ static struct bpf_lru_node *bpf_common_lru_pop_free(struct bpf_lru *lru,
if (node)
__local_list_add_pending(lru, loc_l, cpu, node, hash);
- raw_spin_unlock_irqrestore(&loc_l->lock, flags);
+ raw_res_spin_unlock_irqrestore(&loc_l->lock, flags);
if (node)
return node;
- /* No free nodes found from the local free list and
+ /*
+ * No free nodes found from the local free list and
* the global LRU list.
*
* Steal from the local free/pending list of the
* current CPU and remote CPU in RR. It starts
* with the loc_l->next_steal CPU.
+ *
+ * Acquire the victim's lock before touching either list. On
+ * acquisition failure (rqspinlock AA or timeout) skip the victim
+ * and try the next CPU.
*/
first_steal = loc_l->next_steal;
@@ -466,24 +480,36 @@ static struct bpf_lru_node *bpf_common_lru_pop_free(struct bpf_lru *lru,
do {
steal_loc_l = per_cpu_ptr(clru->local_list, steal);
- raw_spin_lock_irqsave(&steal_loc_l->lock, flags);
-
- node = __local_list_pop_free(steal_loc_l);
- if (!node)
- node = __local_list_pop_pending(lru, steal_loc_l);
-
- raw_spin_unlock_irqrestore(&steal_loc_l->lock, flags);
+ if (!raw_res_spin_lock_irqsave(&steal_loc_l->lock, flags)) {
+ node = __local_list_pop_free(steal_loc_l);
+ if (!node)
+ node = __local_list_pop_pending(lru, steal_loc_l);
+ raw_res_spin_unlock_irqrestore(&steal_loc_l->lock, flags);
+ }
steal = cpumask_next_wrap(steal, cpu_possible_mask);
} while (!node && steal != first_steal);
loc_l->next_steal = steal;
- if (node) {
- raw_spin_lock_irqsave(&loc_l->lock, flags);
- __local_list_add_pending(lru, loc_l, cpu, node, hash);
- raw_spin_unlock_irqrestore(&loc_l->lock, flags);
+ if (!node)
+ return NULL;
+
+ if (raw_res_spin_lock_irqsave(&loc_l->lock, flags)) {
+ /*
+ * The local pending lock can't be acquired (rqspinlock AA
+ * or timeout). Return the stolen node to the per-CPU
+ * free_llist instead of orphaning it; the next pop_free on
+ * this CPU will pick it up.
+ */
+ node->type = BPF_LRU_LOCAL_LIST_T_FREE;
+ bpf_lru_node_clear_ref(node);
+ WRITE_ONCE(node->pending_free, 0);
+ llist_add(&node->llist, &loc_l->free_llist);
+ return NULL;
}
+ __local_list_add_pending(lru, loc_l, cpu, node, hash);
+ raw_res_spin_unlock_irqrestore(&loc_l->lock, flags);
return node;
}
@@ -511,18 +537,24 @@ static void bpf_common_lru_push_free(struct bpf_lru *lru,
loc_l = per_cpu_ptr(lru->common_lru.local_list, node->cpu);
- raw_spin_lock_irqsave(&loc_l->lock, flags);
+ if (raw_res_spin_lock_irqsave(&loc_l->lock, flags)) {
+ WRITE_ONCE(node->pending_free, 1);
+ return;
+ }
if (unlikely(node->type != BPF_LRU_LOCAL_LIST_T_PENDING)) {
- raw_spin_unlock_irqrestore(&loc_l->lock, flags);
+ raw_res_spin_unlock_irqrestore(&loc_l->lock,
+ flags);
goto check_lru_list;
}
node->type = BPF_LRU_LOCAL_LIST_T_FREE;
bpf_lru_node_clear_ref(node);
- list_move(&node->list, local_free_list(loc_l));
+ list_del(&node->list);
+
+ raw_res_spin_unlock_irqrestore(&loc_l->lock, flags);
- raw_spin_unlock_irqrestore(&loc_l->lock, flags);
+ llist_add(&node->llist, &loc_l->free_llist);
return;
}
@@ -538,11 +570,14 @@ static void bpf_percpu_lru_push_free(struct bpf_lru *lru,
l = per_cpu_ptr(lru->percpu_lru, node->cpu);
- raw_spin_lock_irqsave(&l->lock, flags);
+ if (raw_res_spin_lock_irqsave(&l->lock, flags)) {
+ WRITE_ONCE(node->pending_free, 1);
+ return;
+ }
__bpf_lru_node_move(l, node, BPF_LRU_LIST_T_FREE);
- raw_spin_unlock_irqrestore(&l->lock, flags);
+ raw_res_spin_unlock_irqrestore(&l->lock, flags);
}
void bpf_lru_push_free(struct bpf_lru *lru, struct bpf_lru_node *node)
@@ -565,6 +600,7 @@ static void bpf_common_lru_populate(struct bpf_lru *lru, void *buf,
node = (struct bpf_lru_node *)(buf + node_offset);
node->type = BPF_LRU_LIST_T_FREE;
+ node->pending_free = 0;
bpf_lru_node_clear_ref(node);
list_add(&node->list, &l->lists[BPF_LRU_LIST_T_FREE]);
buf += elem_size;
@@ -594,6 +630,7 @@ again:
node = (struct bpf_lru_node *)(buf + node_offset);
node->cpu = cpu;
node->type = BPF_LRU_LIST_T_FREE;
+ node->pending_free = 0;
bpf_lru_node_clear_ref(node);
list_add(&node->list, &l->lists[BPF_LRU_LIST_T_FREE]);
i++;
@@ -618,14 +655,12 @@ void bpf_lru_populate(struct bpf_lru *lru, void *buf, u32 node_offset,
static void bpf_lru_locallist_init(struct bpf_lru_locallist *loc_l, int cpu)
{
- int i;
-
- for (i = 0; i < NR_BPF_LRU_LOCAL_LIST_T; i++)
- INIT_LIST_HEAD(&loc_l->lists[i]);
+ INIT_LIST_HEAD(&loc_l->pending_list);
+ init_llist_head(&loc_l->free_llist);
loc_l->next_steal = cpu;
- raw_spin_lock_init(&loc_l->lock);
+ raw_res_spin_lock_init(&loc_l->lock);
}
static void bpf_lru_list_init(struct bpf_lru_list *l)
@@ -640,7 +675,7 @@ static void bpf_lru_list_init(struct bpf_lru_list *l)
l->next_inactive_rotation = &l->lists[BPF_LRU_LIST_T_INACTIVE];
- raw_spin_lock_init(&l->lock);
+ raw_res_spin_lock_init(&l->lock);
}
int bpf_lru_init(struct bpf_lru *lru, bool percpu, u32 hash_offset,
diff --git a/kernel/bpf/bpf_lru_list.h b/kernel/bpf/bpf_lru_list.h
index fe2661a58ea9..8d0ee61622af 100644
--- a/kernel/bpf/bpf_lru_list.h
+++ b/kernel/bpf/bpf_lru_list.h
@@ -6,11 +6,11 @@
#include <linux/cache.h>
#include <linux/list.h>
-#include <linux/spinlock_types.h>
+#include <linux/llist.h>
+#include <asm/rqspinlock.h>
#define NR_BPF_LRU_LIST_T (3)
#define NR_BPF_LRU_LIST_COUNT (2)
-#define NR_BPF_LRU_LOCAL_LIST_T (2)
#define BPF_LOCAL_LIST_T_OFFSET NR_BPF_LRU_LIST_T
enum bpf_lru_list_type {
@@ -22,10 +22,22 @@ enum bpf_lru_list_type {
};
struct bpf_lru_node {
- struct list_head list;
+ /*
+ * A node is in at most one list at a time. The free path on the
+ * per-CPU locallist uses an llist, so share storage via a union.
+ */
+ union {
+ struct list_head list;
+ struct llist_node llist;
+ };
u16 cpu;
u8 type;
u8 ref;
+ /*
+ * Marks nodes whose *_push_free() lock acquire failed; reclaimed
+ * by flush/shrink which honor the flag instead of del_from_htab().
+ */
+ u8 pending_free;
};
struct bpf_lru_list {
@@ -34,13 +46,14 @@ struct bpf_lru_list {
/* The next inactive list rotation starts from here */
struct list_head *next_inactive_rotation;
- raw_spinlock_t lock ____cacheline_aligned_in_smp;
+ rqspinlock_t lock ____cacheline_aligned_in_smp;
};
struct bpf_lru_locallist {
- struct list_head lists[NR_BPF_LRU_LOCAL_LIST_T];
+ struct list_head pending_list;
+ struct llist_head free_llist;
u16 next_steal;
- raw_spinlock_t lock;
+ rqspinlock_t lock;
};
struct bpf_common_lru {
diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c
index c5c925f00202..1433809bb166 100644
--- a/kernel/bpf/bpf_lsm.c
+++ b/kernel/bpf/bpf_lsm.c
@@ -51,6 +51,9 @@ BTF_ID(func, bpf_lsm_key_getsecurity)
#ifdef CONFIG_AUDIT
BTF_ID(func, bpf_lsm_audit_rule_match)
#endif
+#ifdef CONFIG_SECURITY_NETWORK_XFRM
+BTF_ID(func, bpf_lsm_xfrm_decode_session)
+#endif
BTF_ID(func, bpf_lsm_ismaclabel)
BTF_ID(func, bpf_lsm_file_alloc_security)
BTF_SET_END(bpf_lsm_disabled_hooks)
@@ -427,6 +430,26 @@ BTF_ID(func, bpf_lsm_audit_rule_known)
BTF_ID(func, bpf_lsm_inode_xattr_skipcap)
BTF_SET_END(bool_lsm_hooks)
+/* hooks returning void */
+#define LSM_HOOK_void(DEFAULT, NAME, ...) BTF_ID(func, bpf_lsm_##NAME)
+#define LSM_HOOK_int(DEFAULT, NAME, ...) /* nothing */
+#define LSM_HOOK(RET, DEFAULT, NAME, ...) LSM_HOOK_##RET(DEFAULT, NAME, __VA_ARGS__)
+BTF_SET_START(void_lsm_hooks)
+#include <linux/lsm_hook_defs.h>
+#undef LSM_HOOK
+#undef LSM_HOOK_void
+#undef LSM_HOOK_int
+BTF_SET_END(void_lsm_hooks)
+
+bool bpf_lsm_hook_returns_errno(u32 btf_id)
+{
+ if (btf_id_set_contains(&bool_lsm_hooks, btf_id))
+ return false;
+ if (btf_id_set_contains(&void_lsm_hooks, btf_id))
+ return false;
+ return true;
+}
+
int bpf_lsm_get_retval_range(const struct bpf_prog *prog,
struct bpf_retval_range *retval_range)
{
diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
index f4eefdacd453..2f87c273e642 100644
--- a/kernel/bpf/cgroup.c
+++ b/kernel/bpf/cgroup.c
@@ -55,6 +55,28 @@ void __init cgroup_bpf_lifetime_notifier_init(void)
&cgroup_bpf_lifetime_nb));
}
+#ifdef CONFIG_BPF_LSM
+struct cgroup_lsm_atype {
+ u32 attach_btf_id;
+ int refcnt;
+ bool returns_errno;
+};
+
+static struct cgroup_lsm_atype cgroup_lsm_atype[CGROUP_LSM_NUM];
+
+static bool cgroup_bpf_hook_returns_errno(enum cgroup_bpf_attach_type atype)
+{
+ if (atype >= CGROUP_LSM_START && atype <= CGROUP_LSM_END)
+ return READ_ONCE(cgroup_lsm_atype[atype - CGROUP_LSM_START].returns_errno);
+ return true;
+}
+#else
+static bool cgroup_bpf_hook_returns_errno(enum cgroup_bpf_attach_type atype)
+{
+ return true;
+}
+#endif
+
/* __always_inline is necessary to prevent indirect call through run_prog
* function pointer.
*/
@@ -83,7 +105,8 @@ bpf_prog_run_array_cg(const struct cgroup_bpf *cgrp,
*(ret_flags) |= (func_ret >> 1);
func_ret &= 1;
}
- if (!func_ret && !IS_ERR_VALUE((long)run_ctx.retval))
+ if (!func_ret && cgroup_bpf_hook_returns_errno(atype) &&
+ !IS_ERR_VALUE((long)run_ctx.retval))
run_ctx.retval = -EPERM;
item++;
}
@@ -156,13 +179,6 @@ unsigned int __cgroup_bpf_run_lsm_current(const void *ctx,
}
#ifdef CONFIG_BPF_LSM
-struct cgroup_lsm_atype {
- u32 attach_btf_id;
- int refcnt;
-};
-
-static struct cgroup_lsm_atype cgroup_lsm_atype[CGROUP_LSM_NUM];
-
static enum cgroup_bpf_attach_type
bpf_cgroup_atype_find(enum bpf_attach_type attach_type, u32 attach_btf_id)
{
@@ -191,10 +207,13 @@ void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype)
lockdep_assert_held(&cgroup_mutex);
- WARN_ON_ONCE(cgroup_lsm_atype[i].attach_btf_id &&
- cgroup_lsm_atype[i].attach_btf_id != attach_btf_id);
-
- cgroup_lsm_atype[i].attach_btf_id = attach_btf_id;
+ if (!cgroup_lsm_atype[i].attach_btf_id) {
+ cgroup_lsm_atype[i].attach_btf_id = attach_btf_id;
+ WRITE_ONCE(cgroup_lsm_atype[i].returns_errno,
+ bpf_lsm_hook_returns_errno(attach_btf_id));
+ } else {
+ WARN_ON_ONCE(cgroup_lsm_atype[i].attach_btf_id != attach_btf_id);
+ }
cgroup_lsm_atype[i].refcnt++;
}
@@ -203,8 +222,10 @@ void bpf_cgroup_atype_put(int cgroup_atype)
int i = cgroup_atype - CGROUP_LSM_START;
cgroup_lock();
- if (--cgroup_lsm_atype[i].refcnt <= 0)
+ if (--cgroup_lsm_atype[i].refcnt <= 0) {
+ WRITE_ONCE(cgroup_lsm_atype[i].returns_errno, true);
cgroup_lsm_atype[i].attach_btf_id = 0;
+ }
WARN_ON_ONCE(cgroup_lsm_atype[i].refcnt < 0);
cgroup_unlock();
}
@@ -918,19 +939,65 @@ static int cgroup_bpf_attach(struct cgroup *cgrp,
return ret;
}
+static int effective_prog_pos(struct cgroup *cgrp,
+ enum cgroup_bpf_attach_type atype,
+ struct bpf_prog_list *target_pl)
+{
+ int cnt = 0, preorder_cnt = 0, fstart, bstart, init_bstart, pos = -1;
+ struct bpf_prog_list *pl;
+ struct cgroup *p = cgrp;
+
+ /* count effective programs to find where the preorder region ends */
+ do {
+ if (cnt == 0 || (p->bpf.flags[atype] & BPF_F_ALLOW_MULTI))
+ cnt += prog_list_length(&p->bpf.progs[atype], &preorder_cnt);
+ p = cgroup_parent(p);
+ } while (p);
+
+ /* replay compute_effective_progs() placement and record target's slot */
+ cnt = 0;
+ p = cgrp;
+ fstart = preorder_cnt;
+ bstart = preorder_cnt - 1;
+ do {
+ if (cnt > 0 && !(p->bpf.flags[atype] & BPF_F_ALLOW_MULTI))
+ continue;
+
+ init_bstart = bstart;
+ hlist_for_each_entry(pl, &p->bpf.progs[atype], node) {
+ if (!prog_list_prog(pl))
+ continue;
+
+ if (pl->flags & BPF_F_PREORDER) {
+ if (pl == target_pl)
+ pos = bstart;
+ bstart--;
+ } else {
+ if (pl == target_pl)
+ pos = fstart;
+ fstart++;
+ }
+ cnt++;
+ }
+
+ /* reverse pre-ordering progs at this cgroup level */
+ if (pos >= bstart + 1 && pos <= init_bstart)
+ pos = bstart + 1 + init_bstart - pos;
+ } while ((p = cgroup_parent(p)));
+
+ return pos;
+}
+
/* Swap updated BPF program for given link in effective program arrays across
* all descendant cgroups. This function is guaranteed to succeed.
*/
static void replace_effective_prog(struct cgroup *cgrp,
enum cgroup_bpf_attach_type atype,
- struct bpf_cgroup_link *link)
+ struct bpf_prog_list *pl)
{
struct bpf_prog_array_item *item;
struct cgroup_subsys_state *css;
struct bpf_prog_array *progs;
- struct bpf_prog_list *pl;
- struct hlist_head *head;
- struct cgroup *cg;
int pos;
css_for_each_descendant_pre(css, &cgrp->self) {
@@ -939,27 +1006,15 @@ static void replace_effective_prog(struct cgroup *cgrp,
if (percpu_ref_is_zero(&desc->bpf.refcnt))
continue;
- /* find position of link in effective progs array */
- for (pos = 0, cg = desc; cg; cg = cgroup_parent(cg)) {
- if (pos && !(cg->bpf.flags[atype] & BPF_F_ALLOW_MULTI))
- continue;
+ pos = effective_prog_pos(desc, atype, pl);
+ if (WARN_ON_ONCE(pos < 0))
+ continue;
- head = &cg->bpf.progs[atype];
- hlist_for_each_entry(pl, head, node) {
- if (!prog_list_prog(pl))
- continue;
- if (pl->link == link)
- goto found;
- pos++;
- }
- }
-found:
- BUG_ON(!cg);
progs = rcu_dereference_protected(
desc->bpf.effective[atype],
lockdep_is_held(&cgroup_mutex));
item = &progs->items[pos];
- WRITE_ONCE(item->prog, link->link.prog);
+ WRITE_ONCE(item->prog, pl->link->link.prog);
}
}
@@ -1003,7 +1058,7 @@ static int __cgroup_bpf_replace(struct cgroup *cgrp,
cgrp->bpf.revisions[atype] += 1;
old_prog = xchg(&link->link.prog, new_prog);
- replace_effective_prog(cgrp, atype, link);
+ replace_effective_prog(cgrp, atype, pl);
bpf_prog_put(old_prog);
return 0;
}
@@ -1070,19 +1125,14 @@ static struct bpf_prog_list *find_detach_entry(struct hlist_head *progs,
* recomputing the array in place.
*
* @cgrp: The cgroup which descendants to travers
- * @prog: A program to detach or NULL
- * @link: A link to detach or NULL
+ * @pl: The prog_list entry being detached
* @atype: Type of detach operation
*/
-static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog *prog,
- struct bpf_cgroup_link *link,
+static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog_list *pl,
enum cgroup_bpf_attach_type atype)
{
struct cgroup_subsys_state *css;
struct bpf_prog_array *progs;
- struct bpf_prog_list *pl;
- struct hlist_head *head;
- struct cgroup *cg;
int pos;
/* recompute effective prog array in place */
@@ -1092,24 +1142,11 @@ static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog *prog,
if (percpu_ref_is_zero(&desc->bpf.refcnt))
continue;
- /* find position of link or prog in effective progs array */
- for (pos = 0, cg = desc; cg; cg = cgroup_parent(cg)) {
- if (pos && !(cg->bpf.flags[atype] & BPF_F_ALLOW_MULTI))
- continue;
-
- head = &cg->bpf.progs[atype];
- hlist_for_each_entry(pl, head, node) {
- if (!prog_list_prog(pl))
- continue;
- if (pl->prog == prog && pl->link == link)
- goto found;
- pos++;
- }
- }
-
+ pos = effective_prog_pos(desc, atype, pl);
/* no link or prog match, skip the cgroup of this layer */
- continue;
-found:
+ if (pos < 0)
+ continue;
+
progs = rcu_dereference_protected(
desc->bpf.effective[atype],
lockdep_is_held(&cgroup_mutex));
@@ -1175,7 +1212,7 @@ static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
/* if update effective array failed replace the prog with a dummy prog*/
pl->prog = old_prog;
pl->link = link;
- purge_effective_progs(cgrp, old_prog, link, atype);
+ purge_effective_progs(cgrp, pl, atype);
}
/* now can actually delete it from this cgroup list */
@@ -1208,7 +1245,7 @@ static int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
/* Must be called with cgroup_mutex held to avoid races. */
static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
- union bpf_attr __user *uattr)
+ union bpf_attr __user *uattr, u32 uattr_size)
{
__u32 __user *prog_attach_flags = u64_to_user_ptr(attr->query.prog_attach_flags);
bool effective_query = attr->query.query_flags & BPF_F_QUERY_EFFECTIVE;
@@ -1259,7 +1296,8 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
return -EFAULT;
if (!effective_query && from_atype == to_atype)
revision = cgrp->bpf.revisions[from_atype];
- if (copy_to_user(&uattr->query.revision, &revision, sizeof(revision)))
+ if (uattr_size >= offsetofend(union bpf_attr, query.revision) &&
+ copy_to_user(&uattr->query.revision, &revision, sizeof(revision)))
return -EFAULT;
if (attr->query.prog_cnt == 0 || !prog_ids || !total_cnt)
/* return early if user requested only program count + flags */
@@ -1312,12 +1350,12 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
}
static int cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
- union bpf_attr __user *uattr)
+ union bpf_attr __user *uattr, u32 uattr_size)
{
int ret;
cgroup_lock();
- ret = __cgroup_bpf_query(cgrp, attr, uattr);
+ ret = __cgroup_bpf_query(cgrp, attr, uattr, uattr_size);
cgroup_unlock();
return ret;
}
@@ -1520,7 +1558,7 @@ out_put_cgroup:
}
int cgroup_bpf_prog_query(const union bpf_attr *attr,
- union bpf_attr __user *uattr)
+ union bpf_attr __user *uattr, u32 uattr_size)
{
struct cgroup *cgrp;
int ret;
@@ -1529,7 +1567,7 @@ int cgroup_bpf_prog_query(const union bpf_attr *attr,
if (IS_ERR(cgrp))
return PTR_ERR(cgrp);
- ret = cgroup_bpf_query(cgrp, attr, uattr);
+ ret = cgroup_bpf_query(cgrp, attr, uattr, uattr_size);
cgroup_put(cgrp);
return ret;
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index de61e1894452..f5d9c27e654d 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -2527,7 +2527,7 @@ static bool __bpf_prog_map_compatible(struct bpf_map *map,
cookie = aux->cgroup_storage[i] ?
aux->cgroup_storage[i]->cookie : 0;
ret = map->owner->storage_cookie[i] == cookie ||
- !cookie;
+ (!cookie && !aux->tail_call_reachable);
}
if (ret &&
map->owner->attach_func_proto != aux->attach_func_proto) {
diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c
index 5b9eac5342a9..dc7b859e8bbf 100644
--- a/kernel/bpf/devmap.c
+++ b/kernel/bpf/devmap.c
@@ -710,6 +710,18 @@ int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb,
if (unlikely(err))
return err;
+ if (dst->xdp_prog && skb_cloned(skb)) {
+ struct sk_buff *nskb;
+
+ nskb = skb_copy(skb, GFP_ATOMIC);
+ if (!nskb)
+ return -ENOMEM;
+
+ nskb->mac_len = skb->mac_len;
+ consume_skb(skb);
+ skb = nskb;
+ }
+
/* Redirect has already succeeded semantically at this point, so we just
* return 0 even if packet is dropped. Helper below takes care of
* freeing skb.
diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c
index f8a3c7eb451e..0391b3bc0073 100644
--- a/kernel/bpf/disasm.c
+++ b/kernel/bpf/disasm.c
@@ -323,7 +323,10 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs,
*/
u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm;
bool is_ptr = insn->src_reg == BPF_PSEUDO_MAP_FD ||
- insn->src_reg == BPF_PSEUDO_MAP_VALUE;
+ insn->src_reg == BPF_PSEUDO_MAP_VALUE ||
+ insn->src_reg == BPF_PSEUDO_MAP_IDX ||
+ insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE ||
+ insn->src_reg == BPF_PSEUDO_BTF_ID;
char tmp[64];
if (is_ptr && !allow_ptr_leaks)
diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c
index 3dd9b4924ae4..74c5ced74032 100644
--- a/kernel/bpf/hashtab.c
+++ b/kernel/bpf/hashtab.c
@@ -242,6 +242,10 @@ static void htab_free_prealloced_fields(struct bpf_htab *htab)
if (IS_ERR_OR_NULL(htab->map.record))
return;
+ /*
+ * Preallocated maps do not have a bpf_mem_alloc destructor, so fully
+ * destroy every element, including the extra elements.
+ */
if (htab_has_extra_elems(htab))
num_entries += num_possible_cpus();
for (i = 0; i < num_entries; i++) {
@@ -834,8 +838,8 @@ static int htab_lru_map_gen_lookup(struct bpf_map *map,
return insn - insn_buf;
}
-static void check_and_free_fields(struct bpf_htab *htab,
- struct htab_elem *elem)
+static void check_and_cancel_fields(struct bpf_htab *htab,
+ struct htab_elem *elem)
{
if (IS_ERR_OR_NULL(htab->map.record))
return;
@@ -845,11 +849,11 @@ static void check_and_free_fields(struct bpf_htab *htab,
int cpu;
for_each_possible_cpu(cpu)
- bpf_obj_free_fields(htab->map.record, per_cpu_ptr(pptr, cpu));
+ bpf_obj_cancel_fields(&htab->map, per_cpu_ptr(pptr, cpu));
} else {
void *map_value = htab_elem_value(elem, htab->map.key_size);
- bpf_obj_free_fields(htab->map.record, map_value);
+ bpf_obj_cancel_fields(&htab->map, map_value);
}
}
@@ -884,7 +888,7 @@ static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node)
htab_unlock_bucket(b, flags);
if (l == tgt_l)
- check_and_free_fields(htab, l);
+ check_and_cancel_fields(htab, l);
return l == tgt_l;
}
@@ -949,7 +953,7 @@ find_first_elem:
static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l)
{
- check_and_free_fields(htab, l);
+ check_and_cancel_fields(htab, l);
if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH)
bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr);
@@ -1002,7 +1006,7 @@ static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l)
if (htab_is_prealloc(htab)) {
bpf_map_dec_elem_count(&htab->map);
- check_and_free_fields(htab, l);
+ check_and_cancel_fields(htab, l);
pcpu_freelist_push(&htab->freelist, &l->fnode);
} else {
dec_elem_count(htab);
@@ -1019,7 +1023,7 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr,
/* copy true value_size bytes */
ptr = this_cpu_ptr(pptr);
copy_map_value(&htab->map, ptr, value);
- bpf_obj_free_fields(htab->map.record, ptr);
+ bpf_obj_cancel_fields(&htab->map, ptr);
} else {
u32 size = round_up(htab->map.value_size, 8);
void *val;
@@ -1029,7 +1033,7 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr,
cpu = map_flags >> 32;
ptr = per_cpu_ptr(pptr, cpu);
copy_map_value(&htab->map, ptr, value);
- bpf_obj_free_fields(htab->map.record, ptr);
+ bpf_obj_cancel_fields(&htab->map, ptr);
return;
}
@@ -1037,7 +1041,7 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr,
ptr = per_cpu_ptr(pptr, cpu);
val = (map_flags & BPF_F_ALL_CPUS) ? value : value + size * cpu;
copy_map_value(&htab->map, ptr, val);
- bpf_obj_free_fields(htab->map.record, ptr);
+ bpf_obj_cancel_fields(&htab->map, ptr);
}
}
}
@@ -1253,11 +1257,11 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value,
if (l_old) {
hlist_nulls_del_rcu(&l_old->hash_node);
- /* l_old has already been stashed in htab->extra_elems, free
- * its special fields before it is available for reuse.
+ /* l_old has already been stashed in htab->extra_elems, cancel
+ * its reusable special fields before it is available for reuse.
*/
if (htab_is_prealloc(htab))
- check_and_free_fields(htab, l_old);
+ check_and_cancel_fields(htab, l_old);
}
htab_unlock_bucket(b, flags);
if (l_old && !htab_is_prealloc(htab))
@@ -1270,7 +1274,7 @@ err:
static void htab_lru_push_free(struct bpf_htab *htab, struct htab_elem *elem)
{
- check_and_free_fields(htab, elem);
+ check_and_cancel_fields(htab, elem);
bpf_map_dec_elem_count(&htab->map);
bpf_lru_push_free(&htab->lru, &elem->lru_node);
}
diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index b5314c9fed3c..c2032af98e08 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -2295,6 +2295,7 @@ void bpf_rb_root_free(const struct btf_field *field, void *rb_root,
struct bpf_spin_lock *spin_lock)
{
struct rb_root_cached orig_root, *root = rb_root;
+ struct bpf_rb_node_kern *node;
struct rb_node *pos, *n;
void *obj;
@@ -2303,14 +2304,20 @@ void bpf_rb_root_free(const struct btf_field *field, void *rb_root,
__bpf_spin_lock_irqsave(spin_lock);
orig_root = *root;
+ bpf_rbtree_postorder_for_each_entry_safe(pos, n, &orig_root.rb_root) {
+ node = rb_entry(pos, struct bpf_rb_node_kern, rb_node);
+ WRITE_ONCE(node->owner, BPF_PTR_POISON);
+ }
*root = RB_ROOT_CACHED;
__bpf_spin_unlock_irqrestore(spin_lock);
bpf_rbtree_postorder_for_each_entry_safe(pos, n, &orig_root.rb_root) {
obj = pos;
obj -= field->graph_root.node_offset;
-
-
+ node = rb_entry(pos, struct bpf_rb_node_kern, rb_node);
+ RB_CLEAR_NODE(pos);
+ /* Ensure __bpf_rbtree_add() sees the node as unlinked. */
+ smp_store_release(&node->owner, NULL);
__bpf_obj_drop_impl(obj, field->graph_root.value_rec, false);
}
}
@@ -2912,11 +2919,13 @@ __bpf_kfunc struct task_struct *bpf_task_from_vpid(s32 vpid)
{
struct task_struct *p;
- rcu_read_lock();
+ guard(rcu)();
+ if (!task_active_pid_ns(current))
+ return NULL;
+
p = find_task_by_vpid(vpid);
if (p)
p = bpf_task_acquire(p);
- rcu_read_unlock();
return p;
}
diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c
index 25c06a011825..188c774a469c 100644
--- a/kernel/bpf/inode.c
+++ b/kernel/bpf/inode.c
@@ -766,10 +766,18 @@ static void bpf_destroy_inode(struct inode *inode)
{
enum bpf_type type;
- if (S_ISLNK(inode->i_mode))
- kfree(inode->i_link);
if (!bpf_inode_type(inode, &type))
bpf_any_put(inode->i_private, type);
+}
+
+/*
+ * Called after RCU grace period - safe to free inode and anything
+ * that might be accessed by RCU pathwalk (inode fields, i_link).
+ */
+static void bpf_free_inode(struct inode *inode)
+{
+ if (S_ISLNK(inode->i_mode))
+ kfree(inode->i_link);
free_inode_nonrcu(inode);
}
@@ -778,6 +786,7 @@ const struct super_operations bpf_super_ops = {
.drop_inode = inode_just_drop,
.show_options = bpf_show_options,
.destroy_inode = bpf_destroy_inode,
+ .free_inode = bpf_free_inode,
};
enum {
diff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c
index 645bd30bc9a9..d2cbab4bdf64 100644
--- a/kernel/bpf/map_in_map.c
+++ b/kernel/bpf/map_in_map.c
@@ -20,7 +20,8 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd)
/* Does not support >1 level map-in-map */
if (inner_map->inner_map_meta)
return ERR_PTR(-EINVAL);
-
+ if (inner_map->excl_prog_sha)
+ return ERR_PTR(-ENOTSUPP);
if (!inner_map->ops->map_meta_equal)
return ERR_PTR(-ENOTSUPP);
@@ -101,6 +102,8 @@ void *bpf_map_fd_get_ptr(struct bpf_map *map,
inner_map = __bpf_map_get(f);
if (IS_ERR(inner_map))
return inner_map;
+ if (inner_map->excl_prog_sha)
+ return ERR_PTR(-ENOTSUPP);
inner_map_meta = map->inner_map_meta;
if (inner_map_meta->ops->map_meta_equal(inner_map_meta, inner_map))
diff --git a/kernel/bpf/map_iter.c b/kernel/bpf/map_iter.c
index 261a03ea73d3..ae0741a09c6d 100644
--- a/kernel/bpf/map_iter.c
+++ b/kernel/bpf/map_iter.c
@@ -112,6 +112,10 @@ static int bpf_iter_attach_map(struct bpf_prog *prog,
map = bpf_map_get_with_uref(linfo->map.map_fd);
if (IS_ERR(map))
return PTR_ERR(map);
+ if (map->excl_prog_sha) {
+ err = -EPERM;
+ goto put_map;
+ }
if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c
index 8478d2c6ed5b..1dd99b3194d7 100644
--- a/kernel/bpf/states.c
+++ b/kernel/bpf/states.c
@@ -437,12 +437,10 @@ static void __clean_func_state(struct bpf_verifier_env *env,
continue;
/*
- * Only destroy spilled_ptr when hi half is dead.
- * If hi half is still live with STACK_SPILL, the
- * spilled_ptr metadata is needed for correct state
- * comparison in stacksafe().
- * is_spilled_reg() is using slot_type[7], but
- * is_spilled_scalar_after() check either slot_type[0] or [4]
+ * Only scalar spills can be degraded to raw stack bytes
+ * when their high half is dead. Pointer spills need the
+ * saved spilled_ptr metadata so partial fills keep
+ * rejecting as non-scalar register fills.
*/
if (!hi_live) {
struct bpf_reg_state *spill = &st->stack[i].spilled_ptr;
@@ -450,6 +448,9 @@ static void __clean_func_state(struct bpf_verifier_env *env,
if (lo_live && stype == STACK_SPILL) {
u8 val = STACK_MISC;
+ if (spill->type != SCALAR_VALUE)
+ continue;
+
/*
* 8 byte spill of scalar 0 where half slot is dead
* should become STACK_ZERO in lo 4 bytes.
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 630d530782fe..96d1b797e152 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -807,6 +807,11 @@ void bpf_obj_free_task_work(const struct btf_record *rec, void *obj)
bpf_task_work_cancel_and_free(obj + rec->task_work_off);
}
+void bpf_obj_cancel_fields(struct bpf_map *map, void *obj)
+{
+ bpf_map_free_internal_structs(map, obj);
+}
+
void bpf_obj_free_fields(const struct btf_record *rec, void *obj)
{
const struct btf_field *fields;
@@ -1572,6 +1577,13 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr)
err = -EFAULT;
goto free_map;
}
+
+ /* See libbpf: emit_signature_match() */
+ BUILD_BUG_ON(offsetof(struct bpf_map, excl) != SHA256_DIGEST_SIZE);
+ BUILD_BUG_ON(!__same_type(map->excl, u32));
+ BUILD_BUG_ON(offsetof(struct bpf_map, sha) != 0);
+ BUILD_BUG_ON(!__same_type(map->sha, u8[SHA256_DIGEST_SIZE]));
+ map->excl = 1;
} else if (attr->excl_prog_hash_size) {
err = -EINVAL;
goto free_map;
@@ -4281,6 +4293,11 @@ static int bpf_raw_tp_link_attach(struct bpf_prog *prog,
if (!btp)
return -ENOENT;
+ if (prog->sleepable && !tracepoint_is_faultable(btp->tp)) {
+ bpf_put_raw_tracepoint(btp);
+ return -EINVAL;
+ }
+
link = kzalloc_obj(*link, GFP_USER);
if (!link) {
err = -ENOMEM;
@@ -4654,7 +4671,7 @@ static int bpf_prog_detach(const union bpf_attr *attr)
#define BPF_PROG_QUERY_LAST_FIELD query.revision
static int bpf_prog_query(const union bpf_attr *attr,
- union bpf_attr __user *uattr)
+ union bpf_attr __user *uattr, u32 uattr_size)
{
if (!bpf_net_capable())
return -EPERM;
@@ -4693,7 +4710,7 @@ static int bpf_prog_query(const union bpf_attr *attr,
case BPF_CGROUP_GETSOCKOPT:
case BPF_CGROUP_SETSOCKOPT:
case BPF_LSM_CGROUP:
- return cgroup_bpf_prog_query(attr, uattr);
+ return cgroup_bpf_prog_query(attr, uattr, uattr_size);
case BPF_LIRC_MODE2:
return lirc_prog_query(attr, uattr);
case BPF_FLOW_DISSECTOR:
@@ -5045,10 +5062,11 @@ static int bpf_prog_get_info_by_fd(struct file *file,
u32 info_len = attr->info.info_len;
struct bpf_prog_kstats stats;
char __user *uinsns;
- u32 ulen;
+ u32 ulen, len;
int err;
- err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len);
+ len = offsetofend(struct bpf_prog_info, attach_btf_id);
+ err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), len, info_len);
if (err)
return err;
info_len = min_t(u32, sizeof(info), info_len);
@@ -5330,10 +5348,11 @@ static int bpf_map_get_info_by_fd(struct file *file,
{
struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info);
struct bpf_map_info info;
- u32 info_len = attr->info.info_len;
+ u32 info_len = attr->info.info_len, len;
int err;
- err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len);
+ len = offsetofend(struct bpf_map_info, hash_size);
+ err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), len, info_len);
if (err)
return err;
info_len = min_t(u32, sizeof(info), info_len);
@@ -6189,7 +6208,7 @@ static int prog_stream_read(union bpf_attr *attr)
return ret;
}
-#define BPF_PROG_ASSOC_STRUCT_OPS_LAST_FIELD prog_assoc_struct_ops.prog_fd
+#define BPF_PROG_ASSOC_STRUCT_OPS_LAST_FIELD prog_assoc_struct_ops.flags
static int prog_assoc_struct_ops(union bpf_attr *attr)
{
@@ -6286,7 +6305,7 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size)
err = bpf_prog_detach(&attr);
break;
case BPF_PROG_QUERY:
- err = bpf_prog_query(&attr, uattr.user);
+ err = bpf_prog_query(&attr, uattr.user, size);
break;
case BPF_PROG_TEST_RUN:
err = bpf_prog_test_run(&attr, uattr.user);
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index ff9b1f68ceca..523f00e609e2 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -3836,7 +3836,8 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
bool sanitize = reg && is_spillable_regtype(reg->type);
for (i = 0; i < size; i++) {
- u8 type = state->stack[spi].slot_type[i];
+ u8 type = state->stack[spi].slot_type[(slot - i) %
+ BPF_REG_SIZE];
if (type != STACK_MISC && type != STACK_ZERO) {
sanitize = true;
@@ -5558,14 +5559,11 @@ static int check_max_stack_depth(struct bpf_verifier_env *env)
static int __check_buffer_access(struct bpf_verifier_env *env,
const char *buf_info,
const struct bpf_reg_state *reg,
- int regno, int off, int size)
+ int regno, int off, int size,
+ u32 *access_end)
{
- if (off < 0) {
- verbose(env,
- "R%d invalid %s buffer access: off=%d, size=%d\n",
- regno, buf_info, off, size);
- return -EACCES;
- }
+ s64 start;
+
if (!tnum_is_const(reg->var_off)) {
char tn_buf[48];
@@ -5576,6 +5574,15 @@ static int __check_buffer_access(struct bpf_verifier_env *env,
return -EACCES;
}
+ start = (s64)reg->var_off.value + off;
+ if (start < 0) {
+ verbose(env,
+ "R%d invalid negative %s buffer offset: off=%d, var_off=%lld\n",
+ regno, buf_info, off, (s64)reg->var_off.value);
+ return -EACCES;
+ }
+
+ *access_end = start + size;
return 0;
}
@@ -5583,14 +5590,14 @@ static int check_tp_buffer_access(struct bpf_verifier_env *env,
const struct bpf_reg_state *reg,
int regno, int off, int size)
{
+ u32 access_end;
int err;
- err = __check_buffer_access(env, "tracepoint", reg, regno, off, size);
+ err = __check_buffer_access(env, "tracepoint", reg, regno, off, size, &access_end);
if (err)
return err;
- env->prog->aux->max_tp_access = max(reg->var_off.value + off + size,
- env->prog->aux->max_tp_access);
+ env->prog->aux->max_tp_access = max(access_end, env->prog->aux->max_tp_access);
return 0;
}
@@ -5602,13 +5609,14 @@ static int check_buffer_access(struct bpf_verifier_env *env,
u32 *max_access)
{
const char *buf_info = type_is_rdonly_mem(reg->type) ? "rdonly" : "rdwr";
+ u32 access_end;
int err;
- err = __check_buffer_access(env, buf_info, reg, regno, off, size);
+ err = __check_buffer_access(env, buf_info, reg, regno, off, size, &access_end);
if (err)
return err;
- *max_access = max(reg->var_off.value + off + size, *max_access);
+ *max_access = max(access_end, *max_access);
return 0;
}
@@ -6453,6 +6461,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
*/
if (info.reg_type == SCALAR_VALUE) {
if (info.is_retval && get_func_retval_range(env->prog, &range)) {
+ mark_reg_unknown(env, regs, value_regno);
err = __mark_reg_s32_range(env, regs, value_regno,
range.minval, range.maxval);
if (err)
@@ -19317,6 +19326,12 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
btp = bpf_get_raw_tracepoint(tname);
if (!btp)
return -EINVAL;
+ if (prog->sleepable && !tracepoint_is_faultable(btp->tp)) {
+ bpf_log(log, "Sleepable program cannot attach to non-faultable tracepoint %s\n",
+ tname);
+ bpf_put_raw_tracepoint(btp);
+ return -EINVAL;
+ }
fname = kallsyms_lookup((unsigned long)btp->bpf_func, NULL, NULL, NULL,
trace_symbol);
bpf_put_raw_tracepoint(btp);
@@ -19533,14 +19548,19 @@ static bool can_be_sleepable(struct bpf_prog *prog)
case BPF_MODIFY_RETURN:
case BPF_TRACE_ITER:
case BPF_TRACE_FSESSION:
+ case BPF_TRACE_RAW_TP:
return true;
default:
return false;
}
}
- return prog->type == BPF_PROG_TYPE_LSM ||
- prog->type == BPF_PROG_TYPE_KPROBE /* only for uprobes */ ||
- prog->type == BPF_PROG_TYPE_STRUCT_OPS;
+ if (prog->type == BPF_PROG_TYPE_LSM)
+ return prog->expected_attach_type != BPF_LSM_CGROUP;
+
+ return prog->type == BPF_PROG_TYPE_KPROBE /* only for uprobes */ ||
+ prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
+ prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT ||
+ prog->type == BPF_PROG_TYPE_TRACEPOINT;
}
static int check_attach_btf_id(struct bpf_verifier_env *env)
@@ -19562,7 +19582,7 @@ static int check_attach_btf_id(struct bpf_verifier_env *env)
}
if (prog->sleepable && !can_be_sleepable(prog)) {
- verbose(env, "Only fentry/fexit/fsession/fmod_ret, lsm, iter, uprobe, and struct_ops programs can be sleepable\n");
+ verbose(env, "Program of this type cannot be sleepable\n");
return -EINVAL;
}
@@ -20209,13 +20229,13 @@ err_unlock:
if (!is_priv)
mutex_unlock(&bpf_verifier_lock);
bpf_clear_insn_aux_data(env, 0, env->prog->len);
- vfree(env->insn_aux_data);
err_free_env:
bpf_stack_liveness_free(env);
kvfree(env->cfg.insn_postorder);
kvfree(env->scc_info);
kvfree(env->succ);
kvfree(env->gotox_tmp_buf);
+ vfree(env->insn_aux_data);
kvfree(env);
return ret;
}
generated by cgit v1.2.3 (git 2.25.1) at 2026-08-03 05:40:42 +0000