diff options
Diffstat (limited to 'kernel/bpf/arraymap.c')
| -rw-r--r-- | kernel/bpf/arraymap.c | 34 |
1 files changed, 30 insertions, 4 deletions
diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index 34865701f7f7..ef315b168b29 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -608,17 +608,42 @@ static int array_map_check_btf(struct bpf_map *map, static int array_map_mmap(struct bpf_map *map, struct vm_area_struct *vma) { struct bpf_array *array = container_of(map, struct bpf_array, map); - pgoff_t pgoff = PAGE_ALIGN(sizeof(*array)) >> PAGE_SHIFT; if (!(map->map_flags & BPF_F_MMAPABLE)) return -EINVAL; - if (vma->vm_pgoff * PAGE_SIZE + (vma->vm_end - vma->vm_start) > + /* use u64 math so the offset cannot overflow on 32-bit archs */ + if ((u64)vma->vm_pgoff * PAGE_SIZE + (vma->vm_end - vma->vm_start) > PAGE_ALIGN((u64)array->map.max_entries * array->elem_size)) return -EINVAL; - return remap_vmalloc_range(vma, array_map_vmalloc_addr(array), - vma->vm_pgoff + pgoff); + /* + * Pages are faulted in on demand by array_map_mmap_fault(). Set the + * same flags that the eager remap_vmalloc_range() path used to set + * via vm_insert_page(), so that e.g. NUMA balancing keeps skipping + * these VMAs. + */ + vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP | VM_MIXEDMAP); + + return 0; +} + +static vm_fault_t array_map_mmap_fault(struct bpf_map *map, + struct vm_fault *vmf) +{ + struct bpf_array *array = container_of(map, struct bpf_array, map); + struct page *page; + + page = vmalloc_to_page(array->value + ((u64)vmf->pgoff << PAGE_SHIFT)); + if (!page) + return VM_FAULT_SIGBUS; + + /* the eager remap_vmalloc_range() flushed via vm_insert_page() */ + flush_dcache_folio(page_folio(page)); + get_page(page); + vmf->page = page; + + return 0; } static bool array_map_meta_equal(const struct bpf_map *meta0, @@ -844,6 +869,7 @@ const struct bpf_map_ops array_map_ops = { .map_direct_value_addr = array_map_direct_value_addr, .map_direct_value_meta = array_map_direct_value_meta, .map_mmap = array_map_mmap, + .map_mmap_fault = array_map_mmap_fault, .map_seq_show_elem = array_map_seq_show_elem, .map_check_btf = array_map_check_btf, .map_lookup_batch = generic_map_lookup_batch, |