The current arena fault handling code accounts for misbehaving BPF programs that free arena memory while it is being accessed by a kfunc as part of another BPF program's execution. This scenario triggers a kernel page fault that is intercepted by BPF arena-specific code. The code confirms that the fault is within the BPF program's arena, installs a scratch page, and marks the fault as handled. This is necessary to prevent BPF programs from being able to trigger an oops. However, the protection against misbehaving BPF programs is currently incomplete: Arena memory is accessed by kernel code not just as part of a kfunc, but also from non-BPF paths. This is the case with sched-ext that includes non-BPF code paths that access arena pointers. These paths do not have an arena associated with them, and a fault there causes an oops. Such a fault can be triggered by a misbehaving BPF program by unmapping the pages nominally owned by the genpool allocator used for cid management. Extend the kernel arena fault handling to consider all arenas in the system. The handler now scans a global arena tree till it finds an arena that includes the address, and fixes up the address. Retain the stream-based reporting code for when the faulting call stack includes the BPF program that owns the arena. Fixes: abdc2516f100 ("sched_ext: Convert ops.set_cmask() to arena-resident cmask") Cc: Tejun Heo Signed-off-by: Emil Tsalapatis --- kernel/bpf/arena.c | 119 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 91 insertions(+), 28 deletions(-) diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 366c4c2b6a41..aff231922e11 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -11,6 +11,9 @@ #include #include #include +#include +#include +#include #include #include "range_tree.h" @@ -47,10 +50,15 @@ #define GUARD_SZ round_up(1ull << sizeof_field(struct bpf_insn, off) * 8, PAGE_SIZE << 1) #define KERN_VM_SZ (SZ_4G + GUARD_SZ) +static struct latch_tree_root arena_tree; +static DEFINE_SPINLOCK(arena_tree_lock); + static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, bool sleepable); struct bpf_arena { struct bpf_map map; + struct latch_tree_node tnode; + struct rcu_work destroy_work; u64 user_vm_start; u64 user_vm_end; struct vm_struct *kern_vm; @@ -142,6 +150,33 @@ struct bpf_map *bpf_prog_arena(struct bpf_prog *prog) return arena ? &arena->map : NULL; } +static __always_inline bool arena_tree_less(struct latch_tree_node *a, + struct latch_tree_node *b) +{ + struct bpf_arena *arena_a = container_of(a, struct bpf_arena, tnode); + struct bpf_arena *arena_b = container_of(b, struct bpf_arena, tnode); + + return bpf_arena_get_kern_vm_start(arena_a) < bpf_arena_get_kern_vm_start(arena_b); +} + +static __always_inline int arena_tree_comp(void *key, struct latch_tree_node *node) +{ + struct bpf_arena *arena = container_of(node, struct bpf_arena, tnode); + unsigned long kbase = bpf_arena_get_kern_vm_start(arena); + unsigned long addr = (unsigned long)key; + + if (addr < kbase) + return -1; + if (addr >= kbase + SZ_4G + GUARD_SZ / 2) + return 1; + return 0; +} + +static const struct latch_tree_ops arena_tree_ops = { + .less = arena_tree_less, + .comp = arena_tree_comp, +}; + static long arena_map_peek_elem(struct bpf_map *map, void *value) { return -EOPNOTSUPP; @@ -360,6 +395,10 @@ static struct bpf_map *arena_map_alloc(union bpf_attr *attr) if (err) goto err_destroy_rt; + spin_lock(&arena_tree_lock); + latch_tree_insert(&arena->tnode, &arena_tree, &arena_tree_ops); + spin_unlock(&arena_tree_lock); + return &arena->map; err_destroy_rt: @@ -400,6 +439,26 @@ static int existing_page_cb(pte_t *ptep, unsigned long addr, void *data) return 0; } +static void arena_map_destroy(struct work_struct *work) +{ + struct bpf_arena *arena = container_of(to_rcu_work(work), struct bpf_arena, + destroy_work); + + /* + * free_vm_area() calls remove_vm_area() that calls free_unmap_vmap_area(). + * It unmaps everything from vmalloc area and clears pgtables. + * Call apply_to_existing_page_range() first to find populated ptes and + * free those pages. + */ + apply_to_existing_page_range(&init_mm, bpf_arena_get_kern_vm_start(arena), + SZ_4G + GUARD_SZ / 2, existing_page_cb, arena); + free_vm_area(arena->kern_vm); + range_tree_destroy(&arena->rt); + __free_page(arena->scratch_page); + free_percpu(arena->stacks); + bpf_map_area_free(arena); +} + static void arena_map_free(struct bpf_map *map) { struct bpf_arena *arena = container_of(map, struct bpf_arena, map); @@ -413,23 +472,16 @@ static void arena_map_free(struct bpf_map *map) if (WARN_ON_ONCE(!list_empty(&arena->vma_list))) return; + spin_lock(&arena_tree_lock); + latch_tree_erase(&arena->tnode, &arena_tree, &arena_tree_ops); + spin_unlock(&arena_tree_lock); + /* Ensure no pending deferred frees */ irq_work_sync(&arena->free_irq); flush_work(&arena->free_work); - /* - * free_vm_area() calls remove_vm_area() that calls free_unmap_vmap_area(). - * It unmaps everything from vmalloc area and clears pgtables. - * Call apply_to_existing_page_range() first to find populated ptes and - * free those pages. - */ - apply_to_existing_page_range(&init_mm, bpf_arena_get_kern_vm_start(arena), - SZ_4G + GUARD_SZ / 2, existing_page_cb, arena); - free_vm_area(arena->kern_vm); - range_tree_destroy(&arena->rt); - __free_page(arena->scratch_page); - free_percpu(arena->stacks); - bpf_map_area_free(arena); + INIT_RCU_WORK(&arena->destroy_work, arena_map_destroy); + queue_rcu_work(system_dfl_wq, &arena->destroy_work); } static void *arena_map_lookup_elem(struct bpf_map *map, void *key) @@ -1357,35 +1409,46 @@ static void __bpf_prog_report_arena_violation(struct bpf_prog *prog, bool write, bool bpf_arena_handle_page_fault(unsigned long addr, bool is_write, unsigned long fault_ip) { + static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + struct latch_tree_node *node; struct bpf_arena *arena; struct bpf_prog *prog; unsigned long kbase; unsigned long page_addr = addr & PAGE_MASK; - - prog = bpf_prog_find_from_stack(); - if (!prog) - return false; - - arena = prog->aux->arena; - /* a prog not using arena may be on stack, so arena can be NULL */ - if (!arena) - return false; - - kbase = bpf_arena_get_kern_vm_start(arena); + bool handled = false; /* * Recovery covers the 4 GiB mappable band plus the upper half-guard. * Lower guard is unreachable from kfuncs; an address there indicates * a different bug class - leave it to the regular kernel oops path. */ - if (page_addr < kbase || page_addr >= kbase + SZ_4G + GUARD_SZ / 2) - return false; + rcu_read_lock(); + prog = bpf_prog_find_from_stack(); + arena = prog ? prog->aux->arena : NULL; + if (!arena || arena_tree_comp((void *)page_addr, &arena->tnode)) { + node = latch_tree_find((void *)page_addr, &arena_tree, &arena_tree_ops); + if (!node) + goto out; + arena = container_of(node, struct bpf_arena, tnode); + } + kbase = bpf_arena_get_kern_vm_start(arena); apply_to_page_range(&init_mm, page_addr, PAGE_SIZE, apply_range_set_scratch_cb, arena->scratch_page); flush_vmap_cache(page_addr, PAGE_SIZE); - __bpf_prog_report_arena_violation(prog, is_write, page_addr - kbase, fault_ip); - return true; + + if (prog && prog->aux->arena == arena) + __bpf_prog_report_arena_violation(prog, is_write, + page_addr - kbase, fault_ip); + else if (__ratelimit(&rs)) + printk_deferred(KERN_ERR "BPF arena %u: %s fault at offset 0x%lx, IP %pS\n", + arena->map.id, is_write ? "WRITE" : "READ", + page_addr - kbase, (void *)fault_ip); + handled = true; +out: + rcu_read_unlock(); + return handled; } void bpf_prog_report_arena_violation(bool write, unsigned long addr, unsigned long fault_ip) -- 2.52.0