The current arena fault handling code accounts for misbehaving BPF programs that free arena memory while it is being accessed by a kfunc as part of another BPF program's execution. This scenario triggers a kernel page fault that is intercepted by BPF arena-specific code. The code confirms that the fault is within the BPF program's arena, installs a scratch page, and marks the fault as handled. This is necessary to prevent BPF programs from being able to trigger an oops. However, the protection against misbehaving BPF programs is currently incomplete: Arena memory is accessed by kernel code not just as part of a kfunc, but also from non-BPF paths. This is the case with sched-ext that includes non-BPF code paths that access arena pointers. These paths do not have an arena associated with them, and a fault there causes an oops. Such a fault can be triggered by a misbehaving BPF program by unmapping the pages nominally owned by the genpool allocator used for cid management. Extend the kernel arena fault handling to consider all arenas in the system. The handler now scans a global arena tree till it finds an arena that includes the address, and fixes up the address. Retain the stream-based reporting code for when the faulting call stack includes the BPF program that owns the arena. Fixes: abdc2516f100 ("sched_ext: Convert ops.set_cmask() to arena-resident cmask") Cc: Tejun Heo Signed-off-by: Emil Tsalapatis --- kernel/bpf/arena.c | 119 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 91 insertions(+), 28 deletions(-) diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 366c4c2b6a41..aff231922e11 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -11,6 +11,9 @@ #include #include #include +#include +#include +#include #include #include "range_tree.h" @@ -47,10 +50,15 @@ #define GUARD_SZ round_up(1ull << sizeof_field(struct bpf_insn, off) * 8, PAGE_SIZE << 1) #define KERN_VM_SZ (SZ_4G + GUARD_SZ) +static struct latch_tree_root arena_tree; +static DEFINE_SPINLOCK(arena_tree_lock); + static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, bool sleepable); struct bpf_arena { struct bpf_map map; + struct latch_tree_node tnode; + struct rcu_work destroy_work; u64 user_vm_start; u64 user_vm_end; struct vm_struct *kern_vm; @@ -142,6 +150,33 @@ struct bpf_map *bpf_prog_arena(struct bpf_prog *prog) return arena ? &arena->map : NULL; } +static __always_inline bool arena_tree_less(struct latch_tree_node *a, + struct latch_tree_node *b) +{ + struct bpf_arena *arena_a = container_of(a, struct bpf_arena, tnode); + struct bpf_arena *arena_b = container_of(b, struct bpf_arena, tnode); + + return bpf_arena_get_kern_vm_start(arena_a) < bpf_arena_get_kern_vm_start(arena_b); +} + +static __always_inline int arena_tree_comp(void *key, struct latch_tree_node *node) +{ + struct bpf_arena *arena = container_of(node, struct bpf_arena, tnode); + unsigned long kbase = bpf_arena_get_kern_vm_start(arena); + unsigned long addr = (unsigned long)key; + + if (addr < kbase) + return -1; + if (addr >= kbase + SZ_4G + GUARD_SZ / 2) + return 1; + return 0; +} + +static const struct latch_tree_ops arena_tree_ops = { + .less = arena_tree_less, + .comp = arena_tree_comp, +}; + static long arena_map_peek_elem(struct bpf_map *map, void *value) { return -EOPNOTSUPP; @@ -360,6 +395,10 @@ static struct bpf_map *arena_map_alloc(union bpf_attr *attr) if (err) goto err_destroy_rt; + spin_lock(&arena_tree_lock); + latch_tree_insert(&arena->tnode, &arena_tree, &arena_tree_ops); + spin_unlock(&arena_tree_lock); + return &arena->map; err_destroy_rt: @@ -400,6 +439,26 @@ static int existing_page_cb(pte_t *ptep, unsigned long addr, void *data) return 0; } +static void arena_map_destroy(struct work_struct *work) +{ + struct bpf_arena *arena = container_of(to_rcu_work(work), struct bpf_arena, + destroy_work); + + /* + * free_vm_area() calls remove_vm_area() that calls free_unmap_vmap_area(). + * It unmaps everything from vmalloc area and clears pgtables. + * Call apply_to_existing_page_range() first to find populated ptes and + * free those pages. + */ + apply_to_existing_page_range(&init_mm, bpf_arena_get_kern_vm_start(arena), + SZ_4G + GUARD_SZ / 2, existing_page_cb, arena); + free_vm_area(arena->kern_vm); + range_tree_destroy(&arena->rt); + __free_page(arena->scratch_page); + free_percpu(arena->stacks); + bpf_map_area_free(arena); +} + static void arena_map_free(struct bpf_map *map) { struct bpf_arena *arena = container_of(map, struct bpf_arena, map); @@ -413,23 +472,16 @@ static void arena_map_free(struct bpf_map *map) if (WARN_ON_ONCE(!list_empty(&arena->vma_list))) return; + spin_lock(&arena_tree_lock); + latch_tree_erase(&arena->tnode, &arena_tree, &arena_tree_ops); + spin_unlock(&arena_tree_lock); + /* Ensure no pending deferred frees */ irq_work_sync(&arena->free_irq); flush_work(&arena->free_work); - /* - * free_vm_area() calls remove_vm_area() that calls free_unmap_vmap_area(). - * It unmaps everything from vmalloc area and clears pgtables. - * Call apply_to_existing_page_range() first to find populated ptes and - * free those pages. - */ - apply_to_existing_page_range(&init_mm, bpf_arena_get_kern_vm_start(arena), - SZ_4G + GUARD_SZ / 2, existing_page_cb, arena); - free_vm_area(arena->kern_vm); - range_tree_destroy(&arena->rt); - __free_page(arena->scratch_page); - free_percpu(arena->stacks); - bpf_map_area_free(arena); + INIT_RCU_WORK(&arena->destroy_work, arena_map_destroy); + queue_rcu_work(system_dfl_wq, &arena->destroy_work); } static void *arena_map_lookup_elem(struct bpf_map *map, void *key) @@ -1357,35 +1409,46 @@ static void __bpf_prog_report_arena_violation(struct bpf_prog *prog, bool write, bool bpf_arena_handle_page_fault(unsigned long addr, bool is_write, unsigned long fault_ip) { + static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + struct latch_tree_node *node; struct bpf_arena *arena; struct bpf_prog *prog; unsigned long kbase; unsigned long page_addr = addr & PAGE_MASK; - - prog = bpf_prog_find_from_stack(); - if (!prog) - return false; - - arena = prog->aux->arena; - /* a prog not using arena may be on stack, so arena can be NULL */ - if (!arena) - return false; - - kbase = bpf_arena_get_kern_vm_start(arena); + bool handled = false; /* * Recovery covers the 4 GiB mappable band plus the upper half-guard. * Lower guard is unreachable from kfuncs; an address there indicates * a different bug class - leave it to the regular kernel oops path. */ - if (page_addr < kbase || page_addr >= kbase + SZ_4G + GUARD_SZ / 2) - return false; + rcu_read_lock(); + prog = bpf_prog_find_from_stack(); + arena = prog ? prog->aux->arena : NULL; + if (!arena || arena_tree_comp((void *)page_addr, &arena->tnode)) { + node = latch_tree_find((void *)page_addr, &arena_tree, &arena_tree_ops); + if (!node) + goto out; + arena = container_of(node, struct bpf_arena, tnode); + } + kbase = bpf_arena_get_kern_vm_start(arena); apply_to_page_range(&init_mm, page_addr, PAGE_SIZE, apply_range_set_scratch_cb, arena->scratch_page); flush_vmap_cache(page_addr, PAGE_SIZE); - __bpf_prog_report_arena_violation(prog, is_write, page_addr - kbase, fault_ip); - return true; + + if (prog && prog->aux->arena == arena) + __bpf_prog_report_arena_violation(prog, is_write, + page_addr - kbase, fault_ip); + else if (__ratelimit(&rs)) + printk_deferred(KERN_ERR "BPF arena %u: %s fault at offset 0x%lx, IP %pS\n", + arena->map.id, is_write ? "WRITE" : "READ", + page_addr - kbase, (void *)fault_ip); + handled = true; +out: + rcu_read_unlock(); + return handled; } void bpf_prog_report_arena_violation(bool write, unsigned long addr, unsigned long fault_ip) -- 2.52.0 Test the arena fault handling code with two new selftests. The first one confirms that a program without an arena (and by extension a code path that doesn't involve BPF at all) properly recovers after accessing an unbacked arena page. The second test deploys the same configuration for a program that has its own arena map. Signed-off-by: Emil Tsalapatis --- .../selftests/bpf/prog_tests/arena_fault.c | 100 ++++++++++++++++++ .../testing/selftests/bpf/progs/arena_fault.c | 65 ++++++++++++ .../selftests/bpf/progs/arena_fault_noarena.c | 17 +++ .../selftests/bpf/test_kmods/bpf_testmod.c | 22 ++++ .../bpf/test_kmods/bpf_testmod_kfunc.h | 3 + 5 files changed, 207 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/arena_fault.c create mode 100644 tools/testing/selftests/bpf/progs/arena_fault.c create mode 100644 tools/testing/selftests/bpf/progs/arena_fault_noarena.c diff --git a/tools/testing/selftests/bpf/prog_tests/arena_fault.c b/tools/testing/selftests/bpf/prog_tests/arena_fault.c new file mode 100644 index 000000000000..8bde1f67e4b4 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/arena_fault.c @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include "arena_fault.skel.h" +#include "arena_fault_noarena.skel.h" + +static int run_syscall_prog(struct bpf_program *prog, const char *name) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + int err; + + err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + if (!ASSERT_OK(err, name)) + return -1; + if (!ASSERT_OK(opts.retval, name)) + return -1; + return 0; +} + +static void test_wrong_arena(void) +{ + struct arena_fault *fault_skel = NULL; + struct arena_fault *stash_skel = NULL; + + stash_skel = arena_fault__open(); + if (!ASSERT_OK_PTR(stash_skel, "open_stash")) + return; + + if (stash_skel->data->skip) { + test__skip(); + goto out; + } + if (!ASSERT_OK(arena_fault__load(stash_skel), "load_stash")) + goto out; + + fault_skel = arena_fault__open_and_load(); + if (!ASSERT_OK_PTR(fault_skel, "open_load_fault")) + goto out; + + if (run_syscall_prog(stash_skel->progs.stash_and_free, "stash_and_free")) + goto out; + run_syscall_prog(fault_skel->progs.fault_stashed_wrong_arena, + "fault_stashed_wrong_arena"); + +out: + if (stash_skel && bpf_program__fd(stash_skel->progs.clear_stashed) >= 0) + run_syscall_prog(stash_skel->progs.clear_stashed, "clear_stashed"); + arena_fault__destroy(fault_skel); + arena_fault__destroy(stash_skel); +} + +static void test_no_arena(void) +{ + struct arena_fault_noarena *fault_skel = NULL; + struct arena_fault *stash_skel = NULL; + + stash_skel = arena_fault__open(); + if (!ASSERT_OK_PTR(stash_skel, "open_stash")) + return; + + if (stash_skel->data->skip) { + test__skip(); + goto out; + } + if (!ASSERT_OK(arena_fault__load(stash_skel), "load_stash")) + goto out; + + fault_skel = arena_fault_noarena__open_and_load(); + if (!ASSERT_OK_PTR(fault_skel, "open_load_noarena")) + goto out; + + if (run_syscall_prog(stash_skel->progs.stash_and_free, "stash_and_free")) + goto out; + run_syscall_prog(fault_skel->progs.fault_stashed_noarena, + "fault_stashed_noarena"); + +out: + if (stash_skel && bpf_program__fd(stash_skel->progs.clear_stashed) >= 0) + run_syscall_prog(stash_skel->progs.clear_stashed, "clear_stashed"); + arena_fault_noarena__destroy(fault_skel); + arena_fault__destroy(stash_skel); +} + +void serial_test_arena_fault(void) +{ +#if !defined(__x86_64__) && !defined(__aarch64__) + test__skip(); + return; +#endif + + if (!env.has_testmod) { + test__skip(); + return; + } + + if (test__start_subtest("wrong_arena")) + test_wrong_arena(); + if (test__start_subtest("no_arena")) + test_no_arena(); +} diff --git a/tools/testing/selftests/bpf/progs/arena_fault.c b/tools/testing/selftests/bpf/progs/arena_fault.c new file mode 100644 index 000000000000..89fde45217dc --- /dev/null +++ b/tools/testing/selftests/bpf/progs/arena_fault.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define BPF_NO_KFUNC_PROTOTYPES +#include +#include +#include "bpf_arena_common.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 2); +} arena SEC(".maps"); + +u64 __arena arena_pad; +volatile u64 observed; + +#ifdef __BPF_FEATURE_ADDR_SPACE_CAST +bool skip SEC(".data") = false; +#else +bool skip = true; +#endif + +SEC("syscall") +int stash_and_free(void *ctx) +{ +#ifdef __BPF_FEATURE_ADDR_SPACE_CAST + u64 __arena *val; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + *val = 42; + bpf_test_arena_fault_stash((u64 *)val); + bpf_arena_free_pages(&arena, val, 1); +#endif + return 0; +} + +SEC("syscall") +int fault_stashed_wrong_arena(void *ctx) +{ +#ifdef __BPF_FEATURE_ADDR_SPACE_CAST + u64 __arena *val; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + *val = 1; + observed = bpf_test_arena_fault_load_stashed(); + bpf_arena_free_pages(&arena, val, 1); +#endif + return observed != 0; +} + +SEC("syscall") +int clear_stashed(void *ctx) +{ + bpf_test_arena_fault_clear_stashed(); + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/arena_fault_noarena.c b/tools/testing/selftests/bpf/progs/arena_fault_noarena.c new file mode 100644 index 000000000000..7bb0e6dbd260 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/arena_fault_noarena.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define BPF_NO_KFUNC_PROTOTYPES +#include +#include +#include "../test_kmods/bpf_testmod_kfunc.h" + +volatile u64 observed; + +SEC("syscall") +int fault_stashed_noarena(void *ctx) +{ + observed = bpf_test_arena_fault_load_stashed(); + return observed != 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 76fbe64755ef..f33a7345f9ce 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -344,6 +344,25 @@ __bpf_kfunc u64 bpf_kfunc_arena_mixed_test(u64 *a__arena, u64 *b__arena__nullabl return *a__arena + (b__arena__nullable ? *b__arena__nullable : 0); } +static u64 *arena_fault_stashed_ptr; + +__bpf_kfunc void bpf_test_arena_fault_stash(u64 *ptr__arena) +{ + WRITE_ONCE(arena_fault_stashed_ptr, ptr__arena); +} + +__bpf_kfunc u64 bpf_test_arena_fault_load_stashed(void) +{ + u64 *ptr = READ_ONCE(arena_fault_stashed_ptr); + + return ptr ? READ_ONCE(*ptr) : U64_MAX; +} + +__bpf_kfunc void bpf_test_arena_fault_clear_stashed(void) +{ + WRITE_ONCE(arena_fault_stashed_ptr, NULL); +} + __bpf_kfunc void bpf_kfunc_dynptr_test(struct bpf_dynptr *ptr, struct bpf_dynptr *ptr__nullable) { @@ -1048,6 +1067,9 @@ BTF_ID_FLAGS(func, bpf_kfunc_arena_cap_nullable_test) BTF_ID_FLAGS(func, bpf_kfunc_arena_args5_test) BTF_ID_FLAGS(func, bpf_kfunc_arena_stack_arg_test) BTF_ID_FLAGS(func, bpf_kfunc_arena_mixed_test) +BTF_ID_FLAGS(func, bpf_test_arena_fault_stash) +BTF_ID_FLAGS(func, bpf_test_arena_fault_load_stashed) +BTF_ID_FLAGS(func, bpf_test_arena_fault_clear_stashed) BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_pass1) BTF_ID_FLAGS(func, bpf_kfunc_dynptr_test) BTF_ID_FLAGS(func, bpf_kfunc_nested_acquire_nonzero_offset_test, KF_ACQUIRE) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index e9902e651d72..9ec69fd57d98 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -212,6 +212,9 @@ __u64 bpf_kfunc_arena_args5_test(__u64 *a__arena, __u64 *b__arena, __u64 bpf_kfunc_arena_stack_arg_test(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, __u64 *f__arena) __ksym; __u64 bpf_kfunc_arena_mixed_test(__u64 *a__arena, __u64 *b__arena__nullable) __ksym; +void bpf_test_arena_fault_stash(__u64 *ptr__arena) __ksym; +__u64 bpf_test_arena_fault_load_stashed(void) __ksym; +void bpf_test_arena_fault_clear_stashed(void) __ksym; int *bpf_kfunc_call_test_get_rdwr_mem(struct prog_test_ref_kfunc *p, const int rdwr_buf_size) __ksym; int *bpf_kfunc_call_test_get_rdonly_mem(struct prog_test_ref_kfunc *p, const int rdonly_buf_size) __ksym; int *bpf_kfunc_call_test_acq_rdonly_mem(struct prog_test_ref_kfunc *p, const int rdonly_buf_size) __ksym; -- 2.52.0