From: Josef Bacik When the last reference to a program goes, __bpf_prog_put_noref() takes its symbols out of the kallsyms tree and frees its line info at once, and waits for the grace periods only with the rest. But the program may still be running: the link or the array it was called from is gone, the call is not over. search_bpf_extables() finds the exception table of a program by its symbol. A load that faults, and that the table is there to fix up, is fatal from then on: Oops: general protection fault, probably for non-canonical address 0xdead0000000006f8: 0000 [#1] SMP NOPTI RIP: 0010:0xffffffffc06008f6 It takes a program that is detached and closed on one CPU while it runs on another. Unwinders and bpf_prog_find_from_stack() do not know the program any more either, and the stack dump of a BPF stream reads the freed line info. Free the line info behind the grace periods the program needs, as it is done with the BTF that its text comes from. Remove the symbols in bpf_prog_free_deferred(), which runs behind them as well. The symbols are looked up under rcu_read_lock(), so the images and the program itself are freed one more grace period later. All that the program refers to is released when it was before. Fixes: 3dec541b2e63 ("bpf: Add support for BTF pointers to x86 JIT") Fixes: d7c431cafcb4 ("bpf: Add dump_stack() analogue to print to BPF stderr") Signed-off-by: Josef Bacik Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 1 + kernel/bpf/core.c | 50 +++++++++++++++++++++++++++++++------------- kernel/bpf/syscall.c | 5 ++--- 3 files changed, 38 insertions(+), 18 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index a10af47978a4..feb7de6483e3 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1971,6 +1971,7 @@ struct bpf_prog_aux { union { struct work_struct work; struct rcu_head rcu; + struct rcu_work rwork; }; struct bpf_stream *stream[2]; struct mutex st_ops_assoc_mutex; diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index e1216927373a..6e32db9777c2 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3085,12 +3085,40 @@ static void bpf_free_used_btfs(struct bpf_prog_aux *aux) kfree(aux->used_btfs); } -static void bpf_prog_free_deferred(struct work_struct *work) +static void bpf_prog_free_images(struct work_struct *work) { struct bpf_prog_aux *aux; int i; + aux = container_of(to_rcu_work(work), struct bpf_prog_aux, rwork); + for (i = 0; i < aux->real_func_cnt; i++) { + /* We can just unlink the subprog poke descriptor table as + * it was originally linked to the main program and is also + * released along with it. + */ + aux->func[i]->aux->poke_tab = NULL; + bpf_jit_free(aux->func[i]); + } + if (aux->real_func_cnt) { + kfree(aux->func); + bpf_prog_unlock_free(aux->prog); + } else { + bpf_jit_free(aux->prog); + } +} + +static void bpf_prog_free_deferred(struct work_struct *work) +{ + struct bpf_prog_aux *aux; + bool ksyms; + aux = container_of(work, struct bpf_prog_aux, work); + /* + * A prog that still runs when its last reference goes needs its symbols, + * by which its exception table is found. It has ended by now. + */ + ksyms = aux->real_func_cnt || !list_empty(&aux->ksym.lnode); + bpf_prog_kallsyms_del_all(aux->prog); #ifdef CONFIG_BPF_SYSCALL bpf_free_kfunc_btf_tab(aux->kfunc_btf_tab); #endif @@ -3109,20 +3137,12 @@ static void bpf_prog_free_deferred(struct work_struct *work) #endif if (aux->dst_trampoline) bpf_trampoline_put(aux->dst_trampoline); - for (i = 0; i < aux->real_func_cnt; i++) { - /* We can just unlink the subprog poke descriptor table as - * it was originally linked to the main program and is also - * released along with it. - */ - aux->func[i]->aux->poke_tab = NULL; - bpf_jit_free(aux->func[i]); - } - if (aux->real_func_cnt) { - kfree(aux->func); - bpf_prog_unlock_free(aux->prog); - } else { - bpf_jit_free(aux->prog); - } + /* The symbols are looked up under rcu_read_lock() */ + INIT_RCU_WORK(&aux->rwork, bpf_prog_free_images); + if (ksyms) + queue_rcu_work(system_percpu_wq, &aux->rwork); + else + bpf_prog_free_images(&aux->rwork.work); } void bpf_prog_free(struct bpf_prog *fp) diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index d026ab779ce8..849bc9136299 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2461,6 +2461,8 @@ static void __bpf_prog_put_rcu(struct rcu_head *rcu) struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); btf_put(aux->btf); + kvfree(aux->jited_linfo); + kvfree(aux->linfo); kvfree(aux->func_info); kfree(aux->func_info_aux); free_uid(aux->user); @@ -2485,10 +2487,7 @@ static void __bpf_prog_put_rcu_tasks(struct rcu_head *rcu) static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) { - bpf_prog_kallsyms_del_all(prog); module_put(prog->aux->mod); - kvfree(prog->aux->jited_linfo); - kvfree(prog->aux->linfo); kfree(prog->aux->kfunc_tab); kfree(prog->aux->ctx_arg_info); if (prog->aux->attach_btf) -- 2.55.0