Add two new files, exception.h and exception.c, to host the exception handling code. Only a few helpers so far: recognising a call to bpf_unwind_resume(), and asking which landing pad, if any, a call site unwinds to. The pad of a call site is kept in insn_aux_data, so the two places that move instructions around -- bpf_patch_insn_data() and verifier_remove_insns() -- learn to keep it in step. Signed-off-by: Yonghong Song --- include/linux/bpf_verifier.h | 7 +++++++ kernel/bpf/Makefile | 2 +- kernel/bpf/exception.c | 30 ++++++++++++++++++++++++++++++ kernel/bpf/exception.h | 12 ++++++++++++ kernel/bpf/fixups.c | 26 +++++++++++++++++++++++++- 5 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 kernel/bpf/exception.c create mode 100644 kernel/bpf/exception.h diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 6ce25c96ebed..6d78c20e6507 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -700,6 +700,11 @@ struct bpf_insn_aux_data { u64 jump_target:1; unsigned int orig_idx; /* original instruction index, initialized once */ + /* + * 1 + the instruction index of the exception cleanup landing pad + * this call site unwinds to, or 0 for none. + */ + u32 cleanup_pad; /* * CFG strongly connected component this instruction belongs to, * zero if it is a singleton SCC. @@ -1588,6 +1593,8 @@ u32 btf_func_arg_align(const struct btf *btf, const struct btf_type *t); int bpf_find_subprog(struct bpf_verifier_env *env, int off); bool bpf_is_throw_kfunc(struct bpf_insn *insn); +bool bpf_is_unwind_kfunc(const struct bpf_insn *insn); +bool bpf_is_unwind_resume_kfunc(const struct bpf_insn *insn); int bpf_compute_const_regs(struct bpf_verifier_env *env); int bpf_prune_dead_branches(struct bpf_verifier_env *env); int bpf_check_cfg(struct bpf_verifier_env *env); diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index c1f9b0d3468d..8a6947b3d13a 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -11,7 +11,7 @@ obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_ obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o bpf_task_storage.o -obj-$(CONFIG_BPF_SYSCALL) += fixups.o cfg.o states.o backtrack.o check_btf.o +obj-$(CONFIG_BPF_SYSCALL) += fixups.o cfg.o states.o backtrack.o check_btf.o exception.o obj-${CONFIG_BPF_LSM} += bpf_inode_storage.o obj-$(CONFIG_BPF_SYSCALL) += disasm.o mprog.o obj-$(CONFIG_BPF_JIT) += trampoline.o diff --git a/kernel/bpf/exception.c b/kernel/bpf/exception.c new file mode 100644 index 000000000000..b19fcbf49b7e --- /dev/null +++ b/kernel/bpf/exception.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include +#include +#include +#include "exception.h" + +BTF_ID_LIST_SINGLE(bpf_unwind_id, func, bpf_unwind) +BTF_ID_LIST_SINGLE(bpf_unwind_resume_id, func, bpf_unwind_resume) + +bool bpf_is_unwind_kfunc(const struct bpf_insn *insn) +{ + return bpf_pseudo_kfunc_call(insn) && insn->off == 0 && + insn->imm == bpf_unwind_id[0]; +} + +bool bpf_is_unwind_resume_kfunc(const struct bpf_insn *insn) +{ + return bpf_pseudo_kfunc_call(insn) && insn->off == 0 && + insn->imm == bpf_unwind_resume_id[0]; +} + +int bpf_exc_pad_of_call(struct bpf_verifier_env *env, u32 idx) +{ + u32 pad = env->insn_aux_data[idx].cleanup_pad; + + return pad ? (int)pad - 1 : -1; +} diff --git a/kernel/bpf/exception.h b/kernel/bpf/exception.h new file mode 100644 index 000000000000..634cb2c15ffc --- /dev/null +++ b/kernel/bpf/exception.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#ifndef _LINUX_BPF_EXCEPTION_H +#define _LINUX_BPF_EXCEPTION_H + +#include + +struct bpf_verifier_env; + +int bpf_exc_pad_of_call(struct bpf_verifier_env *env, u32 idx); + +#endif /* _LINUX_BPF_EXCEPTION_H */ diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 37cf130ebb57..5b7fe4ba610b 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -268,11 +268,18 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, data[i].non_stack_access = data[off + cnt - 1].non_stack_access; data[off + cnt - 1].non_stack_access = false; + data[i].cleanup_pad = data[off + cnt - 1].cleanup_pad; + data[off + cnt - 1].cleanup_pad = 0; } else if (bpf_is_mem_insn(insn + i)) { data[i].non_stack_access = true; } } + if (env->cleanup_info_cnt) + for (i = 0; i < prog_len; i++) + if (data[i].cleanup_pad > off + 1) + data[i].cleanup_pad += cnt - 1; + /* * Last slot instruction could be a newly generated * BPF_ST/BPF_LDX/BPF_STX, systematically mark it for non-stack access @@ -619,6 +626,7 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) struct bpf_insn_aux_data *aux_data = env->insn_aux_data; unsigned int orig_prog_len = env->prog->len; int err; + u32 i; if (bpf_rewrite_must_abort()) return -EINTR; @@ -647,6 +655,17 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) sizeof(*aux_data) * (orig_prog_len - off - cnt)); env->insn_aux_data_len -= cnt; + if (env->cleanup_info_cnt) { + for (i = 0; i < env->insn_aux_data_len; i++) { + u32 pad = aux_data[i].cleanup_pad; + + if (pad > off + cnt) + aux_data[i].cleanup_pad = pad - cnt; + else if (pad > off) + aux_data[i].cleanup_pad = 0; + } + } + return 0; } @@ -752,7 +771,7 @@ int bpf_opt_remove_nops(struct bpf_verifier_env *env) struct bpf_insn *insn = env->prog->insnsi; int insn_cnt = env->prog->len; bool is_may_goto_0, is_ja; - int i, err; + int i, j, err; for (i = 0; i < insn_cnt; i++) { is_may_goto_0 = !memcmp(&insn[i], &MAY_GOTO_0, sizeof(MAY_GOTO_0)); @@ -763,6 +782,11 @@ int bpf_opt_remove_nops(struct bpf_verifier_env *env) if (aux[i].indirect_target) continue; + if (env->cleanup_info_cnt) + for (j = 0; j < insn_cnt; j++) + if (env->insn_aux_data[j].cleanup_pad == i + 1) + env->insn_aux_data[j].cleanup_pad = i + 2; + err = verifier_remove_insns(env, i, 1); if (err) return err; -- 2.53.0-Meta