Some plumbing work is done before bpf_check_cfg(). More specifically,
insn_aux_data records cleanup_throw_site for every bpf_throw() and
cleanup_resume_site for every bpf_unwind_resume() -- the two calls a JIT
lowers its own way rather than as calls -- and cleanup_pad, the landing pad
a frame resumes at, for every call within the [begin_off, end_off) range of
a cleanup record. Subsequent commits consume all three.

Marking the two calls here, rather than recognising them in the JIT, is
what makes the recognition exact: by the time a JIT runs,
bpf_fixup_kfunc_call() has rewritten every other kfunc's imm into an offset
from __bpf_call_base, and a BTF id compared against one of those offsets
could match an unrelated call.

bpf_prepare_cleanup_exceptions() runs before bpf_check_cfg(), because what
it produces is what the CFG walk consumes. It refuses a table on an
offloaded program, on a program whose JIT cannot dispatch landing pads or
which the JIT was not asked to compile, and on a program that also installs
an exception callback -- two different answers to what runs on the way out.
bpf_jit_supports_cleanup_pads() is weak here and says no; the arch patches
provide the real ones.

Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
---
 include/linux/bpf_verifier.h |  2 ++
 include/linux/filter.h       |  1 +
 kernel/bpf/core.c            |  5 +++
 kernel/bpf/exception.c       | 60 ++++++++++++++++++++++++++++++++++++
 kernel/bpf/exception.h       |  1 +
 kernel/bpf/fixups.c          |  6 ++++
 kernel/bpf/verifier.c        |  6 ++++
 7 files changed, 81 insertions(+)

diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 325a80ffcbe2..fdee9da6b45d 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -686,6 +686,8 @@ struct bpf_insn_aux_data {
 	bool needs_zext; /* alu op needs to clear upper bits */
 	bool non_sleepable; /* helper/kfunc may be called from non-sleepable context */
 	bool is_iter_next; /* bpf_iter_<type>_next() kfunc call */
+	bool cleanup_throw_site; /* call to bpf_throw() */
+	bool cleanup_resume_site; /* call to bpf_unwind_resume() */
 	bool call_with_percpu_alloc_ptr; /* {this,per}_cpu_ptr() with prog percpu alloc */
 	u8 alu_state; /* used in combination with alu_limit */
 	/* true if STX or LDX instruction is a part of a spill/fill
diff --git a/include/linux/filter.h b/include/linux/filter.h
index 422284b4fa96..2582a7606e46 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1242,6 +1242,7 @@ bool bpf_jit_supports_stack_args(void);
 bool bpf_jit_supports_arena_args(void);
 bool bpf_jit_supports_far_kfunc_call(void);
 bool bpf_jit_supports_exceptions(void);
+bool bpf_jit_supports_cleanup_pads(void);
 bool bpf_jit_supports_ptr_xchg(void);
 bool bpf_jit_supports_arena(void);
 bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena);
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 227211166dcc..a379cd1ec4c6 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -3475,6 +3475,11 @@ void __weak arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp,
 {
 }
 
+bool __weak bpf_jit_supports_cleanup_pads(void)
+{
+	return false;
+}
+
 bool __weak bpf_jit_supports_timed_may_goto(void)
 {
 	return false;
diff --git a/kernel/bpf/exception.c b/kernel/bpf/exception.c
index 4b3ac93e98c1..67af78baa558 100644
--- a/kernel/bpf/exception.c
+++ b/kernel/bpf/exception.c
@@ -18,6 +18,66 @@ static bool insn_is_unwind_resume(const struct bpf_insn *insn)
 	       insn->imm == bpf_unwind_resume_id[0];
 }
 
+static void cleanup_mark_kfunc_sites(struct bpf_verifier_env *env)
+{
+	u32 i;
+
+	for (i = 0; i < env->prog->len; i++) {
+		struct bpf_insn *insn = &env->prog->insnsi[i];
+
+		if (bpf_is_throw_kfunc(insn))
+			env->insn_aux_data[i].cleanup_throw_site = true;
+		else if (insn_is_unwind_resume(insn))
+			env->insn_aux_data[i].cleanup_resume_site = true;
+	}
+}
+
+static void cleanup_mark_call_sites(struct bpf_verifier_env *env)
+{
+	u32 i, j;
+
+	for (i = 0; i < env->cleanup_info_cnt; i++) {
+		struct bpf_cleanup_info *rec = &env->cleanup_info[i];
+
+		for (j = rec->begin_off; j < rec->end_off; j++) {
+			struct bpf_insn *insn = &env->prog->insnsi[j];
+
+			if (!bpf_pseudo_call(insn) && !bpf_is_throw_kfunc(insn))
+				continue;
+			env->insn_aux_data[j].cleanup_pad = rec->landing_pad_off + 1;
+		}
+	}
+}
+
+int bpf_prepare_cleanup_exceptions(struct bpf_verifier_env *env)
+{
+	if (!env->cleanup_info_cnt)
+		return 0;
+
+	if (bpf_prog_is_offloaded(env->prog->aux)) {
+		verbose(env,
+			"exception cleanup is not supported for offloaded programs\n");
+		return -EINVAL;
+	}
+
+	if (!bpf_jit_supports_cleanup_pads() || !env->prog->jit_requested) {
+		verbose(env,
+			"exception cleanup needs a JIT that can dispatch landing pads\n");
+		return -EOPNOTSUPP;
+	}
+	env->prog->jit_required = 1;
+
+	if (env->exception_callback_subprog) {
+		verbose(env,
+			"exception cleanup table cannot be combined with an exception callback\n");
+		return -EINVAL;
+	}
+
+	cleanup_mark_kfunc_sites(env);
+	cleanup_mark_call_sites(env);
+	return 0;
+}
+
 bool bpf_is_unwind_resume_kfunc(const struct bpf_insn *insn)
 {
 	return insn_is_unwind_resume(insn);
diff --git a/kernel/bpf/exception.h b/kernel/bpf/exception.h
index 0f2b9624a2ce..f51383fd775c 100644
--- a/kernel/bpf/exception.h
+++ b/kernel/bpf/exception.h
@@ -7,6 +7,7 @@
 
 struct bpf_verifier_env;
 
+int bpf_prepare_cleanup_exceptions(struct bpf_verifier_env *env);
 int bpf_cleanup_pad_of_call(struct bpf_verifier_env *env, u32 idx);
 
 #endif /* _LINUX_BPF_EXCEPTION_H */
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index 5e257d5fc0ef..18b8812f88a7 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -256,6 +256,12 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env,
 			data[i].non_stack_access =
 				data[off + cnt - 1].non_stack_access;
 			data[off + cnt - 1].non_stack_access = false;
+			data[i].cleanup_throw_site =
+				data[off + cnt - 1].cleanup_throw_site;
+			data[off + cnt - 1].cleanup_throw_site = false;
+			data[i].cleanup_resume_site =
+				data[off + cnt - 1].cleanup_resume_site;
+			data[off + cnt - 1].cleanup_resume_site = false;
 			data[i].cleanup_pad = data[off + cnt - 1].cleanup_pad;
 			data[off + cnt - 1].cleanup_pad = 0;
 		} else if (bpf_is_mem_insn(insn + i)) {
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index f6878f903bc1..a7a3c4b4d975 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -37,6 +37,7 @@
 
 #include "diagnostics.h"
 #include "disasm.h"
+#include "exception.h"
 
 static const struct bpf_verifier_ops * const bpf_verifier_ops[] = {
 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
@@ -21712,6 +21713,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	if (ret < 0)
 		goto skip_full_check;
 
+	/* The CFG needs an edge from a call in a cleanup range to its pad. */
+	ret = bpf_prepare_cleanup_exceptions(env);
+	if (ret < 0)
+		goto skip_full_check;
+
 	/* Validate instructions and resolve the program's referenced resources. */
 	ret = check_and_resolve_insns(env);
 	if (ret < 0)
-- 
2.53.0-Meta