Add a bpf_subprog_info parameter to bpf_int_jit_compile() so that JIT backends can access per-subprog metadata directly, such as stack_arg_cnt and arg_cnt, without encoding them into bpf_prog_aux. Previously, subprog properties needed by the JIT were copied into bpf_prog_aux fields (e.g., out_stack_arg_cnt). Passing bpf_subprog_info directly avoids adding more fields to bpf_prog_aux and gives JIT backends a single place to query subprog metadata. In jit_subprogs(), each subprog's corresponding bpf_subprog_info is passed to bpf_int_jit_compile(). For the single-subprog path in bpf_prog_jit_compile(), &env->subprog_info[0] is passed when env is available, or NULL when env is NULL. The env is NULL for internal kernel BPF programs created via bpf_prog_create() (e.g., the PTP packet classifier in net/core/ptp_classifier.c) which bypass the verifier and never use stack arguments. All architecture JIT implementations are updated to accept the new parameter. No functional change for architectures that do not yet use subprog_info. Signed-off-by: Yonghong Song --- arch/arc/net/bpf_jit_core.c | 3 ++- arch/arm/net/bpf_jit_32.c | 3 ++- arch/arm64/net/bpf_jit_comp.c | 3 ++- arch/loongarch/net/bpf_jit.c | 3 ++- arch/mips/net/bpf_jit_comp.c | 3 ++- arch/parisc/net/bpf_jit_core.c | 3 ++- arch/powerpc/net/bpf_jit_comp.c | 3 ++- arch/riscv/net/bpf_jit_core.c | 3 ++- arch/s390/net/bpf_jit_comp.c | 3 ++- arch/sparc/net/bpf_jit_comp_64.c | 3 ++- arch/x86/net/bpf_jit_comp.c | 3 ++- arch/x86/net/bpf_jit_comp32.c | 3 ++- include/linux/filter.h | 4 +++- kernel/bpf/core.c | 9 ++++++--- kernel/bpf/fixups.c | 4 ++-- 15 files changed, 35 insertions(+), 18 deletions(-) diff --git a/arch/arc/net/bpf_jit_core.c b/arch/arc/net/bpf_jit_core.c index 639a2736f029..771de9a8147d 100644 --- a/arch/arc/net/bpf_jit_core.c +++ b/arch/arc/net/bpf_jit_core.c @@ -1400,7 +1400,8 @@ static struct bpf_prog *do_extra_pass(struct bpf_prog *prog) * (re)locations involved that their addresses are not known * during the first run. */ -struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog, + struct bpf_subprog_info *subprog_info) { vm_dump(prog); diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index 9ede81afbc50..17692af0f11f 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -2148,7 +2148,8 @@ bool bpf_jit_needs_zext(void) return true; } -struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog, + struct bpf_subprog_info *subprog_info) { struct bpf_binary_header *header; struct jit_ctx ctx; diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 0816c40fc7af..c9bdeef31ab9 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -2003,7 +2003,8 @@ struct arm64_jit_data { struct jit_ctx ctx; }; -struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog, + struct bpf_subprog_info *subprog_info) { int image_size, prog_size, extable_size, extable_align, extable_offset; struct bpf_binary_header *header; diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 24913dc7f4e8..416dabe9947f 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -2166,7 +2166,8 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, return ret < 0 ? ret : ret * LOONGARCH_INSN_SIZE; } -struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog, + struct bpf_subprog_info *subprog_info) { bool extra_pass = false; u8 *image_ptr, *ro_image_ptr; diff --git a/arch/mips/net/bpf_jit_comp.c b/arch/mips/net/bpf_jit_comp.c index 6ee4abe6a1f7..2f85c90bba07 100644 --- a/arch/mips/net/bpf_jit_comp.c +++ b/arch/mips/net/bpf_jit_comp.c @@ -909,7 +909,8 @@ bool bpf_jit_needs_zext(void) return true; } -struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog, + struct bpf_subprog_info *subprog_info) { struct bpf_binary_header *header = NULL; struct jit_context ctx; diff --git a/arch/parisc/net/bpf_jit_core.c b/arch/parisc/net/bpf_jit_core.c index 172770132440..b24275aef8db 100644 --- a/arch/parisc/net/bpf_jit_core.c +++ b/arch/parisc/net/bpf_jit_core.c @@ -41,7 +41,8 @@ bool bpf_jit_needs_zext(void) return true; } -struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog, + struct bpf_subprog_info *subprog_info) { unsigned int prog_size = 0, extable_size = 0; bool extra_pass = false; diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 53ab97ad6074..5a414da47e4e 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -162,7 +162,8 @@ static void priv_stack_check_guard(void __percpu *priv_stack_ptr, int alloc_size } } -struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp, + struct bpf_subprog_info *subprog_info) { u32 proglen; u32 alloclen; diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index 4365d07aaf54..3ae8adf0f697 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -41,7 +41,8 @@ bool bpf_jit_needs_zext(void) return true; } -struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog, + struct bpf_subprog_info *subprog_info) { unsigned int prog_size = 0, extable_size = 0; bool extra_pass = false; diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index 14eaaa5b2185..bc29f405161a 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -2337,7 +2337,8 @@ static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit, /* * Compile eBPF program "fp" */ -struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp, + struct bpf_subprog_info *subprog_info) { struct bpf_binary_header *header; struct s390_jit_data *jit_data; diff --git a/arch/sparc/net/bpf_jit_comp_64.c b/arch/sparc/net/bpf_jit_comp_64.c index 2fa0e9375127..90bc01d069fe 100644 --- a/arch/sparc/net/bpf_jit_comp_64.c +++ b/arch/sparc/net/bpf_jit_comp_64.c @@ -1477,7 +1477,8 @@ struct sparc64_jit_data { struct jit_ctx ctx; }; -struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog, + struct bpf_subprog_info *subprog_info) { struct sparc64_jit_data *jit_data; struct bpf_binary_header *header; diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index ea9e707e8abf..e9718faa0124 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -3715,7 +3715,8 @@ struct x64_jit_data { #define MAX_PASSES 20 #define PADDING_PASSES (MAX_PASSES - 5) -struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog, + struct bpf_subprog_info *subprog_info) { struct bpf_binary_header *rw_header = NULL; struct bpf_binary_header *header = NULL; diff --git a/arch/x86/net/bpf_jit_comp32.c b/arch/x86/net/bpf_jit_comp32.c index 852baf2e4db4..e89558a27834 100644 --- a/arch/x86/net/bpf_jit_comp32.c +++ b/arch/x86/net/bpf_jit_comp32.c @@ -2518,7 +2518,8 @@ bool bpf_jit_needs_zext(void) return true; } -struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog, + struct bpf_subprog_info *subprog_info) { struct bpf_binary_header *header = NULL; int proglen, oldproglen = 0; diff --git a/include/linux/filter.h b/include/linux/filter.h index a515a9769078..3576ee523bee 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -37,6 +37,7 @@ struct xdp_buff; struct sock_reuseport; struct ctl_table; struct ctl_table_header; +struct bpf_subprog_info; /* ArgX, context and stack frame pointer register positions. Note, * Arg1, Arg2, Arg3, etc are used as argument mappings of function @@ -1177,7 +1178,8 @@ u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); ((u64 (*)(u64, u64, u64, u64, u64, const struct bpf_insn *)) \ (void *)__bpf_call_base) -struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog); +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog, + struct bpf_subprog_info *subprog_info); void bpf_jit_compile(struct bpf_prog *prog); bool bpf_jit_needs_zext(void); bool bpf_jit_inlines_helper_call(s32 imm); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index e6b836f846eb..dcba6f166a91 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -2547,7 +2547,8 @@ static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struc struct bpf_insn_aux_data *orig_insn_aux; if (!bpf_prog_need_blind(prog)) - return bpf_int_jit_compile(env, prog); + return bpf_int_jit_compile(env, prog, + env ? &env->subprog_info[0] : NULL); if (env) { /* @@ -2569,7 +2570,8 @@ static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struc if (IS_ERR(prog)) goto out_restore; - prog = bpf_int_jit_compile(env, prog); + prog = bpf_int_jit_compile(env, prog, + env ? &env->subprog_info[0] : NULL); if (prog->jited) { bpf_jit_prog_release_other(prog, orig_prog); if (env) @@ -3145,7 +3147,8 @@ const struct bpf_func_proto bpf_tail_call_proto = { * It is encouraged to implement bpf_int_jit_compile() instead, so that * eBPF and implicitly also cBPF can get JITed! */ -struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) +struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog, + struct bpf_subprog_info *subprog_info) { return prog; } diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 19056016eed8..62f1823f089e 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -1162,7 +1162,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) func[i]->aux->token = prog->aux->token; if (!i) func[i]->aux->exception_boundary = env->seen_exception; - func[i] = bpf_int_jit_compile(env, func[i]); + func[i] = bpf_int_jit_compile(env, func[i], &env->subprog_info[i]); if (!func[i]->jited) { err = -ENOTSUPP; goto out_free; @@ -1206,7 +1206,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) } for (i = 0; i < env->subprog_cnt; i++) { old_bpf_func = func[i]->bpf_func; - tmp = bpf_int_jit_compile(env, func[i]); + tmp = bpf_int_jit_compile(env, func[i], &env->subprog_info[i]); if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) { verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); err = -ENOTSUPP; -- 2.53.0-Meta