From: Xu Kuohai Pass bpf_verifier_env to bpf_int_jit_compile(). The follow-up patch will use env->insn_aux_data in the JIT stage to detect indirect jump targets. Since bpf_prog_select_runtime() can be called by cbpf and lib/test_bpf.c code without verifier, introduce helper __bpf_prog_select_runtime() to accept the env parameter. Remove the call to bpf_prog_select_runtime() in bpf_prog_load(), and switch to call __bpf_prog_select_runtime() in the verifier, with env variable passed. The original bpf_prog_select_runtime() is preserved for cbpf and lib/test_bpf.c, where env is NULL. Now all constants blinding calls are moved into the verifier, except the cbpf and lib/test_bpf.c cases. The instructions arrays are adjusted by bpf_patch_insn_data() function for normal cases, so there is no need to call adjust_insn_arrays() in bpf_jit_blind_constants(). Remove it. Reviewed-by: Anton Protopopov Signed-off-by: Xu Kuohai --- arch/arc/net/bpf_jit_core.c | 2 +- arch/arm/net/bpf_jit_32.c | 2 +- arch/arm64/net/bpf_jit_comp.c | 2 +- arch/loongarch/net/bpf_jit.c | 2 +- arch/mips/net/bpf_jit_comp.c | 2 +- arch/parisc/net/bpf_jit_core.c | 2 +- arch/powerpc/net/bpf_jit_comp.c | 2 +- arch/riscv/net/bpf_jit_core.c | 2 +- arch/s390/net/bpf_jit_comp.c | 2 +- arch/sparc/net/bpf_jit_comp_64.c | 2 +- arch/x86/net/bpf_jit_comp.c | 2 +- arch/x86/net/bpf_jit_comp32.c | 2 +- include/linux/filter.h | 17 +++++- kernel/bpf/core.c | 93 +++++++++++++++++--------------- kernel/bpf/syscall.c | 4 -- kernel/bpf/verifier.c | 36 +++++++------ 16 files changed, 98 insertions(+), 76 deletions(-) diff --git a/arch/arc/net/bpf_jit_core.c b/arch/arc/net/bpf_jit_core.c index 973ceae48675..639a2736f029 100644 --- a/arch/arc/net/bpf_jit_core.c +++ b/arch/arc/net/bpf_jit_core.c @@ -1400,7 +1400,7 @@ static struct bpf_prog *do_extra_pass(struct bpf_prog *prog) * (re)locations involved that their addresses are not known * during the first run. */ -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) { vm_dump(prog); diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index e6b1bb2de627..1628b6fc70a4 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -2142,7 +2142,7 @@ bool bpf_jit_needs_zext(void) return true; } -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) { struct bpf_binary_header *header; struct jit_ctx ctx; diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index cd5a72fff500..7212ec89dfe3 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -2006,7 +2006,7 @@ struct arm64_jit_data { struct jit_ctx ctx; }; -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) { int image_size, prog_size, extable_size, extable_align, extable_offset; struct bpf_binary_header *header; diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index fcc8c0c29fb0..5149ce4cef7e 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -1920,7 +1920,7 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, return ret < 0 ? ret : ret * LOONGARCH_INSN_SIZE; } -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) { bool extra_pass = false; u8 *image_ptr, *ro_image_ptr; diff --git a/arch/mips/net/bpf_jit_comp.c b/arch/mips/net/bpf_jit_comp.c index d2b6c955f18e..6ee4abe6a1f7 100644 --- a/arch/mips/net/bpf_jit_comp.c +++ b/arch/mips/net/bpf_jit_comp.c @@ -909,7 +909,7 @@ bool bpf_jit_needs_zext(void) return true; } -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) { struct bpf_binary_header *header = NULL; struct jit_context ctx; diff --git a/arch/parisc/net/bpf_jit_core.c b/arch/parisc/net/bpf_jit_core.c index 35dca372b5df..172770132440 100644 --- a/arch/parisc/net/bpf_jit_core.c +++ b/arch/parisc/net/bpf_jit_core.c @@ -41,7 +41,7 @@ bool bpf_jit_needs_zext(void) return true; } -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) { unsigned int prog_size = 0, extable_size = 0; bool extra_pass = false; diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 711028bebea3..27fecb4cc063 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -129,7 +129,7 @@ bool bpf_jit_needs_zext(void) return true; } -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp) { u32 proglen; u32 alloclen; diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index 527baa50dc68..768ac686b359 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -41,7 +41,7 @@ bool bpf_jit_needs_zext(void) return true; } -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) { unsigned int prog_size = 0, extable_size = 0; bool extra_pass = false; diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index 2dfc279b1be2..94128fe6be23 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -2312,7 +2312,7 @@ static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit, /* * Compile eBPF program "fp" */ -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp) { struct bpf_binary_header *header; struct s390_jit_data *jit_data; diff --git a/arch/sparc/net/bpf_jit_comp_64.c b/arch/sparc/net/bpf_jit_comp_64.c index e83e29137566..2fa0e9375127 100644 --- a/arch/sparc/net/bpf_jit_comp_64.c +++ b/arch/sparc/net/bpf_jit_comp_64.c @@ -1477,7 +1477,7 @@ struct sparc64_jit_data { struct jit_ctx ctx; }; -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) { struct sparc64_jit_data *jit_data; struct bpf_binary_header *header; diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 77d00a8dec87..72d9a5faa230 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -3713,7 +3713,7 @@ struct x64_jit_data { #define MAX_PASSES 20 #define PADDING_PASSES (MAX_PASSES - 5) -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) { struct bpf_binary_header *rw_header = NULL; struct bpf_binary_header *header = NULL; diff --git a/arch/x86/net/bpf_jit_comp32.c b/arch/x86/net/bpf_jit_comp32.c index 5f259577614a..852baf2e4db4 100644 --- a/arch/x86/net/bpf_jit_comp32.c +++ b/arch/x86/net/bpf_jit_comp32.c @@ -2518,7 +2518,7 @@ bool bpf_jit_needs_zext(void) return true; } -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) { struct bpf_binary_header *header = NULL; int proglen, oldproglen = 0; diff --git a/include/linux/filter.h b/include/linux/filter.h index d396e55c9a1d..83f37d38c5c1 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1107,6 +1107,8 @@ static inline int sk_filter_reason(struct sock *sk, struct sk_buff *skb, return sk_filter_trim_cap(sk, skb, 1, reason); } +struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct bpf_prog *fp, + int *err); struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err); void bpf_prog_free(struct bpf_prog *fp); @@ -1152,7 +1154,7 @@ u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); ((u64 (*)(u64, u64, u64, u64, u64, const struct bpf_insn *)) \ (void *)__bpf_call_base) -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog); +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog); void bpf_jit_compile(struct bpf_prog *prog); bool bpf_jit_needs_zext(void); bool bpf_jit_inlines_helper_call(s32 imm); @@ -1187,12 +1189,25 @@ struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off, #ifdef CONFIG_BPF_SYSCALL struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, const struct bpf_insn *patch, u32 len); +struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env); +void bpf_restore_insn_aux_data(struct bpf_verifier_env *env, + struct bpf_insn_aux_data *orig_insn_aux); #else static inline struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, const struct bpf_insn *patch, u32 len) { return ERR_PTR(-ENOTSUPP); } + +static inline struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env) +{ + return NULL; +} + +static inline void bpf_restore_insn_aux_data(struct bpf_verifier_env *env, + struct bpf_insn_aux_data *orig_insn_aux) +{ +} #endif /* CONFIG_BPF_SYSCALL */ int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index cc61fe57b98d..093ab0f68c81 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -1489,23 +1489,6 @@ void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other) bpf_prog_clone_free(fp_other); } -static void adjust_insn_arrays(struct bpf_prog *prog, u32 off, u32 len) -{ -#ifdef CONFIG_BPF_SYSCALL - struct bpf_map *map; - int i; - - if (len <= 1) - return; - - for (i = 0; i < prog->aux->used_map_cnt; i++) { - map = prog->aux->used_maps[i]; - if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY) - bpf_insn_array_adjust(map, off, len); - } -#endif -} - /* Now this function is used only to blind the main prog and must be invoked only when * bpf_prog_need_blind() returns true. */ @@ -1577,12 +1560,6 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bp if (env) env->prog = clone; - else - /* Instructions arrays must be updated using absolute xlated offsets. - * The arrays have already been adjusted by bpf_patch_insn_data() when - * env is not NULL. - */ - adjust_insn_arrays(clone, i, rewritten); /* Walk new program and skip insns we just inserted. */ insn = clone->insnsi + i + insn_delta; @@ -2551,47 +2528,63 @@ static bool bpf_prog_select_interpreter(struct bpf_prog *fp) return select_interpreter; } -static struct bpf_prog *bpf_prog_jit_compile(struct bpf_prog *prog) +static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) { #ifdef CONFIG_BPF_JIT bool blinded = false; struct bpf_prog *orig_prog = prog; + struct bpf_insn_aux_data *orig_insn_aux; if (bpf_prog_need_blind(orig_prog)) { - prog = bpf_jit_blind_constants(NULL, orig_prog); + if (env) { + /* If env is not NULL, we are called from the end of bpf_check(), at this + * point, only insn_aux_data is used after failure, so we only restore it + * here. + */ + orig_insn_aux = bpf_dup_insn_aux_data(env); + if (!orig_insn_aux) + return orig_prog; + } + prog = bpf_jit_blind_constants(env, orig_prog); /* If blinding was requested and we failed during blinding, we must fall * back to the interpreter. */ - if (IS_ERR(prog)) - return orig_prog; + if (IS_ERR(prog)) { + prog = orig_prog; + if (env) + goto out_restore; + else + return prog; + } blinded = true; } - prog = bpf_int_jit_compile(prog); + prog = bpf_int_jit_compile(env, prog); if (blinded) { if (!prog->jited) { bpf_jit_prog_release_other(orig_prog, prog); prog = orig_prog; + if (env) + goto out_restore; } else { bpf_jit_prog_release_other(prog, orig_prog); + if (env) + goto out_free; } } + + return prog; + +out_restore: + bpf_restore_insn_aux_data(env, orig_insn_aux); +out_free: + kvfree(orig_insn_aux); #endif return prog; } -/** - * bpf_prog_select_runtime - select exec runtime for BPF program - * @fp: bpf_prog populated with BPF program - * @err: pointer to error variable - * - * Try to JIT eBPF program, if JIT is not available, use interpreter. - * The BPF program will be executed via bpf_prog_run() function. - * - * Return: the &fp argument along with &err set to 0 for success or - * a negative errno code on failure - */ -struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) +struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct bpf_prog *fp, + int *err) { /* In case of BPF to BPF calls, verifier did all the prep * work with regards to JITing, etc. @@ -2619,7 +2612,7 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) if (*err) return fp; - fp = bpf_prog_jit_compile(fp); + fp = bpf_prog_jit_compile(env, fp); bpf_prog_jit_attempt_done(fp); if (!fp->jited && jit_needed) { *err = -ENOTSUPP; @@ -2645,6 +2638,22 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) return fp; } + +/** + * bpf_prog_select_runtime - select exec runtime for BPF program + * @fp: bpf_prog populated with BPF program + * @err: pointer to error variable + * + * Try to JIT eBPF program, if JIT is not available, use interpreter. + * The BPF program will be executed via bpf_prog_run() function. + * + * Return: the &fp argument along with &err set to 0 for success or + * a negative errno code on failure + */ +struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) +{ + return __bpf_prog_select_runtime(NULL, fp, err); +} EXPORT_SYMBOL_GPL(bpf_prog_select_runtime); static unsigned int __bpf_prog_ret1(const void *ctx, @@ -3132,7 +3141,7 @@ const struct bpf_func_proto bpf_tail_call_proto = { * It is encouraged to implement bpf_int_jit_compile() instead, so that * eBPF and implicitly also cBPF can get JITed! */ -struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog) +struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) { return prog; } diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index e1505c9cd09e..553dca175640 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -3090,10 +3090,6 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) if (err < 0) goto free_used_maps; - prog = bpf_prog_select_runtime(prog, &err); - if (err < 0) - goto free_used_maps; - err = bpf_prog_mark_insn_arrays_ready(prog); if (err < 0) goto free_used_maps; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 66cef3744fde..5084a754a748 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -22983,7 +22983,7 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env) return 0; } -static u32 *dup_subprog_starts(struct bpf_verifier_env *env) +static u32 *bpf_dup_subprog_starts(struct bpf_verifier_env *env) { u32 *starts = NULL; @@ -22995,13 +22995,13 @@ static u32 *dup_subprog_starts(struct bpf_verifier_env *env) return starts; } -static void restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_starts) +static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_starts) { for (int i = 0; i < env->subprog_cnt; i++) env->subprog_info[i].start = orig_starts[i]; } -static struct bpf_insn_aux_data *dup_insn_aux_data(struct bpf_verifier_env *env) +struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env) { size_t size; @@ -23009,8 +23009,8 @@ static struct bpf_insn_aux_data *dup_insn_aux_data(struct bpf_verifier_env *env) return kvmemdup(env->insn_aux_data, size, GFP_KERNEL_ACCOUNT); } -static void restore_insn_aux_data(struct bpf_verifier_env *env, - struct bpf_insn_aux_data *orig_insn_aux) +void bpf_restore_insn_aux_data(struct bpf_verifier_env *env, + struct bpf_insn_aux_data *orig_insn_aux) { /* the expanded elements are zero-filled, so no special handling is required */ vfree(env->insn_aux_data); @@ -23153,7 +23153,7 @@ static int __jit_subprogs(struct bpf_verifier_env *env) func[i]->aux->might_sleep = env->subprog_info[i].might_sleep; if (!i) func[i]->aux->exception_boundary = env->seen_exception; - func[i] = bpf_int_jit_compile(func[i]); + func[i] = bpf_int_jit_compile(env, func[i]); if (!func[i]->jited) { err = -ENOTSUPP; goto out_free; @@ -23197,7 +23197,7 @@ static int __jit_subprogs(struct bpf_verifier_env *env) } for (i = 0; i < env->subprog_cnt; i++) { old_bpf_func = func[i]->bpf_func; - tmp = bpf_int_jit_compile(func[i]); + tmp = bpf_int_jit_compile(env, func[i]); if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) { verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); err = -ENOTSUPP; @@ -23297,12 +23297,12 @@ static int jit_subprogs(struct bpf_verifier_env *env) prog = orig_prog = env->prog; if (bpf_prog_need_blind(orig_prog)) { - orig_insn_aux = dup_insn_aux_data(env); + orig_insn_aux = bpf_dup_insn_aux_data(env); if (!orig_insn_aux) { err = -ENOMEM; goto out_cleanup; } - orig_subprog_starts = dup_subprog_starts(env); + orig_subprog_starts = bpf_dup_subprog_starts(env); if (!orig_subprog_starts) { err = -ENOMEM; goto out_free_aux; @@ -23347,8 +23347,8 @@ static int jit_subprogs(struct bpf_verifier_env *env) return 0; out_restore: - restore_subprog_starts(env, orig_subprog_starts); - restore_insn_aux_data(env, orig_insn_aux); + bpf_restore_subprog_starts(env, orig_subprog_starts); + bpf_restore_insn_aux_data(env, orig_insn_aux); kvfree(orig_subprog_starts); out_free_aux: kvfree(orig_insn_aux); @@ -26523,6 +26523,14 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3 adjust_btf_func(env); + /* extension progs temporarily inherit the attach_type of their targets + for verification purposes, so set it back to zero before returning + */ + if (env->prog->type == BPF_PROG_TYPE_EXT) + env->prog->expected_attach_type = 0; + + env->prog = __bpf_prog_select_runtime(env, env->prog, &ret); + err_release_maps: if (ret) release_insn_arrays(env); @@ -26534,12 +26542,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3 if (!env->prog->aux->used_btfs) release_btfs(env); - /* extension progs temporarily inherit the attach_type of their targets - for verification purposes, so set it back to zero before returning - */ - if (env->prog->type == BPF_PROG_TYPE_EXT) - env->prog->expected_attach_type = 0; - *prog = env->prog; module_put(env->attach_btf_mod); -- 2.43.0