Create callsites tables and store jit indexes of RET_NULL calls to poke them later with dummy functions. Additional to jit indexes, meta data about helpers/kfuncs/loops is stored. Later this could be extended to remaining potential long running iterator helpers/kfuncs. Signed-off-by: Raj Sahu Signed-off-by: Siddharth Chintamaneni --- arch/x86/net/bpf_jit_comp.c | 9 +++ include/linux/bpf_verifier.h | 1 + kernel/bpf/core.c | 5 +- kernel/bpf/verifier.c | 135 +++++++++++++++++++++++++++++++---- 4 files changed, 137 insertions(+), 13 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 7e3fca164620..107a44729675 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -3733,6 +3733,15 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) } if (!image || !prog->is_func || extra_pass) { + + if (addrs) { + struct bpf_term_patch_call_sites *patch_call_sites = prog->term_states->patch_call_sites; + for (int i = 0; i < patch_call_sites->call_sites_cnt; i++) { + struct call_aux_states *call_states = patch_call_sites->call_states + i; + call_states->jit_call_idx = addrs[call_states->call_bpf_insn_idx]; + } + } + if (image) bpf_prog_fill_jited_linfo(prog, addrs + 1); out_addrs: diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 020de62bd09c..2c8bfde8191a 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -677,6 +677,7 @@ struct bpf_subprog_info { bool is_cb: 1; bool is_async_cb: 1; bool is_exception_cb: 1; + bool is_bpf_loop_cb_non_inline: 1; bool args_cached: 1; /* true if bpf_fastcall stack region is used by functions that can't be inlined */ bool keep_fastcall_stack: 1; diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 740b5a3a6b55..93442ab2acde 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -136,6 +136,7 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag fp->term_states = term_states; fp->term_states->patch_call_sites = patch_call_sites; fp->term_states->patch_call_sites->call_sites_cnt = 0; + fp->term_states->patch_call_sites->call_states = NULL; fp->term_states->prog = fp; #ifdef CONFIG_CGROUP_BPF @@ -314,8 +315,10 @@ void __bpf_prog_free(struct bpf_prog *fp) kfree(fp->aux); } if (fp->term_states) { - if (fp->term_states->patch_call_sites) + if (fp->term_states->patch_call_sites) { + vfree(fp->term_states->patch_call_sites->call_states); kfree(fp->term_states->patch_call_sites); + } kfree(fp->term_states); } free_percpu(fp->stats); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index b9394f8fac0e..1d27208e1078 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3491,6 +3491,7 @@ static int add_subprog_and_kfunc(struct bpf_verifier_env *env) * logic. 'subprog_cnt' should not be increased. */ subprog[env->subprog_cnt].start = insn_cnt; + subprog[env->subprog_cnt].is_bpf_loop_cb_non_inline = false; if (env->log.level & BPF_LOG_LEVEL2) for (i = 0; i < env->subprog_cnt; i++) @@ -11319,19 +11320,30 @@ static bool loop_flag_is_zero(struct bpf_verifier_env *env) static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno) { struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state; + struct bpf_subprog_info *prev_info, *info = subprog_info(env, subprogno); if (!state->initialized) { state->initialized = 1; state->fit_for_inline = loop_flag_is_zero(env); state->callback_subprogno = subprogno; + if (!state->fit_for_inline) + info->is_bpf_loop_cb_non_inline = 1; return; } - if (!state->fit_for_inline) + if (!state->fit_for_inline) { + info->is_bpf_loop_cb_non_inline = 1; return; + } state->fit_for_inline = (loop_flag_is_zero(env) && state->callback_subprogno == subprogno); + + if (state->callback_subprogno != subprogno) { + info->is_bpf_loop_cb_non_inline = 1; + prev_info = subprog_info(env, state->callback_subprogno); + prev_info->is_bpf_loop_cb_non_inline = 1; + } } /* Returns whether or not the given map type can potentially elide @@ -21120,6 +21132,9 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, int i, patch_len, delta = 0, len = env->prog->len; struct bpf_insn *insns = env->prog->insnsi; struct bpf_prog *new_prog; + struct bpf_term_aux_states *term_states = env->prog->term_states; + u32 call_sites_cnt = term_states->patch_call_sites->call_sites_cnt; + struct call_aux_states *call_states = term_states->patch_call_sites->call_states; bool rnd_hi32; rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32; @@ -21205,6 +21220,15 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, insns = new_prog->insnsi; aux = env->insn_aux_data; delta += patch_len - 1; + + /* Adust call instruction offsets + * w.r.t adj_idx + */ + for (int iter = 0; iter < call_sites_cnt; iter++) { + if (call_states[iter].call_bpf_insn_idx < adj_idx) + continue; + call_states[iter].call_bpf_insn_idx += patch_len - 1; + } } return 0; @@ -21597,6 +21621,26 @@ static int jit_subprogs(struct bpf_verifier_env *env) func[i]->aux->func_info_cnt = prog->aux->func_info_cnt; func[i]->aux->poke_tab = prog->aux->poke_tab; func[i]->aux->size_poke_tab = prog->aux->size_poke_tab; + func[i]->aux->is_bpf_loop_cb_non_inline = env->subprog_info[i].is_bpf_loop_cb_non_inline; + + if (prog->term_states->patch_call_sites->call_sites_cnt != 0) { + int call_sites_cnt = 0; + struct call_aux_states *func_call_states; + func_call_states = vzalloc(sizeof(*func_call_states) * len); + if (!func_call_states) + goto out_free; + for (int iter = 0; iter < prog->term_states->patch_call_sites->call_sites_cnt; iter++) { + struct call_aux_states call_states = prog->term_states->patch_call_sites->call_states[iter]; + if (call_states.call_bpf_insn_idx >= subprog_start + && call_states.call_bpf_insn_idx < subprog_end) { + func_call_states[call_sites_cnt] = call_states; + func_call_states[call_sites_cnt].call_bpf_insn_idx -= subprog_start; + call_sites_cnt++; + } + } + func[i]->term_states->patch_call_sites->call_sites_cnt = call_sites_cnt; + func[i]->term_states->patch_call_sites->call_states = func_call_states; + } for (j = 0; j < prog->aux->size_poke_tab; j++) { struct bpf_jit_poke_descriptor *poke; @@ -21886,15 +21930,21 @@ static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux, } static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, - struct bpf_insn *insn_buf, int insn_idx, int *cnt) + struct bpf_insn *insn_buf, int insn_idx, int *cnt, int *kfunc_btf_id) { const struct bpf_kfunc_desc *desc; + struct bpf_kfunc_call_arg_meta meta; + int err; if (!insn->imm) { verbose(env, "invalid kernel function call not eliminated in verifier pass\n"); return -EINVAL; } + err = fetch_kfunc_meta(env, insn, &meta, NULL); + if (err) + return err; + *cnt = 0; /* insn->imm has the btf func_id. Replace it with an offset relative to @@ -21908,8 +21958,11 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return -EFAULT; } - if (!bpf_jit_supports_far_kfunc_call()) + if (!bpf_jit_supports_far_kfunc_call()) { + if (meta.kfunc_flags & KF_RET_NULL) + *kfunc_btf_id = insn->imm; insn->imm = BPF_CALL_IMM(desc->addr); + } if (insn->off) return 0; if (desc->func_id == special_kfunc_list[KF_bpf_obj_new_impl] || @@ -22019,6 +22072,13 @@ static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *pat return 0; } +static bool is_bpf_loop_call(struct bpf_insn *insn) +{ + return insn->code == (BPF_JMP | BPF_CALL) && + insn->src_reg == 0 && + insn->imm == BPF_FUNC_loop; +} + /* Do various post-verification rewrites in a single program pass. * These rewrites simplify JIT and interpreter implementations. */ @@ -22039,6 +22099,12 @@ static int do_misc_fixups(struct bpf_verifier_env *env) struct bpf_subprog_info *subprogs = env->subprog_info; u16 stack_depth = subprogs[cur_subprog].stack_depth; u16 stack_depth_extra = 0; + u32 call_sites_cnt = 0; + struct call_aux_states *call_states; + + call_states = vzalloc(sizeof(*call_states) * prog->len); + if (!call_states) + return -ENOMEM; if (env->seen_exception && !env->exception_callback_subprog) { struct bpf_insn *patch = insn_buf; @@ -22368,11 +22434,12 @@ static int do_misc_fixups(struct bpf_verifier_env *env) if (insn->src_reg == BPF_PSEUDO_CALL) goto next_insn; if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { - ret = fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt); + int kfunc_btf_id = 0; + ret = fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt, &kfunc_btf_id); if (ret) return ret; if (cnt == 0) - goto next_insn; + goto store_call_indices; new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) @@ -22381,6 +22448,12 @@ static int do_misc_fixups(struct bpf_verifier_env *env) delta += cnt - 1; env->prog = prog = new_prog; insn = new_prog->insnsi + i + delta; +store_call_indices: + if (kfunc_btf_id != 0) { + call_states[call_sites_cnt].call_bpf_insn_idx = i + delta; + call_states[call_sites_cnt].is_helper_kfunc = 1; + call_sites_cnt++; + } goto next_insn; } @@ -22859,6 +22932,15 @@ static int do_misc_fixups(struct bpf_verifier_env *env) func_id_name(insn->imm), insn->imm); return -EFAULT; } + + if ((fn->ret_type & PTR_MAYBE_NULL) || is_bpf_loop_call(insn)) { + call_states[call_sites_cnt].call_bpf_insn_idx = i + delta; + if (is_bpf_loop_call(insn)) + call_states[call_sites_cnt].is_bpf_loop = 1; + else + call_states[call_sites_cnt].is_helper_kfunc = 1; + call_sites_cnt++; + } insn->imm = fn->func - __bpf_call_base; next_insn: if (subprogs[cur_subprog + 1].start == i + delta + 1) { @@ -22879,6 +22961,8 @@ static int do_misc_fixups(struct bpf_verifier_env *env) insn++; } + env->prog->term_states->patch_call_sites->call_sites_cnt = call_sites_cnt; + env->prog->term_states->patch_call_sites->call_states = call_states; env->prog->aux->stack_depth = subprogs[0].stack_depth; for (i = 0; i < env->subprog_cnt; i++) { int delta = bpf_jit_supports_timed_may_goto() ? 2 : 1; @@ -23014,17 +23098,12 @@ static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, call_insn_offset = position + 12; callback_offset = callback_start - call_insn_offset - 1; new_prog->insnsi[call_insn_offset].imm = callback_offset; + /* Marking offset field to identify loop cb */ + new_prog->insnsi[call_insn_offset].off = 0x1; return new_prog; } -static bool is_bpf_loop_call(struct bpf_insn *insn) -{ - return insn->code == (BPF_JMP | BPF_CALL) && - insn->src_reg == 0 && - insn->imm == BPF_FUNC_loop; -} - /* For all sub-programs in the program (including main) check * insn_aux_data to see if there are bpf_loop calls that require * inlining. If such calls are found the calls are replaced with a @@ -24584,6 +24663,35 @@ static int compute_scc(struct bpf_verifier_env *env) return err; } +static int fix_call_sites(struct bpf_verifier_env *env) +{ + int err = 0, i, subprog; + struct bpf_insn *insn; + struct bpf_prog *prog = env->prog; + struct bpf_term_aux_states *term_states = env->prog->term_states; + u32 *call_sites_cnt = &term_states->patch_call_sites->call_sites_cnt; + struct call_aux_states *call_states = term_states->patch_call_sites->call_states; + + if (!env->subprog_cnt) + return 0; + for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { + if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn)) + continue; + + subprog = find_subprog(env, i + insn->imm + 1); + if (subprog < 0) + return -EFAULT; + + if (insn->off == 0x1) { + call_states[*call_sites_cnt].call_bpf_insn_idx = i; + call_states[*call_sites_cnt].is_bpf_loop_cb_inline = 1; + *call_sites_cnt = *call_sites_cnt + 1; + prog->insnsi[i].off = 0x0; /* Removing the marker */ + } + } + return err; +} + int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u32 uattr_size) { u64 start_time = ktime_get_ns(); @@ -24769,6 +24877,9 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3 : false; } + if (ret == 0) + ret = fix_call_sites(env); + if (ret == 0) ret = fixup_call_args(env); -- 2.43.0