bpf_optimize_bpf_loop() patches the program for every bpf_loop() call that it inlines, then points the call in the patch to the callback with the start of the callback subprog after patching. Each inlining costs O(prog->len), and the call can't be pointed to the callback before the patch is applied. Queue the patches and commit them once. Set the call relative to the patch placed at the bpf_loop() call in the unpatched program, adding the growth of the patch itself for a callback behind it. Signed-off-by: Qiliang Yuan --- kernel/bpf/fixups.c | 62 +++++++++++++++++++++-------------------------------- 1 file changed, 24 insertions(+), 38 deletions(-) diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 84aaaa4632aa3..217771aca0161 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -2845,11 +2845,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) return 0; } -static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, - int position, - s32 stack_base, - u32 callback_subprogno, - u32 *total_cnt) +static int inline_bpf_loop(struct bpf_verifier_env *env, int position, + s32 stack_base, u32 callback_subprogno) { s32 r6_offset = stack_base + 0 * BPF_REG_SIZE; s32 r7_offset = stack_base + 1 * BPF_REG_SIZE; @@ -2859,9 +2856,8 @@ static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, int reg_loop_ctx = BPF_REG_8; struct bpf_insn *insn_buf = env->insn_buf; - struct bpf_prog *new_prog; u32 callback_start; - u32 call_insn_offset; + struct bpf_insn *last; s32 callback_offset; u32 cnt = 0; @@ -2887,9 +2883,7 @@ static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, * if reg_loop_cnt >= reg_loop_max skip the loop body */ insn_buf[cnt++] = BPF_JMP_REG(BPF_JGE, reg_loop_cnt, reg_loop_max, 5); - /* callback call, - * correct callback offset would be set after patching - */ + /* callback call, the offset is set below */ insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_1, reg_loop_cnt); insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_2, reg_loop_ctx); insn_buf[cnt++] = BPF_CALL_REL(0); @@ -2906,19 +2900,19 @@ static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, r7_offset); insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_10, r8_offset); - *total_cnt = cnt; - new_prog = bpf_patch_insn_data(env, position, insn_buf, cnt); - if (!new_prog) - return new_prog; - - /* callback start is known only after patching */ + /* + * Note: insn_buf[12] is the BPF_CALL_REL instruction. Jumps out of a + * patch are relative to it in place, which takes the place of 'cnt' + * insns behind 'position'. + */ callback_start = env->subprog_info[callback_subprogno].start; - /* Note: insn_buf[12] is an offset of BPF_CALL_REL instruction */ - call_insn_offset = position + 12; - callback_offset = callback_start - call_insn_offset - 1; - new_prog->insnsi[call_insn_offset].imm = callback_offset; + callback_offset = callback_start - position; + if (callback_start > position) + callback_offset += cnt - 1; + insn_buf[12].imm = callback_offset - 12 - 1; - return new_prog; + last = bpf_patch_list_add(env, position, insn_buf, cnt); + return PTR_ERR_OR_ZERO(last); } static bool is_bpf_loop_call(struct bpf_insn *insn) @@ -2940,7 +2934,7 @@ static bool is_bpf_loop_call(struct bpf_insn *insn) int bpf_optimize_bpf_loop(struct bpf_verifier_env *env) { struct bpf_subprog_info *subprogs = env->subprog_info; - int i, cur_subprog = 0, cnt, delta = 0; + int i, cur_subprog = 0, err; struct bpf_insn *insn = env->prog->insnsi; int insn_cnt = env->prog->len; u16 stack_depth = subprogs[cur_subprog].stack_depth; @@ -2949,26 +2943,18 @@ int bpf_optimize_bpf_loop(struct bpf_verifier_env *env) for (i = 0; i < insn_cnt; i++, insn++) { struct bpf_loop_inline_state *inline_state = - &env->insn_aux_data[i + delta].loop_inline_state; + &env->insn_aux_data[i].loop_inline_state; if (is_bpf_loop_call(insn) && inline_state->fit_for_inline) { - struct bpf_prog *new_prog; - stack_depth_extra = BPF_REG_SIZE * 3 + stack_depth_roundup; - new_prog = inline_bpf_loop(env, - i + delta, - -(stack_depth + stack_depth_extra), - inline_state->callback_subprogno, - &cnt); - if (!new_prog) - return -ENOMEM; - - delta += cnt - 1; - env->prog = new_prog; - insn = new_prog->insnsi + i + delta; + err = inline_bpf_loop(env, i, + -(stack_depth + stack_depth_extra), + inline_state->callback_subprogno); + if (err) + return err; } - if (subprogs[cur_subprog + 1].start == i + delta + 1) { + if (subprogs[cur_subprog + 1].start == i + 1) { subprogs[cur_subprog].stack_depth += stack_depth_extra; cur_subprog++; stack_depth = subprogs[cur_subprog].stack_depth; @@ -2979,7 +2965,7 @@ int bpf_optimize_bpf_loop(struct bpf_verifier_env *env) env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; - return 0; + return bpf_patch_list_commit(env); } /* Remove unnecessary spill/fill pairs, members of fastcall pattern, -- 2.43.0