bpf_iter_num_next() is called on every iteration of a bpf_for() loop and is the hot path of the numeric open-coded iterator. It only advances the on-stack iterator state and returns a pointer to it, so open-coding it in the verifier removes a function call from each loop iteration. Inline it in bpf_fixup_kfunc_call() by replacing the call with an equivalent instruction sequence. R1 holds the pointer to the on-stack bpf_iter_num; the returned pointer to s->cur is R1 itself since s->cur is the first member. s->cur and s->end are int, so the kfunc's (s64)(s->cur + 1) >= s->end test is equivalent to a signed 32-bit comparison of (s->cur + 1) against s->end: s->cur + 1 is computed as a 32-bit int in the kfunc as well, and sign-extending both sides of a comparison of two int values does not change its result. The inlined code therefore uses a 32-bit compare and needs no sign extension. Signed-off-by: Puranjay Mohan --- kernel/bpf/verifier.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ff76a1ed04556..6fb8fdd5d5b21 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -19807,6 +19807,34 @@ static int inline_bpf_iter_num_new(struct bpf_insn *insn_buf) return i; } +/* + * Inline bpf_iter_num_next(). R1 holds the pointer to the iterator. Keep in sync with the + * kfunc in kernel/bpf/bpf_iter.c. + */ +static int inline_bpf_iter_num_next(struct bpf_insn *insn_buf) +{ + int i = 0; + + /* + * s->cur and s->end are int, so the (s64)(s->cur + 1) >= s->end check is equivalent to a + * signed 32-bit comparison of (s->cur + 1) against s->end and needs no sign extension. + */ + insn_buf[i++] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1, 0); + insn_buf[i++] = BPF_ALU32_IMM(BPF_ADD, BPF_REG_0, 1); + insn_buf[i++] = BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 4); + /* if ((s32)(s->cur + 1) >= (s32)s->end) goto done; */ + insn_buf[i++] = BPF_JMP32_REG(BPF_JSGE, BPF_REG_0, BPF_REG_2, 3); + /* s->cur++; return &s->cur; */ + insn_buf[i++] = BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0); + insn_buf[i++] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1); + insn_buf[i++] = BPF_JMP_A(2); + /* done: s->cur = s->end = 0; return NULL; */ + insn_buf[i++] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0); + insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, 0); + + return i; +} + int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_insn *insn_buf, int insn_idx, int *cnt) { @@ -19938,6 +19966,8 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, *cnt = 6; } else if (desc->func_id == special_kfunc_list[KF_bpf_iter_num_new]) { *cnt = inline_bpf_iter_num_new(insn_buf); + } else if (desc->func_id == special_kfunc_list[KF_bpf_iter_num_next]) { + *cnt = inline_bpf_iter_num_next(insn_buf); } if (env->insn_aux_data[insn_idx].arg_prog) { -- 2.53.0-Meta