From: Tejun Heo Implement arena argument rebasing for kfunc calls on x86. R12 already holds kern_vm_start whenever the prog has an arena, so each tagged argument costs two instructions emitted right before the call: movl %eN, %eN /* truncate, clear the upper 32 bits */ addq %r12, %rN A nullable argument tests the truncated value and jumps over the add: movl %eN, %eN testl %eN, %eN jz 1f addq %r12, %rN 1: addq carries a REX prefix for every argument register and is always three bytes, so the jz displacement is constant. The sequence is native code generated after constant blinding has run on the BPF instruction stream, so blinding never sees the rebase and needs no special handling. bpf_jit_supports_arena_args() is not flipped yet; that happens when the struct_ops trampoline side is in place as well. Signed-off-by: Tejun Heo Signed-off-by: Kumar Kartikeya Dwivedi --- arch/x86/net/bpf_jit_comp.c | 50 +++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index b2feec81e231..08013c2fcfaa 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1678,6 +1678,50 @@ static int emit_spectre_bhb_barrier(u8 **pprog, u8 *ip, return 0; } +/* + * Rebase the __arena args of a kfunc call to arena kernel addresses, + * rN = kern_vm_start + (u32)rN, with R12 holding kern_vm_start. A nullable + * arg preserves NULL by skipping the add, tested on the truncated value as + * arena NULL is offset 0. Return the number of emitted bytes. + */ +static int emit_kfunc_arena_args(struct bpf_prog *bpf_prog, + const struct bpf_insn *insn, u8 **pprog) +{ + const struct btf_func_model *fm; + u8 *prog = *pprog; + u8 *start = prog; + int i; + + fm = bpf_jit_find_kfunc_model(bpf_prog, insn); + if (!fm) + return -EINVAL; + + for (i = 0; i < min_t(int, fm->nr_args, MAX_BPF_FUNC_REG_ARGS); i++) { + u8 flags = fm->arg_flags[i]; + u32 reg = BPF_REG_1 + i; + + if (!(flags & BTF_FMODEL_ARENA_ARG)) + continue; + if (WARN_ON_ONCE(!bpf_prog->aux->arena)) + return -EINVAL; + + /* mov eN, eN: truncate and clear the upper 32 bits */ + emit_mov_reg(&prog, false, reg, reg); + if (flags & BTF_FMODEL_NULLABLE_ARG) { + /* test eN, eN; jz over the 3-byte add */ + maybe_emit_mod(&prog, reg, reg, false); + EMIT2(0x85, add_2reg(0xC0, reg, reg)); + EMIT2(X86_JE, 3); + } + /* add rN, r12 */ + maybe_emit_mod(&prog, reg, X86_REG_R12, true); + EMIT2(0x01, add_2reg(0xC0, reg, X86_REG_R12)); + } + + *pprog = prog; + return prog - start; +} + static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image, int oldproglen, struct jit_context *ctx, bool jmp_padding) { @@ -2583,6 +2627,12 @@ st: insn_off = insn->off; } if (!imm32) return -EINVAL; + if (src_reg == BPF_PSEUDO_KFUNC_CALL) { + err = emit_kfunc_arena_args(bpf_prog, insn, &prog); + if (err < 0) + return err; + ip += err; + } if (priv_frame_ptr) { push_r9(&prog); ip += 2; -- 2.53.0