From: Tejun Heo Implement the struct_ops arena argument conversion on x86. save_args() gains the conversion map from bpf_tramp_collect_arena_args() and, as it copies each native argument into the BPF ctx, routes a marked slot through RAX: movl %esrc, %eax /* truncate and clear the upper 32 bits */ subl $base_lo, %eax movq %rax, ctx_slot A nullable slot tests the full 64-bit kernel pointer first: movq %rsrc, %rax testq %rax, %rax jz 1f subl $base_lo, %eax 1: movq %rax, ctx_slot The 32-bit subtraction is sufficient since (u32)(kaddr - base) == (u32)kaddr - (u32)base, and it clears the upper half as the JITs require of arena pointer registers. Stack-passed arguments already reload through RAX, so only the subtraction (and the NULL test) is inserted there. The marked slots are tracked with a running slot counter shared by the register and stack branches, matching the flattened ctx offsets in ctx_arg_info. The size probe reruns the same emission with the same tnodes, so the image size matches by construction. With both the kfunc and struct_ops directions implemented, flip bpf_jit_supports_arena_args() on for x86. Signed-off-by: Tejun Heo Signed-off-by: Kumar Kartikeya Dwivedi --- arch/x86/net/bpf_jit_comp.c | 64 +++++++++++++++++++++++++++++++++---- 1 file changed, 57 insertions(+), 7 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 08013c2fcfaa..dfb5335ad837 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -3043,12 +3043,39 @@ static int get_nr_used_regs(const struct btf_func_model *m) return nr_used_regs; } +/* + * Convert an arena kernel address into the arena pointer form on its way + * into the BPF ctx, rax = (u32)(src - kern_vm_start). A nullable arg + * preserves NULL, tested on the full 64-bit kernel pointer. The 32-bit + * subtraction both truncates and clears the upper half, so the stored + * value satisfies the JIT invariant for arena pointer registers. + */ +static void emit_arena_arg_conv(u8 **pprog, u32 src_reg, bool nullable, u32 base_lo) +{ + u8 *prog = *pprog; + + if (nullable) { + if (src_reg != BPF_REG_0) + emit_mov_reg(&prog, true, BPF_REG_0, src_reg); + /* test rax, rax; jz over the 5-byte sub */ + EMIT3(0x48, 0x85, 0xC0); + EMIT2(X86_JE, 5); + } else if (src_reg != BPF_REG_0) { + emit_mov_reg(&prog, false, BPF_REG_0, src_reg); + } + /* sub eax, base_lo */ + EMIT1_off32(0x2D, base_lo); + + *pprog = prog; +} + static void save_args(const struct btf_func_model *m, u8 **prog, - int stack_size, bool for_call_origin, u32 flags) + int stack_size, bool for_call_origin, u32 flags, + const struct bpf_tramp_arena_args *aargs) { int arg_regs, first_off = 0, nr_regs = 0, nr_stack_slots = 0; bool use_jmp = bpf_trampoline_use_jmp(flags); - int i, j; + int i, j, slot = 0; /* Store function arguments to stack. * For a function that accepts two pointers the sequence will be: @@ -3089,6 +3116,10 @@ static void save_args(const struct btf_func_model *m, u8 **prog, for (j = 0; j < arg_regs; j++) { emit_ldx(prog, BPF_DW, BPF_REG_0, BPF_REG_FP, nr_stack_slots * 8 + 16 + (!use_jmp) * 8); + if (aargs && (aargs->slots & BIT(slot))) + emit_arena_arg_conv(prog, BPF_REG_0, + aargs->nullable_slots & BIT(slot), + (u32)aargs->kern_vm_start); emit_stx(prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -stack_size); @@ -3096,6 +3127,7 @@ static void save_args(const struct btf_func_model *m, u8 **prog, first_off = stack_size; stack_size -= 8; nr_stack_slots++; + slot++; } } else { /* Only copy the arguments on-stack to current @@ -3104,16 +3136,24 @@ static void save_args(const struct btf_func_model *m, u8 **prog, */ if (for_call_origin) { nr_regs += arg_regs; + slot += arg_regs; continue; } /* copy the arguments from regs into stack */ for (j = 0; j < arg_regs; j++) { - emit_stx(prog, BPF_DW, BPF_REG_FP, - nr_regs == 5 ? X86_REG_R9 : BPF_REG_1 + nr_regs, - -stack_size); + u32 src = nr_regs == 5 ? X86_REG_R9 : BPF_REG_1 + nr_regs; + + if (aargs && (aargs->slots & BIT(slot))) { + emit_arena_arg_conv(prog, src, + aargs->nullable_slots & BIT(slot), + (u32)aargs->kern_vm_start); + src = BPF_REG_0; + } + emit_stx(prog, BPF_DW, BPF_REG_FP, src, -stack_size); stack_size -= 8; nr_regs++; + slot++; } } } @@ -3404,11 +3444,13 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im struct bpf_tramp_nodes *fentry = &tnodes[BPF_TRAMP_FENTRY]; struct bpf_tramp_nodes *fexit = &tnodes[BPF_TRAMP_FEXIT]; struct bpf_tramp_nodes *fmod_ret = &tnodes[BPF_TRAMP_MODIFY_RETURN]; + struct bpf_tramp_arena_args aargs; void *orig_call = func_addr; int cookie_off, cookie_cnt; u8 **branches = NULL; u64 func_meta; u8 *prog; + bool has_aargs; bool save_ret; /* @@ -3419,6 +3461,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im WARN_ON_ONCE((flags & BPF_TRAMP_F_INDIRECT) && (flags & ~(BPF_TRAMP_F_INDIRECT | BPF_TRAMP_F_RET_FENTRY_RET))); + has_aargs = bpf_tramp_collect_arena_args(tnodes, flags, &aargs); + /* extra registers for struct arguments */ for (i = 0; i < m->nr_args; i++) { if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) @@ -3556,7 +3600,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im emit_store_stack_imm64(&prog, BPF_REG_0, -ip_off, (long)func_addr); } - save_args(m, &prog, regs_off, false, flags); + save_args(m, &prog, regs_off, false, flags, + has_aargs ? &aargs : NULL); if (flags & BPF_TRAMP_F_CALL_ORIG) { /* arg1: mov rdi, im */ @@ -3598,7 +3643,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im if (flags & BPF_TRAMP_F_CALL_ORIG) { restore_regs(m, &prog, regs_off); - save_args(m, &prog, arg_stack_off, true, flags); + save_args(m, &prog, arg_stack_off, true, flags, NULL); if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) { /* Before calling the original function, load the @@ -4099,6 +4144,11 @@ bool bpf_jit_supports_stack_args(void) return true; } +bool bpf_jit_supports_arena_args(void) +{ + return true; +} + void *bpf_arch_text_copy(void *dst, void *src, size_t len) { if (text_poke_copy(dst, src, len) == NULL) -- 2.53.0