Currently, LoongArch bpf trampoline supports up to 8 function arguments. According to the statistics from commit 473e3150e30a ("bpf, x86: allow function arguments up to 12 for TRACING"), there are over 200 functions accept 9 to 12 arguments, so add 12 arguments support for trampoline. The initial aim is to pass the following related testcases: sudo ./test_progs -a tracing_struct/struct_many_args sudo ./test_progs -a fentry_test/fentry_many_args sudo ./test_progs -a fexit_test/fexit_many_args but there exist some other problems now, maybe it is related with the following failed testcase: sudo ./test_progs -t module_attach Signed-off-by: Tiezhu Yang --- arch/loongarch/net/bpf_jit.c | 57 ++++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 12 deletions(-) diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 7bd4b132755a..805fa6dba2d8 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -1340,26 +1340,48 @@ int bpf_arch_text_invalidate(void *dst, size_t len) return ret; } -static void store_args(struct jit_ctx *ctx, int nregs, int args_off) +static void store_args(struct jit_ctx *ctx, int nr_arg_slots, int args_off) { int i; - for (i = 0; i < nregs; i++) { - emit_insn(ctx, std, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off); + for (i = 0; i < nr_arg_slots; i++) { + if (i < LOONGARCH_MAX_REG_ARGS) { + emit_insn(ctx, std, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off); + } else { + /* skip slots for T0 and FP of traced function */ + emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, + 16 + (i - LOONGARCH_MAX_REG_ARGS) * 8); + emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -args_off); + } args_off -= 8; } } -static void restore_args(struct jit_ctx *ctx, int nregs, int args_off) +static void restore_args(struct jit_ctx *ctx, int nr_reg_args, int args_off) { int i; - for (i = 0; i < nregs; i++) { + for (i = 0; i < nr_reg_args; i++) { emit_insn(ctx, ldd, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off); args_off -= 8; } } +static void restore_stk_args(struct jit_ctx *ctx, int nr_stk_args, + int args_off, int stk_arg_off) +{ + int i; + + for (i = 0; i < nr_stk_args; i++) { + emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, + -(args_off - LOONGARCH_MAX_REG_ARGS * 8)); + emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, + -stk_arg_off); + args_off -= 8; + stk_arg_off -= 8; + } +} + static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_link *l, const struct btf_func_model *m, int args_off, int retval_off, int run_ctx_off, bool save_ret) @@ -1477,7 +1499,7 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i void *func_addr, u32 flags) { int i, ret, save_ret; - int stack_size = 0, nregs = m->nr_args; + int stack_size = 0, nr_arg_slots = 0, stk_arg_off; int retval_off, args_off, nregs_off, ip_off, run_ctx_off, sreg_off, tcc_ptr_off; bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT; void *orig_call = func_addr; @@ -1511,9 +1533,13 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i * FP - sreg_off [ callee saved reg ] * * FP - tcc_ptr_off [ tail_call_cnt_ptr ] + * + * [ stack_argN ] + * [ ... ] + * FP - stk_arg_off [ stack_arg1 ] BPF_TRAMP_F_CALL_ORIG */ - if (m->nr_args > LOONGARCH_MAX_REG_ARGS) + if (m->nr_args > MAX_BPF_FUNC_ARGS) return -ENOTSUPP; /* extra regiters for struct arguments */ @@ -1544,7 +1570,7 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i } /* Room of trampoline frame to store args */ - stack_size += nregs * 8; + stack_size += nr_arg_slots * 8; args_off = stack_size; /* Room of trampoline frame to store args number */ @@ -1570,8 +1596,14 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i tcc_ptr_off = stack_size; } + if ((flags & BPF_TRAMP_F_CALL_ORIG) && (nr_arg_slots - LOONGARCH_MAX_REG_ARGS > 0)) + stack_size += (nr_arg_slots - LOONGARCH_MAX_REG_ARGS) * 8; + stack_size = round_up(stack_size, 16); + /* Room for args on stack must be at the top of stack */ + stk_arg_off = stack_size; + if (is_struct_ops) { /* * For the trampoline called directly, just handle @@ -1613,10 +1645,10 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i } /* store arg regs count */ - move_imm(ctx, LOONGARCH_GPR_T1, nregs, false); + move_imm(ctx, LOONGARCH_GPR_T1, nr_arg_slots, false); emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -nregs_off); - store_args(ctx, nregs, args_off); + store_args(ctx, nr_arg_slots, args_off); /* To traced function */ /* Ftrace jump skips 2 NOP instructions */ @@ -1648,7 +1680,8 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i } if (flags & BPF_TRAMP_F_CALL_ORIG) { - restore_args(ctx, nregs, args_off); + restore_args(ctx, min_t(int, nr_arg_slots, LOONGARCH_MAX_REG_ARGS), args_off); + restore_stk_args(ctx, nr_arg_slots - LOONGARCH_MAX_REG_ARGS, args_off, stk_arg_off); if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off); @@ -1685,7 +1718,7 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i } if (flags & BPF_TRAMP_F_RESTORE_REGS) - restore_args(ctx, nregs, args_off); + restore_args(ctx, min_t(int, nr_arg_slots, LOONGARCH_MAX_REG_ARGS), args_off); if (save_ret) { emit_insn(ctx, ldd, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off); -- 2.42.0