From: Feng Jiang Implement bpf_jit_supports_stack_args() on RV64 JIT to allow bpf subprogs and kfuncs to pass and receive more than 5 arguments. In the riscv abi, the first 8 arguments are passed in registers a0 to a7, and arguments 9+ reside on the stack. To align bpf stack arguments with this calling convention and unify bpf2bpf calls with kfuncs, map the first 3 bpf stack arguments (6th to 8th) directly to a5 to a7, and store or load the remaining arguments (9th+) at SP or FP. Reserve outgoing stack space in the prologue accordingly when stack_arg_cnt exceeds 3. In addition, update kfunc argument sign extension to handle all 8 register arguments as well as any arguments passed on the stack. Co-developed-by: Pu Lehui Signed-off-by: Feng Jiang Signed-off-by: Pu Lehui --- arch/riscv/net/bpf_jit.h | 1 + arch/riscv/net/bpf_jit_comp64.c | 79 +++++++++++++++++++++++++++++---- arch/riscv/net/bpf_jit_core.c | 6 +++ 3 files changed, 77 insertions(+), 9 deletions(-) diff --git a/arch/riscv/net/bpf_jit.h b/arch/riscv/net/bpf_jit.h index 419b9d795f2a..039877f286fc 100644 --- a/arch/riscv/net/bpf_jit.h +++ b/arch/riscv/net/bpf_jit.h @@ -82,6 +82,7 @@ struct rv_jit_context { unsigned long flags; int stack_size; int tcc_offset; + int stack_arg_sz; u64 arena_vm_start; u64 user_vm_start; }; diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index aac128b9f0a4..cda99c1f9ffe 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -498,6 +498,18 @@ static void emit_ldx(u8 rd, s16 off, u8 rs, u8 size, bool sign_ext, ctx->ex_jmp_off = ctx->ninsns; } +static void emit_stack_arg_ldx(u8 rd, s16 off, struct rv_jit_context *ctx) +{ + int idx = off / 8 - 1; + + if (idx < 3) { + emit_mv(rd, RV_REG_A5 + idx, ctx); + return; + } + + emit_ldx_insn(rd, (idx - 3) * 8, RV_REG_FP, BPF_DW, false, ctx); +} + static void emit_st(u8 rd, s16 off, s32 imm, u8 size, struct rv_jit_context *ctx) { emit_imm(RV_REG_T1, imm, ctx); @@ -515,6 +527,19 @@ static void emit_st(u8 rd, s16 off, s32 imm, u8 size, struct rv_jit_context *ctx ctx->ex_jmp_off = ctx->ninsns; } +static void emit_stack_arg_st(s16 off, s32 imm, struct rv_jit_context *ctx) +{ + int idx = -off / 8 - 1; + + emit_imm(RV_REG_T1, imm, ctx); + if (idx < 3) { + emit_mv(RV_REG_A5 + idx, RV_REG_T1, ctx); + return; + } + + emit_stx_insn(RV_REG_SP, (idx - 3) * 8, RV_REG_T1, BPF_DW, ctx); +} + static void emit_stx(u8 rd, s16 off, u8 rs, u8 size, struct rv_jit_context *ctx) { if (is_12b_int(off)) { @@ -531,6 +556,18 @@ static void emit_stx(u8 rd, s16 off, u8 rs, u8 size, struct rv_jit_context *ctx) ctx->ex_jmp_off = ctx->ninsns; } +static void emit_stack_arg_stx(s16 off, u8 rs, struct rv_jit_context *ctx) +{ + int idx = -off / 8 - 1; + + if (idx < 3) { + emit_mv(RV_REG_A5 + idx, rs, ctx); + return; + } + + emit_stx_insn(RV_REG_SP, (idx - 3) * 8, rs, BPF_DW, ctx); +} + static int emit_atomic_ld_st(u8 rd, u8 rs, const struct bpf_insn *insn, struct rv_jit_context *ctx) { @@ -1824,11 +1861,21 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, return -EINVAL; for (idx = 0; idx < fm->nr_args; idx++) { - u8 reg = bpf_to_rv_reg(BPF_REG_1 + idx, ctx); bool sign = fm->arg_flags[idx] & BTF_FMODEL_SIGNED_ARG; - - if (sign_extend(reg, reg, fm->arg_size[idx], sign, ctx)) - return -EINVAL; + u8 arg_sz = fm->arg_size[idx]; + + if (arg_sz == 8 || (arg_sz != 4 && !sign)) + continue; + + if (idx < RV_MAX_REG_ARGS) { + if (sign_extend(RV_REG_A0 + idx, RV_REG_A0 + idx, arg_sz, sign, ctx)) + return -EINVAL; + } else { + emit_ld(RV_REG_T1, (idx - RV_MAX_REG_ARGS) * 8, RV_REG_SP, ctx); + if (sign_extend(RV_REG_T1, RV_REG_T1, arg_sz, sign, ctx)) + return -EINVAL; + emit_sd(RV_REG_SP, (idx - RV_MAX_REG_ARGS) * 8, RV_REG_T1, ctx); + } } } @@ -1927,7 +1974,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, rs = RV_REG_T2; } - emit_ldx(rd, off, rs, BPF_SIZE(code), sign_ext, ctx); + if (is_stack_arg_ldx(insn)) + emit_stack_arg_ldx(rd, off, ctx); + else + emit_ldx(rd, off, rs, BPF_SIZE(code), sign_ext, ctx); ret = add_exception_handler(insn, rd, ctx); if (ret) @@ -1957,7 +2007,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, rd = RV_REG_T3; } - emit_st(rd, off, imm, BPF_SIZE(code), ctx); + if (is_stack_arg_st(insn)) + emit_stack_arg_st(off, imm, ctx); + else + emit_st(rd, off, imm, BPF_SIZE(code), ctx); ret = add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx); if (ret) @@ -1979,7 +2032,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, rd = RV_REG_T2; } - emit_stx(rd, off, rs, BPF_SIZE(code), ctx); + if (is_stack_arg_stx(insn)) + emit_stack_arg_stx(off, rs, ctx); + else + emit_stx(rd, off, rs, BPF_SIZE(code), ctx); ret = add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx); if (ret) @@ -2029,9 +2085,9 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) { - int i, stack_adjust = 0, store_offset, bpf_stack_adjust; + int i, stack_adjust = 0, store_offset, bpf_stack_adjust = ctx->stack_arg_sz; - bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, STACK_ALIGN); + bpf_stack_adjust += round_up(ctx->prog->aux->stack_depth, STACK_ALIGN); if (bpf_stack_adjust) mark_fp(ctx); @@ -2190,3 +2246,8 @@ bool bpf_jit_supports_timed_may_goto(void) { return true; } + +bool bpf_jit_supports_stack_args(void) +{ + return true; +} diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index cbfcd287ea16..ed48005568c8 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -48,6 +48,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr int pass = 0, prev_ninsns = 0, i; struct rv_jit_data *jit_data; struct rv_jit_context *ctx; + u16 stack_arg_cnt; if (!prog->jit_requested) return prog; @@ -71,6 +72,11 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr ctx->arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); + + stack_arg_cnt = bpf_out_stack_arg_cnt(env, prog); + /* First 3 stack args in regs, rest on stack */ + ctx->stack_arg_sz = stack_arg_cnt > 3 ? round_up((stack_arg_cnt - 3) * 8, STACK_ALIGN) : 0; + ctx->prog = prog; ctx->offset = kvzalloc_objs(int, prog->len); if (!ctx->offset) -- 2.34.1