From: Alexei Starovoitov r3 is x2. The epilogue doesn't touch it, so say so in bpf_jit_supports_unwind(). Like x86, replace 'if r3 != 0 goto pad' after a call with an extable entry: bpf arm64 before arm64 after call inner bl inner bl inner if r3 != 0 goto pad mov x8, x0 mov x8, x0 w0 = w6 cmp x2, #0 mov w8, w19 b.ne pad mov w8, w19 The callee returns to the pad past 'mov x8, x0'. That's fine: r0 is x8 and the epilogue of the callee copies it to x0 and leaves x8 as it is. An exit with r3 != 0 fixes LR in its frame record. With pointer authentication it's signed with SP at the entry of the prog, which is FP + 16: ldr x17, [x29, #8] add x16, x29, #16 autia1716 mov x0, x17 bl bpf_unwind_target mov x17, x0 add x16, x29, #16 pacia1716 str x17, [x29, #8] So the lookup gets an authenticated address, and autiasp in the epilogue checks the new one. autia1716 and pacia1716 are hints like paciasp and autiasp: nops when the cpu doesn't have pointer authentication. RET is not subject to BTI. The private stack pointer is in a callee saved register. Unlike x86 there is nothing to undo after the call, so it doesn't get in the way. Tested in qemu with -cpu max and -cpu cortex-a72, i.e. with and without pointer authentication. Signed-off-by: Alexei Starovoitov --- arch/arm64/net/bpf_jit.h | 2 + arch/arm64/net/bpf_jit_comp.c | 110 ++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/arch/arm64/net/bpf_jit.h b/arch/arm64/net/bpf_jit.h index b2fe6e6dcf44..50029dd9baa6 100644 --- a/arch/arm64/net/bpf_jit.h +++ b/arch/arm64/net/bpf_jit.h @@ -316,6 +316,8 @@ #define A64_PACIASP A64_HINT(AARCH64_INSN_HINT_PACIASP) #define A64_AUTIASP A64_HINT(AARCH64_INSN_HINT_AUTIASP) +#define A64_PACIA1716 A64_HINT(AARCH64_INSN_HINT_PACIA_1716) +#define A64_AUTIA1716 A64_HINT(AARCH64_INSN_HINT_AUTIA_1716) /* BTI */ #define A64_BTI_C A64_HINT(AARCH64_INSN_HINT_BTIC) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 506c4201fec8..9b702930486a 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -1221,6 +1222,86 @@ static int add_exception_handler(const struct bpf_insn *insn, return 0; } +/* + * See bpf_jit_unwind_site(). The branch after the call that was just emitted is + * not emitted. Remember where it goes. The pad is an index into ctx->offset. + */ +static int add_unwind_exentry(struct jit_ctx *ctx, int pad) +{ + struct exception_table_entry *ex; + off_t ins_offset; + + if (!ctx->image) + /* First pass */ + return 0; + + if (!ctx->prog->aux->extable || + WARN_ON_ONCE(ctx->exentry_idx >= ctx->prog->aux->num_exentries)) + return -EINVAL; + + ex = &ctx->prog->aux->extable[ctx->exentry_idx]; + ins_offset = (long)&ctx->ro_image[ctx->idx] - 1 - (long)&ex->insn; + if (WARN_ON_ONCE(ins_offset >= 0 || ins_offset < INT_MIN)) + return -ERANGE; + + /* switch ex to rw buffer for writing */ + ex = (void *)ctx->image + ((void *)ex - (void *)ctx->ro_image); + ex->insn = ins_offset; + ex->fixup = (ctx->offset[pad] - ctx->idx) * AARCH64_INSN_SIZE; + ex->type = EX_TYPE_BPF; + + ctx->exentry_idx++; + return 0; +} + +static u64 bpf_unwind_target(u64 ret_addr) +{ + const struct exception_table_entry *e = search_bpf_extables(ret_addr - 1); + + return e ? ret_addr + e->fixup : ret_addr; +} + +/* + * Make an exit with r3 != 0 return to the pad. + * LR in the frame record is signed with SP at the entry of the prog, which is + * FP + 16. The x16/x17 forms are hints like PACIASP: nops without pointer auth. + * r0, r2 and r3 are what the caller may look at. + */ +static void emit_unwind_ret_addr(struct jit_ctx *ctx, bool look_at_r3) +{ + const bool ptr_auth = IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL); + const u8 r0 = bpf2a64[BPF_REG_0]; + const u8 r2 = bpf2a64[BPF_REG_2]; + const u8 r3 = bpf2a64[BPF_REG_3]; + __le32 *branch = ctx->image + ctx->idx; + + if (look_at_r3) + emit(A64_NOP, ctx); + + emit(A64_PUSH(r0, r2, A64_SP), ctx); + emit(A64_PUSH(r3, A64_ZR, A64_SP), ctx); + + emit(A64_LDR64I(A64_R(17), A64_FP, 8), ctx); + if (ptr_auth) { + emit(A64_ADD_I(1, A64_R(16), A64_FP, 16), ctx); + emit(A64_AUTIA1716, ctx); + } + emit(A64_MOV(1, A64_R(0), A64_R(17)), ctx); + emit_call((u64)bpf_unwind_target, ctx); + emit(A64_MOV(1, A64_R(17), A64_R(0)), ctx); + if (ptr_auth) { + emit(A64_ADD_I(1, A64_R(16), A64_FP, 16), ctx); + emit(A64_PACIA1716, ctx); + } + emit(A64_STR64I(A64_R(17), A64_FP, 8), ctx); + + emit(A64_POP(r3, A64_ZR, A64_SP), ctx); + emit(A64_POP(r0, r2, A64_SP), ctx); + + if (look_at_r3 && ctx->image) + *branch = cpu_to_le32(A64_CBZ(1, r3, &ctx->image[ctx->idx] - branch)); +} + static const struct bpf_jit_arg_abi arm64_arg_abi = { .nr_arg_regs = 8, .even_reg_align = true, @@ -1378,6 +1459,7 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn int off_adj; int ret; bool sign_extend; + enum bpf_unwind_exit unw; if (bpf_insn_is_indirect_target(env, ctx->prog, i)) emit_bti(A64_BTI_J, ctx); @@ -1781,6 +1863,13 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn emit_kfunc_arg_moves(ctx, fm); } emit_call(func_addr, ctx); + if (bpf_jit_unwind_site(env, ctx->prog, i)) { + ret = add_unwind_exentry(ctx, i + 2 + insn[1].off); + if (ret) + return ret; + emit(A64_MOV(1, r0, A64_R(0)), ctx); + return 1; + } /* * Call to arch_bpf_timed_may_goto() is emitted by the * verifier and called with custom calling convention with @@ -1799,6 +1888,13 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn * counter and the private stack are callee saved. */ emit(A64_BLR(dst), ctx); + if (bpf_jit_unwind_site(env, ctx->prog, i)) { + ret = add_unwind_exentry(ctx, i + 2 + insn[1].off); + if (ret) + return ret; + emit(A64_MOV(1, bpf2a64[BPF_REG_0], A64_R(0)), ctx); + return 1; + } emit(A64_MOV(1, bpf2a64[BPF_REG_0], A64_R(0)), ctx); break; /* tail call */ @@ -1808,6 +1904,9 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn break; /* function return */ case BPF_JMP | BPF_EXIT: + unw = bpf_jit_unwind_exit(env, ctx->prog, i); + if (unw != BPF_EXIT_RETURNS) + emit_unwind_ret_addr(ctx, unw == BPF_EXIT_LOOK_AT_R3); /* Optimization: when last instruction is EXIT, simply fallthrough to epilogue. */ if (i == ctx->prog->len - 1) @@ -2240,6 +2339,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr int body_idx; int exentry_idx; int out_cnt; + int i; if (!prog->jit_requested) return prog; @@ -2313,6 +2413,11 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr build_epilogue(&ctx, was_classic); build_plt(&ctx); + /* The extra pass doesn't get here, so the count is adjusted once. */ + for (i = 0; i < prog->len; i++) + if (bpf_jit_unwind_site(env, prog, i)) + prog->aux->num_exentries++; + extable_align = __alignof__(struct exception_table_entry); extable_size = prog->aux->num_exentries * sizeof(struct exception_table_entry); @@ -2464,6 +2569,11 @@ bool bpf_jit_supports_kfunc_ret_reg_pair(void) return true; } +bool bpf_jit_supports_unwind(void) +{ + return true; +} + const struct bpf_jit_arg_abi *bpf_jit_arg_abi(void) { return &arm64_arg_abi; -- 2.55.0