From: Alexei Starovoitov bpf arm64 call inner bl inner mov x8, x0 unwind_goto pad w0 = w6 mov w8, w19 Like on x86 unwind_goto is an extable entry and no code. unwind_exit returns to the pad past 'mov x8, x0'. That's fine: r0 is x8, and the epilogue of the callee copies it to x0 and leaves x8 as it is. unwind_exit fixes LR in its frame record. With pointer authentication it's signed with SP at the entry of the prog, which is FP + 16: stp x8, x1, [sp, #-16]! ldr x17, [x29, #8] add x16, x29, #16 autia1716 mov x0, x17 bl bpf_unwind_target mov x17, x0 add x16, x29, #16 pacia1716 str x17, [x29, #8] ldp x8, x1, [sp], #16 So the lookup gets an authenticated address, and autiasp in the epilogue checks the new one. autia1716 and pacia1716 are hints like paciasp and autiasp: nops when the cpu doesn't have pointer authentication. RET is not subject to BTI. r0 and r2 are what a subprog returns, so x8 and x1 are saved around the lookup. Like on x86 unwind_exit of the main prog is exit, unless it's an extension. The private stack pointer is in a callee saved register. Unlike x86 there is nothing to undo after the call. Tested in qemu with -cpu max and -cpu cortex-a72, i.e. with and without pointer authentication. Signed-off-by: Alexei Starovoitov --- arch/arm64/net/bpf_jit.h | 2 + arch/arm64/net/bpf_jit_comp.c | 93 +++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/arch/arm64/net/bpf_jit.h b/arch/arm64/net/bpf_jit.h index b2fe6e6dcf44..50029dd9baa6 100644 --- a/arch/arm64/net/bpf_jit.h +++ b/arch/arm64/net/bpf_jit.h @@ -316,6 +316,8 @@ #define A64_PACIASP A64_HINT(AARCH64_INSN_HINT_PACIASP) #define A64_AUTIASP A64_HINT(AARCH64_INSN_HINT_AUTIASP) +#define A64_PACIA1716 A64_HINT(AARCH64_INSN_HINT_PACIA_1716) +#define A64_AUTIA1716 A64_HINT(AARCH64_INSN_HINT_AUTIA_1716) /* BTI */ #define A64_BTI_C A64_HINT(AARCH64_INSN_HINT_BTIC) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 1d5f4cf88d33..45ed69b83265 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -1221,6 +1222,85 @@ static int add_exception_handler(const struct bpf_insn *insn, return 0; } +/* + * There is no branch for unwind_goto, only an extable entry + * { .insn = ret_addr - 1, .fixup = pad - ret_addr } + * where ret_addr is the return address of the call in front of it. unwind_exit + * replaces LR in its frame record with bpf_unwind_target() of it. + * ret_addr - 1, because what follows the call can have an entry of its own. + * No insn starts there, so a fault cannot hit the entry. + * 'bl foo; mov x8, x0' is in front of unwind_goto. The pad doesn't miss the mov: + * r0 is x8 and the epilogue of the callee copies it to x0 and leaves x8 as it is. + */ +static int add_unwind_exentry(struct jit_ctx *ctx, int pad) +{ + int ret_addr = ctx->idx - 1; + struct exception_table_entry *ex; + off_t ins_offset; + + if (!ctx->image) + /* First pass */ + return 0; + + if (!ctx->prog->aux->extable || + WARN_ON_ONCE(ctx->exentry_idx >= ctx->prog->aux->num_exentries)) + return -EINVAL; + + ex = &ctx->prog->aux->extable[ctx->exentry_idx]; + ins_offset = (long)&ctx->ro_image[ret_addr] - 1 - (long)&ex->insn; + if (WARN_ON_ONCE(ins_offset >= 0 || ins_offset < INT_MIN)) + return -ERANGE; + + /* switch ex to rw buffer for writing */ + ex = (void *)ctx->image + ((void *)ex - (void *)ctx->ro_image); + ex->insn = ins_offset; + ex->fixup = (ctx->offset[pad] - ret_addr) * AARCH64_INSN_SIZE; + ex->type = EX_TYPE_BPF; + + ctx->exentry_idx++; + return 0; +} + +static u64 bpf_unwind_target(u64 ret_addr) +{ + const struct exception_table_entry *e = search_bpf_extables(ret_addr - 1); + + /* The verifier made sure that unwind_goto follows the call */ + if (WARN_ON_ONCE(!e)) + return ret_addr; + return ret_addr + e->fixup; +} + +/* + * unwind_exit: fix LR in the frame record. It's signed with SP at the entry of + * the prog, which is FP + 16. The x16/x17 forms are hints like PACIASP: nops + * without pointer auth. r0 and r2 are returned. + */ +static void emit_unwind_ret_addr(struct jit_ctx *ctx) +{ + const bool ptr_auth = IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL); + const u8 r0 = bpf2a64[BPF_REG_0]; + const u8 r2 = bpf2a64[BPF_REG_2]; + + emit(A64_PUSH(r0, r2, A64_SP), ctx); + + emit(A64_LDR64I(A64_R(17), A64_FP, 8), ctx); + if (ptr_auth) { + emit(A64_ADD_I(1, A64_R(16), A64_FP, 16), ctx); + emit(A64_AUTIA1716, ctx); + } + emit(A64_MOV(1, A64_R(0), A64_R(17)), ctx); + emit_call((u64)bpf_unwind_target, ctx); + emit(A64_MOV(1, A64_R(17), A64_R(0)), ctx); + if (ptr_auth) { + emit(A64_ADD_I(1, A64_R(16), A64_FP, 16), ctx); + emit(A64_PACIA1716, ctx); + } + emit(A64_STR64I(A64_R(17), A64_FP, 8), ctx); + + emit(A64_POP(r0, r2, A64_SP), ctx); +} + static const struct bpf_jit_arg_abi arm64_arg_abi = { .nr_arg_regs = 8, .even_reg_align = true, @@ -1806,8 +1886,16 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn if (emit_bpf_tail_call(ctx)) return -EFAULT; break; + /* unwind_goto. The verifier expanded may_goto. */ + case BPF_JMP | BPF_JCOND: + ret = add_unwind_exentry(ctx, i + 1 + off); + if (ret) + return ret; + break; /* function return */ case BPF_JMP | BPF_EXIT: + if (bpf_is_unwind_exit(insn) && bpf_prog_can_unwind(ctx->prog)) + emit_unwind_ret_addr(ctx); /* Optimization: when last instruction is EXIT, simply fallthrough to epilogue. */ if (i == ctx->prog->len - 1) @@ -2466,6 +2554,11 @@ bool bpf_jit_supports_kfunc_ret_reg_pair(void) return true; } +bool bpf_jit_supports_unwind(void) +{ + return true; +} + const struct bpf_jit_arg_abi *bpf_jit_arg_abi(void) { return &arm64_arg_abi; -- 2.55.0