Add a native entry for bpf_throw() on arm64, as on x86-64. The verifier redirects a call to bpf_throw() to bpf_throw_entry(), which: - pushes a frame record whose return address points into the program; - saves the BPF registers of the throw site next to it as a struct bpf_throw_regs: R0 from x8, R1-R5 from x0-x4, and R6-R9 from x19-x22; - calls bpf_throw_unwind() with it. Nothing uses the registers yet. Signed-off-by: Kumar Kartikeya Dwivedi --- arch/arm64/net/Makefile | 1 + arch/arm64/net/bpf_jit_comp.c | 8 ++++++++ arch/arm64/net/bpf_throw.S | 30 ++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+) create mode 100644 arch/arm64/net/bpf_throw.S diff --git a/arch/arm64/net/Makefile b/arch/arm64/net/Makefile index 3ae382bfca87..70d18f067d01 100644 --- a/arch/arm64/net/Makefile +++ b/arch/arm64/net/Makefile @@ -3,3 +3,4 @@ # ARM64 networking code # obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o bpf_timed_may_goto.o +obj-$(CONFIG_BPF_SYSCALL) += bpf_throw.o diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 83a858c6d472..33cc50268f3a 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -3965,6 +3966,13 @@ bool bpf_jit_supports_timed_may_goto(void) return true; } +#ifdef CONFIG_BPF_SYSCALL +void *bpf_arch_throw_entry(void) +{ + return (void *)bpf_throw_entry; +} +#endif + bool bpf_jit_inlines_helper_call(s32 imm) { switch (imm) { diff --git a/arch/arm64/net/bpf_throw.S b/arch/arm64/net/bpf_throw.S new file mode 100644 index 000000000000..93f2d208ccc8 --- /dev/null +++ b/arch/arm64/net/bpf_throw.S @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include + +/* + * Native entry of the bpf_throw() kfunc. The verifier retargets the call to + * bpf_throw() here, so the BPF register file of the throw site is still intact + * on entry and x30 is the return address into the program. Push a frame record + * whose link register is that address and save the file below it as a struct + * bpf_throw_regs: r[i] sits at [sp + 16 + 8 * i], with R0 in x8, R1-R5 in + * x0-x4 and R6-R9 in x19-x22. Then hand it to bpf_throw_unwind(), which never + * returns. + */ +SYM_FUNC_START(bpf_throw_entry) + stp x29, x30, [sp, #-(16 + BPF_THROW_REGS_SIZE)]! + mov x29, sp + stp x8, x0, [sp, #16] + stp x1, x2, [sp, #32] + stp x3, x4, [sp, #48] + stp x19, x20, [sp, #64] + stp x21, x22, [sp, #80] + + /* bpf_throw_unwind(regs, cookie); the cookie is R1 of the caller. */ + mov x1, x0 + add x0, sp, #16 + bl bpf_throw_unwind + brk #0x1 +SYM_FUNC_END(bpf_throw_entry) -- 2.53.0