Add bpf_throw_entry, which the verifier calls instead of bpf_throw(). It saves the BPF registers of the throw site, R0-R9, on the stack in the layout of struct bpf_throw_regs, passes the cookie from R1, and calls bpf_throw_unwind(). That function does not return, so a ud2 follows the call. The unwinder needs the callee-saved R6-R9 to restore them for the frames it discards. The entry also captures R0-R5: - later patches describe which register holds a reference at a throw site, and any of R0-R9 can hold one there; - bpf_throw() does not return, so the compiler has no reason to move a reference out of a caller-saved register before the call. R0-R5 are stored before R6-R9, so the unwinder can index the saved registers by BPF register number. An architecture may append callee-saved registers that do not map to a BPF register after R9. None does yet. The JIT calls the entry directly through __bpf_call_base + imm, so it needs no ENDBR. objtool generates ORC data for its frame, so the stack walk in bpf_throw_unwind() passes through the entry to the BPF frames, where the frame pointer fallback of the ORC unwinder takes over as before. The entry calls bpf_throw_unwind(), which is only built with CONFIG_BPF_SYSCALL. Build the entry with CONFIG_BPF_SYSCALL as well, and have the JIT return it only then. The JIT itself is built with CONFIG_BPF_JIT alone, for cBPF. Signed-off-by: Kumar Kartikeya Dwivedi --- arch/x86/net/Makefile | 1 + arch/x86/net/bpf_jit_comp.c | 8 +++++++ arch/x86/net/bpf_throw.S | 43 +++++++++++++++++++++++++++++++++++++ include/linux/bpf_throw.h | 17 ++++++++++----- 4 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 arch/x86/net/bpf_throw.S diff --git a/arch/x86/net/Makefile b/arch/x86/net/Makefile index dddbefc0f439..daef0aa8b28c 100644 --- a/arch/x86/net/Makefile +++ b/arch/x86/net/Makefile @@ -7,4 +7,5 @@ ifeq ($(CONFIG_X86_32),y) obj-$(CONFIG_BPF_JIT) += bpf_jit_comp32.o else obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o bpf_timed_may_goto.o + obj-$(CONFIG_BPF_SYSCALL) += bpf_throw.o endif diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 19fef39f1445..bdfc5329a00c 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -5135,6 +5135,14 @@ void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp #endif } +#ifdef CONFIG_BPF_SYSCALL +/* arch/x86/net/bpf_throw.S, which calls bpf_throw_unwind(); both need the syscall. */ +void *bpf_arch_throw_entry(void) +{ + return (void *)bpf_throw_entry; +} +#endif + void bpf_arch_poke_desc_update(struct bpf_jit_poke_descriptor *poke, struct bpf_prog *new, struct bpf_prog *old) { diff --git a/arch/x86/net/bpf_throw.S b/arch/x86/net/bpf_throw.S new file mode 100644 index 000000000000..67983e5077a6 --- /dev/null +++ b/arch/x86/net/bpf_throw.S @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include + + .code64 + .section .text, "ax" + +/* + * Native entry of the bpf_throw() kfunc. The verifier retargets the call to + * bpf_throw() here, so the BPF register file of the throw site is still intact + * on entry. Capture it on the stack as a struct bpf_throw_regs and hand it to + * bpf_throw_unwind(), which never returns. + */ +SYM_FUNC_START(bpf_throw_entry) + ANNOTATE_NOENDBR + + /* Setup frame. */ + pushq %rbp + movq %rsp, %rbp + + /* Save R9..R0 so that [rsp + 8 * i] is r[i]. */ + pushq %r15 /* R9 */ + pushq %r14 /* R8 */ + pushq %r13 /* R7 */ + pushq %rbx /* R6 */ + pushq %r8 /* R5 */ + pushq %rcx /* R4 */ + pushq %rdx /* R3 */ + pushq %rsi /* R2 */ + pushq %rdi /* R1 */ + pushq %rax /* R0 */ + + /* bpf_throw_unwind(regs, cookie); the cookie is R1 of the caller. */ + movq %rdi, %rsi + movq %rsp, %rdi + + /* Emit call depth accounting for call below. */ + CALL_DEPTH_ACCOUNT + call bpf_throw_unwind + ud2 +SYM_FUNC_END(bpf_throw_entry) diff --git a/include/linux/bpf_throw.h b/include/linux/bpf_throw.h index 57d974458750..2bb2c18796ac 100644 --- a/include/linux/bpf_throw.h +++ b/include/linux/bpf_throw.h @@ -6,21 +6,28 @@ #ifndef _LINUX_BPF_THROW_H #define _LINUX_BPF_THROW_H +/* + * The callee-saved registers of the architecture that no BPF register maps + * to, which an entry captures after R9. None yet. + */ +#define BPF_THROW_NR_EXTRA 0 /* The callee-saved registers the unwinder tracks: R6..R9. */ #define BPF_THROW_NR_CALLEE_SAVED 4 -/* The size of struct bpf_throw_regs, for the entries. */ -#define BPF_THROW_REGS_SIZE (8 * BPF_THROW_NR_CALLEE_SAVED) +/* The size of struct bpf_throw_regs, for the entries: R0..R9 and the extra ones. */ +#define BPF_THROW_REGS_SIZE (8 * (10 + BPF_THROW_NR_EXTRA)) #ifndef __ASSEMBLER__ #include #include +#include /* - * The callee-saved registers captured at a throw site, indexed like - * bpf_prog_aux::callee_reg_off. + * The register file captured at a throw site: the BPF registers R0..R9, then + * the extra callee-saved registers of the architecture; from R6 on, indexed + * like bpf_prog_aux::callee_reg_off. */ struct bpf_throw_regs { - u64 saved[BPF_THROW_NR_CALLEE_SAVED]; + u64 r[BPF_REG_10 + BPF_THROW_NR_EXTRA]; }; static_assert(sizeof(struct bpf_throw_regs) == BPF_THROW_REGS_SIZE); -- 2.53.0