From: Alexei Starovoitov arch_bpf_trampoline_size() emits the trampoline into a scratch page to learn its size. The emitter does not look at the end of the image while it writes. Only the WARN_ON_ONCE() behind the last instruction does. BPF_MAX_TRAMP_LINKS programs fit in a page in the common case. With call depth tracking each call takes 9 more bytes, and they may not fit. Then the attach fails with the warning and -EFAULT, or the size pass writes past its page. The JITs of the other architectures have a pass that counts and stores nothing. Do the same. Without an image the cursor starts at NULL, and emit_code() only moves it. Two places of the trampoline code stored on their own: the template of the call depth accounting and the jump over a program. They go through emit_code() now. There is nothing to overflow. bpf_trampoline_update() sees a size above PAGE_SIZE and returns -E2BIG, as on arm64. The scratch page is gone. Fixes: b2e9dfe54be4 ("x86/bpf: Emit call depth accounting if required") Signed-off-by: Alexei Starovoitov --- arch/x86/net/bpf_jit_comp.c | 54 +++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 8b5772262e13..d10a36daeeec 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -9,12 +9,10 @@ #include #include #include -#include #include #include #include #include -#include #include #include #include @@ -37,23 +35,15 @@ void __asan_store8(void *p); static bool all_callee_regs_used[4] = {true, true, true, true}; -static void *trampoline_size_image; - -static int __init init_trampoline_size_image(void) +static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len) { /* - * The generated trampoline contains calls and jumps with 32bit relative - * offsets, so the scratch image must be in the execmem range. - * On x86, module data and executable memory share the same address range, - * so using EXECMEM_MODULE_DATA to get writable memory. + * arch_bpf_trampoline_size() has no image. Its cursor starts at NULL, + * so it is not a kernel address: count, do not store. */ - trampoline_size_image = execmem_alloc(EXECMEM_MODULE_DATA, PAGE_SIZE); - return trampoline_size_image ? 0 : -ENOMEM; -} -late_initcall(init_trampoline_size_image); + if ((long)ptr >= 0) + return ptr + len; -static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len) -{ if (len == 1) *ptr = bytes; else if (len == 2) @@ -598,10 +588,24 @@ static int emit_call(u8 **pprog, void *func, void *ip) return emit_patch(pprog, func, ip, 0xE8); } +/* x86_call_depth_emit_accounting() stores. Let emit_code() decide about that. */ +static int emit_call_depth_accounting(u8 **pprog, void *func, void *ip) +{ + u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY]; + u8 *prog = *pprog, *end = temp; + int i, len; + + len = x86_call_depth_emit_accounting(&end, func, ip); + for (i = 0; i < len; i++) + EMIT1(temp[i]); + *pprog = prog; + return len; +} + static int emit_rsb_call(u8 **pprog, void *func, void *ip) { OPTIMIZER_HIDE_VAR(func); - ip += x86_call_depth_emit_accounting(pprog, func, ip); + ip += emit_call_depth_accounting(pprog, func, ip); return emit_patch(pprog, func, ip, 0xE8); } @@ -4075,8 +4079,7 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); /* replace 2 nops with JE insn, since jmp target is known */ - jmp_insn[0] = X86_JE; - jmp_insn[1] = prog - jmp_insn - 2; + emit_code(jmp_insn, X86_JE + ((prog - jmp_insn - 2) << 8), 2); /* arg1: mov rdi, progs[i] */ emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p); @@ -4392,7 +4395,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im * Direct-call fentry stub, as such it needs accounting for the * __fentry__ call. */ - x86_call_depth_emit_accounting(&prog, NULL, image); + emit_call_depth_accounting(&prog, NULL, image); } EMIT1(0x55); /* push rbp */ EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */ @@ -4547,7 +4550,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im } emit_return(&prog, image + (prog - (u8 *)rw_image)); /* Make sure the trampoline generation logic doesn't overflow */ - if (WARN_ON_ONCE(prog > (u8 *)rw_image_end - BPF_INSN_SAFETY)) { + if (rw_image && WARN_ON_ONCE(prog > (u8 *)rw_image_end - BPF_INSN_SAFETY)) { ret = -EFAULT; goto cleanup; } @@ -4607,12 +4610,11 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, { struct bpf_tramp_image im = {}; - if (!trampoline_size_image) - return -ENOMEM; - - return __arch_prepare_bpf_trampoline(&im, trampoline_size_image, - trampoline_size_image + PAGE_SIZE, - trampoline_size_image, m, flags, + /* + * No image: emit_code() counts. The calls and jumps have 32-bit offsets + * from where the image is going to be. func_addr is in their range. + */ + return __arch_prepare_bpf_trampoline(&im, NULL, NULL, func_addr, m, flags, tnodes, func_addr); } -- 2.55.0