From: Alexei Starovoitov Add a prog with 32 global functions that call the next one in both branches of an 'if'. It has 2^32 chains of calls, and its load took 7 minutes. Add a prog with a tail call in an exception callback that calls nothing. It was accepted. The other tests pass before and after the change. No test looked at these rules: the number of frames and the bytes of the stack in arena when the second caller has the longer chain, which frame is reported when libbpf puts a callee in front of its caller, stack arguments two calls above a tail call, no private stack for a subprog with stack arguments, and which of two nested callbacks takes a stack in arena. Signed-off-by: Alexei Starovoitov --- .../bpf/progs/test_global_func_deep_stack.c | 49 ++++++++++++ .../bpf/progs/verifier_arena_stack.c | 61 +++++++++++++++ .../bpf/progs/verifier_large_stack.c | 17 ++++ .../bpf/progs/verifier_private_stack.c | 40 ++++++++++ .../selftests/bpf/progs/verifier_tailcall.c | 78 +++++++++++++++++++ 5 files changed, 245 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/test_global_func_deep_stack.c b/tools/testing/selftests/bpf/progs/test_global_func_deep_stack.c index edb8a223a3cb..3a852809b078 100644 --- a/tools/testing/selftests/bpf/progs/test_global_func_deep_stack.c +++ b/tools/testing/selftests/bpf/progs/test_global_func_deep_stack.c @@ -92,6 +92,55 @@ int XCAT(f, n)(unsigned long a) \ F(37) +#define ASM_FUNC(linkage, name, insns) \ +linkage __naked __noinline __used int name(void) \ +{ \ + asm volatile (insns "r0 = 0; exit;" \ + :: __imm(bpf_get_prandom_u32) : __clobber_all); \ +} + +/* + * Every function calls the next one in both branches of an 'if': 32 calls are + * executed, and there are 2^32 chains of calls. + */ +#undef F_0 +#undef FN +#define F_0 ASM_FUNC(, either0, "") +#define FN(n, prev) \ + ASM_FUNC(, either##n, \ + "call %[bpf_get_prandom_u32];" \ + "if r0 == 0 goto +2;" \ + "call either" #prev ";" \ + "goto +1;" \ + "call either" #prev ";") +F(32) + +SEC("syscall") +__success +__load_if_JITed() +int global_func_many_chains(void *ctx) +{ + return either32(); +} + +/* + * r1 is the number of calls that follow, so main -> fs7 .. fs0 and main -> fs15 .. fs8 + * are executed: 9 frames each. main -> fs15 .. fs0 is in the call graph: 17 frames. + * libbpf puts fs7() .. fs0() in front of their second caller fs8(). + */ +#undef F_0 +#undef FN +#define F_0 ASM_FUNC(static, fs0, "") +#define FN(n, prev) ASM_FUNC(static, fs##n, "if r1 == 0 goto +2; r1 += -1; call fs" #prev ";") +F(15) + +SEC("syscall") +__failure __msg("the call stack of 16 frames is too deep !") +__naked int static_func_deep_stack_second_caller(void) +{ + asm volatile ("r1 = 7; call fs7; r1 = 7; call fs15; exit;" ::: __clobber_all); +} + /* Ensure that even 32 levels deep, the function verifies. */ SEC("syscall") __success diff --git a/tools/testing/selftests/bpf/progs/verifier_arena_stack.c b/tools/testing/selftests/bpf/progs/verifier_arena_stack.c index a23ab1ed2f96..1d676da4b5ea 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena_stack.c @@ -745,6 +745,65 @@ __naked void callback_names_backward(void) : __clobber_all); } +/* adds 10 to what the context points to. callee_with_frame() is two calls away */ +static __naked __noinline __used long far_from_r12_cb(void) +{ + asm volatile ( + "r6 = r2;" + "call callee_in_the_middle;" + "r1 = *(u64 *)(r6 + 0);" + "r1 += r0;" + "*(u64 *)(r6 + 0) = r1;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +static __naked __noinline __used long loops_far_from_r12_cb(void) +{ + asm volatile ( + "r3 = r2;" + "r1 = 2;" + "r2 = far_from_r12_cb ll;" + "r4 = 0;" + "call %[bpf_loop];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_loop) + : __clobber_all); +} + +SEC("syscall") +__description("arena stack: the callback that is next to r12 takes a stack") +__success __retval(60) +__load_if_JITed() +/* the end of the main program, the first insn of loops_far_from_r12_cb() */ +__xlated("r0 = r6") +__xlated("exit") +__xlated("r3 = r2") +__xlated("...") +/* the end of the code that takes a stack, the first insn of far_from_r12_cb() */ +__xlated("r12 = *(u32 *)(r{{[0-9]}} +8)") +__xlated("r6 = r2") +__naked void callback_in_callback(void) +{ + asm volatile ( + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "r1 = 3;" + "r2 = loops_far_from_r12_cb ll;" + "r3 = r10;" + "r3 += -8;" + "r4 = 0;" + "call %[bpf_loop];" + "r0 = *(u64 *)(r10 - 8);" + "exit;" + : + : __imm(bpf_loop) + : __clobber_all); +} + SEC("syscall") __description("arena stack: call of a callback that uses r12") __failure __msg("loop_cb() uses R12 and is both called and a callback") @@ -1009,10 +1068,12 @@ __naked void depth_max(void) SEC("syscall") __description("arena stack: more than 32K") __failure __msg("combined size of the frames in the stack in arena is 32784. Too large") +/* frame_16() has two callers. The one that has more in the stack in arena counts */ __naked void depth_too_large(void) { asm volatile ( ".8byte %[adj];" + "call frame_16;" "call frame_16k_calls_16;" "exit;" : diff --git a/tools/testing/selftests/bpf/progs/verifier_large_stack.c b/tools/testing/selftests/bpf/progs/verifier_large_stack.c index c2a3ddf5ac91..7e4f5e3463a2 100644 --- a/tools/testing/selftests/bpf/progs/verifier_large_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_large_stack.c @@ -112,6 +112,23 @@ __naked void five_frames_of_512(void) " ::: __clobber_all); } +/* libbpf puts frame_512_leaf() in front of frame_512_depth_2(), which goes over the budget */ +SEC("socket") +__description("the frame that exceeds the 2 KiB budget is reported, not its callee") +__load_if_large_stack() +__failure __msg("combined stack size of 3 calls is 2064. Too large") +__naked void callee_in_front_of_caller(void) +{ + asm volatile (" \ + r1 = 5; \ + *(u64 *)(r10 - 1040) = r1; \ + call frame_512_leaf; \ + call frame_512_depth_3; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + __used __naked static void frame_1536_leaf(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_private_stack.c b/tools/testing/selftests/bpf/progs/verifier_private_stack.c index bbc09f516d92..aa15361bcca5 100644 --- a/tools/testing/selftests/bpf/progs/verifier_private_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_private_stack.c @@ -70,6 +70,46 @@ __naked void no_private_stack_nested(void) " ::: __clobber_all); } +#ifdef __BPF_FEATURE_STACK_ARGUMENT +__used +__naked static long subprog_6args(long a, long b, long c, long d, long e, long f) +{ + asm volatile (" \ + r0 = *(u64 *)(r11 + 8); \ + *(u64 *)(r10 - 64) = r0; \ + exit; \ +" ::: __clobber_all); +} + +/* x86-64 has one register for the private stack and for the sixth argument */ +SEC("raw_tp") +__description("No private stack, stack arguments") +__success +__load_if_JITed() +__arch_x86_64 +__log_level(4) __msg("stack depth max 128") +__naked void no_private_stack_stack_args(void) +{ + asm volatile (" \ + r1 = 42; \ + *(u64 *)(r10 - 64) = r1; \ + *(u64 *)(r11 - 8) = r1; \ + call subprog_6args; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} +#else +SEC("raw_tp") +__description("No private stack, stack arguments") +__skip("stack_arg is not supported by compiler") +__success +int no_private_stack_stack_args(void) +{ + return 0; +} +#endif + __used __naked static void cumulative_stack_depth_subprog(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_tailcall.c b/tools/testing/selftests/bpf/progs/verifier_tailcall.c index 51687da97225..42b90b692dc5 100644 --- a/tools/testing/selftests/bpf/progs/verifier_tailcall.c +++ b/tools/testing/selftests/bpf/progs/verifier_tailcall.c @@ -50,6 +50,84 @@ static int subprog_tail_call(void) : __clobber_all); } +/* The exception callback has no ctx, so its tail call can only be dead code */ +__used __noinline +int exception_cb_tail_call(__u64 cookie) +{ + asm volatile (" \ + r0 = 0; \ + if r0 == 0 goto +4; \ + r2 = %[jmp_table] ll; \ + r3 = 0; \ + call %[bpf_tail_call]; \ +" : + : __imm(bpf_tail_call), + __imm_addr(jmp_table) + : __clobber_common); + return 0; +} + +SEC("tc") +__description("tail call in an async callback that calls nothing") +__failure __msg("cannot tail call within callback") +__attribute__((btf_decl_tag("exception_callback:exception_cb_tail_call"))) +__naked void tail_call_in_async_callback(void) +{ + asm volatile (" \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ + defined(__BPF_FEATURE_STACK_ARGUMENT) + +__used __noinline +static long subprog_6args(long a, long b, long c, long d, long e, long f) +{ + return f; +} + +__used __naked +static int calls_subprog_tail_call(void) +{ + asm volatile (" \ + call subprog_tail_call; \ + exit; \ +" ::: __clobber_all); +} + +SEC("tc") +__description("stack arguments two calls above a tail call") +__failure __msg("tail_calls are not allowed in programs with stack args") +__load_if_JITed() +__naked void stack_args_above_tail_call(void) +{ + asm volatile (" \ + r6 = r1; \ + *(u64 *)(r11 - 8) = 6; \ + call subprog_6args; \ + r1 = r6; \ + call calls_subprog_tail_call; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +#else + +SEC("tc") +__description("stack arguments two calls above a tail call") +__skip("stack_arg is not supported by compiler or jit") +__success +int stack_args_above_tail_call(void) +{ + return 0; +} + +#endif + /* * A tail call unwinds only the frame of the subprog doing it, so the * frames of its callers stay on the stack. With up to 33 tail calls in -- 2.55.0