From: Abhishek Dubey Implement JIT support for fsession in powerpc64 trampoline. The trampoline stack now accommodate session cookies and function metadata in place of function argument. fentry/fexit programs consume corresponding function metadata. This mirrors existing x86 behavior and enable session cookies on powerpc64. Patch rebased over: https://lore.kernel.org/bpf/20260220063933.196141-1-hbathini@linux.ibm.com v1->v2: No change since v1 [v1]: https://lore.kernel.org/bpf/20260216155310.38457-1-adubey@linux.ibm.com Signed-off-by: Abhishek Dubey --- arch/powerpc/net/bpf_jit.h | 4 +- arch/powerpc/net/bpf_jit_comp.c | 69 ++++++++++++++++++++++++++----- arch/powerpc/net/bpf_jit_comp64.c | 25 +++++++++++ 3 files changed, 87 insertions(+), 11 deletions(-) diff --git a/arch/powerpc/net/bpf_jit.h b/arch/powerpc/net/bpf_jit.h index a232f3fb73be..f32de8704d4d 100644 --- a/arch/powerpc/net/bpf_jit.h +++ b/arch/powerpc/net/bpf_jit.h @@ -218,7 +218,9 @@ void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx); void bpf_jit_build_fentry_stubs(u32 *image, struct codegen_context *ctx); void bpf_jit_realloc_regs(struct codegen_context *ctx); int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr); - +void prepare_for_fsession_fentry(u32 *image, struct codegen_context *ctx, int cookie_cnt, + int cookie_off, int retval_off); +void store_func_meta(u32 *image, struct codegen_context *ctx, u64 func_meta, int func_meta_off); int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass, struct codegen_context *ctx, int insn_idx, int jmp_off, int dst_reg, u32 code); diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 08c992436ace..6d334c0c3d85 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -540,6 +540,11 @@ bool bpf_jit_supports_private_stack(void) return IS_ENABLED(CONFIG_PPC64); } +bool bpf_jit_supports_fsession(void) +{ + return IS_ENABLED(CONFIG_PPC64); +} + bool bpf_jit_supports_arena(void) { return IS_ENABLED(CONFIG_PPC64); @@ -812,12 +817,16 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im struct bpf_tramp_links *tlinks, void *func_addr) { - int regs_off, nregs_off, ip_off, run_ctx_off, retval_off, nvr_off, alt_lr_off, r4_off = 0; + int regs_off, func_meta_off, ip_off, run_ctx_off, retval_off; + int nvr_off, alt_lr_off, r4_off = 0; struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN]; struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY]; struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT]; int i, ret, nr_regs, retaddr_off, bpf_frame_size = 0; struct codegen_context codegen_ctx, *ctx; + int cookie_off, cookie_cnt, cookie_ctx_off; + int fsession_cnt = bpf_fsession_cnt(tlinks); + u64 func_meta; u32 *image = (u32 *)rw_image; ppc_inst_t branch_insn; u32 *branches = NULL; @@ -853,9 +862,11 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im * [ reg argN ] * [ ... ] * regs_off [ reg_arg1 ] prog ctx context - * nregs_off [ args count ] ((u64 *)prog_ctx)[-1] + * func_meta_off [ args count ] ((u64 *)prog_ctx)[-1] * ip_off [ traced function ] ((u64 *)prog_ctx)[-2] + * [ stack cookieN ] * [ ... ] + * cookie_off [ stack cookie1 ] * run_ctx_off [ bpf_tramp_run_ctx ] * [ reg argN ] * [ ... ] @@ -887,16 +898,21 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im run_ctx_off = bpf_frame_size; bpf_frame_size += round_up(sizeof(struct bpf_tramp_run_ctx), SZL); + /* room for session cookies */ + cookie_off = bpf_frame_size; + cookie_cnt = bpf_fsession_cookie_cnt(tlinks); + bpf_frame_size += cookie_cnt * 8; + /* Room for IP address argument */ ip_off = bpf_frame_size; if (flags & BPF_TRAMP_F_IP_ARG) bpf_frame_size += SZL; - /* Room for args count */ - nregs_off = bpf_frame_size; + /* Room for function metadata, arg regs count */ + func_meta_off = bpf_frame_size; bpf_frame_size += SZL; - /* Room for args */ + /* Room for arg egs */ regs_off = bpf_frame_size; bpf_frame_size += nr_regs * SZL; @@ -990,9 +1006,9 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im EMIT(PPC_RAW_STL(_R3, _R1, retaddr_off)); } - /* Save function arg count -- see bpf_get_func_arg_cnt() */ - EMIT(PPC_RAW_LI(_R3, nr_regs)); - EMIT(PPC_RAW_STL(_R3, _R1, nregs_off)); + /* Save function arg regs count -- see bpf_get_func_arg_cnt() */ + func_meta = nr_regs; + store_func_meta(image, ctx, func_meta, func_meta_off); /* Save nv regs */ EMIT(PPC_RAW_STL(_R25, _R1, nvr_off)); @@ -1006,10 +1022,28 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im return ret; } - for (i = 0; i < fentry->nr_links; i++) + if (fsession_cnt) { + /* + * Clear all the session cookies' values + * Clear the return value to make sure fentry always get 0 + */ + prepare_for_fsession_fentry(image, ctx, cookie_cnt, cookie_off, retval_off); + } + + cookie_ctx_off = (regs_off - cookie_off) / 8; + + for (i = 0; i < fentry->nr_links; i++) { + if (bpf_prog_calls_session_cookie(fentry->links[i])) { + u64 meta = func_meta | (cookie_ctx_off << BPF_TRAMP_COOKIE_INDEX_SHIFT); + + store_func_meta(image, ctx, meta, func_meta_off); + cookie_ctx_off--; + } + if (invoke_bpf_prog(image, ro_image, ctx, fentry->links[i], regs_off, retval_off, run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET)) return -EINVAL; + } if (fmod_ret->nr_links) { branches = kcalloc(fmod_ret->nr_links, sizeof(u32), GFP_KERNEL); @@ -1071,12 +1105,27 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im image[branches[i]] = ppc_inst_val(branch_insn); } - for (i = 0; i < fexit->nr_links; i++) + /* set the "is_return" flag for fsession */ + func_meta |= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT); + if (fsession_cnt) + store_func_meta(image, ctx, func_meta, func_meta_off); + + cookie_ctx_off = (regs_off - cookie_off) / 8; + + for (i = 0; i < fexit->nr_links; i++) { + if (bpf_prog_calls_session_cookie(fexit->links[i])) { + u64 meta = func_meta | (cookie_ctx_off << BPF_TRAMP_COOKIE_INDEX_SHIFT); + + store_func_meta(image, ctx, meta, func_meta_off); + cookie_ctx_off--; + } + if (invoke_bpf_prog(image, ro_image, ctx, fexit->links[i], regs_off, retval_off, run_ctx_off, false)) { ret = -EINVAL; goto cleanup; } + } if (flags & BPF_TRAMP_F_CALL_ORIG) { if (ro_image) /* image is NULL for dummy pass */ diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index 17aec341f5eb..1e2ac6c4f331 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -179,6 +179,31 @@ static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg) BUG(); } +void prepare_for_fsession_fentry(u32 *image, struct codegen_context *ctx, int cookie_cnt, + int cookie_off, int retval_off) +{ + EMIT(PPC_RAW_LI(bpf_to_ppc(TMP_REG_1), 0)); + + for (int i = 0; i < cookie_cnt; i++) + EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, cookie_off + 8 * i)); + EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, retval_off)); +} + +void store_func_meta(u32 *image, struct codegen_context *ctx, + u64 func_meta, int func_meta_off) +{ + /* + * Store func_meta to stack at [R1 + func_meta_off] = func_meta + * + * func_meta : + * bit[63]: is_return flag + * byte[1]: cookie offset from ctx + * byte[0]: args count + */ + PPC_LI64(bpf_to_ppc(TMP_REG_1), func_meta); + EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, func_meta_off)); +} + void bpf_jit_realloc_regs(struct codegen_context *ctx) { } -- 2.52.0 From: Abhishek Dubey Allow get_func_args, get_func_ip & fsession selftests to run on powerpc64. # ./test_progs -t fsession #135/1 fsession_test/fsession_test:OK #135/2 fsession_test/fsession_reattach:OK #135/3 fsession_test/fsession_cookie:OK #135 fsession_test:OK Summary: 1/3 PASSED, 0 SKIPPED, 0 FAILED # ./test_progs -t get_func #138 get_func_args_test:OK #139 get_func_ip_test:OK Summary: 2/0 PASSED, 0 SKIPPED, 0 FAILED v1->v2: Resolve merge conflict [v1]: https://lore.kernel.org/bpf/20260216155310.38457-1-adubey@linux.ibm.com Signed-off-by: Abhishek Dubey --- tools/testing/selftests/bpf/progs/get_func_args_test.c | 3 ++- tools/testing/selftests/bpf/progs/get_func_ip_test.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/progs/get_func_args_test.c b/tools/testing/selftests/bpf/progs/get_func_args_test.c index 075a1180ec26..1fe1fb0548ea 100644 --- a/tools/testing/selftests/bpf/progs/get_func_args_test.c +++ b/tools/testing/selftests/bpf/progs/get_func_args_test.c @@ -167,7 +167,8 @@ int BPF_PROG(tp_test2) } __u64 test7_result = 0; -#if defined(bpf_target_x86) || defined(bpf_target_arm64) || defined(bpf_target_riscv) +#if defined(bpf_target_x86) || defined(bpf_target_arm64) || \ + defined(bpf_target_riscv) || defined(bpf_target_powerpc64) SEC("fsession/bpf_fentry_test1") int BPF_PROG(test7) { diff --git a/tools/testing/selftests/bpf/progs/get_func_ip_test.c b/tools/testing/selftests/bpf/progs/get_func_ip_test.c index 45eaa54d1ac7..be6b1a073b11 100644 --- a/tools/testing/selftests/bpf/progs/get_func_ip_test.c +++ b/tools/testing/selftests/bpf/progs/get_func_ip_test.c @@ -106,7 +106,8 @@ int BPF_URETPROBE(test8, int ret) __u64 test9_entry_result = 0; __u64 test9_exit_result = 0; -#if defined(bpf_target_x86) || defined(bpf_target_arm64) || defined(bpf_target_riscv) +#if defined(bpf_target_x86) || defined(bpf_target_arm64) || \ + defined(bpf_target_riscv) || defined(bpf_target_powerpc64) SEC("fsession/bpf_fentry_test1") int BPF_PROG(test9, int a) { -- 2.52.0 From: Abhishek Dubey Extend JIT support of fsession in powerpc64 trampoline, since ppc64 and ppc32 shares common trampoline implementation. Arch specific helpers handle 64-bit data copy using 32 bit regs. Need to validate fsession support along with trampoline support. v1->v2: No change since v1 [v1]: https://lore.kernel.org/bpf/20260216155310.38457-1-adubey@linux.ibm.com Signed-off-by: Abhishek Dubey --- arch/powerpc/net/bpf_jit_comp.c | 8 ++++++- arch/powerpc/net/bpf_jit_comp32.c | 35 +++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 6d334c0c3d85..8e6bfba37a67 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -542,7 +542,13 @@ bool bpf_jit_supports_private_stack(void) bool bpf_jit_supports_fsession(void) { - return IS_ENABLED(CONFIG_PPC64); + /* + * TODO: Remove after validating support + * for fsession and trampoline on ppc32. + */ + if (IS_ENABLED(CONFIG_PPC32)) + return -EOPNOTSUPP; + return true; } bool bpf_jit_supports_arena(void) diff --git a/arch/powerpc/net/bpf_jit_comp32.c b/arch/powerpc/net/bpf_jit_comp32.c index 3087e744fb25..72dc2a55d66c 100644 --- a/arch/powerpc/net/bpf_jit_comp32.c +++ b/arch/powerpc/net/bpf_jit_comp32.c @@ -123,6 +123,41 @@ void bpf_jit_realloc_regs(struct codegen_context *ctx) } } +void prepare_for_fsession_fentry(u32 *image, struct codegen_context *ctx, int cookie_cnt, + int cookie_off, int retval_off) +{ + /* + * Set session cookies value + * Clear cookies field on stack + * Ensure retval to be cleared on fentry + */ + EMIT(PPC_RAW_LI(bpf_to_ppc(TMP_REG), 0)); + + for (int i = 0; i < cookie_cnt; i++) { + EMIT(PPC_RAW_STW(bpf_to_ppc(TMP_REG), _R1, cookie_off + 4 * i)); + EMIT(PPC_RAW_STW(bpf_to_ppc(TMP_REG), _R1, cookie_off + 4 * i + 4)); + } + + EMIT(PPC_RAW_STW(bpf_to_ppc(TMP_REG), _R1, retval_off)); + EMIT(PPC_RAW_STW(bpf_to_ppc(TMP_REG), _R1, retval_off + 4)); +} + +void store_func_meta(u32 *image, struct codegen_context *ctx, + u64 func_meta, int func_meta_off) +{ + /* + * Store func_meta to stack: [R1 + func_meta_off] = func_meta + * func_meta := argument count in first byte + cookie value + */ + /* Store lower word */ + EMIT(PPC_RAW_LI(bpf_to_ppc(TMP_REG), (u32)func_meta)); + EMIT(PPC_RAW_STW(bpf_to_ppc(TMP_REG), _R1, func_meta_off)); + + /* Store upper word */ + EMIT(PPC_RAW_LI(bpf_to_ppc(TMP_REG), (u32)(func_meta >> 32))); + EMIT(PPC_RAW_STW(bpf_to_ppc(TMP_REG), _R1, func_meta_off + 4)); +} + void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) { int i; -- 2.52.0