Arena pointer kfunc calls and struct_ops callbacks need different JIT support. The former rebases BPF arena offsets before a kfunc call, while the latter converts kernel pointers when an indirect trampoline builds a callback context. A single bpf_jit_supports_arena_args() hook forces an architecture to implement both paths at once. That ties bpf_arena_alloc_pages() conversion to struct_ops trampoline support and prevents the paths from being enabled and reviewed independently. Replace it with separate kfunc and struct_ops capability hooks. Make the verifier query the hook for the path it is checking, and have x86-64 and arm64 advertise both capabilities to preserve their current behavior. Signed-off-by: Kumar Kartikeya Dwivedi --- arch/arm64/net/bpf_jit_comp.c | 7 ++++++- arch/x86/net/bpf_jit_comp.c | 7 ++++++- include/linux/filter.h | 3 ++- kernel/bpf/core.c | 7 ++++++- kernel/bpf/verifier.c | 4 ++-- 5 files changed, 22 insertions(+), 6 deletions(-) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 3aa3ea0bc30b..eecaa0027a95 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -2398,7 +2398,12 @@ bool bpf_jit_supports_stack_args(void) return true; } -bool bpf_jit_supports_arena_args(void) +bool bpf_jit_supports_arena_kfunc_args(void) +{ + return true; +} + +bool bpf_jit_supports_arena_struct_ops_args(void) { return true; } diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 48429fae0641..6e89f1c8738b 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -4174,7 +4174,12 @@ bool bpf_jit_supports_stack_args(void) return true; } -bool bpf_jit_supports_arena_args(void) +bool bpf_jit_supports_arena_kfunc_args(void) +{ + return true; +} + +bool bpf_jit_supports_arena_struct_ops_args(void) { return true; } diff --git a/include/linux/filter.h b/include/linux/filter.h index 6e746b0a0930..907d774fd355 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1239,7 +1239,8 @@ bool bpf_jit_supports_percpu_insn(void); bool bpf_jit_supports_kfunc_call(void); bool bpf_jit_supports_kfunc_ret_reg_pair(void); bool bpf_jit_supports_stack_args(void); -bool bpf_jit_supports_arena_args(void); +bool bpf_jit_supports_arena_kfunc_args(void); +bool bpf_jit_supports_arena_struct_ops_args(void); bool bpf_jit_supports_far_kfunc_call(void); bool bpf_jit_supports_exceptions(void); bool bpf_jit_supports_ptr_xchg(void); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 5db77d7915df..92b0a3bf27be 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3297,7 +3297,12 @@ bool __weak bpf_jit_supports_stack_args(void) return false; } -bool __weak bpf_jit_supports_arena_args(void) +bool __weak bpf_jit_supports_arena_kfunc_args(void) +{ + return false; +} + +bool __weak bpf_jit_supports_arena_struct_ops_args(void) { return false; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index e036ae20bf6b..6402e94c2097 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -12013,7 +12013,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, else if (is_kfunc_arg_callback(env, meta->btf, &args[arg])) arg_type = KF_ARG_PTR_TO_CALLBACK; else if (is_kfunc_arg_arena(meta->btf, &args[arg])) { - if (!bpf_jit_supports_arena_args()) { + if (!bpf_jit_supports_arena_kfunc_args()) { verbose(env, "JIT does not support kfunc %s() with arena pointer arguments\n", meta->func_name); return -ENOTSUPP; @@ -19778,7 +19778,7 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) if (info->refcounted) has_refcounted_arg = true; if (base_type(info->reg_type) == PTR_TO_ARENA) { - if (!bpf_jit_supports_arena_args()) { + if (!bpf_jit_supports_arena_struct_ops_args()) { verbose(env, "JIT does not support arena arguments\n"); return -ENOTSUPP; } -- 2.53.0 store_args() reads stack-passed arguments relative to FP assuming the trampoline was entered through the fentry call from a traced function. In that path, the trampoline pushes the parent frame before establishing its final FP, so the incoming stack arguments start at FP + 16. An indirect trampoline for a struct_ops callback is called through a function pointer. Its prologue allocates only the trampoline frame and sets FP to the incoming SP. The RISC-V ABI places the first stack argument at that incoming SP, so the arguments start at FP, not FP + 16. Every stack-passed argument of a callback with more than eight argument slots is therefore read two slots late. Pass the prologue-dependent offset to store_args(), using zero for a direct struct_ops trampoline and 16 for the fentry path. Fixes: 6801b0aef79d ("riscv, bpf: Add 12-argument support for RV64 bpf trampoline") Cc: Björn Töpel Cc: Pu Lehui Signed-off-by: Kumar Kartikeya Dwivedi --- arch/riscv/net/bpf_jit_comp64.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 47c7bf431ba8..b1084f30f3ce 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -857,7 +857,8 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, return ret; } -static void store_args(int nr_arg_slots, int args_off, struct rv_jit_context *ctx) +static void store_args(int nr_arg_slots, int args_off, int stack_args_off, + struct rv_jit_context *ctx) { int i; @@ -865,8 +866,8 @@ static void store_args(int nr_arg_slots, int args_off, struct rv_jit_context *ct if (i < RV_MAX_REG_ARGS) { emit_sd(RV_REG_FP, -args_off, RV_REG_A0 + i, ctx); } else { - /* skip slots for T0 and FP of traced function */ - emit_ld(RV_REG_T1, 16 + (i - RV_MAX_REG_ARGS) * 8, RV_REG_FP, ctx); + emit_ld(RV_REG_T1, stack_args_off + + (i - RV_MAX_REG_ARGS) * 8, RV_REG_FP, ctx); emit_sd(RV_REG_FP, -args_off, RV_REG_T1, ctx); } args_off -= 8; @@ -1152,7 +1153,12 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, func_meta = nr_arg_slots; emit_store_stack_imm64(RV_REG_T1, -func_meta_off, func_meta, ctx); - store_args(nr_arg_slots, args_off, ctx); + /* + * A direct struct_ops call has its first stack argument at the incoming + * SP, which the trampoline keeps as FP. The fentry path pushes the + * parent frame first, so its incoming stack arguments start at FP + 16. + */ + store_args(nr_arg_slots, args_off, is_struct_ops ? 0 : 16, ctx); if (bpf_fsession_cnt(tnodes)) { /* clear all session cookies' value */ -- 2.53.0 A BPF arena pointer is represented as a zero-extended 32-bit offset while a kfunc receives a kernel address. Kfuncs whose BTF argument names carry the __arena suffix therefore need the JIT to rebase those offsets immediately before the native call. RV_REG_ARENA already holds kern_vm_start whenever the program uses an arena. Zero-extend each tagged argument and add that base: zext.w aN, aN add aN, s7, aN For an __arena__nullable argument, branch over the fixed-width add when the truncated offset is zero so that NULL remains NULL. An unconditionally tagged zero is intentionally converted to the arena base. The sequence is emitted as native code after the BPF instruction stream has been blinded, and its size depends only on the function model and enabled ISA extensions. Advertise the kfunc capability independently; struct_ops argument conversion is not enabled by this change. Cc: Björn Töpel Cc: Pu Lehui Signed-off-by: Kumar Kartikeya Dwivedi --- arch/riscv/net/bpf_jit_comp64.c | 40 +++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index b1084f30f3ce..c97d13a3eae4 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -714,6 +714,37 @@ static int sign_extend(u8 rd, u8 rs, u8 sz, bool sign, struct rv_jit_context *ct return 0; } +/* + * Rebase the __arena args of a kfunc call to arena kernel addresses, + * aN = kern_vm_start + (u32)aN, with RV_REG_ARENA holding kern_vm_start. + * A nullable arg preserves NULL by skipping the add, tested on the + * truncated value as arena NULL is offset 0. + */ +static int emit_kfunc_arena_args(struct rv_jit_context *ctx, + const struct btf_func_model *fm) +{ + int i; + + for (i = 0; i < min_t(int, fm->nr_args, MAX_BPF_FUNC_REG_ARGS); i++) { + u8 flags = fm->arg_flags[i]; + u8 reg = bpf_to_rv_reg(BPF_REG_1 + i, ctx); + + if (!(flags & BTF_FMODEL_ARENA_ARG)) + continue; + if (WARN_ON_ONCE(!ctx->arena_vm_start)) + return -EINVAL; + + emit_zextw(reg, reg, ctx); + if (flags & BTF_FMODEL_NULLABLE_ARG) { + /* Skip the fixed-width add so that NULL stays NULL. */ + emit(rv_beq(reg, RV_REG_ZERO, 4), ctx); + } + emit(rv_add(reg, RV_REG_ARENA, reg), ctx); + } + + return 0; +} + #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) #define BPF_FIXUP_REG_MASK GENMASK(31, 27) #define REG_DONT_CLEAR_MARKER 0 /* RV_REG_ZERO unused in pt_regmap */ @@ -1834,6 +1865,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, if (sign_extend(reg, reg, fm->arg_size[idx], sign, ctx)) return -EINVAL; } + + ret = emit_kfunc_arena_args(ctx, fm); + if (ret) + return ret; } /* restore TCC to RV_REG_TCC before bpf2bpf call */ @@ -2132,6 +2167,11 @@ bool bpf_jit_supports_kfunc_ret_reg_pair(void) return true; } +bool bpf_jit_supports_arena_kfunc_args(void) +{ + return true; +} + bool bpf_jit_supports_ptr_xchg(void) { return true; -- 2.53.0 A struct_ops callback receives native kernel addresses, while its BPF program expects an arena pointer argument as a zero-extended 32-bit offset. Convert arguments marked with BTF_FMODEL_ARENA_ARG while the trampoline copies them into the BPF context. bpf_tramp_arena_base() supplies the known base only for the single-program indirect trampoline. Materialize its low 32 bits once in t2, subtract it from each tagged argument through t1, and zero-extend the result before storing it. For a nullable argument, preserve the full native pointer in t1 and branch over the variable-length subtraction sequence when it is NULL. Walk the function model by argument while keeping a separate ABI slot index. This keeps the arena flags aligned with the correct native register or stack slot when an earlier argument occupies two slots, including the case where a 16-byte argument straddles a7 and the stack. Registered and stack-passed arena pointers use the same conversion helper. bpf_tramp_arena_base() returns zero for tracing trampolines, so their emitted argument-save sequence is unchanged. An indirect trampoline cannot call the original function, which ensures a converted pointer never escapes back into a native callback. Advertise the struct_ops capability independently now that the reverse conversion is implemented. Cc: Björn Töpel Cc: Pu Lehui Signed-off-by: Kumar Kartikeya Dwivedi --- arch/riscv/net/bpf_jit_comp64.c | 93 +++++++++++++++++++++++++++++---- 1 file changed, 82 insertions(+), 11 deletions(-) diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index c97d13a3eae4..8db992a285f6 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -888,20 +888,75 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, return ret; } -static void store_args(int nr_arg_slots, int args_off, int stack_args_off, +/* + * Convert an arena kernel address into the arena pointer form on its way + * into the BPF ctx, dst = (u32)(src - kern_vm_start). A nullable arg + * preserves NULL, tested on the full 64-bit kernel pointer. The final + * zero-extension makes the stored value satisfy the JIT invariant for arena + * pointer registers. + */ +static void emit_arena_arg_conv(u8 dst, u8 src, bool nullable, u8 base, + struct rv_jit_context *ctx) +{ + int branch_off = 0; + + if (nullable) { + if (dst != src) + emit_mv(dst, src, ctx); + branch_off = ctx->ninsns; + /* Patched below once the variable-length conversion is emitted. */ + emit(rv_nop(), ctx); + src = dst; + } + + emit_sub(dst, src, base, ctx); + emit_zextw(dst, dst, ctx); + + if (nullable && ctx->insns) { + u32 insn = rv_beq(dst, RV_REG_ZERO, ctx->ninsns - branch_off); + + *(u32 *)(ctx->insns + branch_off) = insn; + } +} + +static void store_args(const struct btf_func_model *m, int args_off, + int stack_args_off, u64 arena_base, struct rv_jit_context *ctx) { - int i; + int i, j, slot = 0; - for (i = 0; i < nr_arg_slots; i++) { - if (i < RV_MAX_REG_ARGS) { - emit_sd(RV_REG_FP, -args_off, RV_REG_A0 + i, ctx); - } else { - emit_ld(RV_REG_T1, stack_args_off + - (i - RV_MAX_REG_ARGS) * 8, RV_REG_FP, ctx); - emit_sd(RV_REG_FP, -args_off, RV_REG_T1, ctx); + /* Only the low 32 bits of the base take part in the subtraction. */ + if (arena_base) + emit_imm(RV_REG_T2, (s32)(u32)arena_base, ctx); + + /* + * Walk arguments and slots together so a 16-byte argument consumes two + * ABI locations before the flags for the following argument are used. + */ + for (i = 0; i < m->nr_args; i++) { + bool arena_arg = arena_base && (m->arg_flags[i] & BTF_FMODEL_ARENA_ARG); + bool nullable = m->arg_flags[i] & BTF_FMODEL_NULLABLE_ARG; + int slots = round_up(m->arg_size[i], 8) / 8; + + for (j = 0; j < slots; j++, slot++) { + u8 src; + + if (slot < RV_MAX_REG_ARGS) { + src = RV_REG_A0 + slot; + } else { + emit_ld(RV_REG_T1, stack_args_off + + (slot - RV_MAX_REG_ARGS) * 8, RV_REG_FP, ctx); + src = RV_REG_T1; + } + + if (arena_arg) { + emit_arena_arg_conv(RV_REG_T1, src, nullable, + RV_REG_T2, ctx); + src = RV_REG_T1; + } + emit_sd(RV_REG_FP, -args_off, src, ctx); + args_off -= 8; } - args_off -= 8; } } @@ -1039,9 +1094,20 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, bool is_struct_ops = is_struct_ops_tramp(fentry); void *orig_call = func_addr; bool save_ret; + u64 arena_base; u64 func_meta; u32 insn; + /* + * F_INDIRECT is only compatible with F_RET_FENTRY_RET. In particular, + * an indirect trampoline never calls the original function with the + * arena arguments converted into their BPF representation. + */ + WARN_ON_ONCE((flags & BPF_TRAMP_F_INDIRECT) && + (flags & ~(BPF_TRAMP_F_INDIRECT | BPF_TRAMP_F_RET_FENTRY_RET))); + + arena_base = bpf_tramp_arena_base(m, tnodes, flags); + /* Two types of generated trampoline stack layout: * * 1. trampoline called from function entry @@ -1189,7 +1255,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, * SP, which the trampoline keeps as FP. The fentry path pushes the * parent frame first, so its incoming stack arguments start at FP + 16. */ - store_args(nr_arg_slots, args_off, is_struct_ops ? 0 : 16, ctx); + store_args(m, args_off, is_struct_ops ? 0 : 16, arena_base, ctx); if (bpf_fsession_cnt(tnodes)) { /* clear all session cookies' value */ @@ -2172,6 +2238,11 @@ bool bpf_jit_supports_arena_kfunc_args(void) return true; } +bool bpf_jit_supports_arena_struct_ops_args(void) +{ + return true; +} + bool bpf_jit_supports_ptr_xchg(void) { return true; -- 2.53.0 Kfuncs with __arena arguments expect kernel addresses, while BPF programs carry arena pointers as zero-extended 32-bit offsets. The s390 JIT does not translate those offsets at the call boundary, so advertising arena kfunc argument support would pass invalid addresses to the callee. Use the kfunc BTF model after the existing argument width normalization to find arena arguments. Load kern_vm_start from the existing arena literal once per call, truncate each tagged argument with llgfr, and add the base. For __arena__nullable, test the truncated value and skip the add so offset zero remains NULL. Advertise the kfunc-specific arena argument capability. Struct_ops trampoline conversion is independent and remains disabled until its own support is added. Cc: Ilya Leoshkevich Cc: Heiko Carstens Cc: Vasily Gorbik Signed-off-by: Kumar Kartikeya Dwivedi --- arch/s390/net/bpf_jit_comp.c | 49 ++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index c46872b071ce..20b6e53999b3 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -946,6 +946,48 @@ static int sign_zero_extend(struct bpf_jit *jit, int r, u8 size, u8 flags) } } +/* + * Rebase the __arena args of a kfunc call to arena kernel addresses, + * rN = kern_vm_start + (u32)rN. A nullable arg preserves NULL by skipping + * the add, tested on the truncated value as arena NULL is offset 0. + */ +static int emit_kfunc_arena_args(struct bpf_jit *jit, struct bpf_prog *fp, + const struct btf_func_model *m) +{ + bool base_loaded = false; + int i; + + for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_REG_ARGS); i++) { + u8 flags = m->arg_flags[i]; + int arg = BPF_REG_1 + i; + + if (!(flags & BTF_FMODEL_ARENA_ARG)) + continue; + if (WARN_ON_ONCE(!fp->aux->arena)) + return -EINVAL; + + if (!base_loaded) { + /* lgrl %w0,kern_arena */ + EMIT6_PCREL_RILB(0xc4080000, REG_W0, jit->kern_arena); + base_loaded = true; + } + + /* llgfr %arg,%arg: truncate and clear the upper 32 bits */ + EMIT4(0xb9160000, arg, arg); + if (flags & BTF_FMODEL_NULLABLE_ARG) { + /* ltgr %arg,%arg */ + EMIT4(0xb9020000, arg, arg); + /* brc 8,1f */ + EMIT4_PCREL_RIC(0xa7040000, 8, jit->prg + 8); + } + /* agr %arg,%w0 */ + EMIT4(0xb9080000, arg, REG_W0); + /* 1: */ + } + + return 0; +} + /* * Compile one eBPF instruction into s390x code * @@ -1867,6 +1909,8 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, m->arg_flags[j])) return -1; } + if (emit_kfunc_arena_args(jit, fp, m)) + return -1; } if ((void *)func == arch_bpf_timed_may_goto) { @@ -2459,6 +2503,11 @@ bool bpf_jit_supports_kfunc_call(void) return true; } +bool bpf_jit_supports_arena_kfunc_args(void) +{ + return true; +} + bool bpf_jit_supports_far_kfunc_call(void) { return true; -- 2.53.0 A struct_ops callback receives a native kernel arena address, while its BPF program expects the corresponding zero-extended 32-bit arena offset in the ctx. The s390 trampoline currently copies native arguments verbatim, so it cannot support callbacks whose stub marks an argument with __arena. Obtain the arena base for the single-program indirect trampoline and convert each tagged argument while copying it into the BPF ctx. Test a nullable source as a full 64-bit kernel pointer, subtract the low 32 bits of kern_vm_start, and zero-extend the result. This preserves NULL and provides the register form required by arena loads. Keep the native argument index separate from the BPF ctx slot index. The former selects r2-r6 or one caller stack slot per s390 ABI argument, while the latter still expands 16-byte arguments to two slots. This also converts arena pointers passed after the fifth argument without shifting later values. bpf_tramp_arena_base() returns a base only for an indirect struct_ops trampoline. Assert the incompatible flag combinations so converted arguments can never be passed back to the original kernel function, and advertise the struct_ops-specific arena argument capability. Cc: Ilya Leoshkevich Cc: Heiko Carstens Cc: Vasily Gorbik Signed-off-by: Kumar Kartikeya Dwivedi --- arch/s390/net/bpf_jit_comp.c | 69 ++++++++++++++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 3 deletions(-) diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index 20b6e53999b3..964f1ad82637 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -2607,6 +2607,34 @@ static void load_imm64(struct bpf_jit *jit, int dst_reg, u64 val) EMIT6_IMM(0xc00d0000, dst_reg, val); } +/* + * Convert an arena kernel address into the arena pointer form on its way + * into the BPF ctx, dst = (u32)(src - kern_vm_start). A nullable arg + * preserves NULL, tested on the full 64-bit kernel pointer. The 32-bit + * subtraction followed by zero-extension keeps the upper half clear. + */ +static void emit_arena_arg_conv(struct bpf_jit *jit, int dst, int src, + bool nullable, u32 base_lo) +{ + if (dst != src) { + /* lgr %dst,%src */ + EMIT4(0xb9040000, dst, src); + } + if (nullable) { + /* ltgr %dst,%dst */ + EMIT4(0xb9020000, dst, dst); + /* brc 8,1f */ + EMIT4_PCREL_RIC(0xa7040000, 8, jit->prg + 16); + } + /* llilf %w1,base_lo */ + EMIT6_IMM(0xc00f0000, REG_W1, base_lo); + /* sr %dst,%w1 */ + EMIT2(0x1b00, dst, REG_W1); + /* llgfr %dst,%dst */ + EMIT4(0xb9160000, dst, dst); + /* 1: */ +} + static void emit_store_stack_imm64(struct bpf_jit *jit, int tmp_reg, int stack_off, u64 imm) { load_imm64(jit, tmp_reg, imm); @@ -2740,6 +2768,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, int cookie_cnt, cookie_off, fsession_cnt; struct bpf_jit *jit = &tjit->common; int arg, bpf_arg_off; + u64 arena_base; u64 func_meta; int i, j; @@ -2749,6 +2778,16 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, if (nr_stack_args > MAX_NR_STACK_ARGS) return -ENOTSUPP; + /* + * F_INDIRECT is only compatible with F_RET_FENTRY_RET. Arena conversion + * relies on the indirect trampoline never calling the original function + * with converted arguments. + */ + WARN_ON_ONCE((flags & BPF_TRAMP_F_INDIRECT) && + (flags & ~(BPF_TRAMP_F_INDIRECT | BPF_TRAMP_F_RET_FENTRY_RET))); + + arena_base = bpf_tramp_arena_base(m, tnodes, flags); + /* Return to %r14 in the struct_ops case. */ if (flags & BPF_TRAMP_F_INDIRECT) flags |= BPF_TRAMP_F_SKIP_FRAME; @@ -2829,14 +2868,33 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, (i - MAX_NR_REG_ARGS) * sizeof(u64); bpf_arg_off = tjit->bpf_args_off + j * sizeof(u64); if (m->arg_size[i] <= 8) { - if (i < MAX_NR_REG_ARGS) + bool arena_arg = arena_base && + (m->arg_flags[i] & BTF_FMODEL_ARENA_ARG); + bool nullable = m->arg_flags[i] & BTF_FMODEL_NULLABLE_ARG; + + if (arena_arg) { + if (i < MAX_NR_REG_ARGS) { + emit_arena_arg_conv(jit, REG_W0, arg, nullable, + (u32)arena_base); + } else { + /* lg %w0,arg(%r15) */ + EMIT6_DISP_LH(0xe3000000, 0x0004, REG_W0, + REG_0, REG_15, arg); + emit_arena_arg_conv(jit, REG_W0, REG_W0, + nullable, (u32)arena_base); + } + /* stg %w0,bpf_arg_off(%r15) */ + EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W0, + REG_0, REG_15, bpf_arg_off); + } else if (i < MAX_NR_REG_ARGS) { /* stg %arg,bpf_arg_off(%r15) */ EMIT6_DISP_LH(0xe3000000, 0x0024, arg, - REG_0, REG_15, bpf_arg_off); - else + REG_0, REG_15, bpf_arg_off); + } else { /* mvc bpf_arg_off(8,%r15),arg(%r15) */ _EMIT6(0xd207f000 | bpf_arg_off, 0xf000 | arg); + } j += 1; } else { if (i < MAX_NR_REG_ARGS) { @@ -3092,6 +3150,11 @@ bool bpf_jit_supports_subprog_tailcalls(void) return true; } +bool bpf_jit_supports_arena_struct_ops_args(void) +{ + return true; +} + bool bpf_jit_supports_arena(void) { return true; -- 2.53.0 LoongArch passes arguments beyond a0-a7 at the caller stack pointer. The trampoline store_args() helper always reads those arguments at FP + 16, which is correct for an fentry trampoline: its prologue leaves FP 16 bytes below the stack pointer at trampoline entry after accounting for the saved parent and traced-function frames. A struct_ops indirect trampoline is entered through a function pointer and only saves its own RA and FP before setting FP to the entry stack pointer. Its stack arguments therefore start at FP, not FP + 16. As a result, every stack-passed struct_ops argument is currently read two slots late. Select the source offset based on whether the trampoline is indirect. This also prepares the stack-passed arena argument path to consume the actual pointer slot. Fixes: c9ebe2016de9 ("LoongArch: BPF: Support up to 12 function arguments for trampoline") Cc: Tiezhu Yang Cc: Huacai Chen Signed-off-by: Kumar Kartikeya Dwivedi --- arch/loongarch/net/bpf_jit.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 29c281bef28e..d193293a0fd2 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -1662,17 +1662,18 @@ int bpf_arch_text_invalidate(void *dst, size_t len) return ret; } -static void store_args(struct jit_ctx *ctx, int nr_arg_slots, int args_off) +static void store_args(struct jit_ctx *ctx, int nr_arg_slots, int args_off, bool is_struct_ops) { + int stack_args_off = is_struct_ops ? 0 : 16; int i; for (i = 0; i < nr_arg_slots; i++) { if (i < LOONGARCH_MAX_REG_ARGS) emit_insn(ctx, std, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off); else { - /* Skip slots for T0 and FP of traced function */ + /* Skip the saved T0 and FP slots for a traced function. */ emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, - 16 + (i - LOONGARCH_MAX_REG_ARGS) * 8); + stack_args_off + (i - LOONGARCH_MAX_REG_ARGS) * 8); emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -args_off); } args_off -= 8; @@ -1995,7 +1996,7 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i func_meta = nr_arg_slots; emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, -func_meta_off, func_meta); - store_args(ctx, nr_arg_slots, args_off); + store_args(ctx, nr_arg_slots, args_off, is_struct_ops); if (bpf_fsession_cnt(tnodes)) { /* clear all session cookies' value */ -- 2.53.0 Kfunc parameters marked with BTF_FMODEL_ARENA_ARG carry a 32-bit arena offset in the BPF register, while the native kfunc expects a directly dereferenceable kernel address. Without JIT conversion, moving arena allocation kfuncs from KF_ARENA_ARGS to BTF argument metadata would make LoongArch pass offsets to functions which dereference them. The LoongArch JIT already keeps the arena kernel mapping base in s6 for arena memory accesses. Reuse it while preparing a pseudo-kfunc call: clear the upper 32 bits of each marked argument, preserve a nullable zero by branching over the addition, and otherwise add the base in s6. Keep the existing ABI extension path unchanged for ordinary kfunc arguments. Advertise the kfunc-specific arena argument capability independently of struct_ops trampoline conversion. Cc: Tiezhu Yang Cc: Huacai Chen Signed-off-by: Kumar Kartikeya Dwivedi --- arch/loongarch/net/bpf_jit.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index d193293a0fd2..33cabaa7353f 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -283,6 +283,11 @@ bool bpf_jit_supports_kfunc_call(void) return true; } +bool bpf_jit_supports_arena_kfunc_args(void) +{ + return true; +} + bool bpf_jit_supports_far_kfunc_call(void) { return true; @@ -1195,9 +1200,22 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext for (i = 0; i < m->nr_args; i++) { u8 reg = regmap[BPF_REG_1 + i]; - bool sign = m->arg_flags[i] & BTF_FMODEL_SIGNED_ARG; - - emit_abi_ext(ctx, reg, m->arg_size[i], sign); + u8 flags = m->arg_flags[i]; + + if (flags & BTF_FMODEL_ARENA_ARG) { + if (WARN_ON_ONCE(!ctx->arena_vm_start)) + return -EINVAL; + + /* rN = kern_vm_start + (u32)rN */ + emit_zext_32(ctx, reg, true); + if (flags & BTF_FMODEL_NULLABLE_ARG) + emit_insn(ctx, beq, reg, LOONGARCH_GPR_ZERO, 2); + emit_insn(ctx, addd, reg, reg, REG_ARENA); + continue; + } + + emit_abi_ext(ctx, reg, m->arg_size[i], + flags & BTF_FMODEL_SIGNED_ARG); } } -- 2.53.0 A struct_ops callback receives native kernel addresses from its caller, but an arena BPF program expects each argument marked with BTF_FMODEL_ARENA_ARG to be a 32-bit offset from its arena mapping. LoongArch currently copies the native pointer into the BPF ctx unchanged. Obtain the mapping base through bpf_tramp_arena_base() and materialize it in t2 while saving arguments. Copy a marked argument through t1, preserve a nullable native NULL by branching over the subtraction, and otherwise subtract the base before clearing the upper 32 bits. Truncating the full address difference produces the required arena offset. Walk the function model by argument while maintaining a separate ABI slot index. This keeps argument flags aligned when an earlier small struct spans two slots, while preserving the existing register and stack slot layout. The preceding indirect-trampoline fix supplies the correct stack source for arguments beyond a0-a7. bpf_tramp_arena_base() only returns a base for the single-program indirect trampoline. Assert that such a trampoline cannot call the original function, which expects unconverted kernel addresses, and advertise the struct_ops-specific arena argument capability. Cc: Tiezhu Yang Cc: Huacai Chen Signed-off-by: Kumar Kartikeya Dwivedi --- arch/loongarch/net/bpf_jit.c | 73 ++++++++++++++++++++++++++++++------ 1 file changed, 61 insertions(+), 12 deletions(-) diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 33cabaa7353f..5020414bfd9f 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -288,6 +288,11 @@ bool bpf_jit_supports_arena_kfunc_args(void) return true; } +bool bpf_jit_supports_arena_struct_ops_args(void) +{ + return true; +} + bool bpf_jit_supports_far_kfunc_call(void) { return true; @@ -1680,21 +1685,55 @@ int bpf_arch_text_invalidate(void *dst, size_t len) return ret; } -static void store_args(struct jit_ctx *ctx, int nr_arg_slots, int args_off, bool is_struct_ops) +/* + * Convert an arena kernel address into a 32-bit arena offset while copying it + * into the BPF ctx. A nullable argument preserves a native NULL. + */ +static void emit_arena_arg_conv(struct jit_ctx *ctx, int dst, int src, bool nullable, int base) +{ + if (dst != src) + move_reg(ctx, dst, src); + if (nullable) + emit_insn(ctx, beq, dst, LOONGARCH_GPR_ZERO, 2); + emit_insn(ctx, subd, dst, dst, base); + emit_zext_32(ctx, dst, true); +} + +static void store_args(struct jit_ctx *ctx, const struct btf_func_model *m, int args_off, + bool is_struct_ops, u64 arena_base) { int stack_args_off = is_struct_ops ? 0 : 16; - int i; + int i, slot = 0; + + if (arena_base) + move_imm(ctx, LOONGARCH_GPR_T2, arena_base, false); - for (i = 0; i < nr_arg_slots; i++) { - if (i < LOONGARCH_MAX_REG_ARGS) - emit_insn(ctx, std, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off); - else { - /* Skip the saved T0 and FP slots for a traced function. */ - emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, - stack_args_off + (i - LOONGARCH_MAX_REG_ARGS) * 8); - emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -args_off); + for (i = 0; i < m->nr_args; i++) { + bool arena_arg = arena_base && (m->arg_flags[i] & BTF_FMODEL_ARENA_ARG); + bool nullable = m->arg_flags[i] & BTF_FMODEL_NULLABLE_ARG; + int slots = round_up(m->arg_size[i], 8) / 8; + + while (slots-- > 0) { + int src; + + if (slot < LOONGARCH_MAX_REG_ARGS) { + src = LOONGARCH_GPR_A0 + slot; + } else { + /* Skip the saved T0 and FP slots for a traced function. */ + emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, + stack_args_off + + (slot - LOONGARCH_MAX_REG_ARGS) * 8); + src = LOONGARCH_GPR_T1; + } + if (arena_arg) { + emit_arena_arg_conv(ctx, LOONGARCH_GPR_T1, src, nullable, + LOONGARCH_GPR_T2); + src = LOONGARCH_GPR_T1; + } + emit_insn(ctx, std, src, LOONGARCH_GPR_FP, -args_off); + slot++; + args_off -= 8; } - args_off -= 8; } } @@ -1868,6 +1907,7 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i struct bpf_tramp_nodes *fexit = &tnodes[BPF_TRAMP_FEXIT]; struct bpf_tramp_nodes *fmod_ret = &tnodes[BPF_TRAMP_MODIFY_RETURN]; u32 **branches = NULL; + u64 arena_base; /* * FP + 8 [ RA to parent func ] return address to parent @@ -1922,6 +1962,15 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY)) return -ENOTSUPP; + /* + * An indirect trampoline never calls the original function. Arena + * conversion relies on this because the original takes kernel addresses. + */ + WARN_ON_ONCE((flags & BPF_TRAMP_F_INDIRECT) && + (flags & ~(BPF_TRAMP_F_INDIRECT | BPF_TRAMP_F_RET_FENTRY_RET))); + + arena_base = bpf_tramp_arena_base(m, tnodes, flags); + /* Room of trampoline frame to store return address and frame pointer */ stack_size = 16; @@ -2014,7 +2063,7 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i func_meta = nr_arg_slots; emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, -func_meta_off, func_meta); - store_args(ctx, nr_arg_slots, args_off, is_struct_ops); + store_args(ctx, m, args_off, is_struct_ops, arena_base); if (bpf_fsession_cnt(tnodes)) { /* clear all session cookies' value */ -- 2.53.0 Kfunc parameters annotated with __arena carry a 32-bit arena offset in a BPF register. The kernel function expects a directly dereferenceable kernel address, so the JIT must add the arena kernel mapping base before making the call. Nullable parameters must preserve offset zero as NULL. PowerPC64 already keeps the arena kernel base in r26 for arena memory accesses. Reuse it in the kfunc ABI preparation path: zero-extend each arena argument to 32 bits, skip the addition for nullable zero, and otherwise add r26. Advertise the kfunc-argument capability on PowerPC64 so arena allocation kfuncs can move to suffix annotations without losing PowerPC support. Cc: Hari Bathini Cc: Christophe Leroy Cc: Naveen N. Rao Signed-off-by: Kumar Kartikeya Dwivedi --- arch/powerpc/net/bpf_jit_comp.c | 5 +++++ arch/powerpc/net/bpf_jit_comp64.c | 18 +++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 7b07b43575f1..911088cbabb9 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -527,6 +527,11 @@ bool bpf_jit_supports_kfunc_call(void) return IS_ENABLED(CONFIG_PPC64); } +bool bpf_jit_supports_arena_kfunc_args(void) +{ + return IS_ENABLED(CONFIG_PPC64); +} + bool bpf_jit_supports_private_stack(void) { return IS_ENABLED(CONFIG_PPC64); diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index fc9db691e820..fc235fbfbeb9 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -624,8 +624,24 @@ static int prepare_for_kfunc_call(const struct bpf_prog *fp, u32 *image, for (i = 0; i < m->nr_args; i++) { /* Note that BPF ABI only allows up to 5 args for kfuncs */ u32 reg = bpf_to_ppc(BPF_REG_1 + i), size = m->arg_size[i]; + u8 flags = m->arg_flags[i]; - if (!(m->arg_flags[i] & BTF_FMODEL_SIGNED_ARG)) { + if (flags & BTF_FMODEL_ARENA_ARG) { + if (WARN_ON_ONCE(!ctx->arena_vm_start)) + return -1; + + /* rN = kern_vm_start + (u32)rN */ + if (zero_extend(image, ctx, reg, reg, 4)) + return -1; + if (flags & BTF_FMODEL_NULLABLE_ARG) { + EMIT(PPC_RAW_CMPLDI(reg, 0)); + PPC_BCC_CONST_SHORT(COND_EQ, 8); + } + EMIT(PPC_RAW_ADD(reg, reg, bpf_to_ppc(ARENA_VM_START))); + continue; + } + + if (!(flags & BTF_FMODEL_SIGNED_ARG)) { if (zero_extend(image, ctx, reg, reg, size)) return -1; } else { -- 2.53.0 The arena allocation kfuncs still identify pointer arguments with KF_ARENA_ARG2. These flags cover only the first two parameters and duplicate the __arena suffix mechanism used by other kfuncs. Annotate the optional allocation address with __arena__nullable. Mark the free and reserve addresses with __arena so a valid address whose low 32 bits are zero is rebased unconditionally instead of becoming NULL. The JIT now passes kernel arena addresses to these kfuncs. Translate them back to the lower-32-bit user addresses expected by the existing arena helpers by subtracting kern_vm_start. This preserves allocation-anywhere, freeing the first page of a 4 GiB arena, and reservation at address zero. Drop KF_ARENA_ARG1 and KF_ARENA_ARG2 from the kernel interface and remove the flags from the arena kfunc sets. KF_ARENA_RET remains responsible for annotating the allocation return value. Keep the affected selftests synchronized with the conversion. Associate an arena before the iterator map-pointer failures so they still reach the intended diagnostics, account for the extra nullable branch in JIT labels, and treat 1ULL << 32 as the same allocation-anywhere request as NULL after the required 32-bit truncation. Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/btf.h | 2 - kernel/bpf/arena.c | 40 ++++++++++++++----- .../selftests/bpf/progs/arena_kfunc_jit.c | 16 ++++---- .../selftests/bpf/progs/verifier_arena.c | 6 +++ .../bpf/progs/verifier_arena_large.c | 4 +- 5 files changed, 46 insertions(+), 22 deletions(-) diff --git a/include/linux/btf.h b/include/linux/btf.h index 89d5a5c4f117..65e5f11dc27e 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -76,8 +76,6 @@ #define KF_RCU_PROTECTED (1 << 11) /* kfunc should be protected by rcu cs when they are invoked */ #define KF_FASTCALL (1 << 12) /* kfunc supports bpf_fastcall protocol */ #define KF_ARENA_RET (1 << 13) /* kfunc returns an arena pointer */ -#define KF_ARENA_ARG1 (1 << 14) /* kfunc takes an arena pointer as its first argument */ -#define KF_ARENA_ARG2 (1 << 15) /* kfunc takes an arena pointer as its second argument */ #define KF_IMPLICIT_ARGS (1 << 16) /* kfunc has implicit arguments supplied by the verifier */ #define KF_SPINLOCK_SAFE (1 << 17) /* kfunc is allowed inside bpf_spin_lock-ed region */ diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 7b6847200b43..6c34a0d34b3f 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -1044,18 +1044,28 @@ static void arena_free_irq(struct irq_work *iw) schedule_work(&arena->free_work); } +static long arena_kaddr_to_uaddr(struct bpf_arena *arena, const void *addr) +{ + if (!addr) + return 0; + + return (long)addr - bpf_arena_get_kern_vm_start(arena); +} + __bpf_kfunc_start_defs(); -__bpf_kfunc void *bpf_arena_alloc_pages(void *p__map, void *addr__ign, u32 page_cnt, - int node_id, u64 flags) +__bpf_kfunc void *bpf_arena_alloc_pages(void *p__map, void *addr__arena__nullable, + u32 page_cnt, int node_id, u64 flags) { struct bpf_map *map = p__map; struct bpf_arena *arena = container_of(map, struct bpf_arena, map); + long addr; if (map->map_type != BPF_MAP_TYPE_ARENA || flags || !page_cnt) return NULL; - return (void *)arena_alloc_pages(arena, (long)addr__ign, page_cnt, node_id, true); + addr = arena_kaddr_to_uaddr(arena, addr__arena__nullable); + return (void *)arena_alloc_pages(arena, addr, page_cnt, node_id, true); } void *bpf_arena_alloc_pages_non_sleepable(void *p__map, void *addr__ign, u32 page_cnt, @@ -1082,14 +1092,20 @@ void *bpf_arena_alloc_pages_sleepable(void *p__map, void *addr__ign, u32 page_cn return (void *)arena_alloc_pages(arena, (long)addr__ign, page_cnt, node_id, true); } -__bpf_kfunc void bpf_arena_free_pages(void *p__map, void *ptr__ign, u32 page_cnt) +/* + * A valid arena address can have zero low 32 bits, so ptr must be rebased + * unconditionally instead of being treated as nullable. + */ +__bpf_kfunc void bpf_arena_free_pages(void *p__map, void *ptr__arena, u32 page_cnt) { struct bpf_map *map = p__map; struct bpf_arena *arena = container_of(map, struct bpf_arena, map); + long ptr; - if (map->map_type != BPF_MAP_TYPE_ARENA || !page_cnt || !ptr__ign) + if (map->map_type != BPF_MAP_TYPE_ARENA || !page_cnt) return; - arena_free_pages(arena, (long)ptr__ign, page_cnt, true); + ptr = arena_kaddr_to_uaddr(arena, ptr__arena); + arena_free_pages(arena, ptr, page_cnt, true); } void bpf_arena_free_pages_non_sleepable(void *p__map, void *ptr__ign, u32 page_cnt) @@ -1102,10 +1118,11 @@ void bpf_arena_free_pages_non_sleepable(void *p__map, void *ptr__ign, u32 page_c arena_free_pages(arena, (long)ptr__ign, page_cnt, false); } -__bpf_kfunc int bpf_arena_reserve_pages(void *p__map, void *ptr__ign, u32 page_cnt) +__bpf_kfunc int bpf_arena_reserve_pages(void *p__map, void *ptr__arena, u32 page_cnt) { struct bpf_map *map = p__map; struct bpf_arena *arena = container_of(map, struct bpf_arena, map); + long ptr; if (map->map_type != BPF_MAP_TYPE_ARENA) return -EINVAL; @@ -1113,14 +1130,15 @@ __bpf_kfunc int bpf_arena_reserve_pages(void *p__map, void *ptr__ign, u32 page_c if (!page_cnt) return 0; - return arena_reserve_pages(arena, (long)ptr__ign, page_cnt); + ptr = arena_kaddr_to_uaddr(arena, ptr__arena); + return arena_reserve_pages(arena, ptr, page_cnt); } __bpf_kfunc_end_defs(); BTF_KFUNCS_START(arena_kfuncs) -BTF_ID_FLAGS(func, bpf_arena_alloc_pages, KF_ARENA_RET | KF_ARENA_ARG2 | KF_SPINLOCK_SAFE) -BTF_ID_FLAGS(func, bpf_arena_free_pages, KF_ARENA_ARG2 | KF_SPINLOCK_SAFE) -BTF_ID_FLAGS(func, bpf_arena_reserve_pages, KF_ARENA_ARG2 | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_arena_alloc_pages, KF_ARENA_RET | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_arena_free_pages, KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_arena_reserve_pages, KF_SPINLOCK_SAFE) BTF_KFUNCS_END(arena_kfuncs) static const struct btf_kfunc_id_set common_kfunc_set = { diff --git a/tools/testing/selftests/bpf/progs/arena_kfunc_jit.c b/tools/testing/selftests/bpf/progs/arena_kfunc_jit.c index b5a01cbc33a7..c9af35c683b3 100644 --- a/tools/testing/selftests/bpf/progs/arena_kfunc_jit.c +++ b/tools/testing/selftests/bpf/progs/arena_kfunc_jit.c @@ -49,15 +49,15 @@ __arch_x86_64 __jited("...") __jited(" movl %edi, %edi") __jited(" testl %edi, %edi") -__jited(" je L0") +__jited(" je L1") __jited(" addq %r12, %rdi") -__jited("L0: callq {{.*}}") +__jited("L1: callq {{.*}}") __arch_arm64 __jited("...") __jited(" mov w0, w0") -__jited(" cbz w0, L0") +__jited(" cbz w0, L1") __jited(" add x0, x28, w0, uxtw") -__jited("L0: {{.*}}") +__jited("L1: {{.*}}") __success int arena_arg_jit_nullable(void *ctx) { @@ -79,9 +79,9 @@ __jited(" movl %ecx, %ecx") __jited(" addq %r12, %rcx") __jited(" movl %r8d, %r8d") __jited(" testl %r8d, %r8d") -__jited(" je L0") +__jited(" je L1") __jited(" addq %r12, %r8") -__jited("L0: callq {{.*}}") +__jited("L1: callq {{.*}}") __arch_arm64 __jited("...") __jited(" add x0, x28, w0, uxtw") @@ -89,9 +89,9 @@ __jited(" add x1, x28, w1, uxtw") __jited(" add x2, x28, w2, uxtw") __jited(" add x3, x28, w3, uxtw") __jited(" mov w4, w4") -__jited(" cbz w4, L0") +__jited(" cbz w4, L1") __jited(" add x4, x28, w4, uxtw") -__jited("L0: {{.*}}") +__jited("L1: {{.*}}") __success int arena_arg_jit_args5(void *ctx) { diff --git a/tools/testing/selftests/bpf/progs/verifier_arena.c b/tools/testing/selftests/bpf/progs/verifier_arena.c index 815f342eb4b0..d76490e059f9 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena.c @@ -445,6 +445,8 @@ int iter_maps1(struct bpf_iter__bpf_map *ctx) if (!map) return 0; + /* Associate an arena before testing the generic map-pointer path. */ + bpf_arena_reserve_pages(&arena, NULL, 0); bpf_arena_alloc_pages(map, NULL, map->max_entries, 0, 0); return 0; } @@ -455,6 +457,8 @@ int iter_maps2(struct bpf_iter__bpf_map *ctx) { struct seq_file *seq = ctx->meta->seq; + /* Associate an arena before testing the generic map-pointer path. */ + bpf_arena_reserve_pages(&arena, NULL, 0); bpf_arena_alloc_pages((void *)seq, NULL, 1, 0, 0); return 0; } @@ -467,6 +471,8 @@ int iter_maps3(struct bpf_iter__bpf_map *ctx) if (!map) return 0; + /* Associate an arena before testing the generic map-pointer path. */ + bpf_arena_reserve_pages(&arena, NULL, 0); bpf_arena_alloc_pages(map->inner_map_meta, NULL, map->max_entries, 0, 0); return 0; } diff --git a/tools/testing/selftests/bpf/progs/verifier_arena_large.c b/tools/testing/selftests/bpf/progs/verifier_arena_large.c index 6ab8730d4878..f6515e0e9b17 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena_large.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena_large.c @@ -49,8 +49,10 @@ int big_alloc1(void *ctx) no_page = bpf_arena_alloc_pages(&arena, (void __arena *)ARENA_SIZE, 1, NUMA_NO_NODE, 0); - if (no_page) + /* Only the low 32 bits contribute, so this is equivalent to NULL. */ + if (!no_page) return 3; + bpf_arena_free_pages(&arena, (void __arena *)no_page, 1); if (*page1 != 1) return 4; if (*page2 != 2) -- 2.53.0 Arena argument suffixes now describe the address-space contract at any parameter position, while the kernel no longer publishes KF_ARENA_ARG1 or KF_ARENA_ARG2. Keeping private copies in resolve_btfids would leave an obsolete second annotation mechanism and silently accept flags the kernel no longer understands. Derive arena arguments exclusively from their suffixes and retain KF_ARENA_RET for functions returning arena pointers. Update the resolver selftest to distinguish return-only flags, unannotated pointer arguments, and suffix-annotated arguments. Signed-off-by: Kumar Kartikeya Dwivedi --- tools/bpf/resolve_btfids/main.c | 22 ++----------------- .../selftests/bpf/prog_tests/resolve_btfids.c | 22 ++++++------------- 2 files changed, 9 insertions(+), 35 deletions(-) diff --git a/tools/bpf/resolve_btfids/main.c b/tools/bpf/resolve_btfids/main.c index 37d7e7224207..3dbf329edb46 100644 --- a/tools/bpf/resolve_btfids/main.c +++ b/tools/bpf/resolve_btfids/main.c @@ -177,8 +177,6 @@ struct object { #define KF_FASTCALL (1 << 12) #define KF_ARENA_RET (1 << 13) -#define KF_ARENA_ARG1 (1 << 14) -#define KF_ARENA_ARG2 (1 << 15) #define KF_IMPLICIT_ARGS (1 << 16) #define KF_IMPL_SUFFIX "_impl" #define TYPE_ATTR_ARENA "address_space(1)" @@ -1317,22 +1315,6 @@ static int process_kfunc_with_implicit_args(struct btf2btf_context *ctx, struct return 0; } -static bool is_arena_arg(const struct btf *btf, const struct kfunc *kfunc, - const struct btf_param *param, u32 idx) -{ - if (is_arena_param(btf, param)) - return true; - - switch (idx) { - case 0: - return kfunc->flags & KF_ARENA_ARG1; - case 1: - return kfunc->flags & KF_ARENA_ARG2; - default: - return false; - } -} - static s32 arena_tag_ptr(struct btf *btf, u32 ptr_id, struct kfunc *kfunc) { const struct btf_type *ptr = btf__type_by_id(btf, ptr_id); @@ -1382,7 +1364,7 @@ static s32 add_arena_tagged_proto(struct btf *btf, struct kfunc *kfunc) int err, i; for (i = 0; i < nr_params; i++) { - if (is_arena_arg(btf, kfunc, ¶ms[i], i)) { + if (is_arena_param(btf, ¶ms[i])) { has_arena_arg = true; break; } @@ -1420,7 +1402,7 @@ static s32 add_arena_tagged_proto(struct btf *btf, struct kfunc *kfunc) for (i = 0; i < nr_params; i++) { t = btf__type_by_id(btf, new_proto_id); tag_params = btf_params(t); - if (!is_arena_arg(btf, kfunc, &tag_params[i], i)) + if (!is_arena_param(btf, &tag_params[i])) continue; id = arena_tag_ptr(btf, tag_params[i].type, kfunc); diff --git a/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c b/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c index 3f9949e8227d..a4381c7fa7da 100644 --- a/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c +++ b/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c @@ -23,13 +23,6 @@ #ifndef KF_ARENA_RET #define KF_ARENA_RET (1 << 13) #endif -#ifndef KF_ARENA_ARG1 -#define KF_ARENA_ARG1 (1 << 14) -#endif -#ifndef KF_ARENA_ARG2 -#define KF_ARENA_ARG2 (1 << 15) -#endif - struct symbol { const char *name; int type; @@ -57,9 +50,8 @@ struct kfunc_symbol { static struct kfunc_symbol kfunc_symbols[] = { { "kfunc_a", -1, 0, 0, false }, { "kfunc_b", -1, KF_FASTCALL, 0, false }, - { "kfunc_c", -1, KF_ARENA_RET | KF_ARENA_ARG1 | KF_ARENA_ARG2, - ARENA_ARG(0) | ARENA_ARG(1), true }, - { "kfunc_d", -1, KF_ARENA_ARG2, ARENA_ARG(1), false }, + { "kfunc_c", -1, KF_ARENA_RET, 0, true }, + { "kfunc_d", -1, 0, 0, false }, { "kfunc_e", -1, 0, ARENA_ARG(0) | ARENA_ARG(1) | ARENA_ARG(2) | ARENA_ARG(3) | ARENA_ARG(4), false }, { "kfunc_f", -1, 0, ARENA_ARG(1), false }, @@ -111,8 +103,8 @@ BTF_SET_END(test_set) BTF_KFUNCS_START(test_kfunc_set) BTF_ID_FLAGS(func, kfunc_a) BTF_ID_FLAGS(func, kfunc_b, KF_FASTCALL) -BTF_ID_FLAGS(func, kfunc_c, KF_ARENA_RET | KF_ARENA_ARG1 | KF_ARENA_ARG2) -BTF_ID_FLAGS(func, kfunc_d, KF_ARENA_ARG2) +BTF_ID_FLAGS(func, kfunc_c, KF_ARENA_RET) +BTF_ID_FLAGS(func, kfunc_d) BTF_ID_FLAGS(func, kfunc_e) BTF_ID_FLAGS(func, kfunc_f) BTF_ID_FLAGS(func, kfunc_g, KF_ARENA_RET) @@ -126,8 +118,8 @@ BTF_KFUNCS_START(test_kfunc_set_rev) BTF_ID_FLAGS(func, kfunc_g, KF_ARENA_RET) BTF_ID_FLAGS(func, kfunc_f) BTF_ID_FLAGS(func, kfunc_e) -BTF_ID_FLAGS(func, kfunc_d, KF_ARENA_ARG2) -BTF_ID_FLAGS(func, kfunc_c, KF_ARENA_RET | KF_ARENA_ARG1 | KF_ARENA_ARG2) +BTF_ID_FLAGS(func, kfunc_d) +BTF_ID_FLAGS(func, kfunc_c, KF_ARENA_RET) BTF_ID_FLAGS(func, kfunc_b, KF_FASTCALL) BTF_ID_FLAGS(func, kfunc_a) BTF_KFUNCS_END(test_kfunc_set_rev) @@ -315,7 +307,7 @@ void test_resolve_btfids(void) } /* - * Check resolve_btfids wrapped exactly the arena-flagged or suffixed + * Check resolve_btfids wrapped exactly the arena return or suffixed * return/args with the address_space(1) type attribute, and left other * pointers/returns untouched. */ -- 2.53.0 The arena kfunc runtime tests are restricted to x86-64 and arm64, so new JIT implementations can compile without exercising rebasing, nullable arguments, five-register calls, or the verifier capability gate. Run those tests on RISC-V, s390, LoongArch, and PowerPC64 as well. Teach test_loader about PowerPC64 so it can select the newly annotated programs. Enable the struct_ops arena tests on RISC-V, s390, and LoongArch, whose indirect trampolines now implement the separate reverse-conversion path. PowerPC64 remains covered only for kfunc arguments because it does not advertise the struct_ops capability. Signed-off-by: Kumar Kartikeya Dwivedi --- .../bpf/prog_tests/test_struct_ops_arena.c | 16 ++++++--- .../testing/selftests/bpf/progs/arena_kfunc.c | 36 +++++++++++++++++++ tools/testing/selftests/bpf/progs/bpf_misc.h | 1 + tools/testing/selftests/bpf/test_loader.c | 5 +++ 4 files changed, 53 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c index 7f9f54ba3fbe..7f44b00af06d 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c +++ b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c @@ -6,7 +6,13 @@ #include "struct_ops_arena_attach.skel.h" #include "struct_ops_arena_fail.skel.h" -#if defined(__x86_64__) || defined(__aarch64__) +#if defined(__x86_64__) || defined(__aarch64__) || \ + (defined(__riscv) && __riscv_xlen == 64) || defined(__s390x__) || \ + defined(__loongarch__) +#define HAVE_ARENA_STRUCT_OPS_ARGS +#endif + +#ifdef HAVE_ARENA_STRUCT_OPS_ARGS /* * Attach callbacks with __arena and __arena__nullable arguments and drive * them through the bpf_testmod_ops3_call_test_arena*() kfuncs. @@ -111,11 +117,11 @@ static void arena_arg_attach(void) void serial_test_struct_ops_arena(void) { /* - * Arena struct_ops arguments need JIT support, currently x86-64 and - * arm64 only. Elsewhere verification fails with "JIT does not support - * arena arguments", so the programs cannot even load. + * Arena struct_ops arguments need JIT support. Elsewhere verification + * fails with "JIT does not support arena arguments", so the programs + * cannot even load. */ -#if defined(__x86_64__) || defined(__aarch64__) +#ifdef HAVE_ARENA_STRUCT_OPS_ARGS if (test__start_subtest("arena_arg")) arena_arg(); if (test__start_subtest("arena_arg_fail")) diff --git a/tools/testing/selftests/bpf/progs/arena_kfunc.c b/tools/testing/selftests/bpf/progs/arena_kfunc.c index bf0d304e0e59..8ee2bd4d6633 100644 --- a/tools/testing/selftests/bpf/progs/arena_kfunc.c +++ b/tools/testing/selftests/bpf/progs/arena_kfunc.c @@ -28,6 +28,10 @@ volatile u64 stash; SEC("syscall") __arch_x86_64 __arch_arm64 +__arch_riscv64 +__arch_s390x +__arch_loongarch +__arch_powerpc64 __success __retval(0) int arena_arg_forms(void *ctx) { @@ -72,6 +76,10 @@ int arena_arg_forms(void *ctx) SEC("syscall") __arch_x86_64 __arch_arm64 +__arch_riscv64 +__arch_s390x +__arch_loongarch +__arch_powerpc64 __success __retval(0) int arena_arg_rebase(void *ctx) { @@ -114,6 +122,10 @@ int arena_arg_rebase(void *ctx) SEC("syscall") __arch_x86_64 __arch_arm64 +__arch_riscv64 +__arch_s390x +__arch_loongarch +__arch_powerpc64 __success __retval(0) int arena_args5(void *ctx) { @@ -146,6 +158,10 @@ int arena_args5(void *ctx) SEC("syscall") __arch_x86_64 __arch_arm64 +__arch_riscv64 +__arch_s390x +__arch_loongarch +__arch_powerpc64 __success __retval(0) int arena_arg_mixed(void *ctx) { @@ -174,6 +190,10 @@ int arena_arg_mixed(void *ctx) SEC("syscall") __arch_x86_64 __arch_arm64 +__arch_riscv64 +__arch_s390x +__arch_loongarch +__arch_powerpc64 __success __retval(0) int arena_arg_unpopulated(void *ctx) { @@ -195,6 +215,10 @@ int arena_arg_unpopulated(void *ctx) SEC("syscall") __arch_x86_64 __arch_arm64 +__arch_riscv64 +__arch_s390x +__arch_loongarch +__arch_powerpc64 __failure __msg("arena pointer requires a program with an associated arena") int arena_arg_no_arena(void *ctx) { @@ -205,6 +229,10 @@ int arena_arg_no_arena(void *ctx) SEC("syscall") __arch_x86_64 __arch_arm64 +__arch_riscv64 +__arch_s390x +__arch_loongarch +__arch_powerpc64 __failure __msg("is not a pointer to arena or scalar") int arena_arg_bad_reg(void *ctx) { @@ -221,6 +249,10 @@ int arena_arg_bad_reg(void *ctx) SEC("syscall") __arch_x86_64 __arch_arm64 +__arch_riscv64 +__arch_s390x +__arch_loongarch +__arch_powerpc64 __failure __msg("arena pointer cannot be a stack argument") int arena_arg_stack(void *ctx) { @@ -232,6 +264,10 @@ int arena_arg_stack(void *ctx) SEC("syscall") __arch_x86_64 __arch_arm64 +__arch_riscv64 +__arch_s390x +__arch_loongarch +__arch_powerpc64 __description("arena_arg_stack: not supported, dummy test") __success int arena_arg_stack(void *ctx) diff --git a/tools/testing/selftests/bpf/progs/bpf_misc.h b/tools/testing/selftests/bpf/progs/bpf_misc.h index 5eacf1b43252..c35359c97e44 100644 --- a/tools/testing/selftests/bpf/progs/bpf_misc.h +++ b/tools/testing/selftests/bpf/progs/bpf_misc.h @@ -159,6 +159,7 @@ #define __arch_riscv64 __arch("RISCV64") #define __arch_s390x __arch("s390x") #define __arch_loongarch __arch("LOONGARCH") +#define __arch_powerpc64 __arch("POWERPC64") #define __caps_unpriv(caps) __test_tag("test_caps_unpriv=" EXPAND_QUOTE(caps)) #define __load_if_JITed() __test_tag("load_mode=jited") #define __load_if_no_JITed() __test_tag("load_mode=no_jited") diff --git a/tools/testing/selftests/bpf/test_loader.c b/tools/testing/selftests/bpf/test_loader.c index 07807757b518..221393f2a8ef 100644 --- a/tools/testing/selftests/bpf/test_loader.c +++ b/tools/testing/selftests/bpf/test_loader.c @@ -378,6 +378,7 @@ enum arch { ARCH_RISCV64 = 0x8, ARCH_S390X = 0x10, ARCH_LOONGARCH = 0x20, + ARCH_POWERPC64 = 0x40, }; static int get_current_arch(void) @@ -392,6 +393,8 @@ static int get_current_arch(void) return ARCH_S390X; #elif defined(__loongarch__) return ARCH_LOONGARCH; +#elif defined(__powerpc64__) + return ARCH_POWERPC64; #endif return ARCH_UNKNOWN; } @@ -585,6 +588,8 @@ static int parse_test_spec(struct test_loader *tester, arch = ARCH_S390X; } else if (strcmp(val, "LOONGARCH") == 0) { arch = ARCH_LOONGARCH; + } else if (strcmp(val, "POWERPC64") == 0) { + arch = ARCH_POWERPC64; } else { PRINT_FAIL("bad arch spec: '%s'\n", val); err = -EINVAL; -- 2.53.0 Arena kfunc calls and struct_ops callbacks perform opposite conversions at different JIT boundaries and are now advertised independently. Document the architectures supporting each capability so users can tell where arena allocation kfuncs and arena struct_ops arguments can load. Signed-off-by: Kumar Kartikeya Dwivedi --- Documentation/bpf/kfuncs.rst | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst index 89dea6b0b024..65e56491cae1 100644 --- a/Documentation/bpf/kfuncs.rst +++ b/Documentation/bpf/kfuncs.rst @@ -301,9 +301,10 @@ An example is given below:: } Calling such a kfunc requires the program to use an arena map and a JIT with -arena argument support (currently x86-64 and arm64); verification fails -otherwise. The program can pass any value without compromising the kernel. A -value that does not point into the arena is a program bug. +arena kfunc argument support (currently x86-64, arm64, RISC-V, s390, +LoongArch, and PowerPC64); verification fails otherwise. The program can pass +any value without compromising the kernel. A value that does not point into +the arena is a program bug. The suffixes have the same meaning on the arguments of struct_ops stub functions, with the conversion running in the opposite direction. The @@ -311,9 +312,11 @@ kernel caller passes the kernel arena address and the trampoline converts it while saving the arguments, so the callback receives an arena pointer it can dereference directly. With ``__arena`` the kernel caller must not pass NULL. With ``__arena__nullable`` a NULL kernel pointer arrives as NULL. -However, there is no obligation to prove to the verifier that such a pointer is -non-NULL before use, in-line with existing semantics of arena pointers used in -a program (or obtained from any other source). +Struct_ops arena argument conversion is a separate JIT capability, currently +provided by x86-64, arm64, RISC-V, s390, and LoongArch. However, there is no +obligation to prove to the verifier that such a pointer is non-NULL before use, +in-line with existing semantics of arena pointers used in a program (or +obtained from any other source). .. _BPF_kfunc_nodef: -- 2.53.0