The legacy epilogue implementation pops the tail call counter (TCC) context via a redundant double-load pattern. It first decrements the load_offset by 2 slots to fetch 'tcc_ptr', and then immediately bumps it back up by 1 slot to load the original 'tcc' value into REG_TCC, unnecessarily overwriting the register. Optimize this sequence by adjusting the load_offset by only 1 slot. This aligns the offset directly with the higher stack slot containing the entry TCC counter (or caller state), allowing us to restore the REG_TCC register safely with a single load. This removes one redundant instruction from the epilogue hot path, improves code readability, and ensures the correct TCC register context is handed back cleanly upon normal return. Fixes: c0fcc955ff82 ("LoongArch: BPF: Fix the tailcall hierarchy") Fixes: ef54c517a937 ("LoongArch: BPF: Implement PROBE_MEM32 pseudo instructions") Cc: stable@vger.kernel.org Signed-off-by: Tiezhu Yang --- arch/loongarch/net/bpf_jit.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 2738b4db1165..93a181312256 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -246,14 +246,8 @@ static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call) emit_insn(ctx, ldd, REG_ARENA, LOONGARCH_GPR_SP, load_offset); } - /* - * When push into the stack, follow the order of tcc then tcc_ptr. - * When pop from the stack, first pop tcc_ptr then followed by tcc. - */ - load_offset -= 2 * sizeof(long); - emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset); - - load_offset += sizeof(long); + /* Only restore the TCC state into REG_TCC from the higher slot */ + load_offset -= sizeof(long); emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset); emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust); -- 2.42.0 Currently, the stack layout places the optional arena register slot above the tail call counter context. When arena_vm_start is dynamically enabled, it shifts the relative offset of the tcc_ptr slot within the stack frame, causing hardcoded tracking macros to mismatch and potentially leading to memory misalignment or corruption. To fix this, move the arena register save and restore sequences below the tail call counter context slots in both build_prologue() and epilogue. Update __build_epilogue() to insert a proper offset decrement to safely skip the unneeded tcc_ptr reading block while accurately aligning with the relocated arena slot at the very bottom. With this patch, the tcc_ptr slot is always positioned at a fixed distance directly underneath the base callee-saved registers that is independent of whether the arena features are on. Fixes: ef54c517a937 ("LoongArch: BPF: Implement PROBE_MEM32 pseudo instructions") Cc: stable@vger.kernel.org Signed-off-by: Tiezhu Yang --- arch/loongarch/net/bpf_jit.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 93a181312256..a12b8278d5d4 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -124,6 +124,9 @@ static void prepare_bpf_tail_call_cnt(struct jit_ctx *ctx, int *store_offset) * | tcc | * +-------------------------+ * | tcc_ptr | + * +-------------------------+ + * | arena | + * | (optional) | * +-------------------------+ <--BPF_REG_FP * | prog->aux->stack_depth | * | (optional) | @@ -145,7 +148,7 @@ static void build_prologue(struct jit_ctx *ctx) stack_adjust += sizeof(long) * 2; if (ctx->arena_vm_start) - stack_adjust += 8; + stack_adjust += sizeof(long); stack_adjust = round_up(stack_adjust, 16); stack_adjust += bpf_stack_adjust; @@ -194,13 +197,13 @@ static void build_prologue(struct jit_ctx *ctx) store_offset -= sizeof(long); emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset); + prepare_bpf_tail_call_cnt(ctx, &store_offset); + if (ctx->arena_vm_start) { store_offset -= sizeof(long); emit_insn(ctx, std, REG_ARENA, LOONGARCH_GPR_SP, store_offset); } - prepare_bpf_tail_call_cnt(ctx, &store_offset); - emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust); if (bpf_stack_adjust) @@ -241,15 +244,18 @@ static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call) load_offset -= sizeof(long); emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset); + /* Only restore the TCC state into REG_TCC from the higher slot */ + load_offset -= sizeof(long); + emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset); + + /* Skip the unused local 'tcc_ptr' slot to align with arena */ + load_offset -= sizeof(long); + if (ctx->arena_vm_start) { load_offset -= sizeof(long); emit_insn(ctx, ldd, REG_ARENA, LOONGARCH_GPR_SP, load_offset); } - /* Only restore the TCC state into REG_TCC from the higher slot */ - load_offset -= sizeof(long); - emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset); - emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust); if (!is_tail_call) { -- 2.42.0 The old macro-based jmp_offset calculation derives the jump distance from a stale prior-pass code stride, which can lead to wrong branch offsets and soft lockups under extra JIT passes. Fix this by calculating the offset directly on the absolute target: "ctx->offset[insn + 1] - ctx->idx". To avoid a false 16-bit range check abort during size estimation, add a "ctx->image == NULL" guard to inject a safe dummy offset. Fixes: cd39d9e6b7e4 ("LoongArch: BPF: Fix jump offset calculation in tailcall") Cc: stable@vger.kernel.org Signed-off-by: Tiezhu Yang --- arch/loongarch/net/bpf_jit.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index a12b8278d5d4..8d6738103f59 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -290,17 +290,13 @@ bool bpf_jit_supports_far_kfunc_call(void) static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) { - int off, tc_ninsn = 0; + int off, jmp_offset; int tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size); u8 a1 = LOONGARCH_GPR_A1; u8 a2 = LOONGARCH_GPR_A2; u8 t1 = LOONGARCH_GPR_T1; u8 t2 = LOONGARCH_GPR_T2; u8 t3 = LOONGARCH_GPR_T3; - const int idx0 = ctx->idx; - -#define cur_offset (ctx->idx - idx0) -#define jmp_offset (tc_ninsn - (cur_offset)) /* * a0: &ctx @@ -310,12 +306,12 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) * if (index >= array->map.max_entries) * goto out; */ - tc_ninsn = insn ? ctx->offset[insn+1] - ctx->offset[insn] : ctx->offset[0]; emit_zext_32(ctx, a2, true); off = offsetof(struct bpf_array, map.max_entries); emit_insn(ctx, ldwu, t1, a1, off); /* bgeu $a2, $t1, jmp_offset */ + jmp_offset = ctx->image ? (ctx->offset[insn + 1] - ctx->idx) : 0; if (emit_tailcall_jmp(ctx, BPF_JGE, a2, t1, jmp_offset) < 0) goto toofar; @@ -326,6 +322,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off); emit_insn(ctx, ldd, t3, REG_TCC, 0); emit_insn(ctx, addid, t2, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT); + jmp_offset = ctx->image ? (ctx->offset[insn + 1] - ctx->idx) : 0; if (emit_tailcall_jmp(ctx, BPF_JSGE, t3, t2, jmp_offset) < 0) goto toofar; @@ -340,6 +337,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) off = offsetof(struct bpf_array, ptrs); emit_insn(ctx, ldd, t2, t2, off); /* beq $t2, $zero, jmp_offset */ + jmp_offset = ctx->image ? (ctx->offset[insn + 1] - ctx->idx) : 0; if (emit_tailcall_jmp(ctx, BPF_JEQ, t2, LOONGARCH_GPR_ZERO, jmp_offset) < 0) goto toofar; @@ -355,8 +353,6 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) toofar: pr_info_once("tail_call: jump too far\n"); return -1; -#undef cur_offset -#undef jmp_offset } static void emit_store_stack_imm64(struct jit_ctx *ctx, int reg, int stack_off, u64 imm64) -- 2.42.0 The current implementation handles combined bpf2bpf and tail calls by checking at runtime whether REG_TCC holds a scalar count or a pointer address via a conditional jump. This adds branch prediction overhead in the hot path of tail call execution. To implement branchless conditional move, use an unsigned comparison (sltui) combined with mask instructions (maskeqz/masknez) to achieve branchless classification and blending of incoming scalar counts and kernel pointers in REG_TCC. This optimization refactors the inner logic of the helper function, unifies the offset decrement at the function entry, and completely removes all runtime branching from the prologue hot path. Signed-off-by: Tiezhu Yang --- arch/loongarch/include/asm/inst.h | 6 ++++ arch/loongarch/net/bpf_jit.c | 47 +++++++++---------------------- 2 files changed, 19 insertions(+), 34 deletions(-) diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm/inst.h index 76b723590023..19feae166e1c 100644 --- a/arch/loongarch/include/asm/inst.h +++ b/arch/loongarch/include/asm/inst.h @@ -97,6 +97,7 @@ enum reg2i6_op { }; enum reg2i12_op { + sltui_op = 0x09, addiw_op = 0x0a, addid_op = 0x0b, lu52id_op = 0x0c, @@ -153,6 +154,8 @@ enum reg3_op { addd_op = 0x21, subw_op = 0x22, subd_op = 0x23, + maskeqz_op = 0x26, + masknez_op = 0x27, nor_op = 0x28, and_op = 0x29, or_op = 0x2a, @@ -644,6 +647,7 @@ static inline void emit_##NAME(union loongarch_instruction *insn, \ insn->reg2i12_format.rj = rj; \ } +DEF_EMIT_REG2I12_FORMAT(sltui, sltui_op) DEF_EMIT_REG2I12_FORMAT(addiw, addiw_op) DEF_EMIT_REG2I12_FORMAT(addid, addid_op) DEF_EMIT_REG2I12_FORMAT(lu52id, lu52id_op) @@ -749,6 +753,8 @@ DEF_EMIT_REG3_FORMAT(divd, divd_op) DEF_EMIT_REG3_FORMAT(modd, modd_op) DEF_EMIT_REG3_FORMAT(divdu, divdu_op) DEF_EMIT_REG3_FORMAT(moddu, moddu_op) +DEF_EMIT_REG3_FORMAT(maskeqz, maskeqz_op) +DEF_EMIT_REG3_FORMAT(masknez, masknez_op) DEF_EMIT_REG3_FORMAT(and, and_op) DEF_EMIT_REG3_FORMAT(or, or_op) DEF_EMIT_REG3_FORMAT(xor, xor_op) diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 8d6738103f59..9e9d98b21812 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -52,50 +52,29 @@ static void prepare_bpf_tail_call_cnt(struct jit_ctx *ctx, int *store_offset) const struct bpf_prog *prog = ctx->prog; const bool is_main_prog = !bpf_is_subprog(prog); + *store_offset -= sizeof(long); if (is_main_prog) { - /* - * LOONGARCH_GPR_T3 = MAX_TAIL_CALL_CNT - * if (REG_TCC > T3 ) - * std REG_TCC -> LOONGARCH_GPR_SP + store_offset - * else - * std REG_TCC -> LOONGARCH_GPR_SP + store_offset - * REG_TCC = LOONGARCH_GPR_SP + store_offset - * - * std REG_TCC -> LOONGARCH_GPR_SP + store_offset - * - * The purpose of this code is to first push the TCC into stack, - * and then push the address of TCC into stack. - * In cases where bpf2bpf and tailcall are used in combination, - * the value in REG_TCC may be a count or an address, - * these two cases need to be judged and handled separately. - */ - emit_insn(ctx, addid, LOONGARCH_GPR_T3, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT); - *store_offset -= sizeof(long); - - emit_cond_jmp(ctx, BPF_JGT, REG_TCC, LOONGARCH_GPR_T3, 4); - - /* - * If REG_TCC < MAX_TAIL_CALL_CNT, the value in REG_TCC is a count, - * push tcc into stack - */ + /* Save entrance TCC state (scalar count or kernel pointer) to local 'tcc' slot */ emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); - /* Push the address of TCC into the REG_TCC */ - emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_SP, *store_offset); - - emit_uncond_jmp(ctx, 2); + /* Compute the absolute pointer to the local 'tcc' slot */ + emit_insn(ctx, addid, LOONGARCH_GPR_T7, LOONGARCH_GPR_SP, *store_offset); /* - * If REG_TCC > MAX_TAIL_CALL_CNT, the value in REG_TCC is an address, - * push tcc_ptr into stack + * Branchless classification and blending: + * Combine interleaved inputs between a scalar count (0 to 33) + * and a kernel pointer address without runtime branching. */ - emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); + emit_insn(ctx, sltui, LOONGARCH_GPR_T8, REG_TCC, MAX_TAIL_CALL_CNT + 1); + emit_insn(ctx, maskeqz, LOONGARCH_GPR_T7, LOONGARCH_GPR_T7, LOONGARCH_GPR_T8); + emit_insn(ctx, masknez, REG_TCC, REG_TCC, LOONGARCH_GPR_T8); + emit_insn(ctx, or, REG_TCC, REG_TCC, LOONGARCH_GPR_T7); } else { - *store_offset -= sizeof(long); + /* Subprograms: backup the verified TCC pointer inherited via REG_TCC */ emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); } - /* Push tcc_ptr into stack */ + /* Store the finalized TCC pointer value securely into the local 'tcc_ptr' slot */ *store_offset -= sizeof(long); emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); } -- 2.42.0