From: Tianci Cao JIT the UHMUL and SHMUL instructions with the one-operand x86-64 mul and imul instructions. These instructions multiply RAX by an explicit source and return the full product in RDX:RAX, so the desired high half is available in RDX. Copy register or immediate sources into AUX_REG, load the destination into RAX, and move RDX back to the BPF destination after multiplication. Preserve RAX and RDX when they hold other live BPF registers, while handling BPF_REG_0 and BPF_REG_3 destinations without restoring over the result. Leave the existing JIT handling of ordinary BPF_MUL unchanged. Add bpf_insn_is_hmul() to identify the high-half multiplication variants. Introduce bpf_jit_supports_hmul() with a weak default returning false, and override it to return true for the x86-64 JIT. This allows the verifier to check JIT support without adding rejection logic to every unsupported backend. Link: https://lore.kernel.org/bpf/CAADnVQ+2O2zgdy11+vxE6dOp+3Br4zZBysHF-Hne7cP+eeucnQ@mail.gmail.com/ Suggested-by: Alexei Starovoitov Signed-off-by: Tianci Cao Co-developed-by: Yazhou Tang Signed-off-by: Yazhou Tang Co-developed-by: Shenghao Yuan Signed-off-by: Shenghao Yuan --- arch/x86/net/bpf_jit_comp.c | 53 +++++++++++++++++++++++++++++++++++++++++++++ include/linux/filter.h | 8 +++++++ kernel/bpf/core.c | 6 +++++ 3 files changed, 67 insertions(+) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 6a2fe3d0039d..2707e8fb9799 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1378,6 +1378,38 @@ static int emit_kasan_check(struct bpf_verifier_env *env, u8 **pprog, return 0; } +/* + * The one-operand x86 mul/imul forms use RAX as the implicit first + * operand and store the 128-bit product in RDX:RAX. + * The JIT uses AUX_REG as the explicit second operand. + */ +static void emit_hmul_aux(u8 **pprog, u32 dst_reg, bool is_signed) +{ + u8 *prog = *pprog; + bool save_rax = dst_reg != BPF_REG_0; + bool save_rdx = dst_reg != BPF_REG_3; + u8 mul_modrm_base = is_signed ? 0xE8 : 0xE0; /* imul /5 or mul /4 */ + + if (save_rax) + EMIT1(0x50); /* push rax */ + if (save_rdx) + EMIT1(0x52); /* push rdx */ + + if (save_rax) + emit_mov_reg(&prog, true, BPF_REG_0, dst_reg); + maybe_emit_1mod(&prog, AUX_REG, true); + EMIT2(0xF7, add_1reg(mul_modrm_base, AUX_REG)); + if (save_rdx) + emit_mov_reg(&prog, true, dst_reg, BPF_REG_3); + + if (save_rdx) + EMIT1(0x5A); /* pop rdx */ + if (save_rax) + EMIT1(0x58); /* pop rax */ + + *pprog = prog; +} + /* LDX: dst_reg = *(u8*)(src_reg + off) */ static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off) { @@ -2916,6 +2948,14 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * case BPF_ALU | BPF_MUL | BPF_K: case BPF_ALU64 | BPF_MUL | BPF_K: + if (bpf_insn_is_hmul(insn)) { + /* imul/mul use AUX_REG as src. */ + emit_mov_imm32(&prog, true, AUX_REG, imm32); + emit_hmul_aux(&prog, dst_reg, + insn->off == BPF_SHMUL); + break; + } + /* * lea for 3, 5 and 9, which LLVM leaves as a * multiplication, but not for r13, which needs a @@ -2951,6 +2991,14 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * case BPF_ALU | BPF_MUL | BPF_X: case BPF_ALU64 | BPF_MUL | BPF_X: + if (bpf_insn_is_hmul(insn)) { + /* imul/mul use AUX_REG as src. */ + emit_mov_reg(&prog, true, AUX_REG, src_reg); + emit_hmul_aux(&prog, dst_reg, + insn->off == BPF_SHMUL); + break; + } + maybe_emit_mod(&prog, src_reg, dst_reg, BPF_CLASS(insn->code) == BPF_ALU64); @@ -5126,6 +5174,11 @@ bool bpf_jit_supports_unwind(void) return true; } +bool bpf_jit_supports_hmul(void) +{ + return true; +} + const struct bpf_jit_arg_abi *bpf_jit_arg_abi(void) { return &x86_arg_abi; diff --git a/include/linux/filter.h b/include/linux/filter.h index e0ee8adf3949..2a2101fb0168 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -178,6 +178,13 @@ static inline u64 bpf_shmul64(s64 a, s64 b) return hi; } +static inline bool bpf_insn_is_hmul(const struct bpf_insn *insn) +{ + return BPF_CLASS(insn->code) == BPF_ALU64 && + BPF_OP(insn->code) == BPF_MUL && + (insn->off == BPF_UHMUL || insn->off == BPF_SHMUL); +} + /* Endianess conversion, cpu_to_{l,b}e(), {l,b}e_to_cpu() */ #define BPF_ENDIAN(TYPE, DST, LEN) \ @@ -1307,6 +1314,7 @@ bool bpf_jit_supports_kfunc_call(void); bool bpf_jit_supports_callx(void); bool bpf_jit_supports_kfunc_ret_reg_pair(void); bool bpf_jit_supports_unwind(void); +bool bpf_jit_supports_hmul(void); bool bpf_jit_supports_stack_args(void); bool bpf_jit_supports_arena_args(void); bool bpf_jit_supports_far_kfunc_call(void); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index f4f8cfa80033..8600c9384217 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3444,6 +3444,12 @@ bool __weak bpf_jit_supports_unwind(void) return false; } +/* uhmul and shmul */ +bool __weak bpf_jit_supports_hmul(void) +{ + return false; +} + /* * How this arch places a by-value kfunc argument, or NULL for one that has * not opted in and so only takes an argument of a single eightbyte, which -- 2.56.0