From: Tianci Cao JIT the UHMUL and SHMUL instructions with the one-operand x86-64 mul and imul instructions. These instructions multiply RAX by an explicit source and return the full product in RDX:RAX, so the desired high half is available in RDX. Copy register or immediate sources into AUX_REG, load the destination into RAX, and move RDX back to the BPF destination after multiplication. Preserve RAX and RDX when they hold other live BPF registers, while handling BPF_REG_0 and BPF_REG_3 destinations without restoring over the result. Leave the existing JIT handling of ordinary BPF_MUL unchanged. Link: https://lore.kernel.org/bpf/CAADnVQ+2O2zgdy11+vxE6dOp+3Br4zZBysHF-Hne7cP+eeucnQ@mail.gmail.com/ Suggested-by: Alexei Starovoitov Signed-off-by: Tianci Cao Co-developed-by: Yazhou Tang Signed-off-by: Yazhou Tang Co-developed-by: Shenghao Yuan Signed-off-by: Shenghao Yuan --- arch/x86/net/bpf_jit_comp.c | 48 +++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 48429fae0641..233a05971cf4 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1110,6 +1110,38 @@ static void maybe_emit_1mod(u8 **pprog, u32 reg, bool is64) *pprog = prog; } +/* + * The one-operand x86 mul/imul forms use RAX as the implicit first + * operand and store the 128-bit product in RDX:RAX. + * The JIT uses AUX_REG as the explicit second operand. + */ +static void emit_hmul_aux(u8 **pprog, u32 dst_reg, bool is_signed) +{ + u8 *prog = *pprog; + bool save_rax = dst_reg != BPF_REG_0; + bool save_rdx = dst_reg != BPF_REG_3; + u8 mul_modrm_base = is_signed ? 0xE8 : 0xE0; /* imul /5 or mul /4 */ + + if (save_rax) + EMIT1(0x50); /* push rax */ + if (save_rdx) + EMIT1(0x52); /* push rdx */ + + if (save_rax) + emit_mov_reg(&prog, true, BPF_REG_0, dst_reg); + maybe_emit_1mod(&prog, AUX_REG, true); + EMIT2(0xF7, add_1reg(mul_modrm_base, AUX_REG)); + if (save_rdx) + emit_mov_reg(&prog, true, dst_reg, BPF_REG_3); + + if (save_rdx) + EMIT1(0x5A); /* pop rdx */ + if (save_rax) + EMIT1(0x58); /* pop rax */ + + *pprog = prog; +} + /* LDX: dst_reg = *(u8*)(src_reg + off) */ static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off) { @@ -2088,6 +2120,14 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * case BPF_ALU | BPF_MUL | BPF_K: case BPF_ALU64 | BPF_MUL | BPF_K: + if (bpf_insn_is_hmul(insn)) { + /* imul/mul use AUX_REG as src. */ + emit_mov_imm32(&prog, true, AUX_REG, imm32); + emit_hmul_aux(&prog, dst_reg, + insn->off == BPF_MUL_VARIANT_SHMUL); + break; + } + maybe_emit_mod(&prog, dst_reg, dst_reg, BPF_CLASS(insn->code) == BPF_ALU64); @@ -2104,6 +2144,14 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * case BPF_ALU | BPF_MUL | BPF_X: case BPF_ALU64 | BPF_MUL | BPF_X: + if (bpf_insn_is_hmul(insn)) { + /* imul/mul use AUX_REG as src. */ + emit_mov_reg(&prog, true, AUX_REG, src_reg); + emit_hmul_aux(&prog, dst_reg, + insn->off == BPF_MUL_VARIANT_SHMUL); + break; + } + maybe_emit_mod(&prog, src_reg, dst_reg, BPF_CLASS(insn->code) == BPF_ALU64); -- 2.43.0