From: Yazhou Tang On RV64, the RISC-V M extension provides MULHU and MULH instructions that return the high 64 bits of an unsigned or signed 64-bit multiplication, respectively. Add the missing MULH encoder and use MULHU and MULH to lower the UHMUL and SHMUL BPF_MUL variants for both register and immediate sources. Keep the existing low-half and ALU32 multiplication lowering unchanged. Suggested-by: Alexei Starovoitov Signed-off-by: Yazhou Tang Co-developed-by: Tianci Cao Signed-off-by: Tianci Cao Co-developed-by: Shenghao Yuan Signed-off-by: Shenghao Yuan --- arch/riscv/net/bpf_jit.h | 5 +++++ arch/riscv/net/bpf_jit_comp64.c | 18 +++++++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/arch/riscv/net/bpf_jit.h b/arch/riscv/net/bpf_jit.h index 5c4f53060c91..21a190a6a6d9 100644 --- a/arch/riscv/net/bpf_jit.h +++ b/arch/riscv/net/bpf_jit.h @@ -436,6 +436,11 @@ static inline u32 rv_mul(u8 rd, u8 rs1, u8 rs2) return rv_r_insn(1, rs2, rs1, 0, rd, 0x33); } +static inline u32 rv_mulh(u8 rd, u8 rs1, u8 rs2) +{ + return rv_r_insn(1, rs2, rs1, 1, rd, 0x33); +} + static inline u32 rv_mulhu(u8 rd, u8 rs1, u8 rs2) { return rv_r_insn(1, rs2, rs1, 3, rd, 0x33); diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index ed0a6f871dea..a773506a9209 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -197,6 +197,19 @@ static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx) emit_addi(rd, rd, lower, ctx); } +static void emit_mul(u8 rd, u8 rs, bool is64, s16 off, + struct rv_jit_context *ctx) +{ + if (!is64) + emit(rv_mulw(rd, rd, rs), ctx); + else if (off == BPF_MUL_VARIANT_UHMUL) + emit(rv_mulhu(rd, rd, rs), ctx); + else if (off == BPF_MUL_VARIANT_SHMUL) + emit(rv_mulh(rd, rd, rs), ctx); + else + emit(rv_mul(rd, rd, rs), ctx); +} + static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) { int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8; @@ -1498,7 +1511,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, break; case BPF_ALU | BPF_MUL | BPF_X: case BPF_ALU64 | BPF_MUL | BPF_X: - emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx); + emit_mul(rd, rs, is64, off, ctx); if (!is64 && !aux->verifier_zext) emit_zextw(rd, rd, ctx); break; @@ -1634,8 +1647,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, case BPF_ALU | BPF_MUL | BPF_K: case BPF_ALU64 | BPF_MUL | BPF_K: emit_imm(RV_REG_T1, imm, ctx); - emit(is64 ? rv_mul(rd, rd, RV_REG_T1) : - rv_mulw(rd, rd, RV_REG_T1), ctx); + emit_mul(rd, RV_REG_T1, is64, off, ctx); if (!is64 && !aux->verifier_zext) emit_zextw(rd, rd, ctx); break; -- 2.55.0