From: Yazhou Tang arm64 provides UMULH and SMULH instructions that return the high 64 bits of an unsigned or signed 64-bit multiplication, respectively. Add encodings for these instructions and use them to lower the UHMUL and SHMUL BPF_MUL variants for both register and immediate sources. Keep the existing low-half and ALU32 multiplication lowering unchanged. Suggested-by: Alexei Starovoitov Signed-off-by: Yazhou Tang Co-developed-by: Tianci Cao Signed-off-by: Tianci Cao Co-developed-by: Shenghao Yuan Signed-off-by: Shenghao Yuan --- arch/arm64/include/asm/insn.h | 4 ++++ arch/arm64/lib/insn.c | 6 ++++++ arch/arm64/net/bpf_jit.h | 6 ++++++ arch/arm64/net/bpf_jit_comp.c | 15 +++++++++++++-- 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h index 1ce75a442638..f38738fa54a2 100644 --- a/arch/arm64/include/asm/insn.h +++ b/arch/arm64/include/asm/insn.h @@ -247,6 +247,8 @@ enum aarch64_insn_data2_type { enum aarch64_insn_data3_type { AARCH64_INSN_DATA3_MADD, AARCH64_INSN_DATA3_MSUB, + AARCH64_INSN_DATA3_SMULH, + AARCH64_INSN_DATA3_UMULH, }; enum aarch64_insn_logic_type { @@ -396,6 +398,8 @@ __AARCH64_INSN_FUNCS(sub_ext, 0x7FE00000, 0x4B200000) __AARCH64_INSN_FUNCS(subs_ext, 0x7FE00000, 0x6B200000) __AARCH64_INSN_FUNCS(madd, 0x7FE08000, 0x1B000000) __AARCH64_INSN_FUNCS(msub, 0x7FE08000, 0x1B008000) +__AARCH64_INSN_FUNCS(smulh, 0x7FE0FC00, 0x1B407C00) +__AARCH64_INSN_FUNCS(umulh, 0x7FE0FC00, 0x1BC07C00) __AARCH64_INSN_FUNCS(udiv, 0x7FE0FC00, 0x1AC00800) __AARCH64_INSN_FUNCS(sdiv, 0x7FE0FC00, 0x1AC00C00) __AARCH64_INSN_FUNCS(lslv, 0x7FE0FC00, 0x1AC02000) diff --git a/arch/arm64/lib/insn.c b/arch/arm64/lib/insn.c index e70ac0238515..dc483d1b8bd5 100644 --- a/arch/arm64/lib/insn.c +++ b/arch/arm64/lib/insn.c @@ -1155,6 +1155,12 @@ u32 aarch64_insn_gen_data3(enum aarch64_insn_register dst, case AARCH64_INSN_DATA3_MSUB: insn = aarch64_insn_get_msub_value(); break; + case AARCH64_INSN_DATA3_SMULH: + insn = aarch64_insn_get_smulh_value(); + break; + case AARCH64_INSN_DATA3_UMULH: + insn = aarch64_insn_get_umulh_value(); + break; default: pr_err("%s: unknown data3 encoding %d\n", __func__, type); return AARCH64_BREAK_FAULT; diff --git a/arch/arm64/net/bpf_jit.h b/arch/arm64/net/bpf_jit.h index b2fe6e6dcf44..70fc329c661d 100644 --- a/arch/arm64/net/bpf_jit.h +++ b/arch/arm64/net/bpf_jit.h @@ -281,6 +281,12 @@ A64_VARIANT(sf), AARCH64_INSN_DATA3_MSUB) /* Rd = Rn * Rm */ #define A64_MUL(sf, Rd, Rn, Rm) A64_MADD(sf, Rd, A64_ZR, Rn, Rm) +/* Rd = high 64 bits of signed Rn * Rm */ +#define A64_SMULH(Rd, Rn, Rm) aarch64_insn_gen_data3(Rd, A64_ZR, Rn, Rm, \ + AARCH64_INSN_VARIANT_64BIT, AARCH64_INSN_DATA3_SMULH) +/* Rd = high 64 bits of unsigned Rn * Rm */ +#define A64_UMULH(Rd, Rn, Rm) aarch64_insn_gen_data3(Rd, A64_ZR, Rn, Rm, \ + AARCH64_INSN_VARIANT_64BIT, AARCH64_INSN_DATA3_UMULH) /* Logical (shifted register) */ #define A64_LOGIC_SREG(sf, Rd, Rn, Rm, type) \ diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 6c04fee46876..054c38775dd2 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -144,6 +144,17 @@ static inline void emit_a64_mov_i(const int is64, const int reg, } } +static inline void emit_a64_mul(const bool is64, const u8 dst, const u8 src, + const s16 off, struct jit_ctx *ctx) +{ + if (is64 && off == BPF_MUL_VARIANT_UHMUL) + emit(A64_UMULH(dst, dst, src), ctx); + else if (is64 && off == BPF_MUL_VARIANT_SHMUL) + emit(A64_SMULH(dst, dst, src), ctx); + else + emit(A64_MUL(is64, dst, dst, src), ctx); +} + static int i64_i16_blocks(const u64 val, bool inverse) { return (((val >> 0) & 0xffff) != (inverse ? 0xffff : 0x0000)) + @@ -1454,7 +1465,7 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn break; case BPF_ALU | BPF_MUL | BPF_X: case BPF_ALU64 | BPF_MUL | BPF_X: - emit(A64_MUL(is64, dst, dst, src), ctx); + emit_a64_mul(is64, dst, src, off, ctx); break; case BPF_ALU | BPF_DIV | BPF_X: case BPF_ALU64 | BPF_DIV | BPF_X: @@ -1583,7 +1594,7 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn case BPF_ALU | BPF_MUL | BPF_K: case BPF_ALU64 | BPF_MUL | BPF_K: emit_a64_mov_i(is64, tmp, imm, ctx); - emit(A64_MUL(is64, dst, dst, tmp), ctx); + emit_a64_mul(is64, dst, tmp, off, ctx); break; case BPF_ALU | BPF_DIV | BPF_K: case BPF_ALU64 | BPF_DIV | BPF_K: -- 2.55.0