The current rv32 bpf jit compiler incorrectly treats BPF_SDIV and BPF_SMOD as unsigned operations. The BPF instruction set allows signed division and modulo by reusing the BPF_DIV and BPF_MOD opcodes with the instruction offset set to 1. Update the emit_alu_r32() function to accept an 'is_sdiv' variable and emit the correct div and rem instructions when the offset is 1. Before this patch: [ 44.161771] test_bpf: #165 ALU_SDIV_X: -6 / 2 = -3 jited:1 ret 2147483645 != -3 (0x7ffffffd != 0xfffffffd)FAIL (1 times) [ 44.167385] test_bpf: #166 ALU_SDIV_K: -6 / 2 = -3 jited:1 ret 2147483645 != -3 (0x7ffffffd != 0xfffffffd)FAIL (1 times) [ 44.171053] test_bpf: #169 ALU_SMOD_X: -7 % 2 = -1 jited:1 ret 1 != -1 (0x1 != 0xffffffff)FAIL (1 times) [ 44.172081] test_bpf: #170 ALU_SMOD_K: -7 % 2 = -1 jited:1 ret 1 != -1 (0x1 != 0xffffffff)FAIL (1 times) After this patch: [ 16.002192] test_bpf: #165 ALU_SDIV_X: -6 / 2 = -3 jited:1 95 PASS [ 16.002983] test_bpf: #166 ALU_SDIV_K: -6 / 2 = -3 jited:1 1059 PASS [ 16.017167] test_bpf: #169 ALU_SMOD_X: -7 % 2 = -1 jited:1 136 PASS [ 16.023002] test_bpf: #170 ALU_SMOD_K: -7 % 2 = -1 jited:1 109 PASS Fixes: ec0e2da95f72 ("bpf: Support new signed div/mod instructions.") Signed-off-by: Kuan-Wei Chiu --- arch/riscv/net/bpf_jit_comp32.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/arch/riscv/net/bpf_jit_comp32.c b/arch/riscv/net/bpf_jit_comp32.c index 592dd86fbf81..7396899ea276 100644 --- a/arch/riscv/net/bpf_jit_comp32.c +++ b/arch/riscv/net/bpf_jit_comp32.c @@ -509,7 +509,7 @@ static void emit_alu_r64(const s8 *dst, const s8 *src, } static void emit_alu_r32(const s8 *dst, const s8 *src, - struct rv_jit_context *ctx, const u8 op) + struct rv_jit_context *ctx, const u8 op, bool is_sdiv) { const s8 *tmp1 = bpf2rv32[TMP_REG_1]; const s8 *tmp2 = bpf2rv32[TMP_REG_2]; @@ -539,10 +539,16 @@ static void emit_alu_r32(const s8 *dst, const s8 *src, emit(rv_mul(lo(rd), lo(rd), lo(rs)), ctx); break; case BPF_DIV: - emit(rv_divu(lo(rd), lo(rd), lo(rs)), ctx); + if (is_sdiv) + emit(rv_div(lo(rd), lo(rd), lo(rs)), ctx); + else + emit(rv_divu(lo(rd), lo(rd), lo(rs)), ctx); break; case BPF_MOD: - emit(rv_remu(lo(rd), lo(rd), lo(rs)), ctx); + if (is_sdiv) + emit(rv_rem(lo(rd), lo(rd), lo(rs)), ctx); + else + emit(rv_remu(lo(rd), lo(rd), lo(rs)), ctx); break; case BPF_LSH: emit(rv_sll(lo(rd), lo(rd), lo(rs)), ctx); @@ -959,6 +965,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, u8 code = insn->code; s16 off = insn->off; s32 imm = insn->imm; + bool is_sdiv = false; const s8 *dst = bpf2rv32[insn->dst_reg]; const s8 *src = bpf2rv32[insn->src_reg]; @@ -1041,7 +1048,9 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, emit_imm32(tmp2, imm, ctx); src = tmp2; } - emit_alu_r32(dst, src, ctx, BPF_OP(code)); + if ((BPF_OP(code) == BPF_DIV || BPF_OP(code) == BPF_MOD) && insn->off == 1) + is_sdiv = true; + emit_alu_r32(dst, src, ctx, BPF_OP(code), is_sdiv); break; case BPF_ALU | BPF_MOV | BPF_K: @@ -1065,7 +1074,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, * src is ignored---choose tmp2 as a dummy register since it * is not on the stack. */ - emit_alu_r32(dst, tmp2, ctx, BPF_OP(code)); + emit_alu_r32(dst, tmp2, ctx, BPF_OP(code), false); break; case BPF_ALU | BPF_END | BPF_FROM_LE: -- 2.54.0.563.g4f69b47b94-goog The current rv32 bpf jit compiler incorrectly treats BPF_MOVSX as a standard zero-extended move operation. The bpf instruction set allows sign-extension moves by reusing the BPF_MOV opcode with the instruction offset set to 8, 16, or 32. Update the bpf_jit_emit_insn() function to check the offset field for both ALU and ALU64 MOV operations. If the offset is non-zero, emit the correct slli and srai instructions to perform the sign extension. Before this patch: [ 19.549705] test_bpf: #82 ALU_MOVSX | BPF_B jited:1 ret 2 != 1 (0x2 != 0x1)FAIL (1 times) [ 19.551354] test_bpf: #83 ALU_MOVSX | BPF_H jited:1 ret 2 != 1 (0x2 != 0x1)FAIL (1 times) [ 19.552576] test_bpf: #84 ALU64_MOVSX | BPF_B jited:1 ret 2 != 1 (0x2 != 0x1)FAIL (1 times) [ 19.553542] test_bpf: #85 ALU64_MOVSX | BPF_H jited:1 ret 2 != 1 (0x2 != 0x1)FAIL (1 times) [ 19.554807] test_bpf: #86 ALU64_MOVSX | BPF_W jited:1 ret 2 != 1 (0x2 != 0x1)FAIL (1 times) After this patch: [ 17.931172] test_bpf: #82 ALU_MOVSX | BPF_B jited:1 125 PASS [ 17.932198] test_bpf: #83 ALU_MOVSX | BPF_H jited:1 124 PASS [ 17.933039] test_bpf: #84 ALU64_MOVSX | BPF_B jited:1 124 PASS [ 17.933918] test_bpf: #85 ALU64_MOVSX | BPF_H jited:1 124 PASS [ 17.934751] test_bpf: #86 ALU64_MOVSX | BPF_W jited:1 122 PASS Fixes: 8100928c8814 ("bpf: Support new sign-extension mov insns") Signed-off-by: Kuan-Wei Chiu --- arch/riscv/net/bpf_jit_comp32.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/arch/riscv/net/bpf_jit_comp32.c b/arch/riscv/net/bpf_jit_comp32.c index 7396899ea276..f8509950fed4 100644 --- a/arch/riscv/net/bpf_jit_comp32.c +++ b/arch/riscv/net/bpf_jit_comp32.c @@ -974,6 +974,24 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, switch (code) { case BPF_ALU64 | BPF_MOV | BPF_X: + if (insn->off != 0) { + const s8 *rd = bpf_get_reg64(dst, tmp1, ctx); + const s8 *rs = bpf_get_reg64(src, tmp2, ctx); + + if (insn->off == 8) { + emit(rv_slli(lo(rd), lo(rs), 24), ctx); + emit(rv_srai(lo(rd), lo(rd), 24), ctx); + } else if (insn->off == 16) { + emit(rv_slli(lo(rd), lo(rs), 16), ctx); + emit(rv_srai(lo(rd), lo(rd), 16), ctx); + } else { + emit(rv_addi(lo(rd), lo(rs), 0), ctx); + } + emit(rv_srai(hi(rd), lo(rd), 31), ctx); + bpf_put_reg64(dst, rd, ctx); + break; + } + fallthrough; case BPF_ALU64 | BPF_ADD | BPF_X: case BPF_ALU64 | BPF_ADD | BPF_K: @@ -1024,6 +1042,20 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, emit_zext64(dst, ctx); break; } + if (insn->off != 0) { + const s8 *rd = bpf_get_reg32(dst, tmp1, ctx); + const s8 *rs = bpf_get_reg32(src, tmp2, ctx); + + if (insn->off == 8) { + emit(rv_slli(lo(rd), lo(rs), 24), ctx); + emit(rv_srai(lo(rd), lo(rd), 24), ctx); + } else if (insn->off == 16) { + emit(rv_slli(lo(rd), lo(rs), 16), ctx); + emit(rv_srai(lo(rd), lo(rd), 16), ctx); + } + bpf_put_reg32(dst, rd, ctx); + break; + } fallthrough; case BPF_ALU | BPF_ADD | BPF_X: -- 2.54.0.563.g4f69b47b94-goog The RV32 BPF JIT compiler currently only supports the BPF_ADD atomic operation. Other 32 bit atomic operations (and, or, xor, xchg) and their BPF_FETCH variants are not supported and gracefully fall back to the interpreter. Since the RISC-V A extension is required for Linux on RV32, we can natively support these 32-bit BPF atomic operations by mapping them directly to the corresponding RISC-V amo*.w instructions. Implement BPF_ADD, BPF_AND, BPF_OR, BPF_XOR, and BPF_XCHG with and without BPF_FETCH. BPF_CMPXCHG requires a more complex lr.w/sc.w loop and is left to fall back to the interpreter. Before this patch: [ 138.862161] test_bpf: Summary: 1054 PASSED, 0 FAILED, [843/1042 JIT'ed] After this patch: [ 157.024124] test_bpf: Summary: 1054 PASSED, 0 FAILED, [902/1042 JIT'ed] Signed-off-by: Kuan-Wei Chiu --- arch/riscv/net/bpf_jit_comp32.c | 50 +++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/arch/riscv/net/bpf_jit_comp32.c b/arch/riscv/net/bpf_jit_comp32.c index f8509950fed4..7fd726a09b26 100644 --- a/arch/riscv/net/bpf_jit_comp32.c +++ b/arch/riscv/net/bpf_jit_comp32.c @@ -877,7 +877,7 @@ static int emit_load_r64(const s8 *dst, const s8 *src, s16 off, static int emit_store_r64(const s8 *dst, const s8 *src, s16 off, struct rv_jit_context *ctx, const u8 size, - const u8 mode) + const u8 mode, s32 imm) { const s8 *tmp1 = bpf2rv32[TMP_REG_1]; const s8 *tmp2 = bpf2rv32[TMP_REG_2]; @@ -902,11 +902,43 @@ static int emit_store_r64(const s8 *dst, const s8 *src, s16 off, case BPF_MEM: emit(rv_sw(RV_REG_T0, 0, lo(rs)), ctx); break; - case BPF_ATOMIC: /* Only BPF_ADD supported */ - emit(rv_amoadd_w(RV_REG_ZERO, lo(rs), RV_REG_T0, 0, 0), - ctx); + case BPF_ATOMIC: + { + bool is_fetch = (imm & BPF_FETCH) || (imm == BPF_XCHG); + s8 fetch_reg = is_fetch ? lo(rs) : RV_REG_ZERO; + int aq = is_fetch ? 1 : 0; + int rl = is_fetch ? 1 : 0; + + switch (imm) { + case BPF_ADD: + case BPF_ADD | BPF_FETCH: + emit(rv_amoadd_w(fetch_reg, lo(rs), RV_REG_T0, aq, rl), ctx); + break; + case BPF_AND: + case BPF_AND | BPF_FETCH: + emit(rv_amoand_w(fetch_reg, lo(rs), RV_REG_T0, aq, rl), ctx); + break; + case BPF_OR: + case BPF_OR | BPF_FETCH: + emit(rv_amoor_w(fetch_reg, lo(rs), RV_REG_T0, aq, rl), ctx); + break; + case BPF_XOR: + case BPF_XOR | BPF_FETCH: + emit(rv_amoxor_w(fetch_reg, lo(rs), RV_REG_T0, aq, rl), ctx); + break; + case BPF_XCHG: + emit(rv_amoswap_w(fetch_reg, lo(rs), RV_REG_T0, aq, rl), ctx); + break; + default: + return -1; + } + if (is_fetch) { + emit(rv_addi(hi(rs), RV_REG_ZERO, 0), ctx); + bpf_put_reg64(src, rs, ctx); + } break; } + } break; case BPF_DW: emit(rv_sw(RV_REG_T0, 0, lo(rs)), ctx); @@ -1308,20 +1340,16 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, } if (emit_store_r64(dst, src, off, ctx, BPF_SIZE(code), - BPF_MODE(code))) + BPF_MODE(code), 0)) return -1; break; case BPF_STX | BPF_ATOMIC | BPF_W: - if (insn->imm != BPF_ADD) { - pr_info_once( - "bpf-jit: not supported: atomic operation %02x ***\n", - insn->imm); + if (insn->imm == BPF_CMPXCHG) return -EFAULT; - } if (emit_store_r64(dst, src, off, ctx, BPF_SIZE(code), - BPF_MODE(code))) + BPF_MODE(code), insn->imm)) return -1; break; -- 2.54.0.563.g4f69b47b94-goog