cmp and cmn take a 12-bit immediate, optionally shifted by 12, so a compare against any other constant moved the constant into a temporary first. LLVM canonicalizes x <= C to x < C + 1 and x >= C to x > C - 1, so compares against a power of two or a mask, such as a length against a page size or a 16-bit limit, reach the JIT as compares against their neighbors and took the extra mov. When the neighbor fits, compare against it with the adjusted condition instead, x < C as x <= C - 1 and x > C as x >= C + 1, unsigned and signed alike. Before/after for if r1 < 4097 goto ... and if r1 > 65535 goto ...: d282002a mov x10, #0x1001 eb0a001f cmp x0, x10 54000063 b.lo ... f140041f cmp x0, #0x1, lsl #12 54000049 b.ls ... d29fffea mov x10, #0xffff eb0a001f cmp x0, x10 54000068 b.hi ... f140401f cmp x0, #0x10, lsl #12 54000042 b.hs ... Signed-off-by: Daniel Borkmann --- arch/arm64/net/bpf_jit_comp.c | 86 ++++++++++++++++++++++++++++++++--- 1 file changed, 80 insertions(+), 6 deletions(-) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index af0b05036d09..606e788facd0 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -477,6 +477,76 @@ static void emit_arena_ldst(const u8 code, const u8 rt, const u8 reg, const s32 a64_ldst_type(code, A64_LDST_REG)), ctx); } +/* + * A compare against an immediate neither cmp nor cmn can encode is one + * against its neighbor with the adjusted condition when that one fits: + * x < C is x <= C - 1 and x > C is x >= C + 1, for the unsigned as for the + * signed conditions, as long as C - 1 or C + 1 does not wrap. LLVM + * canonicalizes x <= C to x < C + 1, so the compares against a power of + * two or a mask plus one arrive here. + */ +static bool adjust_cmp_imm(u8 *op, s32 *imm) +{ + s64 c = *imm; + u8 new_op; + + switch (*op) { + case BPF_JLT: + case BPF_JGE: + if (!c) + return false; + fallthrough; + case BPF_JSLT: + case BPF_JSGE: + c--; + break; + case BPF_JGT: + case BPF_JLE: + if (c == -1) + return false; + fallthrough; + case BPF_JSGT: + case BPF_JSLE: + c++; + break; + default: + return false; + } + /* Past a signed limit the neighbor does not fit the 32-bit immediate */ + if (c != (s32)c || (!is_addsub_imm(c) && !is_addsub_imm(-(u32)c))) + return false; + + switch (*op) { + case BPF_JLT: + new_op = BPF_JLE; + break; + case BPF_JGE: + new_op = BPF_JGT; + break; + case BPF_JSLT: + new_op = BPF_JSLE; + break; + case BPF_JSGE: + new_op = BPF_JSGT; + break; + case BPF_JGT: + new_op = BPF_JGE; + break; + case BPF_JLE: + new_op = BPF_JLT; + break; + case BPF_JSGT: + new_op = BPF_JSGE; + break; + default: + new_op = BPF_JSLT; + break; + } + *op = new_op; + *imm = c; + return true; +} + /* generated main prog prologue: * bti c // if CONFIG_ARM64_BTI_KERNEL * mov x9, lr @@ -1600,6 +1670,8 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn const int i = insn - ctx->prog->insnsi; const bool is64 = BPF_CLASS(code) == BPF_ALU64 || BPF_CLASS(code) == BPF_JMP; + u8 jmp_op = BPF_OP(code); + s32 cmp_imm = imm; u8 jmp_cond; s32 jmp_offset; u32 a64_insn; @@ -1876,7 +1948,7 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn emit_cond_jmp: jmp_offset = bpf2a64_offset(i, off, ctx); check_imm19(jmp_offset); - switch (BPF_OP(code)) { + switch (jmp_op) { case BPF_JEQ: jmp_cond = A64_COND_EQ; break; @@ -1938,12 +2010,14 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn case BPF_JMP32 | BPF_JSLT | BPF_K: case BPF_JMP32 | BPF_JSGE | BPF_K: case BPF_JMP32 | BPF_JSLE | BPF_K: - if (is_addsub_imm(imm)) { - emit(A64_CMP_I(is64, dst, imm), ctx); - } else if (is_addsub_imm(-(u32)imm)) { - emit(A64_CMN_I(is64, dst, -imm), ctx); + if (!is_addsub_imm(imm) && !is_addsub_imm(-(u32)imm)) + adjust_cmp_imm(&jmp_op, &cmp_imm); + if (is_addsub_imm(cmp_imm)) { + emit(A64_CMP_I(is64, dst, cmp_imm), ctx); + } else if (is_addsub_imm(-(u32)cmp_imm)) { + emit(A64_CMN_I(is64, dst, -cmp_imm), ctx); } else { - emit_a64_mov_i(is64, tmp, imm, ctx); + emit_a64_mov_i(is64, tmp, cmp_imm, ctx); emit(A64_CMP(is64, dst, tmp), ctx); } goto emit_cond_jmp; -- 2.43.0