A load or store with a negative offset, or with one that is not a multiple of the access size, does not fit the scaled unsigned 12-bit immediate form and was emitted as a mov of the offset into a temporary followed by the register offset form. LDUR and STUR take a signed 9-bit byte offset, so use them for the offsets from -256 to 255 the other form cannot encode, which saves the mov. The unscaled forms have the same latency and issue on the same pipelines as the scaled ones on Neoverse N1 through V3. LLVM emits such offsets for every 32-bit and wider header field behind the 14-byte Ethernet header of a packet parser, and negative ones for the container of a list or rbtree node. Before/after for r0 = *(u32 *)(r1 + 26) and r0 = *(u64 *)(r1 - 16): d280034a mov x10, #0x1a b86a6808 ldr w8, [x0, x10] b841a008 ldur w8, [x0, #26] 928001ea mov x10, #0xfffffffffffffff0 f86a6808 ldr x8, [x0, x10] f85f0008 ldur x8, [x0, #-16] Signed-off-by: Daniel Borkmann --- arch/arm64/net/bpf_jit_comp.c | 53 +++++++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 41e2d1aef71f..2a9bbd600bc0 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -357,6 +357,12 @@ static bool is_lsi_offset(int offset, int scale) return true; } +/* The unscaled forms take a signed 9-bit byte offset instead */ +static bool is_lsu_offset(int offset) +{ + return offset >= -SZ_256 && offset < SZ_256; +} + /* The A64 size of a BPF load or store, and the scale of its offset */ static enum aarch64_insn_size_type a64_ldst_size(const u8 code, int *scale) { @@ -376,18 +382,36 @@ static enum aarch64_insn_size_type a64_ldst_size(const u8 code, int *scale) } } -/* The A64 type of a BPF load or store, in the immediate or register offset form */ -static enum aarch64_insn_ldst_type a64_ldst_type(const u8 code, bool reg_offset) -{ +/* The forms of a load or store: scaled unsigned immediate, unscaled, register offset */ +enum a64_ldst_form { A64_LDST_IMM, A64_LDST_UNSCALED, A64_LDST_REG }; + +/* The A64 type of a BPF load or store in the given form */ +static enum aarch64_insn_ldst_type a64_ldst_type(const u8 code, enum a64_ldst_form form) +{ + static const enum aarch64_insn_ldst_type types[][3] = { + [A64_LDST_IMM] = { + AARCH64_INSN_LDST_LOAD_IMM_OFFSET, + AARCH64_INSN_LDST_SIGNED_LOAD_IMM_OFFSET, + AARCH64_INSN_LDST_STORE_IMM_OFFSET, + }, + [A64_LDST_UNSCALED] = { + AARCH64_INSN_LDST_LOAD_UNSCALED_IMM_OFFSET, + AARCH64_INSN_LDST_SIGNED_LOAD_UNSCALED_IMM_OFFSET, + AARCH64_INSN_LDST_STORE_UNSCALED_IMM_OFFSET, + }, + [A64_LDST_REG] = { + AARCH64_INSN_LDST_LOAD_REG_OFFSET, + AARCH64_INSN_LDST_SIGNED_LOAD_REG_OFFSET, + AARCH64_INSN_LDST_STORE_REG_OFFSET, + }, + }; + if (BPF_CLASS(code) != BPF_LDX) - return reg_offset ? AARCH64_INSN_LDST_STORE_REG_OFFSET : - AARCH64_INSN_LDST_STORE_IMM_OFFSET; + return types[form][2]; if (BPF_MODE(code) == BPF_MEMSX || BPF_MODE(code) == BPF_PROBE_MEMSX || BPF_MODE(code) == BPF_PROBE_MEM32SX) - return reg_offset ? AARCH64_INSN_LDST_SIGNED_LOAD_REG_OFFSET : - AARCH64_INSN_LDST_SIGNED_LOAD_IMM_OFFSET; - return reg_offset ? AARCH64_INSN_LDST_LOAD_REG_OFFSET : - AARCH64_INSN_LDST_LOAD_IMM_OFFSET; + return types[form][1]; + return types[form][0]; } /* @@ -400,6 +424,7 @@ static void emit_ldst(const u8 code, const u8 rt, const u8 rn, const s32 off, const u8 tmp, struct jit_ctx *ctx) { enum aarch64_insn_size_type size; + enum aarch64_insn_ldst_type type; s32 off_adj = off; u8 rn_adj = rn; int scale; @@ -416,12 +441,16 @@ static void emit_ldst(const u8 code, const u8 rt, const u8 rn, const s32 off, } if (is_lsi_offset(off_adj, scale)) { - emit(aarch64_insn_gen_load_store_imm(rt, rn_adj, off_adj, size, - a64_ldst_type(code, false)), ctx); + type = a64_ldst_type(code, A64_LDST_IMM); + emit(aarch64_insn_gen_load_store_imm(rt, rn_adj, off_adj, size, type), ctx); + } else if (is_lsu_offset(off_adj)) { + type = a64_ldst_type(code, A64_LDST_UNSCALED); + emit(aarch64_insn_gen_load_store_unscaled(rt, rn_adj, off_adj, size, type), ctx); } else { + type = a64_ldst_type(code, A64_LDST_REG); emit_a64_mov_i(1, tmp, off, ctx); emit(aarch64_insn_gen_load_store_reg(rt, rn, tmp, AARCH64_INSN_EXTEND_UXTX, - false, size, a64_ldst_type(code, true)), ctx); + false, size, type), ctx); } } -- 2.43.0