From: Alexei Starovoitov A prog that has a stack in arena takes the per-cpu stack each time it starts. It does this with fetch_add on a per-cpu word. Only this cpu accesses the word. Thus the insn must be atomic only for an interrupt or for a different task on the same cpu. But the fetch_add of BPF is atomic for all cpus. On x86 it has the lock prefix, which waits for the stores in the store buffer. Add BPF_ATOMIC_LOCAL, a flag in imm of fetch_add. It means that the insn is atomic only for code that runs on this cpu. One x86 insn is atomic for such code. Thus the x86 JIT does not emit the lock prefix: BPF_ADD | BPF_FETCH lock xadd BPF_ADD | BPF_FETCH | BPF_ATOMIC_LOCAL xadd The flag is for per-cpu memory only. It is correct there because progs do not migrate. Only the verifier emits the flag, and only when the JIT reports bpf_jit_supports_local_atomics(). The other JITs get the usual fetch_add. The first user is "bpf: Support BPF_REG_ARENA_SP in the verifier". The verifier continues to reject the flag in the insns of a prog. The atomics of progs get the same code. The time that the flag saves depends on the stores that are in the store buffer when the prog starts. On x86-64 (Zen 5), in nsec per run: hot caches 0.44 after a store to 16 Mbyte of data (L3) 1.6 after a store that misses all caches 47 Signed-off-by: Alexei Starovoitov --- arch/x86/net/bpf_jit_comp.c | 10 +++++++++- include/linux/filter.h | 8 ++++++++ kernel/bpf/core.c | 6 ++++++ kernel/bpf/disasm.c | 7 +++++++ 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 8ade87bd56e3..711d774b40e1 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1625,7 +1625,10 @@ static int emit_atomic_rmw(u8 **pprog, u32 atomic_op, { u8 *prog = *pprog; - if (atomic_op != BPF_XCHG) + /* one insn is atomic for what runs on this cpu */ + if (atomic_op & BPF_ATOMIC_LOCAL) + atomic_op &= ~BPF_ATOMIC_LOCAL; + else if (atomic_op != BPF_XCHG) EMIT1(0xF0); /* lock prefix */ maybe_emit_mod(&prog, dst_reg, src_reg, bpf_size == BPF_DW); @@ -5187,6 +5190,11 @@ bool bpf_jit_supports_private_stack(void) return true; } +bool bpf_jit_supports_local_atomics(void) +{ + return true; +} + void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie) { #if defined(CONFIG_UNWINDER_ORC) diff --git a/include/linux/filter.h b/include/linux/filter.h index abc759ad14a6..7b730b6488ec 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -81,6 +81,13 @@ struct ctl_table_header; /* unused opcode to mark special atomic instruction */ #define BPF_PROBE_ATOMIC 0xe0 +/* + * Flag in imm of BPF_ATOMIC fetch_add: atomic for what runs on this cpu only, + * an interrupt or another task. The memory is per-cpu and the program + * doesn't migrate. It's cheaper where one insn without a lock does it. + */ +#define BPF_ATOMIC_LOCAL 0x1000 + /* unused opcode to mark special ldsx instruction. Same as BPF_NOSPEC */ #define BPF_PROBE_MEM32SX 0xc0 @@ -1330,6 +1337,7 @@ bool bpf_jit_supports_ptr_xchg(void); bool bpf_jit_supports_arena(void); bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena); bool bpf_jit_supports_private_stack(void); +bool bpf_jit_supports_local_atomics(void); bool bpf_jit_supports_large_stack(void); bool bpf_jit_supports_timed_may_goto(void); bool bpf_jit_supports_fsession(void); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 06d6d45b1c0e..1466a85c2b1a 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3874,6 +3874,12 @@ bool __weak bpf_jit_supports_private_stack(void) return false; } +/* Whether JIT supports BPF_ATOMIC_LOCAL */ +bool __weak bpf_jit_supports_local_atomics(void) +{ + return false; +} + /* * Return TRUE if the JIT lays out frames of up to MAX_BPF_STACK_JIT bytes. * Its prologue, epilogue and tail call sequences must encode such frame diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c index 0b1bbb27e9e4..2e377a7ea262 100644 --- a/kernel/bpf/disasm.c +++ b/kernel/bpf/disasm.c @@ -9,6 +9,7 @@ /* Only defined by the non-UAPI linux/filter.h, which this file cannot use. */ #define BPF_PROBE_ATOMIC 0xe0 +#define BPF_ATOMIC_LOCAL 0x1000 #define __BPF_FUNC_STR_FN(x) [BPF_FUNC_ ## x] = __stringify(bpf_ ## x) static const char * const func_id_str[] = { @@ -257,6 +258,12 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, insn->src_reg); + else if (atomic && insn->imm == (BPF_ADD | BPF_FETCH | BPF_ATOMIC_LOCAL)) + verbose(cbs->private_data, "(%02x) r%d = local%s_fetch_add((%s *)(r%d %+d), r%d)", + insn->code, insn->src_reg, + BPF_SIZE(insn->code) == BPF_DW ? "64" : "", + bpf_ldst_string[BPF_SIZE(insn->code) >> 3], + insn->dst_reg, insn->off, insn->src_reg); else if (atomic && (insn->imm == BPF_ADD || insn->imm == BPF_AND || insn->imm == BPF_OR || insn->imm == BPF_XOR)) { -- 2.55.0