From: Chenguang Zhao Per the Linux Kernel Memory Model, value-returning atomic RMW operations must provide sequentially consistent ordering (a full memory barrier). On LoongArch, plain AMO instructions and bare ll/sc loops do not satisfy this requirement by themselves. Update emit_atomic_rmw() to emit barrier-carrying instructions for all value-returning BPF atomics: - BPF_FETCH (ADD/AND/OR/XOR): use am*_db.{b,h,w,d} - BPF_XCHG: use amswap_db.{b,h,w,d} - BPF_CMPXCHG: emit dbar 0x700 after the ll/sc loop, matching __WEAK_LLSC_MB in cmpxchg.h Add the corresponding instruction encodings and emit helpers to inst.h. Non-value-returning RMW ops (plain BPF_ADD, BPF_AND, etc.) are left as weakly ordered, consistent with LKMM. Signed-off-by: Chenguang Zhao --- arch/loongarch/include/asm/inst.h | 18 +++++++++++++++++ arch/loongarch/net/bpf_jit.c | 32 +++++++++++++++++-------------- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm/inst.h index 76b723590023..bdbc17d07110 100644 --- a/arch/loongarch/include/asm/inst.h +++ b/arch/loongarch/include/asm/inst.h @@ -199,6 +199,10 @@ enum reg3_op { amswaph_op = 0x70b9, amaddb_op = 0x70ba, amaddh_op = 0x70bb, + amswapdbb_op = 0x70bc, + amswapdbh_op = 0x70bd, + amadddbb_op = 0x70be, + amadddbh_op = 0x70bf, amswapw_op = 0x70c0, amswapd_op = 0x70c1, amaddw_op = 0x70c2, @@ -783,6 +787,20 @@ DEF_EMIT_REG3_FORMAT(amswapb, amswapb_op) DEF_EMIT_REG3_FORMAT(amswaph, amswaph_op) DEF_EMIT_REG3_FORMAT(amswapw, amswapw_op) DEF_EMIT_REG3_FORMAT(amswapd, amswapd_op) +DEF_EMIT_REG3_FORMAT(amswapdbb, amswapdbb_op) +DEF_EMIT_REG3_FORMAT(amswapdbh, amswapdbh_op) +DEF_EMIT_REG3_FORMAT(amadddbb, amadddbb_op) +DEF_EMIT_REG3_FORMAT(amadddbh, amadddbh_op) +DEF_EMIT_REG3_FORMAT(amadddbw, amadddbw_op) +DEF_EMIT_REG3_FORMAT(amadddbd, amadddbd_op) +DEF_EMIT_REG3_FORMAT(amanddbw, amanddbw_op) +DEF_EMIT_REG3_FORMAT(amanddbd, amanddbd_op) +DEF_EMIT_REG3_FORMAT(amordbw, amordbw_op) +DEF_EMIT_REG3_FORMAT(amordbd, amordbd_op) +DEF_EMIT_REG3_FORMAT(amxordbw, amxordbw_op) +DEF_EMIT_REG3_FORMAT(amxordbd, amxordbd_op) +DEF_EMIT_REG3_FORMAT(amswapdbw, amswapdbw_op) +DEF_EMIT_REG3_FORMAT(amswapdbd, amswapdbd_op) #define DEF_EMIT_REG3SA2_FORMAT(NAME, OP) \ static inline void emit_##NAME(union loongarch_instruction *insn, \ diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 29c281bef28e..5417a5624327 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -8,6 +8,9 @@ #include #include "bpf_jit.h" +/* dbar hint for ll/sc completion ordering, see __WEAK_LLSC_MB */ +#define DBAR_LLSC_MB 0x700 + #define LOONGARCH_MAX_REG_ARGS 8 #define LOONGARCH_SAVE_RA_NINSNS 1 @@ -433,7 +436,7 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx) pr_err_once("bpf-jit: amadd.b instruction is not supported\n"); return -EINVAL; } - emit_insn(ctx, amaddb, src, t1, t3); + emit_insn(ctx, amadddbb, src, t1, t3); emit_zext_32(ctx, src, true); break; case BPF_H: @@ -441,39 +444,39 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx) pr_err_once("bpf-jit: amadd.h instruction is not supported\n"); return -EINVAL; } - emit_insn(ctx, amaddh, src, t1, t3); + emit_insn(ctx, amadddbh, src, t1, t3); emit_zext_32(ctx, src, true); break; case BPF_W: - emit_insn(ctx, amaddw, src, t1, t3); + emit_insn(ctx, amadddbw, src, t1, t3); emit_zext_32(ctx, src, true); break; case BPF_DW: - emit_insn(ctx, amaddd, src, t1, t3); + emit_insn(ctx, amadddbd, src, t1, t3); break; } break; case BPF_AND | BPF_FETCH: if (isdw) { - emit_insn(ctx, amandd, src, t1, t3); + emit_insn(ctx, amanddbd, src, t1, t3); } else { - emit_insn(ctx, amandw, src, t1, t3); + emit_insn(ctx, amanddbw, src, t1, t3); emit_zext_32(ctx, src, true); } break; case BPF_OR | BPF_FETCH: if (isdw) { - emit_insn(ctx, amord, src, t1, t3); + emit_insn(ctx, amordbd, src, t1, t3); } else { - emit_insn(ctx, amorw, src, t1, t3); + emit_insn(ctx, amordbw, src, t1, t3); emit_zext_32(ctx, src, true); } break; case BPF_XOR | BPF_FETCH: if (isdw) { - emit_insn(ctx, amxord, src, t1, t3); + emit_insn(ctx, amxordbd, src, t1, t3); } else { - emit_insn(ctx, amxorw, src, t1, t3); + emit_insn(ctx, amxordbw, src, t1, t3); emit_zext_32(ctx, src, true); } break; @@ -485,7 +488,7 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx) pr_err_once("bpf-jit: amswap.b instruction is not supported\n"); return -EINVAL; } - emit_insn(ctx, amswapb, src, t1, t3); + emit_insn(ctx, amswapdbb, src, t1, t3); emit_zext_32(ctx, src, true); break; case BPF_H: @@ -493,15 +496,15 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx) pr_err_once("bpf-jit: amswap.h instruction is not supported\n"); return -EINVAL; } - emit_insn(ctx, amswaph, src, t1, t3); + emit_insn(ctx, amswapdbh, src, t1, t3); emit_zext_32(ctx, src, true); break; case BPF_W: - emit_insn(ctx, amswapw, src, t1, t3); + emit_insn(ctx, amswapdbw, src, t1, t3); emit_zext_32(ctx, src, true); break; case BPF_DW: - emit_insn(ctx, amswapd, src, t1, t3); + emit_insn(ctx, amswapdbd, src, t1, t3); break; } break; @@ -524,6 +527,7 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx) emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -6); emit_zext_32(ctx, r0, true); } + emit_insn(ctx, dbar, DBAR_LLSC_MB); break; default: pr_err_once("bpf-jit: invalid atomic read-modify-write opcode %02x\n", imm); -- 2.25.1 From: Chenguang Zhao The BPF verifier can lower bpf_kptr_xchg() to BPF_XCHG when the JIT advertises ptr xchg support. With ordered amswap_db.* emission from the previous patch, declare that LoongArch bpf JIT supports this inlining. Signed-off-by: Chenguang Zhao --- arch/loongarch/net/bpf_jit.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 5417a5624327..686d333146ce 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -2396,6 +2396,11 @@ bool bpf_jit_supports_fsession(void) return true; } +bool bpf_jit_supports_ptr_xchg(void) +{ + return true; +} + /* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */ bool bpf_jit_supports_subprog_tailcalls(void) { -- 2.25.1 From: Chenguang Zhao Run the kptr_xchg_inline functional test on LoongArch64 now that the bpf JIT can inline bpf_kptr_xchg() with correct memory ordering. Signed-off-by: Chenguang Zhao --- tools/testing/selftests/bpf/prog_tests/kptr_xchg_inline.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/prog_tests/kptr_xchg_inline.c b/tools/testing/selftests/bpf/prog_tests/kptr_xchg_inline.c index 7def158da9eb..1215d6edd590 100644 --- a/tools/testing/selftests/bpf/prog_tests/kptr_xchg_inline.c +++ b/tools/testing/selftests/bpf/prog_tests/kptr_xchg_inline.c @@ -14,7 +14,8 @@ void test_kptr_xchg_inline(void) int err; #if !(defined(__x86_64__) || defined(__aarch64__) || \ - (defined(__riscv) && __riscv_xlen == 64)) + (defined(__riscv) && __riscv_xlen == 64) || \ + (defined(__loongarch__) && __loongarch_grlen == 64)) test__skip(); return; #endif -- 2.25.1 From: Chenguang Zhao Add a bpf selftest benchmark that exercises bpf_kptr_xchg() in a tight loop so helper vs inlined JIT paths can be compared on supported architectures. Signed-off-by: Chenguang Zhao --- tools/testing/selftests/bpf/Makefile | 2 + tools/testing/selftests/bpf/bench.c | 2 + .../selftests/bpf/benchs/bench_kptr_xchg.c | 96 +++++++++++++++++++ .../selftests/bpf/progs/kptr_xchg_bench.c | 49 ++++++++++ 4 files changed, 149 insertions(+) create mode 100644 tools/testing/selftests/bpf/benchs/bench_kptr_xchg.c create mode 100644 tools/testing/selftests/bpf/progs/kptr_xchg_bench.c diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index b642ee489ea6..6dfc81748584 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -980,6 +980,7 @@ $(OUTPUT)/bench_lpm_trie_map.o: $(OUTPUT)/lpm_trie_bench.skel.h $(OUTPUT)/lpm_tr $(OUTPUT)/bench_bpf_nop.o: $(OUTPUT)/bpf_nop_bench.skel.h bench_bpf_timing.h $(OUTPUT)/bench_xdp_lb.o: $(OUTPUT)/xdp_lb_bench.skel.h bench_bpf_timing.h $(OUTPUT)/bench_bpf_timing.o: bench_bpf_timing.h +$(OUTPUT)/bench_kptr_xchg.o: $(OUTPUT)/kptr_xchg_bench.skel.h $(OUTPUT)/bench.o: bench.h testing_helpers.h $(BPFOBJ) $(OUTPUT)/bench: LDLIBS += -lm $(OUTPUT)/bench: $(OUTPUT)/bench.o \ @@ -1005,6 +1006,7 @@ $(OUTPUT)/bench: $(OUTPUT)/bench.o \ $(OUTPUT)/bench_bpf_timing.o \ $(OUTPUT)/bench_bpf_nop.o \ $(OUTPUT)/bench_xdp_lb.o \ + $(OUTPUT)/bench_kptr_xchg.o \ $(OUTPUT)/usdt_1.o \ $(OUTPUT)/usdt_2.o \ # diff --git a/tools/testing/selftests/bpf/bench.c b/tools/testing/selftests/bpf/bench.c index 3d9d2cd7764b..550290d6701a 100644 --- a/tools/testing/selftests/bpf/bench.c +++ b/tools/testing/selftests/bpf/bench.c @@ -582,6 +582,7 @@ extern const struct bench bench_lpm_trie_delete; extern const struct bench bench_lpm_trie_free; extern const struct bench bench_bpf_nop; extern const struct bench bench_xdp_lb; +extern const struct bench bench_kptr_xchg; static const struct bench *benchs[] = { &bench_count_global, @@ -665,6 +666,7 @@ static const struct bench *benchs[] = { &bench_lpm_trie_free, &bench_bpf_nop, &bench_xdp_lb, + &bench_kptr_xchg, }; static void find_benchmark(void) diff --git a/tools/testing/selftests/bpf/benchs/bench_kptr_xchg.c b/tools/testing/selftests/bpf/benchs/bench_kptr_xchg.c new file mode 100644 index 000000000000..b8a0d346fda6 --- /dev/null +++ b/tools/testing/selftests/bpf/benchs/bench_kptr_xchg.c @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2026. Loongson Technology Corporation Limited */ +#include +#include "bench.h" +#include "kptr_xchg_bench.skel.h" + +static struct ctx { + struct kptr_xchg_bench *skel; +} ctx; + +static struct { + __u32 nr_loops; +} args = { + .nr_loops = 256, +}; + +enum { + ARG_NR_LOOPS = 7000, +}; + +static const struct argp_option opts[] = { + { "nr_loops", ARG_NR_LOOPS, "nr_loops", 0, + "Set number of bpf_kptr_xchg() calls per trigger"}, + {}, +}; + +static error_t parse_arg(int key, char *arg, struct argp_state *state) +{ + switch (key) { + case ARG_NR_LOOPS: + args.nr_loops = strtol(arg, NULL, 10); + break; + default: + return ARGP_ERR_UNKNOWN; + } + + return 0; +} + +static const struct argp bench_kptr_xchg_argp = { + .options = opts, + .parser = parse_arg, +}; + +static void validate(void) +{ + if (env.consumer_cnt != 0) { + fprintf(stderr, "benchmark doesn't support consumer!\n"); + exit(1); + } +} + +static void *producer(void *input) +{ + while (true) + syscall(__NR_getpgid); + + return NULL; +} + +static void measure(struct bench_res *res) +{ + res->hits = atomic_swap(&ctx.skel->bss->hits, 0); +} + +static void setup(void) +{ + struct bpf_link *link; + + setup_libbpf(); + + ctx.skel = kptr_xchg_bench__open_and_load(); + if (!ctx.skel) { + fprintf(stderr, "failed to open skeleton\n"); + exit(1); + } + + ctx.skel->data->nr_loops = args.nr_loops; + + link = bpf_program__attach(ctx.skel->progs.benchmark); + if (!link) { + fprintf(stderr, "failed to attach program!\n"); + exit(1); + } +} + +const struct bench bench_kptr_xchg = { + .name = "kptr-xchg", + .argp = &bench_kptr_xchg_argp, + .validate = validate, + .setup = setup, + .producer_thread = producer, + .measure = measure, + .report_progress = ops_report_progress, + .report_final = ops_report_final, +}; diff --git a/tools/testing/selftests/bpf/progs/kptr_xchg_bench.c b/tools/testing/selftests/bpf/progs/kptr_xchg_bench.c new file mode 100644 index 000000000000..363883073e2c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/kptr_xchg_bench.c @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2026. Loongson Technology Corporation Limited */ +#include "vmlinux.h" +#include + +#include "bpf_experimental.h" +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +#define MAX_XCHG_LOOPS 4096 + +struct bin_data { + char blob[32]; +}; + +#define private(name) SEC(".bss." #name) __hidden __attribute__((aligned(8))) + +private(kptr) struct bin_data __kptr *ptr; +u32 nr_loops = 256; +long hits; + +SEC("fentry/" SYS_PREFIX "sys_getpgid") +int benchmark(void *ctx) +{ + struct bin_data *old; + u32 i; + + for (i = 0; i < MAX_XCHG_LOOPS; i++) { + if (i >= nr_loops) + break; + + old = bpf_kptr_xchg(&ptr, NULL); + if (old) + bpf_obj_drop(old); + } + + __sync_add_and_fetch(&hits, i); + return 0; +} + +/* + * BTF FUNC records are not generated for kfuncs referenced only through + * optimized paths. Keep bpf_obj_drop() visible to libbpf's kfunc linker. + */ +void __btf_root(void) +{ + bpf_obj_drop(NULL); +} -- 2.25.1