This commit adds support for BPF dynamic code modification on the LoongArch architecture.: 1. Implement bpf_arch_text_poke() for runtime instruction patching. 2. Add bpf_arch_text_copy() for instruction block copying. 3. Create bpf_arch_text_invalidate() for code invalidation. On LoongArch, since symbol addresses in the direct mapping region cannot be reached via relative jump instructions from the paged mapping region, we use the move_imm+jirl instruction pair as absolute jump instructions. These require 2-5 instructions, so we reserve 5 NOP instructions in the program as placeholders for function jumps. larch_insn_text_copy is solely used for BPF. The use of larch_insn_text_copy() requires page_size alignment. Currently, only the size of the trampoline is page-aligned. Co-developed-by: George Guo Signed-off-by: George Guo Signed-off-by: Chenghao Duan --- arch/loongarch/include/asm/inst.h | 1 + arch/loongarch/kernel/inst.c | 27 ++++++++ arch/loongarch/net/bpf_jit.c | 104 ++++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+) diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm/inst.h index 2ae96a35d..88bb73e46 100644 --- a/arch/loongarch/include/asm/inst.h +++ b/arch/loongarch/include/asm/inst.h @@ -497,6 +497,7 @@ void arch_simulate_insn(union loongarch_instruction insn, struct pt_regs *regs); int larch_insn_read(void *addr, u32 *insnp); int larch_insn_write(void *addr, u32 insn); int larch_insn_patch_text(void *addr, u32 insn); +int larch_insn_text_copy(void *dst, void *src, size_t len); u32 larch_insn_gen_nop(void); u32 larch_insn_gen_b(unsigned long pc, unsigned long dest); diff --git a/arch/loongarch/kernel/inst.c b/arch/loongarch/kernel/inst.c index 674e3b322..7df63a950 100644 --- a/arch/loongarch/kernel/inst.c +++ b/arch/loongarch/kernel/inst.c @@ -4,6 +4,7 @@ */ #include #include +#include #include #include @@ -218,6 +219,32 @@ int larch_insn_patch_text(void *addr, u32 insn) return ret; } +int larch_insn_text_copy(void *dst, void *src, size_t len) +{ + int ret; + unsigned long flags; + unsigned long dst_start, dst_end, dst_len; + + dst_start = round_down((unsigned long)dst, PAGE_SIZE); + dst_end = round_up((unsigned long)dst + len, PAGE_SIZE); + dst_len = dst_end - dst_start; + + set_memory_rw(dst_start, dst_len / PAGE_SIZE); + raw_spin_lock_irqsave(&patch_lock, flags); + + ret = copy_to_kernel_nofault(dst, src, len); + if (ret) + pr_err("%s: operation failed\n", __func__); + + raw_spin_unlock_irqrestore(&patch_lock, flags); + set_memory_rox(dst_start, dst_len / PAGE_SIZE); + + if (!ret) + flush_icache_range((unsigned long)dst, (unsigned long)dst + len); + + return ret; +} + u32 larch_insn_gen_nop(void) { return INSN_NOP; diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 7032f11d3..5e6ae7e0e 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -4,8 +4,12 @@ * * Copyright (C) 2022 Loongson Technology Corporation Limited */ +#include #include "bpf_jit.h" +#define LOONGARCH_LONG_JUMP_NINSNS 5 +#define LOONGARCH_LONG_JUMP_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4) + #define REG_TCC LOONGARCH_GPR_A6 #define TCC_SAVED LOONGARCH_GPR_S5 @@ -88,6 +92,7 @@ static u8 tail_call_reg(struct jit_ctx *ctx) */ static void build_prologue(struct jit_ctx *ctx) { + int i; int stack_adjust = 0, store_offset, bpf_stack_adjust; bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); @@ -98,6 +103,10 @@ static void build_prologue(struct jit_ctx *ctx) stack_adjust = round_up(stack_adjust, 16); stack_adjust += bpf_stack_adjust; + /* Reserve space for the move_imm + jirl instruction */ + for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++) + emit_insn(ctx, nop); + /* * First instruction initializes the tail call count (TCC). * On tail call we skip this instruction, and the TCC is @@ -1367,3 +1376,98 @@ bool bpf_jit_supports_subprog_tailcalls(void) { return true; } + +static int emit_jump_and_link(struct jit_ctx *ctx, u8 rd, u64 target) +{ + if (!target) { + pr_err("bpf_jit: jump target address is error\n"); + return -EFAULT; + } + + move_imm(ctx, LOONGARCH_GPR_T1, target, false); + emit_insn(ctx, jirl, rd, LOONGARCH_GPR_T1, 0); + + return 0; +} + +static int gen_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call) +{ + struct jit_ctx ctx; + + ctx.idx = 0; + ctx.image = (union loongarch_instruction *)insns; + + if (!target) { + emit_insn((&ctx), nop); + emit_insn((&ctx), nop); + return 0; + } + + return emit_jump_and_link(&ctx, is_call ? LOONGARCH_GPR_T0 : LOONGARCH_GPR_ZERO, + (unsigned long)target); +} + +int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, + void *old_addr, void *new_addr) +{ + u32 old_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP}; + u32 new_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP}; + bool is_call = poke_type == BPF_MOD_CALL; + int ret; + + if (!is_kernel_text((unsigned long)ip) && + !is_bpf_text_address((unsigned long)ip)) + return -ENOTSUPP; + + ret = gen_jump_or_nops(old_addr, ip, old_insns, is_call); + if (ret) + return ret; + + if (memcmp(ip, old_insns, LOONGARCH_LONG_JUMP_NBYTES)) + return -EFAULT; + + ret = gen_jump_or_nops(new_addr, ip, new_insns, is_call); + if (ret) + return ret; + + mutex_lock(&text_mutex); + if (memcmp(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES)) + ret = larch_insn_text_copy(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES); + mutex_unlock(&text_mutex); + return ret; +} + +int bpf_arch_text_invalidate(void *dst, size_t len) +{ + int i; + int ret = 0; + u32 *inst; + + inst = kvmalloc(len, GFP_KERNEL); + if (!inst) + return -ENOMEM; + + for (i = 0; i < (len/sizeof(u32)); i++) + inst[i] = INSN_BREAK; + + mutex_lock(&text_mutex); + if (larch_insn_text_copy(dst, inst, len)) + ret = -EINVAL; + mutex_unlock(&text_mutex); + + kvfree(inst); + return ret; +} + +void *bpf_arch_text_copy(void *dst, void *src, size_t len) +{ + int ret; + + mutex_lock(&text_mutex); + ret = larch_insn_text_copy(dst, src, len); + mutex_unlock(&text_mutex); + if (ret) + return ERR_PTR(-EINVAL); + + return dst; +} -- 2.25.1