A trampoline managed by ftrace is determined by its target, that said ftrace_managed is already known when the trampoline is initialized. so, set 'ftrace_managed = tr->ip != 0' in bpf_trampoline_get() instead of relying on the first regular or tracing_multi attachment to set it. This makes the backend selection independent of attachment order and keeps the registration callbacks focused on installing the trampoline. No functional change intended. Signed-off-by: Leon Hwang --- kernel/bpf/trampoline.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 24edb59238f3..17f2a8133f61 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -475,18 +475,11 @@ static int register_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *im void *data __maybe_unused) { void *new_addr = im->image; - void *ip = tr->func.addr; - unsigned long faddr; int ret; - faddr = ftrace_location((unsigned long)ip); - if (faddr) { + if (tr->func.ftrace_managed) { if (!tr->fops) return -ENOTSUPP; - tr->func.ftrace_managed = true; - } - - if (tr->func.ftrace_managed) { ret = direct_ops_add(tr, new_addr); } else { ret = bpf_trampoline_update_fentry(tr, 0, NULL, new_addr); @@ -1286,6 +1279,7 @@ struct bpf_trampoline *bpf_trampoline_get(u64 key, memcpy(&tr->func.model, &tgt_info->fmodel, sizeof(tgt_info->fmodel)); tr->func.addr = (void *)tgt_info->tgt_addr; + tr->func.ftrace_managed = tr->ip != 0; out: trampoline_unlock(tr); return tr; @@ -1649,7 +1643,6 @@ static int register_fentry_multi(struct bpf_trampoline *tr, struct bpf_tramp_ima if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); - tr->func.ftrace_managed = true; ftrace_hash_add(data->reg, data->entry, ip, addr); tr->cur_image = im; return 0; -- 2.55.0 Since 'tr->ip' is already a 'ftrace_location()'-ed IP in bpf_trampoline_lookup(), it is unnecessary to call ftrace_location() again in register/unregister/modify_fentry_multi(). No functional changes intended. Signed-off-by: Leon Hwang --- kernel/bpf/trampoline.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 17f2a8133f61..21d460866dfe 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -1629,8 +1629,8 @@ static void ftrace_hash_add(struct ftrace_hash *hash, struct ftrace_func_entry * static int register_fentry_multi(struct bpf_trampoline *tr, struct bpf_tramp_image *im, void *ptr) { unsigned long addr = (unsigned long) im->image; - unsigned long ip = ftrace_location(tr->ip); struct bpf_tracing_multi_data *data = ptr; + unsigned long ip = tr->ip; /* * Different BTF ids get different trampolines, but they can still @@ -1651,13 +1651,12 @@ static int register_fentry_multi(struct bpf_trampoline *tr, struct bpf_tramp_ima static int unregister_fentry_multi(struct bpf_trampoline *tr, u32 orig_flags, void *ptr) { unsigned long addr = (unsigned long) tr->cur_image->image; - unsigned long ip = ftrace_location(tr->ip); struct bpf_tracing_multi_data *data = ptr; if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); - ftrace_hash_add(data->unreg, data->entry, ip, addr); + ftrace_hash_add(data->unreg, data->entry, tr->ip, addr); tr->cur_image = NULL; return 0; } @@ -1666,13 +1665,12 @@ static int modify_fentry_multi(struct bpf_trampoline *tr, u32 orig_flags, struct bool lock_direct_mutex, void *ptr) { unsigned long addr = (unsigned long) im->image; - unsigned long ip = ftrace_location(tr->ip); struct bpf_tracing_multi_data *data = ptr; if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); - ftrace_hash_add(data->modify, data->entry, ip, addr); + ftrace_hash_add(data->modify, data->entry, tr->ip, addr); tr->cur_image = im; return 0; } -- 2.55.0 In the following commit, it also needs to prepare both old_insn and new_insn when add text-poke batch support. So, factor out bpf_arch_text_poke_prepare_insns() helper for text-poke batch support. No functional changes intended. Signed-off-by: Leon Hwang --- arch/x86/net/bpf_jit_comp.c | 38 ++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 083fcd6cf15b..74033373dc06 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -646,13 +646,12 @@ static int emit_jump(u8 **pprog, void *func, void *ip) return emit_patch(pprog, func, ip, 0xE9); } -static int __bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, - enum bpf_text_poke_type new_t, - void *old_addr, void *new_addr) +static int bpf_arch_text_poke_prepare_insns(void *ip, enum bpf_text_poke_type old_t, + enum bpf_text_poke_type new_t, + void *old_addr, void *new_addr, + u8 *old_insn, u8 *new_insn) { const u8 *nop_insn = x86_nops[5]; - u8 old_insn[X86_PATCH_SIZE]; - u8 new_insn[X86_PATCH_SIZE]; u8 *prog; int ret; @@ -676,18 +675,31 @@ static int __bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, return ret; } - ret = -EBUSY; - mutex_lock(&text_mutex); + return 0; +} + +static int __bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, + enum bpf_text_poke_type new_t, + void *old_addr, void *new_addr) +{ + u8 old_insn[X86_PATCH_SIZE]; + u8 new_insn[X86_PATCH_SIZE]; + int ret; + + ret = bpf_arch_text_poke_prepare_insns(ip, old_t, new_t, old_addr, new_addr, old_insn, + new_insn); + if (ret) + return ret; + + guard(mutex)(&text_mutex); + if (memcmp(ip, old_insn, X86_PATCH_SIZE)) - goto out; - ret = 1; + return -EBUSY; if (memcmp(ip, new_insn, X86_PATCH_SIZE)) { smp_text_poke_single(ip, new_insn, X86_PATCH_SIZE, NULL); - ret = 0; + return 0; } -out: - mutex_unlock(&text_mutex); - return ret; + return 1; } int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, -- 2.55.0 In the following commit, it also needs to check the target IP and adds ENDBR_INSN_SIZE when add text-poke batch support. So, factor out bpf_arch_text_poke_check_ip() helper for text-poke batch support. No functional changes intended. Signed-off-by: Leon Hwang --- arch/x86/net/bpf_jit_comp.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 74033373dc06..c80d3161d458 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -702,10 +702,10 @@ static int __bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, return 1; } -int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, - enum bpf_text_poke_type new_t, void *old_addr, - void *new_addr) +static int bpf_arch_text_poke_check_ip(void **ipp) { + void *ip = *ipp; + if (!is_kernel_text((long)ip) && !is_bpf_text_address((long)ip)) /* BPF poking in modules is not supported */ @@ -716,7 +716,20 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, * with an ENDBR instruction. */ if (is_endbr(ip)) - ip += ENDBR_INSN_SIZE; + *ipp += ENDBR_INSN_SIZE; + + return 0; +} + +int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, + enum bpf_text_poke_type new_t, void *old_addr, + void *new_addr) +{ + int ret; + + ret = bpf_arch_text_poke_check_ip(&ip); + if (ret) + return ret; return __bpf_arch_text_poke(ip, old_t, new_t, old_addr, new_addr); } -- 2.55.0 Introduce bpf_arch_text_poke_batch() to patch multiple targets in batch, which utilizes smp_text_poke_batch_add/finish() on x86_64. Thereafter, it will able to text-poke multiple bpf progs at once via bpf_arch_text_poke_batch(). Signed-off-by: Leon Hwang --- arch/x86/net/bpf_jit_comp.c | 41 +++++++++++++++++++++++++++++++++++++ include/linux/bpf.h | 9 ++++++++ kernel/bpf/core.c | 5 +++++ 3 files changed, 55 insertions(+) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index c80d3161d458..b2040ae09273 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -734,6 +734,47 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, return __bpf_arch_text_poke(ip, old_t, new_t, old_addr, new_addr); } +int bpf_arch_text_poke_batch(struct bpf_text_poke **pokes, u32 cnt) +{ + u8 old_insn[X86_PATCH_SIZE]; + u8 new_insn[X86_PATCH_SIZE]; + int err; + u32 i; + + for (i = 0; i < cnt; i++) { + err = bpf_arch_text_poke_check_ip(&pokes[i]->ip); + if (err) + return err; + } + + guard(mutex)(&text_mutex); + + for (i = 0; i < cnt; i++) { + struct bpf_text_poke *poke = pokes[i]; + + err = bpf_arch_text_poke_prepare_insns(poke->ip, poke->old_t, poke->new_t, + poke->old_addr, poke->new_addr, old_insn, + new_insn); + if (err) + return err; + if (memcmp(poke->ip, old_insn, X86_PATCH_SIZE)) + return -EBUSY; + } + + for (i = 0; i < cnt; i++) { + struct bpf_text_poke *poke = pokes[i]; + + bpf_arch_text_poke_prepare_insns(poke->ip, poke->old_t, poke->new_t, + poke->old_addr, poke->new_addr, old_insn, + new_insn); + if (memcmp(poke->ip, new_insn, X86_PATCH_SIZE)) + smp_text_poke_batch_add(poke->ip, new_insn, X86_PATCH_SIZE, NULL); + } + + smp_text_poke_batch_finish(); + return 0; +} + #define EMIT_LFENCE() EMIT3(0x0F, 0xAE, 0xE8) static void __emit_indirect_jump(u8 **pprog, int reg, bool ereg) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 54144372281c..5b1a189e6e89 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -4231,6 +4231,15 @@ enum bpf_text_poke_type { BPF_MOD_JUMP, }; +struct bpf_text_poke { + void *ip; + void *old_addr; + void *new_addr; + enum bpf_text_poke_type old_t; + enum bpf_text_poke_type new_t; +}; + +int bpf_arch_text_poke_batch(struct bpf_text_poke **pokes, u32 cnt); int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, enum bpf_text_poke_type new_t, void *old_addr, void *new_addr); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 05c89396119a..49b794878c11 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3484,6 +3484,11 @@ int __weak skb_copy_bits(const struct sk_buff *skb, int offset, void *to, return -EFAULT; } +int __weak bpf_arch_text_poke_batch(struct bpf_text_poke **pokes, u32 cnt) +{ + return -EOPNOTSUPP; +} + int __weak bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, enum bpf_text_poke_type new_t, void *old_addr, void *new_addr) -- 2.55.0 Before calling bpf_arch_text_poke_batch(), check the duplicated target pokes by sorting the pokes. Sorting pokes is needed for smp_text_poke_batch_add() on x86_64 for better performance, see arch/x86/kernel/alternative.c::smp_text_poke_batch_add(). If the arch does not implement bpf_arch_text_poke_batch(), fallback to text-poke each target via bpf_arch_text_poke(). Signed-off-by: Leon Hwang --- kernel/bpf/trampoline.c | 51 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 21d460866dfe..a2fd0bd8c685 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -13,6 +13,7 @@ #include #include #include +#include /* dummy _ops. The verifier will operate on target program's ops. */ const struct bpf_verifier_ops bpf_extension_verifier_ops = { @@ -1863,6 +1864,56 @@ void bpf_trampoline_multi_detach(struct bpf_prog *prog, CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS && CONFIG_BPF_SYSCALL */ +static int bpf_text_poke_cmp(const void *a, const void *b) +{ + const struct bpf_text_poke *pa = *(struct bpf_text_poke * const *)a; + const struct bpf_text_poke *pb = *(struct bpf_text_poke * const *)b; + unsigned long ipa = (unsigned long) pa->ip; + unsigned long ipb = (unsigned long) pb->ip; + + return (ipa > ipb) - (ipa < ipb); +} + +static int bpf_text_poke_batch(struct bpf_text_poke **pokes, u32 cnt) +{ + int i, err; + + /* + * Sort pokes for two purposes: + * 1. Check duplicated poke target IPs. + * 2. Sorted poke target IPs would be friendly for poking them on x86, + * see arch/x86/kernel/alternative.c::smp_text_poke_batch_add(). + */ + sort_nonatomic(pokes, cnt, sizeof(*pokes), bpf_text_poke_cmp, NULL); + + for (i = 1; i < cnt; i++) + if (pokes[i - 1]->ip == pokes[i]->ip) + return -EINVAL; + + err = bpf_arch_text_poke_batch(pokes, cnt); + if (err != -EOPNOTSUPP) + return err; + + for (i = 0; i < cnt; i++) { + struct bpf_text_poke *p = pokes[i]; + + err = bpf_arch_text_poke(p->ip, p->old_t, p->new_t, p->old_addr, p->new_addr); + if (err) + goto rollback; + } + + return 0; + +rollback: + for (i--; i >= 0; i--) { + struct bpf_text_poke *p = pokes[i]; + + WARN_ON_ONCE(bpf_arch_text_poke(p->ip, p->new_t, p->old_t, p->new_addr, + p->old_addr)); + } + return err; +} + static int __init init_trampolines(void) { int i; -- 2.55.0 When adding tracing_multi link support for bpf progs, it must handle tgt_prog first in bpf_check_attach_target() to avoid the case: the btf_id is a valid BTF ID of tgt_prog, and also is "bpf_multi_func_btf_id[0]". Signed-off-by: Leon Hwang --- kernel/bpf/verifier.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 2f190abe3448..f2af2bbdd6fb 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -22284,22 +22284,24 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, return -EOPNOTSUPP; } - /* - * *.multi programs don't need an address during program - * verification, we just take the module ref if needed. - */ - if (is_tracing_multi_id(prog, btf_id)) { + if (tgt_prog) { + if (subprog == 0) + addr = (long)tgt_prog->bpf_func; + else + addr = (long)tgt_prog->aux->func[subprog]->bpf_func; + } else if (is_tracing_multi_id(prog, btf_id)) { + /* + * *.multi programs don't need an address during program + * verification, we just take the module ref if needed. + * *.multi programs don't have tgt_prog during program + * verification. + */ if (btf_is_module(btf)) { mod = btf_try_get_module(btf); if (!mod) return -ENOENT; } addr = 0; - } else if (tgt_prog) { - if (subprog == 0) - addr = (long) tgt_prog->bpf_func; - else - addr = (long) tgt_prog->aux->func[subprog]->bpf_func; } else { if (btf_is_module(btf)) { mod = btf_try_get_module(btf); -- 2.55.0 The following helpers are not relied on the CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS guard. * trampoline_lock_all() * trampoline_unlock_all() * bpf_trampoline_multi_attach_init() Move them out of the CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS guard in order to add tracing_multi link support for bpf progs. Keep the CONFIG_BPF_SYSCALL guard to avoid compiling warning when CONFIG_BPF_SYSCALL is disabled. Signed-off-by: Leon Hwang --- kernel/bpf/trampoline.c | 48 ++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index a2fd0bd8c685..bb832af8703f 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -60,6 +60,32 @@ static void trampoline_unlock(struct bpf_trampoline *tr) mutex_unlock(select_trampoline_lock(tr)); } +#ifdef CONFIG_BPF_SYSCALL + +static void trampoline_lock_all(void) +{ + int i; + + for (i = 0; i < TRAMPOLINE_LOCKS_TABLE_SIZE; i++) + mutex_lock(&trampoline_locks[i].mutex); +} + +static void trampoline_unlock_all(void) +{ + int i; + + for (i = 0; i < TRAMPOLINE_LOCKS_TABLE_SIZE; i++) + mutex_unlock(&trampoline_locks[i].mutex); +} + +static void bpf_trampoline_multi_attach_init(struct bpf_trampoline *tr) +{ + tr->multi_attach.old_image = tr->cur_image; + tr->multi_attach.old_flags = tr->flags; +} + +#endif /* CONFIG_BPF_SYSCALL */ + struct bpf_trampoline_ops { int (*register_fentry)(struct bpf_trampoline *tr, struct bpf_tramp_image *im, void *data); int (*unregister_fentry)(struct bpf_trampoline *tr, u32 orig_flags, void *data); @@ -1574,22 +1600,6 @@ int __weak arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, defined(CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS) && \ defined(CONFIG_BPF_SYSCALL) -static void trampoline_lock_all(void) -{ - int i; - - for (i = 0; i < TRAMPOLINE_LOCKS_TABLE_SIZE; i++) - mutex_lock(&trampoline_locks[i].mutex); -} - -static void trampoline_unlock_all(void) -{ - int i; - - for (i = 0; i < TRAMPOLINE_LOCKS_TABLE_SIZE; i++) - mutex_unlock(&trampoline_locks[i].mutex); -} - static void remove_tracing_multi_data(struct bpf_tracing_multi_data *data) { ftrace_hash_remove(data->reg); @@ -1682,12 +1692,6 @@ static const struct bpf_trampoline_ops trampoline_multi_ops = { .modify_fentry = modify_fentry_multi, }; -static void bpf_trampoline_multi_attach_init(struct bpf_trampoline *tr) -{ - tr->multi_attach.old_image = tr->cur_image; - tr->multi_attach.old_flags = tr->flags; -} - static void bpf_trampoline_multi_attach_free(struct bpf_trampoline *tr) { /* -- 2.55.0 The existing tracing_multi helpers update ftrace-managed kernel function targets through the ftrace direct-call API. BPF prog targets are not ftrace-managed and require text pokes at their entry points. Add bpf_trampoline_multi_prog_attach/detach() helpers that prepare each target trampoline, apply the text pokes in batch, and then commit the staged trampoline images. Roll back the added programs and trampoline images when attach fails. Set BPF_TRAMP_F_TAIL_CALL_CTX for tail-call-reachable target bpf progs so the generated trampolines preserve the tail_call_cnt. Signed-off-by: Leon Hwang --- include/linux/bpf.h | 45 ++++++--- kernel/bpf/trampoline.c | 216 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 246 insertions(+), 15 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 5b1a189e6e89..a37f926023fd 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1496,9 +1496,10 @@ struct bpf_trampoline { struct bpf_tramp_image *cur_image; /* Images not freed yet, cur_image and older ones still in use */ struct list_head images; - /* Used as temporary old image storage for multi_attach */ + /* Used as temporary images storage for multi_attach */ struct { struct bpf_tramp_image *old_image; + struct bpf_tramp_image *new_image; u32 old_flags; } multi_attach; }; @@ -1616,6 +1617,9 @@ int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids, struct bpf_tracing_multi_link *link); void bpf_trampoline_multi_detach(struct bpf_prog *prog, struct bpf_tracing_multi_link *link); +int bpf_trampoline_multi_prog_attach(struct bpf_prog *prog, u32 *ids, u64 *keys, + struct bpf_tracing_multi_link *link); +void bpf_trampoline_multi_prog_detach(struct bpf_tracing_multi_link *link); void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags); /* @@ -1739,6 +1743,14 @@ static inline void bpf_trampoline_multi_detach(struct bpf_prog *prog, struct bpf_tracing_multi_link *link) { } + +static inline int bpf_trampoline_multi_prog_attach(struct bpf_prog *prog, u32 *ids, u64 *keys, + struct bpf_tracing_multi_link *link) +{ + return -EOPNOTSUPP; +} + +static inline void bpf_trampoline_multi_prog_detach(struct bpf_tracing_multi_link *link) {} static inline void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags) {} #endif @@ -2089,10 +2101,25 @@ struct bpf_tracing_link { struct bpf_prog *tgt_prog; }; +enum bpf_text_poke_type { + BPF_MOD_NOP, + BPF_MOD_CALL, + BPF_MOD_JUMP, +}; + +struct bpf_text_poke { + void *ip; + void *old_addr; + void *new_addr; + enum bpf_text_poke_type old_t; + enum bpf_text_poke_type new_t; +}; + struct bpf_tracing_multi_node { struct bpf_tramp_node node; struct bpf_trampoline *trampoline; struct ftrace_func_entry entry; + struct bpf_text_poke poke; }; struct bpf_tracing_multi_data { @@ -2107,6 +2134,8 @@ struct bpf_tracing_multi_link { struct bpf_tracing_multi_data data; u64 *cookies; struct bpf_tramp_node *fexits; + struct bpf_text_poke **pokes; + struct bpf_prog **tgt_progs; int nodes_cnt; struct bpf_tracing_multi_node nodes[] __counted_by(nodes_cnt); }; @@ -4225,20 +4254,6 @@ static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type, } #endif /* CONFIG_INET */ -enum bpf_text_poke_type { - BPF_MOD_NOP, - BPF_MOD_CALL, - BPF_MOD_JUMP, -}; - -struct bpf_text_poke { - void *ip; - void *old_addr; - void *new_addr; - enum bpf_text_poke_type old_t; - enum bpf_text_poke_type new_t; -}; - int bpf_arch_text_poke_batch(struct bpf_text_poke **pokes, u32 cnt); int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, enum bpf_text_poke_type new_t, void *old_addr, diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index bb832af8703f..e57c687955fd 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -81,6 +81,7 @@ static void trampoline_unlock_all(void) static void bpf_trampoline_multi_attach_init(struct bpf_trampoline *tr) { tr->multi_attach.old_image = tr->cur_image; + tr->multi_attach.new_image = NULL; tr->multi_attach.old_flags = tr->flags; } @@ -1708,6 +1709,7 @@ static void bpf_trampoline_multi_attach_free(struct bpf_trampoline *tr) bpf_tramp_image_put(tr->multi_attach.old_image); tr->multi_attach.old_image = NULL; + tr->multi_attach.new_image = NULL; tr->multi_attach.old_flags = 0; } @@ -1719,6 +1721,7 @@ static void bpf_trampoline_multi_attach_rollback(struct bpf_trampoline *tr) tr->flags = tr->multi_attach.old_flags; tr->multi_attach.old_image = NULL; + tr->multi_attach.new_image = NULL; tr->multi_attach.old_flags = 0; } @@ -1868,6 +1871,8 @@ void bpf_trampoline_multi_detach(struct bpf_prog *prog, CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS && CONFIG_BPF_SYSCALL */ +#ifdef CONFIG_BPF_SYSCALL + static int bpf_text_poke_cmp(const void *a, const void *b) { const struct bpf_text_poke *pa = *(struct bpf_text_poke * const *)a; @@ -1918,6 +1923,217 @@ static int bpf_text_poke_batch(struct bpf_text_poke **pokes, u32 cnt) return err; } +static enum bpf_text_poke_type bpf_trampoline_poke_type(const struct bpf_tramp_image *im, u32 flags) +{ + if (!im) + return BPF_MOD_NOP; + + return bpf_trampoline_use_jmp(flags) ? BPF_MOD_JUMP : BPF_MOD_CALL; +} + +static void bpf_text_poke_init(struct bpf_text_poke *poke, struct bpf_trampoline *tr) +{ + struct bpf_tramp_image *old_image = tr->multi_attach.old_image; + struct bpf_tramp_image *new_image = tr->multi_attach.new_image; + + poke->ip = tr->func.addr; + poke->old_addr = old_image ? old_image->image : NULL; + poke->new_addr = new_image ? new_image->image : NULL; + poke->old_t = bpf_trampoline_poke_type(old_image, tr->multi_attach.old_flags); + poke->new_t = bpf_trampoline_poke_type(new_image, tr->flags); +} + +static int register_fentry_multi_prog(struct bpf_trampoline *tr, + struct bpf_tramp_image *im, + void *data __maybe_unused) +{ + tr->multi_attach.new_image = im; + return 0; +} + +static int unregister_fentry_multi_prog(struct bpf_trampoline *tr, + u32 orig_flags __maybe_unused, + void *data __maybe_unused) +{ + tr->multi_attach.new_image = NULL; + return 0; +} + +static int modify_fentry_multi_prog(struct bpf_trampoline *tr, + u32 orig_flags __maybe_unused, + struct bpf_tramp_image *im, + bool lock_direct_mutex __maybe_unused, + void *data __maybe_unused) +{ + tr->multi_attach.new_image = im; + return 0; +} + +static const struct bpf_trampoline_ops trampoline_multi_prog_ops = { + .register_fentry = register_fentry_multi_prog, + .unregister_fentry = unregister_fentry_multi_prog, + .modify_fentry = modify_fentry_multi_prog, +}; + +static void bpf_trampoline_multi_prog_commit(struct bpf_trampoline *tr) +{ + struct bpf_tramp_image *old_image = tr->multi_attach.old_image; + + tr->cur_image = tr->multi_attach.new_image; + tr->multi_attach.old_image = NULL; + tr->multi_attach.new_image = NULL; + tr->multi_attach.old_flags = 0; + + if (old_image) + bpf_tramp_image_put(old_image); +} + +static void bpf_trampoline_multi_prog_rollback(struct bpf_trampoline *tr) +{ + if (tr->multi_attach.new_image) + bpf_tramp_image_put(tr->multi_attach.new_image); + + tr->flags = tr->multi_attach.old_flags; + tr->multi_attach.old_image = NULL; + tr->multi_attach.new_image = NULL; + tr->multi_attach.old_flags = 0; +} + +#define for_each_mnode_cnt(mnode, link, cnt) \ + for (i = 0, mnode = &link->nodes[i]; i < cnt; i++, mnode = &link->nodes[i]) + +#define for_each_mnode(mnode, link) \ + for_each_mnode_cnt(mnode, link, link->nodes_cnt) + +static int bpf_trampoline_multi_prog_prepare(struct bpf_prog *prog, u32 *ids, u64 *keys, + struct bpf_tracing_multi_link *link) +{ + struct bpf_attach_target_info tgt_info = {}; + struct bpf_tracing_multi_node *mnode; + struct bpf_trampoline *tr; + int i, cnt, err; + + for_each_mnode(mnode, link) { + err = bpf_check_attach_target(NULL, prog, link->tgt_progs[i], ids[i], &tgt_info); + if (err) + goto put_trampolines; + + tr = bpf_trampoline_get(keys[i], &tgt_info); + if (!tr) { + err = -ENOMEM; + goto put_trampolines; + } + + mnode->trampoline = tr; + mnode->node.link = &link->link; + mnode->node.cookie = link->cookies ? link->cookies[i] : 0; + + if (prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI) { + link->fexits[i].link = &link->link; + link->fexits[i].cookie = link->cookies ? link->cookies[i] : 0; + } + } + + return 0; + +put_trampolines: + cnt = i; + for_each_mnode_cnt(mnode, link, cnt) + bpf_trampoline_put(mnode->trampoline); + return err; +} + +int bpf_trampoline_multi_prog_attach(struct bpf_prog *prog, u32 *ids, u64 *keys, + struct bpf_tracing_multi_link *link) +{ + struct bpf_tracing_multi_node *mnode; + int i, err, linked_cnt = 0; + + err = bpf_trampoline_multi_prog_prepare(prog, ids, keys, link); + if (err) + return err; + + trampoline_lock_all(); + + for_each_mnode(mnode, link) { + bpf_trampoline_multi_attach_init(mnode->trampoline); + + if (link->tgt_progs[i]->aux->tail_call_reachable) + mnode->trampoline->flags |= BPF_TRAMP_F_TAIL_CALL_CTX; + + err = __bpf_trampoline_link_prog(&mnode->node, mnode->trampoline, NULL, + &trampoline_multi_prog_ops, NULL); + if (err) { + bpf_trampoline_multi_prog_rollback(mnode->trampoline); + goto rollback; + } + linked_cnt++; + } + + for_each_mnode(mnode, link) { + link->pokes[i] = &mnode->poke; + bpf_text_poke_init(link->pokes[i], mnode->trampoline); + } + + err = bpf_text_poke_batch(link->pokes, link->nodes_cnt); + if (err) + goto rollback; + + for_each_mnode(mnode, link) + bpf_trampoline_multi_prog_commit(mnode->trampoline); + + trampoline_unlock_all(); + return 0; + +rollback: + for_each_mnode_cnt(mnode, link, linked_cnt) { + bpf_trampoline_remove_prog(mnode->trampoline, &mnode->node); + bpf_trampoline_multi_prog_rollback(mnode->trampoline); + } + + trampoline_unlock_all(); + + for_each_mnode(mnode, link) + bpf_trampoline_put(mnode->trampoline); + return err; +} + +void bpf_trampoline_multi_prog_detach(struct bpf_tracing_multi_link *link) +{ + struct bpf_tracing_multi_node *mnode; + int i, err; + + trampoline_lock_all(); + + for_each_mnode(mnode, link) { + bpf_trampoline_multi_attach_init(mnode->trampoline); + err = __bpf_trampoline_unlink_prog(&mnode->node, mnode->trampoline, NULL, + &trampoline_multi_prog_ops, NULL); + if (WARN_ONCE(err, "__bpf_trampoline_unlink_prog failed: %d\n", err)) + bpf_trampoline_multi_prog_rollback(mnode->trampoline); + } + + for_each_mnode(mnode, link) { + link->pokes[i] = &mnode->poke; + bpf_text_poke_init(link->pokes[i], mnode->trampoline); + } + + WARN_ON_ONCE(bpf_text_poke_batch(link->pokes, link->nodes_cnt)); + + for_each_mnode(mnode, link) + bpf_trampoline_multi_prog_commit(mnode->trampoline); + + trampoline_unlock_all(); + + for_each_mnode(mnode, link) + bpf_trampoline_put(mnode->trampoline); +} + +#undef for_each_mnode_cnt +#undef for_each_mnode + +#endif /* CONFIG_BPF_SYSCALL */ + static int __init init_trampolines(void) { int i; -- 2.55.0 When adding tracing_multi link support for bpf progs, it will use the same UAPI of the tracing_multi link for kernel functions in bpf_tracing_multi_attach(). However, the tracing_multi link support for bpf progs does not require CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS. So, shrink its guard scope to add new code in bpf_tracing_multi_attach() for the tracing_multi link support for bpf progs. To simplify the change, validate CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS only here, because the execution path in trampoline of the tracing_multi link support for kernel functions is already guarded with the CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS and CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS. Meanwhile, provide a stub of bpf_trampoline_multi_detach() out of the CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS and CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS guard. Signed-off-by: Leon Hwang --- kernel/bpf/trampoline.c | 5 +++++ kernel/trace/bpf_trace.c | 15 +++------------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index e57c687955fd..6e18f69a63ad 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -1867,6 +1867,11 @@ void bpf_trampoline_multi_detach(struct bpf_prog *prog, #undef for_each_mnode_cnt #undef for_each_mnode +#else + +void bpf_trampoline_multi_detach(struct bpf_prog *prog, + struct bpf_tracing_multi_link *link) {} + #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS && CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS && CONFIG_BPF_SYSCALL */ diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 195f78db9bda..69559158ab0b 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -3679,9 +3679,6 @@ __bpf_kfunc int bpf_copy_from_user_task_str_dynptr(const struct bpf_dynptr *dptr __bpf_kfunc_end_defs(); -#if defined(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS) && \ - defined(CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS) - static void bpf_tracing_multi_link_release(struct bpf_link *link) { struct bpf_tracing_multi_link *tr_link = @@ -3858,6 +3855,9 @@ int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) if (attr->link_create.flags || attr->link_create.target_fd) return -EINVAL; + if (!IS_ENABLED(CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS)) + return -EOPNOTSUPP; + ids = kvmalloc_objs(*ids, cnt); if (!ids) return -ENOMEM; @@ -3924,12 +3924,3 @@ int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) kvfree(link); return err; } - -#else - -int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) -{ - return -EOPNOTSUPP; -} - -#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS && CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS */ -- 2.55.0 Similar to the tracing_multi link support for kernel functions, add support for bpf progs by reusing the 'ids' fields as BTF ids, and adding fds array to indicate the target bpf prog. Then, for each pair of fd+id, they will be the attach targets of tracing_multi link. When attaching bpf prog, it attaches to the targets via bpf_trampoline_multi_prog_attach(), that would work on all trampoline-supported archs. Since tracing_multi prog cannot recognize its targets at load time, reject attaching to any other tracing prog to avoid two-levels nested tracing attachment: e.g. fentry <- fentry.multi <- another fentry. Assisted-by: Codex:gpt-5.6-sol Signed-off-by: Leon Hwang --- include/uapi/linux/bpf.h | 2 + kernel/trace/bpf_trace.c | 177 ++++++++++++++++++++++++++++----- tools/include/uapi/linux/bpf.h | 2 + 3 files changed, 156 insertions(+), 25 deletions(-) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 6f70db7515dd..94302a3b680c 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -1927,6 +1927,8 @@ union bpf_attr { __aligned_u64 ids; __aligned_u64 cookies; __u32 cnt; + __u32 :32; + __aligned_u64 fds; } tracing_multi; }; } link_create; diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 69559158ab0b..d4af42e25fe5 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -3679,12 +3679,19 @@ __bpf_kfunc int bpf_copy_from_user_task_str_dynptr(const struct bpf_dynptr *dptr __bpf_kfunc_end_defs(); +static void bpf_put_progs(struct bpf_prog **progs, int cnt); + static void bpf_tracing_multi_link_release(struct bpf_link *link) { struct bpf_tracing_multi_link *tr_link = container_of(link, struct bpf_tracing_multi_link, link); - bpf_trampoline_multi_detach(link->prog, tr_link); + if (tr_link->tgt_progs) { + bpf_trampoline_multi_prog_detach(tr_link); + bpf_put_progs(tr_link->tgt_progs, tr_link->nodes_cnt); + } else { + bpf_trampoline_multi_detach(link->prog, tr_link); + } } static void bpf_tracing_multi_link_dealloc(struct bpf_link *link) @@ -3692,6 +3699,8 @@ static void bpf_tracing_multi_link_dealloc(struct bpf_link *link) struct bpf_tracing_multi_link *tr_link = container_of(link, struct bpf_tracing_multi_link, link); + kvfree(tr_link->pokes); + kvfree(tr_link->tgt_progs); kvfree(tr_link->fexits); kvfree(tr_link->cookies); kvfree(tr_link); @@ -3787,64 +3796,129 @@ static const struct bpf_link_ops bpf_tracing_multi_link_lops = { #endif }; -static int ids_cmp_r(const void *pa, const void *pb, const void *priv __maybe_unused) +struct tracing_multi_sort_data { + u64 *keys; + struct bpf_prog **progs; + u32 *ids; + u64 *cookies; +}; + +static int keys_cmp_r(const void *pa, const void *pb, const void *priv __maybe_unused) { - u32 a = *(u32 *) pa; - u32 b = *(u32 *) pb; + const u64 *key_a = pa, *key_b = pb; - return (a > b) - (a < b); + return (*key_a > *key_b) - (*key_a < *key_b); } -static void ids_swap_r(void *a, void *b, int size __maybe_unused, - const void *priv __maybe_unused) +static void keys_swap_r(void *a, void *b, int size __maybe_unused, + const void *priv __maybe_unused) { + const struct tracing_multi_sort_data *data = priv; + struct bpf_prog **prog_a, **prog_b; u64 *cookie_a, *cookie_b, *cookies; - u32 *id_a = a, *id_b = b, *ids; - void **data = (void **) priv; + u64 *key_a = a, *key_b = b, *keys; + u32 *id_a, *id_b, *ids; + + keys = data->keys; + ids = data->ids; + cookies = data->cookies; + id_a = ids + (key_a - keys); + id_b = ids + (key_b - keys); - ids = data[0]; - cookies = data[1]; + if (data->progs) { + prog_a = data->progs + (key_a - keys); + prog_b = data->progs + (key_b - keys); + swap(*prog_a, *prog_b); + } if (cookies) { cookie_a = cookies + (id_a - ids); cookie_b = cookies + (id_b - ids); swap(*cookie_a, *cookie_b); } + swap(*id_a, *id_b); + swap(*key_a, *key_b); } -static int check_dup_ids(u32 *ids, u64 *cookies, u32 cnt) +static int check_dup_keys(u64 *keys, struct bpf_prog **progs, u32 *ids, + u64 *cookies, u32 cnt) { - void *data[2] = { ids, cookies }; - int err = 0; + struct tracing_multi_sort_data data = { + .keys = keys, + .progs = progs, + .ids = ids, + .cookies = cookies, + }; /* - * Sort ids array (together with cookies array if defined) - * and check it for duplicates. The ids and cookies arrays - * are left sorted. + * Sort trampoline keys together with target programs, ids, and cookies, + * then check for duplicates. The arrays are left sorted. */ - sort_r_nonatomic(ids, cnt, sizeof(ids[0]), ids_cmp_r, ids_swap_r, data); + sort_r_nonatomic(keys, cnt, sizeof(keys[0]), keys_cmp_r, keys_swap_r, &data); + + for (int i = 1; i < cnt; i++) + if (keys[i] == keys[i - 1]) + return -EINVAL; - for (int i = 1; i < cnt; i++) { - if (ids[i] == ids[i - 1]) { + return 0; +} + +static void bpf_put_progs(struct bpf_prog **progs, int cnt) +{ + int i; + + if (!progs) + return; + + for (i = 0; i < cnt; i++) + bpf_prog_put(progs[i]); +} + +static int bpf_get_progs(struct bpf_prog **progs, int *fds, int cnt) +{ + int err, i, put_cnt = 0; + struct bpf_prog *prog; + + for (i = 0; i < cnt; i++) { + prog = bpf_prog_get(fds[i]); + if (IS_ERR(prog)) { + err = PTR_ERR(prog); + goto error; + } + + progs[i] = prog; + put_cnt++; + + if (prog->type == BPF_PROG_TYPE_TRACING) { err = -EINVAL; - break; + goto error; } } + return 0; + +error: + bpf_put_progs(progs, put_cnt); return err; } int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) { struct bpf_tracing_multi_link *link = NULL; + struct btf *btf = prog->aux->attach_btf; struct bpf_tramp_node *fexits = NULL; struct bpf_link_primer link_primer; + struct bpf_text_poke **pokes = NULL; + u64 *cookies = NULL, *keys = NULL; + struct bpf_prog **progs = NULL; u32 cnt, *ids = NULL; u64 __user *ucookies; - u64 *cookies = NULL; u32 __user *uids; + int __user *ufds; + int *fds = NULL; int err; + ufds = u64_to_user_ptr(attr->link_create.tracing_multi.fds); uids = u64_to_user_ptr(attr->link_create.tracing_multi.ids); cnt = attr->link_create.tracing_multi.cnt; @@ -3855,7 +3929,7 @@ int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) if (attr->link_create.flags || attr->link_create.target_fd) return -EINVAL; - if (!IS_ENABLED(CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS)) + if (!ufds && !IS_ENABLED(CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS)) return -EOPNOTSUPP; ids = kvmalloc_objs(*ids, cnt); @@ -3867,6 +3941,37 @@ int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) goto error; } + if (ufds) { + fds = kvmalloc_objs(*fds, cnt); + if (!fds) { + err = -ENOMEM; + goto error; + } + + if (copy_from_user(fds, ufds, cnt * sizeof(*fds))) { + err = -EFAULT; + goto error; + } + + progs = kvmalloc_objs(*progs, cnt); + if (!progs) { + err = -ENOMEM; + goto error; + } + + err = bpf_get_progs(progs, fds, cnt); + if (err) { + cnt = 0; + goto error; + } + + pokes = kvmalloc_objs(*pokes, cnt); + if (!pokes) { + err = -ENOMEM; + goto error; + } + } + ucookies = u64_to_user_ptr(attr->link_create.tracing_multi.cookies); if (ucookies) { cookies = kvmalloc_objs(*cookies, cnt); @@ -3880,7 +3985,16 @@ int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) } } - err = check_dup_ids(ids, cookies, cnt); + keys = kvmalloc_objs(*keys, cnt); + if (!keys) { + err = -ENOMEM; + goto error; + } + + for (int i = 0; i < cnt; i++) + keys[i] = bpf_trampoline_compute_key(progs ? progs[i] : NULL, btf, ids[i]); + + err = check_dup_keys(keys, progs, ids, cookies, cnt); if (err) goto error; @@ -3908,19 +4022,32 @@ int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) link->nodes_cnt = cnt; link->cookies = cookies; link->fexits = fexits; + link->tgt_progs = progs; + link->pokes = pokes; - err = bpf_trampoline_multi_attach(prog, ids, link); + if (progs) + err = bpf_trampoline_multi_prog_attach(prog, ids, keys, link); + else + err = bpf_trampoline_multi_attach(prog, ids, link); kvfree(ids); + kvfree(keys); + kvfree(fds); if (err) { + bpf_put_progs(progs, cnt); bpf_link_cleanup(&link_primer); return err; } return bpf_link_settle(&link_primer); error: + bpf_put_progs(progs, cnt); kvfree(fexits); kvfree(cookies); kvfree(ids); + kvfree(keys); + kvfree(fds); + kvfree(progs); + kvfree(pokes); kvfree(link); return err; } diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 6f70db7515dd..94302a3b680c 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -1927,6 +1927,8 @@ union bpf_attr { __aligned_u64 ids; __aligned_u64 cookies; __u32 cnt; + __u32 :32; + __aligned_u64 fds; } tracing_multi; }; } link_create; -- 2.55.0 By design, a tracing_multi link does not support both kernel functions and bpf progs at the same time. In libbpf, the bpf_tracing_multi_opts options provides two new fields for tracing bpf progs: * 'fds': the fd array of the target bpf progs. One fd must be provided multiple times for multiple targets. * 'funcs': the function name array of the target bpf progs. The 'cnt' field indicates the size of both 'fds' and 'funcs' arrays. The 'funcs' will be translated to 'ids' by 'libbpf_find_prog_btf_id()'. At last, 'fds'+'ids' will be passed to kernel. Signed-off-by: Leon Hwang --- tools/lib/bpf/bpf.c | 1 + tools/lib/bpf/bpf.h | 2 ++ tools/lib/bpf/libbpf.c | 41 +++++++++++++++++++++++++++++++++++++---- tools/lib/bpf/libbpf.h | 4 +++- 4 files changed, 43 insertions(+), 5 deletions(-) diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index b49822d212ae..4a78ccba9df2 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c @@ -852,6 +852,7 @@ int bpf_link_create(int prog_fd, int target_fd, attr.link_create.tracing_multi.ids = ptr_to_u64(OPTS_GET(opts, tracing_multi.ids, 0)); attr.link_create.tracing_multi.cookies = ptr_to_u64(OPTS_GET(opts, tracing_multi.cookies, 0)); attr.link_create.tracing_multi.cnt = OPTS_GET(opts, tracing_multi.cnt, 0); + attr.link_create.tracing_multi.fds = ptr_to_u64(OPTS_GET(opts, tracing_multi.fds, 0)); if (!OPTS_ZEROED(opts, tracing_multi)) return libbpf_err(-EINVAL); break; diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h index 826d9cc9ab65..cc37cf622dd8 100644 --- a/tools/lib/bpf/bpf.h +++ b/tools/lib/bpf/bpf.h @@ -474,6 +474,8 @@ struct bpf_link_create_opts { const __u32 *ids; const __u64 *cookies; __u32 cnt; + __u32 :32; + const int *fds; } tracing_multi; }; size_t :0; diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index cb09ded90773..edc5e8e813f3 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -13671,10 +13671,12 @@ bpf_program__attach_tracing_multi(const struct bpf_program *prog, const char *pa { LIBBPF_OPTS(bpf_link_create_opts, lopts); int prog_fd, link_fd, err, cnt; + struct bpf_link *link = NULL; __u32 *free_ids = NULL; - struct bpf_link *link; const __u64 *cookies; + const char **funcs; const __u32 *ids; + const int *fds; if (!OPTS_VALID(opts, bpf_tracing_multi_opts)) return libbpf_err_ptr(-EINVAL); @@ -13689,12 +13691,22 @@ bpf_program__attach_tracing_multi(const struct bpf_program *prog, const char *pa cnt = OPTS_GET(opts, cnt, 0); ids = OPTS_GET(opts, ids, NULL); cookies = OPTS_GET(opts, cookies, NULL); + fds = OPTS_GET(opts, fds, NULL); + funcs = OPTS_GET(opts, funcs, NULL); - if (!!ids != !!cnt) + if (ids && fds) + return libbpf_err_ptr(-EINVAL); + if (!!fds != !!funcs) + return libbpf_err_ptr(-EINVAL); + if (fds && !cnt) return libbpf_err_ptr(-EINVAL); - if (pattern && (ids || cookies)) + if (ids && !cnt) return libbpf_err_ptr(-EINVAL); - if (!pattern && !ids) + if (!fds && !ids && cnt) + return libbpf_err_ptr(-EINVAL); + if (pattern && (fds || ids || cookies)) + return libbpf_err_ptr(-EINVAL); + if (!pattern && !ids && !fds) return libbpf_err_ptr(-EINVAL); if (pattern) { @@ -13704,11 +13716,32 @@ bpf_program__attach_tracing_multi(const struct bpf_program *prog, const char *pa if (cnt == 0) return libbpf_err_ptr(-EINVAL); ids = (const __u32 *) free_ids; + } else if (fds) { + size_t cap = 0; + int i; + + err = libbpf_ensure_mem((void **) &free_ids, &cap, sizeof(*free_ids), cnt); + if (err) + return libbpf_err_ptr(err); + + for (i = 0; i < cnt; i++) { + if (!funcs[i]) { + err = -EINVAL; + goto error; + } + err = libbpf_find_prog_btf_id(funcs[i], fds[i], prog->obj->token_fd); + if (err < 0) + goto error; + + free_ids[i] = err; + } + ids = (const __u32 *) free_ids; } lopts.tracing_multi.ids = ids; lopts.tracing_multi.cookies = cookies; lopts.tracing_multi.cnt = cnt; + lopts.tracing_multi.fds = fds; link = calloc(1, sizeof(*link)); if (!link) { diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h index 03f794f4adb0..612f3e17ebc2 100644 --- a/tools/lib/bpf/libbpf.h +++ b/tools/lib/bpf/libbpf.h @@ -746,10 +746,12 @@ struct bpf_tracing_multi_opts { const __u32 *ids; const __u64 *cookies; size_t cnt; + const int *fds; + const char **funcs; size_t :0; }; -#define bpf_tracing_multi_opts__last_field cnt +#define bpf_tracing_multi_opts__last_field funcs LIBBPF_API struct bpf_link * bpf_program__attach_tracing_multi(const struct bpf_program *prog, const char *pattern, -- 2.55.0 Show the target bpf prog info in the fdinfo. $ cat /proc/self/fdinfo/13 pos: 0 flags: 02000000 mnt_id: 18 ino: 1045 link_type: tracing_multi link_id: 9 prog_tag: 9a7f9e3ab6f138ab prog_id: 25 attach_type: 61 cnt: 2 prog-id btf-id cookie func 21 4 16706819 target_1 22 5 16706819 target_2 Different from the fdinfo of link whose targets are kernel functions, here shows 'prog-id' instead of 'obj-id' to describe the ID of the target bpf progs. And, 'func' shows the concrete function name of the target bpf progs, because the function name can be retrieved by bpftool via 'prog-id' + 'btf-id'. Signed-off-by: Leon Hwang --- kernel/trace/bpf_trace.c | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index d4af42e25fe5..8b1a52350cfd 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -3761,26 +3761,45 @@ static int bpf_tracing_multi_link_fill_link_info(const struct bpf_link *link, } #ifdef CONFIG_PROC_FS +static const char *bpf_prog_func_name(struct bpf_prog *prog, u32 btf_id) +{ + const struct btf *btf = prog->aux->btf; + const struct btf_type *t; + + t = btf_type_by_id(btf, btf_id); + if (!t || !btf_type_is_func(t)) + return ""; + + return btf_name_by_offset(btf, t->name_off); +} + static void bpf_tracing_multi_show_fdinfo(const struct bpf_link *link, struct seq_file *seq) { struct bpf_tracing_multi_link *tr_link = container_of(link, struct bpf_tracing_multi_link, link); bool has_cookies = !!tr_link->cookies; + bool has_progs = !!tr_link->tgt_progs; seq_printf(seq, "attach_type:\t%u\n", tr_link->link.attach_type); seq_printf(seq, "cnt:\t%u\n", tr_link->nodes_cnt); - seq_printf(seq, "%s\t %s\t %s\t %s\n", "obj-id", "btf-id", "cookie", "func"); + seq_printf(seq, "%s\t %s\t %s\t %s\n", has_progs ? "prog-id" : "obj-id", "btf-id", "cookie", + "func"); for (int i = 0; i < tr_link->nodes_cnt; i++) { struct bpf_tracing_multi_node *mnode = &tr_link->nodes[i]; + u64 cookie = has_cookies ? tr_link->cookies[i] : 0; u32 btf_id, obj_id; bpf_trampoline_unpack_key(mnode->trampoline->key, &obj_id, &btf_id); - seq_printf(seq, "%u\t %u\t %llu\t %pS\n", - obj_id, btf_id, - has_cookies ? tr_link->cookies[i] : 0, - (void *) mnode->trampoline->ip); + if (has_progs) + seq_printf(seq, "%u\t %u\t %llu\t %s\n", + obj_id, btf_id, cookie, + bpf_prog_func_name(tr_link->tgt_progs[i], btf_id)); + else + seq_printf(seq, "%u\t %u\t %llu\t %pS\n", + obj_id, btf_id, cookie, + (void *) mnode->trampoline->ip); cond_resched(); } -- 2.55.0 Show bpf prog info in the link info. $ bpftool link 9: tracing_multi prog 25 attach_type trace_fsession_multi btf_obj_id 1 count 2 prog_id func_btf_id addr cookie func [module] 21 4 ffffffffc080064c 16706819 target_1 [bpf] 22 5 ffffffffc08006d4 16706819 target_2 [bpf] pids test_progs(98) Signed-off-by: Leon Hwang --- include/uapi/linux/bpf.h | 7 +++- kernel/trace/bpf_trace.c | 26 ++++++++++---- tools/bpf/bpftool/link.c | 63 +++++++++++++++++++++++++++------- tools/include/uapi/linux/bpf.h | 7 +++- 4 files changed, 82 insertions(+), 21 deletions(-) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 94302a3b680c..df66206af3cf 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -6891,6 +6891,10 @@ struct bpf_btf_info { __u32 kernel_btf; } __attribute__((aligned(8))); +enum { + BPF_F_TRACING_MULTI_TGT_PROGS = (1U << 0), +}; + struct bpf_link_info { __u32 type; __u32 id; @@ -6976,10 +6980,11 @@ struct bpf_link_info { __u32 attach_type; __u32 count; /* in/out: tracing_multi target count */ __u32 btf_obj_id; - __u32 :32; + __u32 flags; __aligned_u64 ids; __aligned_u64 addrs; __aligned_u64 cookies; + __aligned_u64 func_btf_ids; } tracing_multi; struct { __u32 type; /* enum bpf_perf_event_type */ diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 8b1a52350cfd..39c911917ae2 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -3709,15 +3709,16 @@ static void bpf_tracing_multi_link_dealloc(struct bpf_link *link) static int bpf_tracing_multi_link_fill_link_info(const struct bpf_link *link, struct bpf_link_info *info) { + u32 __user *fids = u64_to_user_ptr(info->tracing_multi.func_btf_ids); u64 __user *ucookies = u64_to_user_ptr(info->tracing_multi.cookies); u64 __user *uaddrs = u64_to_user_ptr(info->tracing_multi.addrs); u32 __user *uids = u64_to_user_ptr(info->tracing_multi.ids); struct bpf_tracing_multi_link *tr_link; u32 ucount = info->tracing_multi.count; - bool has_cookies, show_addrs; + bool has_cookies, show_addrs, has_progs; int err = 0; - if ((uids || ucookies || uaddrs) && !ucount) + if ((uids || ucookies || uaddrs || fids) && !ucount) return -EINVAL; tr_link = container_of(link, struct bpf_tracing_multi_link, link); @@ -3725,9 +3726,13 @@ static int bpf_tracing_multi_link_fill_link_info(const struct bpf_link *link, info->tracing_multi.attach_type = tr_link->link.attach_type; info->tracing_multi.count = tr_link->nodes_cnt; info->tracing_multi.btf_obj_id = btf_obj_id(tr_link->link.prog->aux->attach_btf); + has_progs = !!tr_link->tgt_progs; + info->tracing_multi.flags = has_progs ? BPF_F_TRACING_MULTI_TGT_PROGS : 0; - if (!uids && !ucookies && !uaddrs) + if (!uids && !ucookies && !uaddrs && !fids) return 0; + if (has_progs != !!fids) + return -EINVAL; if (ucount < tr_link->nodes_cnt) err = -ENOSPC; @@ -3739,20 +3744,27 @@ static int bpf_tracing_multi_link_fill_link_info(const struct bpf_link *link, for (int i = 0; i < ucount; i++) { struct bpf_tracing_multi_node *mnode = &tr_link->nodes[i]; + u32 prog_id, btf_id; u64 addr, cookie; - u32 id; - bpf_trampoline_unpack_key(mnode->trampoline->key, NULL, &id); + bpf_trampoline_unpack_key(mnode->trampoline->key, &prog_id, &btf_id); - addr = show_addrs ? mnode->trampoline->ip : 0; + if (!show_addrs) + addr = 0; + else if (has_progs) + addr = (u64) mnode->trampoline->func.addr; + else + addr = mnode->trampoline->ip; cookie = has_cookies ? tr_link->cookies[i] : 0; - if (uids && put_user(id, uids + i)) + if (uids && put_user(has_progs ? prog_id : btf_id, uids + i)) return -EFAULT; if (uaddrs && put_user(addr, uaddrs + i)) return -EFAULT; if (ucookies && put_user(cookie, ucookies + i)) return -EFAULT; + if (has_progs && put_user((u32) mnode->trampoline->key, fids + i)) + return -EFAULT; cond_resched(); } diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c index 5cffa2d283b4..da7e430b14c8 100644 --- a/tools/bpf/bpftool/link.c +++ b/tools/bpf/bpftool/link.c @@ -422,12 +422,31 @@ show_uprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr) jsonw_end_array(json_wtr); } +#define BPF_PROG_PREFIX "bpf_prog_" +#define BPF_PROG_PREFIX_LEN (sizeof(BPF_PROG_PREFIX) - 1) + +static const char * +sym_name_trim_prefix(struct kernel_sym *sym, bool is_prog) +{ + int prefix_len; + + if (!is_prog) + return sym->name; + + /* Ref kernel/bpf/core.c::bpf_prog_ksym_set_name() */ + prefix_len = BPF_PROG_PREFIX_LEN; + prefix_len += BPF_TAG_SIZE * 2; + if (sym->name[prefix_len] != '\0') + prefix_len += 1; /* skip the '_' */ + return sym->name + prefix_len; +} + static void show_tracing_multi_json(struct bpf_link_info *info, json_writer_t *wtr) { - bool is_ibt_enabled = is_x86_ibt_enabled(), show_symbol; + bool is_ibt_enabled = is_x86_ibt_enabled(), show_symbol, tgt_progs; __u64 *addrs, *cookies; - __u32 i, *ids; + __u32 i, *ids, *fids; if (!dd.sym_count) kernel_syms_load(&dd); @@ -443,6 +462,8 @@ show_tracing_multi_json(struct bpf_link_info *info, json_writer_t *wtr) ids = u64_to_u32_arr(info->tracing_multi.ids); addrs = u64_to_arr(info->tracing_multi.addrs); cookies = u64_to_arr(info->tracing_multi.cookies); + fids = u64_to_u32_arr(info->tracing_multi.func_btf_ids); + tgt_progs = info->tracing_multi.flags & BPF_F_TRACING_MULTI_TGT_PROGS; for (i = 0; i < info->tracing_multi.count; i++) { struct kernel_sym *sym; @@ -451,10 +472,12 @@ show_tracing_multi_json(struct bpf_link_info *info, json_writer_t *wtr) sym = show_symbol ? find_kernel_sym_by_addr(addr, is_ibt_enabled) : NULL; jsonw_start_object(wtr); - jsonw_uint_field(wtr, "id", ids[i]); + jsonw_uint_field(wtr, tgt_progs ? "prog_id" : "id", ids[i]); + if (tgt_progs) + jsonw_uint_field(wtr, "func_btf_id", fids[i]); jsonw_uint_field(wtr, "addr", addr); if (sym) { - jsonw_string_field(wtr, "func", sym->name); + jsonw_string_field(wtr, "func", sym_name_trim_prefix(sym, tgt_progs)); if (sym->module[0] == '\0') { jsonw_name(wtr, "module"); jsonw_null(wtr); @@ -903,9 +926,9 @@ static void show_uprobe_multi_plain(struct bpf_link_info *info) static void show_tracing_multi_plain(struct bpf_link_info *info) { - bool is_ibt_enabled = is_x86_ibt_enabled(), show_symbol; + bool is_ibt_enabled = is_x86_ibt_enabled(), show_symbol, tgt_progs; __u64 *addrs, *cookies; - __u32 i, *ids; + __u32 i, *ids, *fids; if (!info->tracing_multi.count) return; @@ -919,12 +942,18 @@ static void show_tracing_multi_plain(struct bpf_link_info *info) printf("btf_obj_id %u ", info->tracing_multi.btf_obj_id); printf("count %u ", info->tracing_multi.count); - printf("\n\t%-16s %-16s %-16s %s", - "btf_id", "addr", "cookie", "func [module]"); + tgt_progs = info->tracing_multi.flags & BPF_F_TRACING_MULTI_TGT_PROGS; + if (tgt_progs) + printf("\n\t%-16s %-16s %-16s %-16s %s", + "prog_id", "func_btf_id", "addr", "cookie", "func [module]"); + else + printf("\n\t%-16s %-16s %-16s %s", + "btf_id", "addr", "cookie", "func [module]"); ids = u64_to_u32_arr(info->tracing_multi.ids); addrs = u64_to_arr(info->tracing_multi.addrs); cookies = u64_to_arr(info->tracing_multi.cookies); + fids = u64_to_u32_arr(info->tracing_multi.func_btf_ids); for (i = 0; i < info->tracing_multi.count; i++) { __u64 addr = addrs[i]; @@ -932,9 +961,13 @@ static void show_tracing_multi_plain(struct bpf_link_info *info) sym = show_symbol ? find_kernel_sym_by_addr(addr, is_ibt_enabled) : NULL; - printf("\n\t%-16u %016llx %-16llu", ids[i], addr, cookies[i]); + if (tgt_progs) + printf("\n\t%-16u %-16u %016llx %-16llu", + ids[i], fids[i], addr, cookies[i]); + else + printf("\n\t%-16u %016llx %-16llu", ids[i], addr, cookies[i]); if (sym) { - printf(" %s", sym->name); + printf(" %s", sym_name_trim_prefix(sym, tgt_progs)); if (sym->module[0] != '\0') printf(" [%s]", sym->module); } @@ -1140,7 +1173,7 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info) static int do_show_link(int fd) { __u64 *ref_ctr_offsets = NULL, *offsets = NULL, *cookies = NULL; - __u32 *ids = NULL; + __u32 *ids = NULL, *func_btf_ids = NULL; struct bpf_link_info info; __u32 len = sizeof(info); char path_buf[PATH_MAX]; @@ -1229,12 +1262,16 @@ static int do_show_link(int fd) if (info.type == BPF_LINK_TYPE_TRACING_MULTI && !info.tracing_multi.ids) { count = info.tracing_multi.count; if (count) { + bool tgt_progs = info.tracing_multi.flags & BPF_F_TRACING_MULTI_TGT_PROGS; + ids = calloc(count, sizeof(__u32)); addrs = calloc(count, sizeof(__u64)); cookies = calloc(count, sizeof(__u64)); - if (!ids || !addrs || !cookies) { + func_btf_ids = tgt_progs ? calloc(count, sizeof(__u32)) : NULL; + if (!ids || !addrs || !cookies || (tgt_progs && !func_btf_ids)) { p_err("mem alloc failed"); close(fd); + free(func_btf_ids); free(cookies); free(addrs); free(ids); @@ -1243,6 +1280,7 @@ static int do_show_link(int fd) info.tracing_multi.ids = ptr_to_u64(ids); info.tracing_multi.addrs = ptr_to_u64(addrs); info.tracing_multi.cookies = ptr_to_u64(cookies); + info.tracing_multi.func_btf_ids = ptr_to_u64(func_btf_ids); goto again; } } @@ -1282,6 +1320,7 @@ static int do_show_link(int fd) show_link_close_plain(fd, &info); free(ref_ctr_offsets); + free(func_btf_ids); free(cookies); free(offsets); free(addrs); diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 94302a3b680c..60998ea1381c 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -6817,6 +6817,10 @@ struct sk_reuseport_md { #define BPF_TAG_SIZE 8 +enum { + BPF_F_TRACING_MULTI_TGT_PROGS = (1U << 0), +}; + struct bpf_prog_info { __u32 type; __u32 id; @@ -6976,10 +6980,11 @@ struct bpf_link_info { __u32 attach_type; __u32 count; /* in/out: tracing_multi target count */ __u32 btf_obj_id; - __u32 :32; + __u32 flags; __aligned_u64 ids; __aligned_u64 addrs; __aligned_u64 cookies; + __aligned_u64 func_btf_ids; } tracing_multi; struct { __u32 type; /* enum bpf_perf_event_type */ -- 2.55.0 Attach to two bpf progs with fentry.multi, fexit.multi, and fsession.multi progs. Assisted-by: Codex:gpt-5.5 Signed-off-by: Leon Hwang --- .../selftests/bpf/prog_tests/tracing_multi.c | 74 +++++++++++++++++++ .../selftests/bpf/progs/tracing_multi_bpf.c | 52 +++++++++++++ 2 files changed, 126 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/tracing_multi_bpf.c diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_multi.c b/tools/testing/selftests/bpf/prog_tests/tracing_multi.c index cc727011f3a2..801136ca63f0 100644 --- a/tools/testing/selftests/bpf/prog_tests/tracing_multi.c +++ b/tools/testing/selftests/bpf/prog_tests/tracing_multi.c @@ -12,6 +12,7 @@ #include "tracing_multi_verifier.skel.h" #include "tracing_multi_bench.skel.h" #include "tracing_multi_rollback.skel.h" +#include "tracing_multi_bpf.skel.h" #include "trace_helpers.h" static __u64 bpf_fentry_test_cookies[] = { @@ -1126,3 +1127,76 @@ void test_tracing_multi_test(void) if (test__start_subtest("fentry_after_multi")) test_fentry_after_multi(); } + +static int run_bpf_target(struct bpf_program *prog, __u32 retval, const char *name) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + int err; + + err = bpf_prog_test_run_opts(bpf_program__fd(prog), &topts); + if (!ASSERT_OK(err, name)) + return err; + + if (!ASSERT_EQ(topts.retval, retval, name)) + return -EINVAL; + + return 0; +} + +static void test_link_api_bpf_prog(void) +{ + const char *funcs[] = { "target_1", "target_2" }; + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + struct tracing_multi_bpf *skel = NULL; + int fds[ARRAY_SIZE(funcs)]; + struct bpf_link *link; + + skel = tracing_multi_bpf__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_bpf__open_and_load")) + return; + + fds[0] = bpf_program__fd(skel->progs.target_1); + fds[1] = bpf_program__fd(skel->progs.target_2); + + opts.fds = fds; + opts.funcs = funcs; + opts.cnt = ARRAY_SIZE(fds); + + link = bpf_program__attach_tracing_multi(skel->progs.test_fentry, NULL, &opts); + if (!ASSERT_OK_PTR(link, "attach_fentry")) + goto cleanup; + skel->links.test_fentry = link; + + link = bpf_program__attach_tracing_multi(skel->progs.test_fexit, NULL, &opts); + if (!ASSERT_OK_PTR(link, "attach_fexit")) + goto cleanup; + skel->links.test_fexit = link; + + link = bpf_program__attach_tracing_multi(skel->progs.test_fsession, NULL, &opts); + if (!ASSERT_OK_PTR(link, "attach_fsession")) + goto cleanup; + skel->links.test_fsession = link; + + if (run_bpf_target(skel->progs.target_1, 1, "target_1")) + goto cleanup; + if (run_bpf_target(skel->progs.target_2, 2, "target_2")) + goto cleanup; + + ASSERT_EQ(skel->bss->target_1_result, 1, "target_1_result"); + ASSERT_EQ(skel->bss->target_2_result, 1, "target_2_result"); + ASSERT_EQ(skel->bss->test_result_fentry, ARRAY_SIZE(funcs), "test_result_fentry"); + ASSERT_EQ(skel->bss->test_result_fexit, ARRAY_SIZE(funcs), "test_result_fexit"); + ASSERT_EQ(skel->bss->test_result_fsession_entry, ARRAY_SIZE(funcs), + "test_result_fsession_entry"); + ASSERT_EQ(skel->bss->test_result_fsession_exit, ARRAY_SIZE(funcs), + "test_result_fsession_exit"); + +cleanup: + tracing_multi_bpf__destroy(skel); +} + +void test_tracing_multi_prog_test(void) +{ + if (test__start_subtest("link_api")) + test_link_api_bpf_prog(); +} diff --git a/tools/testing/selftests/bpf/progs/tracing_multi_bpf.c b/tools/testing/selftests/bpf/progs/tracing_multi_bpf.c new file mode 100644 index 000000000000..3e02602558ad --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_multi_bpf.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include + +char _license[] SEC("license") = "GPL"; + +__u64 test_result_fentry; +__u64 test_result_fexit; +__u64 test_result_fsession_entry; +__u64 test_result_fsession_exit; +__u64 target_1_result; +__u64 target_2_result; + +SEC("syscall") +int target_1(void *ctx) +{ + target_1_result++; + return 1; +} + +SEC("syscall") +int target_2(void *ctx) +{ + target_2_result++; + return 2; +} + +SEC("fentry.multi") +int BPF_PROG(test_fentry) +{ + test_result_fentry++; + return 0; +} + +SEC("fexit.multi") +int BPF_PROG(test_fexit) +{ + test_result_fexit++; + return 0; +} + +SEC("fsession.multi") +int BPF_PROG(test_fsession) +{ + if (bpf_session_is_return(ctx)) + test_result_fsession_exit++; + else + test_result_fsession_entry++; + + return 0; +} -- 2.55.0 Tests against two libbpf failures and two kernel failures: * libbpf: 'ids' + 'fds'. * libbpf: 'pattern' + 'fds'. * kernel: 'fds' + invalid 'ids'. * kernel: duplicate targets via different fd for the same bpf prog + func. Assisted-by: Codex:gpt-5.5 Signed-off-by: Leon Hwang --- tools/lib/bpf/libbpf.c | 2 +- tools/lib/bpf/libbpf_internal.h | 1 + .../selftests/bpf/prog_tests/tracing_multi.c | 100 ++++++++++++++++++ 3 files changed, 102 insertions(+), 1 deletion(-) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index edc5e8e813f3..72e2459e9339 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -11604,7 +11604,7 @@ int libbpf_find_vmlinux_btf_id(const char *name, return libbpf_err(err); } -static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd, int token_fd) +int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd, int token_fd) { struct bpf_prog_info info; __u32 info_len = sizeof(info); diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index 4e45dfacb548..d4e899181bdd 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h @@ -439,6 +439,7 @@ int btf_load_into_kernel(struct btf *btf, char *log_buf, size_t log_sz, __u32 log_level, int token_fd); struct btf *btf_load_from_kernel(__u32 id, struct btf *base_btf, int token_fd); +int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd, int token_fd); struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf); void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type, diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_multi.c b/tools/testing/selftests/bpf/prog_tests/tracing_multi.c index 801136ca63f0..d428b5e2600c 100644 --- a/tools/testing/selftests/bpf/prog_tests/tracing_multi.c +++ b/tools/testing/selftests/bpf/prog_tests/tracing_multi.c @@ -1195,8 +1195,108 @@ static void test_link_api_bpf_prog(void) tracing_multi_bpf__destroy(skel); } +static void test_attach_api_bpf_prog_fails(void) +{ + static const char * const kfuncs[] = { + "bpf_fentry_test1", + }; + const char *bpf_funcs[] = { "target_1" }; + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + LIBBPF_OPTS(bpf_link_create_opts, lopts); + struct tracing_multi_bpf *skel = NULL; + int fd, target_id, token_fd; + int fds[2], dup_fd = -1; + struct bpf_link *link; + __u32 target_ids[2]; + __u32 *ids = NULL; + long err; + + skel = tracing_multi_bpf__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_bpf__open_and_load")) + return; + + ids = get_ids(kfuncs, ARRAY_SIZE(kfuncs), NULL); + if (!ASSERT_OK_PTR(ids, "get_ids")) + goto cleanup; + + fds[0] = bpf_program__fd(skel->progs.target_1); + + /* fail#1 (libbpf) kernel ids and BPF prog fds */ + LIBBPF_OPTS_RESET(opts, + .ids = ids, + .fds = fds, + .funcs = bpf_funcs, + .cnt = ARRAY_SIZE(bpf_funcs), + ); + + link = bpf_program__attach_tracing_multi(skel->progs.test_fentry, NULL, &opts); + err = libbpf_get_error(link); + if (!ASSERT_EQ(err, -EINVAL, "libbpf_ids_fds")) { + if (!err) + bpf_link__destroy(link); + goto cleanup; + } + + /* fail#2 (libbpf) kernel pattern and BPF prog fds */ + LIBBPF_OPTS_RESET(opts, + .fds = fds, + .funcs = bpf_funcs, + .cnt = ARRAY_SIZE(bpf_funcs), + ); + + link = bpf_program__attach_tracing_multi(skel->progs.test_fentry, + "bpf_fentry_test*", &opts); + err = libbpf_get_error(link); + if (!ASSERT_EQ(err, -EINVAL, "libbpf_pattern_fds")) { + if (!err) + bpf_link__destroy(link); + goto cleanup; + } + + /* fail#3 (kernel) kernel id with BPF prog fd */ + lopts.tracing_multi.ids = ids; + lopts.tracing_multi.fds = fds; + lopts.tracing_multi.cnt = ARRAY_SIZE(bpf_funcs); + fd = bpf_link_create(bpf_program__fd(skel->progs.test_fentry), 0, + BPF_TRACE_FENTRY_MULTI, &lopts); + ASSERT_EQ(fd, -EINVAL, "kernel_id_bpf_fd"); + if (fd >= 0) + close(fd); + + /* fail#4 (kernel) duplicate target through different fds */ + token_fd = bpf_object__token_fd(skel->obj); + token_fd = token_fd < 0 ? 0 : token_fd; + target_id = libbpf_find_prog_btf_id("target_1", fds[0], token_fd); + if (!ASSERT_GT(target_id, 0, "target_1_btf_id")) + goto cleanup; + + dup_fd = dup(fds[0]); + if (!ASSERT_GE(dup_fd, 0, "dup_target_fd")) + goto cleanup; + + fds[1] = dup_fd; + target_ids[0] = target_id; + target_ids[1] = target_id; + lopts.tracing_multi.ids = target_ids; + lopts.tracing_multi.fds = fds; + lopts.tracing_multi.cnt = ARRAY_SIZE(fds); + fd = bpf_link_create(bpf_program__fd(skel->progs.test_fentry), 0, + BPF_TRACE_FENTRY_MULTI, &lopts); + ASSERT_EQ(fd, -EINVAL, "duplicate_bpf_target"); + if (fd >= 0) + close(fd); + +cleanup: + if (dup_fd >= 0) + close(dup_fd); + tracing_multi_bpf__destroy(skel); + free(ids); +} + void test_tracing_multi_prog_test(void) { if (test__start_subtest("link_api")) test_link_api_bpf_prog(); + if (test__start_subtest("attach_api_fails")) + test_attach_api_bpf_prog_fails(); } -- 2.55.0 Verify that the cookie also works for bpf prog targets of tracing_multi link. Assisted-by: Codex:gpt-5.5 Signed-off-by: Leon Hwang --- .../selftests/bpf/prog_tests/tracing_multi.c | 21 +++++++++++++++++-- .../selftests/bpf/progs/tracing_multi_bpf.c | 13 ++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_multi.c b/tools/testing/selftests/bpf/prog_tests/tracing_multi.c index d428b5e2600c..bcf933760bc5 100644 --- a/tools/testing/selftests/bpf/prog_tests/tracing_multi.c +++ b/tools/testing/selftests/bpf/prog_tests/tracing_multi.c @@ -1143,9 +1143,12 @@ static int run_bpf_target(struct bpf_program *prog, __u32 retval, const char *na return 0; } -static void test_link_api_bpf_prog(void) +static void test_link_api_bpf_prog(bool test_cookies) { const char *funcs[] = { "target_1", "target_2" }; + __u64 fentry_cookies[] = { 0xfeed01, 0xfeed02 }; + __u64 fexit_cookies[] = { 0xfeed11, 0xfeed12 }; + __u64 fsession_cookies[] = { 0xfeed21, 0xfeed22 }; LIBBPF_OPTS(bpf_tracing_multi_opts, opts); struct tracing_multi_bpf *skel = NULL; int fds[ARRAY_SIZE(funcs)]; @@ -1158,27 +1161,39 @@ static void test_link_api_bpf_prog(void) fds[0] = bpf_program__fd(skel->progs.target_1); fds[1] = bpf_program__fd(skel->progs.target_2); + skel->bss->test_cookies = test_cookies; + opts.fds = fds; opts.funcs = funcs; opts.cnt = ARRAY_SIZE(fds); + opts.cookies = test_cookies ? fentry_cookies : NULL; link = bpf_program__attach_tracing_multi(skel->progs.test_fentry, NULL, &opts); if (!ASSERT_OK_PTR(link, "attach_fentry")) goto cleanup; skel->links.test_fentry = link; + opts.cookies = test_cookies ? fexit_cookies : NULL; link = bpf_program__attach_tracing_multi(skel->progs.test_fexit, NULL, &opts); if (!ASSERT_OK_PTR(link, "attach_fexit")) goto cleanup; skel->links.test_fexit = link; + opts.cookies = test_cookies ? fsession_cookies : NULL; link = bpf_program__attach_tracing_multi(skel->progs.test_fsession, NULL, &opts); if (!ASSERT_OK_PTR(link, "attach_fsession")) goto cleanup; skel->links.test_fsession = link; + skel->bss->fentry_cookie = fentry_cookies[0]; + skel->bss->fexit_cookie = fexit_cookies[0]; + skel->bss->fsession_cookie = fsession_cookies[0]; if (run_bpf_target(skel->progs.target_1, 1, "target_1")) goto cleanup; + + skel->bss->fentry_cookie = fentry_cookies[1]; + skel->bss->fexit_cookie = fexit_cookies[1]; + skel->bss->fsession_cookie = fsession_cookies[1]; if (run_bpf_target(skel->progs.target_2, 2, "target_2")) goto cleanup; @@ -1296,7 +1311,9 @@ static void test_attach_api_bpf_prog_fails(void) void test_tracing_multi_prog_test(void) { if (test__start_subtest("link_api")) - test_link_api_bpf_prog(); + test_link_api_bpf_prog(false); if (test__start_subtest("attach_api_fails")) test_attach_api_bpf_prog_fails(); + if (test__start_subtest("cookies")) + test_link_api_bpf_prog(true); } diff --git a/tools/testing/selftests/bpf/progs/tracing_multi_bpf.c b/tools/testing/selftests/bpf/progs/tracing_multi_bpf.c index 3e02602558ad..651b5d86203f 100644 --- a/tools/testing/selftests/bpf/progs/tracing_multi_bpf.c +++ b/tools/testing/selftests/bpf/progs/tracing_multi_bpf.c @@ -5,6 +5,10 @@ char _license[] SEC("license") = "GPL"; +bool test_cookies; +__u64 fentry_cookie; +__u64 fexit_cookie; +__u64 fsession_cookie; __u64 test_result_fentry; __u64 test_result_fexit; __u64 test_result_fsession_entry; @@ -29,20 +33,25 @@ int target_2(void *ctx) SEC("fentry.multi") int BPF_PROG(test_fentry) { - test_result_fentry++; + if (!test_cookies || bpf_get_attach_cookie(ctx) == fentry_cookie) + test_result_fentry++; return 0; } SEC("fexit.multi") int BPF_PROG(test_fexit) { - test_result_fexit++; + if (!test_cookies || bpf_get_attach_cookie(ctx) == fexit_cookie) + test_result_fexit++; return 0; } SEC("fsession.multi") int BPF_PROG(test_fsession) { + if (test_cookies && bpf_get_attach_cookie(ctx) != fsession_cookie) + return 0; + if (bpf_session_is_return(ctx)) test_result_fsession_exit++; else -- 2.55.0 When rollbacking, the prepared trampolines should be reverted. In this test, the first bpf prog will be attached with the trampoline, and the second bpf prog won't be attached because the bpf prog has been attached by a fentry.multi prog, that causes the rollback procedure. Assisted-by: Codex:gpt-5.5 Signed-off-by: Leon Hwang --- .../selftests/bpf/prog_tests/tracing_multi.c | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_multi.c b/tools/testing/selftests/bpf/prog_tests/tracing_multi.c index bcf933760bc5..dfa24e229ff7 100644 --- a/tools/testing/selftests/bpf/prog_tests/tracing_multi.c +++ b/tools/testing/selftests/bpf/prog_tests/tracing_multi.c @@ -1308,6 +1308,67 @@ static void test_attach_api_bpf_prog_fails(void) free(ids); } +static void test_rollback_bpf_prog(void) +{ + int token_fd, prog_fd, fd = -1, pre_fd = -1; + LIBBPF_OPTS(bpf_link_create_opts, lopts); + struct tracing_multi_bpf *skel = NULL; + __u32 target_ids[2], ids[2]; + int target_fds[2], fds[2]; + struct bpf_program *prog; + int first, second; + + skel = tracing_multi_bpf__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_bpf__open_and_load")) + return; + + token_fd = bpf_object__token_fd(skel->obj); + token_fd = token_fd < 0 ? 0 : token_fd; + target_fds[0] = bpf_program__fd(skel->progs.target_1); + target_fds[1] = bpf_program__fd(skel->progs.target_2); + target_ids[0] = libbpf_find_prog_btf_id("target_1", target_fds[0], token_fd); + if (!ASSERT_GT(target_ids[0], 0, "target_1_btf_id")) + goto cleanup; + target_ids[1] = libbpf_find_prog_btf_id("target_2", target_fds[1], token_fd); + if (!ASSERT_GT(target_ids[1], 0, "target_2_btf_id")) + goto cleanup; + + first = target_fds[0] < target_fds[1] ? 0 : 1; + second = 1 - first; + fds[0] = target_fds[first]; + fds[1] = target_fds[second]; + ids[0] = target_ids[first]; + ids[1] = target_ids[second]; + + prog_fd = bpf_program__fd(skel->progs.test_fentry); + lopts.tracing_multi.ids = &ids[1]; + lopts.tracing_multi.fds = &fds[1]; + lopts.tracing_multi.cnt = 1; + pre_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_FENTRY_MULTI, &lopts); + if (!ASSERT_GE(pre_fd, 0, "pre_attach")) + goto cleanup; + + lopts.tracing_multi.ids = ids; + lopts.tracing_multi.fds = fds; + lopts.tracing_multi.cnt = ARRAY_SIZE(ids); + + fd = bpf_link_create(prog_fd, 0, BPF_TRACE_FENTRY_MULTI, &lopts); + if (!ASSERT_EQ(fd, -EBUSY, "bpf_link_create")) + goto cleanup; + + prog = first == 0 ? skel->progs.target_1 : skel->progs.target_2; + if (run_bpf_target(prog, first + 1, "rollback_target")) + goto cleanup; + ASSERT_EQ(skel->bss->test_result_fentry, 0, "test_result_fentry"); + +cleanup: + if (fd >= 0) + close(fd); + if (pre_fd >= 0) + close(pre_fd); + tracing_multi_bpf__destroy(skel); +} + void test_tracing_multi_prog_test(void) { if (test__start_subtest("link_api")) @@ -1316,4 +1377,6 @@ void test_tracing_multi_prog_test(void) test_attach_api_bpf_prog_fails(); if (test__start_subtest("cookies")) test_link_api_bpf_prog(true); + if (test__start_subtest("rollback")) + test_rollback_bpf_prog(); } -- 2.55.0 Verify the link info works well for bpf prog targets of tracing_multi link. Assisted-by: Codex:gpt-5.5 Signed-off-by: Leon Hwang --- .../selftests/bpf/prog_tests/fill_link_info.c | 169 ++++++++++++++++-- 1 file changed, 151 insertions(+), 18 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/fill_link_info.c b/tools/testing/selftests/bpf/prog_tests/fill_link_info.c index 0918321c8e63..422e94f13409 100644 --- a/tools/testing/selftests/bpf/prog_tests/fill_link_info.c +++ b/tools/testing/selftests/bpf/prog_tests/fill_link_info.c @@ -3,11 +3,13 @@ #include #include +#include #include #include #include #include "trace_helpers.h" #include "test_fill_link_info.skel.h" +#include "tracing_multi_bpf.skel.h" #include "bpf/libbpf_internal.h" #define TP_CAT "sched" @@ -39,8 +41,19 @@ struct tmulti_target { __u64 addr; __u64 cookie; __u32 id; + __u32 func_btf_id; }; +static const char *tmulti_bpf_funcs[] = { + "target_1", + "target_2", +}; + +static __u64 tmulti_bpf_fentry_cookies[] = { 0xfeed01, 0xfeed02 }; +static __u64 tmulti_bpf_fexit_cookies[] = { 0xfeed11, 0xfeed12 }; +static __u64 tmulti_bpf_fsession_cookies[] = { 0xfeed21, 0xfeed22 }; +#define TRACING_MULTI_BPF_CNT ARRAY_SIZE(tmulti_bpf_funcs) + #define UPROBE_FILE "/proc/self/exe" static ssize_t uprobe_offset; /* uprobe attach point */ @@ -421,14 +434,28 @@ static int tmulti_target_cmp(const void *a, const void *b) return (ta->id > tb->id) - (ta->id < tb->id); } +static int setup_tmulti_btf_obj_id(const struct bpf_program *prog, __u32 *btf_obj_id) +{ + struct bpf_prog_info prog_info = {}; + __u32 len = sizeof(prog_info); + int err; + + err = bpf_prog_get_info_by_fd(bpf_program__fd(prog), &prog_info, &len); + if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd")) + return -1; + if (!ASSERT_GT(prog_info.attach_btf_obj_id, 0, "attach_btf_obj_id")) + return -1; + + *btf_obj_id = prog_info.attach_btf_obj_id; + return 0; +} + static int setup_tmulti_targets(const struct bpf_program *prog, struct tmulti_target *targets, __u32 *btf_obj_id) { - struct bpf_prog_info prog_info; - __u32 len = sizeof(prog_info); struct btf *btf; - int err, i; + int i; __s32 id; btf = btf__load_vmlinux_btf(); @@ -446,13 +473,8 @@ static int setup_tmulti_targets(const struct bpf_program *prog, targets[i].id = id; } - memset(&prog_info, 0, len); - err = bpf_prog_get_info_by_fd(bpf_program__fd(prog), &prog_info, &len); - if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd")) - goto error; - if (!ASSERT_GT(prog_info.attach_btf_obj_id, 0, "attach_btf_obj_id")) + if (setup_tmulti_btf_obj_id(prog, btf_obj_id)) goto error; - *btf_obj_id = prog_info.attach_btf_obj_id; /* * The kernel tracing multi attach sorts ids. We sort as well, @@ -467,13 +489,49 @@ static int setup_tmulti_targets(const struct bpf_program *prog, return -1; } +static int setup_tmulti_bpf_prog_targets(const struct bpf_program *prog, int token_fd, + const __u64 *cookies, + const struct bpf_program * const *target_progs, + struct tmulti_target *targets, + __u32 *btf_obj_id) +{ + struct bpf_prog_info prog_info = {}; + __u32 len = sizeof(prog_info), func_btf_id; + int err, i, prog_fd; + + for (i = 0; i < TRACING_MULTI_BPF_CNT; i++) { + memset(&prog_info, 0, sizeof(prog_info)); + len = sizeof(prog_info); + prog_info.jited_ksyms = ptr_to_u64(&targets[i].addr); + prog_info.nr_jited_ksyms = 1; + prog_fd = bpf_program__fd(target_progs[i]); + err = bpf_prog_get_info_by_fd(prog_fd, &prog_info, &len); + if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd")) + return -1; + + func_btf_id = libbpf_find_prog_btf_id(tmulti_bpf_funcs[i], prog_fd, token_fd); + if (!ASSERT_GT(func_btf_id, 0, "libbpf_find_prog_btf_id")) + return -1; + + targets[i].cookie = cookies[i]; + targets[i].id = prog_info.id; + targets[i].func_btf_id = func_btf_id; + } + + qsort(targets, TRACING_MULTI_BPF_CNT, sizeof(targets[0]), tmulti_target_cmp); + return setup_tmulti_btf_obj_id(prog, btf_obj_id); +} + static int verify_tracing_multi_link_info(int fd, const struct bpf_program *prog, const struct tmulti_target *targets, - __u32 btf_obj_id, bool has_cookies) + __u32 btf_obj_id, __u32 count, + bool has_cookies, bool tgt_progs) { +#define TMULTI_CNT (TRACING_MULTI_CNT > TRACING_MULTI_BPF_CNT ? \ + TRACING_MULTI_CNT : TRACING_MULTI_BPF_CNT) enum bpf_attach_type attach_type = bpf_program__expected_attach_type(prog); - __u64 addrs[TRACING_MULTI_CNT], cookies[TRACING_MULTI_CNT]; - __u32 ids[TRACING_MULTI_CNT]; + __u64 addrs[TMULTI_CNT], cookies[TMULTI_CNT]; + __u32 ids[TMULTI_CNT], fids[TMULTI_CNT]; struct bpf_link_info info; __u32 len = sizeof(info); int err, i; @@ -487,16 +545,21 @@ static int verify_tracing_multi_link_info(int fd, const struct bpf_program *prog return -1; ASSERT_EQ(info.tracing_multi.attach_type, attach_type, "info.tracing_multi.attach_type"); - ASSERT_EQ(info.tracing_multi.count, TRACING_MULTI_CNT, "info.tracing_multi.count"); + ASSERT_EQ(info.tracing_multi.count, count, "info.tracing_multi.count"); + if (tgt_progs) + ASSERT_TRUE(info.tracing_multi.flags & BPF_F_TRACING_MULTI_TGT_PROGS, + "info.tracing_multi.flags"); memset(ids, 0, sizeof(ids)); + memset(fids, 0, sizeof(fids)); memset(cookies, 0, sizeof(cookies)); memset(addrs, 0, sizeof(addrs)); info.tracing_multi.ids = ptr_to_u64(ids); info.tracing_multi.addrs = ptr_to_u64(addrs); info.tracing_multi.cookies = has_cookies ? ptr_to_u64(cookies) : 0; - info.tracing_multi.count = TRACING_MULTI_CNT; + info.tracing_multi.func_btf_ids = tgt_progs ? ptr_to_u64(fids) : 0; + info.tracing_multi.count = count; err = bpf_link_get_info_by_fd(fd, &info, &len); if (!ASSERT_OK(err, "bpf_link_get_info_by_fd")) @@ -506,14 +569,20 @@ static int verify_tracing_multi_link_info(int fd, const struct bpf_program *prog return -1; ASSERT_EQ(info.tracing_multi.attach_type, attach_type, "info.tracing_multi.attach_type"); - ASSERT_EQ(info.tracing_multi.count, TRACING_MULTI_CNT, "info.tracing_multi.count"); + ASSERT_EQ(info.tracing_multi.count, count, "info.tracing_multi.count"); ASSERT_EQ(info.tracing_multi.btf_obj_id, btf_obj_id, "tracing_multi.btf_obj_id"); + if (tgt_progs) + ASSERT_TRUE(info.tracing_multi.flags & BPF_F_TRACING_MULTI_TGT_PROGS, + "info.tracing_multi.flags"); - for (i = 0; i < TRACING_MULTI_CNT; i++) { + for (i = 0; i < count; i++) { ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.ids"); ASSERT_EQ(cookies[i], has_cookies ? targets[i].cookie : 0, "tracing_multi.cookies"); - if (targets[i].addr) { + if (tgt_progs) { + ASSERT_EQ(addrs[i], targets[i].addr, "tracing_multi.addrs"); + ASSERT_EQ(fids[i], targets[i].func_btf_id, "tracing_multi.func_btf_ids"); + } else if (targets[i].addr) { struct ksym *ksym; if (!ASSERT_NEQ(addrs[i], 0, "tracing_multi.addrs")) @@ -624,13 +693,73 @@ static void test_tracing_multi_fill_link_info(struct test_fill_link_info *skel, verify_tracing_multi_invalid_user_buffer(link_fd, targets); } else { err = verify_tracing_multi_link_info(link_fd, skel->progs.tmulti_run, - targets, btf_obj_id, has_cookies); + targets, btf_obj_id, TRACING_MULTI_CNT, + has_cookies, false); ASSERT_OK(err, "verify_tracing_multi_link_info"); } bpf_link__destroy(link); } +static void test_tracing_multi_bpf_fill_link_info_one(struct tracing_multi_bpf *skel, + struct bpf_program *prog, + __u64 *cookies, + bool has_cookies) +{ + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + const struct bpf_program *target_progs[] = { + skel->progs.target_1, + skel->progs.target_2, + }; + struct tmulti_target targets[TRACING_MULTI_BPF_CNT] = {}; + int fds[TRACING_MULTI_BPF_CNT], token_fd, err, i; + struct bpf_link *link; + __u32 btf_obj_id; + + token_fd = bpf_object__token_fd(skel->obj); + token_fd = token_fd < 0 ? 0 : token_fd; + + if (setup_tmulti_bpf_prog_targets(prog, token_fd, cookies, target_progs, targets, + &btf_obj_id)) + return; + + for (i = 0; i < TRACING_MULTI_BPF_CNT; i++) + fds[i] = bpf_program__fd(target_progs[i]); + + opts.fds = fds; + opts.funcs = tmulti_bpf_funcs; + opts.cnt = ARRAY_SIZE(fds); + opts.cookies = has_cookies ? cookies : NULL; + + link = bpf_program__attach_tracing_multi(prog, NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_tracing_multi")) + return; + + err = verify_tracing_multi_link_info(bpf_link__fd(link), prog, targets, btf_obj_id, + TRACING_MULTI_BPF_CNT, has_cookies, true); + ASSERT_OK(err, "verify_tracing_multi_link_info"); + + bpf_link__destroy(link); +} + +static void test_tracing_multi_bpf_fill_link_info(bool has_cookies) +{ + struct tracing_multi_bpf *skel; + + skel = tracing_multi_bpf__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_bpf__open_and_load")) + return; + + test_tracing_multi_bpf_fill_link_info_one(skel, skel->progs.test_fentry, + tmulti_bpf_fentry_cookies, has_cookies); + test_tracing_multi_bpf_fill_link_info_one(skel, skel->progs.test_fexit, + tmulti_bpf_fexit_cookies, has_cookies); + test_tracing_multi_bpf_fill_link_info_one(skel, skel->progs.test_fsession, + tmulti_bpf_fsession_cookies, has_cookies); + + tracing_multi_bpf__destroy(skel); +} + #define SEC(name) __attribute__((section(name), used)) static short uprobe_link_info_sema_1 SEC(".probes"); @@ -879,6 +1008,10 @@ void test_fill_link_info(void) test_tracing_multi_fill_link_info(skel, false, false); test_tracing_multi_fill_link_info(skel, true, false); } + if (test__start_subtest("tracing_multi_bpf_link_info")) { + test_tracing_multi_bpf_fill_link_info(false); + test_tracing_multi_bpf_fill_link_info(true); + } if (test__start_subtest("tracing_multi_invalid_ubuff")) test_tracing_multi_fill_link_info(skel, true, true); -- 2.55.0 Positively verify that a tail call from a BPF subprog works when the subprog is traced by fentry.multi. Assisted-by: Codex:gpt-5.6-sol Signed-off-by: Leon Hwang --- .../selftests/bpf/prog_tests/tailcalls.c | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/tailcalls.c b/tools/testing/selftests/bpf/prog_tests/tailcalls.c index 88d223110ecd..3b122520711b 100644 --- a/tools/testing/selftests/bpf/prog_tests/tailcalls.c +++ b/tools/testing/selftests/bpf/prog_tests/tailcalls.c @@ -16,7 +16,9 @@ #include "tailcall_sleepable.skel.h" #include "tailcall_callback.skel.h" #include "tailcall_bpf2bpf2.skel.h" +#include "tailcall_bpf2bpf4.skel.h" #include "tailcall_bpf2bpf_fexit.skel.h" +#include "tracing_multi_bpf.skel.h" /* test_tailcall_1 checks basic functionality by patching multiple locations * in a single program for a single tail call slot with nop->jmp, jmp->nop @@ -2065,6 +2067,66 @@ static void test_tailcall_large_stack(void) tailcall_large_stack__destroy(skel); } +/* + * Verify that a tail call from a BPF subprog works when the subprog is + * traced by fentry.multi. + */ +static void test_tailcall_bpf2bpf_fentry_multi(void) +{ + static const char *funcs[] = { "subprog_tail_2" }; + struct tracing_multi_bpf *skel_tracing = NULL; + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + struct tailcall_bpf2bpf4 *skel_tc; + struct bpf_link *link = NULL; + int fds[ARRAY_SIZE(funcs)]; + char buff[128] = {}; + int err, key; + LIBBPF_OPTS(bpf_test_run_opts, topts, + .data_in = buff, + .data_size_in = sizeof(buff), + .repeat = 1, + ); + + skel_tc = tailcall_bpf2bpf4__open_and_load(); + if (!ASSERT_OK_PTR(skel_tc, "tailcall_bpf2bpf4__open_and_load")) + return; + + skel_tracing = tracing_multi_bpf__open_and_load(); + if (!ASSERT_OK_PTR(skel_tracing, "tracing_multi_bpf__open_and_load")) + goto out; + + key = 0; + fds[0] = bpf_program__fd(skel_tc->progs.classifier_2); + err = bpf_map_update_elem(bpf_map__fd(skel_tc->maps.jmp_table), &key, &fds[0], BPF_ANY); + if (!ASSERT_OK(err, "bpf_map_update_elem")) + goto out; + + key = 2; + err = bpf_map_update_elem(bpf_map__fd(skel_tc->maps.jmp_table), &key, &fds[0], BPF_ANY); + if (!ASSERT_OK(err, "bpf_map_update_elem_loop")) + goto out; + + opts.fds = fds; + opts.funcs = funcs; + opts.cnt = ARRAY_SIZE(fds); + link = bpf_program__attach_tracing_multi(skel_tracing->progs.test_fentry, NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_tracing_multi")) + goto out; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel_tc->progs.entry), &topts); + if (!ASSERT_OK(err, "bpf_prog_test_run_opts")) + goto out; + + ASSERT_EQ(topts.retval, sizeof(buff) * 3, "retval"); + ASSERT_EQ(skel_tc->bss->count, 33, "count"); + ASSERT_EQ(skel_tracing->bss->test_result_fentry, 33, "fentry.multi count"); + +out: + bpf_link__destroy(link); + tracing_multi_bpf__destroy(skel_tracing); + tailcall_bpf2bpf4__destroy(skel_tc); +} + void test_tailcalls(void) { if (test__start_subtest("tailcall_1")) @@ -2138,4 +2200,6 @@ void test_tailcalls(void) test_tailcall_bpf2bpf_fexit_links(); if (test__start_subtest("tailcall_large_stack")) test_tailcall_large_stack(); + if (test__start_subtest("tailcall_bpf2bpf_fentry_multi")) + test_tailcall_bpf2bpf_fentry_multi(); } -- 2.55.0 Add an x86-64 one-shot generic benchmark comparing 1000 individual fentry link creations with one fentry.multi link creation for the same 1000 bpf prog targets. fentry.multi is much faster than fentry when attaching to 1000 targets on a 16c16g QEMU VM on an AMD EPYC 9745 server. ./bench tracing-multi-attach-progs Setting up benchmark 'tracing-multi-attach-progs'... tracing-multi-attach-progs: prepared 1000 identical BPF program targets tracing-multi-attach-progs: fentry created and attached 1000 programs/links in 338.171ms tracing-multi-attach-progs: fentry.multi created one program and attached one 1000-target link in 47.345ms tracing-multi-attach-progs: fentry.multi creation/attachment speedup is 7.14x Signed-off-by: Leon Hwang --- tools/testing/selftests/bpf/Makefile | 1 + tools/testing/selftests/bpf/bench.c | 5 + .../benchs/bench_tracing_multi_attach_progs.c | 193 ++++++++++++++++++ .../bpf/progs/tracing_multi_attach_progs.c | 24 +++ 4 files changed, 223 insertions(+) create mode 100644 tools/testing/selftests/bpf/benchs/bench_tracing_multi_attach_progs.c create mode 100644 tools/testing/selftests/bpf/progs/tracing_multi_attach_progs.c diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 80d8bbbc3084..23bed732d1bd 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -625,6 +625,7 @@ $(OUTPUT)/bench_bloom_filter_map.o: $(OUTPUT)/bloom_filter_bench.skel.h $(OUTPUT)/bench_bpf_loop.o: $(OUTPUT)/bpf_loop_bench.skel.h $(OUTPUT)/bench_bpf_for.o: $(OUTPUT)/bpf_for_bench.skel.h $(OUTPUT)/bench_strncmp.o: $(OUTPUT)/strncmp_bench.skel.h +$(OUTPUT)/bench_tracing_multi_attach_progs.o: $(OUTPUT)/tracing_multi_attach_progs.skel.h $(OUTPUT)/bench_bpf_hashmap_full_update.o: $(OUTPUT)/bpf_hashmap_full_update_bench.skel.h $(OUTPUT)/bench_local_storage.o: $(OUTPUT)/local_storage_bench.skel.h $(OUTPUT)/bench_local_storage_rcu_tasks_trace.o: $(OUTPUT)/local_storage_rcu_tasks_trace_bench.skel.h diff --git a/tools/testing/selftests/bpf/bench.c b/tools/testing/selftests/bpf/bench.c index d5f494184f9d..c7502187e654 100644 --- a/tools/testing/selftests/bpf/bench.c +++ b/tools/testing/selftests/bpf/bench.c @@ -529,6 +529,9 @@ extern const struct bench bench_trig_fmodret; extern const struct bench bench_trig_tp; extern const struct bench bench_trig_rawtp; +/* tracing_multi link attachment benchmark */ +extern const struct bench bench_tracing_multi_attach_progs; + /* uprobe/uretprobe benchmarks */ extern const struct bench bench_trig_uprobe_nop; extern const struct bench bench_trig_uretprobe_nop; @@ -615,6 +618,8 @@ static const struct bench *benchs[] = { &bench_trig_fmodret, &bench_trig_tp, &bench_trig_rawtp, + /* tracing_multi */ + &bench_tracing_multi_attach_progs, /* uprobes */ &bench_trig_uprobe_nop, &bench_trig_uretprobe_nop, diff --git a/tools/testing/selftests/bpf/benchs/bench_tracing_multi_attach_progs.c b/tools/testing/selftests/bpf/benchs/bench_tracing_multi_attach_progs.c new file mode 100644 index 000000000000..6cc6851eb488 --- /dev/null +++ b/tools/testing/selftests/bpf/benchs/bench_tracing_multi_attach_progs.c @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "bench.h" +#include "testing_helpers.h" +#include "tracing_multi_attach_progs.skel.h" +#include "bpf/libbpf_internal.h" + +#define TARGET_CNT 1000 + +static struct ctx { + struct tracing_multi_attach_progs *targets[TARGET_CNT]; + struct tracing_multi_attach_progs *fentries[TARGET_CNT]; + struct tracing_multi_attach_progs *fentry_multi; + int target_fds[TARGET_CNT]; + __u32 target_btf_ids[TARGET_CNT]; + int fentry_prog_fds[TARGET_CNT]; + int fentry_multi_prog_fd; +} ctx; + +static void prepare(void) +{ + int err, i; + + setup_libbpf(); + + for (i = 0; i < TARGET_CNT; i++) { + struct tracing_multi_attach_progs *target; + int btf_id, fd, obj_token_fd; + + target = tracing_multi_attach_progs__open(); + if (!target) { + fprintf(stderr, "failed to open target program %d\n", i); + exit(1); + } + + bpf_program__set_autoload(target->progs.dummy_fentry, false); + bpf_program__set_autoload(target->progs.dummy_fentry_multi, false); + err = tracing_multi_attach_progs__load(target); + if (err) { + fprintf(stderr, "failed to load target program %d: %s\n", + i, strerror(-err)); + exit(1); + } + + fd = bpf_program__fd(target->progs.target); + obj_token_fd = bpf_object__token_fd(target->obj); + obj_token_fd = obj_token_fd < 0 ? 0 : obj_token_fd; + btf_id = libbpf_find_prog_btf_id("target", fd, obj_token_fd); + if (btf_id <= 0) { + fprintf(stderr, "failed to find target BTF ID: %s\n", + strerror(-btf_id)); + exit(1); + } + + ctx.targets[i] = target; + ctx.target_fds[i] = fd; + ctx.target_btf_ids[i] = btf_id; + } +} + +static long create_and_attach_fentry(void) +{ + int link_fds[TARGET_CNT]; + long start_ns, end_ns; + int err, i, j; + + start_ns = get_time_ns(); + + /* An ordinary fentry program is bound to one target when it is loaded. */ + for (i = 0; i < TARGET_CNT; i++) { + struct tracing_multi_attach_progs *fentry; + + fentry = tracing_multi_attach_progs__open(); + if (!fentry) { + fprintf(stderr, "failed to open fentry program %d\n", i); + exit(1); + } + + bpf_program__set_autoload(fentry->progs.target, false); + bpf_program__set_autoload(fentry->progs.dummy_fentry_multi, false); + err = bpf_program__set_attach_target(fentry->progs.dummy_fentry, ctx.target_fds[i], + "target"); + err = err ?: tracing_multi_attach_progs__load(fentry); + if (err) { + fprintf(stderr, "failed to load fentry program %d: %s\n", + i, strerror(-err)); + exit(1); + } + + ctx.fentries[i] = fentry; + ctx.fentry_prog_fds[i] = bpf_program__fd(fentry->progs.dummy_fentry); + } + + for (i = 0; i < TARGET_CNT; i++) { + link_fds[i] = bpf_link_create(ctx.fentry_prog_fds[i], 0, BPF_TRACE_FENTRY, NULL); + if (link_fds[i] < 0) { + err = errno; + for (j = 0; j < i; j++) + close(link_fds[j]); + fprintf(stderr, "failed to create fentry link %d: %s\n", + i, strerror(err)); + exit(1); + } + } + end_ns = get_time_ns(); + + for (i = 0; i < TARGET_CNT; i++) + close(link_fds[i]); + + return end_ns - start_ns; +} + +static long create_and_attach_fentry_multi(void) +{ + LIBBPF_OPTS(bpf_link_create_opts, opts); + long start_ns, end_ns; + int err, link_fd; + + opts.tracing_multi.ids = ctx.target_btf_ids; + opts.tracing_multi.fds = ctx.target_fds; + opts.tracing_multi.cnt = TARGET_CNT; + + start_ns = get_time_ns(); + + ctx.fentry_multi = tracing_multi_attach_progs__open(); + if (!ctx.fentry_multi) { + fprintf(stderr, "failed to open fentry.multi program\n"); + exit(1); + } + + bpf_program__set_autoload(ctx.fentry_multi->progs.target, false); + bpf_program__set_autoload(ctx.fentry_multi->progs.dummy_fentry, false); + err = tracing_multi_attach_progs__load(ctx.fentry_multi); + if (err) { + fprintf(stderr, "failed to load fentry.multi program: %s\n", + strerror(-err)); + exit(1); + } + + ctx.fentry_multi_prog_fd = bpf_program__fd(ctx.fentry_multi->progs.dummy_fentry_multi); + link_fd = bpf_link_create(ctx.fentry_multi_prog_fd, 0, BPF_TRACE_FENTRY_MULTI, &opts); + end_ns = get_time_ns(); + if (link_fd < 0) { + err = errno; + fprintf(stderr, "failed to create fentry.multi link: %s\n", + strerror(err)); + exit(1); + } + + close(link_fd); + return end_ns - start_ns; +} + +static void cleanup(void) +{ + int i; + + tracing_multi_attach_progs__destroy(ctx.fentry_multi); + for (i = 0; i < TARGET_CNT; i++) { + tracing_multi_attach_progs__destroy(ctx.fentries[i]); + tracing_multi_attach_progs__destroy(ctx.targets[i]); + } +} + +static void setup(void) +{ + double fentry_multi_ms, fentry_ms; + long fentry_multi_ns, fentry_ns; + + prepare(); + fentry_ns = create_and_attach_fentry(); + fentry_multi_ns = create_and_attach_fentry_multi(); + fentry_ms = fentry_ns / 1000000.0; + fentry_multi_ms = fentry_multi_ns / 1000000.0; + + printf("%s: prepared %d identical BPF program targets\n", + bench->name, TARGET_CNT); + printf("%s: fentry created and attached %d programs/links in %-.3lfms\n", + bench->name, TARGET_CNT, fentry_ms); + printf("%s: fentry.multi created one program and attached one %d-target link in %-.3lfms\n", + bench->name, TARGET_CNT, fentry_multi_ms); + printf("%s: fentry.multi creation/attachment speedup is %-.2lfx\n", + bench->name, fentry_ms / fentry_multi_ms); + + cleanup(); + exit(0); +} + +const struct bench bench_tracing_multi_attach_progs = { + .name = "tracing-multi-attach-progs", + .setup = setup, +}; diff --git a/tools/testing/selftests/bpf/progs/tracing_multi_attach_progs.c b/tools/testing/selftests/bpf/progs/tracing_multi_attach_progs.c new file mode 100644 index 000000000000..0ea4513904cd --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_multi_attach_progs.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include + +char _license[] SEC("license") = "GPL"; + +SEC("syscall") +int target(void *ctx) +{ + return 0; +} + +SEC("fentry") +int BPF_PROG(dummy_fentry) +{ + return 0; +} + +SEC("fentry.multi") +int BPF_PROG(dummy_fentry_multi) +{ + return 0; +} -- 2.55.0