Detect exact function names (no wildcards) in bpf_program__attach_kprobe_multi_opts() and bypass kallsyms parsing, passing the symbol directly to the kernel via syms[] array. This benefits all callers, not just kprobe.session. When the pattern contains no '*' or '?' characters, set syms to point directly at the pattern string and cnt to 1, skipping the expensive /proc/kallsyms or available_filter_functions parsing (~150ms per function). Error code normalization: the fast path returns ESRCH from kernel's ftrace_lookup_symbols(), while the slow path returns ENOENT from userspace kallsyms parsing. Convert ESRCH to ENOENT in the bpf_link_create error path to maintain API consistency - both paths now return identical error codes for "symbol not found". Signed-off-by: Andrey Grodzovsky --- tools/lib/bpf/libbpf.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 0be7017800fe..80278385bc9c 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -12041,7 +12041,15 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, if (addrs && syms) return libbpf_err_ptr(-EINVAL); - if (pattern) { + /* + * Exact function name (no wildcards): bypass kallsyms parsing + * and pass the symbol directly to the kernel via syms[] array. + * The kernel's ftrace_lookup_symbols() resolves it efficiently. + */ + if (pattern && !strpbrk(pattern, "*?")) { + syms = &pattern; + cnt = 1; + } else if (pattern) { if (has_available_filter_functions_addrs()) err = libbpf_available_kprobes_parse(&res); else @@ -12084,6 +12092,14 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, link_fd = bpf_link_create(prog_fd, 0, attach_type, &lopts); if (link_fd < 0) { err = -errno; + /* + * Normalize error code: when exact name bypasses kallsyms + * parsing, kernel returns ESRCH from ftrace_lookup_symbols(). + * Convert to ENOENT for API consistency with the pattern + * matching path which returns ENOENT from userspace. + */ + if (err == -ESRCH) + err = -ENOENT; pr_warn("prog '%s': failed to attach: %s\n", prog->name, errstr(err)); goto error; -- 2.34.1 When ftrace_lookup_symbols() is called with a single symbol (cnt == 1), use kallsyms_lookup_name() for O(log N) binary search instead of the full linear scan via kallsyms_on_each_symbol(). ftrace_lookup_symbols() was designed for batch resolution of many symbols in a single pass. For large cnt this is efficient: a single O(N) walk over all symbols with O(log cnt) binary search into the sorted input array. But for cnt == 1 it still decompresses all ~200K kernel symbols only to match one. kallsyms_lookup_name() uses the sorted kallsyms index and needs only ~17 decompressions for a single lookup. This is the common path for kprobe.session with exact function names, where libbpf sends one symbol per BPF_LINK_CREATE syscall. If binary lookup fails (duplicate symbol names where the first match is not ftrace-instrumented), the function falls through to the existing linear scan path. Before (cnt=1, 50 kprobe.session programs): Attach: 858 ms (kallsyms_expand_symbol 25% of CPU) After: Attach: 52 ms (16x faster) Signed-off-by: Andrey Grodzovsky --- kernel/trace/ftrace.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 827fb9a0bf0d..13906af8098a 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -9263,6 +9263,15 @@ static int kallsyms_callback(void *data, const char *name, unsigned long addr) * @addrs array, which needs to be big enough to store at least @cnt * addresses. * + * For a single symbol (cnt == 1), uses kallsyms_lookup_name() which + * performs an O(log N) binary search via the sorted kallsyms index. + * This avoids the full O(N) linear scan over all kernel symbols that + * the multi-symbol path requires. + * + * For multiple symbols, uses a single-pass linear scan via + * kallsyms_on_each_symbol() with binary search into the sorted input + * array. + * * Returns: 0 if all provided symbols are found, -ESRCH otherwise. */ int ftrace_lookup_symbols(const char **sorted_syms, size_t cnt, unsigned long *addrs) @@ -9270,6 +9279,19 @@ int ftrace_lookup_symbols(const char **sorted_syms, size_t cnt, unsigned long *a struct kallsyms_data args; int found_all; + /* Fast path: single symbol uses O(log N) binary search */ + if (cnt == 1) { + addrs[0] = kallsyms_lookup_name(sorted_syms[0]); + if (addrs[0] && ftrace_location(addrs[0])) + return 0; + /* + * Binary lookup can fail for duplicate symbol names + * where the first match is not ftrace-instrumented. + * Retry with linear scan. + */ + } + + /* Batch path: single-pass O(N) linear scan */ memset(addrs, 0, sizeof(*addrs) * cnt); args.addrs = addrs; args.syms = sorted_syms; -- 2.34.1 Extend existing kprobe_multi_test subtests to validate the kprobe.session exact function name optimization: In kprobe_multi_session.c, add test_kprobe_syms which attaches a kprobe.session program to an exact function name (bpf_fentry_test1) exercising the fast syms[] path that bypasses kallsyms parsing. It calls session_check() so bpf_fentry_test1 is hit by both the wildcard and exact probes, and test_session_skel_api validates kprobe_session_result[0] == 4 (entry + return from each probe). In test_attach_api_fails, add fail_7 and fail_8 verifying error code consistency between the wildcard pattern path (slow, parses kallsyms) and the exact function name path (fast, uses syms[] array). Both paths must return -ENOENT for non-existent functions. Signed-off-by: Andrey Grodzovsky --- .../bpf/prog_tests/kprobe_multi_test.c | 33 +++++++++++++++++-- .../bpf/progs/kprobe_multi_session.c | 10 ++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c b/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c index 9caef222e528..ea605245ba14 100644 --- a/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c +++ b/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c @@ -327,6 +327,30 @@ static void test_attach_api_fails(void) if (!ASSERT_EQ(saved_error, -E2BIG, "fail_6_error")) goto cleanup; + /* fail_7 - non-existent wildcard pattern (slow path) */ + LIBBPF_OPTS_RESET(opts); + + link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual, + "__nonexistent_func_xyz_*", + &opts); + saved_error = -errno; + if (!ASSERT_ERR_PTR(link, "fail_7")) + goto cleanup; + + if (!ASSERT_EQ(saved_error, -ENOENT, "fail_7_error")) + goto cleanup; + + /* fail_8 - non-existent exact name (fast path), same error as wildcard */ + link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual, + "__nonexistent_func_xyz_123", + &opts); + saved_error = -errno; + if (!ASSERT_ERR_PTR(link, "fail_8")) + goto cleanup; + + if (!ASSERT_EQ(saved_error, -ENOENT, "fail_8_error")) + goto cleanup; + cleanup: bpf_link__destroy(link); kprobe_multi__destroy(skel); @@ -355,8 +379,13 @@ static void test_session_skel_api(void) ASSERT_OK(err, "test_run"); ASSERT_EQ(topts.retval, 0, "test_run"); - /* bpf_fentry_test1-4 trigger return probe, result is 2 */ - for (i = 0; i < 4; i++) + /* + * bpf_fentry_test1 is hit by both the wildcard probe and the exact + * name probe (test_kprobe_syms), so entry + return fires twice: 4. + * bpf_fentry_test2-4 are hit only by the wildcard probe: 2. + */ + ASSERT_EQ(skel->bss->kprobe_session_result[0], 4, "kprobe_session_result"); + for (i = 1; i < 4; i++) ASSERT_EQ(skel->bss->kprobe_session_result[i], 2, "kprobe_session_result"); /* bpf_fentry_test5-8 trigger only entry probe, result is 1 */ diff --git a/tools/testing/selftests/bpf/progs/kprobe_multi_session.c b/tools/testing/selftests/bpf/progs/kprobe_multi_session.c index bd8b7fb7061e..d52a65b40bbf 100644 --- a/tools/testing/selftests/bpf/progs/kprobe_multi_session.c +++ b/tools/testing/selftests/bpf/progs/kprobe_multi_session.c @@ -76,3 +76,13 @@ int test_kprobe(struct pt_regs *ctx) { return session_check(ctx); } + +/* + * Exact function name (no wildcards) - exercises the fast syms[] path + * in bpf_program__attach_kprobe_multi_opts() which bypasses kallsyms parsing. + */ +SEC("kprobe.session/bpf_fentry_test1") +int test_kprobe_syms(struct pt_regs *ctx) +{ + return session_check(ctx); +} -- 2.34.1