Expose perf_event_aux_pause() as an external function, this will be used by BPF kfunc in a sequential change. Signed-off-by: Leo Yan --- include/linux/perf_event.h | 1 + kernel/events/core.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index ec9d96025683958e909bb2463439dc69634f4ceb..9445c36e9f23e9090c7bf26c8085d2e4f308d38b 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1252,6 +1252,7 @@ extern int perf_event_read_local(struct perf_event *event, u64 *value, u64 *enabled, u64 *running); extern u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running); +extern void perf_event_aux_pause(struct perf_event *event, bool pause); extern struct perf_callchain_entry *perf_callchain(struct perf_event *event, struct pt_regs *regs); diff --git a/kernel/events/core.c b/kernel/events/core.c index 22fdf0c187cd970f59ad8d6de9c1b231f68ec2cb..75c194007ace911e6f91a27738c04dd0840bb3fb 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -8409,7 +8409,7 @@ static void __perf_event_aux_pause(struct perf_event *event, bool pause) } } -static void perf_event_aux_pause(struct perf_event *event, bool pause) +void perf_event_aux_pause(struct perf_event *event, bool pause) { struct perf_buffer *rb; -- 2.34.1 The bpf_perf_event_aux_pause kfunc is introduced for pause and resume the Perf AUX trace used by eBPF programs. An attached tracepoint (e.g., ftrace tracepoint or a dynamic tracepoint using uprobe or kprobe) can invoke bpf_perf_event_aux_pause() to pause or resume AUX trace. This is useful for fine-grained tracing. Signed-off-by: Leo Yan --- kernel/trace/bpf_trace.c | 55 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 3ae52978cae61a5d60b43c764d3e267bd32e1085..fbed1f9a92cce666e2a61d6bfceaabfd5efade1c 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -704,6 +704,61 @@ static const struct bpf_func_proto bpf_perf_event_output_proto = { .arg5_type = ARG_CONST_SIZE_OR_ZERO, }; +__bpf_kfunc_start_defs(); + +__bpf_kfunc int bpf_perf_event_aux_pause(void *p__map, u64 flags, bool pause) +{ + struct bpf_map *map = p__map; + struct bpf_array *array = container_of(map, struct bpf_array, map); + unsigned int cpu = smp_processor_id(); + u64 index = flags & BPF_F_INDEX_MASK; + struct bpf_event_entry *ee; + + if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) + return -EINVAL; + + /* Disabling IRQ avoids race condition with perf event flows. */ + guard(irqsave)(); + + if (unlikely(flags & ~(BPF_F_INDEX_MASK))) + return -EINVAL; + + if (index == BPF_F_CURRENT_CPU) + index = cpu; + + if (unlikely(index >= array->map.max_entries)) + return -E2BIG; + + ee = READ_ONCE(array->ptrs[index]); + if (!ee) + return -ENOENT; + + if (!has_aux(ee->event)) + return -EINVAL; + + perf_event_aux_pause(ee->event, pause); + return 0; +} + +__bpf_kfunc_end_defs(); + +BTF_KFUNCS_START(perf_event_kfunc_set_ids) +BTF_ID_FLAGS(func, bpf_perf_event_aux_pause, KF_TRUSTED_ARGS) +BTF_KFUNCS_END(perf_event_kfunc_set_ids) + +static const struct btf_kfunc_id_set bpf_perf_event_kfunc_set = { + .owner = THIS_MODULE, + .set = &perf_event_kfunc_set_ids, +}; + +static int __init bpf_perf_event_kfuncs_init(void) +{ + return register_btf_kfunc_id_set(BPF_PROG_TYPE_UNSPEC, + &bpf_perf_event_kfunc_set); +} + +late_initcall(bpf_perf_event_kfuncs_init); + static DEFINE_PER_CPU(int, bpf_event_output_nest_level); struct bpf_nested_pt_regs { struct pt_regs regs[3]; -- 2.34.1 Introduce a BPF program to trigger AUX pause and resume. Once a attached tracepoint is hit, the BPF program calls the bpf_perf_event_aux_pause() kfunc for controlling AUX trace. Signed-off-by: Leo Yan --- tools/perf/Makefile.perf | 1 + tools/perf/util/bpf_skel/auxtrace_pause.bpf.c | 156 ++++++++++++++++++++++++++ 2 files changed, 157 insertions(+) diff --git a/tools/perf/Makefile.perf b/tools/perf/Makefile.perf index d4c7031b01a77f4a42326e4c9a88d8a352143178..8fdd24ba4c25ff4a69925a8e0c85bc78dd4fda47 100644 --- a/tools/perf/Makefile.perf +++ b/tools/perf/Makefile.perf @@ -1179,6 +1179,7 @@ SKELETONS += $(SKEL_OUT)/kwork_trace.skel.h $(SKEL_OUT)/sample_filter.skel.h SKELETONS += $(SKEL_OUT)/kwork_top.skel.h $(SKEL_OUT)/syscall_summary.skel.h SKELETONS += $(SKEL_OUT)/bench_uprobe.skel.h SKELETONS += $(SKEL_OUT)/augmented_raw_syscalls.skel.h +SKELETONS += $(SKEL_OUT)/auxtrace_pause.skel.h $(SKEL_TMP_OUT) $(LIBAPI_OUTPUT) $(LIBBPF_OUTPUT) $(LIBPERF_OUTPUT) $(LIBSUBCMD_OUTPUT) $(LIBSYMBOL_OUTPUT): $(Q)$(MKDIR) -p $@ diff --git a/tools/perf/util/bpf_skel/auxtrace_pause.bpf.c b/tools/perf/util/bpf_skel/auxtrace_pause.bpf.c new file mode 100644 index 0000000000000000000000000000000000000000..37f8e1d096a3098415f6469014652dea948044d2 --- /dev/null +++ b/tools/perf/util/bpf_skel/auxtrace_pause.bpf.c @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +// Copyright 2025 Arm Limited +#include "vmlinux.h" +#include +#include + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} cpu_filter SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} task_filter SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(int)); + __uint(max_entries, 1); +} events SEC(".maps"); + +int enabled = 0; + +const volatile int has_cpu = 0; +const volatile int has_task = 0; +const volatile int per_thread = 0; + +int bpf_perf_event_aux_pause(void *map, u64 flags, bool pause) __ksym; + +static int event_aux_pause(void) +{ + __u64 flag; + __u32 cpu = bpf_get_smp_processor_id(); + + if (!enabled) + return 0; + + if (has_cpu) { + __u8 *ok; + + ok = bpf_map_lookup_elem(&cpu_filter, &cpu); + if (!ok) + return 0; + } + + if (has_task) { + __u32 pid = bpf_get_current_pid_tgid() & 0xffffffff; + __u8 *ok; + + ok = bpf_map_lookup_elem(&task_filter, &pid); + if (!ok) + return 0; + } + + flag = per_thread ? 0 : BPF_F_CURRENT_CPU; + bpf_perf_event_aux_pause(&events, flag, true); + return 0; +} + +static int event_aux_resume(void) +{ + __u64 flag; + __u32 cpu = bpf_get_smp_processor_id(); + + if (!enabled) + return 0; + + if (has_cpu) { + __u8 *ok; + + ok = bpf_map_lookup_elem(&cpu_filter, &cpu); + if (!ok) + return 0; + } + + if (has_task) { + __u32 pid = bpf_get_current_pid_tgid() & 0xffffffff; + __u8 *ok; + + ok = bpf_map_lookup_elem(&task_filter, &pid); + if (!ok) + return 0; + } + + flag = per_thread ? 0 : BPF_F_CURRENT_CPU; + bpf_perf_event_aux_pause(&events, flag, false); + return 0; +} + +SEC("kprobe/func_pause") +int BPF_PROG(kprobe_event_pause) +{ + return event_aux_pause(); +} + +SEC("kprobe/func_resume") +int BPF_PROG(kprobe_event_resume) +{ + return event_aux_resume(); +} + +SEC("kretprobe/func_pause") +int BPF_PROG(kretprobe_event_pause) +{ + return event_aux_pause(); +} + +SEC("kretprobe/func_resume") +int BPF_PROG(kretprobe_event_resume) +{ + return event_aux_resume(); +} + +SEC("uprobe/func_pause") +int BPF_PROG(uprobe_event_pause) +{ + return event_aux_pause(); +} + +SEC("uprobe/func_resume") +int BPF_PROG(uprobe_event_resume) +{ + return event_aux_resume(); +} + +SEC("uretprobe/func_pause") +int BPF_PROG(uretprobe_event_pause) +{ + return event_aux_pause(); +} + +SEC("uretprobe/func_resume") +int BPF_PROG(uretprobe_event_resume) +{ + return event_aux_resume(); +} + +SEC("tp/func_pause") +int BPF_PROG(tp_event_pause) +{ + return event_aux_pause(); +} + +SEC("tp/func_resume") +int BPF_PROG(tp_event_resume) +{ + return event_aux_resume(); +} + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; -- 2.34.1 This commit adds support for the BPF userspace program for AUX pause and resume. A list is maintained to track trigger points; each trigger point attaches to BPF programs when a session is opened and detaches when the session is closed. auxtrace__update_bpf_map() updates the AUX perf event pointer in the BPF map. The BPF kernel program then retrieves the event handler from the map to control AUX tracing. The auxtrace__set_bpf_filter() function updates the CPU and task filters for the BPF kernel program. Signed-off-by: Leo Yan --- tools/perf/util/Build | 4 + tools/perf/util/auxtrace.h | 43 ++++ tools/perf/util/bpf_auxtrace_pause.c | 408 +++++++++++++++++++++++++++++++++++ 3 files changed, 455 insertions(+) diff --git a/tools/perf/util/Build b/tools/perf/util/Build index 7910d908c814feec5e5e008f3a8b45384d796432..8ab29136344c3d37178f94aa1bd4b70ab54a7ab4 100644 --- a/tools/perf/util/Build +++ b/tools/perf/util/Build @@ -186,6 +186,10 @@ ifeq ($(CONFIG_LIBTRACEEVENT),y) perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf_kwork_top.o endif +ifeq ($(CONFIG_AUXTRACE),y) + perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf_auxtrace_pause.o +endif + perf-util-$(CONFIG_LIBELF) += symbol-elf.o perf-util-$(CONFIG_LIBELF) += probe-file.o perf-util-$(CONFIG_LIBELF) += probe-event.o diff --git a/tools/perf/util/auxtrace.h b/tools/perf/util/auxtrace.h index b0db84d27b255dc2f1aff446012598b045bbd5d3..52831e501dea1ebe476aed103a920b77d400e5f7 100644 --- a/tools/perf/util/auxtrace.h +++ b/tools/perf/util/auxtrace.h @@ -907,4 +907,47 @@ void itrace_synth_opts__clear_time_range(struct itrace_synth_opts *opts #endif +#if defined(HAVE_AUXTRACE_SUPPORT) && defined(HAVE_BPF_SKEL) + +int auxtrace__prepare_bpf(struct auxtrace_record *itr, const char *str); +int auxtrace__set_bpf_filter(struct evlist *evlist, struct record_opts *opts); +int auxtrace__enable_bpf(void); +int auxtrace__cleanup_bpf(void); +int auxtrace__update_bpf_map(struct evsel *evsel, int cpu_map_idx, int fd); + +#else /* HAVE_AUXTRACE_SUPPORT && HAVE_BPF_SKEL */ + +static inline int auxtrace__prepare_bpf(struct auxtrace_record *itr + __maybe_unused, + const char *str __maybe_unused) +{ + return -EINVAL; +} + +static inline int auxtrace__set_bpf_filter(struct evlist *evlist __maybe_unused, + struct record_opts *opts + __maybe_unused) +{ + return -EINVAL; +} + +static inline int auxtrace__enable_bpf(void) +{ + return -EINVAL; +} + +static inline int auxtrace__cleanup_bpf(void) +{ + return -EINVAL; +} + +static int auxtrace__update_bpf_map(struct evsel *evsel __maybe_unused, + int cpu_map_idx __maybe_unused, + int fd __maybe_unused) +{ + return -EINVAL; +} + +#endif + #endif diff --git a/tools/perf/util/bpf_auxtrace_pause.c b/tools/perf/util/bpf_auxtrace_pause.c new file mode 100644 index 0000000000000000000000000000000000000000..ed77b1e19dcf9da65cacf98def349c0ce9f83d46 --- /dev/null +++ b/tools/perf/util/bpf_auxtrace_pause.c @@ -0,0 +1,408 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* Copyright 2024 Arm Limited */ + +#include +#include +#include +#include + +#include + +#include "util/auxtrace.h" +#include "util/cpumap.h" +#include "util/thread_map.h" +#include "util/debug.h" +#include "util/evlist.h" +#include "util/bpf_counter.h" +#include "util/record.h" +#include "util/target.h" + +#include "util/bpf_skel/auxtrace_pause.skel.h" + +/* The valid controlling type is "p" (pause) and "r" (resume) */ +#define is_attach_kprobe(str) \ + (!strcmp((str), "kprobe") || !strcmp((str), "kretprobe")) +#define is_attach_uprobe(str) \ + (!strcmp((str), "uprobe") || !strcmp((str), "uretprobe")) +#define is_attach_tracepoint(str) \ + (!strcmp((str), "tp") || !strcmp((str), "tracepoint")) + +/* The valid controlling type is "p" (pause) and "r" (resume) */ +#define is_valid_ctrl_type(str) \ + (!strcmp((str), "p") || !strcmp((str), "r")) + +static struct auxtrace_pause_bpf *skel; + +struct trigger_entry { + struct list_head list; + char *arg0; + char *arg1; + char *arg2; + char *arg3; +}; + +static int trigger_entry_num; +static LIST_HEAD(trigger_list); +static struct bpf_link **trigger_links; + +static void auxtrace__free_bpf_trigger_list(void) +{ + struct trigger_entry *entry, *next; + + list_for_each_entry_safe(entry, next, &trigger_list, list) { + free(entry->arg0); + free(entry->arg1); + free(entry->arg2); + free(entry->arg3); + free(entry); + } + + trigger_entry_num = 0; +} + +static int auxtrace__alloc_bpf_trigger_list(const char *str) +{ + char *cmd_str; + char *substr, *saveptr1; + struct trigger_entry *entry; + int ret = 0; + + if (!str) + return -EINVAL; + + cmd_str = strdup(str); + if (!cmd_str) + return -ENOMEM; + + substr = strtok_r(cmd_str, ",", &saveptr1); + for ( ; substr != NULL; substr = strtok_r(NULL, ",", &saveptr1)) { + char *fmt1_str, *fmt2_str, *fmt3_str, *fmt4_str, *fmt; + + entry = zalloc(sizeof(*entry)); + if (!entry) { + ret = -ENOMEM; + goto out; + } + + /* + * A trigger is expressed with several fields with separator ":". + * The first field is specified for attach types, it can be one + * of types listed below: + * kprobe / kretprobe + * uprobe / uretprobe + * tp / tracepoint + * + * The kprobe and kretprobe trigger format is: + * {kprobe|kretprobe}:{p|r}:function_name + * + * The uprobe and uretprobe trigger format is: + * {uprobe|uretprobe}:{p|r}:executable:function_name + * + * Tracepoint trigger format is: + * {tp|tracepoint}:{p|r}:category:tracepint_name + * + * The last field is used to express the controlling type: "p" + * means aux pause and "r" is for aux resume. + */ + fmt1_str = strtok_r(substr, ":", &fmt); + fmt2_str = strtok_r(NULL, ":", &fmt); + fmt3_str = strtok_r(NULL, ":", &fmt); + if (!fmt1_str || !fmt2_str || !fmt3_str) { + pr_err("Failed to parse bpf aux pause string: %s\n", + substr); + ret = -EINVAL; + goto out; + } + + entry->arg0 = strdup(fmt1_str); + entry->arg1 = strdup(fmt2_str); + entry->arg2 = strdup(fmt3_str); + if (!entry->arg0 || !entry->arg1 || !entry->arg2) { + ret = -ENOMEM; + goto out; + } + + if (!is_attach_kprobe(entry->arg0) && + !is_attach_uprobe(entry->arg0) && + !is_attach_tracepoint(entry->arg0)) { + pr_err("Failed to support bpf aux pause attach: %s\n", + entry->arg0); + ret = -EINVAL; + goto out; + } + + if (!is_valid_ctrl_type(entry->arg1)) { + pr_err("Failed to support bpf aux pause ctrl: %s\n", + entry->arg1); + ret = -EINVAL; + goto out; + } + + if (!is_attach_kprobe(entry->arg0)) { + fmt4_str = strtok_r(NULL, ":", &fmt); + if (!fmt4_str) { + ret = -ENOMEM; + goto out; + } + + entry->arg3 = strdup(fmt4_str); + if (!entry->arg3) { + ret = -ENOMEM; + goto out; + } + } + + if (ret) + goto out; + + list_add(&entry->list, &trigger_list); + trigger_entry_num++; + } + + free(cmd_str); + return 0; + +out: + free(cmd_str); + if (entry) { + free(entry->arg0); + free(entry->arg1); + free(entry->arg2); + free(entry->arg3); + free(entry); + } + auxtrace__free_bpf_trigger_list(); + return ret; +} + +int auxtrace__prepare_bpf(struct auxtrace_record *itr, const char *str) +{ + int ret; + + if (!itr || !str) + return 0; + + skel = auxtrace_pause_bpf__open(); + if (!skel) { + pr_err("Failed to open func latency skeleton\n"); + return -1; + } + + ret = auxtrace__alloc_bpf_trigger_list(str); + if (ret) { + auxtrace_pause_bpf__destroy(skel); + skel = NULL; + return ret; + } + + return 0; +} + +static struct bpf_link *auxtrace__attach_bpf_prog(struct trigger_entry *entry) +{ + struct bpf_link *link = NULL; + LIBBPF_OPTS(bpf_uprobe_opts, uprobe_opts); + + if (!strcmp(entry->arg0, "kprobe")) { + if (!strcmp(entry->arg1, "p")) { + link = bpf_program__attach_kprobe( + skel->progs.kprobe_event_pause, + false, entry->arg2); + } else if (!strcmp(entry->arg1, "r")) { + link = bpf_program__attach_kprobe( + skel->progs.kprobe_event_resume, + false, entry->arg2); + } + } else if (!strcmp(entry->arg0, "kretprobe")) { + if (!strcmp(entry->arg1, "p")) { + link = bpf_program__attach_kprobe( + skel->progs.kretprobe_event_pause, + true, entry->arg2); + } else if (!strcmp(entry->arg1, "r")) { + link = bpf_program__attach_kprobe( + skel->progs.kretprobe_event_resume, + true, entry->arg2); + } + } else if (!strcmp(entry->arg0, "uprobe")) { + uprobe_opts.func_name = entry->arg3; + uprobe_opts.retprobe = false; + if (!strcmp(entry->arg1, "p")) { + link = bpf_program__attach_uprobe_opts( + skel->progs.uprobe_event_pause, + -1, entry->arg2, 0, &uprobe_opts); + } else if (!strcmp(entry->arg1, "r")) { + link = bpf_program__attach_uprobe_opts( + skel->progs.uprobe_event_resume, + -1, entry->arg2, 0, &uprobe_opts); + } + } else if (!strcmp(entry->arg0, "uretprobe")) { + uprobe_opts.func_name = entry->arg3; + uprobe_opts.retprobe = true; + if (!strcmp(entry->arg1, "p")) { + link = bpf_program__attach_uprobe_opts( + skel->progs.uretprobe_event_pause, + -1, entry->arg2, 0, &uprobe_opts); + } else if (!strcmp(entry->arg1, "r")) { + link = bpf_program__attach_uprobe_opts( + skel->progs.uretprobe_event_resume, + -1, entry->arg2, 0, &uprobe_opts); + } + + } else if (is_attach_tracepoint(entry->arg0)) { + if (!strcmp(entry->arg1, "p")) { + link = bpf_program__attach_tracepoint( + skel->progs.tp_event_pause, + entry->arg2, entry->arg3); + } else if (!strcmp(entry->arg1, "r")) { + link = bpf_program__attach_tracepoint( + skel->progs.tp_event_resume, + entry->arg2, entry->arg3); + } + } + + return link; +} + +int auxtrace__set_bpf_filter(struct evlist *evlist, struct record_opts *opts) +{ + int fd, err; + int i, ncpus = 1, ntasks = 1; + struct trigger_entry *trigger_entry; + struct target *target; + + if (!skel) + return 0; + + if (!opts) + return -EINVAL; + + target = &opts->target; + + if (target__has_cpu(target)) { + ncpus = perf_cpu_map__nr(evlist->core.user_requested_cpus); + bpf_map__set_max_entries(skel->maps.cpu_filter, ncpus); + skel->rodata->has_cpu = 1; + } + + if (target__has_task(target) || target__none(target)) { + ntasks = perf_thread_map__nr(evlist->core.threads); + bpf_map__set_max_entries(skel->maps.task_filter, ntasks); + skel->rodata->has_task = 1; + } + + if (target->per_thread) + skel->rodata->per_thread = 1; + + bpf_map__set_max_entries(skel->maps.events, libbpf_num_possible_cpus()); + + err = auxtrace_pause_bpf__load(skel); + if (err) { + pr_err("Failed to load func latency skeleton: %d\n", err); + goto out; + } + + if (target__has_cpu(target)) { + u32 cpu; + u8 val = 1; + + fd = bpf_map__fd(skel->maps.cpu_filter); + + for (i = 0; i < ncpus; i++) { + cpu = perf_cpu_map__cpu(evlist->core.user_requested_cpus, i).cpu; + bpf_map_update_elem(fd, &cpu, &val, BPF_ANY); + } + } + + if (target__has_task(target) || target__none(target)) { + u32 pid; + u8 val = 1; + + fd = bpf_map__fd(skel->maps.task_filter); + + for (i = 0; i < ntasks; i++) { + pid = perf_thread_map__pid(evlist->core.threads, i); + bpf_map_update_elem(fd, &pid, &val, BPF_ANY); + } + } + + trigger_links = zalloc(sizeof(*trigger_links) * trigger_entry_num); + if (!trigger_links) + return -ENOMEM; + + i = 0; + list_for_each_entry(trigger_entry, &trigger_list, list) { + trigger_links[i] = auxtrace__attach_bpf_prog(trigger_entry); + err = libbpf_get_error(trigger_links[i]); + if (err) { + pr_err("Failed to attach bpf program to aux pause entry\n"); + pr_err(" arg0=%s arg1=%s arg2=%s arg3=%s\n", + trigger_entry->arg0, trigger_entry->arg1, + trigger_entry->arg2, trigger_entry->arg3); + trigger_links[i] = NULL; + goto out; + } + i++; + } + + return 0; + +out: + for (i = 0; i < trigger_entry_num; i++) { + if (!trigger_links[i]) + continue; + bpf_link__destroy(trigger_links[i]); + } + + return err; +} + +int auxtrace__enable_bpf(void) +{ + if (!skel) + return 0; + + skel->bss->enabled = 1; + return 0; +} + +int auxtrace__cleanup_bpf(void) +{ + int i; + + if (!skel) + return 0; + + for (i = 0; i < trigger_entry_num; i++) { + if (!trigger_links[i]) + continue; + bpf_link__destroy(trigger_links[i]); + } + + auxtrace__free_bpf_trigger_list(); + auxtrace_pause_bpf__destroy(skel); + return 0; +} + +int auxtrace__update_bpf_map(struct evsel *evsel, int cpu_map_idx, int fd) +{ + int ret; + + if (!skel) + return 0; + + if (!evsel->needs_auxtrace_mmap) + return 0; + + ret = bpf_map_update_elem(bpf_map__fd(skel->maps.events), + &cpu_map_idx, &fd, BPF_ANY); + if (ret) { + pr_err("Failed to update BPF map for auxtrace: %s.\n", + strerror(errno)); + if (errno == EOPNOTSUPP) + pr_err(" Try to disable inherit mode with option '-i'.\n"); + return ret; + } + + return 0; +} -- 2.34.1 This commit introduces an option "--bpf-aux-pause" for loading BPF program to trigger AUX pause and resume. After: perf record -e cs_etm/aux-action=start-paused/ \ --bpf-aux-pause="kretprobe:p:__arm64_sys_openat,kprobe:r:__arm64_sys_openat,tp:r:sched:sched_switch" \ -a -- ls perf record -e cs_etm/aux-action=start-paused/ \ --bpf-aux-pause="kretprobe:p:__arm64_sys_openat,kprobe:r:__arm64_sys_openat,tp:r:sched:sched_switch" \ --per-thread -- ls Signed-off-by: Leo Yan --- tools/perf/builtin-record.c | 20 +++++++++++++++++++- tools/perf/util/evsel.c | 6 ++++++ tools/perf/util/record.h | 1 + 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c index 8059bce85a510b2dddc66f1b8b0013276840eddc..793609c0a59e8abcd248608f55e0af03a253138d 100644 --- a/tools/perf/builtin-record.c +++ b/tools/perf/builtin-record.c @@ -880,7 +880,12 @@ static int record__auxtrace_init(struct record *rec) if (err) return err; - return auxtrace_parse_filters(rec->evlist); + err = auxtrace_parse_filters(rec->evlist); + if (err) + return err; + + return auxtrace__prepare_bpf(rec->itr, + rec->opts.auxtrace_bpf_aux_pause_opts); } #else @@ -2506,6 +2511,10 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) evlist__config(rec->evlist, opts, &callchain_param); + err = auxtrace__set_bpf_filter(rec->evlist, opts); + if (err) + goto out_free_threads; + /* Debug message used by test scripts */ pr_debug3("perf record opening and mmapping events\n"); if (record__open(rec) != 0) { @@ -2579,6 +2588,9 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) if (record__start_threads(rec)) goto out_free_threads; + if (auxtrace__enable_bpf()) + goto out_free_threads; + /* * When perf is starting the traced process, all the events * (apart from group members) have enable_on_exec=1 set, @@ -2907,6 +2919,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) } out_delete_session: + auxtrace__cleanup_bpf(); #ifdef HAVE_EVENTFD_SUPPORT if (done_fd >= 0) { fd = done_fd; @@ -3629,6 +3642,11 @@ static struct option __record_options[] = { OPT_CALLBACK(0, "off-cpu-thresh", &record.opts, "ms", "Dump off-cpu samples if off-cpu time exceeds this threshold (in milliseconds). (Default: 500ms)", record__parse_off_cpu_thresh), + OPT_STRING_OPTARG(0, "bpf-aux-pause", &record.opts.auxtrace_bpf_aux_pause_opts, + "{kprobe|kretprobe}:{p|r}:function_name\n" + "\t\t\t {uprobe|uretprobe}:{p|r}:executable:function_name\n" + "\t\t\t {tp|tracepoint}:{p|r}:category:tracepoint\n", + "Enable AUX pause with BPF backend", ""), OPT_END() }; diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c index d55482f094bf95ac7b5c5173c1341baeb0fa9c93..f240e48f41a3e7ca5ba81733efc58a25c5c829ba 100644 --- a/tools/perf/util/evsel.c +++ b/tools/perf/util/evsel.c @@ -2650,6 +2650,12 @@ static int evsel__open_cpu(struct evsel *evsel, struct perf_cpu_map *cpus, bpf_counter__install_pe(evsel, idx, fd); + /* Update event info into BPF map for AUX trace */ + if (auxtrace__update_bpf_map(evsel, idx, fd)) { + err = -EINVAL; + goto out_close; + } + if (unlikely(test_attr__enabled())) { test_attr__open(&evsel->core.attr, pid, cpu, fd, group_fd, evsel->open_flags); diff --git a/tools/perf/util/record.h b/tools/perf/util/record.h index ea3a6c4657eefb743dc3d54b0b791ea39117cc10..1b1c2ed7fcadae8b56408b3b1b154faef3996eb3 100644 --- a/tools/perf/util/record.h +++ b/tools/perf/util/record.h @@ -65,6 +65,7 @@ struct record_opts { size_t auxtrace_snapshot_size; const char *auxtrace_snapshot_opts; const char *auxtrace_sample_opts; + const char *auxtrace_bpf_aux_pause_opts; bool sample_transaction; bool use_clockid; clockid_t clockid; -- 2.34.1 Documents the usage of the --bpf-aux-pause option and provides examples. Signed-off-by: Leo Yan --- tools/perf/Documentation/perf-record.txt | 51 ++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tools/perf/Documentation/perf-record.txt b/tools/perf/Documentation/perf-record.txt index 612612fa2d8041b94860035ed9cb01557a20b6b7..5aee20bfd03bda72bddabf42005b9678309414ad 100644 --- a/tools/perf/Documentation/perf-record.txt +++ b/tools/perf/Documentation/perf-record.txt @@ -544,6 +544,57 @@ must be an AUX area event. Samples on other events will be created containing data from the AUX area. Optionally sample size may be specified, otherwise it defaults to 4KiB. +--bpf-aux-pause=[=OPTIONS]:: +Specify trace events for triggering AUX pause with a BPF program. A trace event +can be static ftrace tracepoint, or dynamic tracepoint by using kprobe, +kretprobe, uprobe or uretprobe. This option must be enabled in combination with +the "aux-action=start-paused" configuration in an AUX event. + +For attaching a kprobe or kretprobe event, the format is: + + {kprobe|kretprobe}:{p|r}:function_name + +The format for attaching a uprobe or uretprobe event is: + + {uprobe|uretprobe}:{p|r}:executable:function_name + +The format for attaching a tracepoint is: + + {tp|tracepoint}:{p|r}:category:tracepint + +The first field is for the trace event type. It supports five types: kprobe, +kretprobe, uprobe, uretprobe, and tracepoint ('tp' is also supported as an +abbreviation for "tracepoint"). The second field specifies whether the action is +pause ("p") or resume ("r"). + +For probes, the "function_name" field is used to specify a function name. In +particular, for a uprobe or uretprobe, an executable path must also be provided. +In the case of a ftrace tracepoint, the "category" and "tracepoint" fields are +used together to provide complete tracepoint information. + +The '--bpf-aux-pause' option does not support inherit mode. In the default +trace mode, it needs to be combined with the '-i' or '--no-inherit' option to +disable inherit mode. + +The syntax supports multiple trace events, with each separated by a comma (,). +For example, users can set up AUX pause on a kernel function with kretprobe and +AUX resume on a tracepoint with the syntax below: + + For default trace mode (with inherit mode disabled): + perf record -e cs_etm/aux-action=start-paused/ \ + --bpf-aux-pause="kretprobe:p:__arm64_sys_openat,tp:r:sched:sched_switch" \ + -i ... + + For system wide trace mode: + perf record -e cs_etm/aux-action=start-paused/ \ + --bpf-aux-pause="kretprobe:p:__arm64_sys_openat,tp:r:sched:sched_switch" \ + -a ... + + For trace with uprobe and uretprobe: + perf record -e cs_etm/aux-action=start-paused/ \ + --bpf-aux-pause="uretprobe:p:~/sort:bubble_sort,uprobe:r:~/sort:bubble_sort" \ + -i --per-thread -- ~/sort + --proc-map-timeout:: When processing pre-existing threads /proc/XXX/mmap, it may take a long time, because the file may be huge. A time out is needed in such cases. -- 2.34.1