From: Slava Imameev BPF programs designated as dynamically loaded can be loaded and attached independently after the initial bpf_object loading and attaching. These programs can also be reloaded and reattached multiple times, enabling more flexible management of a resident BPF program set. A key motivation for this feature is to reduce load times for utilities that include hundreds of BPF programs. When the selection of a resident BPF program set cannot be determined at the time of bpf_object loading and attaching, all BPF programs would otherwise need to be marked as autoload, leading to unnecessary overhead. This patch addresses that inefficiency. Signed-off-by: Slava Imameev Signed-off-by: Andrey Grodzovsky --- tools/lib/bpf/libbpf.c | 144 ++++++++++++++++++++++++++++++++++++--- tools/lib/bpf/libbpf.h | 4 ++ tools/lib/bpf/libbpf.map | 2 + 3 files changed, 141 insertions(+), 9 deletions(-) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 821d854d43a9..c3fe1de43210 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -725,6 +725,7 @@ struct bpf_object { bool has_subcalls; bool has_rodata; + bool has_dynload_progs; struct bpf_gen *gen_loader; @@ -8056,7 +8057,7 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog log_buf = prog->log_buf; log_buf_size = prog->log_size; own_log_buf = false; - } else if (obj->log_buf) { + } else if (obj->log_buf && prog->load_type != BPF_PROG_LOAD_TYPE_DYNAMIC) { log_buf = obj->log_buf; log_buf_size = obj->log_size; own_log_buf = false; @@ -8403,6 +8404,7 @@ bpf_object__load_progs(struct bpf_object *obj, int log_level) pr_debug("prog '%s': skipped auto-loading\n", prog->name); continue; } + prog->log_level |= log_level; if (obj->gen_loader) @@ -9076,15 +9078,22 @@ static void bpf_object_cleanup_btf(struct bpf_object *obj) obj->btf_modules_loaded = false; zfree(&obj->btf_modules); - /* clean up vmlinux BTF */ - btf__free(obj->btf_vmlinux); - obj->btf_vmlinux = NULL; + /* The btf_vmlinux data is needed for dynamically loaded programs, + * so defer freeing it in that case to the end of the object lifetime. + */ + if (!obj->has_dynload_progs) { + btf__free(obj->btf_vmlinux); + obj->btf_vmlinux = NULL; + } } static void bpf_object_post_load_cleanup(struct bpf_object *obj) { - /* clean up fd_array */ - zfree(&obj->fd_array); + /* The fd array is needed for dynamically loaded programs, + * so defer freeing it in that case to the end of the object lifetime. + */ + if (!obj->has_dynload_progs || !obj->fd_array_cnt) + zfree(&obj->fd_array); /* clean up BTF */ bpf_object_cleanup_btf(obj); @@ -9678,6 +9687,8 @@ void bpf_object__close(struct bpf_object *obj) close(obj->jumptable_maps[i].fd); zfree(&obj->jumptable_maps); + zfree(&obj->fd_array); + free(obj); } @@ -9805,8 +9816,16 @@ bool bpf_program__autoload(const struct bpf_program *prog) int bpf_program__set_autoload(struct bpf_program *prog, bool autoload) { - return bpf_program__set_load_type(prog, - autoload ? BPF_PROG_LOAD_TYPE_AUTO : BPF_PROG_LOAD_TYPE_DISABLED); + enum bpf_prog_load_type type = prog->load_type; + + if (autoload) + type = BPF_PROG_LOAD_TYPE_AUTO; + else if (prog->load_type == BPF_PROG_LOAD_TYPE_AUTO) + type = BPF_PROG_LOAD_TYPE_DISABLED; + else + return 0; /* Otherwise, keep the current load type. */ + + return bpf_program__set_load_type(prog, type); } bool bpf_program__autoattach(const struct bpf_program *prog) @@ -15220,12 +15239,73 @@ void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s) free(s); } +static int bpf_program__set_dynamicload(struct bpf_program *prog) +{ + struct bpf_object *obj; + const char *attach_name; + + obj = prog->obj; + if (!obj) + return libbpf_err(-EINVAL); + + /* Dynamically-loaded programs are not supported for gen_loader. + * This is because bpf_object_load_prog is not called for + * dynamicload programs, so dynamicload programs are not visible + * to gen_loader. For this reason, prevent calling + * bpf_program__set_dynamicload when gen_loader was used to + * generate a BPF object loader. + * A gen_loader implementation is being called for autoloaded + * programs and defines its own model for loading BPF programs. + * To pass a BPF program to gen_loader, set the program's load type + * to LD_AUTOLOAD. + */ + if (obj->gen_loader) + return libbpf_err(-ENOTSUP); + + if (prog_is_subprog(obj, prog)) + return libbpf_err(-EINVAL); + + attach_name = strchr(prog->sec_name, '/'); + if (!attach_name || strchr(attach_name, ':')) { + /* Only reject programs that require BTF-based attach target + * resolution (indicated by the SEC_ATTACH_BTF flag). Such + * programs need the section name parsed for the attach target + * function name (after '/') and optionally the module name + * (before ':') for libbpf_find_attach_btf_id. + * + * Programs like SEC("classifier"), SEC("socket"), etc. do + * not require BTF attach resolution and can safely use + * dynamic loading despite having no '/' in their section + * name. The BTF guard in libbpf_prepare_prog_load (checking + * SEC_ATTACH_BTF) is the authoritative check; this is an + * early-reject for programs that would fail there. + */ + long flags = prog->sec_def ? + (long)prog->sec_def->cookie : SEC_ATTACH_BTF; + if ((flags & SEC_ATTACH_BTF) && !prog->attach_btf_id) + return libbpf_err(-EINVAL); + } + + obj->has_dynload_progs = true; + prog->load_type = BPF_PROG_LOAD_TYPE_DYNAMIC; + prog->autoattach = false; + + return 0; +} + int bpf_program__set_load_type(struct bpf_program *prog, enum bpf_prog_load_type type) { if (prog->obj->state >= OBJ_LOADED) return libbpf_err(-EINVAL); - prog->load_type = type; + switch (type) { + case BPF_PROG_LOAD_TYPE_DYNAMIC: + return bpf_program__set_dynamicload(prog); + default: + prog->load_type = type; + break; + } + return 0; } @@ -15233,3 +15313,49 @@ enum bpf_prog_load_type bpf_program__load_type(const struct bpf_program *prog) { return prog->load_type; } + +/* + * This function must be called after bpf_object__load_progs. + * Dynamically-loaded program data is initialized on object load. + * Post-load initialization is not supported. + */ +int +bpf_program__load_dynamically(struct bpf_program *prog, int extra_log_level) +{ + int err; + struct bpf_object *obj; + + obj = prog->obj; + if (!obj || obj->state < OBJ_LOADED) + return libbpf_err(-EINVAL); + + if (prog_is_subprog(obj, prog) || prog->load_type != BPF_PROG_LOAD_TYPE_DYNAMIC) + return libbpf_err(-EINVAL); + + prog->log_level |= extra_log_level; + + err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt, + obj->license, obj->kern_version, &prog->fd); + if (err) { + pr_warn("prog '%s': failed to dynamically load: %d\n", prog->name, err); + prog->log_level &= ~extra_log_level; + return err; + } + + prog->log_level &= ~extra_log_level; + return 0; +} + +int bpf_program__unload_dynamically(struct bpf_program *prog) +{ + int err; + + if (!prog || prog->load_type != BPF_PROG_LOAD_TYPE_DYNAMIC) + return libbpf_err(-EINVAL); + + /* Close the file descriptor but retain the program's data to + * support reloading the program if it is required again. + */ + err = zclose(prog->fd); + return err ? libbpf_err(-errno) : 0; +} diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h index 700634664e32..54d755861289 100644 --- a/tools/lib/bpf/libbpf.h +++ b/tools/lib/bpf/libbpf.h @@ -2083,15 +2083,19 @@ LIBBPF_API int bpf_program__clone(struct bpf_program *prog, const struct bpf_pro * * - BPF_PROG_LOAD_TYPE_DISABLED: the program is not loaded. * - BPF_PROG_LOAD_TYPE_AUTO: the program is autoloaded when the bpf_object is loaded. + * - BPF_PROG_LOAD_TYPE_DYNAMIC: the program is loaded and attached dynamically. */ enum bpf_prog_load_type { BPF_PROG_LOAD_TYPE_DISABLED = 0, BPF_PROG_LOAD_TYPE_AUTO, + BPF_PROG_LOAD_TYPE_DYNAMIC, }; LIBBPF_API int bpf_program__set_load_type(struct bpf_program *prog, enum bpf_prog_load_type loadtype); LIBBPF_API enum bpf_prog_load_type bpf_program__load_type(const struct bpf_program *prog); +LIBBPF_API int bpf_program__load_dynamically(struct bpf_program *prog, int extra_log_level); +LIBBPF_API int bpf_program__unload_dynamically(struct bpf_program *prog); #ifdef __cplusplus } /* extern "C" */ diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map index 4d63dac8c3b7..a0389c58efe9 100644 --- a/tools/lib/bpf/libbpf.map +++ b/tools/lib/bpf/libbpf.map @@ -464,4 +464,6 @@ LIBBPF_1.8.0 { btf__new_empty_opts; bpf_program__load_type; bpf_program__set_load_type; + bpf_program__load_dynamically; + bpf_program__unload_dynamically; } LIBBPF_1.7.0; -- 2.34.1