From: Slava Imameev BPF programs designated as manually loaded can be loaded and attached independently after the initial bpf_object loading and attaching. These programs can also be reloaded and reattached multiple times, enabling more flexible management of a resident BPF program set. A key motivation for this feature is to reduce load times for utilities that include hundreds of BPF programs. When the selection of a resident BPF program set cannot be determined at the time of bpf_object loading and attaching, all BPF programs would otherwise need to be marked as autoload, leading to unnecessary overhead. This patch addresses that inefficiency. A manual-strategy program is loaded via bpf_program__load() and unloaded via bpf_program__unload(). It only requires that bpf object is already in PREPARE state (BTF loaded, maps created, relocations applied) rather than requiring a full bpf_object__load(). Manual programs are skipped by the object's own autoload pass. A few key changes described below: gen_loader is rejected if any manual program is present since this loader is finalized during bpf_object__load() and so it will miss any manual programs loaded later. Two program log related APIs were relaxed to work post OBJ_LOADED state to fix regression where verifier logs were lost in case of bpf_object__prepare failure. It makes sense to relax those for MANUAL programs as they can load post OBJ_LOADED state so the log buff still can be set at this point. Also add proper documentation to the setters. Failures with BPF_PROG_TYPE_STRUCT_OPS were encountered and so in several places code related to struct_ops logic was altered to take into account struct_ops program with MANUAL strategy set. The general idea is to reject this type of programs for manual loading since those programs cannot load/unload independently of each other per a given struct ops. Hence, SEC("!...") style declaration is rejected for them while bpf_program__set_load_strategy() does accept them to allow certain flexibility in the API future usage. Assisted-by: Claude:claude-sonnet-5 Signed-off-by: Slava Imameev Signed-off-by: Andrey Grodzovsky --- tools/lib/bpf/libbpf.c | 205 ++++++++++++++++++++++++++++++++++----- tools/lib/bpf/libbpf.h | 56 +++++++++++ tools/lib/bpf/libbpf.map | 1 + 3 files changed, 240 insertions(+), 22 deletions(-) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index e4cdbc2f467d..1d5d59b95c4e 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -758,6 +758,7 @@ struct bpf_object { bool has_subcalls; bool has_rodata; + bool has_manual_progs; struct bpf_gen *gen_loader; @@ -870,7 +871,7 @@ static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn); static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx); static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx); -void bpf_program__unload(struct bpf_program *prog) +static void bpf_program_unload_full(struct bpf_program *prog) { if (!prog) return; @@ -882,12 +883,30 @@ void bpf_program__unload(struct bpf_program *prog) zfree(&prog->subprogs); } +void bpf_program__unload(struct bpf_program *prog) +{ + if (!prog) + return; + + /* + * MANUAL programs retain their data here so bpf_program__load() + * can reload them later; object teardown paths call + * bpf_program_unload_full() instead to always release it fully. + */ + if (prog->load_strategy == BPF_PROG_LOAD_STRATEGY_MANUAL) { + zclose(prog->fd); + return; + } + + bpf_program_unload_full(prog); +} + static void bpf_program__exit(struct bpf_program *prog) { if (!prog) return; - bpf_program__unload(prog); + bpf_program_unload_full(prog); zfree(&prog->name); zfree(&prog->sec_name); zfree(&prog->insns); @@ -1228,6 +1247,14 @@ static int bpf_object_adjust_struct_ops_autoload(struct bpf_object *obj) if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) continue; + /* + * If the user set the load strategy to MANUAL, don't guess it + * from map autocreate: a MANUAL program is loaded explicitly + * with bpf_program__load(), which rejects any other strategy. + */ + if (prog->load_strategy == BPF_PROG_LOAD_STRATEGY_MANUAL) + continue; + for (j = 0; j < obj->nr_maps; ++j) { const struct btf_type *type; @@ -5874,6 +5901,20 @@ static int init_prog_array_slots(struct bpf_object *obj, struct bpf_map *map) continue; targ_prog = map->init_slots[i]; + + /* + * Be careful about manual progs, reject if they weren't loaded before + * this step to not insert invalid FDs into the map, otherwise we will + * get a much less clear rejection from the kernel on the element update + * below. + */ + if (targ_prog->load_strategy == BPF_PROG_LOAD_STRATEGY_MANUAL && + targ_prog->fd < 0) { + pr_warn("map '%s': slot [%u] prog '%s' is MANUAL but not loaded\n", + map->name, i, targ_prog->name); + return -EINVAL; + } + fd = bpf_program__fd(targ_prog); err = bpf_map_update_elem(map->fd, &i, &fd, 0); @@ -9330,6 +9371,8 @@ static int bpf_object_prepare_progs(struct bpf_object *obj) for (i = 0; i < obj->nr_programs; i++) { prog = &obj->programs[i]; + if (prog->load_strategy == BPF_PROG_LOAD_STRATEGY_MANUAL) + obj->has_manual_progs = true; err = bpf_object__sanitize_prog(obj, prog); if (err) return err; @@ -9356,6 +9399,18 @@ static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object prog->type = prog->sec_def->prog_type; prog->expected_attach_type = prog->sec_def->expected_attach_type; + /* + * struct_ops programs declared SEC("!...") are rejected here + * at declarative parse time for manual loading since those programs cannot + * load/unload independently of each other per a given struct ops. + */ + if (prog->type == BPF_PROG_TYPE_STRUCT_OPS && + prog->load_strategy == BPF_PROG_LOAD_STRATEGY_MANUAL) { + pr_warn("prog '%s': struct_ops programs do not support manual loading\n", + prog->name); + return -EINVAL; + } + /* sec_def can have custom callback which should be called * after bpf_program is initialized to adjust its properties */ @@ -9556,7 +9611,7 @@ static int bpf_object_unload(struct bpf_object *obj) } for (i = 0; i < obj->nr_programs; i++) - bpf_program__unload(&obj->programs[i]); + bpf_program_unload_full(&obj->programs[i]); return 0; } @@ -9947,7 +10002,7 @@ static int bpf_object__resolve_externs(struct bpf_object *obj, return 0; } -static void bpf_map_prepare_vdata(const struct bpf_map *map) +static int bpf_map_prepare_vdata(const struct bpf_map *map) { const struct btf_type *type; struct bpf_struct_ops *st_ops; @@ -9963,16 +10018,32 @@ static void bpf_map_prepare_vdata(const struct bpf_map *map) if (!prog) continue; + /* + * Manual struct_ops programs don't get along well with manual + * loading: all the progs belonging to the same struct_ops must + * load and unload together, which is the opposite of the manual + * load concept. + * Still, we allow manual progs here as long as they were loaded + * before this step because this way they do work. + */ + if (prog->load_strategy == BPF_PROG_LOAD_STRATEGY_MANUAL && prog->fd < 0) { + pr_warn("map '%s': struct_ops member prog '%s' is MANUAL but not loaded\n", + map->name, prog->name); + return -EINVAL; + } + prog_fd = bpf_program__fd(prog); kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i]; *(unsigned long *)kern_data = prog_fd; } + + return 0; } static int bpf_object_prepare_struct_ops(struct bpf_object *obj) { struct bpf_map *map; - int i; + int i, err; for (i = 0; i < obj->nr_maps; i++) { map = &obj->maps[i]; @@ -9983,7 +10054,9 @@ static int bpf_object_prepare_struct_ops(struct bpf_object *obj) if (!map->autocreate) continue; - bpf_map_prepare_vdata(map); + err = bpf_map_prepare_vdata(map); + if (err) + return err; } return 0; @@ -9999,10 +10072,19 @@ static void bpf_object_unpin(struct bpf_object *obj) bpf_map__unpin(&obj->maps[i], NULL); } -static void bpf_object_cleanup_btf(struct bpf_object *obj) +static void bpf_object_cleanup_btf(struct bpf_object *obj, bool force) { int i; + /* + * Module BTF fds and btf_vmlinux are still needed by + * programs that have not been manually loaded yet, so defer freeing + * them to the end of the object lifetime, unless the caller forces + * immediate cleanup. + */ + if (!force && obj->has_manual_progs) + return; + /* clean up module BTFs */ for (i = 0; i < obj->btf_module_cnt; i++) { close(obj->btf_modules[i].fd); @@ -10019,15 +10101,42 @@ static void bpf_object_cleanup_btf(struct bpf_object *obj) obj->btf_vmlinux = NULL; } -static void bpf_object_post_load_cleanup(struct bpf_object *obj) +static void bpf_object_post_load_cleanup(struct bpf_object *obj, bool force) { - /* clean up fd_array, and the slots handed out with it */ - zfree(&obj->fd_array); - obj->fd_array_cap = 0; - obj->fd_array_used = 0; + /* + * The fd array, and the slots handed out with it, are needed for + * manually loaded programs, so defer freeing them in that case + * to the end of the object lifetime. + */ + if (force || !obj->has_manual_progs) { + zfree(&obj->fd_array); + obj->fd_array_cap = 0; + obj->fd_array_used = 0; + } /* clean up BTF */ - bpf_object_cleanup_btf(obj); + bpf_object_cleanup_btf(obj, force); +} + +static int bpf_object_check_gen_loader(struct bpf_object *obj) +{ + struct bpf_program *prog; + + if (!obj->gen_loader) + return 0; + + /* + * gen_loader only records programs loaded by bpf_object__load_progs(), + * which skips MANUAL ones. + */ + bpf_object__for_each_program(prog, obj) { + if (prog->load_strategy != BPF_PROG_LOAD_STRATEGY_MANUAL) + continue; + pr_warn("prog '%s': gen_loader does not support manually-loaded programs\n", + prog->name); + return -EOPNOTSUPP; + } + return 0; } static int bpf_object_prepare(struct bpf_object *obj, const char *target_btf_path) @@ -10039,7 +10148,8 @@ static int bpf_object_prepare(struct bpf_object *obj, const char *target_btf_pat return -EINVAL; } - err = bpf_object_prepare_token(obj); + err = bpf_object_check_gen_loader(obj); + err = err ? : bpf_object_prepare_token(obj); err = err ? : bpf_object__probe_loading(obj); err = err ? : bpf_object__load_vmlinux_btf(obj, false); err = err ? : bpf_object__resolve_externs(obj, obj->kconfig); @@ -10122,7 +10232,7 @@ static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const ch err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps); } - bpf_object_post_load_cleanup(obj); + bpf_object_post_load_cleanup(obj, false); obj->state = OBJ_LOADED; /* doesn't matter if successfully or not */ if (err) { @@ -10590,7 +10700,7 @@ void bpf_object__close(struct bpf_object *obj) * bpf_object__load(), we need to clean up stuff that is normally * cleaned up at the end of loading step */ - bpf_object_post_load_cleanup(obj); + bpf_object_post_load_cleanup(obj, true); usdt_manager_free(obj->usdt_man); obj->usdt_man = NULL; @@ -10599,7 +10709,6 @@ void bpf_object__close(struct bpf_object *obj) bpf_object__elf_finish(obj); bpf_object_unload(obj); btf__free(obj->btf); - btf__free(obj->btf_vmlinux); btf_ext__free(obj->btf_ext); for (i = 0; i < obj->nr_maps; i++) @@ -10925,7 +11034,13 @@ __u32 bpf_program__log_level(const struct bpf_program *prog) int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level) { - if (prog->obj->state >= OBJ_LOADED) + /* + * Here and below, OBJ_LOADED is not a relevant restriction for + * manual programs as those can load after the bpf object is + * already loaded. + */ + if (prog->obj->state >= OBJ_LOADED && + prog->load_strategy != BPF_PROG_LOAD_STRATEGY_MANUAL) return libbpf_err(-EBUSY); prog->log_level = log_level; @@ -10944,7 +11059,8 @@ int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log return libbpf_err(-EINVAL); if (log_size > UINT_MAX) return libbpf_err(-EINVAL); - if (prog->obj->state >= OBJ_LOADED) + if (prog->obj->state >= OBJ_LOADED && + prog->load_strategy != BPF_PROG_LOAD_STRATEGY_MANUAL) return libbpf_err(-EBUSY); prog->log_buf = log_buf; @@ -13730,7 +13846,7 @@ static int collect_func_ids_by_glob(const struct bpf_program *prog, const char * err = collect_btf_func_ids_by_glob(btf, pattern, ids); cleanup: - bpf_object_cleanup_btf(obj); + bpf_object_cleanup_btf(obj, false); return err; } @@ -16287,10 +16403,26 @@ void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s) int bpf_program__set_load_strategy(struct bpf_program *prog, enum bpf_prog_load_strategy strategy) { - if (prog->obj->state >= OBJ_PREPARED) + struct bpf_object *obj = prog->obj; + + if (obj->state >= OBJ_PREPARED) return libbpf_err(-EINVAL); - prog->load_strategy = strategy; + switch (strategy) { + case BPF_PROG_LOAD_STRATEGY_DISABLED: + case BPF_PROG_LOAD_STRATEGY_AUTO: + prog->load_strategy = strategy; + break; + case BPF_PROG_LOAD_STRATEGY_MANUAL: + if (prog_is_subprog(obj, prog)) + return libbpf_err(-EINVAL); + + prog->load_strategy = BPF_PROG_LOAD_STRATEGY_MANUAL; + break; + default: + return libbpf_err(-EINVAL); + } + return 0; } @@ -16298,3 +16430,32 @@ enum bpf_prog_load_strategy bpf_program__load_strategy(const struct bpf_program { return prog->load_strategy; } + +int +bpf_program__load(struct bpf_program *prog) +{ + int err; + struct bpf_object *obj = prog->obj; + + /* + * This function cannot be called before OBJ_PREPARED - + * Program's data is initialized on object prepare. + */ + if (obj->state < OBJ_PREPARED) + return libbpf_err(-EINVAL); + + if (prog_is_subprog(obj, prog) || prog->load_strategy != BPF_PROG_LOAD_STRATEGY_MANUAL) + return libbpf_err(-EINVAL); + + if (prog->fd >= 0) + return libbpf_err(-EBUSY); + + err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt, + obj->license, obj->kern_version, &prog->fd); + if (err) { + pr_warn("prog '%s': failed to load: %s\n", prog->name, errstr(err)); + return libbpf_err(err); + } + + return 0; +} diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h index a91e43cbba20..dd5380bc355e 100644 --- a/tools/lib/bpf/libbpf.h +++ b/tools/lib/bpf/libbpf.h @@ -473,6 +473,17 @@ LIBBPF_API int bpf_program__pin(struct bpf_program *prog, const char *path); * @return 0, on success; negative error code, otherwise */ LIBBPF_API int bpf_program__unpin(struct bpf_program *prog, const char *path); + +/** + * @brief **bpf_program__unload()** unloads a BPF program, closing its fd. + * + * If the program's load strategy is BPF_PROG_LOAD_STRATEGY_MANUAL, only the + * fd is closed and the program's data is retained so it can be reloaded + * later via bpf_program__load(). For any other load strategy, the program's + * data is also freed and it cannot be reloaded. + * + * @param prog BPF program to unload + */ LIBBPF_API void bpf_program__unload(struct bpf_program *prog); struct bpf_link; @@ -1055,8 +1066,39 @@ LIBBPF_API int bpf_program__clear_flags(struct bpf_program *prog, __u32 flags); * interactions. */ LIBBPF_API __u32 bpf_program__log_level(const struct bpf_program *prog); + +/** + * @brief **bpf_program__set_log_level()** sets the verifier log level used + * when the BPF program is loaded. + * + * Fails with -EBUSY once the enclosing bpf_object is loaded, unless the + * program's load strategy is BPF_PROG_LOAD_STRATEGY_MANUAL, as such a program + * can still be loaded afterwards with bpf_program__load(). + * + * @param prog BPF program to set the log level for + * @param log_level verifier log level, 0 requests no logging on the first + * load attempt + * @return 0, on success; negative error code, otherwise + */ LIBBPF_API int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level); LIBBPF_API const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_size); + +/** + * @brief **bpf_program__set_log_buf()** sets the buffer the verifier log of + * the BPF program is written to when it is loaded. + * + * Fails with -EBUSY once the enclosing bpf_object is loaded, unless the + * program's load strategy is BPF_PROG_LOAD_STRATEGY_MANUAL, as such a program + * can still be loaded afterwards with bpf_program__load(). + * + * @param prog BPF program to set the log buffer for + * @param log_buf buffer for the verifier log, or NULL to use the object's + * kernel_log_buf + * @param log_size size of @log_buf in bytes; must be 0 if @log_buf is NULL and + * non-zero otherwise + * @return 0, on success; -EINVAL if @log_buf and @log_size are inconsistent or + * @log_size exceeds UINT_MAX; -EBUSY as described above + */ LIBBPF_API int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size); LIBBPF_API struct bpf_func_info *bpf_program__func_info(const struct bpf_program *prog); @@ -2122,10 +2164,14 @@ LIBBPF_API int bpf_program__clone(struct bpf_program *prog, const struct bpf_pro * * - BPF_PROG_LOAD_STRATEGY_DISABLED: the program is not loaded. * - BPF_PROG_LOAD_STRATEGY_AUTO: the program is autoloaded when the bpf_object is loaded. + * - BPF_PROG_LOAD_STRATEGY_MANUAL: the program is loaded and attached manually. + * It is skipped by the skeleton's auto-attach, whatever its autoattach + * setting is. */ enum bpf_prog_load_strategy { BPF_PROG_LOAD_STRATEGY_DISABLED = 0, BPF_PROG_LOAD_STRATEGY_AUTO, + BPF_PROG_LOAD_STRATEGY_MANUAL, }; /** @@ -2150,6 +2196,16 @@ LIBBPF_API int bpf_program__set_load_strategy(struct bpf_program *prog, */ LIBBPF_API enum bpf_prog_load_strategy bpf_program__load_strategy(const struct bpf_program *prog); +/** + * @brief **bpf_program__load()** loads a MANUAL BPF program. The enclosing + * bpf_object must already be prepared. + * + * @param prog BPF program to load; non MANUAL programs, already loaded + * programs and subprograms are rejected. + * @return 0 on success; negative error code otherwise + */ +LIBBPF_API int bpf_program__load(struct bpf_program *prog); + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map index c06fc6b68efd..a45666f546fa 100644 --- a/tools/lib/bpf/libbpf.map +++ b/tools/lib/bpf/libbpf.map @@ -476,6 +476,7 @@ LIBBPF_1.8.0 { LIBBPF_1.9.0 { global: + bpf_program__load; bpf_program__load_strategy; bpf_program__set_load_strategy; } LIBBPF_1.8.0; -- 2.34.1