A kernel function bpf_foo with KF_MAGIC_ARGS flag is expected to have two types in BTF: * `bpf_foo` with a function prototype that omits __magic arguments * `bpf_foo_impl` with a function prototype that matches kernel declaration, but doesn't have a ksym associated with its name In order to support magic kfuncs the verifier has to know how to resolve calls both of `bpf_foo` and `bpf_foo_impl` to the correct BTF function prototype and address. In add_kfunc_call() kfunc flags are inspected to detect a magic kfunc or its _impl, and then the address and func_proto are adjusted for the kfunc descriptor. In fetch_kfunc_meta() similar logic is used to fixup the contents of struct bpf_kfunc_call_arg_meta. In check_kfunc_call() reset the subreg_def of registers holding magic arguments to correctly track zero extensions. Signed-off-by: Ihor Solodrai --- include/linux/btf.h | 1 + kernel/bpf/verifier.c | 123 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 120 insertions(+), 4 deletions(-) diff --git a/include/linux/btf.h b/include/linux/btf.h index 9c64bc5e5789..3fe20514692f 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -79,6 +79,7 @@ #define KF_ARENA_RET (1 << 13) /* kfunc returns an arena pointer */ #define KF_ARENA_ARG1 (1 << 14) /* kfunc takes an arena pointer as its first argument */ #define KF_ARENA_ARG2 (1 << 15) /* kfunc takes an arena pointer as its second argument */ +#define KF_MAGIC_ARGS (1 << 16) /* kfunc signature is different from its BPF signature */ /* * Tag marking a kernel function as a kfunc. This is meant to minimize the diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index cb1b483be0fa..fcf0872b9e3d 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3263,17 +3263,68 @@ static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset) return btf_vmlinux ?: ERR_PTR(-ENOENT); } +/* + * magic_kfuncs is used as a list of (foo, foo_impl) pairs + */ +BTF_ID_LIST(magic_kfuncs) +BTF_ID_UNUSED +BTF_ID_LIST_END(magic_kfuncs) + +static s32 magic_kfunc_by_impl(s32 impl_func_id) +{ + int i; + + for (i = 1; i < BTF_ID_LIST_SIZE(magic_kfuncs); i += 2) { + if (magic_kfuncs[i] == impl_func_id) + return magic_kfuncs[i - 1]; + } + return -ENOENT; +} + +static s32 impl_by_magic_kfunc(s32 func_id) +{ + int i; + + for (i = 0; i < BTF_ID_LIST_SIZE(magic_kfuncs); i += 2) { + if (magic_kfuncs[i] == func_id) + return magic_kfuncs[i + 1]; + } + return -ENOENT; +} + +static const struct btf_type *find_magic_kfunc_proto(struct btf *desc_btf, s32 func_id) +{ + const struct btf_type *impl_func, *func_proto; + u32 impl_func_id; + + impl_func_id = impl_by_magic_kfunc(func_id); + if (impl_func_id < 0) + return NULL; + + impl_func = btf_type_by_id(desc_btf, impl_func_id); + if (!impl_func || !btf_type_is_func(impl_func)) + return NULL; + + func_proto = btf_type_by_id(desc_btf, impl_func->type); + if (!func_proto || !btf_type_is_func_proto(func_proto)) + return NULL; + + return func_proto; +} + static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset) { - const struct btf_type *func, *func_proto; + const struct btf_type *func, *func_proto, *tmp_func; struct bpf_kfunc_btf_tab *btf_tab; + const char *func_name, *tmp_name; struct btf_func_model func_model; struct bpf_kfunc_desc_tab *tab; struct bpf_prog_aux *prog_aux; struct bpf_kfunc_desc *desc; - const char *func_name; struct btf *desc_btf; unsigned long addr; + u32 *kfunc_flags; + s32 tmp_func_id; int err; prog_aux = env->prog->aux; @@ -3349,8 +3400,37 @@ static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset) return -EINVAL; } + kfunc_flags = btf_kfunc_flags(desc_btf, func_id, env->prog); func_name = btf_name_by_offset(desc_btf, func->name_off); addr = kallsyms_lookup_name(func_name); + + /* This may be an _impl kfunc with KF_MAGIC_ARGS counterpart */ + if (unlikely(!addr && !kfunc_flags)) { + tmp_func_id = magic_kfunc_by_impl(func_id); + if (tmp_func_id < 0) + return -EACCES; + tmp_func = btf_type_by_id(desc_btf, tmp_func_id); + if (!tmp_func || !btf_type_is_func(tmp_func)) + return -EACCES; + tmp_name = btf_name_by_offset(desc_btf, tmp_func->name_off); + addr = kallsyms_lookup_name(tmp_name); + } + + /* + * Note that kfunc_flags may be NULL at this point, which means that we couldn't find + * func_id in any relevant kfunc_id_set. This most likely indicates an invalid kfunc call. + * However we don't want to fail the verification here, because invalid calls may be + * eliminated as dead code later. + */ + if (unlikely(kfunc_flags && KF_MAGIC_ARGS & *kfunc_flags)) { + func_proto = find_magic_kfunc_proto(desc_btf, func_id); + if (!func_proto) { + verbose(env, "cannot find _impl proto for kernel function %s\n", + func_name); + return -EINVAL; + } + } + if (!addr) { verbose(env, "cannot find address for kernel function %s\n", func_name); @@ -12051,6 +12131,11 @@ static bool is_kfunc_arg_irq_flag(const struct btf *btf, const struct btf_param return btf_param_match_suffix(btf, arg, "__irq_flag"); } +static bool is_kfunc_arg_magic(const struct btf *btf, const struct btf_param *arg) +{ + return btf_param_match_suffix(btf, arg, "__magic"); +} + static bool is_kfunc_arg_prog(const struct btf *btf, const struct btf_param *arg) { return btf_param_match_suffix(btf, arg, "__prog"); @@ -13613,6 +13698,7 @@ static int fetch_kfunc_meta(struct bpf_verifier_env *env, u32 func_id, *kfunc_flags; const char *func_name; struct btf *desc_btf; + s32 tmp_func_id; if (kfunc_name) *kfunc_name = NULL; @@ -13632,10 +13718,28 @@ static int fetch_kfunc_meta(struct bpf_verifier_env *env, func_proto = btf_type_by_id(desc_btf, func->type); kfunc_flags = btf_kfunc_flags_if_allowed(desc_btf, func_id, env->prog); - if (!kfunc_flags) { - return -EACCES; + if (unlikely(!kfunc_flags)) { + /* + * An _impl kfunc with KF_MAGIC_ARGS counterpart + * does not have its own kfunc flags. + */ + tmp_func_id = magic_kfunc_by_impl(func_id); + if (tmp_func_id < 0) + return -EACCES; + kfunc_flags = btf_kfunc_flags_if_allowed(desc_btf, tmp_func_id, env->prog); + if (!kfunc_flags) + return -EACCES; + } else if (unlikely(KF_MAGIC_ARGS & *kfunc_flags)) { + /* + * An actual func_proto of a kfunc with KF_MAGIC_ARGS flag + * can be found through the corresponding _impl kfunc. + */ + func_proto = find_magic_kfunc_proto(desc_btf, func_id); } + if (!func_proto) + return -EACCES; + memset(meta, 0, sizeof(*meta)); meta->btf = desc_btf; meta->func_id = func_id; @@ -14189,6 +14293,17 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, for (i = 0; i < nargs; i++) { u32 regno = i + 1; + /* + * Magic arguments are set after main verification pass. + * For correct tracking of zero-extensions we have to reset subreg_def for such + * args. Otherwise mark_btf_func_reg_size() will be inspecting subreg_def of regs + * from an earlier (irrelevant) point in the program, which may lead to an error + * in opt_subreg_zext_lo32_rnd_hi32(). + */ + if (unlikely(KF_MAGIC_ARGS & meta.kfunc_flags + && is_kfunc_arg_magic(desc_btf, &args[i]))) + regs[regno].subreg_def = DEF_NOT_SUBREG; + t = btf_type_skip_modifiers(desc_btf, args[i].type, NULL); if (btf_type_is_ptr(t)) mark_btf_func_reg_size(env, regno, sizeof(void *)); -- 2.51.1