specialize_kfunc() currently updates the canonical kfunc descriptor in place. It is not currently possible to swtich different specializations of a kfunc per call site in the same program. In fact, specializations are order-dependent: Once a function is specialized, all subsequent call sites are specialized even if they wouldn't trigger specialization themselves. This is especially an issue for bpf_arena_alloc_pages() that is specialized into its non-sleepable for all call sites after a single non-sleepable one. Allow per-call site kfunc specialization for JITs that use near calls. Implement this by keeping two versions of the kfunc table, one with just the initial kfuncs and one with all valid specializations for the program. We currently assume 2 concurrent specializations for each kfunc. This is a conservative estimate, since most of them do not specialize at all. Signed-off-by: Emil Tsalapatis --- include/linux/bpf_verifier.h | 13 ++++--- kernel/bpf/verifier.c | 67 +++++++++++++++++++++++++++++++++--- 2 files changed, 71 insertions(+), 9 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 92f528c45605..e36936936418 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1714,6 +1714,8 @@ enum bpf_reg_arg_type { }; #define MAX_KFUNC_DESCS 256 +/* Each kfunc can have its canonical and one specialized call target. */ +#define MAX_KFUNC_CALL_DESCS (MAX_KFUNC_DESCS * 2) struct bpf_kfunc_desc { struct btf_func_model func_model; @@ -1726,12 +1728,15 @@ struct bpf_kfunc_desc { struct bpf_kfunc_desc_tab { u32 nr_descs; + u32 nr_base_descs; /* Sorted by func_id (BTF ID) and offset (fd_array offset) during - * verification. JITs do lookups by bpf_insn, where func_id may not be - * available, therefore at the end of verification do_misc_fixups() - * sorts this by imm and offset. + * verification. The first nr_base_descs entries are the canonical + * descriptors used for verifier lookups. Call specialization may append + * immutable descriptors for additional targets. Near-call JITs look up + * descriptors by imm and offset after do_misc_fixups() sorts the table. * - * Grown one entry at a time by bpf_add_kfunc_call(). + * Grown one entry at a time by bpf_add_kfunc_call() and during + * call specialization. */ struct bpf_kfunc_desc descs[]; }; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a7c9e2d8965d..5e7c589991e9 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2570,7 +2570,7 @@ find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset) struct bpf_kfunc_desc_tab *tab; tab = prog->aux->kfunc_tab; - return bsearch(&desc, tab->descs, tab->nr_descs, + return bsearch(&desc, tab->descs, tab->nr_base_descs, sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off); } @@ -2920,10 +2920,12 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) if (find_kfunc_desc(env->prog, func_id, offset)) return 0; - if (tab->nr_descs == MAX_KFUNC_DESCS) { + if (tab->nr_base_descs == MAX_KFUNC_DESCS) { verbose(env, "too many different kernel function calls\n"); return -E2BIG; } + if (WARN_ON_ONCE(tab->nr_descs != tab->nr_base_descs)) + return -EFAULT; err = fetch_kfunc_meta(env, func_id, offset, &kfunc); if (err) @@ -2981,7 +2983,8 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) desc->addr = addr; desc->func_model = func_model; tab->nr_descs++; - sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), + tab->nr_base_descs++; + sort(tab->descs, tab->nr_base_descs, sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off, NULL); return 0; } @@ -21299,6 +21302,40 @@ static int specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc return 0; } +static int add_kfunc_desc_target(struct bpf_verifier_env *env, + const struct bpf_kfunc_desc *target_desc) +{ + struct bpf_kfunc_desc desc = *target_desc; + struct bpf_kfunc_desc_tab *new_tab; + struct bpf_kfunc_desc_tab *tab; + struct bpf_prog_aux *prog_aux; + u32 i; + + prog_aux = env->prog->aux; + tab = prog_aux->kfunc_tab; + for (i = 0; i < tab->nr_descs; i++) { + if (tab->descs[i].func_id == desc.func_id && + tab->descs[i].offset == desc.offset && + tab->descs[i].addr == desc.addr) + return 0; + } + + if (tab->nr_descs == MAX_KFUNC_CALL_DESCS) { + verbose(env, "too many different kernel function call targets\n"); + return -E2BIG; + } + + new_tab = krealloc(tab, struct_size(tab, descs, tab->nr_descs + 1), + GFP_KERNEL_ACCOUNT); + if (!new_tab) + return -ENOMEM; + tab = new_tab; + prog_aux->kfunc_tab = tab; + + tab->descs[tab->nr_descs++] = desc; + return 0; +} + static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux, u16 struct_meta_reg, u16 node_offset_reg, @@ -21319,7 +21356,10 @@ static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux, int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_insn *insn_buf, int insn_idx, int *cnt) { + struct bpf_kfunc_desc desc_copy; struct bpf_kfunc_desc *desc; + unsigned long call_imm; + bool near_call; int err; if (!insn->imm) { @@ -21340,12 +21380,29 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return -EFAULT; } + near_call = !bpf_jit_supports_far_kfunc_call(); + if (near_call) { + desc_copy = *desc; + desc = &desc_copy; + } + err = specialize_kfunc(env, desc, insn_idx); if (err) return err; - if (!bpf_jit_supports_far_kfunc_call()) - insn->imm = BPF_CALL_IMM(desc->addr); + if (near_call) { + call_imm = BPF_CALL_IMM(desc->addr); + if ((unsigned long)(s32)call_imm != call_imm) { + verbose(env, "address of kernel func_id %u is out of range\n", + desc->func_id); + return -EINVAL; + } + insn->imm = call_imm; + + err = add_kfunc_desc_target(env, desc); + if (err) + return err; + } if (is_bpf_obj_new_kfunc(desc->func_id) || is_bpf_percpu_obj_new_kfunc(desc->func_id)) { struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; -- 2.52.0