From: Mykyta Yatsenko File dynptr reads may sleep when the requested folios are not in the page cache. To avoid sleeping in non-sleepable contexts while still supporting valid sleepable use, given that dynptrs are non-sleepable by default, enable sleeping only when bpf_dynptr_from_file() is invoked from a sleepable context. This change: * Introduces a sleepable constructor: bpf_dynptr_from_file_sleepable() * Detects whether the kfunc is called in a sleepable context and stores the result in bpf_insn_aux_data (kfunc_in_sleepable_ctx) * Rewrites bpf_dynptr_from_file() calls to the sleepable variant when kfunc_in_sleepable_ctx is set Signed-off-by: Mykyta Yatsenko --- include/linux/bpf.h | 3 +++ include/linux/bpf_verifier.h | 2 ++ kernel/bpf/helpers.c | 5 +++++ kernel/bpf/verifier.c | 12 +++++++++--- 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index bd70117b8e84..9da7460e078c 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -663,6 +663,9 @@ int map_check_no_btf(const struct bpf_map *map, bool bpf_map_meta_equal(const struct bpf_map *meta0, const struct bpf_map *meta1); +int bpf_dynptr_from_file_sleepable(struct file *file, u32 flags, + struct bpf_dynptr *ptr__uninit); + extern const struct bpf_map_ops bpf_map_offload_ops; /* bpf_type_flag contains a set of flags that are applicable to the values of diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 4c497e839526..6078d5e9b535 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -581,6 +581,8 @@ struct bpf_insn_aux_data { u32 scc; /* registers alive before this instruction. */ u16 live_regs_before; + /* kfunc is called in sleepable context */ + bool kfunc_in_sleepable_ctx; }; #define MAX_USED_MAPS 64 /* max number of maps accessed by one eBPF program */ diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 4bba516599c7..f452e22333fe 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -4288,6 +4288,11 @@ __bpf_kfunc int bpf_dynptr_from_file(struct file *file, u32 flags, struct bpf_dy return make_file_dynptr(file, flags, MAY_NOT_SLEEP, (struct bpf_dynptr_kern *)ptr__uninit); } +int bpf_dynptr_from_file_sleepable(struct file *file, u32 flags, struct bpf_dynptr *ptr__uninit) +{ + return make_file_dynptr(file, flags, MAY_SLEEP, (struct bpf_dynptr_kern *)ptr__uninit); +} + __bpf_kfunc int bpf_dynptr_file_discard(struct bpf_dynptr *dynptr) { struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)dynptr; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index aacefa3d0544..82762eab3f17 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3105,7 +3105,8 @@ struct bpf_kfunc_btf_tab { static unsigned long kfunc_call_imm(unsigned long func_addr, u32 func_id); -static void specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc); +static void specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc, + int insn_idx); static int kfunc_desc_cmp_by_id_off(const void *a, const void *b) { @@ -13833,6 +13834,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, insn_aux = &env->insn_aux_data[insn_idx]; insn_aux->is_iter_next = is_iter_next_kfunc(&meta); + insn_aux->kfunc_in_sleepable_ctx = in_sleepable(env); if (!insn->off && (insn->imm == special_kfunc_list[KF_bpf_res_spin_lock] || @@ -21832,7 +21834,8 @@ static unsigned long kfunc_call_imm(unsigned long func_addr, u32 func_id) } /* replace a generic kfunc with a specialized version if necessary */ -static void specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc) +static void specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc, + int insn_idx) { struct bpf_prog_aux *prog_aux = env->prog->aux; struct bpf_kfunc_desc_tab *tab = prog_aux->kfunc_tab; @@ -21872,6 +21875,9 @@ static void specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc } else if (func_id == special_kfunc_list[KF_bpf_remove_dentry_xattr]) { if (bpf_lsm_has_d_inode_locked(prog)) addr = (unsigned long)bpf_remove_dentry_xattr_locked; + } else if (func_id == special_kfunc_list[KF_bpf_dynptr_from_file]) { + if (env->insn_aux_data[insn_idx].kfunc_in_sleepable_ctx) + addr = (unsigned long)bpf_dynptr_from_file_sleepable; } if (!addr) /* Nothing to patch with */ @@ -21924,7 +21930,7 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return -EFAULT; } - specialize_kfunc(env, desc); + specialize_kfunc(env, desc, insn_idx); if (!bpf_jit_supports_far_kfunc_call()) insn->imm = BPF_CALL_IMM(desc->addr); -- 2.51.0