copy_remote_vm_str() gets the target address space from a struct task_struct. This does not work for an address space that exists but is not yet associated with a task_struct, such as the mm held by struct linux_binprm during exec. Add copy_remote_mm_str(), which operates directly on a struct mm_struct. Signed-off-by: Anastasios Papagiannis --- include/linux/mm.h | 2 ++ mm/memory.c | 33 +++++++++++++++++++++++++++++---- mm/nommu.c | 33 +++++++++++++++++++++++++++++---- 3 files changed, 60 insertions(+), 8 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 485df9c2dbdd..eede435bf4a3 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3222,6 +3222,8 @@ extern int access_remote_vm(struct mm_struct *mm, unsigned long addr, void *buf, int len, unsigned int gup_flags); #ifdef CONFIG_BPF_SYSCALL +extern int copy_remote_mm_str(struct mm_struct *mm, unsigned long addr, + void *buf, int len, unsigned int gup_flags); extern int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, void *buf, int len, unsigned int gup_flags); #endif diff --git a/mm/memory.c b/mm/memory.c index 6b8280cfc1db..4c5f0b629889 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -7141,12 +7141,15 @@ EXPORT_SYMBOL_GPL(access_process_vm); * Copy a string from another process's address space as given in mm. * If there is any error return -EFAULT. */ -static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr, +static int __copy_remote_mm_str(struct mm_struct *mm, unsigned long addr, void *buf, int len, unsigned int gup_flags) { void *old_buf = buf; int err = 0; + if (unlikely(len == 0)) + return 0; + *(char *)buf = '\0'; if (mmap_read_lock_killable(mm)) @@ -7218,6 +7221,27 @@ static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr, return buf - old_buf; } +/** + * copy_remote_mm_str - copy a string from a remote address space. + * @mm: the remote address space + * @addr: start address to read from + * @buf: destination buffer + * @len: number of bytes to copy + * @gup_flags: flags modifying lookup behaviour + * + * The caller must hold a reference on @mm. + * + * Return: number of bytes copied from @addr (source) to @buf (destination), + * not including the trailing NUL. If @len is zero, return 0 without accessing + * @buf. Otherwise, @buf is always NUL-terminated. On any error, return + * -EFAULT. + */ +int copy_remote_mm_str(struct mm_struct *mm, unsigned long addr, + void *buf, int len, unsigned int gup_flags) +{ + return __copy_remote_mm_str(mm, addr, buf, len, gup_flags); +} + /** * copy_remote_vm_str - copy a string from another process's address space. * @tsk: the task of the target address space @@ -7229,8 +7253,9 @@ static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr, * The caller must hold a reference on @mm. * * Return: number of bytes copied from @addr (source) to @buf (destination); - * not including the trailing NUL. Always guaranteed to leave NUL-terminated - * buffer. On any error, return -EFAULT. + * not including the trailing NUL. If @len is zero, return 0 without accessing + * @buf. Otherwise, @buf is always NUL-terminated. On any error, return + * -EFAULT. */ int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, void *buf, int len, unsigned int gup_flags) @@ -7247,7 +7272,7 @@ int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, return -EFAULT; } - ret = __copy_remote_vm_str(mm, addr, buf, len, gup_flags); + ret = copy_remote_mm_str(mm, addr, buf, len, gup_flags); mmput(mm); diff --git a/mm/nommu.c b/mm/nommu.c index ed3934bc2de4..94e3709e95fd 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -1716,13 +1716,16 @@ EXPORT_SYMBOL_GPL(access_process_vm); * Copy a string from another process's address space as given in mm. * If there is any error return -EFAULT. */ -static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr, +static int __copy_remote_mm_str(struct mm_struct *mm, unsigned long addr, void *buf, int len) { unsigned long addr_end; struct vm_area_struct *vma; int ret = -EFAULT; + if (unlikely(len == 0)) + return 0; + *(char *)buf = '\0'; if (mmap_read_lock_killable(mm)) @@ -1752,6 +1755,27 @@ static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr, return ret; } +/** + * copy_remote_mm_str - copy a string from a remote address space. + * @mm: the remote address space + * @addr: start address to read from + * @buf: destination buffer + * @len: number of bytes to copy + * @gup_flags: flags modifying lookup behaviour (unused) + * + * The caller must hold a reference on @mm. + * + * Return: number of bytes copied from @addr (source) to @buf (destination), + * not including the trailing NUL. If @len is zero, return 0 without accessing + * @buf. Otherwise, @buf is always NUL-terminated. On any error, return + * -EFAULT. + */ +int copy_remote_mm_str(struct mm_struct *mm, unsigned long addr, + void *buf, int len, unsigned int gup_flags) +{ + return __copy_remote_mm_str(mm, addr, buf, len); +} + /** * copy_remote_vm_str - copy a string from another process's address space. * @tsk: the task of the target address space @@ -1763,8 +1787,9 @@ static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr, * The caller must hold a reference on @mm. * * Return: number of bytes copied from @addr (source) to @buf (destination); - * not including the trailing NUL. Always guaranteed to leave NUL-terminated - * buffer. On any error, return -EFAULT. + * not including the trailing NUL. If @len is zero, return 0 without accessing + * @buf. Otherwise, @buf is always NUL-terminated. On any error, return + * -EFAULT. */ int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, void *buf, int len, unsigned int gup_flags) @@ -1781,7 +1806,7 @@ int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, return -EFAULT; } - ret = __copy_remote_vm_str(mm, addr, buf, len); + ret = copy_remote_mm_str(mm, addr, buf, len, gup_flags); mmput(mm); -- 2.55.0 Once mmput() drops the final reference to bprm->mm, the pointer must no longer remain accessible through struct linux_binprm. The successful exec path and the bprm initialization error path already clear bprm->mm when ownership is transferred or released. Do the same in free_bprm() before calling mmput(). This is required for BPF kfuncs where bprm->mm is either NULL or points to a live mm_struct to ensure safe access. Signed-off-by: Anastasios Papagiannis Reviewed-by: Sun Jian --- fs/exec.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fs/exec.c b/fs/exec.c index a14f28b15607..4b4d8f0627a7 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1453,9 +1453,12 @@ void bprm_drop_loader(struct linux_binprm *bprm) static void free_bprm(struct linux_binprm *bprm) { - if (bprm->mm) { + struct mm_struct *mm = bprm->mm; + + if (mm) { acct_arg_size(bprm, 0); - mmput(bprm->mm); + bprm->mm = NULL; + mmput(mm); } if (bprm->user_ns) put_user_ns(bprm->user_ns); -- 2.55.0 On CONFIG_MMU kernels, when security_bprm_check() runs, the argument and environment strings for the exec have been copied into bprm->mm. The new address space is not associated with a task_struct until exec_mmap(), so existing BPF user memory helpers cannot access it. Add bpf_copy_from_user_mm() and bpf_copy_from_user_mm_str() kfuncs. Both take a struct mm_struct pointer directly, allowing callers to access trusted address spaces that are not associated with a task_struct. bpf_copy_from_user_mm() has similar semantics to bpf_copy_from_user_task(). bpf_copy_from_user_mm_str() copies one NUL-terminated string and returns its size including the NUL terminator. It accepts BPF_F_PAD_ZEROS to clear unused destination bytes on success. Refactor bpf_copy_from_user_task() and bpf_copy_from_user_task_str() to acquire the task's mm with get_task_mm() and delegate to the corresponding mm-based implementations. No behavior change is intended for the existing task-based interfaces. Register both new kfuncs and mark them KF_SLEEPABLE because accessing a remote address space can fault. Signed-off-by: Anastasios Papagiannis --- kernel/bpf/helpers.c | 142 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 113 insertions(+), 29 deletions(-) diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index b3cc5c8fc875..d3c564437ad0 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -32,6 +32,10 @@ #include "../../lib/kstrtox.h" +__bpf_kfunc int bpf_copy_from_user_mm(void *dst, u32 dst__sz, + const void __user *unsafe_ptr__ign, + struct mm_struct *mm, u64 flags); + /* If kernel subsystem is allowing eBPF programs to call this function, * inside its own verifier_ops->get_func_proto() callback it should return * bpf_map_lookup_elem_proto, so that verifier can properly check the arguments @@ -682,22 +686,15 @@ const struct bpf_func_proto bpf_copy_from_user_proto = { BPF_CALL_5(bpf_copy_from_user_task, void *, dst, u32, size, const void __user *, user_ptr, struct task_struct *, tsk, u64, flags) { + struct mm_struct *mm; int ret; - /* flags is not used yet */ - if (unlikely(flags)) - return -EINVAL; - - if (unlikely(!size)) - return 0; - - ret = access_process_vm(tsk, (unsigned long)user_ptr, dst, size, 0); - if (ret == size) - return 0; + mm = get_task_mm(tsk); + ret = bpf_copy_from_user_mm(dst, size, user_ptr, mm, flags); + if (mm) + mmput(mm); - memset(dst, 0, size); - /* Return -EFAULT for partial read */ - return ret < 0 ? ret : -EFAULT; + return ret; } const struct bpf_func_proto bpf_copy_from_user_task_proto = { @@ -3658,6 +3655,100 @@ __bpf_kfunc int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void __user return ret + 1; } +/** + * bpf_copy_from_user_mm() - Copy data from an address space + * @dst: Destination address, in kernel space + * @dst__sz: Number of bytes to copy + * @unsafe_ptr__ign: Source address in the address space + * @mm: Address space to copy from + * @flags: Reserved for future use; must be zero + * + * Copies data from the user address space associated with @mm. The destination + * is zeroed if an attempted copy cannot be completed in full. Unsupported + * flags return -EINVAL without modifying @dst. + * + * Return: 0 on success, -EINVAL if @flags is non-zero, or -EFAULT if the copy + * fails or is partial. + */ +__bpf_kfunc int bpf_copy_from_user_mm(void *dst, u32 dst__sz, + const void __user *unsafe_ptr__ign, + struct mm_struct *mm, u64 flags) +{ + int ret; + + if (unlikely(flags)) + return -EINVAL; + + if (unlikely(!dst__sz)) + return 0; + + if (unlikely(!mm)) { + memset(dst, 0, dst__sz); + return -EFAULT; + } + + ret = access_remote_vm(mm, (unsigned long)unsafe_ptr__ign, + dst, dst__sz, 0); + if (ret == dst__sz) + return 0; + + memset(dst, 0, dst__sz); + return ret < 0 ? ret : -EFAULT; +} + +/** + * bpf_copy_from_user_mm_str() - Copy a string from an address space + * @dst: Destination address, in kernel space. This buffer must be + * at least @dst__sz bytes long + * @dst__sz: Maximum number of bytes to copy, including the trailing NUL + * @unsafe_ptr__ign: Source address in the address space + * @mm: Address space to copy from + * @flags: The only supported flag is BPF_F_PAD_ZEROS + * + * Copies a NUL-terminated string from the user address space associated with + * @mm. If the string is too long, @dst is still NUL-terminated unless @dst__sz + * is zero. + * + * If the flags are valid and BPF_F_PAD_ZEROS is set, the unused portion of + * @dst is cleared on success and all of @dst is cleared on a copy failure. + * Unsupported flags return -EINVAL without modifying @dst. + * + * Return: The number of copied bytes including the NUL terminator on success, + * or a negative error code on failure. + */ +__bpf_kfunc int bpf_copy_from_user_mm_str(void *dst, u32 dst__sz, + const void __user *unsafe_ptr__ign, + struct mm_struct *mm, u64 flags) +{ + int ret; + + if (unlikely(flags & ~BPF_F_PAD_ZEROS)) + return -EINVAL; + + if (unlikely(dst__sz == 0)) + return 0; + + if (unlikely(!mm)) { + if (flags & BPF_F_PAD_ZEROS) + memset(dst, 0, dst__sz); + else + *(char *)dst = '\0'; + return -EFAULT; + } + + ret = copy_remote_mm_str(mm, (unsigned long)unsafe_ptr__ign, dst, dst__sz, 0); + if (ret < 0) { + if (flags & BPF_F_PAD_ZEROS) + memset(dst, 0, dst__sz); + return ret; + } + + if (flags & BPF_F_PAD_ZEROS) + memset(dst + ret, 0, dst__sz - ret); + + return ret + 1; +} + /** * bpf_copy_from_user_task_str() - Copy a string from an task's address space * @dst: Destination address, in kernel space. This buffer must be @@ -3681,25 +3772,16 @@ __bpf_kfunc int bpf_copy_from_user_task_str(void *dst, u32 dst__sz, const void __user *unsafe_ptr__ign, struct task_struct *tsk, u64 flags) { + struct mm_struct *mm; int ret; - if (unlikely(flags & ~BPF_F_PAD_ZEROS)) - return -EINVAL; - - if (unlikely(dst__sz == 0)) - return 0; + mm = get_task_mm(tsk); + ret = bpf_copy_from_user_mm_str(dst, dst__sz, unsafe_ptr__ign, + mm, flags); + if (mm) + mmput(mm); - ret = copy_remote_vm_str(tsk, (unsigned long)unsafe_ptr__ign, dst, dst__sz, 0); - if (ret < 0) { - if (flags & BPF_F_PAD_ZEROS) - memset(dst, 0, dst__sz); - return ret; - } - - if (flags & BPF_F_PAD_ZEROS) - memset(dst + ret, 0, dst__sz - ret); - - return ret + 1; + return ret; } /* Keep unsigned long in prototype so that kfunc is usable when emitted to @@ -4924,6 +5006,8 @@ BTF_ID_FLAGS(func, bpf_iter_bits_new, KF_ITER_NEW) BTF_ID_FLAGS(func, bpf_iter_bits_next, KF_ITER_NEXT | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_iter_bits_destroy, KF_ITER_DESTROY) BTF_ID_FLAGS(func, bpf_copy_from_user_str, KF_SLEEPABLE) +BTF_ID_FLAGS(func, bpf_copy_from_user_mm, KF_SLEEPABLE) +BTF_ID_FLAGS(func, bpf_copy_from_user_mm_str, KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_copy_from_user_task_str, KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_get_kmem_cache) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW | KF_SLEEPABLE) -- 2.55.0 Currently, a trusted-or-null pointer (i.e. PTR_TO_BTF_ID|PTR_TRUSTED|PTR_MAYBE_NULL) has to be checked for NULL before it can be dereferenced. Marking a field from PTR_TO_BTF_ID typing to trusted-or-null can reject programs that previously dereferenced the pointer directly. This is useful as we need to mark new fields as trusted in order to pass those as arguments to kfuncs. Allow reads through pointers marked as PTR_TO_BTF_ID|PTR_TRUSTED|PTR_MAYBE_NULL without an explicit NULL check. Treat these pointers as potentially faulting so the reads happen through BPF_PROBE_MEM. If a read produces another BTF pointer, clear its trusted flags and mark it as PTR_UNTRUSTED. This applies only to reads. Other cases still require an explicit NULL check. After such a check, the pointer retains PTR_TRUSTED and can be used normally. The unchecked read path has two consequences: 1. It uses BPF_PROBE_MEM, which is slower than a normal load. An explicit NULL check refines the pointer to PTR_TRUSTED and allows a normal load. 2. A faulting read returns zero, which is indistinguishable from a legitimately zero-valued field. Programs that need to distinguish those cases must check the pointer before reading the field. The next patch updates current tests and also introduces more checks to ensure this change does not break anything. Signed-off-by: Anastasios Papagiannis --- include/linux/bpf_verifier.h | 9 ++++++++- kernel/bpf/verifier.c | 12 +++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index ae9f606539f4..d3d7254cc1f6 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1326,6 +1326,11 @@ static inline bool bpf_is_ptr_to_mem_or_btf_id(enum bpf_reg_type type) } } +static inline bool bpf_is_trusted_or_null_btf_ptr(enum bpf_reg_type type) +{ + return type == (PTR_TO_BTF_ID | PTR_TRUSTED | PTR_MAYBE_NULL); +} + static inline bool bpf_may_fault_on_deref(enum bpf_reg_type type) { /* @@ -1333,7 +1338,9 @@ static inline bool bpf_may_fault_on_deref(enum bpf_reg_type type) * protection, that is, the ones bpf_convert_ctx_accesses() has to * turn a BPF_LDX into a BPF_PROBE_MEM one for. */ - return type == PTR_TO_BTF_ID || (type_flag(type) & PTR_UNTRUSTED); + return type == PTR_TO_BTF_ID || + (type_flag(type) & PTR_UNTRUSTED) || + bpf_is_trusted_or_null_btf_ptr(type); } static inline bool bpf_prog_has_arena_ctx_arg(const struct bpf_prog *prog) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 8f585ceb2cd5..386401fe051d 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6116,6 +6116,15 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, if (ret != PTR_TO_BTF_ID) { /* just mark; */ + } else if (bpf_is_trusted_or_null_btf_ptr(reg->type)) { + /* + * An unchecked load through a trusted-or-NULL pointer is + * fault-protected. Any pointer derived from that load must be + * untrusted, as a fault produces a NULL value. + */ + clear_trusted_flags(&flag); + flag |= PTR_UNTRUSTED; + } else if (type_flag(reg->type) & PTR_UNTRUSTED) { /* If this is an untrusted pointer, all pointers formed by walking it * also inherit the untrusted flag. @@ -6591,7 +6600,8 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b if (!err && t == BPF_READ && value_regno >= 0) mark_reg_unknown(env, regs, value_regno); } else if (base_type(reg->type) == PTR_TO_BTF_ID && - !type_may_be_null(reg->type)) { + (!type_may_be_null(reg->type) || + (t == BPF_READ && bpf_is_trusted_or_null_btf_ptr(reg->type)))) { err = check_ptr_to_btf_access(env, regs, reg, argno, off, size, t, value_regno); } else if (reg->type == CONST_PTR_TO_MAP) { -- 2.55.0 Update verifier tests that expected an unchecked trusted-or-null BTF pointer dereference to fail. Cover scalar reads and chained reads through BTF and memory pointers. Add a runtime test which verifies that non-NULL reads return the field value and NULL reads return zero. Verify that pointer arithmetic, stores, atomic RMW operations, and BPF_LOAD_ACQ accesses remain prohibited. Assert that a BTF pointer derived from an unchecked trusted-or-null load is PTR_UNTRUSTED. Verify that attempting to NULL-check the derived pointer remains rejected and that it cannot be passed to a kfunc requiring an RCU pointer. Keep the existing NULL-check tests to verify that an explicit check of the original pointer recovers normal trusted pointer behavior. Signed-off-by: Anastasios Papagiannis --- .../selftests/bpf/prog_tests/bpf_iter.c | 6 +- .../prog_tests/test_struct_ops_maybe_null.c | 13 ++-- .../bpf/prog_tests/tp_btf_nullable.c | 28 +++++++ .../selftests/bpf/progs/raw_tp_null_fail.c | 75 ++++++++++++++++++- .../bpf/progs/test_tp_btf_nullable.c | 45 ++++++++++- .../bpf/progs/test_tp_btf_nullable_runtime.c | 35 +++++++++ .../selftests/bpf/progs/verifier_lsm.c | 18 ++++- .../selftests/bpf/progs/verifier_vfs_accept.c | 14 ++++ .../selftests/bpf/progs/verifier_vfs_reject.c | 14 ---- .../selftests/bpf/test_kmods/bpf_testmod.c | 1 + .../sched_ext/maybe_null_fail_dsp.bpf.c | 5 +- 11 files changed, 219 insertions(+), 35 deletions(-) create mode 100644 tools/testing/selftests/bpf/progs/test_tp_btf_nullable_runtime.c diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_iter.c b/tools/testing/selftests/bpf/prog_tests/bpf_iter.c index c69080ca14f5..99a16a1add70 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_iter.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_iter.c @@ -39,10 +39,10 @@ static void test_btf_id_or_null(void) struct bpf_iter_test_kern3 *skel; skel = bpf_iter_test_kern3__open_and_load(); - if (!ASSERT_ERR_PTR(skel, "bpf_iter_test_kern3__open_and_load")) { - bpf_iter_test_kern3__destroy(skel); + if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern3__open_and_load")) return; - } + + bpf_iter_test_kern3__destroy(skel); } static void do_dummy_read_opts(struct bpf_program *prog, struct bpf_iter_attach_opts *opts) diff --git a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_maybe_null.c b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_maybe_null.c index 01dc2613c8a5..45af6f00ad90 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_maybe_null.c +++ b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_maybe_null.c @@ -19,15 +19,16 @@ static void maybe_null(void) struct_ops_maybe_null__destroy(skel); } -/* Test that the verifier rejects a program that access a nullable pointer - * without a check beforehand. +/* + * Test that the verifier accepts a fault-protected read through a nullable + * trusted pointer without an explicit NULL check. */ -static void maybe_null_fail(void) +static void maybe_null_no_check(void) { struct struct_ops_maybe_null_fail *skel; skel = struct_ops_maybe_null_fail__open_and_load(); - if (ASSERT_ERR_PTR(skel, "struct_ops_module_fail__open_and_load")) + if (!ASSERT_OK_PTR(skel, "struct_ops_maybe_null_fail__open_and_load")) return; struct_ops_maybe_null_fail__destroy(skel); @@ -41,6 +42,6 @@ void test_struct_ops_maybe_null(void) */ if (test__start_subtest("maybe_null")) maybe_null(); - if (test__start_subtest("maybe_null_fail")) - maybe_null_fail(); + if (test__start_subtest("maybe_null_no_check")) + maybe_null_no_check(); } diff --git a/tools/testing/selftests/bpf/prog_tests/tp_btf_nullable.c b/tools/testing/selftests/bpf/prog_tests/tp_btf_nullable.c index accc42e01f8a..825fe7a92d74 100644 --- a/tools/testing/selftests/bpf/prog_tests/tp_btf_nullable.c +++ b/tools/testing/selftests/bpf/prog_tests/tp_btf_nullable.c @@ -2,6 +2,31 @@ #include #include "test_tp_btf_nullable.skel.h" +#include "test_tp_btf_nullable_runtime.skel.h" + +static void test_nullable_runtime(void) +{ + struct test_tp_btf_nullable_runtime *skel; + + skel = test_tp_btf_nullable_runtime__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + skel->bss->monitored_tid = sys_gettid(); + + if (!ASSERT_OK(test_tp_btf_nullable_runtime__attach(skel), "attach")) + goto out; + + if (!ASSERT_OK(trigger_module_test_read(2), "trigger")) + goto out; + + ASSERT_EQ(skel->bss->calls, 2, "calls"); + ASSERT_EQ(skel->bss->nonnull_len, 2, "nonnull_len"); + ASSERT_EQ(skel->bss->null_len, 0, "null_len"); + +out: + test_tp_btf_nullable_runtime__destroy(skel); +} void test_tp_btf_nullable(void) { @@ -11,4 +36,7 @@ void test_tp_btf_nullable(void) } RUN_TESTS(test_tp_btf_nullable); + + if (test__start_subtest("runtime")) + test_nullable_runtime(); } diff --git a/tools/testing/selftests/bpf/progs/raw_tp_null_fail.c b/tools/testing/selftests/bpf/progs/raw_tp_null_fail.c index 0d58114a4955..1ce81bcf1626 100644 --- a/tools/testing/selftests/bpf/progs/raw_tp_null_fail.c +++ b/tools/testing/selftests/bpf/progs/raw_tp_null_fail.c @@ -2,23 +2,90 @@ /* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */ #include +#include #include #include "bpf_misc.h" char _license[] SEC("license") = "GPL"; -/* Ensure module parameter has PTR_MAYBE_NULL */ +extern struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; +extern void bpf_task_release(struct task_struct *p) __ksym; + +/* + * Ensure the module tracepoint argument is trusted-or-NULL while allowing + * a fault-protected read without an explicit NULL check. + */ SEC("tp_btf/bpf_testmod_test_raw_tp_null_tp") -__failure __msg("R1 invalid mem access 'trusted_ptr_or_null_'") +__success __log_level(2) +__msg("R1=trusted_ptr_or_null_sk_buff") int test_raw_tp_null_bpf_testmod_test_raw_tp_null_arg_1(void *ctx) { asm volatile("r1 = *(u64 *)(r1 +0); r1 = *(u64 *)(r1 +0);" ::: __clobber_all); return 0; } -/* Check NULL marking */ +/* + * Ensure sched_pi_setprio's second argument is trusted-or-NULL while allowing + * a fault-protected read without an explicit NULL check. + */ SEC("tp_btf/sched_pi_setprio") -__failure __msg("R1 invalid mem access 'trusted_ptr_or_null_'") +__success __log_level(2) +__msg("R1=trusted_ptr_or_null_task_struct") int test_raw_tp_null_sched_pi_setprio_arg_2(void *ctx) { asm volatile("r1 = *(u64 *)(r1 +8); r1 = *(u64 *)(r1 +0);" ::: __clobber_all); return 0; } + +SEC("tp_btf/sched_pi_setprio") +__failure __log_level(2) +__msg("R1=untrusted_ptr_task_struct") +__msg("R1 must be a rcu pointer") +int BPF_PROG(trusted_or_null_walk_is_untrusted, struct task_struct *task, + struct task_struct *pi_task) +{ + struct task_struct *parent, *acquired; + + parent = pi_task->real_parent; + acquired = bpf_task_acquire(parent); + if (acquired) + bpf_task_release(acquired); + return 0; +} + +SEC("tp_btf/sched_pi_setprio") +__failure __msg("R1 must be a rcu pointer") +int BPF_PROG(derived_ptr_null_check_does_not_restore_trust, + struct task_struct *task, struct task_struct *pi_task) +{ + struct task_struct *parent, *acquired; + + parent = pi_task->real_parent; + if (!parent) + return 0; + + acquired = bpf_task_acquire(parent); + if (acquired) + bpf_task_release(acquired); + + return 0; +} + +/* + * In contrast, checking the original trusted-or-NULL pointer removes + * PTR_MAYBE_NULL while retaining PTR_TRUSTED. + */ +SEC("tp_btf/sched_pi_setprio") +__success +int BPF_PROG(original_ptr_null_check_retains_trust, + struct task_struct *task, struct task_struct *pi_task) +{ + struct task_struct *acquired; + + if (!pi_task) + return 0; + + acquired = bpf_task_acquire(pi_task); + if (acquired) + bpf_task_release(acquired); + + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/test_tp_btf_nullable.c b/tools/testing/selftests/bpf/progs/test_tp_btf_nullable.c index cf0547a613ff..b7914224ba19 100644 --- a/tools/testing/selftests/bpf/progs/test_tp_btf_nullable.c +++ b/tools/testing/selftests/bpf/progs/test_tp_btf_nullable.c @@ -7,7 +7,7 @@ #include "bpf_misc.h" SEC("tp_btf/bpf_testmod_test_nullable_bare_tp") -__failure __msg("R1 invalid mem access 'trusted_ptr_or_null_'") +__success int BPF_PROG(handle_tp_btf_nullable_bare1, struct bpf_testmod_test_read_ctx *nullable_ctx) { return nullable_ctx->len; @@ -21,4 +21,47 @@ int BPF_PROG(handle_tp_btf_nullable_bare2, struct bpf_testmod_test_read_ctx *nul return 0; } +SEC("tp_btf/bpf_testmod_test_nullable_bare_tp") +__success +int BPF_PROG(handle_tp_btf_nullable_mem, struct bpf_testmod_test_read_ctx *nullable_ctx) +{ + return nullable_ctx->buf[0]; +} + +SEC("tp_btf/bpf_testmod_test_nullable_bare_tp") +__failure __msg("pointer arithmetic on trusted_ptr_or_null_ prohibited") +int BPF_PROG(handle_tp_btf_nullable_arith, struct bpf_testmod_test_read_ctx *nullable_ctx) +{ + asm volatile("%[ctx] += 1" : [ctx] "+r"(nullable_ctx)); + return nullable_ctx->len; +} + +SEC("tp_btf/bpf_testmod_test_nullable_bare_tp") +__failure __msg("invalid mem access 'trusted_ptr_or_null_'") +int BPF_PROG(handle_tp_btf_nullable_atomic_rmw, + struct bpf_testmod_test_read_ctx *nullable_ctx) +{ + asm volatile ("r1 = %[ctx];" + "w2 = 1;" + "lock *(u32 *)(r1 + %[len]) += w2;" + : + : [ctx] "r"(nullable_ctx), + __imm_const(len, + offsetof(struct bpf_testmod_test_read_ctx, + len)) + : "r1", "r2", "memory"); + return 0; +} + +#ifdef __BPF_FEATURE_LOAD_ACQ_STORE_REL +SEC("tp_btf/bpf_testmod_test_nullable_bare_tp") +__failure +__msg("BPF_ATOMIC loads from R{{[0-9]+}} trusted_ptr_or_null_") +int BPF_PROG(handle_tp_btf_nullable_load_acquire, + struct bpf_testmod_test_read_ctx *nullable_ctx) +{ + return __atomic_load_n(&nullable_ctx->len, __ATOMIC_ACQUIRE); +} +#endif + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_tp_btf_nullable_runtime.c b/tools/testing/selftests/bpf/progs/test_tp_btf_nullable_runtime.c new file mode 100644 index 000000000000..5c9c7f94040d --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_tp_btf_nullable_runtime.c @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" +#include +#include +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +int monitored_tid; +int calls; +__u64 nonnull_len; +__u64 null_len; + +SEC("tp_btf/bpf_testmod_test_nullable_bare_tp") +int BPF_PROG(handle_nullable_runtime, + struct bpf_testmod_test_read_ctx *nullable_ctx) +{ + __u32 tid = bpf_get_current_pid_tgid(); + __u64 len; + int call; + + if (tid != monitored_tid) + return 0; + + len = nullable_ctx->len; + call = calls++; + + if (call == 0) + nonnull_len = len; + else if (call == 1) + null_len = len; + + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/verifier_lsm.c b/tools/testing/selftests/bpf/progs/verifier_lsm.c index c724bf389f5c..fac133d90f5e 100644 --- a/tools/testing/selftests/bpf/progs/verifier_lsm.c +++ b/tools/testing/selftests/bpf/progs/verifier_lsm.c @@ -162,13 +162,13 @@ __naked int disabled_hook_test3(void *ctx) SEC("lsm/mmap_file") __description("not null checking nullable pointer in bpf_lsm_mmap_file") -__failure __msg("R1 invalid mem access 'trusted_ptr_or_null_'") +__success int BPF_PROG(no_null_check, struct file *file) { - struct inode *inode; + ino_t ino; - inode = file->f_inode; - __sink(inode); + ino = file->f_inode->i_ino; + __sink(ino); return 0; } @@ -188,6 +188,16 @@ int BPF_PROG(null_check, struct file *file) return 0; } +SEC("lsm/mmap_file") +__description("store through trusted-or-null file is rejected") +__failure +__msg("R{{[0-9]+}} invalid mem access 'trusted_ptr_or_null_'") +int BPF_PROG(store_through_trusted_or_null_file, struct file *file) +{ + file->f_flags = 0; + return 0; +} + SEC("lsm_cgroup/file_open") __description("sleepable lsm_cgroup program is rejected") __failure __msg("Program of this type cannot be sleepable") diff --git a/tools/testing/selftests/bpf/progs/verifier_vfs_accept.c b/tools/testing/selftests/bpf/progs/verifier_vfs_accept.c index 55398c04290a..17c1542cc7e1 100644 --- a/tools/testing/selftests/bpf/progs/verifier_vfs_accept.c +++ b/tools/testing/selftests/bpf/progs/verifier_vfs_accept.c @@ -100,4 +100,18 @@ int BPF_PROG(inode_rename, struct inode *old_dir, struct dentry *old_dentry, return 0; } +SEC("lsm.s/inode_rename") +__success __log_level(2) +__msg("R{{[0-9]+}}=trusted_ptr_or_null_inode") +int BPF_PROG(inode_rename_no_null_check, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) +{ + ino_t ino = new_dentry->d_inode->i_ino; + + if (ino == 0) + return -EACCES; + return 0; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c b/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c index 8f0c45421f89..2a0813258183 100644 --- a/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c +++ b/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c @@ -159,18 +159,4 @@ int BPF_PROG(path_d_path_kfunc_non_lsm, struct path *path, struct file *f) return 0; } -SEC("lsm.s/inode_rename") -__failure __msg("invalid mem access 'trusted_ptr_or_null_'") -int BPF_PROG(inode_rename, struct inode *old_dir, struct dentry *old_dentry, - struct inode *new_dir, struct dentry *new_dentry, - unsigned int flags) -{ - struct inode *inode = new_dentry->d_inode; - ino_t ino; - - ino = inode->i_ino; - if (ino == 0) - return -EACCES; - return 0; -} char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 2380b6cbdead..2da9c2464c25 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -651,6 +651,7 @@ bpf_testmod_test_read(struct file *file, struct kobject *kobj, if (bpf_testmod_loop_test(101) > 100) trace_bpf_testmod_test_read(current, &ctx); + trace_bpf_testmod_test_nullable_bare_tp(&ctx); trace_bpf_testmod_test_nullable_bare_tp(NULL); /* Magic number to enable writable tp */ diff --git a/tools/testing/selftests/sched_ext/maybe_null_fail_dsp.bpf.c b/tools/testing/selftests/sched_ext/maybe_null_fail_dsp.bpf.c index ec724d7b33d1..ecaa36355cae 100644 --- a/tools/testing/selftests/sched_ext/maybe_null_fail_dsp.bpf.c +++ b/tools/testing/selftests/sched_ext/maybe_null_fail_dsp.bpf.c @@ -7,14 +7,13 @@ char _license[] SEC("license") = "GPL"; -u64 vtime_test; - void BPF_STRUCT_OPS(maybe_null_running, struct task_struct *p) {} void BPF_STRUCT_OPS(maybe_null_fail_dispatch, s32 cpu, struct task_struct *p) { - vtime_test = p->scx.dsq_vtime; + /* Pointer arithmetic on a trusted-or-NULL pointer must be rejected. */ + asm volatile("%[p] += 0" : [p] "+r"(p)); } SEC(".struct_ops.link") -- 2.55.0 Mark linux_binprm->mm as a trusted-or-null nested pointer so BPF programs can pass it to kfuncs after a NULL check. The field is either NULL or points to a live mm_struct whenever BPF can access a linux_binprm. On successful exec, exec_mmap() installs the new address space before begin_new_exec() clears bprm->mm. The bprm_mm_init() error path clears the field before mmdrop(), and free_bprm() clears it before mmput(), as ensured by an earlier patch in this series. Signed-off-by: Anastasios Papagiannis Reviewed-by: Sun Jian --- kernel/bpf/verifier.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 386401fe051d..584c79326b9c 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5958,6 +5958,10 @@ BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct dentry) { struct inode *d_inode; }; +BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct linux_binprm) { + struct mm_struct *mm; +}; + BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket) { struct sock *sk; }; @@ -6012,6 +6016,7 @@ static bool type_is_trusted_or_null(struct bpf_verifier_env *env, { BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct dentry)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct linux_binprm)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct vm_area_struct)); return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, -- 2.55.0 Add a sleepable BPF LSM program attached to bprm_check_security to test bpf_copy_from_user_mm() and bpf_copy_from_user_mm_str() on CONFIG_MMU kernels. Starting at bprm->p, verify that bpf_copy_from_user_mm() can copy the contiguous NUL-separated argument and environment data. Then use bpf_copy_from_user_mm_str() to read each argument and environment string separately, advancing the offset by the length returned from each call. Skip the test on !CONFIG_MMU. In that configuration, exec argument and environment strings remain in bprm->page[] until the binary loader transfers them to the new process stack, so they are not accessible through bprm->mm at the bprm_check_security hook. Signed-off-by: Anastasios Papagiannis --- .../bpf/prog_tests/copy_from_user_bprm.c | 72 ++++++++++ .../selftests/bpf/progs/copy_from_user_bprm.c | 123 ++++++++++++++++++ 2 files changed, 195 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/copy_from_user_bprm.c create mode 100644 tools/testing/selftests/bpf/progs/copy_from_user_bprm.c diff --git a/tools/testing/selftests/bpf/prog_tests/copy_from_user_bprm.c b/tools/testing/selftests/bpf/prog_tests/copy_from_user_bprm.c new file mode 100644 index 000000000000..b2325b193576 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/copy_from_user_bprm.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include + +#include + +#include "copy_from_user_bprm.skel.h" + +void test_copy_from_user_bprm(void) +{ + char arg0[] = "first"; + char arg1[] = "second-argument"; + char env0[] = "SOME_ENV=a"; + char env1[] = "OTHER_ENV=something"; + struct copy_from_user_bprm *skel; + pid_t child; + int status; + + skel = copy_from_user_bprm__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + /* + * On !CONFIG_MMU, exec strings are held in bprm->page[] rather than + * being mapped in bprm->mm. + */ + if (!skel->kconfig->CONFIG_MMU) { + printf("%s:SKIP: test requires CONFIG_MMU\n", __func__); + test__skip(); + goto out; + } + + if (!ASSERT_OK(copy_from_user_bprm__attach(skel), "attach")) + goto out; + + child = fork(); + if (!ASSERT_GE(child, 0, "fork")) + goto out; + + if (!child) { + char *const argv[] = { arg0, arg1, NULL }; + char *const envp[] = { env0, env1, NULL }; + + skel->bss->monitored_pid = getpid(); + execve("/bin/true", argv, envp); + _exit(errno); + } + + if (!ASSERT_EQ(waitpid(child, &status, 0), child, "waitpid")) + goto out; + + if (ASSERT_TRUE(WIFEXITED(status), "child_exited")) + ASSERT_EQ(WEXITSTATUS(status), EPERM, "exec_errno"); + + ASSERT_EQ(skel->bss->bprm_argc, 2, "bprm_argc"); + ASSERT_EQ(skel->bss->bprm_envc, 2, "bprm_envc"); + ASSERT_EQ(skel->bss->data_len_match, 1, "data_len_match"); + ASSERT_EQ(skel->bss->invalid_flags_ret, -EINVAL, "invalid_flags_ret"); + ASSERT_EQ(skel->bss->copy_ret, 0, "copy_ret"); + ASSERT_EQ(skel->bss->str_arg0_ret, sizeof(arg0), "str_arg0_ret"); + ASSERT_EQ(skel->bss->str_arg1_ret, sizeof(arg1), "str_arg1_ret"); + ASSERT_EQ(skel->bss->str_env0_ret, sizeof(env0), "str_env0_ret"); + ASSERT_EQ(skel->bss->str_env1_ret, sizeof(env1), "str_env1_ret"); + ASSERT_EQ(skel->bss->data_match, 1, "data_match"); + ASSERT_EQ(skel->bss->str_args_match, 1, "str_args_match"); + ASSERT_EQ(skel->bss->str_envs_match, 1, "str_envs_match"); + +out: + copy_from_user_bprm__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/progs/copy_from_user_bprm.c b/tools/testing/selftests/bpf/progs/copy_from_user_bprm.c new file mode 100644 index 000000000000..b334a157419e --- /dev/null +++ b/tools/testing/selftests/bpf/progs/copy_from_user_bprm.c @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" + +#include +#include +#include +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +static const char expected_data[] = "first\0second-argument\0" + "SOME_ENV=a\0OTHER_ENV=something"; +static const char expected_arg0[] = "first"; +static const char expected_arg1[] = "second-argument"; +static const char expected_env0[] = "SOME_ENV=a"; +static const char expected_env1[] = "OTHER_ENV=something"; + +int monitored_pid; +int bprm_argc; +int bprm_envc; +int data_len_match; +int invalid_flags_ret; +int copy_ret; +int str_arg0_ret; +int str_arg1_ret; +int str_env0_ret; +int str_env1_ret; +int data_match; +int str_args_match; +int str_envs_match; + +extern bool CONFIG_MMU __kconfig __weak; + +extern int bpf_copy_from_user_mm(void *dst, u32 dst__sz, + const void *unsafe_ptr__ign, + struct mm_struct *mm, u64 flags) __ksym; + +extern int bpf_copy_from_user_mm_str(void *dst, u32 dst__sz, + const void *unsafe_ptr__ign, + struct mm_struct *mm, u64 flags) __ksym; + +SEC("lsm.s/bprm_check_security") +int BPF_PROG(check_exec_args, struct linux_binprm *bprm) +{ + u32 pid = bpf_get_current_pid_tgid() >> 32; + char data[sizeof(expected_data)] = {}; + struct mm_struct *mm; + char arg0[32] = {}; + char arg1[32] = {}; + char env0[32] = {}; + char env1[32] = {}; + u64 offset = 0; + u64 data_len; + + if (!CONFIG_MMU) + return 0; + + if (pid != monitored_pid) + return 0; + + mm = bprm->mm; + if (!mm) + return 0; + + bprm_argc = bprm->argc; + bprm_envc = bprm->envc; + + /* this is the total size of args and envs starting from bprm->p */ + data_len = bprm->exec - bprm->p; + data_len_match = data_len == sizeof(expected_data); + + invalid_flags_ret = bpf_copy_from_user_mm(data, + sizeof(data), (void *)bprm->p, mm, ~0ULL); + + copy_ret = bpf_copy_from_user_mm(data, sizeof(data), (void *)bprm->p, + mm, 0); + if (copy_ret) + return 0; + + data_match = + !__builtin_memcmp(data, expected_data, sizeof(expected_data)); + + /* arg0 is at bprm->p */ + str_arg0_ret = bpf_copy_from_user_mm_str(arg0, sizeof(arg0), + (void *)(bprm->p + offset), + mm, BPF_F_PAD_ZEROS); + if (str_arg0_ret != sizeof(expected_arg0)) + return 0; + offset += str_arg0_ret; + + /* arg1 is at bprm->p + sizeof(arg0) */ + str_arg1_ret = bpf_copy_from_user_mm_str(arg1, sizeof(arg1), + (void *)(bprm->p + offset), + mm, BPF_F_PAD_ZEROS); + if (str_arg1_ret != sizeof(expected_arg1)) + return 0; + offset += str_arg1_ret; + + /* env0 is at bprm->p + sizeof(arg0) + sizeof(arg1) */ + str_env0_ret = bpf_copy_from_user_mm_str(env0, sizeof(env0), + (void *)(bprm->p + offset), + mm, BPF_F_PAD_ZEROS); + if (str_env0_ret != sizeof(expected_env0)) + return 0; + offset += str_env0_ret; + + /* env1 is at bprm->p + sizeof(arg0) + sizeof(arg1) + sizeof(env0) */ + str_env1_ret = bpf_copy_from_user_mm_str(env1, sizeof(env1), + (void *)(bprm->p + offset), + mm, BPF_F_PAD_ZEROS); + if (str_env1_ret != sizeof(expected_env1)) + return 0; + + str_args_match = + !__builtin_memcmp(arg0, expected_arg0, sizeof(expected_arg0)) && + !__builtin_memcmp(arg1, expected_arg1, sizeof(expected_arg1)); + str_envs_match = + !__builtin_memcmp(env0, expected_env0, sizeof(expected_env0)) && + !__builtin_memcmp(env1, expected_env1, sizeof(expected_env1)); + + return data_match && str_args_match && str_envs_match ? -EPERM : 0; +} -- 2.55.0