5.15-stable review patch. If anyone has any objections, please let me know. ------------------ From: Lorenzo Stoakes (ARM) [ Upstream commit 838bb60dc21553e32df90f2f3bac5b905376e7aa ] The MREMAP_DONTUNMAP feature is highly unusual in that it permits mremap() operations that keep the original VMA in place. Historically this has led to a lot of bugs where non-obvious interactions occur between existing mremap() operations and the original VMA. Fix another of these - self-merge. Self-merge occurs when a VMA is moved in front of or behind itself and the attributes of the VMA permit such a merge. Practically this can only happen for unfaulted anonymous VMAs due to the page offset equality requirement for merge: |------------| | | | v |...........||-----------||...........| | || unfaulted || | |...........||-----------||...........| ^ | | | |------------| This becomes problematic if the VMA is configured by the user to mlock-on-fault, i.e. the VMA_LOCKED_BIT, VMA_LOCKONFAULT_BIT VMA flags are set. MREMAP_DONTUNMAP clears mlock flags for the source VMA and maintains them for the destination VMA. Self-merge makes this impossible (there is only one VMA) and incorrectly clears the destination VMA's mlock flags. This causes a leak in mm->locked_vm as clearing this flag does not decrement the counter and the VMA no longer has VMA_LOCKED_BIT set so it is not decremented on unmap. Resolve this by simply disallowing a self-merge in this case - the source and destination VMAs are kept distinct and then are able to have distinct mlock() flags. Update to make the now-redundant self-merge check a VM_WARN_ON_ONCE() instead to guard against future regressions. Fixes: e346b3813067 ("mm/mremap: add MREMAP_DONTUNMAP to mremap()") Signed-off-by: Lorenzo Stoakes (ARM) Signed-off-by: Andrew Morton Reviewed-by: Pedro Falcato Acked-by: Kiryl Shutsemau (Meta) Reviewed-by: Jose A. Perez de Azpillaga Tested-by: Anirudh Srinivasan Cc: [ ljs - Converted to work with vma_merge(), tweaked commit msg. ] Signed-off-by: Lorenzo Stoakes (ARM) Signed-off-by: Sasha Levin --- include/linux/mm.h | 2 +- mm/mmap.c | 23 +++++++++++++++++++---- mm/mremap.c | 7 +++++-- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 071dd864a7b2b..bd69dd6c5e9d5 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2581,7 +2581,7 @@ extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *, extern void unlink_file_vma(struct vm_area_struct *); extern struct vm_area_struct *copy_vma(struct vm_area_struct **, unsigned long addr, unsigned long len, pgoff_t pgoff, - bool *need_rmap_locks); + bool *need_rmap_locks, bool keep_source); extern void exit_mmap(struct mm_struct *); static inline int check_data_rlimit(unsigned long rlim, diff --git a/mm/mmap.c b/mm/mmap.c index 52ab2f7f6632a..53fa4f5167dd0 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -3281,7 +3281,7 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) */ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, unsigned long addr, unsigned long len, pgoff_t pgoff, - bool *need_rmap_locks) + bool *need_rmap_locks, bool keep_source) { struct vm_area_struct *vma = *vmap; unsigned long vma_start = vma->vm_start; @@ -3289,6 +3289,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, struct vm_area_struct *new_vma, *prev; struct rb_node **rb_link, *rb_parent; bool faulted_in_anon_vma = true; + bool can_self_merge = false; /* * If anonymous vma has not yet been faulted, update new pgoff @@ -3297,13 +3298,27 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) { pgoff = addr >> PAGE_SHIFT; faulted_in_anon_vma = false; + can_self_merge = true; } if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent)) return NULL; /* should never get here */ - new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags, - vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma), - vma->vm_userfaultfd_ctx); + + /* + * If the original VMA is kept (MREMAP_DONTUNMAP), the source and + * destination VMA must be treated distinctly. + * + * A merge violates this, so in this case disallow a self-merge. A + * self-merge can only occur if the new range is immediately adjacent to + * the source VMA, so simply do not attempt to merge in that case. + */ + if (can_self_merge && keep_source && + (addr == vma->vm_end || vma->vm_start == addr + len)) + new_vma = NULL; + else + new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags, + vma->anon_vma, vma->vm_file, pgoff, + vma_policy(vma), vma->vm_userfaultfd_ctx); if (new_vma) { /* * Source vma may have been merged into new_vma diff --git a/mm/mremap.c b/mm/mremap.c index f7a5f1106699d..5d8b8309ae2ac 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -629,7 +629,7 @@ static unsigned long move_vma(struct vm_area_struct *vma, new_pgoff = vma->vm_pgoff + ((old_addr - vma->vm_start) >> PAGE_SHIFT); new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff, - &need_rmap_locks); + &need_rmap_locks, flags & MREMAP_DONTUNMAP); if (!new_vma) { if (unlikely(flags & MREMAP_DONTUNMAP && vm_flags & VM_ACCOUNT)) vm_unacct_memory(new_len >> PAGE_SHIFT); @@ -686,6 +686,9 @@ static unsigned long move_vma(struct vm_area_struct *vma, untrack_pfn_moved(vma); if (unlikely(!err && (flags & MREMAP_DONTUNMAP))) { + /* Self-merge is disallowed. */ + VM_WARN_ON_ONCE(new_vma == vma); + /* We always clear VM_LOCKED[ONFAULT] on the old vma */ vma->vm_flags &= VM_LOCKED_CLEAR_MASK; @@ -693,7 +696,7 @@ static unsigned long move_vma(struct vm_area_struct *vma, * anon_vma links of the old vma is no longer needed after its page * table has been moved. */ - if (new_vma != vma && vma->vm_start == old_addr && + if (vma->vm_start == old_addr && vma->vm_end == (old_addr + old_len)) { const pgoff_t pgoff_unfaulted = vma->vm_start >> PAGE_SHIFT; -- 2.53.0