From: Dave Hansen Previously, the per-VMA locking could fail in the face of writers which necessitate a fallback to mmap_lock. The new vma_start_read_unlocked() will wait for writers instead of failing. Use the new helper. Wait for writers. Remove the fallback to mmap_lock. Signed-off-by: Dave Hansen Reviewed-by: Alice Ryhl Acked-by: Lorenzo Stoakes (ARM) Cc: Andrew Morton Cc: Liam R. Howlett Cc: Vlastimil Babka Cc: Shakeel Butt Cc: linux-mm@kvack.org Cc: Greg Kroah-Hartman Cc: Arve Hjønnevåg Cc: Todd Kjos Cc: Christian Brauner Cc: Carlos Llamas Cc: Alice Ryhl Cc: David S. Miller Cc: David Ahern Cc: netdev@vger.kernel.org Signed-off-by: Suren Baghdasaryan --- drivers/android/binder/page_range.rs | 19 +++---------------- drivers/android/binder_alloc.c | 17 +++++------------ rust/kernel/mm.rs | 27 +++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 28 deletions(-) diff --git a/drivers/android/binder/page_range.rs b/drivers/android/binder/page_range.rs index e82a5523804f..f7ad88a0d806 100644 --- a/drivers/android/binder/page_range.rs +++ b/drivers/android/binder/page_range.rs @@ -439,22 +439,9 @@ unsafe fn use_page_slow(&self, i: usize) -> Result<()> { // workqueue. let mm = MmWithUser::into_mmput_async(self.mm.mmget_not_zero().ok_or(ESRCH)?); { - let vma_read; - let mmap_read; - let vma = if let Some(ret) = mm.lock_vma_under_rcu(vma_addr) { - vma_read = ret; - check_vma(&vma_read, self) - } else { - mmap_read = mm.mmap_read_lock(); - mmap_read - .vma_lookup(vma_addr) - .and_then(|vma| check_vma(vma, self)) - }; - - match vma { - Some(vma) => vma.vm_insert_page(user_page_addr, &new_page)?, - None => return Err(ESRCH), - } + let vma_read_guard = mm.vma_start_read_unlocked(vma_addr).ok_or(ESRCH)?; + let vma = check_vma(&vma_read_guard, self).ok_or(ESRCH)?; + vma.vm_insert_page(user_page_addr, &new_page)?; } let inner = self.lock.lock(); diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c index c13a588c37de..efa23f6b8360 100644 --- a/drivers/android/binder_alloc.c +++ b/drivers/android/binder_alloc.c @@ -259,21 +259,14 @@ static int binder_page_insert(struct binder_alloc *alloc, struct vm_area_struct *vma; int ret = -ESRCH; - /* attempt per-vma lock first */ - vma = lock_vma_under_rcu(mm, addr); - if (vma) { - if (binder_alloc_is_mapped(alloc)) - ret = vm_insert_page(vma, addr, page); - vma_end_read(vma); + vma = vma_start_read_unlocked(mm, addr); + if (!vma) return ret; - } - /* fall back to mmap_lock */ - mmap_read_lock(mm); - vma = vma_lookup(mm, addr); - if (vma && binder_alloc_is_mapped(alloc)) + if (binder_alloc_is_mapped(alloc)) ret = vm_insert_page(vma, addr, page); - mmap_read_unlock(mm); + + vma_end_read(vma); return ret; } diff --git a/rust/kernel/mm.rs b/rust/kernel/mm.rs index f4fa54616085..11466fb304df 100644 --- a/rust/kernel/mm.rs +++ b/rust/kernel/mm.rs @@ -186,6 +186,33 @@ pub fn lock_vma_under_rcu(&self, vma_addr: usize) -> Option> { }) } + /// Find the VMA covering 'address' and read-lock it. + /// + /// The fast path does not take mmap_lock. Waits for writers to finish if the + /// VMA is being modified by taking mmap_lock. + /// Use when mmap_lock is not held, otherwise use vma_start_read_locked(). + /// Nothing prevents VMAs being unmapped/mapped before or after the VMA is + /// looked up, if a stronger guarantee is required, take an mmap_lock. + /// + /// Return: If a VMA exists which spans @address, return that VMA, read-locked. + /// If no VMA is mapped there or, very unlikely, a reference count overflow + /// occurred, return NULL. + #[inline] + pub fn vma_start_read_unlocked(&self, vma_addr: usize) -> Option> { + // SAFETY: We may invoke `vma_start_read_unlocked` because we know this `mm` has non-zero + // `mm_users`. + let vma = unsafe { bindings::vma_start_read_unlocked(self.as_raw(), vma_addr) }; + if vma.is_null() { + return None; + } + Some(VmaReadGuard { + // SAFETY: If `vma_start_read_unlocked` returns a non-null ptr, then it points at a + // valid vma. The vma is stable for as long as the vma read lock is held. + vma: unsafe { VmaRef::from_raw(vma) }, + _nts: NotThreadSafe, + }) + } + /// Lock the mmap read lock. #[inline] pub fn mmap_read_lock(&self) -> MmapReadGuard<'_> { -- 2.55.0.691.gc56d675ccc-goog