From: Xu Xin (ZTE) Purpose ======= Let's add find_mergeable_vma_locked(), which is similar to find_tcp_vma(), using the universal per-VMA locking helper, so that we can avoid mmap_read_lock() to reduce contention. To be used in KSM code to replace find_mergeable_vma() with mmap_read_lock(), the helper find_mergeable_vma_locked() uses the universal per-VMA locking allowing us to lock a struct vm_area_struct without taking the process-wide mmap lock in read mode. Performance =========== As a test, I construct a testcase which follows the approach: Create one victim and several churner threads sharing one mm_struct; The victim registers a 32 MiB anonymous VM_MERGEABLE region containing 8192 pages totally:churners hammer mmap_lock via mmap/munmap repeatedly; ksmd merges the victim's pages; Compare latency baseline VS this per-VMA patch. Before patched After Patched 0 churner: 1.627 seconds 1.426 seconds 4 churners: 72.45 seconds 36.61 seconds In conclusion, when no mmap_lock contention (0 churner), there is little difference between the baseline kernel and the per-VMA optimized kernel; But under interference from 4 churner threads, the merge time of the per-VMA KSM-optimized kernel is significantly reduced by 50%. Signed-off-by: Xu Xin (ZTE) --- mm/ksm.c | 64 +++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 22 deletions(-) diff --git a/mm/ksm.c b/mm/ksm.c index dda105681d7f..1d85769ec7db 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -765,15 +765,36 @@ static bool vma_ksm_compatible(struct vm_area_struct *vma) return ksm_compatible(vma->vm_file, vma->flags); } -static struct vm_area_struct *find_mergeable_vma(struct mm_struct *mm, - unsigned long addr) +/** + * find_mergeable_vma_locked() - Find the VMA covering 'address' which is + * VM_MERGEABLE and read-lock it by per-VMA locks. Please use vma_end_read() + * to unlock vma after finishing reading the VMA (non-NULL). + * + * Return: If a VMA exists which spans @address, return that VMA, read-locked. + * If no VMA is mapped there or, very unlikely, a reference count overflow + * occurred, return NULL, and no read-locked. + * + * IMPORTANT: If a VMA exists but is not VM_MERGEABLE or has no anon_vma, + * this function releases the per-VMA read lock before returning NULL. + * Callers must NOT call vma_end_read() on a NULL return value. + */ +static struct vm_area_struct *find_mergeable_vma_locked(struct mm_struct *mm, + unsigned long address) { struct vm_area_struct *vma; + if (ksm_test_exit(mm)) return NULL; - vma = vma_lookup(mm, addr); - if (!vma || !(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma) + + vma = vma_start_read_unlocked(mm, address); + if (!vma) + return NULL; + + if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma) { + vma_end_read(vma); return NULL; + } + return vma; } @@ -799,11 +820,12 @@ static void break_cow(struct ksm_rmap_item *rmap_item) */ rmap_item->linear_page_index = 0; - mmap_read_lock(mm); - vma = find_mergeable_vma(mm, addr); - if (vma) - break_ksm(vma, addr, addr + PAGE_SIZE, PGWALK_RDLOCK); - mmap_read_unlock(mm); + vma = find_mergeable_vma_locked(mm, addr); + if (!vma) + return; + + break_ksm(vma, addr, addr + PAGE_SIZE, PGWALK_VMA_RDLOCK_VERIFY); + vma_end_read(vma); } static struct page *get_mergeable_page(struct ksm_rmap_item *rmap_item) @@ -814,13 +836,12 @@ static struct page *get_mergeable_page(struct ksm_rmap_item *rmap_item) struct page *page = NULL; struct folio *folio; struct folio_walk fw = { - .walk_lock = PGWALK_RDLOCK, + .walk_lock = PGWALK_VMA_RDLOCK_VERIFY, }; - mmap_read_lock(mm); - vma = find_mergeable_vma(mm, addr); + vma = find_mergeable_vma_locked(mm, addr); if (!vma) - goto out; + return NULL; folio = folio_walk_start(&fw, vma, addr, 0); if (folio) { @@ -831,12 +852,12 @@ static struct page *get_mergeable_page(struct ksm_rmap_item *rmap_item) } folio_walk_end(&fw, vma); } -out: + if (page) { flush_anon_page(vma, page, addr); flush_dcache_page(page); } - mmap_read_unlock(mm); + vma_end_read(vma); return page; } @@ -1568,14 +1589,14 @@ static int try_to_merge_with_zero_page(struct ksm_rmap_item *rmap_item, if (ksm_use_zero_pages && (rmap_item->oldchecksum == zero_checksum)) { struct vm_area_struct *vma; - mmap_read_lock(mm); - vma = find_mergeable_vma(mm, rmap_item->address); + vma = find_mergeable_vma_locked(mm, rmap_item->address); if (vma) { err = try_to_merge_one_page(vma, page, ZERO_PAGE(rmap_item->address)); trace_ksm_merge_one_page( page_to_pfn(ZERO_PAGE(rmap_item->address)), rmap_item, mm, err); + vma_end_read(vma); } else { /* * If the vma is out of date, we do not need to @@ -1583,7 +1604,6 @@ static int try_to_merge_with_zero_page(struct ksm_rmap_item *rmap_item, */ err = 0; } - mmap_read_unlock(mm); } return err; @@ -1602,10 +1622,9 @@ static int try_to_merge_with_ksm_page(struct ksm_rmap_item *rmap_item, struct vm_area_struct *vma; int err = -EFAULT; - mmap_read_lock(mm); - vma = find_mergeable_vma(mm, rmap_item->address); + vma = find_mergeable_vma_locked(mm, rmap_item->address); if (!vma) - goto out; + goto out_trace; err = try_to_merge_one_page(vma, page, kpage); if (err) @@ -1625,7 +1644,8 @@ static int try_to_merge_with_ksm_page(struct ksm_rmap_item *rmap_item, rmap_item->linear_page_index = linear_anon_page_index(vma, rmap_item->address); get_anon_vma(vma->anon_vma); out: - mmap_read_unlock(mm); + vma_end_read(vma); +out_trace: trace_ksm_merge_with_ksm_page(kpage, page_to_pfn(kpage ? kpage : page), rmap_item, mm, err); return err; -- 2.25.1