From: Vernon Yang MADV_COLD and MADV_PAGEOUT hold the mmap_lock read lock while walking the target range. When a concurrent mmap_lock write lock is held, MADV_COLD/PAGEOUT must wait and execute serially, even for unrelated VMAs. MADV_FREE/DONTNEED have already achieved this goal using the per_vma read lock. With only simple modifications, MADV_COLD/PAGEOUT can also use the per_vma read lock to replace the mmap_lock read lock. Start 16 threads running mmap/unmap on separate VMAs, and benchmark the time taken by 10,000 madvise(MADV_COLD/PAGEOUT) calls on another VMA[1]. before after MADV_COLD 8.09 s 0.07 s MADV_PAGEOUT 11.68 s 0.40 s [1] https://github.com/vernon2gh/app_and_module/blob/main/madvise/app.c Signed-off-by: Vernon Yang --- mm/madvise.c | 52 ++++++++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/mm/madvise.c b/mm/madvise.c index cd9bb077072c..795a8a6fb237 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -566,10 +566,19 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, return 0; } -static const struct mm_walk_ops cold_walk_ops = { - .pmd_entry = madvise_cold_or_pageout_pte_range, - .walk_lock = PGWALK_RDLOCK, -}; +static inline enum page_walk_lock get_walk_lock(enum madvise_lock_mode mode) +{ + switch (mode) { + case MADVISE_VMA_READ_LOCK: + return PGWALK_VMA_RDLOCK_VERIFY; + case MADVISE_MMAP_READ_LOCK: + return PGWALK_RDLOCK; + default: + /* Other modes don't require fixing up the walk_lock */ + WARN_ON_ONCE(1); + return PGWALK_RDLOCK; + } +} static void madvise_cold_page_range(struct mmu_gather *tlb, struct madvise_behavior *madv_behavior) @@ -577,13 +586,17 @@ static void madvise_cold_page_range(struct mmu_gather *tlb, { struct vm_area_struct *vma = madv_behavior->vma; struct madvise_behavior_range *range = &madv_behavior->range; + struct mm_walk_ops walk_ops = { + .pmd_entry = madvise_cold_or_pageout_pte_range, + }; struct madvise_walk_private walk_private = { .pageout = false, .tlb = tlb, }; + walk_ops.walk_lock = get_walk_lock(madv_behavior->lock_mode); tlb_start_vma(tlb, vma); - walk_page_range_vma(vma, range->start, range->end, &cold_walk_ops, + walk_page_range_vma(vma, range->start, range->end, &walk_ops, &walk_private); tlb_end_vma(tlb, vma); } @@ -611,15 +624,20 @@ static long madvise_cold(struct madvise_behavior *madv_behavior) static void madvise_pageout_page_range(struct mmu_gather *tlb, struct vm_area_struct *vma, - struct madvise_behavior_range *range) + struct madvise_behavior *madv_behavior) { + struct madvise_behavior_range *range = &madv_behavior->range; + struct mm_walk_ops walk_ops = { + .pmd_entry = madvise_cold_or_pageout_pte_range, + }; struct madvise_walk_private walk_private = { .pageout = true, .tlb = tlb, }; + walk_ops.walk_lock = get_walk_lock(madv_behavior->lock_mode); tlb_start_vma(tlb, vma); - walk_page_range_vma(vma, range->start, range->end, &cold_walk_ops, + walk_page_range_vma(vma, range->start, range->end, &walk_ops, &walk_private); tlb_end_vma(tlb, vma); } @@ -644,7 +662,7 @@ static long madvise_pageout(struct madvise_behavior *madv_behavior) lru_add_drain(); tlb_gather_mmu(&tlb, madv_behavior->mm); - madvise_pageout_page_range(&tlb, vma, &madv_behavior->range); + madvise_pageout_page_range(&tlb, vma, madv_behavior); tlb_finish_mmu(&tlb); return 0; @@ -783,20 +801,6 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, return 0; } -static inline enum page_walk_lock get_walk_lock(enum madvise_lock_mode mode) -{ - switch (mode) { - case MADVISE_VMA_READ_LOCK: - return PGWALK_VMA_RDLOCK_VERIFY; - case MADVISE_MMAP_READ_LOCK: - return PGWALK_RDLOCK; - default: - /* Other modes don't require fixing up the walk_lock */ - WARN_ON_ONCE(1); - return PGWALK_RDLOCK; - } -} - static int madvise_free_single_vma(struct madvise_behavior *madv_behavior) { struct mm_struct *mm = madv_behavior->mm; @@ -1748,8 +1752,6 @@ static enum madvise_lock_mode get_lock_mode(struct madvise_behavior *madv_behavi switch (madv_behavior->behavior) { case MADV_REMOVE: case MADV_WILLNEED: - case MADV_COLD: - case MADV_PAGEOUT: case MADV_POPULATE_READ: case MADV_POPULATE_WRITE: case MADV_COLLAPSE: @@ -1759,6 +1761,8 @@ static enum madvise_lock_mode get_lock_mode(struct madvise_behavior *madv_behavi case MADV_DONTNEED: case MADV_DONTNEED_LOCKED: case MADV_FREE: + case MADV_COLD: + case MADV_PAGEOUT: return MADVISE_VMA_READ_LOCK; default: return MADVISE_MMAP_WRITE_LOCK; -- 2.53.0