Each pass of the PTE loop validates the entry, batches the PTEs that map one large folio, decides whether a partly mapped large folio must be split, and ages or isolates the folio. The split path also drops and retakes the PTE lock in the middle of that sequence. Move the per-batch work into madvise_lru_pte_batch_locked(), which runs under the PTE lock and returns a folio that needs splitting locked and referenced. The folio lookup goes into madvise_lru_pte_folio() and the MADV_COLD young-bit clearing into madvise_cold_ptes(). The loop drops the lock, splits the folio and maps the PTE table again at the same address. No functional change intended. Signed-off-by: Gregory Price (Meta) --- mm/madvise.c | 151 +++++++++++++++++++++++++++++---------------------- 1 file changed, 86 insertions(+), 65 deletions(-) diff --git a/mm/madvise.c b/mm/madvise.c index a9e5a3e2fab4..0c0a840102eb 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -407,6 +407,77 @@ static int madvise_lru_split_folio(struct folio *folio) return err; } +static void madvise_cold_ptes(struct mmu_gather *tlb, + struct vm_area_struct *vma, pte_t *pte, unsigned long addr, + int nr, pte_t ptent) +{ + if (!pte_young(ptent)) + return; + + clear_young_dirty_ptes(vma, addr, pte, nr, CYDP_CLEAR_YOUNG); + tlb_remove_tlb_entries(tlb, pte, nr, addr); +} + +/* Return the normal folio mapped by a present PTE, or NULL. */ +static struct folio *madvise_lru_pte_folio(struct vm_area_struct *vma, + unsigned long addr, pte_t ptent) +{ + struct folio *folio; + + if (pte_none(ptent) || !pte_present(ptent)) + return NULL; + + folio = vm_normal_folio(vma, addr, ptent); + if (!folio || folio_is_zone_device(folio)) + return NULL; + return folio; +} + +/* Return a split candidate locked and referenced for use after the PTL drop. */ +static struct folio *madvise_lru_pte_batch_locked(pte_t *pte, + unsigned long addr, unsigned long end, struct mm_walk *walk, + struct list_head *folio_list, bool pageout_anon_only, int *nr) +{ + const struct madvise_walk_private *private = walk->private; + struct vm_area_struct *vma = walk->vma; + pte_t ptent = ptep_get(pte); + struct folio *folio; + + *nr = 1; + folio = madvise_lru_pte_folio(vma, addr, ptent); + if (!folio) + return NULL; + + /* Split a large folio that the range only partly maps. */ + if (folio_test_large(folio)) { + *nr = madvise_folio_pte_batch(addr, end, folio, pte, &ptent); + if (*nr < folio_nr_pages(folio)) { + if (madvise_lru_folio_is_filtered(folio, pageout_anon_only)) + return NULL; + if (!folio_trylock(folio)) + return NULL; + folio_get(folio); + return folio; + } + } + + /* + * Skip non-LRU folios and folios mapped elsewhere. A large folio here + * is fully mapped, so it is exclusive if its mapcount equals its page + * count. + */ + if (!folio_test_lru(folio) || + folio_mapcount(folio) != folio_nr_pages(folio)) + return NULL; + if (pageout_anon_only && !folio_test_anon(folio)) + return NULL; + + if (!private->pageout) + madvise_cold_ptes(private->tlb, vma, pte, addr, *nr, ptent); + madvise_lru_folio(folio, private->pageout, folio_list); + return NULL; +} + #ifdef CONFIG_TRANSPARENT_HUGEPAGE static void madvise_cold_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, pmd_t orig_pmd) @@ -502,7 +573,7 @@ static int madvise_lru_pmd_entry(pmd_t *pmd, unsigned long addr, bool pageout = private->pageout; struct mm_struct *mm = tlb->mm; struct vm_area_struct *vma = walk->vma; - pte_t *start_pte, *pte, ptent; + pte_t *start_pte, *pte; spinlock_t *ptl; struct folio *folio = NULL; LIST_HEAD(folio_list); @@ -526,9 +597,6 @@ static int madvise_lru_pmd_entry(pmd_t *pmd, unsigned long addr, flush_tlb_batched_pending(mm); lazy_mmu_mode_enable(); for (; addr < end; pte += nr, addr += nr * PAGE_SIZE) { - nr = 1; - ptent = ptep_get(pte); - if (++batch_count == SWAP_CLUSTER_MAX) { batch_count = 0; if (need_resched()) { @@ -539,69 +607,22 @@ static int madvise_lru_pmd_entry(pmd_t *pmd, unsigned long addr, } } - if (pte_none(ptent)) - continue; - - if (!pte_present(ptent)) - continue; - - folio = vm_normal_folio(vma, addr, ptent); - if (!folio || folio_is_zone_device(folio)) - continue; - - /* - * If we encounter a large folio, only split it if it is not - * fully mapped within the range we are operating on. Otherwise - * leave it as is so that it can be swapped out whole. If we - * fail to split a folio, leave it in place and advance to the - * next pte in the range. - */ - if (folio_test_large(folio)) { - nr = madvise_folio_pte_batch(addr, end, folio, pte, &ptent); - if (nr < folio_nr_pages(folio)) { - int err; - - if (madvise_lru_folio_is_filtered(folio, pageout_anon_only)) - continue; - if (!folio_trylock(folio)) - continue; - folio_get(folio); - lazy_mmu_mode_disable(); - pte_unmap_unlock(start_pte, ptl); - start_pte = NULL; - err = madvise_lru_split_folio(folio); - start_pte = pte = - pte_offset_map_lock(mm, pmd, addr, &ptl); - if (!start_pte) - break; - flush_tlb_batched_pending(mm); - lazy_mmu_mode_enable(); - if (!err) - nr = 0; - continue; - } - } - - /* - * Do not interfere with other mappings of this folio and - * non-LRU folio. If we have a large folio at this point, we - * know it is fully mapped so if its mapcount is the same as its - * number of pages, it must be exclusive. - */ - if (!folio_test_lru(folio) || - folio_mapcount(folio) != folio_nr_pages(folio)) + folio = madvise_lru_pte_batch_locked(pte, addr, end, walk, + &folio_list, pageout_anon_only, &nr); + if (!folio) continue; - if (pageout_anon_only && !folio_test_anon(folio)) - continue; - - if (!pageout && pte_young(ptent)) { - clear_young_dirty_ptes(vma, addr, pte, nr, - CYDP_CLEAR_YOUNG); - tlb_remove_tlb_entries(tlb, pte, nr, addr); - } - - madvise_lru_folio(folio, pageout, &folio_list); + lazy_mmu_mode_disable(); + pte_unmap_unlock(start_pte, ptl); + start_pte = NULL; + if (!madvise_lru_split_folio(folio)) + nr = 0; + start_pte = pte_offset_map_lock(mm, pmd, addr, &ptl); + if (!start_pte) + break; + pte = start_pte; + flush_tlb_batched_pending(mm); + lazy_mmu_mode_enable(); } out: -- 2.56.0