Refactor anonymous swapbacked folio unmap to ttu_anon_swapbacked_folio(). While at it, rename the remaining subpage variables to page; folios have pages, not subpages. No functional change intended. Signed-off-by: Dev Jain --- mm/rmap.c | 115 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 68 insertions(+), 47 deletions(-) diff --git a/mm/rmap.c b/mm/rmap.c index 4465c88ddb77b..2c910d4575981 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -2112,6 +2112,66 @@ static bool ttu_anon_lazyfree_folio(struct vm_area_struct *vma, return true; } +static pte_t swp_pte_prepare(swp_entry_t entry, pte_t old_pte, + bool anon_exclusive) +{ + pte_t swp_pte = swp_entry_to_pte(entry); + + if (anon_exclusive) + swp_pte = pte_swp_mkexclusive(swp_pte); + + if (likely(pte_present(old_pte))) { + if (pte_soft_dirty(old_pte)) + swp_pte = pte_swp_mksoft_dirty(swp_pte); + if (pte_uffd(old_pte)) + swp_pte = pte_swp_mkuffd(swp_pte); + } else { + /* Device-exclusive entry */ + if (pte_swp_soft_dirty(old_pte)) + swp_pte = pte_swp_mksoft_dirty(swp_pte); + if (pte_swp_uffd(old_pte)) + swp_pte = pte_swp_mkuffd(swp_pte); + } + + return swp_pte; +} + +static bool ttu_anon_swapbacked_folio(struct vm_area_struct *vma, + struct folio *folio, struct page *page, unsigned long address, + pte_t *ptep, pte_t pteval) +{ + const bool anon_exclusive = folio_test_anon(folio) && + PageAnonExclusive(page); + swp_entry_t entry = page_swap_entry(page); + struct mm_struct *mm = vma->vm_mm; + + if (folio_dup_swap(folio, page) < 0) + return false; + + /* + * arch_unmap_one() is expected to be a NOP on + * architectures where we could have PFN swap PTEs, + * so we'll not check/care. + */ + if (arch_unmap_one(mm, vma, address, pteval) < 0) { + folio_put_swap(folio, page); + return false; + } + + /* See folio_try_share_anon_rmap(): clear PTE first. */ + if (anon_exclusive && folio_try_share_anon_rmap_pte(folio, page)) { + folio_put_swap(folio, page); + return false; + } + + mm_prepare_for_swap_entries(mm); + dec_mm_counter(mm, MM_ANONPAGES); + inc_mm_counter(mm, MM_SWAPENTS); + set_pte_at(mm, address, ptep, + swp_pte_prepare(entry, pteval, anon_exclusive)); + return true; +} + /* * @arg: enum ttu_flags will be passed to this argument */ @@ -2120,9 +2180,9 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, { struct mm_struct *mm = vma->vm_mm; DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); - bool anon_exclusive, ret = true; + bool ret = true; pte_t pteval; - struct page *subpage; + struct page *page; struct mmu_notifier_range range; enum ttu_flags flags = (enum ttu_flags)(long)arg; unsigned long nr_pages = 1, end_addr; @@ -2233,9 +2293,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, pfn = softleaf_to_pfn(entry); } - subpage = folio_page(folio, pfn - folio_pfn(folio)); - anon_exclusive = folio_test_anon(folio) && - PageAnonExclusive(subpage); + page = folio_page(folio, pfn - folio_pfn(folio)); if (likely(pte_present(pteval))) { nr_pages = folio_unmap_pte_batch(folio, &pvmw, flags, pteval); @@ -2274,7 +2332,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, /* With TTU_HWPOISON, we only expect small folios here. */ if (folio_test_hwpoison(folio) && (flags & TTU_HWPOISON)) { - pteval = swp_entry_to_pte(make_hwpoison_entry(subpage)); + pteval = swp_entry_to_pte(make_hwpoison_entry(page)); dec_mm_counter(mm, mm_counter(folio)); set_pte_at(mm, address, pvmw.pte, pteval); } else if (likely(pte_present(pteval)) && pte_unused(pteval) && @@ -2291,8 +2349,6 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, */ dec_mm_counter(mm, mm_counter(folio)); } else if (folio_test_anon(folio)) { - swp_entry_t entry = page_swap_entry(subpage); - pte_t swp_pte; /* * Store the swap location in the pte. * See handle_pte_fault() ... @@ -2313,47 +2369,12 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, goto finish_unmap; } - if (folio_dup_swap(folio, subpage) < 0) { - set_pte_at(mm, address, pvmw.pte, pteval); - goto walk_abort; - } - - /* - * arch_unmap_one() is expected to be a NOP on - * architectures where we could have PFN swap PTEs, - * so we'll not check/care. - */ - if (arch_unmap_one(mm, vma, address, pteval) < 0) { - folio_put_swap(folio, subpage); - set_pte_at(mm, address, pvmw.pte, pteval); - goto walk_abort; - } - - /* See folio_try_share_anon_rmap(): clear PTE first. */ - if (anon_exclusive && - folio_try_share_anon_rmap_pte(folio, subpage)) { - folio_put_swap(folio, subpage); + if (!ttu_anon_swapbacked_folio(vma, folio, page, address, + pvmw.pte, pteval)) { set_pte_at(mm, address, pvmw.pte, pteval); goto walk_abort; } - mm_prepare_for_swap_entries(mm); - dec_mm_counter(mm, MM_ANONPAGES); - inc_mm_counter(mm, MM_SWAPENTS); - swp_pte = swp_entry_to_pte(entry); - if (anon_exclusive) - swp_pte = pte_swp_mkexclusive(swp_pte); - if (likely(pte_present(pteval))) { - if (pte_soft_dirty(pteval)) - swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_uffd(pteval)) - swp_pte = pte_swp_mkuffd(swp_pte); - } else { - if (pte_swp_soft_dirty(pteval)) - swp_pte = pte_swp_mksoft_dirty(swp_pte); - if (pte_swp_uffd(pteval)) - swp_pte = pte_swp_mkuffd(swp_pte); - } - set_pte_at(mm, address, pvmw.pte, swp_pte); + goto finish_unmap; } else { /* * This is a locked file-backed folio, @@ -2369,7 +2390,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, add_mm_counter(mm, mm_counter_file(folio), -nr_pages); } finish_unmap: - folio_remove_rmap_ptes(folio, subpage, nr_pages, vma); + folio_remove_rmap_ptes(folio, page, nr_pages, vma); if (vma->vm_flags & VM_LOCKED) mlock_drain_local(); folio_put_refs(folio, nr_pages); -- 2.43.0