From: Wandun Chen remove_migration_pte() restores PTE-mapped large folios one PTE at a time, constructing each new PTE and updating the folio references and rmap separately. Add migration_pte_batch() to find adjacent migration entries with consecutive PFNs and matching type and flags. When a batch covers the whole folio, the rmap helpers already call mlock_vma_folio(). Skip the explicit call to avoid double mlock accounting. Device-private PTEs encode PFNs as swap offsets. set_ptes() does not handle this encoding when constructing subsequent PTEs, such as on arm64 with a 16KB page granule, leaving device-private destinations unbatched. The next patch enables batching for these entries by advancing their softleaf offsets. Suggested-by: David Hildenbrand (Arm) Link: https://lore.kernel.org/lkml/5f9c449e-43b3-4732-9c14-8199418cace0@kernel.org/ Signed-off-by: Wandun Chen --- mm/internal.h | 6 ++++++ mm/migrate.c | 35 +++++++++++++++++++++++++++++------ 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/mm/internal.h b/mm/internal.h index 9f78c12882cb..e6ecf307d532 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -552,6 +552,12 @@ static inline int swap_pte_batch(pte_t *start_ptep, int max_nr, pte_t pte) return __softleaf_pte_batch(start_ptep, max_nr, pte); } +static inline int migration_pte_batch(pte_t *start_ptep, int max_nr, pte_t pte) +{ + VM_WARN_ON(!softleaf_is_migration(softleaf_from_pte(pte))); + + return __softleaf_pte_batch(start_ptep, max_nr, pte); +} #endif /* CONFIG_MMU */ void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio, diff --git a/mm/migrate.c b/mm/migrate.c index 3c39d309e6bd..e000ef77ea87 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -358,6 +358,7 @@ static bool remove_migration_pte(struct folio *folio, while (page_vma_mapped_walk(&pvmw)) { rmap_t rmap_flags = RMAP_NONE; unsigned long idx = 0; + int nr = 1; softleaf_t entry; struct page *new; pte_t old_pte; @@ -441,17 +442,34 @@ static bool remove_migration_pte(struct folio *folio, } else #endif { + if (folio_test_large(rmap_walk_arg->folio) && + !is_device_private_page(new)) { + unsigned long end_addr = pmd_addr_end(pvmw.address, vma->vm_end); + unsigned long max_nr = (end_addr - pvmw.address) >> PAGE_SHIFT; + + max_nr = min_t(unsigned long, max_nr, + PTRS_PER_PTE - pte_index(pvmw.address)); + + max_nr = min_t(unsigned long, max_nr, + folio_nr_pages(rmap_walk_arg->folio) - idx); + + nr = migration_pte_batch(pvmw.pte, max_nr, old_pte); + } + if (nr > 1) + folio_ref_add(folio, nr - 1); + if (folio_test_anon(folio)) - folio_add_anon_rmap_pte(folio, new, vma, - pvmw.address, rmap_flags); + folio_add_anon_rmap_ptes(folio, new, nr, vma, + pvmw.address, rmap_flags); else - folio_add_file_rmap_pte(folio, new, vma); - set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); + folio_add_file_rmap_ptes(folio, new, nr, vma); + set_ptes(vma->vm_mm, pvmw.address, pvmw.pte, pte, nr); - ptes++; + ptes += nr; } if (READ_ONCE(vma->vm_flags) & VM_LOCKED) { if (folio_test_large(folio) && + nr < folio_nr_pages(folio) && ptes == folio_nr_pages(folio) && !(pvmw.flags & PVMW_PGTABLE_CROSSED)) mlock_vma_folio(folio, vma); @@ -462,7 +480,12 @@ static bool remove_migration_pte(struct folio *folio, compound_order(new)); /* No need to invalidate - it was non-present before */ - update_mmu_cache(vma, pvmw.address, pvmw.pte); + update_mmu_cache_range(NULL, vma, pvmw.address, pvmw.pte, nr); + + if (nr == folio_nr_pages(folio)) { + page_vma_mapped_walk_done(&pvmw); + break; + } } return true; -- 2.43.0