This fixes no bug: it makes fork copy a PMD the way copy_present_ptes() copies a PTE. Until now copy_huge_pmd() skipped every folio PMD of a non-anonymous VMA and left the child to fault them in again, while the PTEs of the same VMA are copied. Treat file folios as the PTE path does: take a reference, duplicate the rmap and account them. fork then pays a reference and an rmap update per PMD, and saves the child a fault per PMD it touches. The next patch sends PFN mappings through the same tail. That covers the VMAs vma_needs_copy() sends here: - private file mappings with an anon_vma; - VM_COPY_ON_FORK ones: uffd-wp and uffd-rwp, VM_MIXEDMAP (fuse DAX), and VM_MAYBE_GUARD, which shared shmem, file and DAX mappings get from a guard region. The DAX huge zero PMD is copied too, with a reference taken by mm_get_huge_zero_folio() as for anonymous VMAs. A migration entry of a file folio is copied and accounted like copy_nonpresent_pte() does, and made read-only in both processes only in a CoW mapping. The copy then gets the same tail as __copy_present_ptes(): write protect both sides in a CoW mapping, mark the child's copy clean in a shared one, and drop the uffd-wp bit where the child is not armed. An anonymous THP is now write protected only when its PMD is writable, as a PTE is. A page table is deposited under the copy where the fault paths deposit one: in anonymous VMAs, and wherever the architecture needs one. Suggested-by: David Hildenbrand Signed-off-by: Konstantin Khorenko --- mm/huge_memory.c | 60 ++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 1e5d68acf62a5..13e3674fb6115 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1898,12 +1898,13 @@ static void copy_huge_non_present_pmd( pmd_t pmd, pgtable_t pgtable) { softleaf_t entry = softleaf_from_pmd(pmd); - struct folio *src_folio; + struct folio *src_folio = softleaf_to_folio(entry); VM_WARN_ON_ONCE(!pmd_is_valid_softleaf(pmd)); - if (softleaf_is_migration_write(entry) || - softleaf_is_migration_read_exclusive(entry)) { + if ((softleaf_is_migration_write(entry) || + softleaf_is_migration_read_exclusive(entry)) && + vma_is_cow_mapping(src_vma)) { entry = make_readable_migration_entry(swp_offset(entry)); pmd = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*src_pmd)) @@ -1927,7 +1928,6 @@ static void copy_huge_non_present_pmd( set_pmd_at(src_mm, addr, src_pmd, pmd); } - src_folio = softleaf_to_folio(entry); VM_WARN_ON(!folio_test_large(src_folio)); folio_get(src_folio); @@ -1939,9 +1939,11 @@ static void copy_huge_non_present_pmd( dst_vma, src_vma); } - add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); - mm_inc_nr_ptes(dst_mm); - pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); + add_mm_counter(dst_mm, mm_counter(src_folio), HPAGE_PMD_NR); + if (pgtable) { + mm_inc_nr_ptes(dst_mm); + pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); + } if (!userfaultfd_protected(dst_vma)) pmd = pmd_swp_clear_uffd(pmd); set_pmd_at(dst_mm, addr, dst_pmd, pmd); @@ -1957,6 +1959,7 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, pmd_t pmd; pgtable_t pgtable = NULL; int ret = -ENOMEM; + bool writable; pmd = pmdp_get_lockless(src_pmd); if (unlikely(pmd_present(pmd) && pmd_special(pmd) && @@ -1977,13 +1980,11 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, goto set_pmd; } - /* Skip if can be re-fill on fault */ - if (!vma_is_anonymous(dst_vma)) - return 0; - - pgtable = pte_alloc_one(dst_mm); - if (unlikely(!pgtable)) - goto out; + if (vma_is_anonymous(dst_vma) || arch_needs_pgtable_deposit()) { + pgtable = pte_alloc_one(dst_mm); + if (unlikely(!pgtable)) + goto out; + } dst_ptl = pmd_lock(dst_mm, dst_pmd); src_ptl = pmd_lockptr(src_mm, src_pmd); @@ -2001,7 +2002,8 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, } if (unlikely(!pmd_trans_huge(pmd))) { - pte_free(dst_mm, pgtable); + if (pgtable) + pte_free(dst_mm, pgtable); goto out_unlock; } /* @@ -2016,7 +2018,7 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, * copy. It just takes a reference. */ mm_get_huge_zero_folio(dst_mm); - goto out_zero_page; + goto copy_pmd; } src_page = pmd_page(pmd); @@ -2024,6 +2026,12 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, src_folio = page_folio(src_page); folio_get(src_folio); + if (!folio_test_anon(src_folio)) { + folio_dup_file_rmap_pmd(src_folio, src_page, dst_vma); + add_mm_counter(dst_mm, mm_counter_file(src_folio), + HPAGE_PMD_NR); + goto copy_pmd; + } if (unlikely(folio_try_dup_anon_rmap_pmd(src_folio, src_page, dst_vma, src_vma))) { /* Page maybe pinned: split and retry the fault on PTEs. */ folio_put(src_folio); @@ -2034,19 +2042,25 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, return -EAGAIN; } add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); -out_zero_page: - mm_inc_nr_ptes(dst_mm); - pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); +copy_pmd: + if (pgtable) { + mm_inc_nr_ptes(dst_mm); + pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); + } - /* See __copy_present_ptes(): restore accessible protection. */ + /* See __copy_present_ptes(). */ + writable = pmd_write(pmd); if (!userfaultfd_protected(dst_vma)) { if (userfaultfd_rwp(src_vma) && pmd_uffd(pmd)) pmd = pmd_modify(pmd, dst_vma->vm_page_prot); pmd = pmd_clear_uffd(pmd); } - - pmdp_set_wrprotect(src_mm, addr, src_pmd); - pmd = pmd_wrprotect(pmd); + if (vma_is_cow_mapping(src_vma) && writable) { + pmdp_set_wrprotect(src_mm, addr, src_pmd); + pmd = pmd_wrprotect(pmd); + } + if (src_vma->vm_flags & VM_SHARED) + pmd = pmd_mkclean(pmd); set_pmd: pmd = pmd_mkold(pmd); set_pmd_at(dst_mm, addr, dst_pmd, pmd); -- 2.43.0