From: Luka Bai Transparent huge page is now properly working with most of the mm framework, and well fused with the folio concept that can be reclaimed or allocated with a large order. However, its deed is not very "estimable". For example, a THP is easily split in many path like partially mapped, swap out or fork + COW(for child processes). In some cases, we may want it to have some concluded result. Since some workloads expect a relatively "stable" THP, while others may want to save memory more rather than the performance benifits. This patch adds some basic helpers and branch in madvise path so that we can add madvise choices on THP to conduct what we do on different types of operations like COW or swap that may split THP, on the level of vma. We transfer the type of configuration using parameters of madvise, analyze it and save the result in vma->vm_flags for later use. Currently the only operation in the list is COW. It decides whether we want to use hugepages for the child process when it writes a spot on the shared anonymous pmd so that we can make sure the THP not being split after writing. This patch only adds the basic setup helpers, the real usage will be added in the later patches. Signed-off-by: Luka Bai --- include/linux/huge_mm.h | 6 ++++++ include/linux/mm.h | 19 +++++++++++++++++++ include/uapi/asm-generic/mman-common.h | 9 +++++++++ mm/madvise.c | 25 +++++++++++++++++++++++++ 4 files changed, 59 insertions(+) diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 48496f09909b..a0ce8c0b81f5 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -6,6 +6,7 @@ #include /* only for vma_is_dax() */ #include +#include vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf); int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, @@ -363,6 +364,11 @@ static inline bool thp_disabled_by_hw(void) return transparent_hugepage_flags & (1 << TRANSPARENT_HUGEPAGE_UNSUPPORTED); } +static inline bool madv_thp_cow(int behavior) +{ + return behavior & MADV_THP_COW; +} + unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, diff --git a/include/linux/mm.h b/include/linux/mm.h index 1d76da6e0791..8a800819cfa2 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -391,6 +391,10 @@ enum { #else DECLARE_VMA_BIT_ALIAS(STACK, GROWSDOWN), #endif +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + DECLARE_VMA_BIT(THP_SETUP_1, 43), + DECLARE_VMA_BIT_ALIAS(THP_COW, THP_SETUP_1), +#endif }; #undef DECLARE_VMA_BIT #undef DECLARE_VMA_BIT_ALIAS @@ -510,6 +514,9 @@ enum { #define VM_DROPPABLE VM_NONE #define VMA_DROPPABLE EMPTY_VMA_FLAGS #endif +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#define VM_THP_COW INIT_VM_FLAG(THP_COW) +#endif /* Bits set in the VMA until the stack is in its final location */ #define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ | VM_STACK_EARLY) @@ -4128,6 +4135,18 @@ extern int do_munmap(struct mm_struct *, unsigned long, size_t, struct list_head *uf); extern int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static inline bool madv_thp_behavior(int behavior) +{ + return behavior >= MADV_THP_SETUP_BASE && behavior < MADV_THP_SETUP_END; +} +#else +static inline bool madv_thp_behavior(int behavior) +{ + return false; +} +#endif + #ifdef CONFIG_MMU extern int __mm_populate(unsigned long addr, unsigned long len, int ignore_errors); diff --git a/include/uapi/asm-generic/mman-common.h b/include/uapi/asm-generic/mman-common.h index ef1c27fa3c57..1617ed374503 100644 --- a/include/uapi/asm-generic/mman-common.h +++ b/include/uapi/asm-generic/mman-common.h @@ -82,6 +82,15 @@ #define MADV_GUARD_INSTALL 102 /* fatal signal on access to range */ #define MADV_GUARD_REMOVE 103 /* unguard range */ +/* for THP setup */ +#define MADV_THP_SETUP_BASE 256 +enum { + MADV_THP_COW_BIT, + MADV_THP_SETUP_MAX_BIT, +}; +#define MADV_THP_COW (MADV_THP_SETUP_BASE + (1 << MADV_THP_COW_BIT)) +#define MADV_THP_SETUP_END (MADV_THP_SETUP_BASE + (1 << MADV_THP_SETUP_MAX_BIT)) + /* compatibility flags */ #define MAP_FILE 0 diff --git a/mm/madvise.c b/mm/madvise.c index 69708e953cf5..5dbfc89682d7 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -1331,6 +1331,25 @@ static bool can_madvise_modify(struct madvise_behavior *madv_behavior) } #endif +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static vm_flags_t madvise_thp_setup(struct madvise_behavior *madv_behavior) +{ + int thp_behavior = madv_behavior->behavior - MADV_THP_SETUP_BASE; + struct vm_area_struct *vma = madv_behavior->vma; + vm_flags_t new_flags = vma->vm_flags; + + if (madv_thp_cow(thp_behavior)) + new_flags |= VM_THP_COW; + + return new_flags; +} +#else +static vm_flags_t madvise_thp_setup(struct madvise_behavior *madv_behavior) +{ + return madv_behavior->vma->vm_flags; +} +#endif + /* * Apply an madvise behavior to a region of a vma. madvise_update_vma * will handle splitting a vm area into separate areas, each area with its own @@ -1427,6 +1446,10 @@ static int madvise_vma_behavior(struct madvise_behavior *madv_behavior) break; } + /* Handle THP behaviors */ + if (madv_thp_behavior(behavior)) + new_flags = madvise_thp_setup(madv_behavior); + /* This is a write operation.*/ VM_WARN_ON_ONCE(madv_behavior->lock_mode != MADVISE_MMAP_WRITE_LOCK); @@ -1555,6 +1578,8 @@ madvise_behavior_valid(int behavior) return true; default: + if (madv_thp_behavior(behavior)) + return true; return false; } } -- 2.52.0 From: Luka Bai We would like to use similar logic of huge anonymous page or huge shmem pages for THP COW: to categorize the strategies into three types: always, never, madvise. If setting up to always, then we always do THP COW for all the existing THPs. If setting up to never, then we never do THP COW. If setting up to madvise, then we follow the setup we introduced in last commit to decide whether we do COW for each individual vma. We add TRANSPARENT_HUGEPAGE_COW_FLAG and TRANSPARENT_HUGEPAGE_REQ_MADV_COW_FLAG that are very similar to the TRANSPARENT_HUGEPAGE_FLAG and TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG which are used to decide whether we do anonymous huge page fault when it permits. And we add sysfs attribute thp_cow_attr as the interface to choose from the three strategies we mentioned before. Signed-off-by: Luka Bai --- .../testing/sysfs-kernel-mm-transparent-hugepage | 1 + Documentation/admin-guide/mm/transhuge.rst | 27 +++++++++++++++ include/linux/huge_mm.h | 2 ++ mm/huge_memory.c | 39 ++++++++++++++++++++++ 4 files changed, 69 insertions(+) diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-transparent-hugepage b/Documentation/ABI/testing/sysfs-kernel-mm-transparent-hugepage index 7bfbb9cc2c11..43a1af13efe0 100644 --- a/Documentation/ABI/testing/sysfs-kernel-mm-transparent-hugepage +++ b/Documentation/ABI/testing/sysfs-kernel-mm-transparent-hugepage @@ -11,6 +11,7 @@ Description: - khugepaged - shmem_enabled - use_zero_page + - thp_cow - subdirectories of the form hugepages-kB, where is the page size of the hugepages supported by the kernel/CPU combination. diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst index 0ef13c451ac8..0926651bad0d 100644 --- a/Documentation/admin-guide/mm/transhuge.rst +++ b/Documentation/admin-guide/mm/transhuge.rst @@ -226,6 +226,33 @@ to "always" or "madvise"), and it'll be automatically shutdown when all THP sizes are disabled (when both the per-size anon control and the top-level control are "never") +Some workloads may want to do copy on write on the pmd size to acquire the +tlb benifit when it tries to write on a shared anonymous pmd sized entry. +They can do so by setting up the thp_cow control. The control is only enabled +when the global THP controls are set to "always" or "madvise" for the +specific memory region:: + +:: + + echo always >/sys/kernel/mm/transparent_hugepage/thp_cow + echo madvise >/sys/kernel/mm/transparent_hugepage/thp_cow + echo never >/sys/kernel/mm/transparent_hugepage/thp_cow + +always + means that the writing process will always do copy on write on + the pmd size. If there is no pmd sized folio available, it will + fallback to the pte size. + +madvise + will do things like ``always`` but only for regions that have + used madvise(MADV_THP_COW). + +never + will not do copy on write on the pmd size no matter what setup + is done using madvise. When a process writes on a shared anonymous + pmd sized entry, it will just allocate a pte sized page and do copy + on write on the pte size. + process THP controls -------------------- diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index a0ce8c0b81f5..2a62f0f92f68 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -57,6 +57,8 @@ enum transparent_hugepage_flag { TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG, TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG, + TRANSPARENT_HUGEPAGE_COW_FLAG, + TRANSPARENT_HUGEPAGE_REQ_MADV_COW_FLAG, }; struct kobject; diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 1f0d0b780943..babca060feca 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -531,6 +531,44 @@ static ssize_t split_underused_thp_store(struct kobject *kobj, static struct kobj_attribute split_underused_thp_attr = __ATTR( shrink_underused, 0644, split_underused_thp_show, split_underused_thp_store); +static ssize_t thp_cow_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + const char *output; + + if (test_bit(TRANSPARENT_HUGEPAGE_COW_FLAG, &transparent_hugepage_flags)) + output = "[always] madvise never"; + else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_COW_FLAG, + &transparent_hugepage_flags)) + output = "always [madvise] never"; + else + output = "always madvise [never]"; + + return sysfs_emit(buf, "%s\n", output); +} + +static ssize_t thp_cow_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + ssize_t ret = count; + + if (sysfs_streq(buf, "always")) { + clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_COW_FLAG, &transparent_hugepage_flags); + set_bit(TRANSPARENT_HUGEPAGE_COW_FLAG, &transparent_hugepage_flags); + } else if (sysfs_streq(buf, "madvise")) { + clear_bit(TRANSPARENT_HUGEPAGE_COW_FLAG, &transparent_hugepage_flags); + set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_COW_FLAG, &transparent_hugepage_flags); + } else if (sysfs_streq(buf, "never")) { + clear_bit(TRANSPARENT_HUGEPAGE_COW_FLAG, &transparent_hugepage_flags); + clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_COW_FLAG, &transparent_hugepage_flags); + } else + ret = -EINVAL; + + return ret; +} +static struct kobj_attribute thp_cow_attr = __ATTR_RW(thp_cow); + static struct attribute *hugepage_attr[] = { &enabled_attr.attr, &defrag_attr.attr, @@ -540,6 +578,7 @@ static struct attribute *hugepage_attr[] = { &shmem_enabled_attr.attr, #endif &split_underused_thp_attr.attr, + &thp_cow_attr.attr, NULL, }; -- 2.52.0 From: Luka Bai We add hugepage_cow_always and hugepage_cow_madvise as two convenient helpers to decide whether we want to do THP COW under each specific circumstance. Also, we add a helper hugepage_cow_enabled to help us know the setup more easily. THP COW is only opened when hugepage is globally enabled or madvise enabled. Signed-off-by: Luka Bai --- include/linux/huge_mm.h | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 2a62f0f92f68..3e5c6da3905b 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -203,6 +203,38 @@ static inline bool hugepage_global_always(void) (1<vm_flags; + + /* anonymous THP need to be enabled first */ + if (!hugepage_global_always() && + (!hugepage_global_enabled() || !(vm_flags & VM_HUGEPAGE))) + return false; + + /* always enables all the THP COW */ + if (hugepage_cow_always()) + return true; + + /* madvise enables THP cow only when vm_flags says so */ + if (hugepage_cow_madvise() && (vm_flags & VM_THP_COW)) + return true; + + return false; +} + static inline int highest_order(unsigned long orders) { return fls_long(orders) - 1; -- 2.52.0 From: Luka Bai Function map_anon_folio_pmd_nopf was able to map new anonymous pages. Like in function do_huge_pmd_anonymous_page, it handles all the mappings and statistics correctly in one call. However, it doesn't support FAULT_FLAG_UNSHARE. Normally, FAULT_FLAG_UNSHARE was set when we just want to separate multiple non-exclusive sharing apart, it follows the copy on write process, since it also does the checking like whether we need to copy memory, or just use the existing one, basically the same work like what COW does. But it doesn't happen because of writing on a RO pte/pmd which is actually permitted to be written to but simply for "unsharing". Hence we need to copy the same permissive and other marker flags into the copied new page table entry just like the old one when doing the duplication, without making it writable. Now, map_anon_folio_pmd_nopf only tries to make the new pmd writable that is not what unsharing wants. We add unsharing support for map_anon_folio_pmd_nopf by passing the vm_fault struct as a parameter and get the unsharing hint. If we are in the unsharing procedure, then we just copy the soft_dirty and uffd_wp flags into the new pmd instead of trying to make the new pmd writable. Signed-off-by: Luka Bai --- include/linux/huge_mm.h | 5 ++--- mm/huge_memory.c | 34 +++++++++++++++++++++++----------- mm/khugepaged.c | 8 +++++++- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 3e5c6da3905b..61f0e614ca52 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -610,9 +610,8 @@ void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address, pmd_t *pmd, bool freeze); bool unmap_huge_pmd_locked(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmdp, struct folio *folio); -void map_anon_folio_pmd_nopf(struct folio *folio, pmd_t *pmd, - struct vm_area_struct *vma, unsigned long haddr); - +void map_anon_folio_pmd_nopf(struct folio *folio, struct vm_fault *vmf, + bool cow); #else /* CONFIG_TRANSPARENT_HUGEPAGE */ static inline bool folio_test_pmd_mappable(struct folio *folio) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index babca060feca..1e661b411b2e 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1423,13 +1423,26 @@ static struct folio *vma_alloc_anon_folio_pmd(struct vm_area_struct *vma, return folio; } -void map_anon_folio_pmd_nopf(struct folio *folio, pmd_t *pmd, - struct vm_area_struct *vma, unsigned long haddr) +void map_anon_folio_pmd_nopf(struct folio *folio, struct vm_fault *vmf, + bool cow) { pmd_t entry; + struct vm_area_struct *vma = vmf->vma; + pmd_t *pmd = vmf->pmd; + pmd_t orig_pmd = vmf->orig_pmd; + unsigned long haddr = vmf->address & HPAGE_PMD_MASK; + const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; entry = folio_mk_pmd(folio, vma->vm_page_prot); - entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); + if (unlikely(cow && unshare)) { + VM_WARN_ON(pmd_write(orig_pmd)); + if (pmd_soft_dirty(orig_pmd)) + entry = pmd_mksoft_dirty(entry); + if (pmd_uffd_wp(orig_pmd)) + entry = pmd_mkuffd_wp(entry); + } else { + entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); + } folio_add_new_anon_rmap(folio, vma, haddr, RMAP_EXCLUSIVE); folio_add_lru_vma(folio, vma); set_pmd_at(vma->vm_mm, haddr, pmd, entry); @@ -1437,19 +1450,18 @@ void map_anon_folio_pmd_nopf(struct folio *folio, pmd_t *pmd, deferred_split_folio(folio, false); } -static void map_anon_folio_pmd_pf(struct folio *folio, pmd_t *pmd, - struct vm_area_struct *vma, unsigned long haddr) +static void map_anon_folio_pmd_pf(struct folio *folio, struct vm_fault *vmf, + bool cow) { - map_anon_folio_pmd_nopf(folio, pmd, vma, haddr); - add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR); + map_anon_folio_pmd_nopf(folio, vmf, cow); + add_mm_counter(vmf->vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR); count_vm_event(THP_FAULT_ALLOC); count_mthp_stat(HPAGE_PMD_ORDER, MTHP_STAT_ANON_FAULT_ALLOC); - count_memcg_event_mm(vma->vm_mm, THP_FAULT_ALLOC); + count_memcg_event_mm(vmf->vma->vm_mm, THP_FAULT_ALLOC); } static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf) { - unsigned long haddr = vmf->address & HPAGE_PMD_MASK; struct vm_area_struct *vma = vmf->vma; struct folio *folio; pgtable_t pgtable; @@ -1483,7 +1495,7 @@ static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf) return ret; } pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable); - map_anon_folio_pmd_pf(folio, vmf->pmd, vma, haddr); + map_anon_folio_pmd_pf(folio, vmf, false); mm_inc_nr_ptes(vma->vm_mm); spin_unlock(vmf->ptl); } @@ -2174,7 +2186,7 @@ static vm_fault_t do_huge_zero_wp_pmd(struct vm_fault *vmf) if (ret) goto release; (void)pmdp_huge_clear_flush(vma, haddr, vmf->pmd); - map_anon_folio_pmd_pf(folio, vmf->pmd, vma, haddr); + map_anon_folio_pmd_pf(folio, vmf, true); goto unlock; release: folio_put(folio); diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 7d48d4fbd5f3..18d309b69d30 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1402,7 +1402,13 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long s if (is_pmd_order(order)) { /* PMD collapse */ pgtable = pmd_pgtable(_pmd); pgtable_trans_huge_deposit(mm, pmd, pgtable); - map_anon_folio_pmd_nopf(folio, pmd, vma, pmd_addr); + struct vm_fault vmf = { + .vma = vma, + .flags = 0, + .address = pmd_addr, + .orig_pmd = pmdp_get(pmd), + }; + map_anon_folio_pmd_nopf(folio, &vmf, false); } else { /* mTHP collapse */ map_anon_folio_pte_nopf(folio, pte, vma, start_addr, /*uffd_wp=*/ false); smp_wmb(); /* make PTEs visible before PMD. See pmd_install() */ -- 2.52.0 From: Luka Bai For pmd mapped anonymous folios, we currently do not do COW for the whole vma region, because we don't want to copy and unshare the full PMD range on the first write fault. That proposal holds for the most workloads, however, that also makes the pmd entry split into 512 4K ptes in the child process after we write on a part of the folio. For example, if process A and B share a pmd sized folio, if B does writing on a small region, its pmd mapping will be split into 511 4K ptes which still point to the original pmd sized folio, and 1 4K pte pointing to the new 4K page. This is quite good for memory utilization, but it also make the tlb gain caused by pmd entry suddenly "vanish" after a simple write, which causes a observable performance decrease in some workloads. And also, it adds some "uncertainty" to the THP since it does splitting transparently in the COW scenorio which sometimes can cause trouble to ones that need stable hugepages. This patch adds support for pmd sized COW of anonymous page with switch controlling. The reason we add switch is that for some scenorio, the performance matters more, but for other workloads maybe the memory waste is more unbearable. So we can use the THP setup to control this configuration, either on the vma level or the global level. The patch is relatively simple, we add function wp_huge_pmd_page_copy to do the hugepage copy on write part, and do the allocation, accouting and cache flushing just like in 4K path. We use the newly reconstructed map_anon_folio_pmd_pf to do the mapping since it can properly support FAULT_FLAG_UNSHARE right now. We remove the ref checking in do_huge_pmd_wp_page, since we have supported copying the pmd folio right now, we'll check the refcount in the following folio_ref_count to make sure if the folio can be exclusively used. If not, we can always do copy on write for this folio just like in do_wp_page when THP COW is enabled. Signed-off-by: Luka Bai --- mm/huge_memory.c | 125 +++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 116 insertions(+), 9 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 1e661b411b2e..a05a4456e5a2 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include "internal.h" @@ -2196,6 +2197,94 @@ static vm_fault_t do_huge_zero_wp_pmd(struct vm_fault *vmf) return ret; } +static vm_fault_t wp_huge_pmd_page_copy(struct vm_fault *vmf, struct folio *old_folio) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + struct folio *new_folio = NULL; + struct page *new_page, *old_page; + unsigned long pmd_address = vmf->address & HPAGE_PMD_MASK; + struct mmu_notifier_range range; + vm_fault_t ret = 0; + int i; + + delayacct_wpcopy_start(); + + old_page = folio_page(old_folio, 0); + ret = vmf_anon_prepare(vmf); + if (unlikely(ret)) { + if (ret != VM_FAULT_RETRY) + ret = VM_FAULT_FALLBACK; + goto out; + } + + new_folio = vma_alloc_anon_folio_pmd(vma, vmf->address); + if (unlikely(!new_folio)) { + ret = VM_FAULT_FALLBACK; + goto out; + } + + if (copy_user_large_folio(new_folio, old_folio, + pmd_address, vma)) { + ret = VM_FAULT_HWPOISON; + goto out; + } + + new_page = folio_page(new_folio, 0); + for (i = 0; i < HPAGE_PMD_NR; i++) + kmsan_copy_page_meta(new_page + i, old_page + i); + + __folio_mark_uptodate(new_folio); + mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, + pmd_address, pmd_address + HPAGE_PMD_SIZE); + mmu_notifier_invalidate_range_start(&range); + + spin_lock(vmf->ptl); + if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd))) { + update_mmu_cache_pmd(vma, pmd_address, vmf->pmd); + ret = 0; + goto out_unlock; + } + + flush_cache_range(vma, pmd_address, pmd_address + HPAGE_PMD_SIZE); + /* + * Clear the pmd entry and flush it first, before updating the + * pmd with the new entry, to keep TLBs on different CPUs in + * sync. + */ + (void)pmdp_huge_clear_flush(vma, pmd_address, vmf->pmd); + /* + * We just temporarily decrement the mm_counter here, and it will be added back in + * map_anon_folio_pmd_pf below. + */ + add_mm_counter(mm, MM_ANONPAGES, -HPAGE_PMD_NR); + map_anon_folio_pmd_pf(new_folio, vmf, true); + folio_remove_rmap_pmd(old_folio, old_page, vma); + + spin_unlock(vmf->ptl); + + mmu_notifier_invalidate_range_end(&range); + /* This put is for the folio_get() in the caller */ + folio_put(old_folio); + free_swap_cache(old_folio); + + /* This put is for decrementing refcount after we switch page table mapping */ + folio_put(old_folio); + + delayacct_wpcopy_end(); + return 0; +out_unlock: + spin_unlock(vmf->ptl); + mmu_notifier_invalidate_range_end(&range); +out: + folio_put(old_folio); + if (new_folio) + folio_put(new_folio); + + delayacct_wpcopy_end(); + return ret; +} + vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf) { const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; @@ -2204,12 +2293,13 @@ vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf) struct page *page; unsigned long haddr = vmf->address & HPAGE_PMD_MASK; pmd_t orig_pmd = vmf->orig_pmd; + vm_fault_t ret; vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd); VM_BUG_ON_VMA(!vma->anon_vma, vma); if (is_huge_zero_pmd(orig_pmd)) { - vm_fault_t ret = do_huge_zero_wp_pmd(vmf); + ret = do_huge_zero_wp_pmd(vmf); if (!(ret & VM_FAULT_FALLBACK)) return ret; @@ -2253,14 +2343,6 @@ vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf) goto reuse; } - /* - * See do_wp_page(): we can only reuse the folio exclusively if - * there are no additional references. Note that we always drain - * the LRU cache immediately after adding a THP. - */ - if (folio_ref_count(folio) > - 1 + folio_test_swapcache(folio) * folio_nr_pages(folio)) - goto unlock_fallback; if (folio_test_swapcache(folio)) folio_free_swap(folio); if (folio_ref_count(folio) == 1) { @@ -2282,6 +2364,31 @@ vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf) return 0; } + /* + * Only do hugepage copy on write if the parameter setup supports it. + */ + if (!hugepage_cow_enabled(vma)) + goto unlock_fallback; + + /* + * For vma without a vm_ops(anonymous vma), there should not be VM_SHARED or + * VM_MAYSHARE types. + */ + VM_WARN_ON_ONCE_VMA(vma->vm_flags & (VM_SHARED | VM_MAYSHARE), vma); + + folio_unlock(folio); + /* + * Copy on write branch here. + * We are about to unlock the ptl here, so we need to get folio before that + * in case the folio gets freed in the meantime. + */ + folio_get(folio); + spin_unlock(vmf->ptl); + ret = wp_huge_pmd_page_copy(vmf, folio); + if (ret & VM_FAULT_FALLBACK) + goto fallback; + return ret; + unlock_fallback: folio_unlock(folio); spin_unlock(vmf->ptl); -- 2.52.0