Add softleaf_to_pmd() as the PMD counterpart to softleaf_to_pte(), completing the symmetry of the softleaf abstraction for page table leaf entries. The upcoming PMD swap entry support needs to construct PMD entries from swap entries. Converting existing swp_entry_to_pmd() callers to softleaf_to_pmd() in a prep patch keeps the feature patches focused on new functionality rather than mixing refactoring with new code. Acked-by: David Hildenbrand (Arm) Reviewed-by: Zi Yan Signed-off-by: Usama Arif --- include/linux/leafops.h | 20 ++++++++++++++++++++ mm/debug_vm_pgtable.c | 4 ++-- mm/huge_memory.c | 12 ++++++------ mm/migrate_device.c | 2 +- 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/include/linux/leafops.h b/include/linux/leafops.h index 992cd8bd8ed0..803d312437df 100644 --- a/include/linux/leafops.h +++ b/include/linux/leafops.h @@ -108,6 +108,21 @@ static inline softleaf_t softleaf_from_pmd(pmd_t pmd) return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry)); } +/** + * softleaf_to_pmd() - Obtain a PMD entry from a leaf entry. + * @entry: Leaf entry. + * + * This generates an architecture-specific PMD entry that can be utilised to + * encode the metadata the leaf entry encodes. + * + * Returns: Architecture-specific PMD entry encoding leaf entry. + */ +static inline pmd_t softleaf_to_pmd(softleaf_t entry) +{ + /* Temporary until swp_entry_t eliminated. */ + return swp_entry_to_pmd(entry); +} + #else static inline softleaf_t softleaf_from_pmd(pmd_t pmd) @@ -115,6 +130,11 @@ static inline softleaf_t softleaf_from_pmd(pmd_t pmd) return softleaf_mk_none(); } +static inline pmd_t softleaf_to_pmd(softleaf_t entry) +{ + return __pmd(0); +} + #endif /** diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c index 23dc3ee09561..18411fb09aab 100644 --- a/mm/debug_vm_pgtable.c +++ b/mm/debug_vm_pgtable.c @@ -758,7 +758,7 @@ static void __init pmd_leaf_soft_dirty_tests(struct pgtable_debug_args *args) return; pr_debug("Validating PMD swap soft dirty\n"); - pmd = swp_entry_to_pmd(args->leaf_entry); + pmd = softleaf_to_pmd(args->leaf_entry); WARN_ON(!pmd_is_huge(pmd)); WARN_ON(!pmd_is_valid_softleaf(pmd)); @@ -829,7 +829,7 @@ static void __init pmd_softleaf_tests(struct pgtable_debug_args *args) return; pr_debug("Validating PMD swap\n"); - pmd1 = swp_entry_to_pmd(args->leaf_entry); + pmd1 = softleaf_to_pmd(args->leaf_entry); WARN_ON(!pmd_is_huge(pmd1)); WARN_ON(!pmd_is_valid_softleaf(pmd1)); diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 2bccb0a53a0a..d51f53cfbe4f 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1819,7 +1819,7 @@ static void copy_huge_non_present_pmd( if (softleaf_is_migration_write(entry) || softleaf_is_migration_read_exclusive(entry)) { entry = make_readable_migration_entry(swp_offset(entry)); - pmd = swp_entry_to_pmd(entry); + pmd = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*src_pmd)) pmd = pmd_swp_mksoft_dirty(pmd); if (pmd_swp_uffd_wp(*src_pmd)) @@ -1832,7 +1832,7 @@ static void copy_huge_non_present_pmd( */ if (softleaf_is_device_private_write(entry)) { entry = make_readable_device_private_entry(swp_offset(entry)); - pmd = swp_entry_to_pmd(entry); + pmd = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*src_pmd)) pmd = pmd_swp_mksoft_dirty(pmd); @@ -2570,12 +2570,12 @@ static void change_non_present_huge_pmd(struct mm_struct *mm, entry = make_readable_exclusive_migration_entry(swp_offset(entry)); else entry = make_readable_migration_entry(swp_offset(entry)); - newpmd = swp_entry_to_pmd(entry); + newpmd = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*pmd)) newpmd = pmd_swp_mksoft_dirty(newpmd); } else if (softleaf_is_device_private_write(entry)) { entry = make_readable_device_private_entry(swp_offset(entry)); - newpmd = swp_entry_to_pmd(entry); + newpmd = softleaf_to_pmd(entry); if (pmd_swp_uffd_wp(*pmd)) newpmd = pmd_swp_mkuffd_wp(newpmd); } else { @@ -4923,7 +4923,7 @@ int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, } /* Set PMD. */ - pmdswp = swp_entry_to_pmd(entry); + pmdswp = softleaf_to_pmd(entry); if (softdirty) pmdswp = pmd_swp_mksoft_dirty(pmdswp); if (uffd_wp) @@ -4976,7 +4976,7 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) else entry = make_readable_device_private_entry( page_to_pfn(new)); - pmde = swp_entry_to_pmd(entry); + pmde = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*pvmw->pmd)) pmde = pmd_swp_mksoft_dirty(pmde); diff --git a/mm/migrate_device.c b/mm/migrate_device.c index ae39173d6a0e..aa948db49501 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -835,7 +835,7 @@ static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate, else swp_entry = make_readable_device_private_entry( page_to_pfn(page)); - entry = swp_entry_to_pmd(swp_entry); + entry = softleaf_to_pmd(swp_entry); } else { if (folio_is_zone_device(folio) && !folio_is_device_coherent(folio)) { -- 2.53.0-Meta When a swap entry is installed in a page table, the mm must be added to init_mm.mmlist so that swapoff can find and unuse its swap entries. This double-checked locking pattern is currently open-coded in try_to_unmap_one() and copy_nonpresent_pte(). Move it into mm_prepare_for_swap_entries() in mm/internal.h and convert both callers so it can be reused by upcoming PMD-level swap entry code paths that also need to register the mm with swapoff. copy_nonpresent_pte() previously inserted into &src_mm->mmlist rather than &init_mm.mmlist, but the insertion point is irrelevant, mmlist is a circular list and swapoff walks it entirely from init_mm.mmlist, so only membership matters, not position. Reviewed-by: Dev Jain Reviewed-by: Zi Yan Signed-off-by: Usama Arif --- mm/internal.h | 13 +++++++++++++ mm/memory.c | 9 +-------- mm/rmap.c | 7 +------ 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/mm/internal.h b/mm/internal.h index 8e0df3d1c6f9..0e3db9451e5c 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1956,4 +1956,17 @@ static inline int get_sysctl_max_map_count(void) bool may_expand_vm(struct mm_struct *mm, const vma_flags_t *vma_flags, unsigned long npages); +/* + * Ensure @mm is on the init_mm.mmlist so swapoff can find it. + */ +static inline void mm_prepare_for_swap_entries(struct mm_struct *mm) +{ + if (list_empty(&mm->mmlist)) { + spin_lock(&mmlist_lock); + if (list_empty(&mm->mmlist)) + list_add(&mm->mmlist, &init_mm.mmlist); + spin_unlock(&mmlist_lock); + } +} + #endif /* __MM_INTERNAL_H */ diff --git a/mm/memory.c b/mm/memory.c index a3fcaf8cfe8f..6637c5b13c9b 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -953,14 +953,7 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, if (swap_dup_entry_direct(entry) < 0) return -EIO; - /* make sure dst_mm is on swapoff's mmlist. */ - if (unlikely(list_empty(&dst_mm->mmlist))) { - spin_lock(&mmlist_lock); - if (list_empty(&dst_mm->mmlist)) - list_add(&dst_mm->mmlist, - &src_mm->mmlist); - spin_unlock(&mmlist_lock); - } + mm_prepare_for_swap_entries(dst_mm); /* Mark the swap entry as shared. */ if (pte_swp_exclusive(orig_pte)) { pte = pte_swp_clear_exclusive(orig_pte); diff --git a/mm/rmap.c b/mm/rmap.c index 1c77d5dc06e9..b93caabd186f 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -2304,12 +2304,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, set_pte_at(mm, address, pvmw.pte, pteval); goto walk_abort; } - if (list_empty(&mm->mmlist)) { - spin_lock(&mmlist_lock); - if (list_empty(&mm->mmlist)) - list_add(&mm->mmlist, &init_mm.mmlist); - spin_unlock(&mmlist_lock); - } + mm_prepare_for_swap_entries(mm); dec_mm_counter(mm, MM_ANONPAGES); inc_mm_counter(mm, MM_SWAPENTS); swp_pte = swp_entry_to_pte(entry); -- 2.53.0-Meta pagemap_pmd_range_thp() assumes that every non-present PMD is a migration entry and unconditionally calls softleaf_to_page(). This will crash on any non-present PMD type that does not encode a PFN, such as the upcoming PMD-level swap entries. Guard the page lookup with softleaf_has_pfn(), matching how pte_to_pagemap_entry() already handles non-present PTEs. Acked-by: David Hildenbrand (Arm) Reviewed-by: Zi Yan Signed-off-by: Usama Arif --- fs/proc/task_mmu.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index d32408f7cd5e..1fb5acd88ad0 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -2129,7 +2129,8 @@ static int pagemap_pmd_range_thp(pmd_t *pmdp, unsigned long addr, flags |= PM_SOFT_DIRTY; if (pmd_swp_uffd_wp(pmd)) flags |= PM_UFFD_WP; - page = softleaf_to_page(entry); + if (softleaf_has_pfn(entry)) + page = softleaf_to_page(entry); } if (page) { -- 2.53.0-Meta change_non_present_huge_pmd() calls softleaf_to_folio() unconditionally at the top of the function. softleaf_to_folio() extracts a PFN from the entry and converts it to a folio pointer, which is only meaningful for migration and device_private entries that encode a real PFN. A swap entry encodes a swap offset instead, so softleaf_to_folio() would produce a bogus pointer and crash on mprotect() when a PMD swap entry is present. Move the call into the migration_write branch where the folio is actually used, so the function is safe for any non-present PMD type. Acked-by: David Hildenbrand (Arm) Reviewed-by: Dev Jain Reviewed-by: Zi Yan Signed-off-by: Usama Arif --- mm/huge_memory.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index d51f53cfbe4f..4aca9cdff73f 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2557,11 +2557,12 @@ static void change_non_present_huge_pmd(struct mm_struct *mm, bool uffd_wp_resolve) { softleaf_t entry = softleaf_from_pmd(*pmd); - const struct folio *folio = softleaf_to_folio(entry); pmd_t newpmd; VM_WARN_ON(!pmd_is_valid_softleaf(*pmd)); if (softleaf_is_migration_write(entry)) { + const struct folio *folio = softleaf_to_folio(entry); + /* * A protection check is difficult so * just be safe and disable write -- 2.53.0-Meta migrate_vma_collect_pmd() calls softleaf_to_folio() on a non-present PMD before checking the entry's type. softleaf_to_folio() converts the entry's offset to a PFN, which is only meaningful for migration or device-private entries. A PMD swap entry's offset is a swap offset, not a PFN, so the lookup would either return a bogus folio pointer or trip pfn_to_page validation on a debug kernel. In the non-device-private path the returned folio is then unused (the OR short-circuits to migrate_vma_collect_skip()), but the lookup itself is already unsafe. Move the softleaf_to_folio() call inside the device-private branch where the folio is actually needed, mirroring the equivalent change_non_present_huge_pmd() fix. Reviewed-by: Zi Yan Signed-off-by: Usama Arif --- mm/migrate_device.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/mm/migrate_device.c b/mm/migrate_device.c index aa948db49501..36287e958b2a 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -166,11 +166,14 @@ static int migrate_vma_collect_huge_pmd(pmd_t *pmdp, unsigned long start, } else if (!pmd_present(*pmdp)) { const softleaf_t entry = softleaf_from_pmd(*pmdp); - folio = softleaf_to_folio(entry); - if (!softleaf_is_device_private(entry) || - !(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_PRIVATE) || - (folio->pgmap->owner != migrate->pgmap_owner)) { + !(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_PRIVATE)) { + spin_unlock(ptl); + return migrate_vma_collect_skip(start, end, walk); + } + + folio = softleaf_to_folio(entry); + if (folio->pgmap->owner != migrate->pgmap_owner) { spin_unlock(ptl); return migrate_vma_collect_skip(start, end, walk); } -- 2.53.0-Meta CONFIG_ARCH_ENABLE_THP_MIGRATION started life gating just PMD-level migration entries, but has grown to gate the entire PMD-level softleaf machinery: migration entries, device-private entries, and soon swap entries. Rename CONFIG_ARCH_ENABLE_THP_MIGRATION to CONFIG_ARCH_SUPPORTS_PMD _SOFTLEAF to make this clear. This is a pure rename: the set of selecting architectures (x86, arm64, s390, riscv, loongarch, and powerpc on PPC_BOOK3S_64) and the gating semantics are unchanged. No functional change intended. Reviewed-by: Zi Yan Signed-off-by: Usama Arif --- arch/arm64/Kconfig | 2 +- arch/arm64/include/asm/pgtable.h | 4 ++-- arch/loongarch/Kconfig | 2 +- arch/powerpc/include/asm/book3s/64/pgtable.h | 2 +- arch/powerpc/platforms/Kconfig.cputype | 2 +- arch/riscv/Kconfig | 2 +- arch/riscv/include/asm/pgtable.h | 8 ++++---- arch/s390/Kconfig | 2 +- arch/s390/include/asm/pgtable.h | 2 +- arch/x86/Kconfig | 2 +- arch/x86/include/asm/pgtable.h | 2 +- include/linux/huge_mm.h | 2 +- include/linux/leafops.h | 8 ++++---- include/linux/pgtable.h | 2 +- include/linux/swapops.h | 6 +++--- mm/Kconfig | 2 +- mm/debug_vm_pgtable.c | 8 ++++---- mm/hmm.c | 4 ++-- mm/huge_memory.c | 2 +- mm/migrate.c | 4 ++-- mm/migrate_device.c | 6 +++--- mm/rmap.c | 2 +- 22 files changed, 38 insertions(+), 38 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b3afe0688919..3eb94f36c77e 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -17,7 +17,7 @@ config ARM64 select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION select ARCH_ENABLE_MEMORY_HOTPLUG select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 - select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE + select ARCH_SUPPORTS_PMD_SOFTLEAF if TRANSPARENT_HUGEPAGE select ARCH_HAS_CACHE_LINE_SIZE select ARCH_HAS_CC_PLATFORM select ARCH_HAS_CPU_CACHE_INVALIDATE_MEMREGION diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 27689c62bd25..984badfa9a74 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -1534,10 +1534,10 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) #define __swp_entry_to_pte(swp) ((pte_t) { (swp).val }) -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF #define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) }) #define __swp_entry_to_pmd(swp) __pmd((swp).val) -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF */ /* * Ensure that there are not more swap files than can be encoded in the kernel diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index d8d252325017..550142b7ee10 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -12,7 +12,7 @@ config LOONGARCH select ARCH_NEEDS_DEFER_KASAN select ARCH_DISABLE_KASAN_INLINE select ARCH_ENABLE_MEMORY_HOTPLUG - select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE + select ARCH_SUPPORTS_PMD_SOFTLEAF if TRANSPARENT_HUGEPAGE select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI select ARCH_HAS_CPU_FINALIZE_INIT select ARCH_HAS_CURRENT_STACK_POINTER diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index e67e64ac6e8c..6f30aa8a6490 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -1060,7 +1060,7 @@ static inline pte_t *pmdp_ptep(pmd_t *pmd) #define pmd_mksoft_dirty(pmd) pte_pmd(pte_mksoft_dirty(pmd_pte(pmd))) #define pmd_clear_soft_dirty(pmd) pte_pmd(pte_clear_soft_dirty(pmd_pte(pmd))) -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF #define pmd_swp_mksoft_dirty(pmd) pte_pmd(pte_swp_mksoft_dirty(pmd_pte(pmd))) #define pmd_swp_soft_dirty(pmd) pte_swp_soft_dirty(pmd_pte(pmd)) #define pmd_swp_clear_soft_dirty(pmd) pte_pmd(pte_swp_clear_soft_dirty(pmd_pte(pmd))) diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platforms/Kconfig.cputype index bac02c83bb3e..4a0fa681bf98 100644 --- a/arch/powerpc/platforms/Kconfig.cputype +++ b/arch/powerpc/platforms/Kconfig.cputype @@ -112,7 +112,7 @@ config PPC_THP depends on PPC_RADIX_MMU || (PPC_64S_HASH_MMU && PAGE_SIZE_64KB) select HAVE_ARCH_TRANSPARENT_HUGEPAGE select HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD - select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE + select ARCH_SUPPORTS_PMD_SOFTLEAF if TRANSPARENT_HUGEPAGE choice prompt "CPU selection" diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 3f0a647218e4..bf7b271cccd8 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -22,7 +22,7 @@ config RISCV select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION select ARCH_ENABLE_MEMORY_HOTPLUG if SPARSEMEM_VMEMMAP select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 - select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE + select ARCH_SUPPORTS_PMD_SOFTLEAF if TRANSPARENT_HUGEPAGE select ARCH_HAS_BINFMT_FLAT select ARCH_HAS_CC_CAN_LINK select ARCH_HAS_CURRENT_STACK_POINTER diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 5d5756bda82e..2aa529e882d3 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -946,7 +946,7 @@ static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd) return pte_pmd(pte_clear_soft_dirty(pmd_pte(pmd))); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF static inline bool pmd_swp_soft_dirty(pmd_t pmd) { return pte_swp_soft_dirty(pmd_pte(pmd)); @@ -961,7 +961,7 @@ static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) { return pte_pmd(pte_swp_clear_soft_dirty(pmd_pte(pmd))); } -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF */ #endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */ static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, @@ -1208,10 +1208,10 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte) return __pte(pte_val(pte) & ~_PAGE_SWP_EXCLUSIVE); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF #define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) }) #define __swp_entry_to_pmd(swp) __pmd((swp).val) -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF */ /* * In the RV64 Linux scheme, we give the user half of the virtual-address space diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 84404e6778d5..0719f68de0f3 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -82,7 +82,7 @@ config S390 select ARCH_CORRECT_STACKTRACE_ON_KRETPROBE select ARCH_ENABLE_MEMORY_HOTPLUG if SPARSEMEM select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 - select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE + select ARCH_SUPPORTS_PMD_SOFTLEAF if TRANSPARENT_HUGEPAGE select ARCH_HAS_CC_CAN_LINK select ARCH_HAS_CPU_FINALIZE_INIT select ARCH_HAS_CURRENT_STACK_POINTER diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 859ce7c7d454..6faccfa63b09 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -903,7 +903,7 @@ static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd) return clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_SOFT_DIRTY)); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF #define pmd_swp_soft_dirty(pmd) pmd_soft_dirty(pmd) #define pmd_swp_mksoft_dirty(pmd) pmd_mksoft_dirty(pmd) #define pmd_swp_clear_soft_dirty(pmd) pmd_clear_soft_dirty(pmd) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index bdad90f210e4..50e00d8417aa 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -70,7 +70,7 @@ config X86 select ARCH_ENABLE_HUGEPAGE_MIGRATION if X86_64 && HUGETLB_PAGE && MIGRATION select ARCH_ENABLE_MEMORY_HOTPLUG if X86_64 select ARCH_ENABLE_SPLIT_PMD_PTLOCK if (PGTABLE_LEVELS > 2) && (X86_64 || X86_PAE) - select ARCH_ENABLE_THP_MIGRATION if X86_64 && TRANSPARENT_HUGEPAGE + select ARCH_SUPPORTS_PMD_SOFTLEAF if X86_64 && TRANSPARENT_HUGEPAGE select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI select ARCH_HAS_CPU_ATTACK_VECTORS if CPU_MITIGATIONS select ARCH_HAS_CACHE_LINE_SIZE diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index ac295ca6c92f..e0fd318d4004 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -1545,7 +1545,7 @@ static inline pte_t pte_swp_clear_soft_dirty(pte_t pte) return pte_clear_flags(pte, _PAGE_SWP_SOFT_DIRTY); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) { return pmd_set_flags(pmd, _PAGE_SWP_SOFT_DIRTY); diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index ad20f7f8c179..1487bf4af1a7 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -567,7 +567,7 @@ static inline struct folio *get_persistent_huge_zero_folio(void) static inline bool thp_migration_supported(void) { - return IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION); + return IS_ENABLED(CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF); } void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address, diff --git a/include/linux/leafops.h b/include/linux/leafops.h index 803d312437df..88888daeb018 100644 --- a/include/linux/leafops.h +++ b/include/linux/leafops.h @@ -81,7 +81,7 @@ static inline pte_t softleaf_to_pte(softleaf_t entry) return swp_entry_to_pte(entry); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF /** * softleaf_from_pmd() - Obtain a leaf entry from a PMD entry. * @pmd: PMD entry. @@ -587,7 +587,7 @@ static inline bool pte_is_uffd_marker(pte_t pte) return false; } -#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_ARCH_ENABLE_THP_MIGRATION) +#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF) /** * pmd_is_device_private_entry() - Check if PMD contains a device private swap @@ -606,14 +606,14 @@ static inline bool pmd_is_device_private_entry(pmd_t pmd) return softleaf_is_device_private(softleaf_from_pmd(pmd)); } -#else /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#else /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF */ static inline bool pmd_is_device_private_entry(pmd_t pmd) { return false; } -#endif /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF */ /** * pmd_is_migration_entry() - Does this PMD entry encode a migration entry? diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index d52d2a976e5a..e38f069c1c91 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -1839,7 +1839,7 @@ static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) #endif #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY -#ifndef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifndef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) { return pmd; diff --git a/include/linux/swapops.h b/include/linux/swapops.h index 8cfc966eae48..705a84154d28 100644 --- a/include/linux/swapops.h +++ b/include/linux/swapops.h @@ -321,7 +321,7 @@ static inline swp_entry_t make_guard_swp_entry(void) struct page_vma_mapped_walk; -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF extern int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, struct page *page); @@ -338,7 +338,7 @@ static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) return __swp_entry_to_pmd(arch_entry); } -#else /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#else /* CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF */ static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, struct page *page) { @@ -358,7 +358,7 @@ static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) return __pmd(0); } -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF */ #endif /* CONFIG_MMU */ #endif /* _LINUX_SWAPOPS_H */ diff --git a/mm/Kconfig b/mm/Kconfig index 9e0ca4824905..04fe5171bb8c 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -703,7 +703,7 @@ config DEVICE_MIGRATION config ARCH_ENABLE_HUGEPAGE_MIGRATION bool -config ARCH_ENABLE_THP_MIGRATION +config ARCH_SUPPORTS_PMD_SOFTLEAF bool config HUGETLB_PAGE_SIZE_VARIABLE diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c index 18411fb09aab..507fbd1ae7e5 100644 --- a/mm/debug_vm_pgtable.c +++ b/mm/debug_vm_pgtable.c @@ -751,7 +751,7 @@ static void __init pmd_leaf_soft_dirty_tests(struct pgtable_debug_args *args) pmd_t pmd; if (!pgtable_supports_soft_dirty() || - !IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION)) + !IS_ENABLED(CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF)) return; if (!has_transparent_hugepage()) @@ -819,7 +819,7 @@ static void __init pte_swap_tests(struct pgtable_debug_args *args) WARN_ON(memcmp(&pte1, &pte2, sizeof(pte1))); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF static void __init pmd_softleaf_tests(struct pgtable_debug_args *args) { swp_entry_t arch_entry; @@ -837,9 +837,9 @@ static void __init pmd_softleaf_tests(struct pgtable_debug_args *args) pmd2 = __swp_entry_to_pmd(arch_entry); WARN_ON(memcmp(&pmd1, &pmd2, sizeof(pmd1))); } -#else /* !CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#else /* !CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF */ static void __init pmd_softleaf_tests(struct pgtable_debug_args *args) { } -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF */ static void __init swap_migration_tests(struct pgtable_debug_args *args) { diff --git a/mm/hmm.c b/mm/hmm.c index c72c9ddfdb2f..4f3f627d2b47 100644 --- a/mm/hmm.c +++ b/mm/hmm.c @@ -331,7 +331,7 @@ static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr, return hmm_vma_fault(addr, end, required_fault, walk); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start, unsigned long end, unsigned long *hmm_pfns, pmd_t pmd) @@ -391,7 +391,7 @@ static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start, return -EFAULT; return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR); } -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF */ static int hmm_vma_walk_pmd(pmd_t *pmdp, unsigned long start, diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 4aca9cdff73f..4b6e441ae931 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -4864,7 +4864,7 @@ static int __init split_huge_pages_debugfs(void) late_initcall(split_huge_pages_debugfs); #endif -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, struct page *page) { diff --git a/mm/migrate.c b/mm/migrate.c index 49e10feeb094..c48dc5127e6a 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -362,7 +362,7 @@ static bool remove_migration_pte(struct folio *folio, idx = linear_page_index(vma, pvmw.address) - pvmw.pgoff; new = folio_page(folio, idx); -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF /* PMD-mapped THP migration entry */ if (!pvmw.pte) { VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) || @@ -545,7 +545,7 @@ void migration_entry_wait_huge(struct vm_area_struct *vma, unsigned long addr, p } #endif -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd) { spinlock_t *ptl; diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 36287e958b2a..2f8b646302c2 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -771,7 +771,7 @@ int migrate_vma_setup(struct migrate_vma *args) } EXPORT_SYMBOL(migrate_vma_setup); -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF /** * migrate_vma_insert_huge_pmd_page: Insert a huge folio into @migrate->vma->vm_mm * at @addr. folio is already allocated as a part of the migration process with @@ -926,7 +926,7 @@ static int migrate_vma_split_unmapped_folio(struct migrate_vma *migrate, migrate->src[i+idx] = migrate_pfn(pfn + i) | flags; return ret; } -#else /* !CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#else /* !CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF */ static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate, unsigned long addr, struct page *page, @@ -947,7 +947,7 @@ static int migrate_vma_split_unmapped_folio(struct migrate_vma *migrate, static unsigned long migrate_vma_nr_pages(unsigned long *src) { unsigned long nr = 1; -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF if (*src & MIGRATE_PFN_COMPOUND) nr = HPAGE_PMD_NR; #else diff --git a/mm/rmap.c b/mm/rmap.c index b93caabd186f..0fb7a1b82cf3 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -2472,7 +2472,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, page_vma_mapped_walk_restart(&pvmw); continue; } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF pmdval = pmdp_get(pvmw.pmd); if (likely(pmd_present(pmdval))) pfn = pmd_pfn(pmdval); -- 2.53.0-Meta