From: Qi Zheng On a 64-bit system, madvise(MADV_DONTNEED) may cause a large number of empty PTE page table pages (such as 100GB+). To resolve this problem, first enable MMU_GATHER_RCU_TABLE_FREE to prepare for enabling the PT_RECLAIM feature, which resolves this problem. Signed-off-by: Qi Zheng Cc: Richard Henderson Cc: Matt Turner --- arch/alpha/Kconfig | 1 + arch/alpha/include/asm/tlb.h | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/arch/alpha/Kconfig b/arch/alpha/Kconfig index 80367f2cf821c..681ed894d9e72 100644 --- a/arch/alpha/Kconfig +++ b/arch/alpha/Kconfig @@ -40,6 +40,7 @@ config ALPHA select MMU_GATHER_NO_RANGE select SPARSEMEM_EXTREME if SPARSEMEM select ZONE_DMA + select MMU_GATHER_RCU_TABLE_FREE help The Alpha is a 64-bit general-purpose processor designed and marketed by the Digital Equipment Corporation of blessed memory, diff --git a/arch/alpha/include/asm/tlb.h b/arch/alpha/include/asm/tlb.h index 4f79e331af5ea..4fe5a901720f0 100644 --- a/arch/alpha/include/asm/tlb.h +++ b/arch/alpha/include/asm/tlb.h @@ -4,7 +4,9 @@ #include -#define __pte_free_tlb(tlb, pte, address) pte_free((tlb)->mm, pte) -#define __pmd_free_tlb(tlb, pmd, address) pmd_free((tlb)->mm, pmd) - +#define __pte_free_tlb(tlb, pte, address) \ + tlb_remove_ptdesc((tlb), page_ptdesc(pte)) +#define __pmd_free_tlb(tlb, pmd, address) \ + tlb_remove_ptdesc((tlb), virt_to_ptdesc(pmd)) + #endif -- 2.20.1 From: Qi Zheng On a 64-bit system, madvise(MADV_DONTNEED) may cause a large number of empty PTE page table pages (such as 100GB+). To resolve this problem, first enable MMU_GATHER_RCU_TABLE_FREE to prepare for enabling the PT_RECLAIM feature, which resolves this problem. Signed-off-by: Qi Zheng Cc: Vineet Gupta --- arch/arc/Kconfig | 1 + arch/arc/include/asm/pgalloc.h | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig index f27e6b90428e4..47db93952386d 100644 --- a/arch/arc/Kconfig +++ b/arch/arc/Kconfig @@ -54,6 +54,7 @@ config ARC select HAVE_ARCH_JUMP_LABEL if ISA_ARCV2 && !CPU_ENDIAN_BE32 select TRACE_IRQFLAGS_SUPPORT select HAVE_EBPF_JIT if ISA_ARCV2 + select MMU_GATHER_RCU_TABLE_FREE config LOCKDEP_SUPPORT def_bool y diff --git a/arch/arc/include/asm/pgalloc.h b/arch/arc/include/asm/pgalloc.h index dfae070fe8d55..b1c6619435613 100644 --- a/arch/arc/include/asm/pgalloc.h +++ b/arch/arc/include/asm/pgalloc.h @@ -72,7 +72,8 @@ static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp) set_p4d(p4dp, __p4d((unsigned long)pudp)); } -#define __pud_free_tlb(tlb, pmd, addr) pud_free((tlb)->mm, pmd) +#define __pud_free_tlb(tlb, pud, addr) \ + tlb_remove_ptdesc((tlb), virt_to_ptdesc(pud)) #endif @@ -83,10 +84,12 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp) set_pud(pudp, __pud((unsigned long)pmdp)); } -#define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) +#define __pmd_free_tlb(tlb, pmd, addr) \ + tlb_remove_ptdesc((tlb), virt_to_ptdesc(pmd)) #endif -#define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte) +#define __pte_free_tlb(tlb, pte, addr) \ + tlb_remove_ptdesc((tlb), page_ptdesc(pte)) #endif /* _ASM_ARC_PGALLOC_H */ -- 2.20.1 From: Qi Zheng On a 64-bit system, madvise(MADV_DONTNEED) may cause a large number of empty PTE page table pages (such as 100GB+). To resolve this problem, first enable MMU_GATHER_RCU_TABLE_FREE to prepare for enabling the PT_RECLAIM feature, which resolves this problem. Signed-off-by: Qi Zheng Cc: Huacai Chen Cc: WANG Xuerui --- arch/loongarch/Kconfig | 1 + arch/loongarch/include/asm/pgalloc.h | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 5b1116733d881..3bf2f2a9cd647 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -210,6 +210,7 @@ config LOONGARCH select USER_STACKTRACE_SUPPORT select VDSO_GETRANDOM select ZONE_DMA32 + select MMU_GATHER_RCU_TABLE_FREE config 32BIT bool diff --git a/arch/loongarch/include/asm/pgalloc.h b/arch/loongarch/include/asm/pgalloc.h index 1c63a9d9a6d35..0539d04bf1525 100644 --- a/arch/loongarch/include/asm/pgalloc.h +++ b/arch/loongarch/include/asm/pgalloc.h @@ -79,7 +79,8 @@ static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address) return pmd; } -#define __pmd_free_tlb(tlb, x, addr) pmd_free((tlb)->mm, x) +#define __pmd_free_tlb(tlb, x, addr) \ + tlb_remove_ptdesc((tlb), virt_to_ptdesc(x)) #endif @@ -99,7 +100,8 @@ static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long address) return pud; } -#define __pud_free_tlb(tlb, x, addr) pud_free((tlb)->mm, x) +#define __pud_free_tlb(tlb, x, addr) \ + tlb_remove_ptdesc((tlb), virt_to_ptdesc(x)) #endif /* __PAGETABLE_PUD_FOLDED */ -- 2.20.1 From: Qi Zheng On a 64-bit system, madvise(MADV_DONTNEED) may cause a large number of empty PTE page table pages (such as 100GB+). To resolve this problem, first enable MMU_GATHER_RCU_TABLE_FREE to prepare for enabling the PT_RECLAIM feature, which resolves this problem. Signed-off-by: Qi Zheng Cc: Thomas Bogendoerfer --- arch/mips/Kconfig | 1 + arch/mips/include/asm/pgalloc.h | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index e8683f58fd3e2..0ee8820a354c4 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -108,6 +108,7 @@ config MIPS select TRACE_IRQFLAGS_SUPPORT select ARCH_HAS_ELFCORE_COMPAT select HAVE_ARCH_KCSAN if 64BIT + select MMU_GATHER_RCU_TABLE_FREE config MIPS_FIXUP_BIGPHYS_ADDR bool diff --git a/arch/mips/include/asm/pgalloc.h b/arch/mips/include/asm/pgalloc.h index 942af87f1cddb..c00f445045f43 100644 --- a/arch/mips/include/asm/pgalloc.h +++ b/arch/mips/include/asm/pgalloc.h @@ -72,7 +72,8 @@ static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address) return pmd; } -#define __pmd_free_tlb(tlb, x, addr) pmd_free((tlb)->mm, x) +#define __pmd_free_tlb(tlb, x, addr) \ + tlb_remove_ptdesc((tlb), virt_to_ptdesc(x)) #endif @@ -98,7 +99,8 @@ static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) set_p4d(p4d, __p4d((unsigned long)pud)); } -#define __pud_free_tlb(tlb, x, addr) pud_free((tlb)->mm, x) +#define __pud_free_tlb(tlb, x, addr) \ + tlb_remove_ptdesc((tlb), virt_to_ptdesc(x)) #endif /* __PAGETABLE_PUD_FOLDED */ -- 2.20.1 From: Qi Zheng On a 64-bit system, madvise(MADV_DONTNEED) may cause a large number of empty PTE page table pages (such as 100GB+). To resolve this problem, first enable MMU_GATHER_RCU_TABLE_FREE to prepare for enabling the PT_RECLAIM feature, which resolves this problem. Signed-off-by: Qi Zheng Cc: "James E.J. Bottomley" Cc: Helge Deller --- arch/parisc/Kconfig | 1 + arch/parisc/include/asm/tlb.h | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index 47fd9662d8005..946cbe21a4118 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -92,6 +92,7 @@ config PARISC select TRACE_IRQFLAGS_SUPPORT select HAVE_FUNCTION_DESCRIPTORS if 64BIT select PCI_MSI_ARCH_FALLBACKS if PCI_MSI + select MMU_GATHER_RCU_TABLE_FREE help The PA-RISC microprocessor is designed by Hewlett-Packard and used diff --git a/arch/parisc/include/asm/tlb.h b/arch/parisc/include/asm/tlb.h index 44235f367674d..ab7d4113df61a 100644 --- a/arch/parisc/include/asm/tlb.h +++ b/arch/parisc/include/asm/tlb.h @@ -5,8 +5,10 @@ #include #if CONFIG_PGTABLE_LEVELS == 3 -#define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) +#define __pmd_free_tlb(tlb, pmd, addr) \ + tlb_remove_ptdesc((tlb), virt_to_ptdesc(pmd)) #endif -#define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte) +#define __pte_free_tlb(tlb, pte, addr) \ + tlb_remove_ptdesc((tlb), page_ptdesc(pte)) #endif -- 2.20.1 From: Qi Zheng On a 64-bit system, madvise(MADV_DONTNEED) may cause a large number of empty PTE page table pages (such as 100GB+). To resolve this problem, first enable MMU_GATHER_RCU_TABLE_FREE to prepare for enabling the PT_RECLAIM feature, which resolves this problem. Signed-off-by: Qi Zheng Cc: Richard Weinberger Cc: Anton Ivanov Cc: Johannes Berg --- arch/um/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/um/Kconfig b/arch/um/Kconfig index 097c6a6265ef3..47a41bc77bb24 100644 --- a/arch/um/Kconfig +++ b/arch/um/Kconfig @@ -41,6 +41,7 @@ config UML select HAVE_SYSCALL_TRACEPOINTS select THREAD_INFO_IN_TASK select SPARSE_IRQ + select MMU_GATHER_RCU_TABLE_FREE config MMU bool -- 2.20.1 From: Qi Zheng Make PT_RECLAIM depend on MMU_GATHER_RCU_TABLE_FREE so that PT_RECLAIM can be enabled by default on all architectures that support MMU_GATHER_RCU_TABLE_FREE. Considering that a large number of PTE page table pages (such as 100GB+) can only be caused on a 64-bit system, let PT_RECLAIM also depend on 64BIT. Signed-off-by: Qi Zheng --- arch/x86/Kconfig | 1 - mm/Kconfig | 6 +----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index eac2e86056902..96bff81fd4787 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -330,7 +330,6 @@ config X86 select FUNCTION_ALIGNMENT_4B imply IMA_SECURE_AND_OR_TRUSTED_BOOT if EFI select HAVE_DYNAMIC_FTRACE_NO_PATCHABLE - select ARCH_SUPPORTS_PT_RECLAIM if X86_64 select ARCH_SUPPORTS_SCHED_SMT if SMP select SCHED_SMT if SMP select ARCH_SUPPORTS_SCHED_CLUSTER if SMP diff --git a/mm/Kconfig b/mm/Kconfig index a5a90b169435d..e795fbd69e50c 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -1440,14 +1440,10 @@ config ARCH_HAS_USER_SHADOW_STACK The architecture has hardware support for userspace shadow call stacks (eg, x86 CET, arm64 GCS or RISC-V Zicfiss). -config ARCH_SUPPORTS_PT_RECLAIM - def_bool n - config PT_RECLAIM bool "reclaim empty user page table pages" default y - depends on ARCH_SUPPORTS_PT_RECLAIM && MMU && SMP - select MMU_GATHER_RCU_TABLE_FREE + depends on MMU_GATHER_RCU_TABLE_FREE && MMU && SMP && 64BIT help Try to reclaim empty user page table pages in paths other than munmap and exit_mmap path. -- 2.20.1