On x86-64, vmemmap_populate() normally calls sync_global_pgds() to keep the page tables in sync. However, when vmemmap optimization for compound devmaps is enabled, vmemmap_populate_compound_pages() is called directly from __populate_section_memmap(), bypassing the architecture- specific vmemmap_populate() entirely. This skips the sync on x86-64 and can later trigger vmemmap-access faults. Fix this by moving the vmemmap_can_optimize() dispatch from __populate_section_memmap() into the generic helpers -- vmemmap_populate_basepages() and vmemmap_populate_hugepages(). This way, the architecture vmemmap_populate() is always invoked first, ensuring any arch-specific post-population steps (e.g. sync_global_pgds()) are executed before returning. Architectures that override vmemmap_populate() (e.g. powerpc) handle the optimization dispatch in their own implementation instead. Fixes: 4917f55b4ef9 ("mm/sparse-vmemmap: improve memory savings for compound devmaps") Signed-off-by: Muchun Song --- arch/powerpc/include/asm/book3s/64/radix.h | 6 ------ arch/powerpc/mm/book3s64/radix_pgtable.c | 16 ++++++++++----- mm/sparse-vmemmap.c | 24 +++++++++++----------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/arch/powerpc/include/asm/book3s/64/radix.h b/arch/powerpc/include/asm/book3s/64/radix.h index bde07c6f900f..2600defa2dc2 100644 --- a/arch/powerpc/include/asm/book3s/64/radix.h +++ b/arch/powerpc/include/asm/book3s/64/radix.h @@ -357,11 +357,5 @@ int radix__remove_section_mapping(unsigned long start, unsigned long end); #define vmemmap_can_optimize vmemmap_can_optimize bool vmemmap_can_optimize(struct vmem_altmap *altmap, struct dev_pagemap *pgmap); #endif - -#define vmemmap_populate_compound_pages vmemmap_populate_compound_pages -int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, - unsigned long start, - unsigned long end, int node, - struct dev_pagemap *pgmap); #endif /* __ASSEMBLER__ */ #endif diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c index 568500343e5f..21fece355fbb 100644 --- a/arch/powerpc/mm/book3s64/radix_pgtable.c +++ b/arch/powerpc/mm/book3s64/radix_pgtable.c @@ -1109,7 +1109,10 @@ static inline pte_t *vmemmap_pte_alloc(pmd_t *pmdp, int node, return pte_offset_kernel(pmdp, address); } - +static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, + unsigned long start, + unsigned long end, int node, + struct dev_pagemap *pgmap); int __meminit radix__vmemmap_populate(unsigned long start, unsigned long end, int node, struct vmem_altmap *altmap, struct dev_pagemap *pgmap) @@ -1122,6 +1125,9 @@ int __meminit radix__vmemmap_populate(unsigned long start, unsigned long end, in pmd_t *pmd; pte_t *pte; + if (vmemmap_can_optimize(altmap, pgmap)) + return vmemmap_populate_compound_pages(page_to_pfn((struct page *)start), + start, end, node, pgmap); /* * If altmap is present, Make sure we align the start vmemmap addr * to PAGE_SIZE so that we calculate the correct start_pfn in @@ -1303,10 +1309,10 @@ static pte_t * __meminit vmemmap_compound_tail_page(unsigned long addr, return pte; } -int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, - unsigned long start, - unsigned long end, int node, - struct dev_pagemap *pgmap) +static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, + unsigned long start, + unsigned long end, int node, + struct dev_pagemap *pgmap) { /* * we want to map things as base page size mapping so that diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index f5245647afee..7f684ed3479e 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -296,10 +296,16 @@ static int __meminit vmemmap_populate_range(unsigned long start, return 0; } +static int __meminit vmemmap_populate_compound_pages(unsigned long start, + unsigned long end, int node, + struct dev_pagemap *pgmap); + int __meminit vmemmap_populate_basepages(unsigned long start, unsigned long end, int node, struct vmem_altmap *altmap, struct dev_pagemap *pgmap) { + if (vmemmap_can_optimize(altmap, pgmap)) + return vmemmap_populate_compound_pages(start, end, node, pgmap); return vmemmap_populate_range(start, end, node, altmap, -1, 0); } @@ -411,6 +417,9 @@ int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end, pud_t *pud; pmd_t *pmd; + if (vmemmap_can_optimize(altmap, pgmap)) + return vmemmap_populate_compound_pages(start, end, node, pgmap); + for (addr = start; addr < end; addr = next) { next = pmd_addr_end(addr, end); @@ -453,7 +462,6 @@ int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end, return 0; } -#ifndef vmemmap_populate_compound_pages /* * For compound pages bigger than section size (e.g. x86 1G compound * pages with 2M subsection size) fill the rest of sections as tail @@ -491,14 +499,14 @@ static pte_t * __meminit compound_section_tail_page(unsigned long addr) return pte; } -static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, - unsigned long start, +static int __meminit vmemmap_populate_compound_pages(unsigned long start, unsigned long end, int node, struct dev_pagemap *pgmap) { unsigned long size, addr; pte_t *pte; int rc; + unsigned long start_pfn = page_to_pfn((struct page *)start); if (reuse_compound_section(start_pfn, pgmap)) { pte = compound_section_tail_page(start); @@ -544,26 +552,18 @@ static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, return 0; } -#endif - struct page * __meminit __populate_section_memmap(unsigned long pfn, unsigned long nr_pages, int nid, struct vmem_altmap *altmap, struct dev_pagemap *pgmap) { unsigned long start = (unsigned long) pfn_to_page(pfn); unsigned long end = start + nr_pages * sizeof(struct page); - int r; if (WARN_ON_ONCE(!IS_ALIGNED(pfn, PAGES_PER_SUBSECTION) || !IS_ALIGNED(nr_pages, PAGES_PER_SUBSECTION))) return NULL; - if (vmemmap_can_optimize(altmap, pgmap)) - r = vmemmap_populate_compound_pages(pfn, start, end, nid, pgmap); - else - r = vmemmap_populate(start, end, nid, altmap, pgmap); - - if (r < 0) + if (vmemmap_populate(start, end, nid, altmap, pgmap)) return NULL; return pfn_to_page(pfn); -- 2.20.1