Several struct page fields have the same initial values for every page in a zone. __init_single_page() initializes this state as part of its per-page setup, while the shared vmemmap tail path open-codes only a subset. Factor the PFN-independent state into memmap_init_page() and use it for both regular and shared tail struct pages. Keep section-, PFN- and instance-specific initialization in __init_single_page(). This also initializes the last cpupid and KASAN tag for shared tail struct pages consistently with regular struct pages. Since memmap_init_page() marks ZONE_DEVICE pages reserved, remove the redundant update from the memory hotplug path. Signed-off-by: Muchun Song --- v3: - Rename memmap_init_page_common() to memmap_init_page() (suggested by Mike Rapoport) - Remove redundant ZONE_DEVICE reserved-page initialization (suggested by Mike Rapoport) v2: - Remove __meminit from memmap_init_page_common() so HVO shared tail pages can call it after init text is freed (reported by Sashiko). --- include/linux/mm.h | 8 -------- mm/mm_init.c | 37 +++++++++++++++++++++---------------- mm/mm_init.h | 1 + mm/sparse-vmemmap.c | 7 +------ 4 files changed, 23 insertions(+), 30 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 8883b780b76d..d8bafcd92791 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3096,14 +3096,6 @@ static inline void set_page_node(struct page *page, unsigned long node) page->flags.f |= (node & NODES_MASK) << NODES_PGSHIFT; } -static inline void set_page_links(struct page *page, enum zone_type zone, - unsigned long node, unsigned long pfn) -{ - set_page_zone(page, zone); - set_page_node(page, node); - set_page_section_from_pfn(page, pfn); -} - /** * folio_nr_pages - The number of pages in the folio. * @folio: The folio. diff --git a/mm/mm_init.c b/mm/mm_init.c index 0129152b74d8..08d6e47bde22 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -602,15 +602,27 @@ static inline void vmemmap_optimization_verify_zone(struct page *page) page_zone_id(page)); } -void __meminit __init_single_page(struct page *page, unsigned long pfn, - unsigned long zone, int nid) +/* Initialize PFN-independent fields common to pages in the same zone. */ +void memmap_init_page(struct page *page, enum zone_type zone, int nid) { mm_zero_struct_page(page); - set_page_links(page, zone, nid, pfn); - init_page_count(page); - atomic_set(&page->_mapcount, -1); + set_page_zone(page, zone); + set_page_node(page, nid); page_cpupid_reset_last(page); page_kasan_tag_reset(page); + atomic_set(&page->_mapcount, -1); +#ifdef CONFIG_ZONE_DEVICE + if (zone == ZONE_DEVICE) + __SetPageReserved(page); +#endif +} + +void __meminit __init_single_page(struct page *page, unsigned long pfn, + unsigned long zone, int nid) +{ + memmap_init_page(page, zone, nid); + set_page_section_from_pfn(page, pfn); + init_page_count(page); INIT_LIST_HEAD(&page->lru); #ifdef WANT_PAGE_VIRTUAL @@ -891,11 +903,11 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone { unsigned long pfn, end_pfn = start_pfn + size; struct page *page; + const struct zone *z = &NODE_DATA(nid)->node_zones[zone]; if (highest_memmap_pfn < end_pfn - 1) highest_memmap_pfn = end_pfn - 1; -#ifdef CONFIG_ZONE_DEVICE /* * Honor reservation requested by the driver for this ZONE_DEVICE * memory. We limit the total number of pages to initialize to just @@ -903,7 +915,7 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone * ZONE_DEVICE page initialization until after we have released * the hotplug lock. */ - if (zone == ZONE_DEVICE) { + if (zone_is_zone_device(z)) { if (!altmap) return; @@ -911,7 +923,6 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone start_pfn += altmap->reserve; end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); } -#endif for (pfn = start_pfn; pfn < end_pfn; ) { const unsigned int order = pfn_to_section_compound_order(pfn); @@ -938,14 +949,8 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone page = pfn_to_page(pfn); __init_single_page(page, pfn, zone, nid); - if (context == MEMINIT_HOTPLUG) { -#ifdef CONFIG_ZONE_DEVICE - if (zone == ZONE_DEVICE) - __SetPageReserved(page); - else -#endif - __SetPageOffline(page); - } + if (context == MEMINIT_HOTPLUG && !zone_is_zone_device(z)) + __SetPageOffline(page); if (pageblock_aligned(pfn)) cond_resched(); diff --git a/mm/mm_init.h b/mm/mm_init.h index 0034e37dbabd..dec22844f41e 100644 --- a/mm/mm_init.h +++ b/mm/mm_init.h @@ -109,6 +109,7 @@ extern bool mirrored_kernelcore; bool memblock_has_mirror(void); void memblock_free_all(void); +void memmap_init_page(struct page *page, enum zone_type zone, int nid); void __meminit __init_single_page(struct page *page, unsigned long pfn, unsigned long zone, int nid); diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index 9b80dd7326f7..51fa77982ca5 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -212,12 +212,7 @@ struct page __ref *vmemmap_shared_tail_page(unsigned int order, struct zone *zon for (int i = 0; i < PAGE_SIZE / sizeof(struct page); i++) { page = (struct page *)addr + i; - mm_zero_struct_page(page); - atomic_set(&page->_mapcount, -1); - set_page_node(page, zone_to_nid(zone)); - set_page_zone(page, zone_idx(zone)); - if (zone_is_zone_device(zone)) - __SetPageReserved(page); + memmap_init_page(page, zone_idx(zone), zone_to_nid(zone)); prep_compound_tail(page, NULL, order); } -- 2.54.0