ZONE_DEVICE initializes compound struct pages while their refcounts remain frozen. The setup is open-coded in memmap_init_compound(), including the per-page initialization and compound tail preparation. Factor the common setup into memmap_init_compound_page_frozen(). Keep the pgmap association and MEMORY_DEVICE_GENERIC refcount handling in the ZONE_DEVICE-specific path. This provides an MM-internal helper for other early memmap users to share. No functional change is intended. Signed-off-by: Muchun Song --- mm/mm_init.c | 137 +++++++++++++++++++++++++-------------------------- mm/mm_init.h | 2 + 2 files changed, 70 insertions(+), 69 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index 6cf74f6840de..458496271a44 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -719,6 +719,40 @@ static __meminit void pageblock_migratetype_init_range(unsigned long pfn, } } +static inline void memmap_init_page_from_template(struct page *page, + unsigned long pfn, const struct zone *zone, struct page *template) +{ + set_page_section_from_pfn(template, pfn); +#ifdef WANT_PAGE_VIRTUAL + if (!is_highmem_idx(zone_idx(zone))) + set_page_address(template, __va(pfn << PAGE_SHIFT)); +#endif + memcpy_nontemporal(page, template, sizeof(*page)); + vmemmap_optimization_verify_zone(page); +} + +void __meminit memmap_init_compound_page_frozen(struct page *head, + unsigned long pfn, const struct zone *zone, unsigned int order) +{ + unsigned long nr_pages = 1UL << order; + struct page template; + + memmap_init_page_common(&template, zone_idx(zone), zone_to_nid(zone)); + prep_compound_tail(&template, head, order); + + /* Initialize the head page. */ + __init_single_page(head, pfn, zone_idx(zone), zone_to_nid(zone)); + set_page_count(head, 0); + __SetPageHead(head); + + if (vmemmap_optimizable_order(pfn_to_section_compound_order(pfn))) + nr_pages = VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES; + /* Initialize the tail pages from PFN-independent template page. */ + for (unsigned long i = 1; i < nr_pages; i++) + memmap_init_page_from_template(head + i, pfn + i, zone, &template); + prep_compound_head(head, order); +} + struct zone __meminit *pfn_to_zone(unsigned long pfn, int nid) { pg_data_t *pgdat = NODE_DATA(nid); @@ -1044,56 +1078,38 @@ static void __init memmap_init(void) } #ifdef CONFIG_ZONE_DEVICE -static void zone_device_page_init_from_template(struct page *page, - unsigned long pfn, struct page *template) -{ - set_page_section_from_pfn(template, pfn); -#ifdef WANT_PAGE_VIRTUAL - if (!is_highmem_idx(ZONE_DEVICE)) - set_page_address(template, __va(pfn << PAGE_SHIFT)); -#endif - memcpy_nontemporal(page, template, sizeof(*page)); - vmemmap_optimization_verify_zone(page); -} - -/* - * With compound page geometry and when struct pages are stored in ram most - * tail pages are reused. Consequently, the amount of unique struct pages to - * initialize is a lot smaller that the total amount of struct pages being - * mapped. This is a paired / mild layering violation with explicit knowledge - * of how the sparse_vmemmap internals handle compound pages in the lack - * of an altmap. - */ -static inline unsigned long compound_nr_pages(unsigned long pfn, - struct dev_pagemap *pgmap) +static void memmap_init_zone_device_head_page(struct page *head, struct dev_pagemap *pgmap) { - const struct mem_section *ms = __pfn_to_section(pfn); - - if (!section_vmemmap_optimizable(ms)) - return pgmap_vmemmap_nr(pgmap); - - return VMEMMAP_OPTIMIZATION_PAGES * (PAGE_SIZE / sizeof(struct page)); + /* + * MEMORY_DEVICE_GENERIC pages start with a refcount of one; other + * ZONE_DEVICE pages raise it when handed out. Use atomic_set() because + * @head may be a stack template, while init_page_count() expects a + * memmap-backed page. + */ + if (pgmap->type == MEMORY_DEVICE_GENERIC) + atomic_set(&head->_refcount, 1); + /* + * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer + * and zone_device_data. It is a bug if a ZONE_DEVICE page is + * ever freed or placed on a driver-private list. + */ + ((struct folio *)head)->pgmap = pgmap; + head->zone_device_data = NULL; } -static void __ref memmap_init_compound(struct page *head, - unsigned long head_pfn, - unsigned long zone_idx, int nid, - struct dev_pagemap *pgmap, - unsigned long nr_pages) +static void memmap_init_zone_device_basepages(unsigned long start, unsigned long end, + const struct zone *zone, struct dev_pagemap *pgmap) { - unsigned long pfn, end_pfn = head_pfn + nr_pages; - unsigned int order = pgmap->vmemmap_shift; struct page template; - memmap_init_page_common(&template, zone_idx, nid); - prep_compound_tail(&template, head, order); + memmap_init_page_common(&template, zone_idx(zone), zone_to_nid(zone)); + memmap_init_zone_device_head_page(&template, pgmap); - __SetPageHead(head); - /* Initialize the tail pages from template. */ - for (pfn = head_pfn + 1; pfn < end_pfn; pfn++) - zone_device_page_init_from_template(pfn_to_page(pfn), pfn, - &template); - prep_compound_head(head, order); + for (unsigned long pfn = start; pfn < end; pfn++) { + memmap_init_page_from_template(pfn_to_page(pfn), pfn, zone, &template); + if (IS_ALIGNED(pfn, PAGES_PER_SECTION)) + cond_resched(); + } } void __ref memmap_init_zone_device(struct zone *zone, @@ -1102,16 +1118,11 @@ void __ref memmap_init_zone_device(struct zone *zone, struct dev_pagemap *pgmap) { unsigned long pfn, end_pfn = start_pfn + nr_pages; - struct pglist_data *pgdat = zone->zone_pgdat; struct vmem_altmap *altmap = pgmap_altmap(pgmap); unsigned int pfns_per_compound = pgmap_vmemmap_nr(pgmap); - unsigned long zone_idx = zone_idx(zone); unsigned long start = jiffies; - int nid = pgdat->node_id; - struct page template; - struct page *page; - if (WARN_ON_ONCE(!pgmap || zone_idx != ZONE_DEVICE)) + if (WARN_ON_ONCE(!pgmap || !zone_is_zone_device(zone))) return; /* @@ -1129,32 +1140,20 @@ void __ref memmap_init_zone_device(struct zone *zone, VM_WARN_ON(!IS_ALIGNED(start_pfn | nr_pages, pfns_per_compound)); - memmap_init_page_common(&template, zone_idx, nid); - page_folio(&template)->pgmap = pgmap; - /* - * MEMORY_DEVICE_GENERIC pages regain a refcount of one in the free - * path. The remaining ZONE_DEVICE types start from zero here and raise - * the count again when the allocator or driver hands the page out. - */ - if (pgmap->type == MEMORY_DEVICE_GENERIC) - atomic_set(&template._refcount, 1); + if (pfns_per_compound == 1) { + memmap_init_zone_device_basepages(start_pfn, end_pfn, zone, pgmap); + goto out; + } - /* Initialize the head pages from template. */ for (pfn = start_pfn; pfn < end_pfn; pfn += pfns_per_compound) { - page = pfn_to_page(pfn); - - zone_device_page_init_from_template(page, pfn, &template); + struct page *head = pfn_to_page(pfn); + memmap_init_compound_page_frozen(head, pfn, zone, pgmap->vmemmap_shift); + memmap_init_zone_device_head_page(head, pgmap); if (IS_ALIGNED(pfn, PAGES_PER_SECTION)) cond_resched(); - - if (pfns_per_compound == 1) - continue; - - memmap_init_compound(page, pfn, zone_idx, nid, pgmap, - compound_nr_pages(pfn, pgmap)); } - +out: pageblock_migratetype_init_range(start_pfn, nr_pages, MIGRATE_MOVABLE, /* isolate */ false, /* atomic */ false); diff --git a/mm/mm_init.h b/mm/mm_init.h index 0df0644c703c..1560ac9e9286 100644 --- a/mm/mm_init.h +++ b/mm/mm_init.h @@ -113,5 +113,7 @@ void __meminit memmap_init_page_common(struct page *page, enum zone_type zone, int nid); void __meminit __init_single_page(struct page *page, unsigned long pfn, unsigned long zone, int nid); +void __meminit memmap_init_compound_page_frozen(struct page *head, unsigned long pfn, + const struct zone *zone, unsigned int order); #endif /* __MM_MM_INIT_H */ -- 2.54.0