memmap_init_zone_device() currently seeds its reusable head and tail templates by fully initializing the first real struct page and copying it back into a stack object. This keeps the first page outside the common loops and leaves __init_zone_device_page() only for template construction. Build both templates directly from the PFN-independent state. Add the head-specific pgmap and refcount state to the head template and the compound metadata to the tail template, then initialize every page through zone_device_page_init_from_template(). This removes the special handling for the first head and tail pages without changing their final state. Signed-off-by: Muchun Song --- v3: - Rebase on zone_device_page_init_from_template() after integrating the template helper series (suggested by Mike Rapoport) v2: - Rebase on memmap_init_page_from_template(). --- mm/mm_init.c | 94 ++++++++-------------------------------------------- 1 file changed, 14 insertions(+), 80 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index 08d6e47bde22..75d8d17601ed 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1036,48 +1036,6 @@ static void __init memmap_init(void) } #ifdef CONFIG_ZONE_DEVICE -static void __ref __init_zone_device_page(struct page *page, unsigned long pfn, - unsigned long zone_idx, int nid, - struct dev_pagemap *pgmap) -{ - - __init_single_page(page, pfn, zone_idx, nid); - - /* - * Mark page reserved as it will need to wait for onlining - * phase for it to be fully associated with a zone. - * - * We can use the non-atomic __set_bit operation for setting - * the flag as we are still initializing the pages. - */ - __SetPageReserved(page); - - /* - * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer - * and zone_device_data. It is a bug if a ZONE_DEVICE page is - * ever freed or placed on a driver-private list. - */ - page_folio(page)->pgmap = pgmap; - page->zone_device_data = NULL; - - /* - * MEMORY_DEVICE_GENERIC pages regain a refcount of 1 in the free - * path. The remaining ZONE_DEVICE types start from 0 here and raise - * the count again when the allocator or driver hands the page out. - */ - switch (pgmap->type) { - case MEMORY_DEVICE_FS_DAX: - case MEMORY_DEVICE_PRIVATE: - case MEMORY_DEVICE_COHERENT: - case MEMORY_DEVICE_PCI_P2PDMA: - set_page_count(page, 0); - break; - - case MEMORY_DEVICE_GENERIC: - break; - } -} - static void zone_device_page_init_from_template(struct page *page, unsigned long pfn, struct page *template) { @@ -1118,30 +1076,13 @@ static void __ref memmap_init_compound(struct page *head, unsigned long pfn, end_pfn = head_pfn + nr_pages; unsigned int order = pgmap->vmemmap_shift; struct page template; - struct page *page; - /* - * We have to initialize the pages, including setting up page links. - * prep_compound_page() does not take care of that, so instead we - * open-code prep_compound_page() so we can take care of initializing - * the pages in the same go. - */ - __SetPageHead(head); - - /* - * All tails of the same compound page share the state established by - * prep_compound_tail(). Reuse one tail template for the whole range and - * refresh only the PFN-dependent fields in that template before each copy. - */ - pfn = head_pfn + 1; - page = pfn_to_page(pfn); - __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); - prep_compound_tail(page, head, order); - set_page_count(page, 0); - memcpy(&template, page, sizeof(*page)); + memmap_init_page(&template, zone_idx, nid); + prep_compound_tail(&template, head, order); - /* Initialize the remaining tail pages from template. */ - for (pfn = head_pfn + 2; pfn < end_pfn; pfn++) + __SetPageHead(head); + /* Initialize the tail pages from template. */ + for (pfn = head_pfn + 1; pfn < end_pfn; pfn++) zone_device_page_init_from_template(pfn_to_page(pfn), pfn, &template); prep_compound_head(head, order); @@ -1175,27 +1116,20 @@ void __ref memmap_init_zone_device(struct zone *zone, nr_pages = end_pfn - start_pfn; } - if (!nr_pages) - return; - VM_WARN_ON(!IS_ALIGNED(start_pfn | nr_pages, pfns_per_compound)); + memmap_init_page(&template, zone_idx, nid); + page_folio(&template)->pgmap = pgmap; /* - * Seed the reusable head-page template from the first real struct - * page. The normal page-init and refcount helpers must operate on - * a real memmap entry rather than a stack object. + * MEMORY_DEVICE_GENERIC pages regain a refcount of one in the free + * path. The remaining ZONE_DEVICE types start from zero here and raise + * the count again when the allocator or driver hands the page out. */ - pfn = start_pfn; - page = pfn_to_page(pfn); - __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); - memcpy(&template, page, sizeof(*page)); - if (pfns_per_compound != 1) - memmap_init_compound(page, pfn, zone_idx, nid, pgmap, - compound_nr_pages(pfn, pgmap)); - pfn += pfns_per_compound; + if (pgmap->type == MEMORY_DEVICE_GENERIC) + atomic_set(&template._refcount, 1); - /* Initialize the remaining head pages from template. */ - for (; pfn < end_pfn; pfn += pfns_per_compound) { + /* Initialize the head pages from template. */ + for (pfn = start_pfn; pfn < end_pfn; pfn += pfns_per_compound) { page = pfn_to_page(pfn); zone_device_page_init_from_template(page, pfn, &template); -- 2.54.0