From: Li Zhe Boot-time HugeTLB allocation has to initialize tail struct pages itself when HugeTLB vmemmap optimization (HVO) is unavailable or fails. For gigantic pages, this can cover a large number of tail pages. Most state installed by __init_single_page() and prep_compound_tail() is identical for every tail page in the same huge page; only PFN-dependent fields need refreshing. Initialize the first tail page normally, then initialize the remaining tails with hugetlb_init_tail_page(). This copies the already initialized first tail page with memcpy() and refreshes only the PFN-dependent fields in the destination page, while avoiding the full __init_single_page() and prep_compound_tail() sequence for state that is identical across tail pages in the same huge page. The fallback path was measured on a v7.4-rc2 base with a temporary jiffies-based printk modeled after the existing timing in memmap_init_zone_device(), using 4K base pages and 64 boot-time 1G hugepages. HVO is unavailable on arm64; on x86-64, hugetlb_free_vmemmap=off was used to exercise the fallback path. arch base with this patch reduction arm64 360.8 ms 256.0 ms 29.05% x86-64 171.0 ms 165.0 ms 3.51% Signed-off-by: Li Zhe Signed-off-by: Muchun Song --- mm/hugetlb.c | 23 ++++++++++++++++++----- mm/mm_init.c | 5 ++++- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 88c9aa700533..ed3af959f615 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3266,6 +3266,7 @@ static void __init hugetlb_folio_init_tail_vmemmap(struct folio *folio, enum zone_type zone = folio_zonenum(folio); int nid = folio_nid(folio); struct page *page = folio_page(folio, start_page_number); + struct page template; unsigned long head_pfn = folio_pfn(folio); unsigned long pfn, end_pfn = head_pfn + end_page_number; unsigned int order = huge_page_order(h); @@ -3274,11 +3275,23 @@ static void __init hugetlb_folio_init_tail_vmemmap(struct folio *folio, * As we marked all tail pages with memblock_reserved_mark_noinit(), * we must initialize them ourselves here. */ - for (pfn = head_pfn + start_page_number; pfn < end_pfn; page++, pfn++) { - __init_single_page(page, pfn, zone, nid); - prep_compound_tail(page, &folio->page, order); - set_page_count(page, 0); - } + if (start_page_number >= end_page_number) + return; + + pfn = head_pfn + start_page_number; + + /* + * Seed a template from the first real tail page and refresh only + * PFN-dependent fields before copying it. + */ + __init_single_page(page, pfn, zone, nid); + prep_compound_tail(page, &folio->page, order); + set_page_count(page, 0); + memcpy(&template, page, sizeof(template)); + + for (pfn++; pfn < end_pfn; pfn++) + memmap_init_page_from_template(pfn_to_page(pfn), pfn, + folio_zone(folio), &template); } static void __init hugetlb_folio_init_vmemmap(struct folio *folio, diff --git a/mm/mm_init.c b/mm/mm_init.c index 3988c19699bc..639880afe0ee 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -720,7 +720,10 @@ void __meminit memmap_init_page_from_template(struct page *page, if (!is_highmem_idx(zone_idx(zone))) set_page_address(template, __va(pfn << PAGE_SHIFT)); #endif - memcpy_nontemporal(page, template, sizeof(*page)); + if(zone_is_zone_device(zone)) + memcpy_nontemporal(page, template, sizeof(*page)); + else + memcpy(page, template, sizeof(*page)); } struct zone __meminit *pfn_to_zone(unsigned long pfn, int nid) -- 2.54.0