Vmemmap optimization operates on one folio at a time, but the current interface exposes a raw address range and requires the HugeTLB caller to prepare part of the operation. This splits the common mechanism across layers and obscures its natural unit of work. Make the folio the unit of vmemmap optimization and move the common mechanics behind a folio-oriented interface. This keeps HugeTLB-specific eligibility and state in the caller while making the remapping code applicable to any eligible folio. No functional change intended. Signed-off-by: Muchun Song Reviewed-by: Lance Yang Reviewed-by: Qi Zheng Reviewed-by: Mike Rapoport (Microsoft) --- v3: - Collect Reviewed-by from Mike Rapoport v2: - Collect Reviewed-by from Lance Yang and Qi Zheng --- mm/hugetlb_vmemmap.c | 64 +++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 34 deletions(-) diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index 7d51df1cb78b..64967f18e3c3 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -233,6 +233,11 @@ static struct page *vmemmap_dup_page_from_list(const struct page *src, return dst; } +static inline unsigned long folio_vmemmap_size(const struct folio *folio) +{ + return folio_nr_pages(folio) * sizeof(struct page); +} + static inline struct page *folio_vmemmap_shared_tail_page(const struct folio *folio) { return vmemmap_shared_tail_page(folio_order(folio), folio_zone(folio)); @@ -302,28 +307,39 @@ static int vmemmap_remap_split(unsigned long start, unsigned long end) } /** - * vmemmap_remap_free - remap a vmemmap range and collect old backing pages - * @start: start address of the vmemmap virtual address range that we want - * to remap. - * @end: end address of the vmemmap virtual address range that we want to - * remap. - * @vmemmap_pages: input/output list whose first entry supplies the new backing - * page for @start; replaced backing pages are added to the list for - * the caller to free. - * @flags: modifications to vmemmap_remap_walk flags + * vmemmap_optimize_folio - optimize the vmemmap backing of a folio + * @folio: folio whose vmemmap pages should be optimized. + * @vmemmap_pages: list holding replacement pages for the remap and displaced + * backing pages for the caller to free. + * @flags: vmemmap remap control flags passed to vmemmap_remap_walk. + * + * Remap @folio's vmemmap so optimizable entries point at the shared tail page + * while non-optimizable entries are copied into replacement pages. Caller is + * responsible for freeing the pages left on the @vmemmap_pages list, even on + * failure. * * Return: %0 on success, negative error code otherwise. */ -static int vmemmap_remap_free(unsigned long start, unsigned long end, +static int vmemmap_optimize_folio(const struct folio *folio, struct list_head *vmemmap_pages, unsigned long flags) { - int ret; + int ret, nid = folio_nid(folio); + unsigned long start = (unsigned long)folio_page(folio, 0); + unsigned long end = start + folio_vmemmap_size(folio); struct vmemmap_remap_walk walk = { .remap_pte = vmemmap_optimize_pte, .vmemmap_pages = vmemmap_pages, .flags = flags, }; + VM_WARN_ON_ONCE_FOLIO(folio_ref_count(folio), folio); + + if (!vmemmap_optimizable_order(folio_order(folio))) + return -EOPNOTSUPP; + + if (alloc_vmemmap_page_list(vmemmap_pages, VMEMMAP_OPTIMIZATION_PAGES, nid)) + return -ENOMEM; + ret = vmemmap_remap_range(start, end, &walk); if (!ret || !walk.nr_walked) return ret; @@ -488,16 +504,12 @@ static int __hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct list_head *vmemmap_pages, unsigned long flags) { - unsigned long vmemmap_start, vmemmap_end; - int nid, ret = 0; + int ret; VM_WARN_ON_ONCE_FOLIO(!folio_test_hugetlb(folio), folio); - VM_WARN_ON_ONCE_FOLIO(folio_ref_count(folio), folio); if (!vmemmap_should_optimize_folio(h, folio)) - return ret; - - nid = folio_nid(folio); + return 0; /* * Very Subtle @@ -511,23 +523,7 @@ static int __hugetlb_vmemmap_optimize_folio(const struct hstate *h, * the TLB and clear the flag below. */ folio_set_hugetlb_vmemmap_optimized(folio); - - if (alloc_vmemmap_page_list(vmemmap_pages, VMEMMAP_OPTIMIZATION_PAGES, nid)) { - ret = -ENOMEM; - goto out; - } - - vmemmap_start = (unsigned long)&folio->page; - vmemmap_end = vmemmap_start + hugetlb_vmemmap_size(h); - - /* - * Remap the vmemmap virtual address range [@vmemmap_start, @vmemmap_end). - * Add pages previously mapping the range to vmemmap_pages list so that - * they can be freed by the caller. - */ - ret = vmemmap_remap_free(vmemmap_start, vmemmap_end, - vmemmap_pages, flags); -out: + ret = vmemmap_optimize_folio(folio, vmemmap_pages, flags); if (ret) folio_clear_hugetlb_vmemmap_optimized(folio); -- 2.54.0