Introduce a ACR_FLAGS_FROZEN flags to indicate that we want to allocate a frozen compound pages by alloc_contig_range(), also provide alloc_contig_frozen_pages() to allocate pages without incrementing their refcount, which may be beneficial to some users (eg hugetlb). Signed-off-by: Kefeng Wang --- include/linux/gfp.h | 6 ++++ mm/page_alloc.c | 85 +++++++++++++++++++++++++-------------------- 2 files changed, 54 insertions(+), 37 deletions(-) diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 5ebf26fcdcfa..d0047b85fe34 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -427,6 +427,7 @@ extern gfp_t vma_thp_gfp_mask(struct vm_area_struct *vma); typedef unsigned int __bitwise acr_flags_t; #define ACR_FLAGS_NONE ((__force acr_flags_t)0) // ordinary allocation request #define ACR_FLAGS_CMA ((__force acr_flags_t)BIT(0)) // allocate for CMA +#define ACR_FLAGS_FROZEN ((__force acr_flags_t)BIT(1)) // allocate for frozen compound pages /* The below functions must be run on a range from a single zone. */ extern int alloc_contig_range_noprof(unsigned long start, unsigned long end, @@ -437,6 +438,11 @@ extern struct page *alloc_contig_pages_noprof(unsigned long nr_pages, gfp_t gfp_ int nid, nodemask_t *nodemask); #define alloc_contig_pages(...) alloc_hooks(alloc_contig_pages_noprof(__VA_ARGS__)) +struct page *alloc_contig_frozen_pages_noprof(unsigned long nr_pages, + gfp_t gfp_mask, int nid, nodemask_t *nodemask); +#define alloc_contig_frozen_pages(...) \ + alloc_hooks(alloc_contig_frozen_pages_noprof(__VA_ARGS__)) + #endif void free_contig_range(unsigned long pfn, unsigned long nr_pages); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index baead29b3e67..0677c49fdff1 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -6854,6 +6854,9 @@ int alloc_contig_range_noprof(unsigned long start, unsigned long end, if (__alloc_contig_verify_gfp_mask(gfp_mask, (gfp_t *)&cc.gfp_mask)) return -EINVAL; + if ((alloc_flags & ACR_FLAGS_FROZEN) && !(gfp_mask & __GFP_COMP)) + return -EINVAL; + /* * What we do here is we mark all pageblocks in range as * MIGRATE_ISOLATE. Because pageblock and max order pages may @@ -6951,7 +6954,8 @@ int alloc_contig_range_noprof(unsigned long start, unsigned long end, check_new_pages(head, order); prep_new_page(head, order, gfp_mask, 0); - set_page_refcounted(head); + if (!(alloc_flags & ACR_FLAGS_FROZEN)) + set_page_refcounted(head); } else { ret = -EINVAL; WARN(true, "PFN range: requested [%lu, %lu), allocated [%lu, %lu)\n", @@ -6963,15 +6967,6 @@ int alloc_contig_range_noprof(unsigned long start, unsigned long end, } EXPORT_SYMBOL(alloc_contig_range_noprof); -static int __alloc_contig_pages(unsigned long start_pfn, - unsigned long nr_pages, gfp_t gfp_mask) -{ - unsigned long end_pfn = start_pfn + nr_pages; - - return alloc_contig_range_noprof(start_pfn, end_pfn, ACR_FLAGS_NONE, - gfp_mask); -} - static bool pfn_range_valid_contig(struct zone *z, unsigned long start_pfn, unsigned long nr_pages) { @@ -7003,31 +6998,8 @@ static bool zone_spans_last_pfn(const struct zone *zone, return zone_spans_pfn(zone, last_pfn); } -/** - * alloc_contig_pages() -- tries to find and allocate contiguous range of pages - * @nr_pages: Number of contiguous pages to allocate - * @gfp_mask: GFP mask. Node/zone/placement hints limit the search; only some - * action and reclaim modifiers are supported. Reclaim modifiers - * control allocation behavior during compaction/migration/reclaim. - * @nid: Target node - * @nodemask: Mask for other possible nodes - * - * This routine is a wrapper around alloc_contig_range(). It scans over zones - * on an applicable zonelist to find a contiguous pfn range which can then be - * tried for allocation with alloc_contig_range(). This routine is intended - * for allocation requests which can not be fulfilled with the buddy allocator. - * - * The allocated memory is always aligned to a page boundary. If nr_pages is a - * power of two, then allocated range is also guaranteed to be aligned to same - * nr_pages (e.g. 1GB request would be aligned to 1GB). - * - * Allocated pages can be freed with free_contig_range() or by manually calling - * __free_page() on each allocated page. - * - * Return: pointer to contiguous pages on success, or NULL if not successful. - */ -struct page *alloc_contig_pages_noprof(unsigned long nr_pages, gfp_t gfp_mask, - int nid, nodemask_t *nodemask) +static struct page *__alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask, + acr_flags_t alloc_flags, int nid, nodemask_t *nodemask) { unsigned long ret, pfn, flags; struct zonelist *zonelist; @@ -7050,8 +7022,8 @@ struct page *alloc_contig_pages_noprof(unsigned long nr_pages, gfp_t gfp_mask, * and cause alloc_contig_range() to fail... */ spin_unlock_irqrestore(&zone->lock, flags); - ret = __alloc_contig_pages(pfn, nr_pages, - gfp_mask); + ret = alloc_contig_range_noprof(pfn, pfn + nr_pages, + alloc_flags, gfp_mask); if (!ret) return pfn_to_page(pfn); spin_lock_irqsave(&zone->lock, flags); @@ -7062,6 +7034,45 @@ struct page *alloc_contig_pages_noprof(unsigned long nr_pages, gfp_t gfp_mask, } return NULL; } + +/** + * alloc_contig_pages() -- tries to find and allocate contiguous range of pages + * @nr_pages: Number of contiguous pages to allocate + * @gfp_mask: GFP mask. Node/zone/placement hints limit the search; only some + * action and reclaim modifiers are supported. Reclaim modifiers + * control allocation behavior during compaction/migration/reclaim. + * @nid: Target node + * @nodemask: Mask for other possible nodes + * + * This routine is a wrapper around alloc_contig_range(). It scans over zones + * on an applicable zonelist to find a contiguous pfn range which can then be + * tried for allocation with alloc_contig_range(). This routine is intended + * for allocation requests which can not be fulfilled with the buddy allocator. + * + * The allocated memory is always aligned to a page boundary. If nr_pages is a + * power of two, then allocated range is also guaranteed to be aligned to same + * nr_pages (e.g. 1GB request would be aligned to 1GB). + * + * Allocated pages can be freed with free_contig_range() or by manually calling + * __free_page() on each allocated page. + * + * Return: pointer to contiguous pages on success, or NULL if not successful. + */ +struct page *alloc_contig_pages_noprof(unsigned long nr_pages, gfp_t gfp_mask, + int nid, nodemask_t *nodemask) +{ + return __alloc_contig_pages(nr_pages, gfp_mask, ACR_FLAGS_NONE, + nid, nodemask); +} + +struct page *alloc_contig_frozen_pages_noprof(unsigned long nr_pages, + gfp_t gfp_mask, int nid, nodemask_t *nodemask) +{ + /* always allocate compound pages without refcount increased */ + return __alloc_contig_pages(nr_pages, gfp_mask | __GFP_COMP, + ACR_FLAGS_FROZEN, nid, nodemask); +} + #endif /* CONFIG_CONTIG_ALLOC */ void free_contig_range(unsigned long pfn, unsigned long nr_pages) -- 2.27.0