A subsequent patch will want to set ALLOC_UNMAPPED in the allocator APIs used by filemap.c. To keep that patch from getting overwhelming, split out the arg plumbing into a separate one. No functional change intended. Signed-off-by: Brendan Jackman --- mm/filemap.c | 2 ++ mm/khugepaged.c | 2 +- mm/mempolicy.c | 31 ++++++++++++++++++++----------- mm/mempolicy.h | 24 ++++++++++++++++++++++++ mm/migrate.c | 2 +- mm/page_alloc.c | 6 +++--- mm/page_alloc.h | 2 +- 7 files changed, 52 insertions(+), 17 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index 341c7c5b28bc7..bf62fee570d8b 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -53,6 +53,8 @@ #include #include "internal.h" +#include "mempolicy.h" +#include "page_alloc.h" #define CREATE_TRACE_POINTS #include diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 27e8f3077e80f..89ce6bcbc376b 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1243,7 +1243,7 @@ static enum scan_result alloc_charge_folio(struct folio **foliop, struct mm_stru int node = collapse_find_target_node(cc); struct folio *folio; - folio = __folio_alloc(gfp, order, node, &cc->alloc_nmask); + folio = __folio_alloc(gfp, order, node, &cc->alloc_nmask, ALLOC_DEFAULT); if (!folio) { *foliop = NULL; if (is_pmd_order(order)) diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 5720f7f54d942..73aeda071f5d7 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -119,6 +119,7 @@ #include #include "internal.h" +#include "mempolicy.h" #include "page_alloc.h" /* Internal flags */ @@ -2406,7 +2407,8 @@ bool mempolicy_in_oom_domain(struct task_struct *tsk, } static struct page *alloc_pages_preferred_many(gfp_t gfp, unsigned int order, - int nid, nodemask_t *nodemask) + int nid, nodemask_t *nodemask, + unsigned int alloc_flags) { struct page *page; gfp_t preferred_gfp; @@ -2420,10 +2422,10 @@ static struct page *alloc_pages_preferred_many(gfp_t gfp, unsigned int order, preferred_gfp = gfp | __GFP_NOWARN; preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL); page = __alloc_frozen_pages_noprof(preferred_gfp, order, nid, nodemask, - ALLOC_DEFAULT); + alloc_flags); if (!page) page = __alloc_frozen_pages_noprof(gfp, order, nid, NULL, - ALLOC_DEFAULT); + alloc_flags); return page; } @@ -2435,11 +2437,12 @@ static struct page *alloc_pages_preferred_many(gfp_t gfp, unsigned int order, * @pol: Pointer to the NUMA mempolicy. * @ilx: Index for interleave mempolicy (also distinguishes alloc_pages()). * @nid: Preferred node (usually numa_node_id() but @mpol may override it). + * @alloc_flags: ALLOC_* flags. * * Return: The page on success or NULL if allocation fails. */ static struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order, - struct mempolicy *pol, pgoff_t ilx, int nid) + struct mempolicy *pol, pgoff_t ilx, int nid, unsigned int alloc_flags) { nodemask_t *nodemask; struct page *page; @@ -2447,7 +2450,7 @@ static struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order, nodemask = policy_nodemask(gfp, pol, ilx, &nid); if (pol->mode == MPOL_PREFERRED_MANY) - return alloc_pages_preferred_many(gfp, order, nid, nodemask); + return alloc_pages_preferred_many(gfp, order, nid, nodemask, alloc_flags); if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && /* filter "hugepage" allocation, unless from alloc_pages() */ @@ -2471,7 +2474,7 @@ static struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order, */ page = __alloc_frozen_pages_noprof( gfp | __GFP_THISNODE | __GFP_NORETRY, order, - nid, NULL, ALLOC_DEFAULT); + nid, NULL, alloc_flags); if (page || !(gfp & __GFP_DIRECT_RECLAIM)) return page; /* @@ -2483,7 +2486,7 @@ static struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order, } } - page = __alloc_frozen_pages_noprof(gfp, order, nid, nodemask, ALLOC_DEFAULT); + page = __alloc_frozen_pages_noprof(gfp, order, nid, nodemask, alloc_flags); if (unlikely(pol->mode == MPOL_INTERLEAVE || pol->mode == MPOL_WEIGHTED_INTERLEAVE) && page) { @@ -2499,11 +2502,11 @@ static struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order, return page; } -struct folio *folio_alloc_mpol_noprof(gfp_t gfp, unsigned int order, - struct mempolicy *pol, pgoff_t ilx, int nid) +struct folio *__folio_alloc_mpol_noprof(gfp_t gfp, unsigned int order, + struct mempolicy *pol, pgoff_t ilx, int nid, unsigned int alloc_flags) { struct page *page = alloc_pages_mpol(gfp | __GFP_COMP, order, pol, - ilx, nid); + ilx, nid, alloc_flags); if (!page) return NULL; @@ -2511,6 +2514,12 @@ struct folio *folio_alloc_mpol_noprof(gfp_t gfp, unsigned int order, return page_rmappable_folio(page); } +struct folio *folio_alloc_mpol_noprof(gfp_t gfp, unsigned int order, + struct mempolicy *pol, pgoff_t ilx, int nid) +{ + return __folio_alloc_mpol_noprof(gfp, order, pol, ilx, nid, ALLOC_DEFAULT); +} + /** * vma_alloc_folio - Allocate a folio for a VMA. * @gfp: GFP flags. @@ -2555,7 +2564,7 @@ struct page *alloc_frozen_pages_noprof(gfp_t gfp, unsigned order) pol = get_task_policy(current); return alloc_pages_mpol(gfp, order, pol, NO_INTERLEAVE_INDEX, - numa_node_id()); + numa_node_id(), ALLOC_DEFAULT); } /** diff --git a/mm/mempolicy.h b/mm/mempolicy.h new file mode 100644 index 0000000000000..92d0bb3640962 --- /dev/null +++ b/mm/mempolicy.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * mm-internal API for mempolicy.c. Public API lives in + * include/linux/mempolicy.h. + */ +#ifndef __MM_MEMPOLICY_H +#define __MM_MEMPOLICY_H + +#include +#include +#include "page_alloc.h" + +#ifdef CONFIG_NUMA +struct folio *__folio_alloc_mpol_noprof(gfp_t gfp, unsigned int order, + struct mempolicy *pol, pgoff_t ilx, int nid, unsigned int alloc_flags); +#else +static inline struct folio *__folio_alloc_mpol_noprof(gfp_t gfp, unsigned int order, + struct mempolicy *pol, pgoff_t ilx, int nid, unsigned int alloc_flags) +{ + return __folio_alloc_noprof(gfp, order, numa_node_id(), NULL, alloc_flags); +} +#endif + +#endif diff --git a/mm/migrate.c b/mm/migrate.c index 0c772571610d3..583e74cd179c0 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -2233,7 +2233,7 @@ struct folio *alloc_migration_target(struct folio *src, unsigned long private) if (is_highmem_idx(zidx) || zidx == ZONE_MOVABLE) gfp_mask |= __GFP_HIGHMEM; - return __folio_alloc(gfp_mask, order, nid, mtc->nmask); + return __folio_alloc(gfp_mask, order, nid, mtc->nmask, ALLOC_DEFAULT); } #ifdef CONFIG_NUMA_MIGRATION diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 5f1dea7eee15b..f39b6af3a6b73 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -5888,7 +5888,7 @@ struct page *alloc_pages_node_noprof(int nid, gfp_t gfp_mask, unsigned int order EXPORT_SYMBOL(alloc_pages_node_noprof); struct folio *__folio_alloc_noprof(gfp_t gfp, unsigned int order, int preferred_nid, - nodemask_t *nodemask) + nodemask_t *nodemask, unsigned int alloc_flags) { struct page *page; @@ -5898,13 +5898,13 @@ struct folio *__folio_alloc_noprof(gfp_t gfp, unsigned int order, int preferred_ warn_if_node_offline(preferred_nid, gfp); page = __alloc_pages_noprof(gfp | __GFP_COMP, order, - preferred_nid, nodemask, ALLOC_DEFAULT); + preferred_nid, nodemask, alloc_flags); return page_rmappable_folio(page); } struct folio *folio_alloc_node_noprof(gfp_t gfp, unsigned int order, int nid) { - return __folio_alloc_noprof(gfp, order, nid, NULL); + return __folio_alloc_noprof(gfp, order, nid, NULL, ALLOC_DEFAULT); } EXPORT_SYMBOL(folio_alloc_node_noprof); diff --git a/mm/page_alloc.h b/mm/page_alloc.h index 5fa8fc527347c..be83a974edc2b 100644 --- a/mm/page_alloc.h +++ b/mm/page_alloc.h @@ -279,7 +279,7 @@ struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, int preferred_n #define __alloc_pages(...) alloc_hooks(__alloc_pages_noprof(__VA_ARGS__)) struct folio *__folio_alloc_noprof(gfp_t gfp, unsigned int order, int preferred_nid, - nodemask_t *nodemask); + nodemask_t *nodemask, unsigned int alloc_flags); #define __folio_alloc(...) alloc_hooks(__folio_alloc_noprof(__VA_ARGS__)) extern void zone_pcp_reset(struct zone *zone); -- 2.54.0