From: "Gregory Price (Meta)" Add ALLOC_ZONELIST_PRIVATE, which selects the local private zonelist independently of __GFP_THISNODE. __GFP_THISNODE still governs OOM behavior separately. Allocations are otherwise still filtered by the task's cpuset, this creates some quirks. Reserve allocations normally drop the nodemask and the cpuset check. A private request keeps both, since private memory has no reserves to grant. A non-blocking __GFP_HIGH request still skips the cpuset check, as it does for ordinary allocations. Signed-off-by: Gregory Price (Meta) --- mm/page_alloc.c | 21 +++++++++++++++------ mm/page_alloc.h | 15 +++++++++++++++ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 0bd21916cea1..a219057931c3 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -4926,16 +4926,24 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, goto got_pg; reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); - if (reserve_flags) + if (reserve_flags) { + unsigned int keep = ALLOC_KSWAPD; + + /* The cpuset still grants a private request's node. */ + if (ac->alloc_flags & ALLOC_ZONELIST_PRIVATE) + keep |= ALLOC_CPUSET; alloc_flags = alloc_flags_cma(gfp_mask) | reserve_flags | - ac->alloc_flags | (alloc_flags & ALLOC_KSWAPD); + ac->alloc_flags | (alloc_flags & keep); + } /* * Reset the nodemask and zonelist iterators if memory policies can be * ignored. These allocations are high priority and system rather than - * user oriented. + * user oriented. A private request keeps its node: private memory has + * no reserves to grant. */ - if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) { + if ((!(alloc_flags & ALLOC_CPUSET) || reserve_flags) && + !(ac->alloc_flags & ALLOC_ZONELIST_PRIVATE)) { ac->nodemask = NULL; ac->preferred_zoneref = first_zones_zonelist(ac->zonelist, ac->highest_zoneidx, ac->nodemask); @@ -5164,7 +5172,7 @@ static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, unsigned int *alloc_flags) { ac->highest_zoneidx = gfp_zone(gfp_mask); - ac->zonelist = node_zonelist(preferred_nid, gfp_mask); + ac->zonelist = select_zonelist(preferred_nid, gfp_mask, *alloc_flags); ac->nodemask = nodemask; ac->migratetype = gfp_migratetype(gfp_mask); @@ -5477,7 +5485,8 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, unsigned int zonelist_iter_cookie; /* Other flags could be supported later if needed. */ - if (WARN_ON(alloc_flags & ~(ALLOC_NOLOCK | ALLOC_NO_CODETAG))) + if (WARN_ON(alloc_flags & ~(ALLOC_NOLOCK | ALLOC_NO_CODETAG | + ALLOC_ZONELIST_PRIVATE))) return NULL; if (!alloc_order_allowed(gfp, order, alloc_flags)) diff --git a/mm/page_alloc.h b/mm/page_alloc.h index 917653ae14a3..c0a0e6abe4cd 100644 --- a/mm/page_alloc.h +++ b/mm/page_alloc.h @@ -56,6 +56,11 @@ * alloc_tag_sub_check(). */ #define ALLOC_NO_CODETAG 0x1000 +#ifdef CONFIG_NUMA +#define ALLOC_ZONELIST_PRIVATE 0x2000 /* use the local private zonelist */ +#else +#define ALLOC_ZONELIST_PRIVATE 0x0 +#endif /* Flags that allow allocations below the min watermark. */ #define ALLOC_RESERVES (ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM) @@ -100,6 +105,16 @@ struct alloc_context { unsigned int reclaim_compact_order; }; +static inline struct zonelist * +select_zonelist(int nid, gfp_t gfp, unsigned int alloc_flags) +{ +#ifdef CONFIG_NUMA + if (alloc_flags & ALLOC_ZONELIST_PRIVATE) + return &NODE_DATA(nid)->node_zonelists[ZONELIST_PRIVATE]; +#endif + return node_zonelist(nid, gfp); +} + /* * This function returns the order of a free page in the buddy system. In * general, page_zone(page)->lock must be held by the caller to prevent the -- 2.56.1