This flag is set unless we can be sure the caller isn't in an atomic context. The allocator will soon start needing to call set_direct_map_* APIs which cannot be called with IRQs off. It will need to do this even before direct reclaim is possible. Despite the fact that, in principle, ALLOC_NOBLOCK is distinct from __GFP_DIRECT_RECLAIM, in order to avoid introducing a GFP flag, just infer the former based on whether the caller set the latter. This means that, in practice, ALLOC_NOBLOCK is just !__GFP_DIRECT_RECLAIM, except that it is not influenced by gfp_allowed_mask. Call it ALLOC_NOBLOCK in order to try and mitigate confusion vs the recently-removed ALLOC_NON_BLOCK, which meant something different. Signed-off-by: Brendan Jackman --- mm/internal.h | 1 + mm/page_alloc.c | 29 ++++++++++++++++++++++------- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/mm/internal.h b/mm/internal.h index 513aba6c00bed813c9e38464aec5a15e65edaa58..c697ed35a8ca3376445d1e4249e9ce03097f15b8 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1315,6 +1315,7 @@ unsigned int reclaim_clean_pages_from_list(struct zone *zone, #define ALLOC_HIGHATOMIC 0x200 /* Allows access to MIGRATE_HIGHATOMIC */ #define ALLOC_TRYLOCK 0x400 /* Only use spin_trylock in allocation path */ #define ALLOC_KSWAPD 0x800 /* allow waking of kswapd, __GFP_KSWAPD_RECLAIM set */ +#define ALLOC_NOBLOCK 0x1000 /* Caller may be atomic */ /* Flags that allow allocations below the min watermark. */ #define ALLOC_RESERVES (ALLOC_HARDER|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index cd47cfaae820ce696d2e6e0c47436e00d3feef60..b0aeb97baa13af038fff0edae33affbbf49e825c 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -4552,6 +4552,8 @@ gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order) (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM)); if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) { + alloc_flags |= ALLOC_NOBLOCK; + /* * Not worth trying to allocate harder for __GFP_NOMEMALLOC even * if it can't schedule. @@ -4745,14 +4747,13 @@ check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac) static inline struct page * __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, - struct alloc_context *ac) + struct alloc_context *ac, unsigned int alloc_flags) { bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM; bool can_compact = gfp_compaction_allowed(gfp_mask); bool nofail = gfp_mask & __GFP_NOFAIL; const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER; struct page *page = NULL; - unsigned int alloc_flags; unsigned long did_some_progress; enum compact_priority compact_priority; enum compact_result compact_result; @@ -4795,7 +4796,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, * kswapd needs to be woken up, and to avoid the cost of setting up * alloc_flags precisely. So we do that now. */ - alloc_flags = gfp_to_alloc_flags(gfp_mask, order); + alloc_flags |= gfp_to_alloc_flags(gfp_mask, order); /* * We need to recalculate the starting point for the zonelist iterator @@ -5045,6 +5046,19 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, return page; } +static inline unsigned int init_alloc_flags(gfp_t gfp_mask, unsigned int flags) +{ + /* + * If the caller allowed __GFP_DIRECT_RECLAIM, they can't be atomic. + * Note this is a separate determination from whether direct + * reclaim is actually allowed, it must happen before applying + * gfp_allowed_mask. + */ + if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) + flags |= ALLOC_NOBLOCK; + return flags; +} + static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, int preferred_nid, nodemask_t *nodemask, struct alloc_context *ac, gfp_t *alloc_gfp, @@ -5121,7 +5135,7 @@ unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid, struct list_head *pcp_list; struct alloc_context ac; gfp_t alloc_gfp; - unsigned int alloc_flags = ALLOC_WMARK_LOW; + unsigned int alloc_flags = init_alloc_flags(gfp, ALLOC_WMARK_LOW); int nr_populated = 0, nr_account = 0; /* @@ -5267,7 +5281,7 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, int preferred_nid, nodemask_t *nodemask) { struct page *page; - unsigned int alloc_flags = ALLOC_WMARK_LOW; + unsigned int alloc_flags = init_alloc_flags(gfp, ALLOC_WMARK_LOW); gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */ struct alloc_context ac = { }; @@ -5319,7 +5333,7 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, */ ac.nodemask = nodemask; - page = __alloc_pages_slowpath(alloc_gfp, order, &ac); + page = __alloc_pages_slowpath(alloc_gfp, order, &ac, alloc_flags); out: if (memcg_kmem_online() && (gfp & __GFP_ACCOUNT) && page && @@ -7684,10 +7698,11 @@ struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned */ gfp_t alloc_gfp = __GFP_NOWARN | __GFP_ZERO | __GFP_NOMEMALLOC | __GFP_COMP | gfp_flags; - unsigned int alloc_flags = ALLOC_TRYLOCK; + unsigned int alloc_flags = init_alloc_flags(gfp_flags, ALLOC_TRYLOCK); struct alloc_context ac = { }; struct page *page; + VM_WARN_ON_ONCE(!(alloc_flags & ALLOC_NOBLOCK)); VM_WARN_ON_ONCE(gfp_flags & ~__GFP_ACCOUNT); /* * In PREEMPT_RT spin_trylock() will call raw_spin_lock() which is -- 2.50.1