For a high-order request, __zone_watermark_ok() accepts a free page of any migratetype. An ALLOC_NOFRAGMENT request below pageblock_order can take a page of its own migratetype or claim a whole free pageblock, but not steal from another migratetype's pageblock. So the check can pass for an allocation that is going to fail. To tell these apart, the check needs the migratetype of the request. Add __zone_watermark_ok_mt(), which takes it, and use it where the caller asks whether this allocation can succeed: - zone_watermark_fast(); - the recheck after node reclaim in get_page_from_freelist(); - compaction_suit_allocation_order(). Other callers keep __zone_watermark_ok(), which counts every migratetype. kswapd and kcompactd balance the zone as a whole. should_reclaim_retry() asks whether the allocation could succeed if all reclaimable pages were freed, and compaction can then free a whole pageblock for it to claim. kcompactd passes MIGRATE_MOVABLE to compaction_suit_allocation_order(). It never sets ALLOC_NOFRAGMENT, so the value is not used. Nothing outside mm/ uses __zone_watermark_ok(). Move its declaration to mm/page_alloc.h, next to the new one. Nothing reads the migratetype yet, so there is no functional change. Preparation for making the high-order check honour ALLOC_NOFRAGMENT. Reviewed-by: Harry Yoo (Meta) Tested-by: Dev Jain Signed-off-by: Kiryl Shutsemau (Meta) --- include/linux/mmzone.h | 3 --- mm/compaction.c | 11 ++++++----- mm/page_alloc.c | 36 +++++++++++++++++++++++++++--------- mm/page_alloc.h | 8 ++++++++ 4 files changed, 41 insertions(+), 17 deletions(-) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index ecd1ebd49756..4d9347a399fa 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -1637,9 +1637,6 @@ static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat) #include void build_all_zonelists(pg_data_t *pgdat); -bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, - int highest_zoneidx, unsigned int alloc_flags, - long free_pages); bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, int highest_zoneidx, unsigned int alloc_flags); diff --git a/mm/compaction.c b/mm/compaction.c index f1b2060eb201..6c553d0e90f8 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -2520,7 +2520,7 @@ bool compaction_zonelist_suitable(struct alloc_context *ac, int order, static enum compact_result compaction_suit_allocation_order(struct zone *zone, unsigned int order, int highest_zoneidx, unsigned int alloc_flags, - bool async, bool kcompactd) + int migratetype, bool async, bool kcompactd) { unsigned long free_pages; unsigned long watermark; @@ -2531,8 +2531,8 @@ compaction_suit_allocation_order(struct zone *zone, unsigned int order, free_pages = zone_page_state(zone, NR_FREE_PAGES); watermark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK); - if (__zone_watermark_ok(zone, order, watermark, highest_zoneidx, - alloc_flags, free_pages)) + if (__zone_watermark_ok_mt(zone, order, watermark, highest_zoneidx, + alloc_flags, free_pages, migratetype)) return COMPACT_SUCCESS; /* @@ -2588,6 +2588,7 @@ compact_zone(struct compact_control *cc, struct capture_control *capc) ret = compaction_suit_allocation_order(cc->zone, cc->order, cc->highest_zoneidx, cc->alloc_flags, + cc->migratetype, cc->mode == MIGRATE_ASYNC, !cc->direct_compaction); if (ret != COMPACT_CONTINUE) @@ -3073,7 +3074,7 @@ static bool kcompactd_node_suitable(pg_data_t *pgdat) ret = compaction_suit_allocation_order(zone, pgdat->kcompactd_max_order, highest_zoneidx, alloc_flags, - false, true); + MIGRATE_MOVABLE, false, true); if (ret == COMPACT_CONTINUE) return true; } @@ -3116,7 +3117,7 @@ static void kcompactd_do_work(pg_data_t *pgdat) ret = compaction_suit_allocation_order(zone, cc.order, zoneid, cc.alloc_flags, - false, true); + MIGRATE_MOVABLE, false, true); if (ret != COMPACT_CONTINUE) continue; diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 9e1261521b6a..0ebfbbb5d7c5 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -3588,9 +3588,10 @@ static inline long __zone_watermark_unusable_free(struct zone *z, * one free page of a suitable size. Checking now avoids taking the zone lock * to check in the allocation paths if no pages are free. */ -bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, - int highest_zoneidx, unsigned int alloc_flags, - long free_pages) +bool __zone_watermark_ok_mt(struct zone *z, unsigned int order, + unsigned long mark, int highest_zoneidx, + unsigned int alloc_flags, long free_pages, + int migratetype) { long min = mark; int o; @@ -3666,6 +3667,20 @@ bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, return false; } +/* + * Return true if free base pages in the zone are above 'mark' regardless + * of migratetype. Use it to check whether the zone as a whole has enough + * free memory. Use __zone_watermark_ok_mt() instead to check whether a + * particular allocation can be served. + */ +bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, + int highest_zoneidx, unsigned int alloc_flags, + long free_pages) +{ + return __zone_watermark_ok_mt(z, order, mark, highest_zoneidx, + alloc_flags, free_pages, MIGRATE_MOVABLE); +} + bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, int highest_zoneidx, unsigned int alloc_flags) { @@ -3698,8 +3713,8 @@ static inline bool zone_watermark_fast(struct zone *z, unsigned int order, return true; } - if (__zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags, - free_pages)) + if (__zone_watermark_ok_mt(z, order, mark, highest_zoneidx, alloc_flags, + free_pages, gfp_migratetype(gfp_mask))) return true; /* @@ -3711,8 +3726,9 @@ static inline bool zone_watermark_fast(struct zone *z, unsigned int order, if (unlikely(!order && (alloc_flags & ALLOC_MIN_RESERVE) && z->watermark_boost && ((alloc_flags & ALLOC_WMARK_MASK) == WMARK_MIN))) { mark = z->_watermark[WMARK_MIN]; - return __zone_watermark_ok(z, order, mark, highest_zoneidx, - alloc_flags, free_pages); + return __zone_watermark_ok_mt(z, order, mark, highest_zoneidx, + alloc_flags, free_pages, + gfp_migratetype(gfp_mask)); } return false; @@ -3932,8 +3948,10 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, continue; /* did we reclaim enough */ - if (!zone_watermark_ok(zone, order, mark, - ac->highest_zoneidx, alloc_flags)) + if (!__zone_watermark_ok_mt(zone, order, mark, + ac->highest_zoneidx, alloc_flags, + zone_page_state(zone, NR_FREE_PAGES), + gfp_migratetype(gfp_mask))) continue; } diff --git a/mm/page_alloc.h b/mm/page_alloc.h index e238ee91ed24..d95420093b15 100644 --- a/mm/page_alloc.h +++ b/mm/page_alloc.h @@ -231,6 +231,14 @@ extern bool free_pages_prepare(struct page *page, unsigned int order); extern int user_min_free_kbytes; +bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, + int highest_zoneidx, unsigned int alloc_flags, + long free_pages); +bool __zone_watermark_ok_mt(struct zone *z, unsigned int order, + unsigned long mark, int highest_zoneidx, + unsigned int alloc_flags, long free_pages, + int migratetype); + struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, int nid, nodemask_t *nodemask, unsigned int alloc_flags); #define __alloc_frozen_pages(...) \ -- 2.54.0