Currently, proactive compaction only evaluates fragmentation relative to COMPACTION_HPAGE_ORDER (typically order-9 for 2MB THP). This makes it unsuitable for systems that primarily need smaller high-order pages, such as order-2 (16KB) for mTHP. Generalize the fragmentation score functions to accept an order parameter: - fragmentation_score_zone(zone, order) - fragmentation_score_zone_weighted(zone, order) - fragmentation_score_node(pgdat, order) Calculate the min order in huge_anon_orders_always to configure the target order for proactive compaction. This enables proactive compaction to maintain free page availability at any order, which is particularly useful for mTHP-enabled systems where order-n (n < 9) allocation pressure is high. Signed-off-by: Bo Zhang --- mm/compaction.c | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/mm/compaction.c b/mm/compaction.c index a049415512c6..a4f87232ad80 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -24,6 +24,7 @@ #include #include #include +#include #include "page_alloc.h" #include "internal.h" @@ -81,6 +82,15 @@ static inline bool is_via_compact_memory(int order) { return false; } #define COMPACTION_HPAGE_ORDER (PMD_SHIFT - PAGE_SHIFT) #endif +static inline int compact_hpage_order(void) +{ + unsigned long orders = READ_ONCE(huge_anon_orders_always); + + if (orders) + return __ffs(orders); + return COMPACTION_HPAGE_ORDER; +} + static struct page *mark_allocated_noprof(struct page *page, unsigned int order, gfp_t gfp_flags) { post_alloc_hook(page, order, __GFP_MOVABLE, ALLOC_DEFAULT); @@ -2208,16 +2218,16 @@ static bool kswapd_is_running(pg_data_t *pgdat) /* * A zone's fragmentation score is the external fragmentation wrt to the - * COMPACTION_HPAGE_ORDER. It returns a value in the range [0, 100]. + * compact_hpage_order(). It returns a value in the range [0, 100]. */ -static unsigned int fragmentation_score_zone(struct zone *zone) +static unsigned int fragmentation_score_zone(struct zone *zone, unsigned int order) { - return extfrag_for_order(zone, COMPACTION_HPAGE_ORDER); + return extfrag_for_order(zone, order); } /* * A weighted zone's fragmentation score is the external fragmentation - * wrt to the COMPACTION_HPAGE_ORDER scaled by the zone's size. It + * wrt to the compact_hpage_order() scaled by the zone's size. It * returns a value in the range [0, 100]. * * The scaling factor ensures that proactive compaction focuses on larger @@ -2225,11 +2235,11 @@ static unsigned int fragmentation_score_zone(struct zone *zone) * ZONE_DMA32. For smaller zones, the score value remains close to zero, * and thus never exceeds the high threshold for proactive compaction. */ -static unsigned int fragmentation_score_zone_weighted(struct zone *zone) +static unsigned int fragmentation_score_zone_weighted(struct zone *zone, unsigned int order) { unsigned long score; - score = zone->present_pages * fragmentation_score_zone(zone); + score = zone->present_pages * fragmentation_score_zone(zone, order); return div64_ul(score, zone->zone_pgdat->node_present_pages + 1); } @@ -2240,7 +2250,7 @@ static unsigned int fragmentation_score_zone_weighted(struct zone *zone) * the node's score falls below the low threshold, or one of the back-off * conditions is met. */ -static unsigned int fragmentation_score_node(pg_data_t *pgdat) +static unsigned int fragmentation_score_node(pg_data_t *pgdat, unsigned int order) { unsigned int score = 0; int zoneid; @@ -2251,7 +2261,7 @@ static unsigned int fragmentation_score_node(pg_data_t *pgdat) zone = &pgdat->node_zones[zoneid]; if (!populated_zone(zone)) continue; - score += fragmentation_score_zone_weighted(zone); + score += fragmentation_score_zone_weighted(zone, order); } return score; @@ -2269,12 +2279,13 @@ static unsigned int fragmentation_score_wmark(bool low) static bool should_proactive_compact_node(pg_data_t *pgdat) { int wmark_high; + unsigned int order = compact_hpage_order(); if (!sysctl_compaction_proactiveness || kswapd_is_running(pgdat)) return false; wmark_high = fragmentation_score_wmark(false); - return fragmentation_score_node(pgdat) > wmark_high; + return fragmentation_score_node(pgdat, order) > wmark_high; } static enum compact_result __compact_finished(struct compact_control *cc) @@ -2311,7 +2322,7 @@ static enum compact_result __compact_finished(struct compact_control *cc) if (kswapd_is_running(pgdat)) return COMPACT_PARTIAL_SKIPPED; - score = fragmentation_score_zone(cc->zone); + score = fragmentation_score_zone(cc->zone, compact_hpage_order()); wmark_low = fragmentation_score_wmark(true); if (score > wmark_low) @@ -3238,10 +3249,11 @@ static int kcompactd(void *p) timeout = default_timeout; if (should_proactive_compact_node(pgdat)) { unsigned int prev_score, score; + unsigned int order = compact_hpage_order(); - prev_score = fragmentation_score_node(pgdat); + prev_score = fragmentation_score_node(pgdat, order); compact_node(pgdat, true); - score = fragmentation_score_node(pgdat); + score = fragmentation_score_node(pgdat, order); /* * Defer proactive compaction if the fragmentation * score did not go down i.e. no progress made. -- 2.34.1 When proactive compaction is triggered via compact_memory and mTHP always-enabled orders are configured, skip isolating folios whose order is >= the minimum always-enabled mTHP order. These folios already satisfy mTHP allocation requirements and migrating them is unnecessary overhead. Signed-off-by: Bo Zhang --- mm/compaction.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mm/compaction.c b/mm/compaction.c index a4f87232ad80..a15ed2576562 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -837,6 +837,12 @@ static bool skip_isolation_on_order(int order, int target_order) */ if (!is_via_compact_memory(target_order) && order >= target_order) return true; + + /* We are compacting for multi-size THP allocation */ + if (is_via_compact_memory(target_order) && order >= compact_hpage_order() && + READ_ONCE(huge_anon_orders_always)) + return true; + /* * We limit memory compaction to pageblocks and won't try * creating free blocks of memory that are larger than that. -- 2.34.1 When the minimum always-enabled mTHP order is below PAGE_ALLOC_COSTLY_ORDER, do not skip proactive compaction even if kswapd is running. For these non-costly mTHP orders, kswapd reclaim alone may not produce the contiguous free blocks needed, so proactive compaction should proceed concurrently to ensure mTHP allocation success. Signed-off-by: Bo Zhang --- mm/compaction.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/compaction.c b/mm/compaction.c index a15ed2576562..15b92475562a 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -2323,9 +2323,10 @@ static enum compact_result __compact_finished(struct compact_control *cc) if (cc->proactive_compaction) { int score, wmark_low; pg_data_t *pgdat; + bool costly = compact_hpage_order() > PAGE_ALLOC_COSTLY_ORDER; pgdat = cc->zone->zone_pgdat; - if (kswapd_is_running(pgdat)) + if (costly && kswapd_is_running(pgdat)) return COMPACT_PARTIAL_SKIPPED; score = fragmentation_score_zone(cc->zone, compact_hpage_order()); -- 2.34.1 To improve the mTHP allocation success rate and reduce fragmentation over time, introduce zone_effective_free_pages() for mTHP-aware free page accounting. Refactor the free page counting logic used by pgdat_balanced() and compaction_suit_allocation_order() into a shared helper function zone_effective_free_pages(). The function selects the appropriate free page metric based on context: - defrag_mode: use NR_FREE_PAGES_BLOCKS (whole pageblock accounting) - mTHP always-enabled: count only free pages in buddy blocks >= the minimum always-enabled mTHP order, since smaller fragments cannot satisfy mTHP allocations - otherwise: use NR_FREE_PAGES Signed-off-by: Bo Zhang --- mm/compaction.c | 38 ++++++++++++++++++++++++++++++++++---- mm/internal.h | 3 +++ mm/vmscan.c | 23 +++++------------------ 3 files changed, 42 insertions(+), 22 deletions(-) diff --git a/mm/compaction.c b/mm/compaction.c index 15b92475562a..29be72597415 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -2528,6 +2528,38 @@ bool compaction_zonelist_suitable(struct alloc_context *ac, int order, return false; } +/** + * zone_effective_free_pages - get free pages relevant to allocation order + * @zone: target zone + * @order: allocation order + * @use_blocks: if true, use NR_FREE_PAGES_BLOCKS + * + * In defrag_mode, watermarks must be met in whole blocks to avoid + * polluting allocator fallbacks. kswapd usually cannot accomplish + * this on its own and needs kcompactd support. + * + * When mTHP always-enabled orders are configured, count only free pages + * in blocks >= min mTHP order, as smaller fragments cannot satisfy mTHP + * allocations. + */ +unsigned long zone_effective_free_pages(struct zone *zone, + unsigned int order, + bool use_blocks) +{ + if (use_blocks) + return zone_page_state(zone, NR_FREE_PAGES_BLOCKS); + + if (READ_ONCE(huge_anon_orders_always) && order == compact_hpage_order()) { + unsigned long free_pages = 0; + + for (int o = order; o < NR_PAGE_ORDERS; o++) + free_pages += zone->free_area[o].nr_free << o; + return free_pages; + } + + return zone_page_state(zone, NR_FREE_PAGES); +} + /* * Should we do compaction for target allocation order. * Return COMPACT_SUCCESS if allocation for target order can be already @@ -2543,10 +2575,8 @@ compaction_suit_allocation_order(struct zone *zone, unsigned int order, unsigned long free_pages; unsigned long watermark; - if (kcompactd && defrag_mode) - free_pages = zone_page_state(zone, NR_FREE_PAGES_BLOCKS); - else - free_pages = zone_page_state(zone, NR_FREE_PAGES); + free_pages = zone_effective_free_pages(zone, order, + kcompactd && defrag_mode); watermark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK); if (__zone_watermark_ok(zone, order, watermark, highest_zoneidx, diff --git a/mm/internal.h b/mm/internal.h index 38b1165212c9..14bb9543879b 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1654,4 +1654,7 @@ static inline bool can_spin_trylock(void) return true; } +unsigned long zone_effective_free_pages(struct zone *zone, + unsigned int order, + bool use_blocks); #endif /* __MM_INTERNAL_H */ diff --git a/mm/vmscan.c b/mm/vmscan.c index c1404a59523d..a419a2c2fca4 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -6966,7 +6966,6 @@ static bool pgdat_balanced(pg_data_t *pgdat, int order, int highest_zoneidx) * meet watermarks. */ for_each_managed_zone_pgdat(zone, pgdat, i, highest_zoneidx) { - enum zone_stat_item item; unsigned long free_pages; if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) @@ -6974,21 +6973,6 @@ static bool pgdat_balanced(pg_data_t *pgdat, int order, int highest_zoneidx) else mark = high_wmark_pages(zone); - /* - * In defrag_mode, watermarks must be met in whole - * blocks to avoid polluting allocator fallbacks. - * - * However, kswapd usually cannot accomplish this on - * its own and needs kcompactd support. Once it's - * reclaimed a compaction gap, and kswapd_shrink_node - * has dropped order, simply ensure there are enough - * base pages for compaction, wake kcompactd & sleep. - */ - if (defrag_mode && order) - item = NR_FREE_PAGES_BLOCKS; - else - item = NR_FREE_PAGES; - /* * When there is a high number of CPUs in the system, * the cumulative error from the vmstat per-cpu cache @@ -7001,9 +6985,12 @@ static bool pgdat_balanced(pg_data_t *pgdat, int order, int highest_zoneidx) * counter won't actually be per-cpu cached. But keep * things simple for now; revisit when somebody cares. */ - free_pages = zone_page_state(zone, item); + free_pages = zone_effective_free_pages(zone, order, + defrag_mode & order); if (zone->percpu_drift_mark && free_pages < zone->percpu_drift_mark) - free_pages = zone_page_state_snapshot(zone, item); + free_pages = zone_page_state_snapshot(zone, + defrag_mode & order ? + NR_FREE_PAGES_BLOCKS : NR_FREE_PAGES); if (__zone_watermark_ok(zone, order, mark, highest_zoneidx, 0, free_pages)) -- 2.34.1