We may have remaining scan budget for anon or file but no folios in populatable generations to scan. In this case, `scan_folios()` may repeatedly scan that type without making progress, even when the other type still has populatable folios. Run aging when a type is drained to make more folios available for reclaim. This also helps honor the scan budget distribution determined by swappiness. Move aging after eviction to detect a lack of scan progress and avoid unnecessary aging when eviction can still make progress. To avoid excessive aging, allow drained aging only once per `try_to_shrink_lruvec()` invocation. Also skip drained aging if another reclaimer has advanced `max_seq` during eviction. At extreme swappiness values such as 1 and 200, reclaim initially focuses on the preferred type. Rely on imbalanced aging once `sc->priority` reaches `DEF_PRIORITY / 2`, rather than triggering drained aging for the unpreferred type. This avoids unnecessary aging and has shown better performance in testing. If `evict_folio_lists()` returns zero, retry eviction once if `max_seq` has advanced since the aging decision. Signed-off-by: Barry Song (Xiaomi) --- mm/vmscan.c | 49 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 1450542d709c..6d942f192e63 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -5175,7 +5175,8 @@ static bool should_abort_scan(struct lruvec *lruvec, struct scan_control *sc) } static long evict_folio_lists(unsigned long nr_to_scan[], struct lruvec *lruvec, - struct scan_control *sc, int swappiness, unsigned long max_batch) + struct scan_control *sc, int swappiness, + unsigned long max_batch, bool *drained) { int type; unsigned long batch, scanned, total_scanned = 0; @@ -5191,6 +5192,8 @@ static long evict_folio_lists(unsigned long nr_to_scan[], struct lruvec *lruvec, batch = min(nr_to_scan[type], max_batch); scanned = evict_folios(batch, lruvec, sc, type, swappiness); + if (drained && !scanned) + *drained = true; nr_to_scan[type] -= min(nr_to_scan[type], scanned); total_scanned += scanned; } @@ -5206,9 +5209,15 @@ static long evict_folio_lists(unsigned long nr_to_scan[], struct lruvec *lruvec, static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) { unsigned long nr[ANON_AND_FILE]; - bool need_rotate = false, should_age = false; + bool need_rotate = false, should_age = false, retried = false; int swappiness = get_swappiness(lruvec, sc); struct mem_cgroup *memcg = lruvec_memcg(lruvec); + /* + * Avoid aging the lagging, unpreferred type at extreme swappiness; + * rely on imbalanced aging once priority reaches AGING_PRIORITY + */ + bool allow_drained_age = sc->priority < DEF_PRIORITY && + swappiness > MIN_SWAPPINESS + 1 && swappiness < MAX_SWAPPINESS; /* * Proactive reclaim initiated by userspace for anonymous memory only. @@ -5226,22 +5235,17 @@ static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) lru_gen_prepare_scan(lruvec, sc, memcg, swappiness, nr); while (nr[LRU_GEN_ANON] > 0 || nr[LRU_GEN_FILE] > 0) { + bool drained = false, do_drained_age; DEFINE_MAX_SEQ(lruvec); + unsigned long new_seq; + long scanned; if (mem_cgroup_below_min(sc->target_mem_cgroup, memcg)) { need_rotate = true; break; } - if (should_run_aging(lruvec, max_seq, sc, swappiness)) { - trace_mm_mglru_aging(lruvec, sc, swappiness); - if (try_to_inc_max_seq(lruvec, max_seq, swappiness, false)) - need_rotate = true; - should_age = true; - } - - if (!evict_folio_lists(nr, lruvec, sc, swappiness, MIN_LRU_BATCH)) - break; + scanned = evict_folio_lists(nr, lruvec, sc, swappiness, MIN_LRU_BATCH, &drained); if (should_abort_scan(lruvec, sc)) break; @@ -5253,6 +5257,27 @@ static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) if (root_reclaim(sc) && should_age) break; + /* + * Allow drained aging only once per invocation and skip it if + * max_seq advanced concurrently during eviction + */ + new_seq = READ_ONCE(lruvec->lrugen.max_seq); + do_drained_age = drained && allow_drained_age && max_seq == new_seq; + allow_drained_age &= !do_drained_age; + if (do_drained_age || should_run_aging(lruvec, new_seq, sc, swappiness)) { + trace_mm_mglru_aging(lruvec, sc, swappiness); + if (try_to_inc_max_seq(lruvec, new_seq, swappiness, false)) + need_rotate = true; + should_age = true; + } + + if (!scanned) { + /* Retry once if aging advanced max_seq */ + if (retried || new_seq == READ_ONCE(lruvec->lrugen.max_seq)) + break; + retried = true; + } + cond_resched(); } @@ -5885,7 +5910,7 @@ static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_co lru_gen_balance_scan(lruvec, swappiness, nr_to_reclaim - sc->nr_reclaimed, nr_to_scan); - if (!evict_folio_lists(nr_to_scan, lruvec, sc, swappiness, MAX_LRU_BATCH)) + if (!evict_folio_lists(nr_to_scan, lruvec, sc, swappiness, MAX_LRU_BATCH, NULL)) return 0; cond_resched(); -- 2.39.3 (Apple Git-146)