shrink_lruvec() charges the per-LRU budget nr[lru] in SWAP_CLUSTER_MAX (32) chunks, but isolate_lru_folios() may scan far more per call: a large folio can jump scan by many pages at once (a PMD-sized folio counts 512), and a zone-ineligible LRU walks the whole list without feeding scan back. The overshoot is never refunded, so shrink_lruvec() keeps charging only 32 per round and rescans the same folios. Have isolate_lru_folios() record its scanned count in sc->nr_isolate_scanned and let shrink_lruvec() subtract the overshoot from the remaining quota so the next round skips already-scanned folios. The field is reset to 0 before each shrink_list() call, as shrink_list() only reaches isolate_lru_folios() on some paths (active + skipped_deactivate, too_many_isolated stall bail out early); a stale value would otherwise be charged. The budget floor stays nr_to_scan via max() so an empty LRU (sc->nr_isolate_scanned = 0) still advances and cannot deadlock. Co-developed-by: Qiuhao Zhang Signed-off-by: Qiuhao Zhang Signed-off-by: Wupeng Ma --- mm/vmscan.c | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 56708d1d2dfd5..147e74f9732d5 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -164,6 +164,9 @@ struct scan_control { /* Incremented by the number of inactive pages that were scanned */ unsigned long nr_scanned; + /* Number of pages that were scanned from isolate_lru_folios() */ + unsigned long nr_isolate_scanned; + /* Number of pages freed so far during a call to shrink_zones() */ unsigned long nr_reclaimed; @@ -1673,7 +1676,6 @@ static __always_inline void update_lru_sizes(struct lruvec *lruvec, * @nr_to_scan: The number of eligible pages to look through on the list. * @lruvec: The LRU vector to pull pages from. * @dst: The temp list to put pages on to. - * @nr_scanned: The number of pages that were scanned. * @sc: The scan_control struct for this reclaim session * @lru: LRU list id for isolating * @@ -1681,8 +1683,7 @@ static __always_inline void update_lru_sizes(struct lruvec *lruvec, */ static unsigned long isolate_lru_folios(unsigned long nr_to_scan, struct lruvec *lruvec, struct list_head *dst, - unsigned long *nr_scanned, struct scan_control *sc, - enum lru_list lru) + struct scan_control *sc, enum lru_list lru) { struct list_head *src = &lruvec->lists[lru]; unsigned long nr_taken = 0; @@ -1766,7 +1767,7 @@ static unsigned long isolate_lru_folios(unsigned long nr_to_scan, skipped += nr_skipped[zid]; } } - *nr_scanned = total_scan; + sc->nr_isolate_scanned = total_scan; trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, nr_to_scan, total_scan, skipped, nr_taken, lru); update_lru_sizes(lruvec, lru, nr_zone_taken); @@ -2014,8 +2015,8 @@ static unsigned long shrink_inactive_list(unsigned long nr_to_scan, lruvec_lock_irq(lruvec); - nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &folio_list, - &nr_scanned, sc, lru); + nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &folio_list, sc, lru); + nr_scanned = sc->nr_isolate_scanned; __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken); item = PGSCAN_KSWAPD + reclaimer_offset(sc); @@ -2071,7 +2072,6 @@ static void shrink_active_list(unsigned long nr_to_scan, enum lru_list lru) { unsigned long nr_taken; - unsigned long nr_scanned; vm_flags_t vm_flags; LIST_HEAD(l_hold); /* The folios which were snipped off */ LIST_HEAD(l_active); @@ -2085,12 +2085,11 @@ static void shrink_active_list(unsigned long nr_to_scan, lruvec_lock_irq(lruvec); - nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &l_hold, - &nr_scanned, sc, lru); + nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &l_hold, sc, lru); __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken); - mod_lruvec_state(lruvec, PGREFILL, nr_scanned); + mod_lruvec_state(lruvec, PGREFILL, sc->nr_isolate_scanned); lruvec_unlock_irq(lruvec); @@ -5920,10 +5919,21 @@ static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) for_each_evictable_lru(lru) { if (nr[lru]) { nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX); - nr[lru] -= nr_to_scan; + sc->nr_isolate_scanned = 0; nr_reclaimed += shrink_list(lru, nr_to_scan, lruvec, sc); + /* + * isolate_lru_folios() may scan far more + * than nr_to_scan when the LRU holds + * ineligible folios (zone-skip) or large + * folios. Charge that overshoot against the + * remaining quota (clamped by min() so it + * cannot go negative) so the next iteration + * does not rescan the same skipped folios. + */ + nr[lru] -= min(nr[lru], + max(nr_to_scan, sc->nr_isolate_scanned)); } } -- 2.43.0