From: Zhang Peng shrink_folio_list() contains a self-contained folio-freeing section: buffer release, lazyfree, __remove_mapping(), and folio_batch draining. Extract it into folio_try_reclaim_free() to reduce the size of shrink_folio_list() and make the freeing step independently readable. Return an explicit result so the caller retains the distinction between activating a folio, keeping it on the inactive list, and reclaiming it. The helper leaves the folio locked when it returns ACTIVATE or KEEP and consumes it when it returns DONE. Group retained folios, deferred frees and the reclaimed count in a reclaim_folio_state shared by the reclaim paths. Initialize it once per shrink_folio_list() invocation, including demotion retries. No functional change. Signed-off-by: Zhang Peng --- mm/vmscan.c | 201 +++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 116 insertions(+), 85 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 54f6a9e62715..9ac125f2e317 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1177,6 +1177,106 @@ static void folio_activate_locked(struct folio *folio, } } +/* Results accumulated across a shrink_folio_list() invocation. */ +struct reclaim_folio_state { + struct list_head ret_folios; + struct folio_batch free_folios; + unsigned int nr_reclaimed; +}; + +enum folio_reclaim_result { + FOLIO_RECLAIM_KEEP, + FOLIO_RECLAIM_ACTIVATE, + FOLIO_RECLAIM_DONE, +}; + +/* + * Called with the folio locked. KEEP and ACTIVATE leave it locked and + * owned by the caller. DONE consumes the caller's reference; the caller + * must not access the folio again, even if freeing is deferred. + */ +static enum folio_reclaim_result folio_try_reclaim_free(struct folio *folio, + struct reclaim_folio_state *state, + struct scan_control *sc) +{ + const unsigned int nr_pages = folio_nr_pages(folio); + struct address_space *mapping = folio_mapping(folio); + + /* + * If the folio has buffers, try to free the buffer mappings + * associated with this folio. If we succeed we try to free + * the folio as well. + * + * We do this even if the folio is dirty. + * filemap_release_folio() does not perform I/O, but it is + * possible for a folio to have the dirty flag set, but it + * is actually clean (all its buffers are clean). This + * happens if the buffers were written out directly, with + * bh_submit(). ext3 will do this, as well as the blockdev + * mapping. filemap_release_folio() will discover that + * cleanness and will drop the buffers and mark the folio + * clean - it can be freed. + * + * Rarely, folios can have buffers and no ->mapping. These + * are the folios which were not successfully invalidated in + * truncate_cleanup_folio(). We try to drop those buffers + * here and if that worked, and the folio is no longer + * mapped into process address space (refcount == 1) it can + * be freed. Otherwise, leave the folio on the LRU so it is + * swappable. + */ + if (folio_needs_release(folio)) { + if (!filemap_release_folio(folio, sc->gfp_mask)) + return FOLIO_RECLAIM_ACTIVATE; + + if (!mapping && folio_ref_count(folio) == 1) { + folio_unlock(folio); + if (folio_put_testzero(folio)) + goto free_it; + + /* + * Rare race with speculative reference. The + * speculative reference will free this folio + * shortly, so we may increment nr_reclaimed here + * and leave it off the LRU. + */ + state->nr_reclaimed += nr_pages; + return FOLIO_RECLAIM_DONE; + } + } + + if (folio_test_lazyfree(folio)) { + /* follow __remove_mapping for reference */ + if (!folio_ref_freeze(folio, 1)) + return FOLIO_RECLAIM_KEEP; + /* + * The folio has only one reference left, which is + * from the isolation. After the caller puts the + * folio back on the lru and drops the reference, the + * folio will be freed anyway. It doesn't matter + * which lru it goes on. So we don't bother checking + * the dirty flag here. + */ + count_vm_events(PGLAZYFREED, nr_pages); + count_memcg_folio_events(folio, PGLAZYFREED, nr_pages); + } else if (!mapping || !__remove_mapping(mapping, folio, true, + sc->target_mem_cgroup)) + return FOLIO_RECLAIM_KEEP; + + folio_unlock(folio); +free_it: + VM_WARN_ON_ONCE_FOLIO(folio_ref_count(folio), folio); + state->nr_reclaimed += nr_pages; + + folio_unqueue_deferred_split(folio); + if (folio_batch_add(&state->free_folios, folio) == 0) { + mem_cgroup_uncharge_folios(&state->free_folios); + try_to_unmap_flush(); + free_unref_folios(&state->free_folios); + } + return FOLIO_RECLAIM_DONE; +} + /* * shrink_folio_list() returns the number of reclaimed pages */ @@ -1185,15 +1285,15 @@ static unsigned int shrink_folio_list(struct list_head *folio_list, struct reclaim_stat *stat, bool ignore_references, struct mem_cgroup *memcg) { - struct folio_batch free_folios; - LIST_HEAD(ret_folios); + struct reclaim_folio_state state = {}; LIST_HEAD(demote_folios); - unsigned int nr_reclaimed = 0, nr_demoted = 0; + unsigned int nr_demoted = 0; unsigned int pgactivate = 0; bool do_demote_pass; struct swap_io_ctx ctx = {}; - folio_batch_init(&free_folios); + INIT_LIST_HEAD(&state.ret_folios); + folio_batch_init(&state.free_folios); memset(stat, 0, sizeof(*stat)); cond_resched(); do_demote_pass = can_demote(pgdat->node_id, sc, memcg); @@ -1564,83 +1664,14 @@ static unsigned int shrink_folio_list(struct list_head *folio_list, } } - /* - * If the folio has buffers, try to free the buffer - * mappings associated with this folio. If we succeed - * we try to free the folio as well. - * - * We do this even if the folio is dirty. - * filemap_release_folio() does not perform I/O, but it - * is possible for a folio to have the dirty flag set, - * but it is actually clean (all its buffers are clean). - * This happens if the buffers were written out directly, - * with bh_submit(). ext3 will do this, as well as - * the blockdev mapping. filemap_release_folio() will - * discover that cleanness and will drop the buffers - * and mark the folio clean - it can be freed. - * - * Rarely, folios can have buffers and no ->mapping. - * These are the folios which were not successfully - * invalidated in truncate_cleanup_folio(). We try to - * drop those buffers here and if that worked, and the - * folio is no longer mapped into process address space - * (refcount == 1) it can be freed. Otherwise, leave - * the folio on the LRU so it is swappable. - */ - if (folio_needs_release(folio)) { - if (!filemap_release_folio(folio, sc->gfp_mask)) - goto activate_locked; - if (!mapping && folio_ref_count(folio) == 1) { - folio_unlock(folio); - if (folio_put_testzero(folio)) - goto free_it; - else { - /* - * rare race with speculative reference. - * the speculative reference will free - * this folio shortly, so we may - * increment nr_reclaimed here (and - * leave it off the LRU). - */ - nr_reclaimed += nr_pages; - continue; - } - } - } - - if (folio_test_lazyfree(folio)) { - /* follow __remove_mapping for reference */ - if (!folio_ref_freeze(folio, 1)) - goto keep_locked; - /* - * The folio has only one reference left, which is - * from the isolation. After the caller puts the - * folio back on the lru and drops the reference, the - * folio will be freed anyway. It doesn't matter - * which lru it goes on. So we don't bother checking - * the dirty flag here. - */ - count_vm_events(PGLAZYFREED, nr_pages); - count_memcg_folio_events(folio, PGLAZYFREED, nr_pages); - } else if (!mapping || !__remove_mapping(mapping, folio, true, - sc->target_mem_cgroup)) + switch (folio_try_reclaim_free(folio, &state, sc)) { + case FOLIO_RECLAIM_ACTIVATE: + goto activate_locked; + case FOLIO_RECLAIM_KEEP: goto keep_locked; - - folio_unlock(folio); -free_it: - /* - * Folio may get swapped out as a whole, need to account - * all pages in it. - */ - nr_reclaimed += nr_pages; - - folio_unqueue_deferred_split(folio); - if (folio_batch_add(&free_folios, folio) == 0) { - mem_cgroup_uncharge_folios(&free_folios); - try_to_unmap_flush(); - free_unref_folios(&free_folios); + case FOLIO_RECLAIM_DONE: + continue; } - continue; activate_locked_split: /* @@ -1654,7 +1685,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list, keep_locked: folio_unlock(folio); keep: - list_add(&folio->lru, &ret_folios); + list_add(&folio->lru, &state.ret_folios); VM_BUG_ON_FOLIO(folio_test_lru(folio) || folio_test_unevictable(folio), folio); } @@ -1662,7 +1693,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list, /* Migrate folios selected for demotion */ nr_demoted = demote_folio_list(&demote_folios, pgdat, memcg); - nr_reclaimed += nr_demoted; + state.nr_reclaimed += nr_demoted; stat->nr_demoted += nr_demoted; /* Folios that could not be demoted are still in @demote_folios */ if (!list_empty(&demote_folios)) { @@ -1693,15 +1724,15 @@ static unsigned int shrink_folio_list(struct list_head *folio_list, pgactivate = stat->nr_activate[0] + stat->nr_activate[1]; - mem_cgroup_uncharge_folios(&free_folios); + mem_cgroup_uncharge_folios(&state.free_folios); try_to_unmap_flush(); - free_unref_folios(&free_folios); + free_unref_folios(&state.free_folios); - list_splice(&ret_folios, folio_list); + list_splice(&state.ret_folios, folio_list); count_vm_events(PGACTIVATE, pgactivate); swap_write_submit(&ctx); - return nr_reclaimed; + return state.nr_reclaimed; } unsigned int reclaim_clean_pages_from_list(struct zone *zone, -- 2.56.0