The swap cache allocator evaluates the refault of every folio it allocates. zswap writeback also allocates through it: the shrinker puts a buffer folio in the swap cache to write the compressed data out, and that allocation is then counted as an anon refault (and, if the eviction looks recent, as an activation) even though nothing faulted the page back in. On a workload that writes back continuously this inflates workingset_refault_anon and workingset_activate_anon substantially. Move the refault evaluation out of the allocator and into the two swap-in callers. zswap writeback keeps allocating the buffer, but no longer reports a refault for it. The callers cannot read the shadow themselves before allocating: the allocation can sleep, so another swap-in may install a folio, have it reclaimed and leave a newer shadow behind, and this caller would then win the insertion but refault against the stale snapshot. Hand the shadow back from __swap_cache_alloc_folio() instead, which already captures it under ci->lock in __swap_cache_add_check(), at the point the insertion that displaces it succeeds. The refault is evaluated before folio_add_lru(), as it was before this patch, so workingset_refault() still sets PG_workingset/PG_active while the folio is off the LRU: folio_add_lru() consumes both when it picks the LRU list, and under MGLRU when it picks the generation. Fixes: aae466b0052e ("mm/swap: implement workingset detection for anonymous LRU") Signed-off-by: Nhat Pham Signed-off-by: Alexandre Ghiti --- mm/swap.h | 3 ++- mm/swap_state.c | 28 +++++++++++++++++++++------- mm/zswap.c | 2 +- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/mm/swap.h b/mm/swap.h index 8679cb61268e..5e959bedf5a3 100644 --- a/mm/swap.h +++ b/mm/swap.h @@ -314,7 +314,8 @@ void *swap_cache_get_shadow(swp_entry_t entry); void swap_cache_del_folio(struct folio *folio); struct folio *__swap_cache_alloc_folio(swp_entry_t target_entry, gfp_t gfp_mask, unsigned long orders, struct vm_fault *vmf, - struct mempolicy *mpol, pgoff_t ilx); + struct mempolicy *mpol, pgoff_t ilx, + void **shadowp); /* Below helpers require the caller to lock and pass in the swap cluster. */ void __swap_cache_add_folio(struct swap_cluster_info *ci, struct folio *folio, swp_entry_t entry); diff --git a/mm/swap_state.c b/mm/swap_state.c index bf8ff2d2dbf1..c95d1ddf3193 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -409,7 +409,8 @@ void __swap_cache_replace_folio(struct swap_cluster_info *ci, static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci, swp_entry_t targ_entry, gfp_t gfp, unsigned int order, struct vm_fault *vmf, - struct mempolicy *mpol, pgoff_t ilx) + struct mempolicy *mpol, pgoff_t ilx, + void **shadowp) { int err; swp_entry_t entry; @@ -483,12 +484,13 @@ static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci, /* memsw uncharges swap when folio is added to swap cache */ memcg1_swapin(folio); - if (shadow) - workingset_refault(folio, shadow); node_stat_mod_folio(folio, NR_FILE_PAGES, nr_pages); lruvec_stat_mod_folio(folio, NR_SWAPCACHE, nr_pages); + if (shadowp) + *shadowp = shadow; + return folio; } @@ -500,6 +502,7 @@ static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci, * @vmf: fault information * @mpol: NUMA memory allocation policy to be applied * @ilx: NUMA interleave index, for use only when MPOL_INTERLEAVE + * @shadowp: Returns the shadow the allocation displaced, NULL to ignore * * Allocate a folio in the swap cache for one swap slot, typically before * doing IO (e.g. swap in or zswap writeback). The swap slot indicated by @@ -515,7 +518,8 @@ static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci, */ struct folio *__swap_cache_alloc_folio(swp_entry_t targ_entry, gfp_t gfp, unsigned long orders, struct vm_fault *vmf, - struct mempolicy *mpol, pgoff_t ilx) + struct mempolicy *mpol, pgoff_t ilx, + void **shadowp) { int order, err; struct folio *ret; @@ -530,7 +534,7 @@ struct folio *__swap_cache_alloc_folio(swp_entry_t targ_entry, gfp_t gfp, do { ret = __swap_cache_alloc(ci, targ_entry, gfp, order, - vmf, mpol, ilx); + vmf, mpol, ilx, shadowp); if (!IS_ERR(ret)) break; err = PTR_ERR(ret); @@ -646,17 +650,22 @@ static struct folio *swap_cache_read_folio(struct swap_io_ctx *ctx, pgoff_t ilx, bool readahead) { struct folio *folio; + void *shadow = NULL; do { folio = swap_cache_get_folio(entry); if (folio) return folio; - folio = __swap_cache_alloc_folio(entry, gfp, BIT(0), NULL, mpol, ilx); + folio = __swap_cache_alloc_folio(entry, gfp, BIT(0), NULL, mpol, + ilx, &shadow); } while (PTR_ERR(folio) == -EEXIST); if (IS_ERR_OR_NULL(folio)) return NULL; + if (shadow) + workingset_refault(folio, shadow); + folio_add_lru(folio); swap_read_folio(ctx, folio); if (readahead) { @@ -688,17 +697,22 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders, { struct swap_io_ctx ctx = {}; struct folio *folio; + void *shadow = NULL; do { folio = swap_cache_get_folio(entry); if (folio) return folio; - folio = __swap_cache_alloc_folio(entry, gfp, orders, vmf, mpol, ilx); + folio = __swap_cache_alloc_folio(entry, gfp, orders, vmf, mpol, + ilx, &shadow); } while (PTR_ERR(folio) == -EEXIST); if (IS_ERR(folio)) return folio; + if (shadow) + workingset_refault(folio, shadow); + folio_add_lru(folio); swap_read_folio(&ctx, folio); swap_read_submit(&ctx); diff --git a/mm/zswap.c b/mm/zswap.c index 0d2efe21f18a..16b78d44fdd0 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -1002,7 +1002,7 @@ static int zswap_writeback_entry(struct zswap_entry *entry, mpol = get_task_policy(current); folio = __swap_cache_alloc_folio(swpentry, GFP_KERNEL, BIT(0), NULL, mpol, - NO_INTERLEAVE_INDEX); + NO_INTERLEAVE_INDEX, NULL); put_swap_device(si); /* -- 2.53.0-Meta