zswap writeback wants a swap cache folio it can free directly once writeback completes, i.e. one that is not on the LRU (dropping the last reference on an LRU folio would trip the free-time page-flag checks). So defer the LRU addition to the callers of __swap_cache_alloc_folio(), no functional change intended. Suggested-by: Kairui Song Signed-off-by: Alexandre Ghiti --- mm/swap.h | 6 +++--- mm/swap_state.c | 20 ++++++++++++-------- mm/zswap.c | 5 +++-- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/mm/swap.h b/mm/swap.h index 77d2d14eda42..fc44daae1de1 100644 --- a/mm/swap.h +++ b/mm/swap.h @@ -304,9 +304,9 @@ bool swap_cache_has_folio(swp_entry_t entry); struct folio *swap_cache_get_folio(swp_entry_t entry); void *swap_cache_get_shadow(swp_entry_t entry); void swap_cache_del_folio(struct folio *folio); -struct folio *swap_cache_alloc_folio(swp_entry_t target_entry, gfp_t gfp_mask, - unsigned long orders, struct vm_fault *vmf, - struct mempolicy *mpol, pgoff_t ilx); +struct folio *__swap_cache_alloc_folio(swp_entry_t target_entry, gfp_t gfp_mask, + unsigned long orders, struct vm_fault *vmf, + struct mempolicy *mpol, pgoff_t ilx); /* Below helpers require the caller to lock and pass in the swap cluster. */ void __swap_cache_add_folio(struct swap_cluster_info *ci, struct folio *folio, swp_entry_t entry); diff --git a/mm/swap_state.c b/mm/swap_state.c index 9c3a5cf99778..90638a8d7232 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -483,13 +483,11 @@ static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci, node_stat_mod_folio(folio, NR_FILE_PAGES, nr_pages); lruvec_stat_mod_folio(folio, NR_SWAPCACHE, nr_pages); - /* Caller will initiate read into locked new_folio */ - folio_add_lru(folio); return folio; } /** - * swap_cache_alloc_folio - Allocate folio for swapped out slot in swap cache. + * __swap_cache_alloc_folio - Allocate folio for swapped out slot in swap cache. * @targ_entry: swap entry indicating the target slot * @gfp: memory allocation flags * @orders: allocation orders, must be non zero @@ -501,13 +499,17 @@ static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci, * doing IO (e.g. swap in or zswap writeback). The swap slot indicated by * @targ_entry must have a non-zero swap count (swapped out). * + * The returned folio is locked and is NOT on the LRU. The caller must either + * add it to the LRU with folio_add_lru() so page reclaim can find it, or free + * it directly once done; a folio left off the LRU is unreclaimable and leaks. + * * Context: Caller must protect the swap device with reference count or locks. * Return: Returns the folio if allocation succeeded and folio is in the swap * cache. Returns error code if failed due to race, OOM or invalid arguments. */ -struct folio *swap_cache_alloc_folio(swp_entry_t targ_entry, gfp_t gfp, - unsigned long orders, struct vm_fault *vmf, - struct mempolicy *mpol, pgoff_t ilx) +struct folio *__swap_cache_alloc_folio(swp_entry_t targ_entry, gfp_t gfp, + unsigned long orders, struct vm_fault *vmf, + struct mempolicy *mpol, pgoff_t ilx) { int order, err; struct folio *ret; @@ -643,12 +645,13 @@ static struct folio *swap_cache_read_folio(swp_entry_t entry, gfp_t gfp, folio = swap_cache_get_folio(entry); if (folio) return folio; - folio = swap_cache_alloc_folio(entry, gfp, BIT(0), NULL, mpol, ilx); + folio = __swap_cache_alloc_folio(entry, gfp, BIT(0), NULL, mpol, ilx); } while (PTR_ERR(folio) == -EEXIST); if (IS_ERR_OR_NULL(folio)) return NULL; + folio_add_lru(folio); swap_read_folio(folio, plug); if (readahead) { folio_set_readahead(folio); @@ -683,12 +686,13 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders, folio = swap_cache_get_folio(entry); if (folio) return folio; - folio = swap_cache_alloc_folio(entry, gfp, orders, vmf, mpol, ilx); + folio = __swap_cache_alloc_folio(entry, gfp, orders, vmf, mpol, ilx); } while (PTR_ERR(folio) == -EEXIST); if (IS_ERR(folio)) return folio; + folio_add_lru(folio); swap_read_folio(folio, NULL); return folio; } diff --git a/mm/zswap.c b/mm/zswap.c index 761cd699e0a3..8163e6c5f76c 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -1000,8 +1000,8 @@ static int zswap_writeback_entry(struct zswap_entry *entry, return -EEXIST; mpol = get_task_policy(current); - folio = swap_cache_alloc_folio(swpentry, GFP_KERNEL, BIT(0), NULL, mpol, - NO_INTERLEAVE_INDEX); + folio = __swap_cache_alloc_folio(swpentry, GFP_KERNEL, BIT(0), NULL, mpol, + NO_INTERLEAVE_INDEX); put_swap_device(si); /* @@ -1013,6 +1013,7 @@ static int zswap_writeback_entry(struct zswap_entry *entry, */ if (IS_ERR(folio)) return PTR_ERR(folio); + folio_add_lru(folio); /* * folio is locked, and the swapcache is now secured against -- 2.53.0-Meta A PG_dropbehind folio is dropped from its cache once writeback completes rather than left for reclaim to find later; this is implemented for file folios in folio_end_dropbehind(). Extend it to swap cache folios. The drop needs the folio and swap cluster locks and may sleep, but writeback can complete in interrupt context, so defer the work to a workqueue that runs once writeback has completed and frees the folios in batches. Suggested-by: Yosry Ahmed Suggested-by: Johannes Weiner Suggested-by: Nhat Pham Signed-off-by: Alexandre Ghiti --- fs/splice.c | 2 +- include/linux/swap.h | 6 ++- mm/filemap.c | 19 +++++++++ mm/swap_state.c | 94 ++++++++++++++++++++++++++++++++++++++++++++ mm/truncate.c | 2 +- mm/vmscan.c | 13 ++++-- 6 files changed, 130 insertions(+), 6 deletions(-) diff --git a/fs/splice.c b/fs/splice.c index 9d8f63e2fd1a..79424bbee1b0 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -90,7 +90,7 @@ static bool page_cache_pipe_buf_try_steal(struct pipe_inode_info *pipe, * If we succeeded in removing the mapping, set LRU flag * and return good. */ - if (remove_mapping(mapping, folio)) { + if (remove_mapping(mapping, folio, false, NULL)) { buf->flags |= PIPE_BUF_FLAG_LRU; return true; } diff --git a/include/linux/swap.h b/include/linux/swap.h index 6d72778e6cc3..25470634dd9d 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -373,7 +373,8 @@ extern unsigned long mem_cgroup_shrink_node(struct mem_cgroup *mem, unsigned long *nr_scanned); extern unsigned long shrink_all_memory(unsigned long nr_pages); extern int vm_swappiness; -long remove_mapping(struct address_space *mapping, struct folio *folio); +long remove_mapping(struct address_space *mapping, struct folio *folio, + bool reclaimed, struct mem_cgroup *target_memcg); #if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA) extern int reclaim_register_node(struct node *node); @@ -467,6 +468,8 @@ void swap_put_entries_direct(swp_entry_t entry, int nr); */ bool folio_free_swap(struct folio *folio); +void swap_writeback_dropbehind_folio(struct folio *folio); + /* Allocate / free (hibernation) exclusive entries */ swp_entry_t swap_alloc_hibernation_slot(int type); void swap_free_hibernation_slot(swp_entry_t entry); @@ -477,6 +480,7 @@ static inline void put_swap_device(struct swap_info_struct *si) } #else /* CONFIG_SWAP */ +static inline void swap_writeback_dropbehind_folio(struct folio *folio) {} static inline struct swap_info_struct *get_swap_device(swp_entry_t entry) { return NULL; diff --git a/mm/filemap.c b/mm/filemap.c index dc3a0e960b9f..aaffacdf5b1f 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1680,6 +1680,8 @@ EXPORT_SYMBOL_GPL(folio_end_writeback_no_dropbehind); */ void folio_end_writeback(struct folio *folio) { + bool swap_dropbehind; + VM_BUG_ON_FOLIO(!folio_test_writeback(folio), folio); /* @@ -1689,7 +1691,24 @@ void folio_end_writeback(struct folio *folio) * reused before the folio_wake_bit(). */ folio_get(folio); + + /* + * Dropbehind swap cache folios are off-LRU, so we must prevent a racing + * swapin from removing the folio from the swap cache and keeping it + * off-LRU: the writeback flag allows that. Afterwards a swapin may win + * the race, but the folio is already queued and the worker puts it back + * on the LRU in that case. + */ + swap_dropbehind = folio_test_swapcache(folio) && + folio_test_dropbehind(folio); + folio_end_writeback_no_dropbehind(folio); + + if (swap_dropbehind) { + swap_writeback_dropbehind_folio(folio); + return; + } + folio_end_dropbehind(folio); folio_put(folio); } diff --git a/mm/swap_state.c b/mm/swap_state.c index 90638a8d7232..b93dd607d24c 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include #include #include @@ -537,6 +539,98 @@ struct folio *__swap_cache_alloc_folio(swp_entry_t targ_entry, gfp_t gfp, return ret; } +static DEFINE_PER_CPU(struct llist_head, swap_dropbehind_llist); + +static bool swap_dropbehind_drop_folio(struct folio *folio) +{ + struct mem_cgroup *memcg; + bool dropped = false; + + folio_lock(folio); + + /* The folio was allocated off the LRU and nothing re-adds it here. */ + VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio); + + rcu_read_lock(); + memcg = folio_memcg(folio); + if (!mem_cgroup_tryget(memcg)) + memcg = NULL; + rcu_read_unlock(); + + /* + * Gate remove_mapping() on folio_test_swapcache(): a racing swapin may + * have freed the swap slot (folio_free_swap()) and dropped the folio from + * the cache, and remove_mapping() must not run on a non-swapcache folio + * (it would trip __remove_mapping()'s mapping == folio_mapping() check). + */ + if (folio_test_swapcache(folio) && !folio_test_writeback(folio) && + remove_mapping(swap_address_space(folio->swap), folio, true, memcg)) { + dropped = true; + } else { + /* Raced: the folio is now owned by the swapin; put it back. */ + folio_clear_dropbehind(folio); + folio_add_lru(folio); + } + + mem_cgroup_put(memcg); + + folio_unlock(folio); + if (!dropped) + folio_put(folio); + return dropped; +} + +/** + * swap_dropbehind_free_batch_folio - free a dropbehind swap cache folio into a batch + * @folio: the off-LRU folio whose writeback has completed + * @fbatch: batch of folios to free, flushed when full + */ +static void swap_dropbehind_free_batch_folio(struct folio *folio, + struct folio_batch *fbatch) +{ + if (swap_dropbehind_drop_folio(folio) && !folio_batch_add(fbatch, folio)) + folios_put(fbatch); +} + +static void swap_dropbehind_workfn(struct work_struct *work) +{ + struct folio_batch fbatch; + struct llist_node *pos, *next; + int cpu; + + folio_batch_init(&fbatch); + for_each_possible_cpu(cpu) { + pos = llist_del_all(per_cpu_ptr(&swap_dropbehind_llist, cpu)); + llist_for_each_safe(pos, next, pos) { + struct folio *folio = container_of((struct list_head *)pos, + struct folio, lru); + swap_dropbehind_free_batch_folio(folio, &fbatch); + } + } + if (fbatch.nr) + folios_put(&fbatch); +} + +static DECLARE_WORK(swap_dropbehind_work, swap_dropbehind_workfn); +static struct workqueue_struct *swap_dropbehind_wq; + +void swap_writeback_dropbehind_folio(struct folio *folio) +{ + llist_add((struct llist_node *)&folio->lru, + raw_cpu_ptr(&swap_dropbehind_llist)); + queue_work(swap_dropbehind_wq, &swap_dropbehind_work); +} + +static int __init swap_dropbehind_init(void) +{ + swap_dropbehind_wq = alloc_workqueue("swap_dropbehind", + WQ_MEM_RECLAIM | WQ_PERCPU, 0); + if (!swap_dropbehind_wq) + return -ENOMEM; + return 0; +} +core_initcall(swap_dropbehind_init); + /* * If we are the only user, then try to free up the swap cache. * diff --git a/mm/truncate.c b/mm/truncate.c index b58ba940be47..37f7d8278c1c 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -336,7 +336,7 @@ long mapping_evict_folio(struct address_space *mapping, struct folio *folio) if (!filemap_release_folio(folio, 0)) return 0; - return remove_mapping(mapping, folio); + return remove_mapping(mapping, folio, false, NULL); } /** diff --git a/mm/vmscan.c b/mm/vmscan.c index 3f3ff25e561a..37b093c4628c 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -789,17 +789,24 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio, * remove_mapping() - Attempt to remove a folio from its mapping. * @mapping: The address space. * @folio: The folio to remove. + * @reclaimed: Whether the folio is being reclaimed (record a shadow). + * @target_memcg: The memcg to charge the eviction shadow to when @reclaimed; + * the caller must keep it alive across the call. Ignored (may + * be NULL) when @reclaimed is false. * * If the folio is dirty, under writeback or if someone else has a ref - * on it, removal will fail. + * on it, removal will fail. When @reclaimed is true, a workingset eviction + * shadow is stored (like page reclaim does) so that a later refault can be + * detected and the folio re-activated. * Return: The number of pages removed from the mapping. 0 if the folio * could not be removed. * Context: The caller should have a single refcount on the folio and * hold its lock. */ -long remove_mapping(struct address_space *mapping, struct folio *folio) +long remove_mapping(struct address_space *mapping, struct folio *folio, + bool reclaimed, struct mem_cgroup *target_memcg) { - if (__remove_mapping(mapping, folio, false, NULL)) { + if (__remove_mapping(mapping, folio, reclaimed, target_memcg)) { /* * Unfreezing the refcount with 1 effectively * drops the pagecache ref for us without requiring another -- 2.53.0-Meta zswap writeback decompresses an entry into a fresh swap cache folio and writes it back. The folio is cold by construction, yet it is left on the LRU for reclaim to find and free later, wasting a reclaim scan and keeping cold memory resident longer than necessary. Allocate the folio off the LRU and mark it PG_dropbehind so the swap dropbehind path frees it from the swap cache once writeback completes. Suggested-by: Johannes Weiner Suggested-by: Nhat Pham Signed-off-by: Alexandre Ghiti --- mm/zswap.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mm/zswap.c b/mm/zswap.c index 8163e6c5f76c..a35671e837f8 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -1013,7 +1013,6 @@ static int zswap_writeback_entry(struct zswap_entry *entry, */ if (IS_ERR(folio)) return PTR_ERR(folio); - folio_add_lru(folio); /* * folio is locked, and the swapcache is now secured against @@ -1046,8 +1045,7 @@ static int zswap_writeback_entry(struct zswap_entry *entry, /* folio is up to date */ folio_mark_uptodate(folio); - /* move it to the tail of the inactive list after end_writeback */ - folio_set_reclaim(folio); + folio_set_dropbehind(folio); /* start writeback */ __swap_writepage(folio, NULL); -- 2.53.0-Meta