Splitting into little patches gets tedious: now that folios on per-cpu fbatches no longer hold an extra reference (and have the lru flag set), most calls to lru_add_drain(), and more importantly lru_add_drain_all(), should be removed. Remove lru_add_drain() from compact_zone(). Remove lru_add_drain_all() from compact_nodes() and compact_store(). Keep lru_add_drain_cpu_zone() in compact_zone(): it was always a bit of a hack, a cheap way to get a local_lock() around pcp drain_pages_zone(). Remove lru_add_drain() and lru_add_drain_all() from generic_fadvise( POSIX_FADV_DONTNEED), but still retry once on failure; remove outdated comment line from mapping_try_invalidate(). Remove every lru_add_drain() and lru_add_drain_all() from khugepaged.c. Remove lru_add_drain_all() from KSM's scan_get_next_rmap_item() restart. Remove lru_add_drain() from wp_can_reuse_anon_folio() and do_swap_page(). Remove lru_add_drain(), lru_add_drain_all() from migrate_device_unmap(). Keep lru_add_drain()s in mm/gup.c populate_vma_page_range() and faultin_page_range(): good housekeeping after a bulk operation. Keep lru_add_drain()s in mm/madvise.c: the ones after a bulk op probably do want to "Push any new pages onto the LRU now", and update the stats; the ones before a bulk op may be trying to stabilize initial conditions, or to minimize draining under ptlock. Keep lru_add_drain_all() in memcg-v1's mem_cgroup_force_empty(): it probably does want to push onto the LRU and update the stats. Keep lru_add_drain() after bulk op in mlock_vma_pages_range(), partly for "Unevictable kB" accuracy; but remove the lru_add_drain() before it. Keep lru_add_drain()s in swap_cluster_readahead(), swap_vma_readahead(): they do want to "Push any new pages onto the LRU now" (akpm 2.5.46). Keep lru_add_drain()s and lru_add_drain_all() throughout mm/vmscan.c: it works on LRUs, so it does need folios to be on an actual LRU. Signed-off-by: Hugh Dickins --- mm/compaction.c | 12 +----------- mm/fadvise.c | 17 +---------------- mm/khugepaged.c | 11 ----------- mm/ksm.c | 12 ------------ mm/memory.c | 14 ++------------ mm/migrate_device.c | 9 --------- mm/mlock.c | 1 - mm/truncate.c | 1 - 8 files changed, 4 insertions(+), 73 deletions(-) diff --git a/mm/compaction.c b/mm/compaction.c index 9e045a90ba21..8ed375b08f3e 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -2652,9 +2652,6 @@ compact_zone(struct compact_control *cc, struct capture_control *capc) trace_mm_compaction_begin(cc, start_pfn, end_pfn, sync); - /* lru_add_drain_all could be expensive with involving other CPUs */ - lru_add_drain(); - while ((ret = compact_finished(cc)) == COMPACT_CONTINUE) { int err; unsigned long iteration_start_pfn = cc->migrate_pfn; @@ -2969,9 +2966,6 @@ static int compact_nodes(void) { int ret, nid; - /* Flush pending updates to the LRU lists */ - lru_add_drain_all(); - for_each_online_node(nid) { ret = compact_node(NODE_DATA(nid), false); if (ret) @@ -3036,12 +3030,8 @@ static ssize_t compact_store(struct device *dev, { int nid = dev->id; - if (nid >= 0 && nid < nr_node_ids && node_online(nid)) { - /* Flush pending updates to the LRU lists */ - lru_add_drain_all(); - + if (nid >= 0 && nid < nr_node_ids && node_online(nid)) compact_node(NODE_DATA(nid), false); - } return count; } diff --git a/mm/fadvise.c b/mm/fadvise.c index b63fe21416ff..f788a5020384 100644 --- a/mm/fadvise.c +++ b/mm/fadvise.c @@ -143,27 +143,12 @@ int generic_fadvise(struct file *file, loff_t offset, loff_t len, int advice) if (end_index >= start_index) { unsigned long nr_failed = 0; - /* - * It's common to FADV_DONTNEED right after - * the read or write that instantiates the - * pages, in which case there will be some - * sitting on the local LRU cache. Try to - * avoid the expensive remote drain and the - * second cache tree walk below by flushing - * them out right away. - */ - lru_add_drain(); - mapping_try_invalidate(mapping, start_index, end_index, &nr_failed); - /* - * The failures may be due to the folio being - * in the LRU cache of a remote CPU. Drain all - * caches and try again. + * Retry if any failures, in case they were transient. */ if (nr_failed) { - lru_add_drain_all(); invalidate_mapping_pages(mapping, start_index, end_index); } diff --git a/mm/khugepaged.c b/mm/khugepaged.c index b237f6e7662a..904a45c11fca 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1225,10 +1225,6 @@ static enum scan_result __collapse_huge_page_swapin(struct mm_struct *mm, if (pte) pte_unmap(pte); - /* Drain LRU cache to remove extra pin on the swapped in pages */ - if (swapped_in) - lru_add_drain(); - result = SCAN_SUCCEED; out: trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, result, @@ -2325,8 +2321,6 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, result = SCAN_FAIL; goto xa_unlocked; } - /* drain lru cache to help folio_isolate_lru() */ - lru_add_drain(); } else if (folio_trylock(folio)) { folio_get(folio); xas_unlock_irq(&xas); @@ -2340,8 +2334,6 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, page_cache_sync_readahead(mapping, &file->f_ra, file, index, end - index); - /* drain lru cache to help folio_isolate_lru() */ - lru_add_drain(); folio = filemap_lock_folio(mapping, index); if (IS_ERR(folio)) { result = SCAN_FAIL; @@ -2981,8 +2973,6 @@ static void khugepaged_do_scan(struct collapse_control *cc) bool wait = true; enum scan_result result = SCAN_SUCCEED; - lru_add_drain_all(); - cc->progress = 0; while (true) { cond_resched(); @@ -3214,7 +3204,6 @@ int madvise_collapse(struct vm_area_struct *vma, unsigned long start, cc->progress = 0; mmgrab(mm); - lru_add_drain_all(); for (addr = hstart; addr < hend; addr += HPAGE_PMD_SIZE) { enum scan_result result = SCAN_FAIL; diff --git a/mm/ksm.c b/mm/ksm.c index b4142746777e..5bae928620ad 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -2626,18 +2626,6 @@ static struct ksm_rmap_item *scan_get_next_rmap_item(struct page **page) advisor_start_scan(); trace_ksm_start_scan(ksm_scan.seqnr, ksm_rmap_items); - /* - * A number of pages can hang around indefinitely in per-cpu - * LRU cache, raised page count preventing write_protect_page - * from merging them. Though it doesn't really matter much, - * it is puzzling to see some stuck in pages_volatile until - * other activity jostles them out, and they also prevented - * LTP's KSM test from succeeding deterministically; so drain - * them here (here rather than on entry to ksm_do_scan(), - * so we don't IPI too often when pages_to_scan is set low). - */ - lru_add_drain_all(); - /* * Whereas stale stable_nodes on the stable_tree itself * get pruned in the regular course of stable_tree_search(), diff --git a/mm/memory.c b/mm/memory.c index 8da0f945141b..1478bee69e30 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4251,9 +4251,6 @@ static bool __wp_can_reuse_large_anon_folio(struct folio *folio, static bool wp_can_reuse_anon_folio(struct folio *folio, struct vm_area_struct *vma) { - const bool maybe_in_lru_cache = !folio_test_lru(folio); - const bool in_swapcache = folio_test_swapcache(folio); - if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && folio_test_large(folio)) return __wp_can_reuse_large_anon_folio(folio, vma); @@ -4264,16 +4261,9 @@ static bool wp_can_reuse_anon_folio(struct folio *folio, * * KSM doesn't necessarily raise the folio refcount. */ - if (folio_test_ksm(folio) || - folio_ref_count(folio) > 1 + maybe_in_lru_cache + in_swapcache) + if (folio_test_ksm(folio)) return false; - if (maybe_in_lru_cache) - /* - * We cannot easily detect+handle references from - * remote LRU caches or references to LRU folios. - */ - lru_add_drain(); - if (folio_ref_count(folio) > 1 + in_swapcache) + if (folio_ref_count(folio) > 1 + folio_test_swapcache(folio)) return false; if (!folio_trylock(folio)) return false; diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 18d097c38853..d3f865904c48 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -575,11 +575,8 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns, struct folio *fault_folio = fault_page ? page_folio(fault_page) : NULL; unsigned long i, restore = 0; - bool allow_drain = true; unsigned long unmapped = 0; - lru_add_drain(); - for (i = 0; i < npages; ) { struct page *page = migrate_pfn_to_page(src_pfns[i]); struct folio *folio; @@ -600,12 +597,6 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns, /* ZONE_DEVICE folios are not on LRU */ if (!folio_is_zone_device(folio)) { - if (!folio_test_lru(folio) && allow_drain) { - /* Drain CPU's lru cache */ - lru_add_drain_all(); - allow_drain = false; - } - if (!folio_isolate_lru(folio)) { src_pfns[i] &= ~MIGRATE_PFN_MIGRATE; restore++; diff --git a/mm/mlock.c b/mm/mlock.c index 97134eff6b56..971430e6251e 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -424,7 +424,6 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma, vma_start_write(vma); vma_flags_reset_once(vma, new_vma_flags); - lru_add_drain(); walk_page_range_vma(vma, start, end, &mlock_walk_ops, NULL); lru_add_drain(); diff --git a/mm/truncate.c b/mm/truncate.c index 4151f7a167e3..7ea5d513f76a 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -571,7 +571,6 @@ unsigned long mapping_try_invalidate(struct address_space *mapping, */ if (!ret) { deactivate_file_folio(folio); - /* Likely in the lru cache of a remote CPU */ if (nr_failed) (*nr_failed)++; } -- 2.51.0