page_cluster and the vm.page-cluster sysctl are only used by swap-in readahead in swap_state.c. Move them out of swap.c together with swap_readahead_setup(), and make page_cluster static to that file. Rename swap_setup() while moving it as well. The helper is internal to MM and now only sets up swap readahead defaults and its sysctl hook, so the more specific name matches its reduced scope. Call swap_readahead_setup() from swap_init() after moving it, keeping the readahead defaults and sysctl registration with swap_state.c initialization. swap_setup() previously lived in mm/swap.c, which is built unconditionally, so the vm.page-cluster sysctl was registered also on CONFIG_SWAP=n kernels. After moving the setup into swap_state.c, which is built only when CONFIG_SWAP is enabled, vm.page-cluster is no longer registered there. The knob only tunes swap-in readahead and had no effect without swap. Suggested-by: Baoquan He Suggested-by: Barry Song Suggested-by: Johannes Weiner Suggested-by: Matthew Wilcox Reviewed-by: Kairui Song Acked-by: Johannes Weiner Acked-by: David Hildenbrand (Arm) Signed-off-by: Jianyue Wu --- include/linux/swap.h | 1 - mm/swap.c | 36 ------------------------------------ mm/swap.h | 2 -- mm/swap_state.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++++-- mm/vmscan.c | 1 - 5 files changed, 48 insertions(+), 42 deletions(-) diff --git a/include/linux/swap.h b/include/linux/swap.h index 3f31b6a56788..45656ca9792d 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -340,7 +340,6 @@ extern void lru_add_drain_cpu_zone(struct zone *zone); extern void lru_add_drain_all(void); void folio_deactivate(struct folio *folio); void folio_mark_lazyfree(struct folio *folio); -extern void swap_setup(void); /* linux/mm/vmscan.c */ extern unsigned long zone_reclaimable_pages(struct zone *zone); diff --git a/mm/swap.c b/mm/swap.c index d25131305c94..c88643e56af7 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -44,10 +44,6 @@ #define CREATE_TRACE_POINTS #include -/* How many pages do we try to swap or page in/out together? As a power of 2 */ -int page_cluster; -static const int page_cluster_max = 31; - struct cpu_fbatches { /* * The following folio batches are grouped together because they are protected @@ -1177,35 +1173,3 @@ void lru_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int lruvec_reparent_lru(child_lruvec, parent_lruvec, lru, nid); } #endif - -static const struct ctl_table swap_sysctl_table[] = { - { - .procname = "page-cluster", - .data = &page_cluster, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ZERO, - .extra2 = (void *)&page_cluster_max, - } -}; - -/* - * Perform any setup for the swap system - */ -void __init swap_setup(void) -{ - unsigned long megs = PAGES_TO_MB(totalram_pages()); - - /* Use a smaller cluster for small-memory machines */ - if (megs < 16) - page_cluster = 2; - else - page_cluster = 3; - /* - * Right now other parts of the system means that we - * _really_ don't want to cluster much more - */ - - register_sysctl_init("vm", swap_sysctl_table); -} diff --git a/mm/swap.h b/mm/swap.h index 44ab8e1e595b..b51ad3071a73 100644 --- a/mm/swap.h +++ b/mm/swap.h @@ -8,8 +8,6 @@ struct mempolicy; struct swap_iocb; struct swap_memcg_table; -extern int page_cluster; - #if defined(MAX_POSSIBLE_PHYSMEM_BITS) #define SWAP_CACHE_PFN_BITS (MAX_POSSIBLE_PHYSMEM_BITS - PAGE_SHIFT) #elif defined(MAX_PHYSMEM_BITS) diff --git a/mm/swap_state.c b/mm/swap_state.c index 6fd6e3415b71..1444d20a40e9 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -22,10 +22,15 @@ #include #include #include +#include #include "internal.h" #include "swap_table.h" #include "swap.h" +/* Swap readahead cluster size, as a power of 2 pages. */ +static int page_cluster; +static const int page_cluster_max = 31; + /* * swapper_space is a fiction, retained to simplify the path through * vmscan's shrink_folio_list. @@ -985,6 +990,35 @@ struct folio *swapin_readahead(swp_entry_t entry, gfp_t gfp_mask, return folio; } +static const struct ctl_table swap_readahead_sysctl_table[] = { + { + .procname = "page-cluster", + .data = &page_cluster, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = (void *)&page_cluster_max, + } +}; + +static void __init swap_readahead_setup(void) +{ + unsigned long megs = PAGES_TO_MB(totalram_pages()); + + /* Use a smaller cluster for small-memory machines */ + if (megs < 16) + page_cluster = 2; + else + page_cluster = 3; + /* + * Right now other parts of the system means that we + * _really_ don't want to cluster much more + */ + + register_sysctl_init("vm", swap_readahead_sysctl_table); +} + #ifdef CONFIG_SYSFS static ssize_t vma_ra_enabled_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) @@ -1014,7 +1048,7 @@ static const struct attribute_group swap_attr_group = { .attrs = swap_attrs, }; -static int __init swap_init(void) +static int __init swap_sysfs_init(void) { int err; struct kobject *swap_kobj; @@ -1037,5 +1071,17 @@ static int __init swap_init(void) kobject_put(swap_kobj); return err; } -subsys_initcall(swap_init); +#else +static int __init swap_sysfs_init(void) +{ + return 0; +} #endif + +static int __init swap_init(void) +{ + swap_readahead_setup(); + + return swap_sysfs_init(); +} +subsys_initcall(swap_init); diff --git a/mm/vmscan.c b/mm/vmscan.c index 1474a7234ea1..74b43c12fbc0 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -7647,7 +7647,6 @@ static int __init kswapd_init(void) { int nid; - swap_setup(); for_each_node_state(nid, N_MEMORY) kswapd_run(nid); register_sysctl_init("vm", vmscan_sysctl_table); -- 2.43.0 Rename mm/swap.c to mm/folio.c so the filename better matches the code's main responsibility. This keeps the implementation split from swap-specific code without changing the published LRU helper interfaces. Update MAINTAINERS and the remaining mm/swap.c documentation references after the rename. Suggested-by: Baoquan He Suggested-by: David Hildenbrand Suggested-by: Matthew Wilcox Reviewed-by: Lorenzo Stoakes Acked-by: Johannes Weiner Acked-by: David Hildenbrand (Arm) Signed-off-by: Jianyue Wu --- Documentation/admin-guide/sysctl/vm.rst | 3 --- Documentation/core-api/mm-api.rst | 2 +- MAINTAINERS | 3 +-- mm/Makefile | 2 +- mm/{swap.c => folio.c} | 10 +++------- 5 files changed, 6 insertions(+), 14 deletions(-) diff --git a/Documentation/admin-guide/sysctl/vm.rst b/Documentation/admin-guide/sysctl/vm.rst index 22cc54cac3b2..5b318d17aa4b 100644 --- a/Documentation/admin-guide/sysctl/vm.rst +++ b/Documentation/admin-guide/sysctl/vm.rst @@ -19,9 +19,6 @@ The files in this directory can be used to tune the operation of the virtual memory (VM) subsystem of the Linux kernel and the writeout of dirty data to disk. -Default values and initialization routines for most of these -files can be found in mm/swap.c. - Currently, these files are in /proc/sys/vm: - admin_reserve_kbytes diff --git a/Documentation/core-api/mm-api.rst b/Documentation/core-api/mm-api.rst index 4df7d5edbee5..c1d03a5a2a19 100644 --- a/Documentation/core-api/mm-api.rst +++ b/Documentation/core-api/mm-api.rst @@ -118,7 +118,7 @@ More Memory Management Functions .. #kernel-doc:: mm/hmm.c (build warnings) .. kernel-doc:: mm/memremap.c .. kernel-doc:: mm/hugetlb.c -.. kernel-doc:: mm/swap.c +.. kernel-doc:: mm/folio.c .. kernel-doc:: mm/memcontrol.c .. #kernel-doc:: mm/memory-tiers.c (build warnings) .. kernel-doc:: mm/shmem.c diff --git a/MAINTAINERS b/MAINTAINERS index d19822d574fd..c3e535ca4bef 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -17085,7 +17085,7 @@ F: Documentation/admin-guide/mm/multigen_lru.rst F: Documentation/mm/multigen_lru.rst F: include/linux/mm_inline.h F: include/linux/mmzone.h -F: mm/swap.c +F: mm/folio.c F: mm/vmscan.c F: mm/workingset.c @@ -17237,7 +17237,6 @@ F: include/linux/swap.h F: include/linux/swapfile.h F: include/linux/swapops.h F: mm/page_io.c -F: mm/swap.c F: mm/swap.h F: mm/swap_table.h F: mm/swap_state.c diff --git a/mm/Makefile b/mm/Makefile index 4fc713867b9b..8e7e964b99a4 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -50,7 +50,7 @@ endif obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ maccess.o page-writeback.o folio-compat.o \ - readahead.o swap.o truncate.o vmscan.o shrinker.o \ + readahead.o folio.o truncate.o vmscan.o shrinker.o \ shmem.o util.o mmzone.o vmstat.o backing-dev.o \ mm_init.o percpu.o slab_common.o \ compaction.o show_mem.o \ diff --git a/mm/swap.c b/mm/folio.c similarity index 99% rename from mm/swap.c rename to mm/folio.c index c88643e56af7..d2937600cf72 100644 --- a/mm/swap.c +++ b/mm/folio.c @@ -1,17 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * linux/mm/swap.c + * linux/mm/folio.c * * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds */ /* - * This file contains the default values for the operation of the - * Linux VM subsystem. Fine-tuning documentation can be found in - * Documentation/admin-guide/sysctl/vm.rst. - * Started 18.12.91 - * Swap aging added 23.2.95, Stephen Tweedie. - * Buffermem limits added 12.3.98, Rik van Riel. + * Folio LRU helpers: add/remove folios from LRU lists, batching, + * activation/deactivation, and page cache release paths. */ #include -- 2.43.0 Keep include/linux/swap.h focused on swap-facing interfaces by moving MM-internal reclaim and workingset declarations into mm/internal.h. Leave the small set of LRU helper declarations that are used outside mm/ in swap.h so this cleanup does not need a new public header under include/linux/. Suggested-by: Barry Song Suggested-by: Baoquan He Acked-by: Johannes Weiner Signed-off-by: Jianyue Wu --- include/linux/swap.h | 75 +++++----------------------------------------------- mm/internal.h | 67 ++++++++++++++++++++++++++++++++++++++++++++++ mm/memfd.c | 1 + 3 files changed, 75 insertions(+), 68 deletions(-) diff --git a/include/linux/swap.h b/include/linux/swap.h index 45656ca9792d..696ed01709c2 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -287,39 +287,19 @@ static inline swp_entry_t page_swap_entry(struct page *page) return entry; } -/* linux/mm/workingset.c */ -bool workingset_test_recent(void *shadow, bool file, bool *workingset, - bool flush); -void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages); -void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg); -void workingset_refault(struct folio *folio, void *shadow); -void workingset_activation(struct folio *folio); - /* linux/mm/page_alloc.c */ extern unsigned long totalreserve_pages; /* Definition of global_zone_page_state not available yet */ #define nr_free_pages() global_zone_page_state(NR_FREE_PAGES) +/* linux/mm/folio.c */ +void folio_add_lru(struct folio *folio); +void folio_mark_accessed(struct folio *folio); +void lru_add_drain_all(void); -/* linux/mm/swap.c */ -void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file, - unsigned int nr_io, unsigned int nr_rotated); -void lru_note_cost_refault(struct folio *); -void folio_add_lru(struct folio *); -void folio_add_lru_vma(struct folio *, struct vm_area_struct *); -void mark_page_accessed(struct page *); -void folio_mark_accessed(struct folio *); - -static inline bool folio_may_be_lru_cached(struct folio *folio) -{ - /* - * Holding PMD-sized folios in per-CPU LRU cache unbalances accounting. - * Holding small numbers of low-order mTHP folios in per-CPU LRU cache - * will be sensible, but nobody has implemented and tested that yet. - */ - return !folio_test_large(folio); -} +/* linux/mm/folio-compat.c */ +void mark_page_accessed(struct page *page); extern atomic_t lru_disable_count; @@ -328,42 +308,6 @@ static inline bool lru_cache_disabled(void) return atomic_read(&lru_disable_count); } -static inline void lru_cache_enable(void) -{ - atomic_dec(&lru_disable_count); -} - -extern void lru_cache_disable(void); -extern void lru_add_drain(void); -extern void lru_add_drain_cpu(int cpu); -extern void lru_add_drain_cpu_zone(struct zone *zone); -extern void lru_add_drain_all(void); -void folio_deactivate(struct folio *folio); -void folio_mark_lazyfree(struct folio *folio); - -/* linux/mm/vmscan.c */ -extern unsigned long zone_reclaimable_pages(struct zone *zone); -extern unsigned long try_to_free_pages(struct zonelist *zonelist, int order, - gfp_t gfp_mask, const nodemask_t *mask); -unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx); - -#define MEMCG_RECLAIM_MAY_SWAP (1 << 1) -#define MEMCG_RECLAIM_PROACTIVE (1 << 2) -#define MIN_SWAPPINESS 0 -#define MAX_SWAPPINESS 200 - -/* Just reclaim from anon folios in proactive memory reclaim */ -#define SWAPPINESS_ANON_ONLY (MAX_SWAPPINESS + 1) - -extern unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, - unsigned long nr_pages, - gfp_t gfp_mask, - unsigned int reclaim_options, - int *swappiness); -extern unsigned long mem_cgroup_shrink_node(struct mem_cgroup *mem, - gfp_t gfp_mask, bool noswap, - pg_data_t *pgdat, - unsigned long *nr_scanned); extern unsigned long shrink_all_memory(unsigned long nr_pages); extern int vm_swappiness; long remove_mapping(struct address_space *mapping, struct folio *folio); @@ -384,11 +328,6 @@ static inline void reclaim_unregister_node(struct node *node) } #endif /* CONFIG_SYSFS && CONFIG_NUMA */ -#ifdef CONFIG_NUMA -extern int sysctl_min_unmapped_ratio; -extern int sysctl_min_slab_ratio; -#endif - void check_move_unevictable_folios(struct folio_batch *fbatch); extern void __meminit kswapd_run(int nid); @@ -544,7 +483,7 @@ static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg) void lru_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid); #else -static inline int mem_cgroup_swappiness(struct mem_cgroup *mem) +static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg) { return READ_ONCE(vm_swappiness); } diff --git a/mm/internal.h b/mm/internal.h index 841c27611627..6afbe6b7252d 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -31,6 +31,73 @@ struct huge_bootmem_page { unsigned long flags; }; +/* mm/workingset.c */ +bool workingset_test_recent(void *shadow, bool file, bool *workingset, + bool flush); +void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages); +void *workingset_eviction(struct folio *folio, + struct mem_cgroup *target_memcg); +void workingset_refault(struct folio *folio, void *shadow); +void workingset_activation(struct folio *folio); + +/* mm/folio.c */ +void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file, + unsigned int nr_io, unsigned int nr_rotated); +void lru_note_cost_refault(struct folio *folio); +void folio_add_lru_vma(struct folio *folio, struct vm_area_struct *vma); + +static inline bool folio_may_be_lru_cached(struct folio *folio) +{ + /* + * Holding PMD-sized folios in per-CPU LRU cache unbalances accounting. + * Holding small numbers of low-order mTHP folios in per-CPU LRU cache + * will be sensible, but nobody has implemented and tested that yet. + */ + return !folio_test_large(folio); +} + +static inline void lru_cache_enable(void) +{ + atomic_dec(&lru_disable_count); +} + +void lru_cache_disable(void); +void lru_add_drain(void); +void lru_add_drain_cpu(int cpu); +void lru_add_drain_cpu_zone(struct zone *zone); +void folio_deactivate(struct folio *folio); +void folio_mark_lazyfree(struct folio *folio); + +/* mm/vmscan.c */ +unsigned long zone_reclaimable_pages(struct zone *zone); +unsigned long try_to_free_pages(struct zonelist *zonelist, int order, + gfp_t gfp_mask, const nodemask_t *mask); +unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, + int zone_idx); + +#define MEMCG_RECLAIM_MAY_SWAP (1 << 1) +#define MEMCG_RECLAIM_PROACTIVE (1 << 2) +#define MIN_SWAPPINESS 0 +#define MAX_SWAPPINESS 200 + +/* Just reclaim from anon folios in proactive memory reclaim */ +#define SWAPPINESS_ANON_ONLY (MAX_SWAPPINESS + 1) + +unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, + unsigned long nr_pages, + gfp_t gfp_mask, + unsigned int reclaim_options, + int *swappiness); +unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, + gfp_t gfp_mask, bool noswap, + pg_data_t *pgdat, + unsigned long *nr_scanned); + +#ifdef CONFIG_NUMA +extern int sysctl_min_unmapped_ratio; +extern int sysctl_min_slab_ratio; +#endif + /* * Maintains state across a page table move. The operation assumes both source * and destination VMAs already exist and are specified by the user. diff --git a/mm/memfd.c b/mm/memfd.c index abe13b291ddc..6c72fe6caef7 100644 --- a/mm/memfd.c +++ b/mm/memfd.c @@ -19,6 +19,7 @@ #include #include #include +#include "internal.h" #include "swap.h" /* -- 2.43.0