From: Kairui Song There are many users that access the swap info array just to iterate through the swap devices to find usable ones. Introduce a generic helper for this to prepare for dropping the lock convention. Also slightly adjust __swap_type_to_info to allow access of swapped off devices by moving the sanity check into its only current caller, and this way makes more sense too: the only caller is __swap_entry_to_info, where we should never see an entry referring to a dead device. Signed-off-by: Kairui Song --- mm/swap.h | 12 +++++----- mm/swapfile.c | 77 ++++++++++++++++++++++++++++++++++++----------------------- 2 files changed, 53 insertions(+), 36 deletions(-) diff --git a/mm/swap.h b/mm/swap.h index b51ad3071a73..08a782f3d5ac 100644 --- a/mm/swap.h +++ b/mm/swap.h @@ -91,16 +91,16 @@ static inline unsigned int swp_cluster_offset(swp_entry_t entry) */ static inline struct swap_info_struct *__swap_type_to_info(int type) { - struct swap_info_struct *si; - - si = READ_ONCE(swap_info[type]); /* rcu_dereference() */ - VM_WARN_ON_ONCE(percpu_ref_is_zero(&si->users)); /* race with swapoff */ - return si; + return READ_ONCE(swap_info[type]); /* rcu_dereference() */ } static inline struct swap_info_struct *__swap_entry_to_info(swp_entry_t entry) { - return __swap_type_to_info(swp_type(entry)); + struct swap_info_struct *si; + + si = __swap_type_to_info(swp_type(entry)); /* rcu_dereference() */ + VM_WARN_ON_ONCE(percpu_ref_is_zero(&si->users)); /* race with swapoff */ + return si; } static inline struct swap_cluster_info *__swap_offset_to_cluster( diff --git a/mm/swapfile.c b/mm/swapfile.c index a4701692d330..8faa4e7c32bf 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -106,6 +106,34 @@ static DEFINE_SPINLOCK(swap_avail_lock); struct swap_info_struct *swap_info[MAX_SWAPFILES]; +static inline struct swap_info_struct *__swap_iter(int *i, unsigned long flag) +{ + lockdep_assert_held(&swap_lock); + while (*i < nr_swapfiles) { + struct swap_info_struct *si = __swap_type_to_info(*i); + + VM_WARN_ON(!si); + (*i)++; + if (flag && !((si->flags & flag) == flag)) + continue; + return si; + } + return NULL; +} + +#define __for_each_swap(si, flag) \ + for (int __i = 0; ((si) = __swap_iter(&__i, flag));) + +/* + * for_each_swap - iterate through all allocated and inuse swap devices + * @si: the iterator + * + * Context: The caller must hold swap_lock. The lock may be dropped during + * the loop body but must be re-acquired before the next iteration. + */ +#define for_each_swap(si) __for_each_swap(si, SWP_USED) +#define for_each_avail_swap(si) __for_each_swap(si, SWP_USED | SWP_WRITEOK) + static struct kmem_cache *swap_table_cachep; /* Protects si->swap_file for /proc/swaps usage */ @@ -2197,26 +2225,18 @@ void swap_free_hibernation_slot(swp_entry_t entry) static int __find_hibernation_swap_type(dev_t device, sector_t offset) { - int type; - - lockdep_assert_held(&swap_lock); + struct swap_info_struct *si; if (!device) return -EINVAL; - for (type = 0; type < nr_swapfiles; type++) { - struct swap_info_struct *sis = swap_info[type]; - - if (!(sis->flags & SWP_WRITEOK)) - continue; - - if (device == sis->bdev->bd_dev) { - struct swap_extent *se = first_se(sis); - - if (se->start_block == offset) - return type; + for_each_avail_swap(si) { + if (device == si->bdev->bd_dev) { + if (first_se(si)->start_block == offset) + return si->type; } } + return -ENODEV; } @@ -2322,16 +2342,13 @@ int find_hibernation_swap_type(dev_t device, sector_t offset) int find_first_swap(dev_t *device) { - int type, ret = -ENODEV; + int ret = -ENODEV; + struct swap_info_struct *si; spin_lock(&swap_lock); - for (type = 0; type < nr_swapfiles; type++) { - struct swap_info_struct *sis = swap_info[type]; - - if (!(sis->flags & SWP_WRITEOK)) - continue; - *device = sis->bdev->bd_dev; - ret = type; + for_each_avail_swap(si) { + *device = si->bdev->bd_dev; + ret = si->type; break; } spin_unlock(&swap_lock); @@ -2817,12 +2834,14 @@ static int try_to_unuse(unsigned int type) */ static void drain_mmlist(void) { + struct swap_info_struct *si; struct list_head *p, *next; - unsigned int type; - for (type = 0; type < nr_swapfiles; type++) - if (swap_usage_in_pages(swap_info[type])) + for_each_swap(si) { + if (swap_usage_in_pages(si)) return; + } + spin_lock(&mmlist_lock); list_for_each_safe(p, next, &init_mm.mmlist) list_del_init(p); @@ -3802,14 +3821,12 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) void si_swapinfo(struct sysinfo *val) { - unsigned int type; + struct swap_info_struct *si; unsigned long nr_to_be_unused = 0; spin_lock(&swap_lock); - for (type = 0; type < nr_swapfiles; type++) { - struct swap_info_struct *si = swap_info[type]; - - if ((si->flags & SWP_USED) && !(si->flags & SWP_WRITEOK)) + for_each_swap(si) { + if (!(si->flags & SWP_WRITEOK)) nr_to_be_unused += swap_usage_in_pages(si); } val->freeswap = atomic_long_read(&nr_swap_pages) + nr_to_be_unused; -- 2.55.0