From: Kairui Song Remove swap_active_head and the associated plist_node. Device identity lookups can use the writable swap_info iterator without ordering. Keep priority ordering for the block-cgroup swap throttle target by walking the priority queue under the topology reader. Choose the first unmasked, writable, non-offlist device with a backing block device. Check WRITEOK and OFFLIST as well as the pointer tag because disable and full transitions can temporarily leave an unmasked entry. This read-only lookup does not consume per-CPU allocation quotas or change retry cursors. It avoids making the throttle disk depend on swapon's swap-type assignment order when devices have different priorities. Signed-off-by: Kairui Song Signed-off-by: Lian Wang --- include/linux/swap.h | 1 - mm/swapfile.c | 59 ++++++++++++++++++++++++-------------------- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/include/linux/swap.h b/include/linux/swap.h index 17b8ec654ee2..bfca46e1fbc6 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -261,7 +261,6 @@ struct swap_info_struct { struct percpu_ref users; /* indicate and keep swap device valid. */ unsigned long flags; /* SWP_USED etc: see above */ signed short prio; /* swap priority of this type */ - struct plist_node list; /* entry in swap_active_head */ signed char type; /* strange name for an index */ unsigned int max; /* size of this swap device */ struct swap_cluster_info *cluster_info; /* array, one entry per cluster */ diff --git a/mm/swapfile.c b/mm/swapfile.c index 5c26682c0bab..7b3520ce1e41 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -41,7 +41,6 @@ #include #include #include -#include #include #include @@ -90,12 +89,6 @@ bool swap_migration_ad_supported; static const char Bad_file[] = "Bad swap file entry "; static const char Bad_offset[] = "Bad swap offset entry "; -/* - * all active swap_info_structs - * protected with swapon_rwsem, and ordered by priority. - */ -static PLIST_HEAD(swap_active_head); - static inline struct swap_info_struct *__swap_iter(int *i, unsigned long flag) { lockdep_assert_held(&swapon_rwsem); @@ -3423,7 +3416,6 @@ static void __swap_device_enable(struct swap_info_struct *si) spin_unlock(&swap_queue_update_lock); atomic_long_add(si->pages, &nr_swap_pages); total_swap_pages += si->pages; - plist_add(&si->list, &swap_active_head); add_to_avail_list(si); } @@ -3443,9 +3435,8 @@ static int swap_device_disable(struct address_space *mapping, int err = -EINVAL; percpu_down_write(&swapon_rwsem); - plist_for_each_entry(si, &swap_active_head, list) { - if ((si->flags & SWP_WRITEOK) && - si->swap_file->f_mapping == mapping) { + for_each_avail_swap(si) { + if (si->swap_file->f_mapping == mapping) { err = 0; break; } @@ -3470,7 +3461,6 @@ static int swap_device_disable(struct address_space *mapping, spin_lock(&swap_queue_update_lock); si->flags &= ~SWP_WRITEOK; spin_unlock(&swap_queue_update_lock); - plist_del(&si->list, &swap_active_head); total_swap_pages -= si->pages; atomic_long_sub(si->pages, &nr_swap_pages); @@ -3801,7 +3791,6 @@ static struct swap_info_struct *alloc_swap_info(void) */ } p->swap_extent_root = RB_ROOT; - plist_node_init(&p->list, 0); p->flags = SWP_USED; percpu_up_write(&swapon_rwsem); if (defer) { @@ -4211,12 +4200,7 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) if (swap_flags & SWAP_FLAG_PREFER) prio = swap_flags & SWAP_FLAG_PRIO_MASK; - /* - * The plist prio is negated because plist ordering is - * low-to-high, while swap ordering is high-to-low - */ si->prio = prio; - si->list.prio = -si->prio; /* * Publish swap_file before making the percpu ref live, then add the device @@ -4342,7 +4326,33 @@ int swap_dup_entry_direct(swp_entry_t entry) #if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP) static bool __has_usable_swap(void) { - return !plist_head_empty(&swap_active_head); + return READ_ONCE(total_swap_pages) > 0; +} + +static struct swap_info_struct *swap_queue_first_usable_bdev(void) +{ + struct swap_prio_ring *ring; + struct swap_info_struct *si; + unsigned int ring_idx, dev_idx; + + lockdep_assert_held(&swapon_rwsem); + for (ring_idx = 0; ring_idx < swap_queue_len; ring_idx++) { + ring = swap_queue[ring_idx]; + for (dev_idx = 0; dev_idx < ring->size; dev_idx++) { + si = READ_ONCE(ring->dev[dev_idx]); + if (swap_device_masked(si)) + continue; + si = swap_device_unmask_ptr(si); + if (!(READ_ONCE(si->flags) & SWP_WRITEOK)) + continue; + if (atomic_long_read(&si->inuse_pages) & + SWAP_USAGE_OFFLIST_BIT) + continue; + if (si->bdev) + return si; + } + } + return NULL; } void __folio_throttle_swaprate(struct folio *folio, gfp_t gfp) @@ -4366,14 +4376,9 @@ void __folio_throttle_swaprate(struct folio *folio, gfp_t gfp) return; percpu_down_read(&swapon_rwsem); - plist_for_each_entry(si, &swap_active_head, list) { - if ((si->flags & SWP_WRITEOK) && - !(atomic_long_read(&si->inuse_pages) & - SWAP_USAGE_OFFLIST_BIT) && si->bdev) { - blkcg_schedule_throttle(si->bdev->bd_disk, true); - break; - } - } + si = swap_queue_first_usable_bdev(); + if (si) + blkcg_schedule_throttle(si->bdev->bd_disk, true); percpu_up_read(&swapon_rwsem); } #endif