From: Baoquan He Grow the mapped xswap range before it runs out. When usage reaches 85% of the mapped range, map more clusters from cluster_alloc_swap_entry(). Grow to about 65% usage. This leaves free space in the mapped range before the next grow at 85%, so one grow is not immediately followed by another. Mapping pages can sleep. Grow before the allocation that needs them, not after it fails. xswap_grow() drops the per-CPU cluster lock around xswap_map_clusters(), because mapping can sleep and the lock only protects the per-CPU cache. xswap is always SWP_SOLIDSTATE, so this path does not use global_cluster_lock. Link the new clusters even if swapoff cleared SWP_WRITEOK while mapping. The caller keeps the device alive, and the allocation path checks the flag before taking a cluster. If try_to_unuse() then fails, the reinserted device needs these clusters; otherwise they go away with the device. After a failed scan, retry the free list once. A concurrent free or grow may have added clusters. Signed-off-by: Baoquan He --- mm/swapfile.c | 100 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/mm/swapfile.c b/mm/swapfile.c index 7b1d16226b7b..ba730fa19194 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -80,7 +80,18 @@ max_t(unsigned long, \ ((XSWAP_UNMAP_BATCH_PAGES * PAGE_SIZE) / \ sizeof(struct swap_cluster_info)), 16) +/* + * Grow thresholds, as a percentage of the mapped range in use. The target + * sits above the point where the shrink starts looking (50%) and below the + * next grow trigger (85%), so a grow cannot leave the range in a state the + * shrink would undo. + */ +#define XSWAP_GROW_UNTIL 65 +#define XSWAP_GROW_WHEN 85 +#define XSWAP_GROW_CHUNKS 4 +static bool xswap_should_grow(struct swap_info_struct *si); +static unsigned long xswap_grow(struct swap_info_struct *si); static int xswap_map_clusters(struct swap_info_struct *si, unsigned long start_idx, unsigned long nr); static void xswap_unmap_clusters(struct swap_info_struct *si, @@ -1359,6 +1370,18 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si, if (order && !(si->flags & SWP_BLKDEV)) return 0; +#ifdef CONFIG_XSWAP + /* + * Top the range up early. Every path below leaves through `done`, + * so this has to come first; mapping pages can sleep, and doing it + * from the allocation that would otherwise fail leaves nothing to + * fall back on. + */ + if ((si->flags & SWP_XSWAP) && (si->flags & SWP_WRITEOK) && + xswap_should_grow(si)) + xswap_grow(si); +#endif + if (!(si->flags & SWP_SOLIDSTATE)) { /* Serialize HDD SWAP allocation for each device. */ spin_lock(&si->global_cluster_lock); @@ -1435,6 +1458,12 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si, if (found) goto done; } + +#ifdef CONFIG_XSWAP + /* A concurrent free or grow may have added clusters; retry once. */ + if (!found && (si->flags & SWP_XSWAP)) + found = alloc_swap_scan_list(si, &si->free_clusters, folio, false); +#endif done: if (!(si->flags & SWP_SOLIDSTATE)) spin_unlock(&si->global_cluster_lock); @@ -4168,6 +4197,77 @@ static void xswap_unmap_range(struct swap_info_struct *si, __free_page(pages[i]); } +static bool xswap_should_grow(struct swap_info_struct *si) +{ + unsigned long mapped = READ_ONCE(si->nr_clusters_mapped); + + if (mapped >= READ_ONCE(si->nr_clusters_max)) + return false; + + return swap_usage_in_pages(si) * 100 >= + mapped * SWAPFILE_CLUSTER * XSWAP_GROW_WHEN; +} + +/* + * Map more of the cluster_info array, up to GROW_UNTIL in use. The caller + * holds percpu_swap_cluster.lock, which this drops while it sleeps. + */ +static unsigned long xswap_grow(struct swap_info_struct *si) +{ + unsigned long mapped = READ_ONCE(si->nr_clusters_mapped); + unsigned long nr_new, want, i; + int ret; + + /* The unmap below is paired with the caller's lock. */ + lockdep_assert_held(&this_cpu_ptr(&percpu_swap_cluster)->lock); + + want = DIV_ROUND_UP(swap_usage_in_pages(si) * 100, + XSWAP_GROW_UNTIL * SWAPFILE_CLUSTER); + if (want <= mapped) + return 0; + + nr_new = rounddown(want - mapped, XSWAP_GROW_CLUSTERS); + if (!nr_new) + nr_new = XSWAP_GROW_CLUSTERS; + if (nr_new > XSWAP_GROW_CHUNKS * XSWAP_GROW_CLUSTERS) + nr_new = XSWAP_GROW_CHUNKS * XSWAP_GROW_CLUSTERS; + nr_new = min(nr_new, READ_ONCE(si->nr_clusters_max) - mapped); + if (!nr_new) + return 0; + + /* + * Mapping pages can sleep. The lock only guards the per-cpu + * cluster cache, which this path does not touch. + */ + local_unlock(&percpu_swap_cluster.lock); + ret = xswap_map_clusters(si, mapped, nr_new); + local_lock(&percpu_swap_cluster.lock); + if (ret) + return 0; + + /* + * Link the clusters even if swapoff cleared SWP_WRITEOK while we + * mapped. The caller keeps the device alive, and nothing takes a + * cluster while WRITEOK is clear. If try_to_unuse() fails, the + * reinserted device needs them. + */ + for (i = mapped; i < mapped + nr_new; i++) { + struct swap_cluster_info *ci = &si->cluster_info[i]; + + /* + * A concurrent grower may have taken these already; + * only add the off-list ones. + */ + spin_lock(&ci->lock); + if (ci->flags == CLUSTER_FLAG_NONE) + move_cluster(si, ci, &si->free_clusters, + CLUSTER_FLAG_FREE); + spin_unlock(&ci->lock); + } + + return nr_new; +} + /* Caller must hold si->xswap_lock. Cannot fail. */ static void xswap_unmap_clusters(struct swap_info_struct *si, unsigned long start_idx, unsigned long nr) -- 2.54.0