From: Baoquan He An xswap cluster_info page can be unmapped at runtime. The shrink path that does this is added by a later patch; this patch establishes the contract it relies on. The allocation and reclaim paths drop ci->lock to reclaim swap cache, then take it again; in that window the cluster can be freed and its cluster_info page can go away. Cover the whole read/drop/reacquire window with RCU, so the cluster pointer stays valid until the lock is taken again. Also take RCU in swap_alloc_fast() and the hibernation slot allocation path, which read cluster_info through a cached offset without holding ci->lock. A cached offset can outlive a shrink that did not see it, so swap_alloc_fast() now loads nr_clusters_mapped with smp_load_acquire() and fails the fast path when the offset is beyond it. On the writer side, xswap_unmap_clusters() publishes the smaller range with smp_store_release() before the memory goes away, invalidates the per-cpu cluster cache, and waits one RCU grace period. A reader that already loaded a cached offset is then either seen by the cache flush or waited out before the PTEs are cleared. Signed-off-by: Baoquan He --- mm/swapfile.c | 82 +++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 73 insertions(+), 9 deletions(-) diff --git a/mm/swapfile.c b/mm/swapfile.c index 2624233b4fc2..bc30ab42203c 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1239,6 +1239,9 @@ static unsigned long alloc_swap_scan_cluster(struct swap_info_struct *si, lockdep_assert_held(&ci->lock); VM_WARN_ON(!cluster_is_usable(ci, order)); + /* ci is used without ci->lock; an xswap unmap waits for this. */ + rcu_read_lock(); + if (end < nr_pages || ci->count + nr_pages > SWAPFILE_CLUSTER) goto out; @@ -1267,6 +1270,7 @@ static unsigned long alloc_swap_scan_cluster(struct swap_info_struct *si, out: relocate_cluster(si, ci); swap_cluster_unlock(ci); + rcu_read_unlock(); if (si->flags & SWP_SOLIDSTATE) { this_cpu_write(percpu_swap_cluster.offset[order], next); this_cpu_write(percpu_swap_cluster.si[order], si); @@ -1310,6 +1314,9 @@ static void swap_reclaim_full_clusters(struct swap_info_struct *si, bool force) to_scan = swap_usage_in_pages(si) / SWAPFILE_CLUSTER; while ((ci = isolate_lock_cluster(si, &si->full_clusters))) { + /* As in alloc_swap_scan_cluster(). */ + rcu_read_lock(); + offset = cluster_offset(si, ci); end = min(si->max, offset + SWAPFILE_CLUSTER); to_scan--; @@ -1334,6 +1341,7 @@ static void swap_reclaim_full_clusters(struct swap_info_struct *si, bool force) relocate_cluster(si, ci); swap_cluster_unlock(ci); + rcu_read_unlock(); if (to_scan <= 0) break; @@ -1664,11 +1672,36 @@ static bool swap_alloc_fast(struct folio *folio) /* * Once allocated, swap_info_struct will never be completely freed, * so checking it's liveness by get_swap_device_info is enough. + * + * The cached offset indexes si->cluster_info, which xswap can + * unmap; cover both the read and the use with RCU. */ + rcu_read_lock(); si = this_cpu_read(percpu_swap_cluster.si[order]); offset = this_cpu_read(percpu_swap_cluster.offset[order]); - if (!si || !offset || !get_swap_device_info(si)) + if (!si || !offset || !get_swap_device_info(si)) { + rcu_read_unlock(); return false; + } +#ifdef CONFIG_XSWAP + if (si->flags & SWP_XSWAP) { + /* + * The cached offset can outlive a shrink that did not see + * it, and the cluster_info page it indexes may be unmapped. + * Fail the fast path; the slow path picks a live cluster. + */ + unsigned long mapped; + + /* Pairs with the release store in xswap_unmap_clusters(). */ + mapped = smp_load_acquire(&si->nr_clusters_mapped); + + if (offset / SWAPFILE_CLUSTER >= mapped) { + put_swap_device(si); + rcu_read_unlock(); + return false; + } + } +#endif ci = swap_cluster_lock(si, offset); if (cluster_is_usable(ci, order)) { @@ -1680,6 +1713,7 @@ static bool swap_alloc_fast(struct folio *folio) } put_swap_device(si); + rcu_read_unlock(); return folio_test_swapcache(folio); } @@ -2472,8 +2506,11 @@ swp_entry_t swap_alloc_hibernation_slot(int type) /* * Try the local cluster first if it matches the device. If * not, try grab a new cluster and override local cluster. + * + * Same RCU requirement as swap_alloc_fast(). */ local_lock(&percpu_swap_cluster.lock); + rcu_read_lock(); pcp_si = this_cpu_read(percpu_swap_cluster.si[0]); pcp_offset = this_cpu_read(percpu_swap_cluster.offset[0]); if (pcp_si == si && pcp_offset) { @@ -2483,6 +2520,7 @@ swp_entry_t swap_alloc_hibernation_slot(int type) else swap_cluster_unlock(ci); } + rcu_read_unlock(); if (!offset) offset = cluster_alloc_swap_entry(si, NULL); local_unlock(&percpu_swap_cluster.lock); @@ -3445,7 +3483,13 @@ static void wait_for_allocation(struct swap_info_struct *si) #ifdef CONFIG_XSWAP if (si->flags & SWP_XSWAP) { - /* Pairs with the smp_store_release() in xswap_map_clusters(). */ + /* + * Pairs with the smp_store_release() in xswap_map_clusters(). + * A grow may publish after this read, but no slot comes from + * those clusters once WRITEOK is clear. Cached clusters are + * inside the range; try_to_unuse()'s rescan catches any slot + * taken after the walk. + */ end = min(end, smp_load_acquire(&si->nr_clusters_mapped) * SWAPFILE_CLUSTER); } @@ -3527,9 +3571,9 @@ static void flush_percpu_swap_cluster(struct swap_info_struct *si) for_each_possible_cpu(cpu) { pcp_si = per_cpu_ptr(percpu_swap_cluster.si, cpu); /* - * Invalidate the percpu swap cluster cache, si->users - * is dead, so no new user will point to it, just flush - * any existing user. + * Invalidate the per-cpu swap cluster cache. Any reader that + * has already loaded an entry is waited out by the caller's + * synchronize_rcu(). */ for (i = 0; i < SWAP_NR_ORDERS; i++) cmpxchg(&pcp_si[i], si, NULL); @@ -4289,12 +4333,33 @@ static void xswap_unmap_clusters(struct swap_info_struct *si, mutex_lock(&si->xswap_lock); + /* + * Publish the smaller range before its memory goes away. Readers + * that use nr_clusters_mapped without xswap_lock pair with this + * store using smp_load_acquire(). + */ + smp_store_release(&si->nr_clusters_mapped, start_idx); + if (vm_start >= vm_end) { - WRITE_ONCE(si->nr_clusters_mapped, start_idx); + /* + * The clusters sit inside one page that the preceding + * cluster shares, so there is nothing to unmap. The page + * goes away with that cluster; publishing the smaller + * cluster count is still correct. + */ mutex_unlock(&si->xswap_lock); return; } + /* + * A per-cpu cluster cache can still hold an offset in this range. + * Invalidate those references, then wait out the readers that have + * already loaded one, so that nobody can dereference cluster_info + * past this point. swapoff() needs the same before it releases. + */ + flush_percpu_swap_cluster(si); + synchronize_rcu(); + npages = (vm_end - vm_start) >> PAGE_SHIFT; if (npages <= totalram_pages()) { @@ -4306,7 +4371,7 @@ static void xswap_unmap_clusters(struct swap_info_struct *si, if (pages) { xswap_unmap_range(si, vm_start, vm_end, pages, npages); kvfree(pages); - goto out_store; + goto out_unlock; } } @@ -4325,8 +4390,7 @@ static void xswap_unmap_clusters(struct swap_info_struct *si, xswap_unmap_range(si, bstart, bend, batch_pages, ARRAY_SIZE(batch_pages)); } -out_store: - WRITE_ONCE(si->nr_clusters_mapped, start_idx); +out_unlock: mutex_unlock(&si->xswap_lock); } -- 2.54.0