When zswap refuses a folio, the folio goes to a physical swap device. A small folio takes one slot there. A large folio is different as it must sit in a contiguous run of slots. This is because swap_read_folio() reads it back with one IO that starts at its first physical slot. So the whole run has to be reserved at once. The problem is that the reservation path has no folio, so it only knows order 0. Pass the order down through cluster_alloc_swap_entry(), and let it fill a range of shadow entries. The xswap device must accept large orders too. Otherwise a large anon folio cannot be swapped out to it at all. Signed-off-by: Baoquan He --- mm/page_io.c | 2 +- mm/swap.h | 8 +-- mm/swapfile.c | 164 +++++++++++++++++++++++++++----------------------- mm/zswap.c | 4 +- 4 files changed, 95 insertions(+), 83 deletions(-) diff --git a/mm/page_io.c b/mm/page_io.c index d521ea335df1..dda68d7e1d94 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -257,7 +257,7 @@ int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio) swp_entry_t phys; /* The pool refused it: fall back to a real swap device. */ - phys = xswap_backend_alloc(entry); + phys = xswap_backend_alloc(entry, folio_order(folio)); if (!phys.val) { folio_mark_dirty(folio); return AOP_WRITEPAGE_ACTIVATE; diff --git a/mm/swap.h b/mm/swap.h index 30b67915c459..4fcbb21a583d 100644 --- a/mm/swap.h +++ b/mm/swap.h @@ -475,8 +475,8 @@ int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio, #ifdef CONFIG_XSWAP swp_entry_t xswap_slot_backend(struct swap_cluster_info *ci, unsigned int slot); -swp_entry_t xswap_backend_alloc(swp_entry_t entry); -void xswap_backend_free(swp_entry_t entry, swp_entry_t phys); +swp_entry_t xswap_backend_alloc(swp_entry_t entry, unsigned int order); +void xswap_backend_free(swp_entry_t entry, swp_entry_t phys, unsigned int order); #else static inline swp_entry_t xswap_slot_backend(struct swap_cluster_info *ci, unsigned int slot) @@ -484,12 +484,12 @@ static inline swp_entry_t xswap_slot_backend(struct swap_cluster_info *ci, return (swp_entry_t){}; } -static inline swp_entry_t xswap_backend_alloc(swp_entry_t entry) +static inline swp_entry_t xswap_backend_alloc(swp_entry_t entry, unsigned int order) { return (swp_entry_t){}; } -static inline void xswap_backend_free(swp_entry_t entry, swp_entry_t phys) +static inline void xswap_backend_free(swp_entry_t entry, swp_entry_t phys, unsigned int order) { } #endif /* CONFIG_XSWAP */ diff --git a/mm/swapfile.c b/mm/swapfile.c index 875b40f2e362..4d72c854c7b3 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1063,10 +1063,10 @@ static bool cluster_scan_range(struct swap_info_struct *si, static bool __swap_cluster_alloc_entries(struct swap_info_struct *si, struct swap_cluster_info *ci, struct folio *folio, - unsigned int ci_off) + unsigned int ci_off, unsigned int order) { - unsigned int order; - unsigned long nr_pages; + unsigned long nr_pages = 1UL << order; + unsigned long i; lockdep_assert_held(&ci->lock); @@ -1075,33 +1075,24 @@ static bool __swap_cluster_alloc_entries(struct swap_info_struct *si, /* * All mm swap allocation starts with a folio (folio_alloc_swap), - * it's also the only allocation path for large orders allocation. - * Such swap slots starts with count == 0 and will be increased - * upon folio unmap. + * it's also the allocation path for large orders. Such swap slots + * starts with count == 0 and will be increased upon folio unmap. * - * Else, it's an exclusive order 0 allocation for a slot that has - * no folio of its own: hibernation, and the physical slot that - * backs an xswap entry. The slot starts with count == 1 and - * never increases. + * Else, it's an exclusive allocation for a range of slots that has no + * folio of its own: hibernation, and the range that backs an xswap + * folio. Each slot starts with count == 1 and never increases. */ + swap_cluster_assert_empty(ci, ci_off, nr_pages, false); if (likely(folio)) { - order = folio_order(folio); - nr_pages = 1 << order; - swap_cluster_assert_empty(ci, ci_off, nr_pages, false); __swap_cache_add_folio(ci, folio, swp_entry(si->type, ci_off + cluster_offset(si, ci))); } else { - order = 0; - nr_pages = 1; - swap_cluster_assert_empty(ci, ci_off, 1, false); - /* Fake shadow placeholder with no flag; no zeromap is used. */ - __swap_table_set(ci, ci_off, __swp_tb_mk_count(shadow_to_swp_tb(NULL, 0), 1)); + /* Fake shadow placeholders with no flag; no zeromap is used. */ + for (i = 0; i < nr_pages; i++) + __swap_table_set(ci, ci_off + i, + __swp_tb_mk_count(shadow_to_swp_tb(NULL, 0), 1)); } - /* - * The first allocation in a cluster makes the - * cluster exclusive to this order - */ if (cluster_is_empty(ci)) ci->order = order; ci->count += nr_pages; @@ -1109,17 +1100,15 @@ static bool __swap_cluster_alloc_entries(struct swap_info_struct *si, return true; } - -/* Try use a new cluster for current CPU and allocate from it. */ static unsigned int alloc_swap_scan_cluster(struct swap_info_struct *si, struct swap_cluster_info *ci, - struct folio *folio, unsigned long offset) + struct folio *folio, unsigned long offset, + unsigned int order) { unsigned int next = SWAP_ENTRY_INVALID, found = SWAP_ENTRY_INVALID; unsigned long start = ALIGN_DOWN(offset, SWAPFILE_CLUSTER); - unsigned int order = likely(folio) ? folio_order(folio) : 0; unsigned long end = start + SWAPFILE_CLUSTER; - unsigned int nr_pages = 1 << order; + unsigned long nr_pages = 1UL << order; bool need_reclaim, ret, usable; lockdep_assert_held(&ci->lock); @@ -1145,7 +1134,7 @@ static unsigned int alloc_swap_scan_cluster(struct swap_info_struct *si, if (!ret) continue; } - if (!__swap_cluster_alloc_entries(si, ci, folio, offset % SWAPFILE_CLUSTER)) + if (!__swap_cluster_alloc_entries(si, ci, folio, offset % SWAPFILE_CLUSTER, order)) break; found = offset; offset += nr_pages; @@ -1176,7 +1165,7 @@ static unsigned int alloc_swap_scan_cluster(struct swap_info_struct *si, static unsigned int alloc_swap_scan_list(struct swap_info_struct *si, struct list_head *list, struct folio *folio, - bool scan_all) + unsigned int order, bool scan_all) { unsigned int found = SWAP_ENTRY_INVALID; @@ -1187,7 +1176,7 @@ static unsigned int alloc_swap_scan_list(struct swap_info_struct *si, if (!ci) break; offset = cluster_offset(si, ci); - found = alloc_swap_scan_cluster(si, ci, folio, offset); + found = alloc_swap_scan_cluster(si, ci, folio, offset, order); if (found) break; } while (scan_all); @@ -1279,17 +1268,16 @@ static void swap_reclaim_work(struct work_struct *work) * cluster for current CPU too. */ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si, - struct folio *folio) + struct folio *folio, unsigned int order) { struct swap_cluster_info *ci; - unsigned int order = likely(folio) ? folio_order(folio) : 0; unsigned int offset = SWAP_ENTRY_INVALID, found = SWAP_ENTRY_INVALID; /* * Swapfile is not block device so unable - * to allocate large entries. + * to allocate large entries. xswap has no backing file, so it can. */ - if (order && !(si->flags & SWP_BLKDEV)) + if (order && !(si->flags & SWP_BLKDEV) && !(si->flags & SWP_XSWAP)) return 0; if (!(si->flags & SWP_SOLIDSTATE)) { @@ -1304,7 +1292,7 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si, if (cluster_is_usable(ci, order)) { if (cluster_is_empty(ci)) offset = cluster_offset(si, ci); - found = alloc_swap_scan_cluster(si, ci, folio, offset); + found = alloc_swap_scan_cluster(si, ci, folio, offset, order); } else { swap_cluster_unlock(ci); } @@ -1318,7 +1306,7 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si, * cluster is the only thing a large order can allocate from. */ if (!folio && order < PMD_ORDER) { - found = alloc_swap_scan_list(si, &si->frag_clusters[order], folio, false); + found = alloc_swap_scan_list(si, &si->frag_clusters[order], folio, order, false); if (found) goto done; } @@ -1328,19 +1316,19 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si, * to spread out the writes. */ if (si->flags & SWP_PAGE_DISCARD) { - found = alloc_swap_scan_list(si, &si->free_clusters, folio, false); + found = alloc_swap_scan_list(si, &si->free_clusters, folio, order, false); if (found) goto done; } if (order < PMD_ORDER) { - found = alloc_swap_scan_list(si, &si->nonfull_clusters[order], folio, true); + found = alloc_swap_scan_list(si, &si->nonfull_clusters[order], folio, order, true); if (found) goto done; } if (!(si->flags & SWP_PAGE_DISCARD)) { - found = alloc_swap_scan_list(si, &si->free_clusters, folio, false); + found = alloc_swap_scan_list(si, &si->free_clusters, folio, order, false); if (found) goto done; } @@ -1356,7 +1344,7 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si, * failure is not critical. Scanning one cluster still * keeps the list rotated and reclaimed (for clean swap cache). */ - found = alloc_swap_scan_list(si, &si->frag_clusters[order], folio, false); + found = alloc_swap_scan_list(si, &si->frag_clusters[order], folio, order, false); if (found) goto done; } @@ -1370,11 +1358,11 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si, * Clusters here have at least one usable slots and can't fail order 0 * allocation, but reclaim may drop si->lock and race with another user. */ - found = alloc_swap_scan_list(si, &si->frag_clusters[o], folio, true); + found = alloc_swap_scan_list(si, &si->frag_clusters[o], folio, order, true); if (found) goto done; - found = alloc_swap_scan_list(si, &si->nonfull_clusters[o], folio, true); + found = alloc_swap_scan_list(si, &si->nonfull_clusters[o], folio, order, true); if (found) goto done; } @@ -1417,7 +1405,7 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si, } found = alloc_swap_scan_list(si, &si->free_clusters, - folio, false); + folio, order, false); } } #endif @@ -1630,7 +1618,7 @@ static bool swap_alloc_fast(struct folio *folio) if (cluster_is_usable(ci, order)) { if (cluster_is_empty(ci)) offset = cluster_offset(si, ci); - alloc_swap_scan_cluster(si, ci, folio, offset); + alloc_swap_scan_cluster(si, ci, folio, offset, order); } else { swap_cluster_unlock(ci); } @@ -1652,7 +1640,7 @@ static void swap_alloc_slow(struct folio *folio) plist_requeue(&si->avail_list, &swap_avail_head); spin_unlock(&swap_avail_lock); if (get_swap_device_info(si)) { - cluster_alloc_swap_entry(si, folio); + cluster_alloc_swap_entry(si, folio, folio_order(folio)); put_swap_device(si); if (folio_test_swapcache(folio)) return; @@ -2443,13 +2431,13 @@ swp_entry_t swap_alloc_hibernation_slot(int type) if (pcp_si == si && pcp_offset) { ci = swap_cluster_lock(si, pcp_offset); if (cluster_is_usable(ci, 0)) - offset = alloc_swap_scan_cluster(si, ci, NULL, pcp_offset); + offset = alloc_swap_scan_cluster(si, ci, NULL, pcp_offset, 0); else swap_cluster_unlock(ci); } rcu_read_unlock(); if (!offset) - offset = cluster_alloc_swap_entry(si, NULL); + offset = cluster_alloc_swap_entry(si, NULL, 0); local_unlock(&percpu_swap_cluster.lock); if (offset) entry = swp_entry(si->type, offset); @@ -4241,32 +4229,33 @@ static void xswap_free_xs_table(struct swap_info_struct *si, unsigned long idx) ci->xs_table = NULL; } -/* Order-0 slot from @si, no folio attached; goes through the per-CPU cache. */ -static unsigned long xswap_alloc_slot_local(struct swap_info_struct *si) +/* A range of 1 << @order slots, no folio attached. */ +static unsigned long xswap_alloc_slot_local(struct swap_info_struct *si, + unsigned int order) { struct swap_cluster_info *ci; unsigned long pcp_offset, offset = SWAP_ENTRY_INVALID; local_lock(&percpu_swap_cluster.lock); - if (this_cpu_read(percpu_swap_cluster.si[0]) == si) { - pcp_offset = this_cpu_read(percpu_swap_cluster.offset[0]); + if (this_cpu_read(percpu_swap_cluster.si[order]) == si) { + pcp_offset = this_cpu_read(percpu_swap_cluster.offset[order]); if (pcp_offset) { ci = swap_cluster_lock(si, pcp_offset); - if (cluster_is_usable(ci, 0)) + if (cluster_is_usable(ci, order)) offset = alloc_swap_scan_cluster(si, ci, NULL, - pcp_offset); + pcp_offset, order); else swap_cluster_unlock(ci); } } if (offset == SWAP_ENTRY_INVALID) - offset = cluster_alloc_swap_entry(si, NULL); + offset = cluster_alloc_swap_entry(si, NULL, order); local_unlock(&percpu_swap_cluster.lock); return offset; } -static swp_entry_t xswap_alloc_phys_slot(void) +static swp_entry_t xswap_alloc_phys_slot(unsigned int order) { struct swap_info_struct *devs[MAX_SWAPFILES], *si; int nr = 0, i; @@ -4291,7 +4280,7 @@ static swp_entry_t xswap_alloc_phys_slot(void) if (!get_swap_device_info(devs[i])) continue; - offset = xswap_alloc_slot_local(devs[i]); + offset = xswap_alloc_slot_local(devs[i], order); put_swap_device(devs[i]); if (offset && offset != SWAP_ENTRY_INVALID) return swp_entry(devs[i]->type, offset); @@ -4317,29 +4306,46 @@ static void xswap_install_rmap(swp_entry_t phys, swp_entry_t entry) } } -swp_entry_t xswap_backend_alloc(swp_entry_t entry) +/* Back the range of 1 << @order slots at @entry with contiguous physical slots. */ +swp_entry_t xswap_backend_alloc(swp_entry_t entry, unsigned int order) { struct swap_info_struct *si = __swap_entry_to_info(entry); struct swap_cluster_info *ci; + unsigned long nr = 1UL << order; unsigned long off = swp_offset(entry); swp_entry_t phys; + unsigned int i; bool recorded = false; - phys = xswap_alloc_phys_slot(); + phys = xswap_alloc_phys_slot(order); if (!phys.val) return phys; - xswap_install_rmap(phys, entry); + /* One IO reads a large folio, so its range has to be contiguous. */ + VM_WARN_ON_ONCE(swp_offset(phys) % nr); + VM_WARN_ON_ONCE(off % SWAPFILE_CLUSTER + nr > SWAPFILE_CLUSTER); + + for (i = 0; i < nr; i++) + xswap_install_rmap(swp_entry(swp_type(phys), swp_offset(phys) + i), + swp_entry(si->type, off + i)); ci = swap_cluster_lock(si, off); if (ci) { - recorded = xswap_slot_set_backend(ci, off % SWAPFILE_CLUSTER, phys); + recorded = true; + for (i = 0; i < nr; i++) { + if (!xswap_slot_set_backend(ci, (off + i) % SWAPFILE_CLUSTER, + swp_entry(swp_type(phys), + swp_offset(phys) + i))) { + recorded = false; + break; + } + } swap_cluster_unlock(ci); } if (!recorded) { /* The backend is unrecorded, so the data would be unreachable. */ - xswap_backend_free(entry, phys); + xswap_backend_free(entry, phys, order); return (swp_entry_t){}; } @@ -4347,27 +4353,33 @@ swp_entry_t xswap_backend_alloc(swp_entry_t entry) } /* Undo xswap_backend_alloc(): the zswap copy is still in place. */ -void xswap_backend_free(swp_entry_t entry, swp_entry_t phys) +void xswap_backend_free(swp_entry_t entry, swp_entry_t phys, unsigned int order) { struct swap_info_struct *si = __swap_entry_to_info(entry); - struct swap_info_struct *psi; - struct swap_cluster_info *ci; + struct swap_info_struct *psi = swap_type_to_info(swp_type(phys)); + unsigned long nr = 1UL << order; unsigned long off = swp_offset(entry); + unsigned long poff = swp_offset(phys); + unsigned int i; - ci = swap_cluster_lock(si, off); - if (ci) { - unsigned int slot = off % SWAPFILE_CLUSTER; - unsigned long *xs = ci->xs_table; + for (i = 0; i < nr; i++) { + struct swap_cluster_info *ci; - if (xs && xs[slot] == phys.val) - WRITE_ONCE(xs[slot], 0); - swap_cluster_unlock(ci); - } + ci = swap_cluster_lock(si, off + i); + if (ci) { + unsigned int slot = (off + i) % SWAPFILE_CLUSTER; + unsigned long *xs = ci->xs_table; - /* xswap_free_phys_slot() clears the reverse mapping. */ - psi = swap_type_to_info(swp_type(phys)); - if (psi) - xswap_free_phys_slot(psi, entry, phys); + if (xs && xs[slot] == phys.val + i) + WRITE_ONCE(xs[slot], 0); + swap_cluster_unlock(ci); + } + + /* xswap_free_phys_slot() clears the reverse mapping. */ + if (psi) + xswap_free_phys_slot(psi, swp_entry(si->type, off + i), + swp_entry(swp_type(phys), poff + i)); + } } static int xswap_map_clusters(struct swap_info_struct *si, diff --git a/mm/zswap.c b/mm/zswap.c index 24904b4c4dce..c79c527e6345 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -1054,7 +1054,7 @@ static int zswap_writeback_entry(struct zswap_entry *entry, /* Reserve the destination before dropping the zswap copy. */ if (is_xswap) { - phys = xswap_backend_alloc(swpentry); + phys = xswap_backend_alloc(swpentry, 0); if (!phys.val) { ret = -ENOMEM; goto out; @@ -1087,7 +1087,7 @@ static int zswap_writeback_entry(struct zswap_entry *entry, out: if (ret) { if (is_xswap && phys.val) - xswap_backend_free(swpentry, phys); + xswap_backend_free(swpentry, phys, 0); swap_cache_del_folio(folio); folio_unlock(folio); } -- 2.54.0