Add migrate_folios_mc_copy() which walks src/dst folio lists in lockstep, copies folio content via folio_mc_copy() and marks the dst with FOLIO_CONTENT_COPIED so the move phase in __migrate_folio() can skip the per-folio copy. The folios_cnt parameter is unused here, present for the offload_copy callback signature compatibility used by later patches in the series. Split unmapped folios into batch-eligible (unmap_batch/dst_batch) and standard (unmap_single/dst_single) lists. A folio is batch-eligible when the migration reason is allowed by the driver, refcount equals expected and it's not a movable_ops page. After TLB flush, batch copy the eligible folios via migrate_folios_mc_copy(), then run the move phase. The FOLIO_CONTENT_COPIED marker is consumed on first read in the __migrate_folio, so a -EAGAIN retry falls back to fresh per-folio copy. Signed-off-by: Shivank Garg --- include/linux/migrate.h | 3 ++ mm/migrate.c | 107 ++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 97 insertions(+), 13 deletions(-) diff --git a/include/linux/migrate.h b/include/linux/migrate.h index 876035df2fdc..e62c974b81d5 100644 --- a/include/linux/migrate.h +++ b/include/linux/migrate.h @@ -71,6 +71,9 @@ void folio_migrate_flags(struct folio *newfolio, struct folio *folio); int folio_migrate_mapping(struct address_space *mapping, struct folio *newfolio, struct folio *folio, int extra_count); int set_movable_ops(const struct movable_operations *ops, enum pagetype type); +int migrate_folios_mc_copy(struct list_head *dst_list, + struct list_head *src_list, + unsigned int __always_unused folios_cnt); /* * To record some information during migration, we use the migrate_info diff --git a/mm/migrate.c b/mm/migrate.c index b3f632575c82..41b732e78a67 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -51,6 +51,12 @@ #include "internal.h" #include "swap.h" +/* For now, never offload. Wired up in later patch. */ +static bool migrate_should_offload(int reason) +{ + return false; +} + static const struct movable_operations *offline_movable_ops; static const struct movable_operations *zsmalloc_movable_ops; @@ -1707,6 +1713,60 @@ static int migrate_hugetlbs(struct list_head *from, new_folio_t get_new_folio, return nr_failed; } +static bool folio_can_batch_copy(struct folio *folio) +{ + /* movable_ops pages have a separate migration path */ + if (unlikely(page_has_movable_ops(&folio->page))) + return false; + + /* + * Only batch when refcount matches the expected refcount. + * Otherwise a GUP holder could modify src between the + * batch copy and the move-phase refcount check, leaving the + * dst folio with a stale copy. + */ + return folio_ref_count(folio) == folio_expected_ref_count(folio) + 1; +} + +/** + * migrate_folios_mc_copy - Copy the contents of a list of folios. + * @dst_list: destination folio list. + * @src_list: source folio list. + * @folios_cnt: unused here, present for callback signature compatibility. + * + * Walk src and dst folio lists in lockstep, copy each folio via + * folio_mc_copy(), and mark the dst with FOLIO_CONTENT_COPIED so the + * move phase in __migrate_folio() skips the per-folio copy. The caller + * must ensure both lists have the same number of entries. + * + * On error, dst folios already copied keep their FOLIO_CONTENT_COPIED + * marker, while the remaining folios fallback to per-folio CPU copy in + * the move phase. + * + * This function may sleep. + * + * Return: 0 on success, negative errno on failure. + */ +int migrate_folios_mc_copy(struct list_head *dst_list, + struct list_head *src_list, + unsigned int __always_unused folios_cnt) +{ + struct folio *src, *dst; + int ret; + + dst = list_first_entry(dst_list, struct folio, lru); + list_for_each_entry(src, src_list, lru) { + ret = folio_mc_copy(dst, src); + if (ret) + return ret; + dst->migrate_info |= FOLIO_CONTENT_COPIED; + dst = list_next_entry(dst, lru); + cond_resched(); + } + + return 0; +} + static void migrate_folios_move(struct list_head *src_folios, struct list_head *dst_folios, free_folio_t put_new_folio, unsigned long private, @@ -1806,9 +1866,13 @@ static int migrate_pages_batch(struct list_head *from, bool is_large = false; struct folio *folio, *folio2, *dst = NULL; int rc, rc_saved = 0, nr_pages; - LIST_HEAD(unmap_folios); - LIST_HEAD(dst_folios); + unsigned int nr_batch = 0; + LIST_HEAD(unmap_batch); + LIST_HEAD(dst_batch); + LIST_HEAD(unmap_single); + LIST_HEAD(dst_single); bool nosplit = (reason == MR_NUMA_MISPLACED); + bool offload = migrate_should_offload(reason); VM_WARN_ON_ONCE(mode != MIGRATE_ASYNC && !list_empty(from) && !list_is_singular(from)); @@ -1902,8 +1966,8 @@ static int migrate_pages_batch(struct list_head *from, private, folio, &dst, mode, ret_folios); /* * The rules are: - * 0: folio will be put on unmap_folios list, - * dst folio put on dst_folios list + * 0: folio put on unmap_batch or unmap_single, + * dst folio put on dst_batch or dst_single * -EAGAIN: stay on the from list * -ENOMEM: stay on the from list * Other errno: put on ret_folios list @@ -1944,7 +2008,7 @@ static int migrate_pages_batch(struct list_head *from, /* nr_failed isn't updated for not used */ stats->nr_thp_failed += thp_retry; rc_saved = rc; - if (list_empty(&unmap_folios)) + if (list_empty(&unmap_batch) && list_empty(&unmap_single)) goto out; else goto move; @@ -1954,8 +2018,14 @@ static int migrate_pages_batch(struct list_head *from, nr_retry_pages += nr_pages; break; case 0: - list_move_tail(&folio->lru, &unmap_folios); - list_add_tail(&dst->lru, &dst_folios); + if (offload && folio_can_batch_copy(folio)) { + list_move_tail(&folio->lru, &unmap_batch); + list_add_tail(&dst->lru, &dst_batch); + nr_batch++; + } else { + list_move_tail(&folio->lru, &unmap_single); + list_add_tail(&dst->lru, &dst_single); + } break; default: /* @@ -1978,17 +2048,26 @@ static int migrate_pages_batch(struct list_head *from, /* Flush TLBs for all unmapped folios */ try_to_unmap_flush(); + /* Batch-copy eligible folios before the move phase */ + if (!list_empty(&unmap_batch)) + migrate_folios_mc_copy(&dst_batch, &unmap_batch, nr_batch); + retry = 1; for (pass = 0; pass < nr_pass && retry; pass++) { retry = 0; thp_retry = 0; nr_retry_pages = 0; - /* Move the unmapped folios */ - migrate_folios_move(&unmap_folios, &dst_folios, - put_new_folio, private, mode, reason, - ret_folios, stats, &retry, &thp_retry, - &nr_failed, &nr_retry_pages); + if (!list_empty(&unmap_batch)) + migrate_folios_move(&unmap_batch, &dst_batch, put_new_folio, + private, mode, reason, ret_folios, stats, + &retry, &thp_retry, &nr_failed, + &nr_retry_pages); + if (!list_empty(&unmap_single)) + migrate_folios_move(&unmap_single, &dst_single, put_new_folio, + private, mode, reason, ret_folios, stats, + &retry, &thp_retry, &nr_failed, + &nr_retry_pages); } nr_failed += retry; stats->nr_thp_failed += thp_retry; @@ -1997,7 +2076,9 @@ static int migrate_pages_batch(struct list_head *from, rc = rc_saved ? : nr_failed; out: /* Cleanup remaining folios */ - migrate_folios_undo(&unmap_folios, &dst_folios, + migrate_folios_undo(&unmap_batch, &dst_batch, + put_new_folio, private, ret_folios); + migrate_folios_undo(&unmap_single, &dst_single, put_new_folio, private, ret_folios); return rc; -- 2.43.0