From: Kairui Song The check only exists to avoid the expensive PMD-splitting unmap of a folio that cannot be split anyway. Move it from __folio_split() into unmap_folio(), right before the PMD split, so both the anon and file split helpers get the early check without repeating it. unmap_folio() now returns -EAGAIN if the check fails and the split helpers propagate the error. folio_split_unmapped() drops its own copy of the check: it works on already unmapped folios and the definitive folio_ref_freeze() in __folio_freeze_split_anon() still catches unexpected references. Signed-off-by: Kairui Song --- mm/huge_memory.c | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index c5279c0d0e59..17bd2c053210 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -3538,13 +3538,17 @@ void vma_adjust_trans_huge(struct vm_area_struct *vma, split_huge_pmd_if_needed(next, end); } -static void unmap_folio(struct folio *folio) +static int unmap_folio(struct folio *folio) { enum ttu_flags ttu_flags = TTU_RMAP_LOCKED | TTU_SYNC | TTU_BATCH_FLUSH; VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); + /* Racy check if we can split the page, before we split PMDs */ + if (folio_expected_ref_count(folio) != folio_ref_count(folio) - 1) + return -EAGAIN; + if (folio_test_pmd_mappable(folio)) ttu_flags |= TTU_SPLIT_HUGE_PMD; @@ -3559,6 +3563,8 @@ static void unmap_folio(struct folio *folio) try_to_unmap(folio, ttu_flags | TTU_IGNORE_MLOCK); try_to_unmap_flush(); + + return 0; } static bool __discard_anon_folio_pmd_locked(struct vm_area_struct *vma, @@ -4006,7 +4012,9 @@ static int __folio_freeze_split_anon(struct folio *folio, if (folio_mapped(folio)) { need_remap = true; - unmap_folio(folio); + ret = unmap_folio(folio); + if (ret) + return ret; } local_irq_disable(); @@ -4104,7 +4112,9 @@ static int __folio_freeze_split_file(struct folio *folio, if (shmem_mapping(mapping)) end = shmem_fallocend(mapping->host, end); - unmap_folio(folio); + ret = unmap_folio(folio); + if (ret) + return ret; xas_lock_irq(xas); @@ -4312,15 +4322,6 @@ static int __folio_split(struct folio *folio, unsigned int new_order, i_mmap_lock_read(mapping); } - /* - * Racy check if we can split the page, before unmap_folio() will - * split PMDs - */ - if (folio_expected_ref_count(folio) != folio_ref_count(folio) - 1) { - ret = -EAGAIN; - goto out_unlock; - } - if (is_anon) ret = __folio_freeze_split_anon(folio, new_order, split_at, true, list, split_type); @@ -4359,7 +4360,6 @@ static int __folio_split(struct folio *folio, unsigned int new_order, free_folio_and_swap_cache(new_folio); } -out_unlock: if (anon_vma) { anon_vma_unlock_write(anon_vma); put_anon_vma(anon_vma); @@ -4407,9 +4407,6 @@ int folio_split_unmapped(struct folio *folio, unsigned int new_order) VM_WARN_ON_ONCE_FOLIO(!folio_test_large(folio), folio); VM_WARN_ON_ONCE_FOLIO(!folio_test_anon(folio), folio); - if (folio_expected_ref_count(folio) != folio_ref_count(folio) - 1) - return -EAGAIN; - return __folio_freeze_split_anon(folio, new_order, &folio->page, false, NULL, SPLIT_TYPE_UNIFORM); } -- 2.55.0