From: Kairui Song Invert the folio_ref_freeze() success check in __folio_freeze_and_split_unmapped() to return early on failure, which removes one level of indentation from the entire success path. This is a pure refactoring with no functional change. It prepares the function to be split into separate helpers for anonymous and file-backed folios in a later patch. Reviewed-by: Zi Yan Reviewed-by: Barry Song Acked-by: David Hildenbrand (Arm) Reviewed-by: Yeoreum Yun Reviewed-by: Kiryl Shutsemau (Meta) Signed-off-by: Kairui Song --- mm/huge_memory.c | 185 +++++++++++++++++++++++++++---------------------------- 1 file changed, 91 insertions(+), 94 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 09cf40357557..ad45b86819fd 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -3991,126 +3991,123 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n pgoff_t end, int *nr_shmem_dropped) { struct folio *end_folio = folio_next(folio); + struct swap_cluster_info *ci = NULL; struct folio *new_folio, *next; int old_order = folio_order(folio); + struct lruvec *lruvec; int ret = 0; VM_WARN_ON_ONCE(!mapping && end); - if (folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1)) { - struct swap_cluster_info *ci = NULL; - struct lruvec *lruvec; + if (!folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1)) + return -EAGAIN; - /* Take off the deferred split queue while frozen and memcg set */ - folio_unqueue_deferred_split(folio); + /* Take off the deferred split queue while frozen and memcg set */ + folio_unqueue_deferred_split(folio); - /* - * deferred_split_scan() takes the folio off the queue before it - * splits it, so the unqueue above finds an empty list and - * leaves PG_partially_mapped set. - * Clear it here: the flag does not survive the split. - */ - if (folio_test_partially_mapped(folio)) { - folio_clear_partially_mapped(folio); - mod_mthp_stat(old_order, - MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); - } + /* + * deferred_split_scan() takes the folio off the queue before it + * splits it, so the unqueue above finds an empty list and + * leaves PG_partially_mapped set. + * Clear it here: the flag does not survive the split. + */ + if (folio_test_partially_mapped(folio)) { + folio_clear_partially_mapped(folio); + mod_mthp_stat(old_order, + MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); + } - if (mapping) { - int nr = folio_nr_pages(folio); - - if (folio_test_pmd_mappable(folio) && - new_order < HPAGE_PMD_ORDER) { - if (folio_test_swapbacked(folio)) { - lruvec_stat_mod_folio(folio, - NR_SHMEM_THPS, -nr); - } else { - lruvec_stat_mod_folio(folio, - NR_FILE_THPS, -nr); - } + if (mapping) { + int nr = folio_nr_pages(folio); + + if (folio_test_pmd_mappable(folio) && + new_order < HPAGE_PMD_ORDER) { + if (folio_test_swapbacked(folio)) { + lruvec_stat_mod_folio(folio, + NR_SHMEM_THPS, -nr); + } else { + lruvec_stat_mod_folio(folio, + NR_FILE_THPS, -nr); } } + } - if (folio_test_swapcache(folio)) - ci = swap_cluster_get_and_lock(folio); - - /* lock lru list/PageCompound, ref frozen by page_ref_freeze */ - if (do_lru) - lruvec = folio_lruvec_lock(folio); + if (folio_test_swapcache(folio)) + ci = swap_cluster_get_and_lock(folio); - ret = __split_unmapped_folio(folio, new_order, split_at, xas, - mapping, split_type); + /* lock lru list/PageCompound, ref frozen by page_ref_freeze */ + if (do_lru) + lruvec = folio_lruvec_lock(folio); - /* - * Unfreeze after-split folios and put them back to the right - * list. @folio should be kept frozon until page cache - * entries are updated with all the other after-split folios - * to prevent others seeing stale page cache entries. - * As a result, new_folio starts from the next folio of - * @folio. - */ - for (new_folio = folio_next(folio); new_folio != end_folio; - new_folio = next) { - unsigned long nr_pages = folio_nr_pages(new_folio); + ret = __split_unmapped_folio(folio, new_order, split_at, xas, + mapping, split_type); - next = folio_next(new_folio); + /* + * Unfreeze after-split folios and put them back to the right + * list. @folio should be kept frozon until page cache + * entries are updated with all the other after-split folios + * to prevent others seeing stale page cache entries. + * As a result, new_folio starts from the next folio of + * @folio. + */ + for (new_folio = folio_next(folio); new_folio != end_folio; + new_folio = next) { + unsigned long nr_pages = folio_nr_pages(new_folio); - zone_device_private_split_cb(folio, new_folio); + next = folio_next(new_folio); - folio_ref_unfreeze(new_folio, - folio_cache_ref_count(new_folio) + 1); + zone_device_private_split_cb(folio, new_folio); - if (do_lru) - lru_add_split_folio(folio, new_folio, lruvec, list); + folio_ref_unfreeze(new_folio, + folio_cache_ref_count(new_folio) + 1); - /* - * Anonymous folio with swap cache. - * NOTE: shmem in swap cache is not supported yet. - */ - if (ci) { - __swap_cache_replace_folio(ci, folio, new_folio); - continue; - } + if (do_lru) + lru_add_split_folio(folio, new_folio, lruvec, list); - /* Anonymous folio without swap cache */ - if (!mapping) - continue; + /* + * Anonymous folio with swap cache. + * NOTE: shmem in swap cache is not supported yet. + */ + if (ci) { + __swap_cache_replace_folio(ci, folio, new_folio); + continue; + } - /* Add the new folio to the page cache. */ - if (new_folio->index < end) { - __xa_store(&mapping->i_pages, new_folio->index, - new_folio, 0); - continue; - } + /* Anonymous folio without swap cache */ + if (!mapping) + continue; - VM_WARN_ON_ONCE(!nr_shmem_dropped); - /* Drop folio beyond EOF: ->index >= end */ - if (shmem_mapping(mapping) && nr_shmem_dropped) - *nr_shmem_dropped += nr_pages; - else if (folio_test_clear_dirty(new_folio)) - folio_account_cleaned( - new_folio, inode_to_wb(mapping->host)); - __filemap_remove_folio(new_folio, NULL); - folio_put_refs(new_folio, nr_pages); + /* Add the new folio to the page cache. */ + if (new_folio->index < end) { + __xa_store(&mapping->i_pages, new_folio->index, + new_folio, 0); + continue; } - zone_device_private_split_cb(folio, NULL); - /* - * Unfreeze @folio only after all page cache entries, which - * used to point to it, have been updated with new folios. - * Otherwise, a parallel folio_try_get() can grab @folio - * and its caller can see stale page cache entries. - */ - folio_ref_unfreeze(folio, folio_cache_ref_count(folio) + 1); + VM_WARN_ON_ONCE(!nr_shmem_dropped); + /* Drop folio beyond EOF: ->index >= end */ + if (shmem_mapping(mapping) && nr_shmem_dropped) + *nr_shmem_dropped += nr_pages; + else if (folio_test_clear_dirty(new_folio)) + folio_account_cleaned(new_folio, + inode_to_wb(mapping->host)); + __filemap_remove_folio(new_folio, NULL); + folio_put_refs(new_folio, nr_pages); + } - if (do_lru) - lruvec_unlock(lruvec); + zone_device_private_split_cb(folio, NULL); + /* + * Unfreeze @folio only after all page cache entries, which + * used to point to it, have been updated with new folios. + * Otherwise, a parallel folio_try_get() can grab @folio + * and its caller can see stale page cache entries. + */ + folio_ref_unfreeze(folio, folio_cache_ref_count(folio) + 1); - if (ci) - swap_cluster_unlock(ci); - } else { - return -EAGAIN; - } + if (do_lru) + lruvec_unlock(lruvec); + if (ci) + swap_cluster_unlock(ci); return ret; } -- 2.55.0