From: Kairui Song Invert the folio_ref_freeze() success check in __folio_freeze_and_split_unmapped() to return early on failure, which removes one level of indentation from the entire success path. This is a pure refactoring with no functional change. It prepares the function to be split into separate helpers for anonymous and file-backed folios in a later patch. Signed-off-by: Kairui Song --- mm/huge_memory.c | 181 +++++++++++++++++++++++++++---------------------------- 1 file changed, 90 insertions(+), 91 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 2fa72158e063..cf8f90b94e42 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -3941,9 +3941,11 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n pgoff_t end, int *nr_shmem_dropped) { struct folio *end_folio = folio_next(folio); + struct swap_cluster_info *ci = NULL; struct folio *new_folio, *next; int old_order = folio_order(folio); struct list_lru_one *lru; + struct lruvec *lruvec; bool dequeue_deferred; int ret = 0; @@ -3964,122 +3966,119 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n lru = list_lru_lock(&deferred_split_lru, folio_nid(folio), &memcg); } - if (folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1)) { - struct swap_cluster_info *ci = NULL; - struct lruvec *lruvec; + if (!folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1)) { if (dequeue_deferred) { - __list_lru_del(&deferred_split_lru, lru, - &folio->_deferred_list, folio_nid(folio)); - if (folio_test_partially_mapped(folio)) { - folio_clear_partially_mapped(folio); - mod_mthp_stat(old_order, - MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); - } list_lru_unlock(lru); rcu_read_unlock(); } + return -EAGAIN; + } - if (mapping) { - int nr = folio_nr_pages(folio); - - if (folio_test_pmd_mappable(folio) && - new_order < HPAGE_PMD_ORDER) { - if (folio_test_swapbacked(folio)) { - lruvec_stat_mod_folio(folio, - NR_SHMEM_THPS, -nr); - } else { - lruvec_stat_mod_folio(folio, - NR_FILE_THPS, -nr); - } - } + if (dequeue_deferred) { + __list_lru_del(&deferred_split_lru, lru, + &folio->_deferred_list, folio_nid(folio)); + if (folio_test_partially_mapped(folio)) { + folio_clear_partially_mapped(folio); + mod_mthp_stat(old_order, + MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); } + list_lru_unlock(lru); + rcu_read_unlock(); + } - if (folio_test_swapcache(folio)) - ci = swap_cluster_get_and_lock(folio); - - /* lock lru list/PageCompound, ref frozen by page_ref_freeze */ - if (do_lru) - lruvec = folio_lruvec_lock(folio); - - ret = __split_unmapped_folio(folio, new_order, split_at, xas, - mapping, split_type); + if (mapping) { + int nr = folio_nr_pages(folio); - /* - * Unfreeze after-split folios and put them back to the right - * list. @folio should be kept frozon until page cache - * entries are updated with all the other after-split folios - * to prevent others seeing stale page cache entries. - * As a result, new_folio starts from the next folio of - * @folio. - */ - for (new_folio = folio_next(folio); new_folio != end_folio; - new_folio = next) { - unsigned long nr_pages = folio_nr_pages(new_folio); + if (folio_test_pmd_mappable(folio) && + new_order < HPAGE_PMD_ORDER) { + if (folio_test_swapbacked(folio)) { + lruvec_stat_mod_folio(folio, + NR_SHMEM_THPS, -nr); + } else { + lruvec_stat_mod_folio(folio, + NR_FILE_THPS, -nr); + } + } + } - next = folio_next(new_folio); + if (folio_test_swapcache(folio)) + ci = swap_cluster_get_and_lock(folio); - zone_device_private_split_cb(folio, new_folio); + /* lock lru list/PageCompound, ref frozen by page_ref_freeze */ + if (do_lru) + lruvec = folio_lruvec_lock(folio); - folio_ref_unfreeze(new_folio, - folio_cache_ref_count(new_folio) + 1); + ret = __split_unmapped_folio(folio, new_order, split_at, xas, + mapping, split_type); - if (do_lru) - lru_add_split_folio(folio, new_folio, lruvec, list); + /* + * Unfreeze after-split folios and put them back to the right + * list. @folio should be kept frozon until page cache + * entries are updated with all the other after-split folios + * to prevent others seeing stale page cache entries. + * As a result, new_folio starts from the next folio of + * @folio. + */ + for (new_folio = folio_next(folio); new_folio != end_folio; + new_folio = next) { + unsigned long nr_pages = folio_nr_pages(new_folio); - /* - * Anonymous folio with swap cache. - * NOTE: shmem in swap cache is not supported yet. - */ - if (ci) { - __swap_cache_replace_folio(ci, folio, new_folio); - continue; - } + next = folio_next(new_folio); - /* Anonymous folio without swap cache */ - if (!mapping) - continue; + zone_device_private_split_cb(folio, new_folio); - /* Add the new folio to the page cache. */ - if (new_folio->index < end) { - __xa_store(&mapping->i_pages, new_folio->index, - new_folio, 0); - continue; - } + folio_ref_unfreeze(new_folio, + folio_cache_ref_count(new_folio) + 1); - VM_WARN_ON_ONCE(!nr_shmem_dropped); - /* Drop folio beyond EOF: ->index >= end */ - if (shmem_mapping(mapping) && nr_shmem_dropped) - *nr_shmem_dropped += nr_pages; - else if (folio_test_clear_dirty(new_folio)) - folio_account_cleaned( - new_folio, inode_to_wb(mapping->host)); - __filemap_remove_folio(new_folio, NULL); - folio_put_refs(new_folio, nr_pages); - } + if (do_lru) + lru_add_split_folio(folio, new_folio, lruvec, list); - zone_device_private_split_cb(folio, NULL); /* - * Unfreeze @folio only after all page cache entries, which - * used to point to it, have been updated with new folios. - * Otherwise, a parallel folio_try_get() can grab @folio - * and its caller can see stale page cache entries. + * Anonymous folio with swap cache. + * NOTE: shmem in swap cache is not supported yet. */ - folio_ref_unfreeze(folio, folio_cache_ref_count(folio) + 1); + if (ci) { + __swap_cache_replace_folio(ci, folio, new_folio); + continue; + } - if (do_lru) - lruvec_unlock(lruvec); + /* Anonymous folio without swap cache */ + if (!mapping) + continue; - if (ci) - swap_cluster_unlock(ci); - } else { - if (dequeue_deferred) { - list_lru_unlock(lru); - rcu_read_unlock(); + /* Add the new folio to the page cache. */ + if (new_folio->index < end) { + __xa_store(&mapping->i_pages, new_folio->index, + new_folio, 0); + continue; } - return -EAGAIN; + + VM_WARN_ON_ONCE(!nr_shmem_dropped); + /* Drop folio beyond EOF: ->index >= end */ + if (shmem_mapping(mapping) && nr_shmem_dropped) + *nr_shmem_dropped += nr_pages; + else if (folio_test_clear_dirty(new_folio)) + folio_account_cleaned(new_folio, + inode_to_wb(mapping->host)); + __filemap_remove_folio(new_folio, NULL); + folio_put_refs(new_folio, nr_pages); } + zone_device_private_split_cb(folio, NULL); + /* + * Unfreeze @folio only after all page cache entries, which + * used to point to it, have been updated with new folios. + * Otherwise, a parallel folio_try_get() can grab @folio + * and its caller can see stale page cache entries. + */ + folio_ref_unfreeze(folio, folio_cache_ref_count(folio) + 1); + + if (do_lru) + lruvec_unlock(lruvec); + if (ci) + swap_cluster_unlock(ci); + return ret; } -- 2.55.0