From: Kairui Song Only file split needs the filemap and xarray handling and related variables. Move them out of __folio_split() into the file helper so the helper is self-contained, and simplify the parameters. No functional change. Signed-off-by: Kairui Song --- mm/huge_memory.c | 102 ++++++++++++++++++++++++------------------------------- 1 file changed, 44 insertions(+), 58 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 17bd2c053210..c4910c0b6018 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -4087,16 +4087,42 @@ static int __folio_freeze_split_anon(struct folio *folio, static int __folio_freeze_split_file(struct folio *folio, unsigned int new_order, struct page *split_at, - struct xa_state *xas, struct address_space *mapping, bool do_lru, struct list_head *list, enum split_type split_type) { + struct address_space *mapping = folio->mapping; + XA_STATE(xas, &mapping->i_pages, folio->index); struct folio *end_folio = folio_next(folio); struct folio *new_folio, *next; int nr_shmem_dropped = 0; + unsigned int min_order; struct lruvec *lruvec; pgoff_t end; - int ret; + gfp_t gfp; + int ret = 0; + + min_order = mapping_min_folio_order(mapping); + if (new_order < min_order) + return -EINVAL; + + gfp = current_gfp_context(mapping_gfp_mask(mapping) & GFP_RECLAIM_MASK); + if (!filemap_release_folio(folio, gfp)) + return -EBUSY; + + mapping_set_update(&xas, mapping); + + if (split_type == SPLIT_TYPE_UNIFORM) { + int old_order = folio_order(folio); + + xas_set_order(&xas, folio->index, new_order); + xas_split_alloc(&xas, folio, old_order, gfp); + if (xas_error(&xas)) { + ret = xas_error(&xas); + goto fail_free; + } + } + + i_mmap_lock_read(mapping); /* Currently device private folios can only back anonymous memory. */ VM_WARN_ON_ONCE_FOLIO(folio_is_device_private(folio), folio); @@ -4114,15 +4140,15 @@ static int __folio_freeze_split_file(struct folio *folio, ret = unmap_folio(folio); if (ret) - return ret; + goto fail_mmap_unlock; - xas_lock_irq(xas); + xas_lock_irq(&xas); /* * Check if the folio is present in page cache. * We assume all tail are present too, if folio is there. */ - if (xas_load(xas) != folio) { + if (xas_load(&xas) != folio) { ret = -EAGAIN; goto fail; } @@ -4149,7 +4175,7 @@ static int __folio_freeze_split_file(struct folio *folio, if (do_lru) lruvec = folio_lruvec_lock(folio); - ret = __split_frozen_folio(folio, new_order, split_at, xas, + ret = __split_frozen_folio(folio, new_order, split_at, &xas, mapping, split_type); /* @@ -4205,9 +4231,19 @@ static int __folio_freeze_split_file(struct folio *folio, * If we want to use try_to_migrate() on file in unmap_folio, * remember to add remap_folio() and adapt it. */ - xas_unlock_irq(xas); + xas_unlock_irq(&xas); +fail_mmap_unlock: if (nr_shmem_dropped) shmem_uncharge(mapping->host, nr_shmem_dropped); + /* + * Drop the mapping while the inode is still pinned. @folio stays + * locked and present in the page cache, so eviction cannot free + * the inode yet, nothing past this point may touch the inode or + * the mapping. + */ + i_mmap_unlock_read(mapping); +fail_free: + xas_destroy(&xas); return ret; } @@ -4236,11 +4272,9 @@ static int __folio_split(struct folio *folio, unsigned int new_order, struct page *split_at, struct page *lock_at, struct list_head *list, enum split_type split_type) { - XA_STATE(xas, &folio->mapping->i_pages, folio->index); struct folio *end_folio = folio_next(folio); bool is_anon = folio_test_anon(folio); struct mem_cgroup *memcg, *old_memcg; - struct address_space *mapping = NULL; struct anon_vma *anon_vma = NULL; int old_order = folio_order(folio); struct folio *new_folio, *next; @@ -4287,60 +4321,15 @@ static int __folio_split(struct folio *folio, unsigned int new_order, goto out; } anon_vma_lock_write(anon_vma); - mapping = NULL; - } else { - unsigned int min_order; - gfp_t gfp; - - mapping = folio->mapping; - min_order = mapping_min_folio_order(mapping); - if (new_order < min_order) { - ret = -EINVAL; - goto out; - } - - gfp = current_gfp_context(mapping_gfp_mask(mapping) & - GFP_RECLAIM_MASK); - - if (!filemap_release_folio(folio, gfp)) { - ret = -EBUSY; - goto out; - } - - mapping_set_update(&xas, mapping); - - if (split_type == SPLIT_TYPE_UNIFORM) { - xas_set_order(&xas, folio->index, new_order); - xas_split_alloc(&xas, folio, old_order, gfp); - if (xas_error(&xas)) { - ret = xas_error(&xas); - goto out; - } - } - - anon_vma = NULL; - i_mmap_lock_read(mapping); } if (is_anon) ret = __folio_freeze_split_anon(folio, new_order, split_at, true, list, split_type); else - ret = __folio_freeze_split_file(folio, new_order, split_at, &xas, mapping, + ret = __folio_freeze_split_file(folio, new_order, split_at, true, list, split_type); - /* - * Drop the mapping while the inode is still pinned. @folio stays - * locked and present in the page cache until the loop below, so - * eviction cannot free the inode yet; @lock_at is not enough, it may - * be a tail beyond EOF that the split already dropped from the page - * cache. Nothing past this point may touch the inode or the mapping. - */ - if (mapping) { - i_mmap_unlock_read(mapping); - mapping = NULL; - } - /* * Unlock all after-split folios except the one containing * @lock_at page. If @folio is not split, it will be kept locked. @@ -4364,14 +4353,11 @@ static int __folio_split(struct folio *folio, unsigned int new_order, anon_vma_unlock_write(anon_vma); put_anon_vma(anon_vma); } - if (mapping) - i_mmap_unlock_read(mapping); out: /* restore to caller's old_memcg */ set_active_memcg(old_memcg); mem_cgroup_put(memcg); out_no_memcg: - xas_destroy(&xas); if (is_pmd_order(old_order)) count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); count_mthp_stat(old_order, !ret ? MTHP_STAT_SPLIT : MTHP_STAT_SPLIT_FAILED); -- 2.55.0