Add a folio-based counterpart to page_snap_context() that reads the snap context from folio->private directly. Convert the three writeback paths that passed &folio->page to page_snap_context() to ceph_folio_snap_context(), removing three open-coded folio-to-page conversions. No functional change. Signed-off-by: Tal Zussman --- fs/ceph/addr.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index e2da3ab9f808..8347366ce5d9 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -75,6 +75,12 @@ static inline struct ceph_snap_context *page_snap_context(struct page *page) return NULL; } +static inline +struct ceph_snap_context *ceph_folio_snap_context(const struct folio *folio) +{ + return folio->private; +} + /* * Dirty a page. Optimistically adjust accounting, on the assumption * that we won't race with invalidate. If we do, readjust. @@ -749,7 +755,7 @@ static int write_folio_nounlock(struct folio *folio, return -EIO; /* verify this is a writeable snap context */ - snapc = page_snap_context(&folio->page); + snapc = ceph_folio_snap_context(folio); if (!snapc) { doutc(cl, "%llx.%llx folio %p not dirty?\n", ceph_vinop(inode), folio); @@ -1171,7 +1177,7 @@ int ceph_check_page_before_write(struct address_space *mapping, } /* only if matching snap context */ - pgsnapc = page_snap_context(&folio->page); + pgsnapc = ceph_folio_snap_context(folio); if (pgsnapc != ceph_wbc->snapc) { doutc(cl, "folio snapc %p %lld != oldest %p %lld\n", pgsnapc, pgsnapc->seq, @@ -1824,7 +1830,7 @@ ceph_find_incompatible(struct folio *folio) folio_wait_writeback(folio); - snapc = page_snap_context(&folio->page); + snapc = ceph_folio_snap_context(folio); if (!snapc || snapc == ci->i_head_snapc) break; -- 2.39.5 Iterate the writeback batch as folios rather than using &folios[i]->page. Use ceph_folio_snap_context() for the snap context check and folio_wait_writeback() instead of wait_on_page_writeback(). This removes one call to compound_head() and the last caller of wait_on_page_writeback(). No functional change. Reviewed-by: Matthew Wilcox (Oracle) Signed-off-by: Tal Zussman --- fs/ceph/addr.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index 8347366ce5d9..967243f425f2 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -1624,7 +1624,7 @@ void ceph_wait_until_current_writes_complete(struct address_space *mapping, struct writeback_control *wbc, struct ceph_writeback_ctl *ceph_wbc) { - struct page *page; + struct folio *folio; unsigned i, nr; if (wbc->sync_mode != WB_SYNC_NONE && @@ -1639,10 +1639,10 @@ void ceph_wait_until_current_writes_complete(struct address_space *mapping, PAGECACHE_TAG_WRITEBACK, &ceph_wbc->fbatch))) { for (i = 0; i < nr; i++) { - page = &ceph_wbc->fbatch.folios[i]->page; - if (page_snap_context(page) != ceph_wbc->snapc) + folio = ceph_wbc->fbatch.folios[i]; + if (ceph_folio_snap_context(folio) != ceph_wbc->snapc) continue; - wait_on_page_writeback(page); + folio_wait_writeback(folio); } folio_batch_release(&ceph_wbc->fbatch); -- 2.39.5 The last caller was converted to folio_wait_writeback(), so drop the wait_on_page_writeback() compatibility wrapper and its declaration. Reviewed-by: Matthew Wilcox (Oracle) Acked-by: David Hildenbrand (Arm) Signed-off-by: Tal Zussman --- include/linux/pagemap.h | 1 - mm/folio-compat.c | 6 ------ 2 files changed, 7 deletions(-) diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 2c3718d592d6..3748dde2b97e 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -1240,7 +1240,6 @@ static inline int folio_wait_locked_killable(struct folio *folio) } void folio_end_read(struct folio *folio, bool success); -void wait_on_page_writeback(struct page *page); void folio_wait_writeback(struct folio *folio); int folio_wait_writeback_killable(struct folio *folio); void end_page_writeback(struct page *page); diff --git a/mm/folio-compat.c b/mm/folio-compat.c index a02179a0bded..ae0271591d37 100644 --- a/mm/folio-compat.c +++ b/mm/folio-compat.c @@ -23,12 +23,6 @@ void end_page_writeback(struct page *page) } EXPORT_SYMBOL(end_page_writeback); -void wait_on_page_writeback(struct page *page) -{ - return folio_wait_writeback(page_folio(page)); -} -EXPORT_SYMBOL_GPL(wait_on_page_writeback); - void mark_page_accessed(struct page *page) { folio_mark_accessed(page_folio(page)); -- 2.39.5 Introduce ceph_fscrypt_pagecache_folio() and ceph_fscrypt_folio_offset(), folio equivalents of ceph_fscrypt_pagecache_page() and ceph_fscrypt_page_offset(), and use them to convert get_writepages_data_length() to folios. This removes the last caller of page_snap_context(), so remove it as well. This also removes a use of page->private and a call to fscrypt_is_bounce_page(). Signed-off-by: Tal Zussman --- fs/ceph/addr.c | 25 +++++++++---------------- fs/ceph/crypto.h | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index 967243f425f2..c81e9da42df9 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -29,9 +29,9 @@ * * There are a few funny things going on here. * - * The page->private field is used to reference a struct - * ceph_snap_context for _every_ dirty page. This indicates which - * snapshot the page was logically dirtied in, and thus which snap + * The folio->private field is used to reference a struct + * ceph_snap_context for _every_ dirty folio. This indicates which + * snapshot the folio was logically dirtied in, and thus which snap * context needs to be associated with the osd write during writeback. * * Similarly, struct ceph_inode_info maintains a set of counters to @@ -68,13 +68,6 @@ static int ceph_netfs_check_write_begin(struct file *file, loff_t pos, unsigned int len, struct folio **foliop, void **_fsdata); -static inline struct ceph_snap_context *page_snap_context(struct page *page) -{ - if (PagePrivate(page)) - return (void *)page->private; - return NULL; -} - static inline struct ceph_snap_context *ceph_folio_snap_context(const struct folio *folio) { @@ -692,7 +685,7 @@ get_oldest_context(struct inode *inode, struct ceph_writeback_ctl *ctl, } static u64 get_writepages_data_length(struct inode *inode, - struct page *page, u64 start) + struct folio *folio, u64 start) { struct ceph_inode_info *ci = ceph_inode(inode); struct ceph_snap_context *snapc; @@ -700,7 +693,7 @@ static u64 get_writepages_data_length(struct inode *inode, u64 end = i_size_read(inode); u64 ret; - snapc = page_snap_context(ceph_fscrypt_pagecache_page(page)); + snapc = ceph_folio_snap_context(ceph_fscrypt_pagecache_folio(folio)); if (snapc != ci->i_head_snapc) { bool found = false; spin_lock(&ci->i_ceph_lock); @@ -715,10 +708,10 @@ static u64 get_writepages_data_length(struct inode *inode, spin_unlock(&ci->i_ceph_lock); WARN_ON(!found); } - if (end > ceph_fscrypt_page_offset(page) + thp_size(page)) - end = ceph_fscrypt_page_offset(page) + thp_size(page); + if (end > ceph_fscrypt_folio_offset(folio) + folio_size(folio)) + end = ceph_fscrypt_folio_offset(folio) + folio_size(folio); ret = end > start ? end - start : 0; - if (ret && fscrypt_is_bounce_page(page)) + if (ret && fscrypt_is_bounce_folio(folio)) ret = round_up(ret, CEPH_FSCRYPT_BLOCK_SIZE); return ret; } @@ -1565,7 +1558,7 @@ int ceph_submit_write(struct address_space *mapping, * data length covers all locked pages */ u64 min_len = len + 1 - thp_size(page); len = get_writepages_data_length(inode, - ceph_wbc->pages[i - 1], + page_folio(ceph_wbc->pages[i - 1]), offset); len = max(len, min_len); } diff --git a/fs/ceph/crypto.h b/fs/ceph/crypto.h index 79cb563fd887..948c8b5dca06 100644 --- a/fs/ceph/crypto.h +++ b/fs/ceph/crypto.h @@ -162,6 +162,11 @@ static inline struct page *ceph_fscrypt_pagecache_page(struct page *page) return fscrypt_is_bounce_page(page) ? fscrypt_pagecache_page(page) : page; } +static inline struct folio *ceph_fscrypt_pagecache_folio(struct folio *folio) +{ + return fscrypt_is_bounce_folio(folio) ? fscrypt_pagecache_folio(folio) : folio; +} + #else /* CONFIG_FS_ENCRYPTION */ static inline void ceph_fscrypt_set_ops(struct super_block *sb) @@ -262,6 +267,11 @@ static inline struct page *ceph_fscrypt_pagecache_page(struct page *page) { return page; } + +static inline struct folio *ceph_fscrypt_pagecache_folio(struct folio *folio) +{ + return folio; +} #endif /* CONFIG_FS_ENCRYPTION */ static inline loff_t ceph_fscrypt_page_offset(struct page *page) @@ -269,4 +279,9 @@ static inline loff_t ceph_fscrypt_page_offset(struct page *page) return page_offset(ceph_fscrypt_pagecache_page(page)); } +static inline loff_t ceph_fscrypt_folio_offset(struct folio *folio) +{ + return folio_pos(ceph_fscrypt_pagecache_folio(folio)); +} + #endif /* _CEPH_CRYPTO_H */ -- 2.39.5 Convert the request assembly loop and error paths in ceph_submit_write() to folios. This drops ceph's uses of the set_page_writeback(), redirty_page_for_writepage(), and unlock_page() compatibility wrappers in the writeback submission path. Add ceph_folio_start_fscache(), a folio counterpart of ceph_set_page_fscache(). The remaining caller of the latter in write_folio_nounlock() will be converted separately. In total, this removes nine calls to compound_head() hidden in the page-based APIs, while adding four explicit ones via page_folio(). Note that get_writepages_data_length() must still be passed the possibly-bounce folio, not the unwrapped pagecache folio, as it checks fscrypt_is_bounce_folio() to round encrypted lengths up to the fscrypt block size. No functional change. Signed-off-by: Tal Zussman --- fs/ceph/addr.c | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index c81e9da42df9..a96986c503bf 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -553,6 +553,11 @@ const struct netfs_request_ops ceph_netfs_ops = { }; #ifdef CONFIG_CEPH_FSCACHE +static void ceph_folio_start_fscache(struct folio *folio) +{ + folio_start_private_2(folio); /* [DEPRECATED] */ +} + static void ceph_set_page_fscache(struct page *page) { folio_start_private_2(page_folio(page)); /* [DEPRECATED] */ @@ -575,6 +580,10 @@ static void ceph_fscache_write_to_cache(struct inode *inode, u64 off, u64 len, b ceph_fscache_write_terminated, inode, true, caching); } #else +static inline void ceph_folio_start_fscache(struct folio *folio) +{ +} + static inline void ceph_set_page_fscache(struct page *page) { } @@ -1436,14 +1445,14 @@ int ceph_submit_write(struct address_space *mapping, struct ceph_client *cl = fsc->client; struct ceph_vino vino = ceph_vino(inode); struct ceph_osd_request *req = NULL; - struct page *page = NULL; + struct folio *folio = NULL; bool caching = ceph_is_cache_enabled(inode); u64 offset; u64 len; unsigned i; new_request: - offset = ceph_fscrypt_page_offset(ceph_wbc->pages[0]); + offset = ceph_fscrypt_folio_offset(page_folio(ceph_wbc->pages[0])); len = ceph_wbc->wsize; req = ceph_osdc_new_request(&fsc->client->osdc, @@ -1467,29 +1476,28 @@ int ceph_submit_write(struct address_space *mapping, BUG_ON(IS_ERR(req)); } - page = ceph_wbc->pages[ceph_wbc->locked_pages - 1]; - BUG_ON(len < ceph_fscrypt_page_offset(page) + thp_size(page) - offset); + folio = page_folio(ceph_wbc->pages[ceph_wbc->locked_pages - 1]); + BUG_ON(len < ceph_fscrypt_folio_offset(folio) + folio_size(folio) - offset); if (!ceph_inc_osd_stopping_blocker(fsc->mdsc)) { for (i = 0; i < folio_batch_count(&ceph_wbc->fbatch); i++) { - struct folio *folio = ceph_wbc->fbatch.folios[i]; + folio = ceph_wbc->fbatch.folios[i]; if (!folio) continue; - page = &folio->page; - redirty_page_for_writepage(wbc, page); - unlock_page(page); + folio_redirty_for_writepage(wbc, folio); + folio_unlock(folio); } for (i = 0; i < ceph_wbc->locked_pages; i++) { - page = ceph_fscrypt_pagecache_page(ceph_wbc->pages[i]); + folio = ceph_fscrypt_pagecache_folio(page_folio(ceph_wbc->pages[i])); - if (!page) + if (!folio) continue; - redirty_page_for_writepage(wbc, page); - unlock_page(page); + folio_redirty_for_writepage(wbc, folio); + folio_unlock(folio); } ceph_osdc_put_request(req); @@ -1506,8 +1514,8 @@ int ceph_submit_write(struct address_space *mapping, for (i = 0; i < ceph_wbc->locked_pages; i++) { u64 cur_offset; - page = ceph_fscrypt_pagecache_page(ceph_wbc->pages[i]); - cur_offset = page_offset(page); + folio = ceph_fscrypt_pagecache_folio(page_folio(ceph_wbc->pages[i])); + cur_offset = folio_pos(folio); /* * Discontinuity in page range? Ceph can handle that by just passing @@ -1540,12 +1548,12 @@ int ceph_submit_write(struct address_space *mapping, ceph_wbc->op_idx++; } - set_page_writeback(page); + folio_start_writeback(folio); if (caching) - ceph_set_page_fscache(page); + ceph_folio_start_fscache(folio); - len += thp_size(page); + len += folio_size(folio); } ceph_fscache_write_to_cache(inode, offset, len, caching); @@ -1556,7 +1564,7 @@ int ceph_submit_write(struct address_space *mapping, /* writepages_finish() clears writeback pages * according to the data length, so make sure * data length covers all locked pages */ - u64 min_len = len + 1 - thp_size(page); + u64 min_len = len + 1 - folio_size(folio); len = get_writepages_data_length(inode, page_folio(ceph_wbc->pages[i - 1]), offset); -- 2.39.5 The last callers of thp_size() have been converted to folio_size(), and thp_order()'s only remaining user was thp_size() itself, so remove both. Signed-off-by: Tal Zussman --- include/linux/mm.h | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 485df9c2dbdd..48e821454ca0 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1966,27 +1966,6 @@ static inline unsigned int page_shift(struct page *page) return PAGE_SHIFT + compound_order(page); } -/** - * thp_order - Order of a transparent huge page. - * @page: Head page of a transparent huge page. - */ -static inline unsigned int thp_order(struct page *page) -{ - VM_BUG_ON_PGFLAGS(PageTail(page), page); - return compound_order(page); -} - -/** - * thp_size - Size of a transparent huge page. - * @page: Head page of a transparent huge page. - * - * Return: Number of bytes in this page. - */ -static inline unsigned long thp_size(struct page *page) -{ - return PAGE_SIZE << thp_order(page); -} - #ifdef CONFIG_MMU /* * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when -- 2.39.5 Use ceph_folio_start_fscache() and only use a struct page pointer at the osd_req_op_extent_osd_data_pages() boundary, which requires a page array. This removes a call to compound_head() in ceph_set_page_fscache(). This was the last user of ceph_set_page_fscache(), so remove it. Signed-off-by: Tal Zussman --- fs/ceph/addr.c | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index a96986c503bf..59e559bdfdff 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -558,11 +558,6 @@ static void ceph_folio_start_fscache(struct folio *folio) folio_start_private_2(folio); /* [DEPRECATED] */ } -static void ceph_set_page_fscache(struct page *page) -{ - folio_start_private_2(page_folio(page)); /* [DEPRECATED] */ -} - static void ceph_fscache_write_terminated(void *priv, ssize_t error) { struct inode *inode = priv; @@ -584,10 +579,6 @@ static inline void ceph_folio_start_fscache(struct folio *folio) { } -static inline void ceph_set_page_fscache(struct page *page) -{ -} - static inline void ceph_fscache_write_to_cache(struct inode *inode, u64 off, u64 len, bool caching) { } @@ -734,7 +725,7 @@ static u64 get_writepages_data_length(struct inode *inode, static int write_folio_nounlock(struct folio *folio, struct writeback_control *wbc) { - struct page *page = &folio->page; + struct page *page; struct inode *inode = folio->mapping->host; struct ceph_inode_info *ci = ceph_inode(inode); struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); @@ -813,7 +804,7 @@ static int write_folio_nounlock(struct folio *folio, folio_start_writeback(folio); if (caching) - ceph_set_page_fscache(&folio->page); + ceph_folio_start_fscache(folio); ceph_fscache_write_to_cache(inode, page_off, len, caching); if (IS_ENCRYPTED(inode)) { @@ -833,9 +824,8 @@ static int write_folio_nounlock(struct folio *folio, /* it may be a short write due to an object boundary */ WARN_ON_ONCE(len > folio_size(folio)); - osd_req_op_extent_osd_data_pages(req, 0, - bounce_page ? &bounce_page : &page, wlen, 0, - false, false); + page = bounce_page ? bounce_page : &folio->page; + osd_req_op_extent_osd_data_pages(req, 0, &page, wlen, 0, false, false); doutc(cl, "%llx.%llx %llu~%llu (%llu bytes, %sencrypted)\n", ceph_vinop(inode), page_off, len, wlen, IS_ENCRYPTED(inode) ? "" : "not "); -- 2.39.5 Convert the page cleanup loop in writepages_finish() to work on folios, resolving the folio directly after fscrypt_finalize_bounce_page() has replaced any bounce page with its pagecache page. This removes a use of detach_page_private() and five calls to compound_head() per page, while adding one back via page_folio(). No functional change. Signed-off-by: Tal Zussman --- fs/ceph/addr.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index 59e559bdfdff..eaf5b3f6d13f 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -896,7 +896,6 @@ static void writepages_finish(struct ceph_osd_request *req) struct ceph_inode_info *ci = ceph_inode(inode); struct ceph_client *cl = ceph_inode_to_client(inode); struct ceph_osd_data *osd_data; - struct page *page; int num_pages, total_pages = 0; int i, j; int rc = req->r_result; @@ -943,31 +942,33 @@ static void writepages_finish(struct ceph_osd_request *req) (u64)osd_data->length); total_pages += num_pages; for (j = 0; j < num_pages; j++) { + struct folio *folio; + fscrypt_finalize_bounce_page(&osd_data->pages[j]); - page = osd_data->pages[j]; - BUG_ON(!page); - WARN_ON(!PageUptodate(page)); + folio = page_folio(osd_data->pages[j]); + BUG_ON(!folio); + WARN_ON(!folio_test_uptodate(folio)); if (atomic_long_dec_return(&fsc->writeback_count) < CONGESTION_OFF_THRESH( fsc->mount_options->congestion_kb)) fsc->write_congested = false; - ceph_put_snap_context(detach_page_private(page)); - end_page_writeback(page); + ceph_put_snap_context(folio_detach_private(folio)); + folio_end_writeback(folio); if (atomic64_dec_return(&mdsc->dirty_folios) <= 0) { wake_up_all(&mdsc->flush_end_wq); WARN_ON(atomic64_read(&mdsc->dirty_folios) < 0); } - doutc(cl, "unlocking %p\n", page); + doutc(cl, "unlocking %p\n", folio); if (remove_page) generic_error_remove_folio(inode->i_mapping, - page_folio(page)); + folio); - unlock_page(page); + folio_unlock(folio); } doutc(cl, "%llx.%llx wrote %llu bytes cleaned %d pages\n", ceph_vinop(inode), osd_data->length, -- 2.39.5 All callers have been converted to the folio equivalents, so remove these unused helpers. This also removes a use of page_offset(). Signed-off-by: Tal Zussman --- fs/ceph/crypto.h | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/fs/ceph/crypto.h b/fs/ceph/crypto.h index 948c8b5dca06..d199dc8410fe 100644 --- a/fs/ceph/crypto.h +++ b/fs/ceph/crypto.h @@ -157,11 +157,6 @@ int ceph_fscrypt_decrypt_extents(struct inode *inode, struct page **page, int ceph_fscrypt_encrypt_pages(struct inode *inode, struct page **page, u64 off, int len); -static inline struct page *ceph_fscrypt_pagecache_page(struct page *page) -{ - return fscrypt_is_bounce_page(page) ? fscrypt_pagecache_page(page) : page; -} - static inline struct folio *ceph_fscrypt_pagecache_folio(struct folio *folio) { return fscrypt_is_bounce_folio(folio) ? fscrypt_pagecache_folio(folio) : folio; @@ -263,22 +258,12 @@ static inline int ceph_fscrypt_encrypt_pages(struct inode *inode, return 0; } -static inline struct page *ceph_fscrypt_pagecache_page(struct page *page) -{ - return page; -} - static inline struct folio *ceph_fscrypt_pagecache_folio(struct folio *folio) { return folio; } #endif /* CONFIG_FS_ENCRYPTION */ -static inline loff_t ceph_fscrypt_page_offset(struct page *page) -{ - return page_offset(ceph_fscrypt_pagecache_page(page)); -} - static inline loff_t ceph_fscrypt_folio_offset(struct folio *folio) { return folio_pos(ceph_fscrypt_pagecache_folio(folio)); -- 2.39.5 This function only operates on folios, so rename it accordingly. Signed-off-by: Tal Zussman --- fs/ceph/addr.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index eaf5b3f6d13f..08d854ed663b 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -1153,10 +1153,10 @@ bool can_next_page_be_processed(struct ceph_writeback_ctl *ceph_wbc, } static -int ceph_check_page_before_write(struct address_space *mapping, - struct writeback_control *wbc, - struct ceph_writeback_ctl *ceph_wbc, - struct folio *folio) +int ceph_check_folio_before_write(struct address_space *mapping, + struct writeback_control *wbc, + struct ceph_writeback_ctl *ceph_wbc, + struct folio *folio) { struct inode *inode = mapping->host; struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); @@ -1340,8 +1340,8 @@ void ceph_process_folio_batch(struct address_space *mapping, else if (!folio_trylock(folio)) break; - rc = ceph_check_page_before_write(mapping, wbc, - ceph_wbc, folio); + rc = ceph_check_folio_before_write(mapping, wbc, + ceph_wbc, folio); if (rc == -ENODATA) { folio_unlock(folio); folio_put(folio); -- 2.39.5