hugetlbfs currently uses an fs-specific way to determine the size of the folios in its page cache. Thanks to the support of block sizes larger than page sizes, we now support this as part of the page cache. It's somewhat more efficient as we have this information directly in the mapping rather than going from inode->i_sb->s_fs_info. We can convert the rest of the hugetlb code to use this at our leisure; this is needed now so that we can use filemap_get_pages() unmodified. Signed-off-by: Matthew Wilcox (Oracle) --- fs/hugetlbfs/inode.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 216e1a0dd0b2..1760df6f2709 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -883,6 +883,16 @@ static struct inode *hugetlbfs_get_root(struct super_block *sb, return inode; } +static void hugetlbfs_init_regular_inode(struct inode *inode) +{ + struct hstate *hstate = hstate_inode(inode); + unsigned int order = huge_page_order(hstate); + + inode->i_op = &hugetlbfs_inode_operations; + inode->i_fop = &hugetlbfs_file_operations; + mapping_set_folio_order_range(inode->i_mapping, order, order); +} + /* * Hugetlbfs is not reclaimable; therefore its i_mmap_rwsem will never * be taken from reclaim -- unlike regular filesystems. This needs an @@ -926,8 +936,7 @@ static struct inode *hugetlbfs_get_inode(struct super_block *sb, init_special_inode(inode, mode, dev); break; case S_IFREG: - inode->i_op = &hugetlbfs_inode_operations; - inode->i_fop = &hugetlbfs_file_operations; + hugetlbfs_init_regular_inode(inode); break; case S_IFDIR: inode->i_op = &hugetlbfs_dir_inode_operations; -- 2.47.3 Checking to see whether a page inside a hugetlb folio is hardware poisoned is more complicated than I realised. The correct logic was in is_raw_hwpoison_page_in_hugepage(). Make is_page_hwpoison() the official entry point for "is this page hwpoison", no matter what kind of folio it belongs to. Rename is_raw_hwpoison_page_in_hugepage() to hugetlb_page_hwpoison(), and remove the attempt to handle non-hugetlb folios from it. Signed-off-by: Matthew Wilcox (Oracle) --- fs/hugetlbfs/inode.c | 4 ++-- include/linux/hugetlb.h | 5 ----- include/linux/page-flags.h | 24 +++++++++++++++++++----- mm/memory-failure.c | 9 ++++----- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 1760df6f2709..725bb945a440 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -198,7 +198,7 @@ static size_t adjust_range_hwpoison(struct folio *folio, size_t offset, struct page *page = folio_page(folio, offset / PAGE_SIZE); size_t safe_bytes; - if (is_raw_hwpoison_page_in_hugepage(page)) + if (hugetlb_page_hwpoison(folio, page)) return 0; /* Safe to read the remaining bytes in this page. */ safe_bytes = PAGE_SIZE - (offset % PAGE_SIZE); @@ -206,7 +206,7 @@ static size_t adjust_range_hwpoison(struct folio *folio, size_t offset, /* Check each remaining page as long as we are not done yet. */ for (; safe_bytes < bytes; safe_bytes += PAGE_SIZE, page++) - if (is_raw_hwpoison_page_in_hugepage(page)) + if (hugetlb_page_hwpoison(folio, page)) break; return min(safe_bytes, bytes); diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 2abaf99321e9..5df0bb618b05 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -1070,11 +1070,6 @@ void hugetlb_register_node(struct node *node); void hugetlb_unregister_node(struct node *node); #endif -/* - * Check if a given raw @page in a hugepage is HWPOISON. - */ -bool is_raw_hwpoison_page_in_hugepage(struct page *page); - static inline unsigned long huge_page_mask_align(struct file *file) { return PAGE_MASK & ~huge_page_mask(hstate_file(file)); diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 7223f6f4e2b4..b0b451b4e632 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -1068,6 +1068,9 @@ static inline bool PageHuge(const struct page *page) return folio_test_hugetlb(page_folio(page)); } +#ifdef CONFIG_MEMORY_FAILURE +bool hugetlb_page_hwpoison(const struct folio *folio, const struct page *page); + /* * Check if a page is currently marked HWPoisoned. Note that this check is * best effort only and inherently racy: there is no way to synchronize with @@ -1075,13 +1078,24 @@ static inline bool PageHuge(const struct page *page) */ static inline bool is_page_hwpoison(const struct page *page) { - const struct folio *folio; + const struct folio *folio = page_folio(page); + + if (folio_test_hugetlb(folio)) + return hugetlb_page_hwpoison(folio, page); + return PageHWPoison(page); +} +#else +static inline bool hugetlb_page_hwpoison(const struct folio *folio, + const struct page *page) +{ + return false; +} - if (PageHWPoison(page)) - return true; - folio = page_folio(page); - return folio_test_hugetlb(folio) && PageHWPoison(&folio->page); +static inline bool is_page_hwpoison(const struct page *page) +{ + return false; } +#endif static inline bool folio_contain_hwpoisoned_page(struct folio *folio) { diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 51508a55c405..0980791108a4 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -1822,19 +1822,18 @@ static inline struct llist_head *raw_hwp_list_head(struct folio *folio) return (struct llist_head *)&folio->_hugetlb_hwpoison; } -bool is_raw_hwpoison_page_in_hugepage(struct page *page) +/* + * Check if a given @page in a hugetlb folio is HWPOISON. + */ +bool hugetlb_page_hwpoison(const struct folio *folio, const struct page *page) { struct llist_head *raw_hwp_head; struct raw_hwp_page *p; - struct folio *folio = page_folio(page); bool ret = false; if (!folio_test_hwpoison(folio)) return false; - if (!folio_test_hugetlb(folio)) - return PageHWPoison(page); - /* * When RawHwpUnreliable is set, kernel lost track of which subpages * are HWPOISON. So return as if ALL subpages are HWPOISONed. -- 2.47.3 From: Jane Chu Add hwpoison handling to filemap_read() such that .read_iter() could make best effort copying data out of clean pages without risking MCE in case page cache contains HWpoison. [1] https://lore.kernel.org/linux-mm/aeZwAz6PcdlqSnJ2@casper.infradead.org/ Suggested-by: Matthew Wilcox Signed-off-by: Jane Chu Signed-off-by: Matthew Wilcox --- mm/filemap.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/mm/filemap.c b/mm/filemap.c index 5af62e6abca5..2b65367574be 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2483,6 +2483,8 @@ static void filemap_get_read_batch(struct address_space *mapping, if (!folio_batch_add(fbatch, folio)) break; + if (folio_contain_hwpoisoned_page(folio)) + break; if (!folio_test_uptodate(folio)) break; if (folio_test_readahead(folio)) @@ -2749,6 +2751,27 @@ static inline bool pos_same_folio(loff_t pos1, loff_t pos2, struct folio *folio) return (pos1 >> shift == pos2 >> shift); } +static size_t adjust_range_hwpoison(const struct folio *folio, size_t offset, + size_t bytes) +{ + const struct page *page = folio_page(folio, offset / PAGE_SIZE); + size_t safe_bytes; + + if (is_page_hwpoison(page)) + return 0; + + /* Safe to read the remaining bytes in this page. */ + safe_bytes = PAGE_SIZE - (offset % PAGE_SIZE); + page++; + + /* Check each remaining page as long as we are not done yet. */ + for (; safe_bytes < bytes; safe_bytes += PAGE_SIZE, page++) + if (is_page_hwpoison(page)) + break; + + return min(safe_bytes, bytes); +} + static void filemap_end_dropbehind_read(struct folio *folio) { if (!folio_test_dropbehind(folio)) @@ -2862,6 +2885,14 @@ ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *iter, if (writably_mapped) flush_dcache_folio(folio); + if (folio_contain_hwpoisoned_page(folio)) { + bytes = adjust_range_hwpoison(folio, offset, bytes); + if (bytes == 0) { + error = -EIO; + break; + } + } + copied = copy_folio_to_iter(folio, offset, bytes, iter); already_read += copied; -- 2.47.3 From: Jane Chu generic_file_read_iter() now handles hugetlb poison and hugetlb page size correctly, so we no longer need a separate hugetlbfs_read_iter(). [1] https://lore.kernel.org/linux-mm/aeZwAz6PcdlqSnJ2@casper.infradead.org/ Suggested-by: Matthew Wilcox Signed-off-by: Jane Chu Signed-off-by: Matthew Wilcox (Oracle) --- fs/hugetlbfs/inode.c | 109 +------------------------------------------ 1 file changed, 1 insertion(+), 108 deletions(-) diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 725bb945a440..a6f0459d79de 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -187,113 +187,6 @@ hugetlb_get_unmapped_area(struct file *file, unsigned long addr, return mm_get_unmapped_area_vmflags(file, addr0, len, pgoff, flags, 0); } -/* - * Someone wants to read @bytes from a HWPOISON hugetlb @folio from @offset. - * Returns the maximum number of bytes one can read without touching the 1st raw - * HWPOISON page. - */ -static size_t adjust_range_hwpoison(struct folio *folio, size_t offset, - size_t bytes) -{ - struct page *page = folio_page(folio, offset / PAGE_SIZE); - size_t safe_bytes; - - if (hugetlb_page_hwpoison(folio, page)) - return 0; - /* Safe to read the remaining bytes in this page. */ - safe_bytes = PAGE_SIZE - (offset % PAGE_SIZE); - page++; - - /* Check each remaining page as long as we are not done yet. */ - for (; safe_bytes < bytes; safe_bytes += PAGE_SIZE, page++) - if (hugetlb_page_hwpoison(folio, page)) - break; - - return min(safe_bytes, bytes); -} - -/* - * Support for read() - Find the page attached to f_mapping and copy out the - * data. This provides functionality similar to filemap_read(). - */ -static ssize_t hugetlbfs_read_iter(struct kiocb *iocb, struct iov_iter *to) -{ - struct file *file = iocb->ki_filp; - struct hstate *h = hstate_file(file); - struct address_space *mapping = file->f_mapping; - struct inode *inode = mapping->host; - unsigned long index = iocb->ki_pos >> huge_page_shift(h); - unsigned long offset = iocb->ki_pos & ~huge_page_mask(h); - unsigned long end_index; - loff_t isize; - ssize_t retval = 0; - - while (iov_iter_count(to)) { - struct folio *folio; - size_t nr, copied, want; - - /* nr is the maximum number of bytes to copy from this page */ - nr = huge_page_size(h); - isize = i_size_read(inode); - if (!isize) - break; - end_index = (isize - 1) >> huge_page_shift(h); - if (index > end_index) - break; - if (index == end_index) { - nr = ((isize - 1) & ~huge_page_mask(h)) + 1; - if (nr <= offset) - break; - } - nr = nr - offset; - - /* Find the folio */ - folio = filemap_lock_hugetlb_folio(h, mapping, index); - if (IS_ERR(folio)) { - /* - * We have a HOLE, zero out the user-buffer for the - * length of the hole or request. - */ - copied = iov_iter_zero(nr, to); - } else { - folio_unlock(folio); - - if (!folio_test_hwpoison(folio)) - want = nr; - else { - /* - * Adjust how many bytes safe to read without - * touching the 1st raw HWPOISON page after - * offset. - */ - want = adjust_range_hwpoison(folio, offset, nr); - if (want == 0) { - folio_put(folio); - retval = -EIO; - break; - } - } - - /* - * We have the folio, copy it to user space buffer. - */ - copied = copy_folio_to_iter(folio, offset, want, to); - folio_put(folio); - } - offset += copied; - retval += copied; - if (copied != nr && iov_iter_count(to)) { - if (!retval) - retval = -EFAULT; - break; - } - index += offset >> huge_page_shift(h); - offset &= ~huge_page_mask(h); - } - iocb->ki_pos = ((loff_t)index << huge_page_shift(h)) + offset; - return retval; -} - static int hugetlbfs_write_begin(const struct kiocb *iocb, struct address_space *mapping, loff_t pos, unsigned len, @@ -1215,7 +1108,7 @@ static void init_once(void *foo) } static const struct file_operations hugetlbfs_file_operations = { - .read_iter = hugetlbfs_read_iter, + .read_iter = generic_file_read_iter, .mmap = hugetlbfs_file_mmap, .fsync = noop_fsync, .get_unmapped_area = hugetlb_get_unmapped_area, -- 2.47.3