Other large folios use the has_hwpoisoned flag. Convert hugetlb to match. This will help us use the per-page hwpoison flag in the future. Also introduce a folio_test_huge_poison(). This has exactly the same meaning as folio_test_has_hwpoisoned() but can be used when we don't have a reference to the folio containing the page. folio_test_has_hwpoisoned() can race with folio splitting / reallocation and trip the assertions in const_folio_flags(). This closes a gap where a page in a previously-poisoned hugetlb folio could be observed to not have the hwpoison bit set. Signed-off-by: Matthew Wilcox (Oracle) --- fs/Kconfig | 2 +- fs/hugetlbfs/inode.c | 2 +- include/linux/page-flags.h | 46 ++++++++++++++++++++++++------ mm/Kconfig | 6 +++- mm/hugetlb.c | 10 +++---- mm/memory-failure.c | 58 +++++++++++++++++++++++++++----------- mm/rmap.c | 43 +++++++++++++++++----------- 7 files changed, 119 insertions(+), 48 deletions(-) diff --git a/fs/Kconfig b/fs/Kconfig index cf6ae64776e6..eddac4ed214b 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -272,7 +272,7 @@ endif # HUGETLBFS config HUGETLB_PAGE def_bool HUGETLBFS - select XARRAY_MULTI + select LARGE_FOLIO config HUGETLB_PAGE_OPTIMIZE_VMEMMAP def_bool HUGETLB_PAGE diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 216e1a0dd0b2..fbac554886c3 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -258,7 +258,7 @@ static ssize_t hugetlbfs_read_iter(struct kiocb *iocb, struct iov_iter *to) } else { folio_unlock(folio); - if (!folio_test_hwpoison(folio)) + if (!folio_test_has_hwpoisoned(folio)) want = nr; else { /* diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 4185a03a45cf..4fab3fbfd430 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -893,14 +893,19 @@ static inline int PageTransCompound(const struct page *page) TESTPAGEFLAG_FALSE(TransCompound, transcompound) #endif -#if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_TRANSPARENT_HUGEPAGE) +#if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_LARGE_FOLIO) /* - * PageHasHWPoisoned indicates that at least one subpage is hwpoisoned in the - * compound page. + * folio_has_hwpoisoned indicates that at least one page is hwpoisoned in the + * folio. That page will usually also have the HWPoison flag set, but this + * is not possible for folios which have HVO (see memory-failure for the + * scheme used in that case). You probably don't want to call this directly; + * use folio_has_hwpoisoned_page() instead. * * This flag is set by hwpoison handler. Cleared by THP split or free page. */ FOLIO_FLAG(has_hwpoisoned, FOLIO_SECOND_PAGE) +FOLIO_TEST_SET_FLAG(has_hwpoisoned, FOLIO_SECOND_PAGE) +FOLIO_TEST_CLEAR_FLAG(has_hwpoisoned, FOLIO_SECOND_PAGE) #else FOLIO_FLAG_FALSE(has_hwpoisoned) #endif @@ -1041,8 +1046,29 @@ PAGE_TYPE_OPS(Slab, slab, slab) #ifdef CONFIG_HUGETLB_PAGE FOLIO_TYPE_OPS(hugetlb, hugetlb) + +#ifdef CONFIG_MEMORY_FAILURE +static inline bool folio_test_huge_poison(const struct folio *folio) +{ + return (READ_ONCE(folio->page.page_type) >> 23) == + ((PGTY_hugetlb << 1) | 1); +} + +static inline void folio_set_huge_poison(struct folio *folio) +{ + folio->page.page_type |= (1 << 23); +} + +static inline void folio_clear_huge_poison(struct folio *folio) +{ + folio->page.page_type &= ~(1 << 23); +} +#else +FOLIO_TEST_FLAG_FALSE(huge_poison) +#endif #else FOLIO_TEST_FLAG_FALSE(hugetlb) +FOLIO_TEST_FLAG_FALSE(huge_poison) #endif PAGE_TYPE_OPS(Zsmalloc, zsmalloc, zsmalloc) @@ -1069,9 +1095,10 @@ static inline bool PageHuge(const struct page *page) } /* - * Check if a page is currently marked HWPoisoned. Note that this check is - * best effort only and inherently racy: there is no way to synchronize with - * failing hardware. + * Check if a page is currently marked HWPoisoned. This check is best + * effort only and inherently racy: there is no way to synchronize with + * failing hardware. The caller may not have a refcount on the folio + * containing the page, so we must be careful to not trip any assertions. */ static inline bool is_page_hwpoison(const struct page *page) { @@ -1080,12 +1107,15 @@ static inline bool is_page_hwpoison(const struct page *page) if (PageHWPoison(page)) return true; folio = page_folio(page); - return folio_test_hugetlb(folio) && PageHWPoison(&folio->page); + if (folio_test_huge_poison(folio)) + return true; + /* In case we raced with hugetlb transferring flags */ + return PageHWPoison(page); } static inline bool folio_has_hwpoisoned_page(const struct folio *folio) { - return folio_test_hwpoison(folio) || + return PageHWPoison(&folio->page) || (folio_test_large(folio) && folio_test_has_hwpoisoned(folio)); } diff --git a/mm/Kconfig b/mm/Kconfig index 9e0ca4824905..e666dd14ca0c 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -843,11 +843,15 @@ config PERSISTENT_HUGE_ZERO_FOLIO config MM_ID def_bool n +config LARGE_FOLIO + def_bool n + select XARRAY_MULTI + menuconfig TRANSPARENT_HUGEPAGE bool "Transparent Hugepage Support" depends on HAVE_ARCH_TRANSPARENT_HUGEPAGE && !PREEMPT_RT select COMPACTION - select XARRAY_MULTI + select LARGE_FOLIO select MM_ID help Transparent Hugepages allows the kernel to use huge pages and diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 47403d02be88..40ae967b9ecc 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1255,7 +1255,7 @@ static struct folio *dequeue_hugetlb_folio_node_exact(struct hstate *h, if (pin && !folio_is_longterm_pinnable(folio)) continue; - if (folio_test_hwpoison(folio)) + if (folio_test_has_hwpoisoned(folio)) continue; if (is_migrate_isolate_page(&folio->page)) @@ -1387,7 +1387,7 @@ static void folio_clear_hugetlb(struct folio *folio) * Move HWPoison flag to each error page * which makes any healthy pages reusable. */ - if (unlikely(folio_test_hwpoison(folio))) + if (unlikely(folio_test_has_hwpoisoned(folio))) folio_clear_hugetlb_hwpoison(folio); __folio_clear_hugetlb(folio); @@ -4003,7 +4003,7 @@ long demote_pool_huge_page(struct hstate *src, nodemask_t *nodes_allowed, struct folio *folio, *next; list_for_each_entry_safe(folio, next, &src->hugepage_freelists[node], lru) { - if (folio_test_hwpoison(folio)) + if (folio_test_has_hwpoisoned(folio)) continue; remove_hugetlb_folio(src, folio, false); @@ -5814,7 +5814,7 @@ static vm_fault_t hugetlb_no_page(struct address_space *mapping, * don't have hwpoisoned swap entry for errored virtual address. * So we need to block hugepage fault by PG_hwpoison bit check. */ - if (unlikely(folio_test_hwpoison(folio))) { + if (unlikely(folio_test_has_hwpoisoned(folio))) { ret = VM_FAULT_HWPOISON_LARGE | VM_FAULT_SET_HINDEX(hstate_index(h)); goto backout_unlocked; @@ -6323,7 +6323,7 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte, ptl = huge_pte_lock(h, dst_mm, dst_pte); ret = -EIO; - if (folio_test_hwpoison(folio)) + if (folio_test_has_hwpoisoned(folio)) goto out_release_unlock; ret = -EEXIST; diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 1dd0e7b99bb1..714e1b398f2c 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -1829,7 +1829,7 @@ bool is_raw_hwpoison_page_in_hugepage(struct page *page) struct folio *folio = page_folio(page); bool ret = false; - if (!folio_test_hwpoison(folio)) + if (!folio_test_has_hwpoisoned(folio)) return false; if (!folio_test_hugetlb(folio)) @@ -1881,6 +1881,24 @@ static unsigned long __folio_free_raw_hwp(struct folio *folio, bool move_flag) #define MF_HUGETLB_FOLIO_PRE_POISONED 3 /* folio already poisoned */ #define MF_HUGETLB_PAGE_PRE_POISONED 4 /* exact page already poisoned */ #define MF_HUGETLB_RETRY 5 /* hugepage is busy, retry */ + +static inline int hugetlb_set_poison(struct folio *folio) +{ + if (folio_test_set_has_hwpoisoned(folio)) + return MF_HUGETLB_FOLIO_PRE_POISONED; + folio_set_huge_poison(folio); + return 0; +} + +static inline int hugetlb_clear_poison(struct folio *folio) +{ + if (!folio_test_has_hwpoisoned(folio)) + return -EBUSY; + folio_clear_huge_poison(folio); + folio_clear_has_hwpoisoned(folio); + return 0; +} + /* * Set hugetlb folio as hwpoisoned, update folio private raw hwpoison list * to keep track of the poisoned pages. @@ -1890,12 +1908,12 @@ static int hugetlb_update_hwpoison(struct folio *folio, struct page *page) struct llist_head *head; struct raw_hwp_page *raw_hwp; struct raw_hwp_page *p; - int ret = folio_test_set_hwpoison(folio) ? MF_HUGETLB_FOLIO_PRE_POISONED : 0; + int ret = hugetlb_set_poison(folio); /* * Once the hwpoison hugepage has lost reliable raw error info, - * there is little meaning to keep additional error info precisely, - * so skip to add additional raw error info. + * there is no point in keeping additional error info precisely, + * so skip adding additional raw error info. */ if (folio_test_hugetlb_raw_hwp_unreliable(folio)) return MF_HUGETLB_FOLIO_PRE_POISONED; @@ -1950,8 +1968,8 @@ void folio_clear_hugetlb_hwpoison(struct folio *folio) return; if (folio_test_hugetlb_vmemmap_optimized(folio)) return; - folio_clear_hwpoison(folio); folio_free_raw_hwp(folio, true); + folio_clear_has_hwpoisoned(folio); } static int get_huge_page_for_hwpoison(unsigned long pfn, int flags, @@ -2104,6 +2122,11 @@ static inline unsigned long folio_free_raw_hwp(struct folio *folio, bool flag) { return 0; } + +static inline int hugetlb_clear_poison(struct folio *folio) +{ + return 0; +} #endif /* CONFIG_HUGETLB_PAGE */ /* Drop the extra refcount in case we come from madvise() */ @@ -2733,8 +2756,10 @@ int unpoison_memory(unsigned long pfn) hugetlb_unlock_irq(); goto unlock_mutex; } + ret = hugetlb_clear_poison(folio); + } else { + ret = TestClearPageHWPoison(p) ? 0 : -EBUSY; } - ret = folio_test_clear_hwpoison(folio) ? 0 : -EBUSY; hugetlb_unlock_irq(); } else if (ghp < 0) { if (ghp == -EHWPOISON) { @@ -2744,17 +2769,18 @@ int unpoison_memory(unsigned long pfn) unpoison_pr_info("%#lx: failed to grab page\n", pfn, &unpoison_rs); } - } else { - if (folio_test_hugetlb(folio)) { - huge = true; - count = folio_free_raw_hwp(folio, false); - if (count == 0) { - folio_put(folio); - goto unlock_mutex; - } - p = &folio->page; + } else if (folio_test_hugetlb(folio)) { + huge = true; + count = folio_free_raw_hwp(folio, false); + if (count == 0) { + folio_put(folio); + goto unlock_mutex; } - + ret = hugetlb_clear_poison(folio); + folio_put(folio); + if (!ret) + folio_put(folio); + } else { folio_put(folio); if (TestClearPageHWPoison(p)) { folio_put(folio); diff --git a/mm/rmap.c b/mm/rmap.c index 1c77d5dc06e9..fd19a0bfbfe7 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1978,6 +1978,22 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio, FPB_RESPECT_WRITE | FPB_RESPECT_SOFT_DIRTY); } +/* + * Since we cannot split a hugetlb folio, we want to insert a poison + * entry into the page table for the whole folio even if only one page + * is poisoned. Otherwise, we've split down to the PTE level and we only + * want to poison the precise page + */ +static bool ttu_create_hwpoison(const struct folio *folio, + const struct page *page, enum ttu_flags flags) +{ + if (!(flags & TTU_HWPOISON)) + return false; + if (folio_test_hugetlb(folio)) + return folio_test_has_hwpoisoned(folio); + return PageHWPoison(page); +} + /* * @arg: enum ttu_flags will be passed to this argument */ @@ -1993,7 +2009,6 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, enum ttu_flags flags = (enum ttu_flags)(long)arg; unsigned long nr_pages = 1, end_addr; unsigned long pfn; - unsigned long hsz = 0; int ptes = 0; /* @@ -2023,9 +2038,6 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, */ adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end); - - /* We need the huge page size for set_huge_pte_at() */ - hsz = huge_page_size(hstate_vma(vma)); } mmu_notifier_invalidate_range_start(&range); @@ -2121,7 +2133,8 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, * The try_to_unmap() is only passed a hugetlb page * in the case where the hugetlb page is poisoned. */ - VM_BUG_ON_PAGE(!PageHWPoison(subpage), subpage); + VM_BUG_ON_FOLIO(!folio_has_hwpoisoned_page(folio), + folio); /* * huge_pmd_unshare may unmap an entire PMD page. * There is no way of knowing exactly which PMDs may @@ -2200,12 +2213,12 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, /* Update high watermark before we lower rss */ update_hiwater_rss(mm); - if (PageHWPoison(subpage) && (flags & TTU_HWPOISON)) { + if (ttu_create_hwpoison(folio, subpage, flags)) { pteval = swp_entry_to_pte(make_hwpoison_entry(subpage)); if (folio_test_hugetlb(folio)) { hugetlb_count_sub(folio_nr_pages(folio), mm); set_huge_pte_at(mm, address, pvmw.pte, pteval, - hsz); + folio_size(folio)); } else { dec_mm_counter(mm, mm_counter(folio)); set_pte_at(mm, address, pvmw.pte, pteval); @@ -2423,7 +2436,6 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, struct mmu_notifier_range range; enum ttu_flags flags = (enum ttu_flags)(long)arg; unsigned long pfn; - unsigned long hsz = 0; /* * When racing against e.g. zap_pte_range() on another cpu, @@ -2452,9 +2464,6 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, */ adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end); - - /* We need the huge page size for set_huge_pte_at() */ - hsz = huge_page_size(hstate_vma(vma)); } mmu_notifier_invalidate_range_start(&range); @@ -2607,14 +2616,14 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, /* Update high watermark before we lower rss */ update_hiwater_rss(mm); - if (PageHWPoison(subpage)) { + if (ttu_create_hwpoison(folio, subpage, TTU_HWPOISON)) { VM_WARN_ON_FOLIO(folio_is_device_private(folio), folio); pteval = swp_entry_to_pte(make_hwpoison_entry(subpage)); if (folio_test_hugetlb(folio)) { hugetlb_count_sub(folio_nr_pages(folio), mm); set_huge_pte_at(mm, address, pvmw.pte, pteval, - hsz); + folio_size(folio)); } else { dec_mm_counter(mm, mm_counter(folio)); set_pte_at(mm, address, pvmw.pte, pteval); @@ -2644,7 +2653,8 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, if (arch_unmap_one(mm, vma, address, pteval) < 0) { if (folio_test_hugetlb(folio)) set_huge_pte_at(mm, address, pvmw.pte, - pteval, hsz); + pteval, + folio_size(folio)); else set_pte_at(mm, address, pvmw.pte, pteval); ret = false; @@ -2657,7 +2667,8 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, if (anon_exclusive && hugetlb_try_share_anon_rmap(folio)) { set_huge_pte_at(mm, address, pvmw.pte, - pteval, hsz); + pteval, + folio_size(folio)); ret = false; page_vma_mapped_walk_done(&pvmw); break; @@ -2703,7 +2714,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, } if (folio_test_hugetlb(folio)) set_huge_pte_at(mm, address, pvmw.pte, swp_pte, - hsz); + folio_size(folio)); else set_pte_at(mm, address, pvmw.pte, swp_pte); trace_set_migration_pte(address, pte_val(swp_pte), -- 2.47.3