Hugetlb migration might have to allocate a temporary folio through alloc_migrate_hugetlb_folio(). Such folios are currently accounted in "nr_huge_pages" but not in "surplus_huge_pages". persistent_huge_pages() can mistake them as persistent huge pages, as it merely calculates the difference. If set_max_huge_pages() shrinks the pool at the wrong time, it will turn hugetlb folios into surplus folios until persistent_huge_pages() returns the desired number. After the migration (successful or unsuccessful) either the new (temporary) folio or the old folio gets freed: this will reduce "nr_huge_pages" but not "surplus_huge_pages", to undo what was done when allocating a temporary folio. This can result in having more surplus folios than actually allocated folios: HugePages_Total: 0 HugePages_Surp: 1 From then on persistent_huge_pages() returns garbage. To fix it, add internal temporary counters and subtract them when calculating persistent pool sizes. Update persistent_huge_pages() and add persistent_huge_pages_node() to make the code easier to read. Count migration targets and rejected surplus allocations as temporary. Transfer the per-node temporary count along with the temporary flag on migration, such that folio freeing will always do the right thing. Take care of calling account_new_hugetlb_folio() only after setting the folios to be temporary. For now, keep temporary folios in nr_huge_pages, so HugePages_Total and Hugetlb continue to include them and overcommit accounting remains unchanged. The new temporary counters are internal for now and are not exported to userspace. Fixes: ab5ac90aecf5 ("mm, hugetlb: do not rely on overcommit limit during migration") Cc: stable@kernel.org Reported-by: Josef Bacik Signed-off-by: David Hildenbrand (Arm) --- include/linux/hugetlb.h | 3 +++ mm/hugetlb.c | 65 +++++++++++++++++++++++++++++-------------------- 2 files changed, 42 insertions(+), 26 deletions(-) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 5029c7241863..d49cb15d13e6 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -701,6 +701,8 @@ struct hstate { unsigned long free_huge_pages; unsigned long resv_huge_pages; unsigned long surplus_huge_pages; + /* Included in nr_huge_pages; not exported separately to userspace. */ + unsigned long temporary_huge_pages; unsigned long nr_overcommit_huge_pages; struct list_head hugepage_activelist; struct list_head hugepage_freelists[MAX_NUMNODES]; @@ -708,6 +710,7 @@ struct hstate { unsigned int nr_huge_pages_node[MAX_NUMNODES]; unsigned int free_huge_pages_node[MAX_NUMNODES]; unsigned int surplus_huge_pages_node[MAX_NUMNODES]; + unsigned int temporary_huge_pages_node[MAX_NUMNODES]; char name[HSTATE_NAME_LEN]; }; diff --git a/mm/hugetlb.c b/mm/hugetlb.c index a78b2487c20c..110a5c2d9e50 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -115,7 +115,7 @@ early_param(str, func##args) /* * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages, - * free_huge_pages, and surplus_huge_pages. + * free_huge_pages, surplus_huge_pages, and temporary_huge_pages. */ __cacheline_aligned_in_smp DEFINE_SPINLOCK(hugetlb_lock); @@ -1337,7 +1337,13 @@ static unsigned long available_huge_pages(struct hstate *h) static unsigned long persistent_huge_pages(struct hstate *h) { - return h->nr_huge_pages - h->surplus_huge_pages; + return h->nr_huge_pages - h->surplus_huge_pages - h->temporary_huge_pages; +} + +static unsigned long persistent_huge_pages_node(struct hstate *h, int nid) +{ + return h->nr_huge_pages_node[nid] - h->surplus_huge_pages_node[nid] - + h->temporary_huge_pages_node[nid]; } static struct folio *dequeue_hugetlb_folio(struct hstate *h, gfp_t gfp_mask, @@ -1422,6 +1428,10 @@ void remove_hugetlb_folio(struct hstate *h, struct folio *folio, h->surplus_huge_pages--; h->surplus_huge_pages_node[nid]--; } + if (folio_test_hugetlb_temporary(folio)) { + h->temporary_huge_pages--; + h->temporary_huge_pages_node[nid]--; + } /* * We can only clear the hugetlb flag after allocating vmemmap @@ -1764,9 +1774,15 @@ void free_huge_folio(struct folio *folio) */ static void account_new_hugetlb_folio(struct hstate *h, struct folio *folio) { + const int nid = folio_nid(folio); + lockdep_assert_held(&hugetlb_lock); h->nr_huge_pages++; - h->nr_huge_pages_node[folio_nid(folio)]++; + h->nr_huge_pages_node[nid]++; + if (folio_test_hugetlb_temporary(folio)) { + h->temporary_huge_pages++; + h->temporary_huge_pages_node[nid]++; + } } void init_new_hugetlb_folio(struct folio *folio) @@ -2113,13 +2129,6 @@ static struct folio *alloc_surplus_hugetlb_folio(struct hstate *h, return NULL; spin_lock_irq(&hugetlb_lock); - /* - * nr_huge_pages needs to be adjusted within the same lock cycle - * as surplus_pages, otherwise it might confuse - * persistent_huge_pages() momentarily. - */ - account_new_hugetlb_folio(h, folio); - /* * We could have raced with the pool size change. * Double check that and simply deallocate the new page @@ -2129,11 +2138,18 @@ static struct folio *alloc_surplus_hugetlb_folio(struct hstate *h, */ if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) { folio_set_hugetlb_temporary(folio); + account_new_hugetlb_folio(h, folio); spin_unlock_irq(&hugetlb_lock); free_huge_folio(folio); return NULL; } + /* + * nr_huge_pages needs to be adjusted within the same lock cycle + * as surplus_pages, otherwise it might confuse + * persistent_huge_pages() momentarily. + */ + account_new_hugetlb_folio(h, folio); h->surplus_huge_pages++; h->surplus_huge_pages_node[folio_nid(folio)]++; @@ -2155,17 +2171,17 @@ static struct folio *alloc_migrate_hugetlb_folio(struct hstate *h, gfp_t gfp_mas if (!folio) return NULL; + /* + * Count temporary folios separately so that pool resizing does not + * mistake them for persistent pages. + */ spin_lock_irq(&hugetlb_lock); + folio_set_hugetlb_temporary(folio); account_new_hugetlb_folio(h, folio); spin_unlock_irq(&hugetlb_lock); /* fresh huge pages are frozen */ folio_ref_unfreeze(folio, 1); - /* - * We do not account these pages as surplus because they are only - * temporary and will be released properly on the last reference - */ - folio_set_hugetlb_temporary(folio); return folio; } @@ -3757,8 +3773,7 @@ static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed, } } else { for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) { - if (h->surplus_huge_pages_node[node] < - h->nr_huge_pages_node[node]) + if (persistent_huge_pages_node(h, node)) goto found; } } @@ -3807,9 +3822,7 @@ static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid, if (nid != NUMA_NO_NODE) { unsigned long old_count = count; - count += persistent_huge_pages(h) - - (h->nr_huge_pages_node[nid] - - h->surplus_huge_pages_node[nid]); + count += persistent_huge_pages(h) - persistent_huge_pages_node(h, nid); /* * User may have specified a large count value which caused the * above calculation to overflow. In this case, they wanted @@ -7440,9 +7453,8 @@ void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, * be final while the old one will be freed so it takes over * the temporary status. * - * Also note that we have to transfer the per-node surplus state - * here as well otherwise the global surplus count will not match - * the per-node's. + * Transfer the per-node temporary count to the old folio's node, + * and any surplus count in the opposite direction. */ if (folio_test_hugetlb_temporary(new_folio)) { int old_nid = folio_nid(old_folio); @@ -7451,13 +7463,14 @@ void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, folio_set_hugetlb_temporary(old_folio); folio_clear_hugetlb_temporary(new_folio); - /* - * There is no need to transfer the per-node surplus state - * when we do not cross the node. + * There is no need to transfer the per-node state when we do + * not cross the node. */ if (new_nid != old_nid) { spin_lock_irq(&hugetlb_lock); + h->temporary_huge_pages_node[new_nid]--; + h->temporary_huge_pages_node[old_nid]++; if (h->surplus_huge_pages_node[old_nid]) { h->surplus_huge_pages_node[old_nid]--; h->surplus_huge_pages_node[new_nid]++; -- 2.43.0