From: Ackerley Tng hugepage_subpool_put_pages() currently has two distinct responsibilities that conflict: 1. When size is specified for the mount, max_hpages != -1: Keep track of total active pages (allocated + reserved) and decrement this count (used_hpages) when a page is freed or allocation fails. 2. When min_size is specified for the mount, min_hpages != -1: Ensure we don't drop below the guaranteed minimum, and restore a reservation (rsv_hpages) if we do. This causes trouble because when allocation fails (refer to alloc_hugetlb_folio()) if gbl_chg = 1 (i.e. no subpool reservation was taken): + To keep used_hpages consistent, HugeTLB needs to call hugepage_subpool_put_pages() to restore undo used_hpages being incremented + But can't call hugepage_subpool_put_pages() if no reservation was consumed. One option would be to conditionally do subpool tracking updates outside of the hugepage_subpool_put_pages() function, but that would spread logic all over. Instead, always track used_hpages, regardless of whether a max_size was requested for the mount, so that the subpool always knows how many pages were allocated through it. Every page allocated through the subpool increments used_hpages, regardless of whether a reservation was taken from it. Conceptually, now, every allocation involving a subpool uses a page from the subpool, which must be returned to the subpool. Every page taken from the subpool tries to use a subpool reservation. Restoring a page to the subpool reservations only if the page was taken from subpool reservations. (If used_hpages >= min_hpages, the page must have not have been taken from the reservations.) Always tracking used_hpages provides the subpool with information of both used and reserved counts to make the correct decision for both max_size and min_size correctly. With used_hpages always tracked, + subpool_is_free() can be simplified, such that the subpool can be declared free if there are no more pages in use. + open-coding in hugetlb_reserve_pages() can be removed. Also update the documentation for used_hpages, since it no longer matters whether the used pages count against the maximum. Also update statfs reporting. Previously, if max_hpages is negative, used_hpages is static at 0, so returning max_hpages - used_hpages returns -1 and is always correct. Now, if the subpool doesn't have a maximum requested size, indicate no limit for free pages (-1). If it does have a maximum size, report the difference between the requested size and the number of used pages. This difference is always positive, because if the mount does have a maximum size, hugepage_subpool_get_pages() ensures that the subpool usage never exceeds the maximum. This fixes a bug in hugetlb_unreserve_pages(), where pages are returned to the subpool regardless of whether it consumed a reservation. The corresponding bug in the failure handling path of alloc_hugetlb_folio() was fixed in a833a693a490e. Fixes: 1c5ecae3a93fa ("hugetlbfs: add minimum size accounting to subpools") Cc: stable@vger.kernel.org Signed-off-by: Ackerley Tng --- fs/hugetlbfs/inode.c | 8 ++++-- include/linux/hugetlb.h | 4 +-- mm/hugetlb.c | 71 +++++++++++++++++++++++-------------------------- 3 files changed, 41 insertions(+), 42 deletions(-) diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 216e1a0dd0b23..26c0187340636 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -1109,8 +1109,12 @@ static int hugetlbfs_statfs(struct dentry *dentry, struct kstatfs *buf) spin_lock_irq(&sbinfo->spool->lock); buf->f_blocks = sbinfo->spool->max_hpages; - free_pages = sbinfo->spool->max_hpages - - sbinfo->spool->used_hpages; + if (sbinfo->spool->max_hpages == -1) { + free_pages = -1; + } else { + free_pages = sbinfo->spool->max_hpages - + sbinfo->spool->used_hpages; + } buf->f_bavail = buf->f_bfree = free_pages; spin_unlock_irq(&sbinfo->spool->lock); buf->f_files = sbinfo->max_inodes; diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 2abaf99321e90..34b9a3e1be0fa 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -38,8 +38,8 @@ struct hugepage_subpool { spinlock_t lock; long count; long max_hpages; /* Maximum huge pages or -1 if no maximum. */ - long used_hpages; /* Used count against maximum, includes */ - /* both allocated and reserved pages. */ + long used_hpages; /* Used page count, includes both */ + /* allocated and reserved pages. */ struct hstate *hstate; long min_hpages; /* Minimum huge pages or -1 if no minimum. */ long rsv_hpages; /* Pages reserved against global pool to */ diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 571212b80835e..eef9610a0593c 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -129,12 +129,8 @@ static inline bool subpool_is_free(struct hugepage_subpool *spool) { if (spool->count) return false; - if (spool->max_hpages != -1) - return spool->used_hpages == 0; - if (spool->min_hpages != -1) - return spool->rsv_hpages == spool->min_hpages; - return true; + return spool->used_hpages == 0; } static inline void unlock_or_release_subpool(struct hugepage_subpool *spool, @@ -205,15 +201,14 @@ static long hugepage_subpool_get_pages(struct hugepage_subpool *spool, spin_lock_irq(&spool->lock); - if (spool->max_hpages != -1) { /* maximum size accounting */ - if ((spool->used_hpages + delta) <= spool->max_hpages) - spool->used_hpages += delta; - else { - ret = -ENOMEM; - goto unlock_ret; - } + if (spool->max_hpages != -1 && + spool->used_hpages + delta > spool->max_hpages) { + ret = -ENOMEM; + goto unlock_ret; } + spool->used_hpages += delta; + /* minimum size accounting */ if (spool->min_hpages != -1 && spool->rsv_hpages) { if (delta > spool->rsv_hpages) { @@ -251,19 +246,24 @@ static long hugepage_subpool_put_pages(struct hugepage_subpool *spool, spin_lock_irqsave(&spool->lock, flags); - if (spool->max_hpages != -1) /* maximum size accounting */ - spool->used_hpages -= delta; + spool->used_hpages -= delta; /* minimum size accounting */ if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) { - if (spool->rsv_hpages + delta <= spool->min_hpages) + /* + * limit is the maximum number of reservations that + * can be restored to this subpool. + */ + long limit = spool->min_hpages - spool->used_hpages; + + if (spool->rsv_hpages + delta <= limit) ret = 0; else - ret = spool->rsv_hpages + delta - spool->min_hpages; + ret = spool->rsv_hpages + delta - limit; spool->rsv_hpages += delta; - if (spool->rsv_hpages > spool->min_hpages) - spool->rsv_hpages = spool->min_hpages; + if (spool->rsv_hpages > limit) + spool->rsv_hpages = limit; } /* @@ -6542,7 +6542,7 @@ long hugetlb_reserve_pages(struct inode *inode, struct vm_area_struct *vma, vma_flags_t vma_flags) { - long chg = -1, add = -1, spool_resv, gbl_resv; + long chg = -1, add = -1, gbl_resv; struct hstate *h = hstate_inode(inode); struct hugepage_subpool *spool = subpool_inode(inode); struct resv_map *resv_map; @@ -6687,26 +6687,21 @@ long hugetlb_reserve_pages(struct inode *inode, } return chg; -out_put_pages: - spool_resv = chg - gbl_reserve; - if (spool_resv) { - /* put sub pool's reservation back, chg - gbl_reserve */ - gbl_resv = hugepage_subpool_put_pages(spool, spool_resv); - /* - * subpool's reserved pages can not be put back due to race, - * return to hstate. - */ - hugetlb_acct_memory(h, -gbl_resv); - } - /* Restore used_hpages for pages that failed global reservation */ - if (gbl_reserve && spool) { - unsigned long flags; + out_put_pages: + /* + * Return all that was requested from the subpool, let subpool + * tell us the new number of reservations that need to be + * returned to the global pool. + */ + gbl_resv = hugepage_subpool_put_pages(spool, chg); + /* + * There may be a difference between the number of + * reservations to consume and the number to restore now if + * there are multiple threads interacting with the subpool - + * restore the difference. + */ + hugetlb_acct_memory(h, -(gbl_resv - gbl_reserve)); - spin_lock_irqsave(&spool->lock, flags); - if (spool->max_hpages != -1) - spool->used_hpages -= gbl_reserve; - unlock_or_release_subpool(spool, flags); - } out_uncharge_cgroup: hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h), chg * pages_per_huge_page(h), h_cg); -- 2.55.0.229.g6434b31f56-goog