Let's add a folio_ref_count_over_limit() check in fast and slow GUP before acquiring additional references or pins. Keep it simple and use the check for all folio types. Also propagate try_grab_folio() errors when grabbing additional references in __get_user_pages() and in memfd_pin_folios(), so hitting the limit returns -ENOMEM instead of -EFAULT and -EINVAL, respectively. Keep the existing invalid-refcount warnings. Fixes: 800d8c63b2e9 ("shmem: add huge pages support") Fixes: 99cb0dbd47a1 ("mm,thp: add read-only THP support for (non-shmem) FS") Fixes: 793917d997df ("mm/readahead: Add large folio readahead") Cc: stable@vger.kernel.org Reviewed-by: Jose A. Perez de Azpillaga Signed-off-by: David Hildenbrand (Arm) --- include/linux/mm.h | 11 +++++------ mm/gup.c | 24 +++++++++++++++--------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index a36ff83589c61..9b11af88de2b9 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2244,6 +2244,9 @@ bool __folio_ref_count_over_limit(unsigned int ref_count); * to pagecache folios (!hugetlb), either before acquiring references or * afterwards with rollback if the operation is refused. * + * Also required when pinning folios without a dedicated pincount that use + * GUP_PIN_COUNTING_BIAS. + * * Checking other folios does no harm. * * This check is an approximation; we leave some headroom such that concurrent @@ -2951,12 +2954,8 @@ static inline bool folio_maybe_dma_pinned(struct folio *folio) return atomic_read(&folio->_pincount) > 0; /* - * folio_ref_count() is signed. If that refcount overflows, then - * folio_ref_count() returns a negative value, and callers will avoid - * further incrementing the refcount. - * - * Here, for that overflow case, use the sign bit to count a little - * bit higher via unsigned math, and thus still get an accurate result. + * Use unsigned math so even an unexpected negative refcount is + * treated as possibly pinned. */ return ((unsigned int)folio_ref_count(folio)) >= GUP_PIN_COUNTING_BIAS; diff --git a/mm/gup.c b/mm/gup.c index ebbfee09b7278..531ef2469228e 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -79,6 +79,8 @@ static inline struct folio *try_get_folio(struct page *page, int refs) folio = page_folio(page); if (WARN_ON_ONCE(folio_ref_count(folio) < 0)) return NULL; + if (unlikely(folio_ref_count_over_limit(folio))) + return NULL; if (unlikely(!folio_ref_try_add(folio, refs))) return NULL; @@ -143,6 +145,10 @@ int __must_check try_grab_folio(struct folio *folio, int refs, if (WARN_ON_ONCE(folio_ref_count(folio) <= 0)) return -ENOMEM; + if ((flags & (FOLL_GET | FOLL_PIN)) && + unlikely(folio_ref_count_over_limit(folio))) + return -ENOMEM; + if (unlikely(!(flags & FOLL_PCI_P2PDMA) && folio_is_pci_p2pdma(folio))) return -EREMOTEIO; @@ -1482,17 +1488,17 @@ static long __get_user_pages(struct mm_struct *mm, struct folio *folio = page_folio(page); /* - * Since we already hold refcount on the - * large folio, this should never fail. + * Grabbing more references can fail once we + * hit folio refcount limits. */ - if (try_grab_folio(folio, page_increm - 1, - gup_flags)) { + ret = try_grab_folio(folio, page_increm - 1, + gup_flags); + if (ret) { /* - * Release the 1st page ref if the - * folio is problematic, fail hard. + * Release the folio reference we + * already took. */ gup_put_folio(folio, 1, gup_flags); - ret = -EFAULT; goto out; } } @@ -3541,10 +3547,10 @@ long memfd_pin_folios(struct file *memfd, loff_t start, loff_t end, break; } - if (try_grab_folio(folio, 1, FOLL_PIN)) { + ret = try_grab_folio(folio, 1, FOLL_PIN); + if (ret) { folio_unlock(folio); folio_batch_release(&fbatch); - ret = -EINVAL; goto err; } -- 2.56.0