The current atomic-based page refcount implementation treats zero counter as dead and requires a compare-and-swap loop in folio_try_get() to prevent incrementing a dead refcount. This CAS loop acts as a serialization point and can become a significant bottleneck during high-frequency file read operations. This patch introduces PAGEREF_FROZEN_BIT to distinguish between a (temporary) zero refcount and a locked (dead/frozen) state. Because now incrementing counter doesn't affect it's locked/unlocked state, it is possible to use an optimistic atomic_add_return() in page_ref_add_unless_zero() that operates independently of the locked bit. The locked state is handled after the increment attempt, eliminating the need for the CAS loop. If locked state is detected after atomic_add(), pageref counter will be reset with CAS loop, eliminating theoretical possibility of overflow. Reviewed-by: Artem Kuzin Co-developed-by: Gorbunov Ivan Signed-off-by: Gorbunov Ivan Signed-off-by: Gladyshev Ilya Acked-by: Linus Torvalds --- include/linux/page-flags.h | 13 +++++++++++++ include/linux/page_ref.h | 28 ++++++++++++++++++++++++---- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 7223f6f4e2b4..ea9904a67334 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -196,6 +196,19 @@ enum pageflags { #define PAGEFLAGS_MASK ((1UL << NR_PAGEFLAGS) - 1) +/* Most significant bit in page refcount */ +#define PAGEREF_FROZEN_BIT BIT(31) + +/* Page reference counter can be in 4 logical states, + * which are described below with their value representation + * state | value + * (1) safe with owners | 1...INT_MAX + * (2) safe with no owners | 0 + * (3) frozen | INT_MIN....-1 + * + * State (2) can be only temporally inside dec_and_test. + */ + #ifndef __GENERATING_BOUNDS_H /* diff --git a/include/linux/page_ref.h b/include/linux/page_ref.h index 24b09c8fbb68..b041894b6659 100644 --- a/include/linux/page_ref.h +++ b/include/linux/page_ref.h @@ -64,12 +64,17 @@ static inline void __page_ref_unfreeze(struct page *page, int v) static inline bool __page_count_is_frozen(int count) { - return count == 0; + return count & PAGEREF_FROZEN_BIT; } static inline int page_ref_count(const struct page *page) { - return atomic_read(&page->_refcount); + int val = atomic_read(&page->_refcount); + + if (unlikely(val & PAGEREF_FROZEN_BIT)) + return 0; + + return val; } /** @@ -191,6 +196,9 @@ static inline int page_ref_sub_and_test(struct page *page, int nr) { int ret = atomic_sub_and_test(nr, &page->_refcount); + if (ret) + ret = !atomic_cmpxchg_relaxed(&page->_refcount, 0, PAGEREF_FROZEN_BIT); + if (page_ref_tracepoint_active(page_ref_mod_and_test)) __page_ref_mod_and_test(page, -nr, ret); return ret; @@ -220,6 +228,9 @@ static inline int page_ref_dec_and_test(struct page *page) { int ret = atomic_dec_and_test(&page->_refcount); + if (ret) + ret = !atomic_cmpxchg_relaxed(&page->_refcount, 0, PAGEREF_FROZEN_BIT); + if (page_ref_tracepoint_active(page_ref_mod_and_test)) __page_ref_mod_and_test(page, -1, ret); return ret; @@ -245,9 +256,18 @@ static inline int folio_ref_dec_return(struct folio *folio) return page_ref_dec_return(&folio->page); } +#define _PAGEREF_FROZEN_LIMIT ((1 << 30) | PAGEREF_FROZEN_BIT) + static inline bool page_ref_add_unless_frozen(struct page *page, int nr) { - bool ret = atomic_add_unless(&page->_refcount, nr, 0); + bool ret = false; + int val = atomic_add_return(nr, &page->_refcount); + // See PAGEREF_FROZEN_BIT declaration in page-flags.h for details + ret = !(val & PAGEREF_FROZEN_BIT); + + /* Undo atomic_add() if counter is locked and scary big */ + while (unlikely((unsigned int)val >= _PAGEREF_FROZEN_LIMIT)) + val = atomic_cmpxchg_relaxed(&page->_refcount, val, PAGEREF_FROZEN_BIT); if (page_ref_tracepoint_active(page_ref_mod_unless)) __page_ref_mod_unless(page, nr, ret); @@ -282,7 +302,7 @@ static inline bool folio_ref_try_add(struct folio *folio, int count) static inline int page_ref_freeze(struct page *page, int count) { - int ret = likely(atomic_cmpxchg(&page->_refcount, count, 0) == count); + int ret = likely(atomic_cmpxchg(&page->_refcount, count, PAGEREF_FROZEN_BIT) == count); if (page_ref_tracepoint_active(page_ref_freeze)) __page_ref_freeze(page, count, ret); -- 2.43.0