From: Kairui Song Introduce folio_inc_lru_refs_fast(): raise refs from 0 to LRU_REFS_REFERENCED via cmpxchg, or just set PG_referenced for the classical LRU as before. No counter update, promotion, and guarteens no other complex operations in following commits. Convert the gup fast-path sites. No behavior change for classical LRU. Tested-by: zhaozhengzhuo Signed-off-by: Kairui Song --- include/linux/mm_inline.h | 25 +++++++++++++++++++++++++ mm/gup.c | 6 +++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 65415eeac802..effa13c63632 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -454,6 +454,31 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, } #endif /* CONFIG_LRU_GEN */ +/** + * folio_inc_lru_refs_fast - Bump folio refs count without promotion. + * @folio: the folio + * + * Raise refs from 0 to LRU_REFS_REFERENCED and leave hotter folios + * untouched. For the classical LRU, a plain PG_referenced set. + */ +static __always_inline void folio_inc_lru_refs_fast(struct folio *folio) +{ + unsigned long new_flags, old_flags; + + if (!lru_gen_enabled()) { + folio_set_referenced(folio); + return; + } + + old_flags = READ_ONCE(*folio_flags(folio, 0)); + do { + if (lru_get_refs_flags(old_flags)) + break; + new_flags = old_flags; + lru_set_refs_flags(&new_flags, LRU_REFS_REFERENCED); + } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); +} + static __always_inline void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio) { diff --git a/mm/gup.c b/mm/gup.c index f166acf794e3..b3f563d41192 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -2896,7 +2896,7 @@ static unsigned long gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, gup_put_folio(folio, 1, flags); goto pte_unmap; } - folio_set_referenced(folio); + folio_inc_lru_refs_fast(folio); pages[nr_pages] = page; nr_pages++; } while (ptep++, addr += PAGE_SIZE, addr != end); @@ -2965,7 +2965,7 @@ static unsigned long gup_fast_pmd_leaf(pmd_t orig, pmd_t *pmdp, for (i = 0; i < nr_pages; i++) *(pages++) = page++; - folio_set_referenced(folio); + folio_inc_lru_refs_fast(folio); return nr_pages; } @@ -3007,7 +3007,7 @@ static unsigned long gup_fast_pud_leaf(pud_t orig, pud_t *pudp, for (i = 0; i < nr_pages; i++) *(pages++) = page++; - folio_set_referenced(folio); + folio_inc_lru_refs_fast(folio); return nr_pages; } -- 2.56.0