From: Kairui Song Instead of keep touching the raw page flags, use helpers for adjusting folio's refs and gen info, make the code easier to debug and understand. Signed-off-by: Kairui Song --- include/linux/mm_inline.h | 79 +++++++++++++++++++++++++++++++++++++++++------ include/linux/mmzone.h | 2 ++ mm/migrate.c | 2 -- mm/swap.c | 17 +++++----- mm/vmscan.c | 52 ++++++++++++++++++------------- 5 files changed, 111 insertions(+), 41 deletions(-) diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 9c8ad8af37de..eade9f2d6afc 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -141,10 +141,42 @@ static inline int lru_tier_from_refs(int refs, bool workingset) return workingset ? MAX_NR_TIERS - 1 : order_base_2(refs); } -static inline int folio_lru_refs(const struct folio *folio) +/** + * lru_gen_from_flags - Return the LRU generation number from folio flags. + * @flags: folio flags + * + * Returns: A number between 0 and LRU_GEN_MAX, inclusive. Returns -1 if the + * flags indicate the folio is off the list (e.g., isolated). + */ +static inline int lru_gen_from_flags(unsigned long flags) { - unsigned long flags = READ_ONCE(*const_folio_flags(folio, 0)); + int gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF); + gen -= 1; + VM_WARN_ON_ONCE(gen != -1 && gen > LRU_GEN_MAX); + return gen; +} + +/** + * lru_gen_set_flags - Set the LRU generation number to specified folio flags. + * @flags: pointer to the folio flags + * @gen: generation number, between 0 and LRU_GEN_MAX, inclusive. + */ +static inline void lru_gen_set_flags(unsigned long *flags, int gen) +{ + VM_WARN_ON_ONCE(gen > LRU_GEN_MAX || gen < 0); + BUILD_BUG_ON((LRU_GEN_MAX + 1) != MAX_NR_GENS); + + *flags &= ~LRU_GEN_MASK; + *flags |= (gen + 1UL) << LRU_GEN_PGOFF; +} + +/** + * lru_refs_from_flags - Return LRU referenced / access count from folio flags. + * @flags: folio flags + */ +static inline int lru_refs_from_flags(unsigned long flags) +{ if (!(flags & BIT(PG_referenced))) return 0; /* @@ -154,18 +186,46 @@ static inline int folio_lru_refs(const struct folio *folio) return ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) + 1; } -static inline int folio_lru_gen(const struct folio *folio) +/** + * lru_refs_set_flags - Set the LRU referenced / access count to specified folio flags. + * @flags: pointer to the folio flags + * @refs: referenced / access count number, between 0 and LRU_REFS_MAX, inclusive. + */ +static inline void lru_refs_set_flags(unsigned long *flags, unsigned int refs) +{ + VM_WARN_ON_ONCE(refs > LRU_REFS_MAX); + + *flags &= ~LRU_REFS_FLAGS; + if (!refs) + return; + *flags |= (BIT(PG_referenced) | ((refs - 1UL) << LRU_REFS_PGOFF)); +} + +static inline int folio_lru_refs(const struct folio *folio) +{ + return lru_refs_from_flags(READ_ONCE(*const_folio_flags(folio, 0))); +} + +static inline void folio_set_lru_refs(struct folio *folio, unsigned int refs) { - unsigned long flags = READ_ONCE(*const_folio_flags(folio, 0)); + unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0)); + + do { + new_flags = old_flags; + lru_refs_set_flags(&new_flags, refs); + } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); +} - return ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; +static inline int folio_lru_gen(const struct folio *folio) +{ + return lru_gen_from_flags(READ_ONCE(*const_folio_flags(folio, 0))); } static inline bool lru_gen_is_active(const struct lruvec *lruvec, int gen) { unsigned long max_seq = lruvec->lrugen.max_seq; - VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); + VM_WARN_ON_ONCE(gen > LRU_GEN_MAX); /* see the comment on MIN_NR_GENS */ return gen == lru_gen_from_seq(max_seq) || gen == lru_gen_from_seq(max_seq - 1); @@ -305,9 +365,7 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, static inline void folio_migrate_refs(struct folio *new, const struct folio *old) { - unsigned long refs = READ_ONCE(*const_folio_flags(old, 0)) & LRU_REFS_MASK; - - set_mask_bits(folio_flags(new, 0), LRU_REFS_MASK, refs); + folio_set_lru_refs(new, folio_lru_refs(old)); } #else /* !CONFIG_LRU_GEN */ @@ -338,7 +396,8 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, static inline void folio_migrate_refs(struct folio *new, const struct folio *old) { - + if (folio_test_referenced(old)) + folio_set_referenced(new); } #endif /* CONFIG_LRU_GEN */ diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 9adb2ad21da5..e4c51961ec27 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -493,7 +493,9 @@ enum lruvec_flags { #ifndef __GENERATING_BOUNDS_H #define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF) +#define LRU_GEN_MAX (BIT(LRU_GEN_WIDTH - 1) - 1) #define LRU_REFS_MASK ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF) +#define LRU_REFS_MAX (BIT(LRU_REFS_WIDTH) - 1) /* * For folios accessed multiple times through file descriptors, diff --git a/mm/migrate.c b/mm/migrate.c index 8a64291ab5b4..23248484a165 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -763,8 +763,6 @@ void folio_migrate_flags(struct folio *newfolio, struct folio *folio) { int cpupid; - if (folio_test_referenced(folio)) - folio_set_referenced(newfolio); if (folio_test_uptodate(folio)) folio_mark_uptodate(newfolio); if (folio_test_clear_active(folio)) { diff --git a/mm/swap.c b/mm/swap.c index e7037ea2c10f..6204496d48f5 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -393,24 +393,26 @@ static void __lru_cache_activate_folio(struct folio *folio) static void lru_gen_inc_refs(struct folio *folio) { unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0)); + int refs; if (folio_test_unevictable(folio)) return; - /* see the comment on LRU_REFS_FLAGS */ - if (!folio_test_referenced(folio)) { - set_mask_bits(folio_flags(folio, 0), LRU_REFS_MASK, BIT(PG_referenced)); + if (!folio_lru_refs(folio)) { + folio_set_lru_refs(folio, 1); return; } + /* see the comment on LRU_REFS_FLAGS */ do { - if ((old_flags & LRU_REFS_MASK) == LRU_REFS_MASK) { + new_flags = old_flags; + refs = lru_refs_from_flags(old_flags); + if (refs == LRU_REFS_MAX) { if (!folio_test_workingset(folio)) folio_set_workingset(folio); return; } - - new_flags = old_flags + BIT(LRU_REFS_PGOFF); + lru_refs_set_flags(&new_flags, refs + 1); } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); } @@ -423,7 +425,8 @@ static bool lru_gen_clear_refs(struct folio *folio) if (gen < 0) return true; - set_mask_bits(folio_flags(folio, 0), LRU_REFS_FLAGS | BIT(PG_workingset), 0); + folio_set_lru_refs(folio, 0); + folio_clear_workingset(folio); rcu_read_lock(); seq = READ_ONCE(folio_lruvec(folio)->lrugen.min_seq[type]); diff --git a/mm/vmscan.c b/mm/vmscan.c index 7a1f08147dee..2ca1d6d80259 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -843,12 +843,14 @@ enum folio_references { static bool lru_gen_set_refs(struct folio *folio) { /* see the comment on LRU_REFS_FLAGS */ - if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) { - set_mask_bits(folio_flags(folio, 0), LRU_REFS_MASK, BIT(PG_referenced)); + if (!folio_lru_refs(folio) && !folio_test_workingset(folio)) { + folio_set_lru_refs(folio, 1); return false; } - set_mask_bits(folio_flags(folio, 0), LRU_REFS_FLAGS, BIT(PG_workingset)); + folio_set_lru_refs(folio, 0); + folio_set_workingset(folio); + return true; } #else @@ -3191,28 +3193,31 @@ static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv) ******************************************************************************/ /* promote pages accessed through page tables */ -static int folio_update_gen(struct folio *folio, int gen) +static int folio_update_gen(struct folio *folio, int new_gen) { unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0)); - - VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); + int old_gen; /* see the comment on LRU_REFS_FLAGS */ - if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) { - set_mask_bits(folio_flags(folio, 0), LRU_REFS_MASK, BIT(PG_referenced)); + if (!lru_refs_from_flags(old_flags) && !folio_test_workingset(folio)) { + folio_set_lru_refs(folio, 1); return -1; } do { + old_gen = lru_gen_from_flags(old_flags); + new_flags = old_flags; + /* lru_gen_del_folio() has isolated this page? */ - if (!(old_flags & LRU_GEN_MASK)) - return -1; + if (old_gen < 0) + break; - new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_FLAGS); - new_flags |= ((gen + 1UL) << LRU_GEN_PGOFF) | BIT(PG_workingset); + lru_gen_set_flags(&new_flags, new_gen); + lru_refs_set_flags(&new_flags, 0); + new_flags |= BIT(PG_workingset); } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); - return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; + return old_gen; } /* protect pages accessed multiple times through file descriptors */ @@ -3220,17 +3225,18 @@ static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio) { int type = folio_is_file_lru(folio); struct lru_gen_folio *lrugen = &lruvec->lrugen; - int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); + int old_gen, new_gen, min_gen = lru_gen_from_seq(lrugen->min_seq[type]); unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0)); - VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio); - do { - new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; + old_gen = lru_gen_from_flags(old_flags); + VM_WARN_ON_ONCE_FOLIO(old_gen < 0, folio); + /* folio_update_gen() has promoted this page? */ - if (new_gen >= 0 && new_gen != old_gen) - return new_gen; + if (old_gen >= 0 && old_gen != min_gen) + return old_gen; + new_flags = old_flags; new_gen = (old_gen + 1) % MAX_NR_GENS; new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_FLAGS); @@ -4639,7 +4645,7 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca /* see the comment on LRU_REFS_FLAGS */ if (!folio_test_referenced(folio)) - set_mask_bits(folio_flags(folio, 0), LRU_REFS_MASK, 0); + folio_set_lru_refs(folio, 0); success = lru_gen_del_folio(lruvec, folio, true); VM_WARN_ON_ONCE_FOLIO(!success, folio); @@ -4854,8 +4860,10 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec, } /* don't add rejected folios to the oldest generation */ - if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]) - set_mask_bits(folio_flags(folio, 0), LRU_REFS_FLAGS, BIT(PG_active)); + if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]) { + folio_set_lru_refs(folio, 0); + folio_set_active(folio); + } } move_folios_to_lru(&list); -- 2.54.0