From: Kairui Song To enable moving file pages in folio_mark_accessed directly and lazily for MGLRU, allow updating the LRU statistic atomically without holding a lock. It may cause temporary counter underflow, which should be fine as we still follow final consistency of the counter, and it only serves as a factor for calculating the reclaim budget in vmscan. A little inaccuracy has no visible effect. Signed-off-by: Kairui Song --- include/linux/memcontrol.h | 6 +++--- include/linux/mm_inline.h | 3 +-- mm/memcontrol.c | 6 +++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 46bf724cae7a..2daf2be5d1e5 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -106,7 +106,7 @@ struct mem_cgroup_per_node { /* Written on every LRU update and on every reclaim iteration. */ __cacheline_group_begin_aligned(memcg_pn_write_hot); - long lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS]; + atomic_long_t lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS]; struct mem_cgroup_reclaim_iter iter; #ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC /* slab stats for nmi context */ @@ -926,8 +926,8 @@ unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec, struct mem_cgroup_per_node *mz; mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); - val = READ_ONCE(mz->lru_zone_size[zone_idx][lru]); - if (WARN_ON_ONCE(val < 0)) + val = atomic_long_read(&mz->lru_zone_size[zone_idx][lru]); + if (val < 0) return 0; return val; diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index ab69b9930893..597f013c8e04 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -47,11 +47,10 @@ static __always_inline void __update_lru_size(struct lruvec *lruvec, { struct pglist_data *pgdat = lruvec_pgdat(lruvec); - lockdep_assert_held(&lruvec->lru_lock); WARN_ON_ONCE(nr_pages != (int)nr_pages); mod_lruvec_state(lruvec, NR_LRU_BASE + lru, nr_pages); - __mod_zone_page_state(&pgdat->node_zones[zid], + mod_zone_page_state(&pgdat->node_zones[zid], NR_ZONE_LRU_BASE + lru, nr_pages); } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 1460cba53588..3cc14d1932f0 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1544,8 +1544,8 @@ struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio, * @zid: zone id of the accounted pages * @nr_pages: positive when adding or negative when removing * - * This function must be called under lru_lock, just before a page is added - * to or just after a page is removed from an lru list. + * This function must be called when a page is added to or removed from + * an lru list. Caller need to protect the lruvec from being freed. */ void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, int zid, long nr_pages) @@ -1556,7 +1556,7 @@ void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, return; mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); - mz->lru_zone_size[zid][lru] += nr_pages; + atomic_long_add(nr_pages, &mz->lru_zone_size[zid][lru]); } /** -- 2.55.0 From: Kairui Song No feature change, convert them to atomic so we can update them without holding the LRU lock. There is no risk of overflow. The reader always compares and uses zero instead if the counter values are negative. It follows final consistency. Signed-off-by: Kairui Song --- include/linux/mm_inline.h | 6 ++---- include/linux/mmzone.h | 2 +- mm/vmscan.c | 27 ++++++++++++--------------- 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 597f013c8e04..f52f02e8e5be 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -265,11 +265,9 @@ static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *foli VM_WARN_ON_ONCE(old_gen == -1 && new_gen == -1); if (old_gen >= 0) - WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone], - lrugen->nr_pages[old_gen][type][zone] - delta); + atomic_long_sub(delta, &lrugen->nr_pages[old_gen][type][zone]); if (new_gen >= 0) - WRITE_ONCE(lrugen->nr_pages[new_gen][type][zone], - lrugen->nr_pages[new_gen][type][zone] + delta); + atomic_long_add(delta, &lrugen->nr_pages[new_gen][type][zone]); /* addition */ if (old_gen < 0) { diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index acd94cecc0d3..de7d3f8059a5 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -576,7 +576,7 @@ struct lru_gen_folio { /* the multi-gen LRU lists, lazily sorted on eviction */ struct list_head folios[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; /* the multi-gen LRU sizes, eventually consistent */ - long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; + atomic_long_t nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; /* the exponential moving average of refaulted */ unsigned long avg_refaulted[ANON_AND_FILE][MAX_NR_TIERS]; /* the exponential moving average of evicted+protected */ diff --git a/mm/vmscan.c b/mm/vmscan.c index aaceed4759ee..32890e628240 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3451,8 +3451,7 @@ static void reset_batch_size(struct lru_gen_mm_walk *walk) continue; walk->nr_pages[gen][type][zone] = 0; - WRITE_ONCE(lrugen->nr_pages[gen][type][zone], - lrugen->nr_pages[gen][type][zone] + delta); + atomic_long_add(delta, &lrugen->nr_pages[gen][type][zone]); if (lru_gen_is_active(lruvec, gen)) lru += LRU_ACTIVE; @@ -4072,11 +4071,8 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness) break; } flush_lru_batch(head, &batch_end, target_list); - - WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone], - lrugen->nr_pages[old_gen][type][zone] - delta); - WRITE_ONCE(lrugen->nr_pages[target_gen][type][zone], - lrugen->nr_pages[target_gen][type][zone] + delta); + atomic_long_sub(delta, &lrugen->nr_pages[old_gen][type][zone]); + atomic_long_add(delta, &lrugen->nr_pages[target_gen][type][zone]); if (!remaining) return false; } @@ -4181,8 +4177,8 @@ static bool inc_max_seq(struct lruvec *lruvec, unsigned long seq, int swappiness for (type = 0; type < ANON_AND_FILE; type++) { for (zone = 0; zone < MAX_NR_ZONES; zone++) { enum lru_list lru = type * LRU_INACTIVE_FILE; - long delta = lrugen->nr_pages[prev][type][zone] - - lrugen->nr_pages[next][type][zone]; + long delta = atomic_long_read(&lrugen->nr_pages[prev][type][zone]) - + atomic_long_read(&lrugen->nr_pages[next][type][zone]); if (!delta) continue; @@ -4300,7 +4296,8 @@ static unsigned long lruvec_evictable_size(struct lruvec *lruvec, int swappiness for (seq = min_seq[type]; seq <= max_seq; seq++) { gen = lru_gen_from_seq(seq); for (zone = 0; zone < MAX_NR_ZONES; zone++) - total += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); + total += max(atomic_long_read(&lrugen->nr_pages[gen][type][zone]), + 0L); } } @@ -4720,7 +4717,7 @@ static void __lru_gen_reparent_memcg(struct lruvec *child_lruvec, struct lruvec for (i = 0; i < get_nr_gens(child_lruvec, type); i++) { int gen = lru_gen_from_seq(child_lrugen->max_seq - i); - long nr_pages = child_lrugen->nr_pages[gen][type][zone]; + long nr_pages = atomic_long_read(&child_lrugen->nr_pages[gen][type][zone]); int child_lru_active = lru_gen_is_active(child_lruvec, gen) ? LRU_ACTIVE : 0; int parent_lru_active = lru_gen_is_active(parent_lruvec, gen) ? LRU_ACTIVE : 0; @@ -4728,9 +4725,8 @@ static void __lru_gen_reparent_memcg(struct lruvec *child_lruvec, struct lruvec list_splice_tail_init(&child_lrugen->folios[gen][type][zone], &parent_lrugen->folios[gen][type][zone]); - WRITE_ONCE(child_lrugen->nr_pages[gen][type][zone], 0); - WRITE_ONCE(parent_lrugen->nr_pages[gen][type][zone], - parent_lrugen->nr_pages[gen][type][zone] + nr_pages); + atomic_long_set(&child_lrugen->nr_pages[gen][type][zone], 0); + atomic_long_add(nr_pages, &parent_lrugen->nr_pages[gen][type][zone]); if (lru_gen_is_active(child_lruvec, gen) != lru_gen_is_active(parent_lruvec, gen)) { __update_lru_size(child_lruvec, lru + child_lru_active, zone, -nr_pages); @@ -5807,7 +5803,8 @@ static int lru_gen_seq_show(struct seq_file *m, void *v) char mark = full && seq < min_seq[type] ? 'x' : ' '; for (zone = 0; zone < MAX_NR_ZONES; zone++) - size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); + size += max(atomic_long_read(&lrugen->nr_pages[gen][type][zone]), + 0L); seq_printf(m, " %10lu%c", size, mark); } -- 2.55.0 From: Kairui Song Add a helper that resolves a stable lruvec for a folio under RCU without taking the lruvec lock. It takes a folio directly so the lruvec lookup happens inside the RCU read-side critical section, which a lruvec-based interface cannot guarantee. No functional change. Signed-off-by: Kairui Song --- include/linux/memcontrol.h | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 2daf2be5d1e5..d8a47fde387e 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -1524,6 +1524,45 @@ static inline void lruvec_lock_irq(struct lruvec *lruvec) spin_lock_irq(&lruvec->lru_lock); } +/** + * folio_lruvec_live_get - get a live lruvec for a folio under RCU + * @folio: the folio + * + * Computes @folio's lruvec and walks up to the nearest live ancestor + * if the folio's memcg is dying. Paired with folio_lruvec_live_put(). + * The result may be stale: RCU keeps it alive but does not pin @folio + * to it. That is fine as the counters are fixed up on reparenting. + * + * Return: the live lruvec, with rcu_read_lock held. + */ +static inline struct lruvec *folio_lruvec_live_get(struct folio *folio) +{ +#ifdef CONFIG_MEMCG + struct lruvec *lruvec; + struct pglist_data *pgdat; + struct mem_cgroup *memcg; + + rcu_read_lock(); + lruvec = folio_lruvec(folio); + pgdat = lruvec_pgdat(lruvec); + memcg = lruvec_memcg(lruvec); + while (unlikely(memcg && css_is_dying(&memcg->css))) { + memcg = parent_mem_cgroup(memcg); + lruvec = mem_cgroup_lruvec(memcg, pgdat); + } + return lruvec; +#else + return folio_lruvec(folio); +#endif +} + +static inline void folio_lruvec_live_put(struct lruvec *lruvec) +{ +#ifdef CONFIG_MEMCG + rcu_read_unlock(); +#endif +} + static inline struct lruvec *lruvec_live_lock_irq(struct lruvec *lruvec) { #ifdef CONFIG_MEMCG -- 2.55.0 From: Kairui Song Complement MGLRU's eviction-time tier-PID protection with access-time frequency-guided promotion. Introduce a unified set of helpers built based on referenced (access) count of a folio. Each access increments a folio's referenced count stored in folio flags (refs), refs still maps to a logarithmic tier just like before, but with more formal bit definitions, a few special thresholds are introduced: LRU_REFS_REFERENCED (1), LRU_REFS_WORKINGSET (2), LRU_REFS_PROTECTED (3), and LRU_REFS_MAX (7). When refs reaches a certain threshold, the folio is promoted proactively instead of waiting for the PID controller to kick in. Also simplify MGLRU's usage of PG_workingset and PG_referenced: they become the low two bits of the refs count, with the higher bits provided by LRU_REFS_MASK. This reduces MGLRU's original refs count bit usage by one, since only one extra bit is now needed to record a max referenced count of 7, and makes MGLRU's refs accounting more accurate. This doesn't affect classical LRU in any way, and it addresses several shortcomings of MGLRU's old tier-only cache protection model: - Long feedback loop: protection only activated after enough re-faults, by which time the hot folios are already evicted, or no longer hot. - Limited tier resolution: once referenced count exceeded the bits limit (8 previously), MGLRU could no longer distinguish hotter folios as they are capped by the tier. And what's worse, PG_workingset forces a folio to stay on tier 3. - Eviction hotness reversion: because PID protection activates upon eviction and always targets the LRU tail, it tends to protect cold tail folios at the expense of hotter head folios. Once the tail folios consume the PID protection budget, head folios lose their protection. - Additionally, the PID cannot distinguish the access time of folios that share the same reference count, and there are only 4 tiers. To achieve a frequency-guided framework, this commit introduces and reworks the LRU_REFS related helpers and definitions; most of the work is done by the helpers below, and their inline comments describe the details. - folio_inc_lru_refs(): Called on any cache access (folio_mark_accessed) or page table access. This is the main helper: it promotes folios according to their access frequency. Promotion is lazy: the gen bits and size counters are updated eagerly, while the list move is deferred to the next isolation. NOTE: For now, the lruvec lock is unconditionally taken on every on-list access to block concurrent aging; a lockless fast path will be implemented very soon in a following commit. - folio_inc_lru_refs_walk(): Used by the PTE walk path during aging, where generations are stable, and promotes folios a bit more aggressively since PTE accesses have a higher promotion bias; this also performs lazy promotion. - folio_inc_lru_refs_isolated(): Used by the rmap check before eviction. The folio is isolated and hence this doesn't perform promotion by itself; the folio will be added back to the right gen upon return according to the access frequency. This path also has a higher promotion bias. The eviction-time folio_inc_gen() still handles PID protection, but the protection ratio is softer than before since proactive promotion is mostly good enough already. The PID gain factors are relaxed from (2:3) to (1:2) and the setpoint now spans the cumulative mass of the tiers below the candidate instead of tier 0 alone. folio_inc_gen() caps refs at WORKINGSET so the folio retains enough history to stay above the cold tier. Tier 1 is the fallback tier for PID, and tier 2 is the fallback tier for frequency-guided promotion. The forced protection for full-refs folios is removed, obsoleted by the proactive promotion. Refaults are now activated purely according to access frequency: the old fault bias applied in folio_add_lru() is simplified, since a folio's access history is a more consistent signal than the context of the faulting task. Page table access is still considered a slightly stronger signal. This also redefines PG_workingset and PG_referenced as the low two bits of the refs count, eliminating the old restriction where LRU_REFS_MASK was only valid when PG_referenced was set, and allows all paths to use the same encoding consistently. Following this idea, a workingset folio is now defined as refs >= LRU_REFS_WORKINGSET (2), matching the active/inactive LRU's definition and giving in-kernel consumers (PSI, readahead) consistent behavior on MGLRU, fixing the longstanding issue that these users don't work well with MGLRU. PG_workingset and PG_referenced are no longer independent flags under MGLRU. Adjusting existing raw folio_test_*() callers to the new semantics is left as follow-ups. Signed-off-by: Kairui Song --- include/linux/mm_inline.h | 83 +++++++------ include/linux/mmzone.h | 144 ++++++++++++++++------ kernel/bounds.c | 2 +- mm/folio.c | 50 +------- mm/vmscan.c | 304 ++++++++++++++++++++++++++++++---------------- mm/workingset.c | 45 ++++--- 6 files changed, 380 insertions(+), 248 deletions(-) diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index f52f02e8e5be..7c8288661edd 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -144,12 +144,13 @@ static inline int lru_hist_from_seq(unsigned long seq) return seq % NR_HIST_GENS; } -static inline int lru_tier_from_refs(int refs, bool workingset) +static inline int lru_tier_from_refs(unsigned int refs) { - VM_WARN_ON_ONCE(refs > BIT(LRU_REFS_WIDTH)); - - /* see the comment on MAX_NR_TIERS */ - return workingset ? MAX_NR_TIERS - 1 : order_base_2(refs); + BUILD_BUG_ON(fls(LRU_REFS_MAX - 1) > MAX_NR_TIERS - 1); + VM_WARN_ON_ONCE(refs > LRU_REFS_MAX); + if (refs < LRU_REFS_WORKINGSET) + return 0; + return fls(refs - 1); } /** @@ -187,21 +188,24 @@ static inline int lru_get_gen_flags(unsigned long flags) * @flags: pointer to the folio flags * @refs: referenced / access count number, between 0 and LRU_REFS_MAX, inclusive. * - * For MGLRU, PG_referenced holds the first ref, and the extra bits hold the - * remaining refs. For classical LRU the extra bits are not used, so it can - * also be seen as the refs count never exceeds 1. In both cases, refs == 1 - * means PG_referenced is set and the extra bits are zero, and refs == 0 means - * PG_referenced and the extra bits are all unset. + * For MGLRU, PG_referenced, PG_workingset are used as the lower two bits of + * refs counter, and extra bits hold the remaining higher bits. For classical + * LRU the extra bits are not used, and the two flags has no direct + * relationship with each other, but this helper can still be used to sync + * them. For both cases, refs == 0 means these two flags and the extra bits + * are all unset. And refs == 1 / 2 / 3 means PG_referenced and PG_workingset + * are set in an bit order way, which is more meaningful for MGLRU though. */ static inline void lru_set_refs_flags(unsigned long *flags, unsigned int refs) { VM_WARN_ON_ONCE(refs > LRU_REFS_MAX); - BUILD_BUG_ON(LRU_REFS_MAX != (LRU_REFS_MASK >> LRU_REFS_PGOFF) + 1); - + BUILD_BUG_ON(LRU_REFS_MASK & (BIT(PG_referenced) | BIT(PG_workingset))); *flags &= ~LRU_REFS_FLAGS; - if (!refs) - return; - *flags |= (BIT(PG_referenced) | ((refs - 1UL) << LRU_REFS_PGOFF)); + if (refs & BIT(0)) + *flags |= BIT(PG_referenced); + if (refs & BIT(1)) + *flags |= BIT(PG_workingset); + *flags |= ((unsigned long)refs >> 2) << LRU_REFS_PGOFF; } /** @@ -212,13 +216,13 @@ static inline void lru_set_refs_flags(unsigned long *flags, unsigned int refs) */ static inline int lru_get_refs_flags(unsigned long flags) { - if (!(flags & BIT(PG_referenced))) - return 0; - /* - * Return the total number of accesses including PG_referenced. Also see - * the comment on LRU_REFS_FLAGS. - */ - return ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) + 1; + int refs; + + /* Return the total number of accesses. See the comment on LRU_REFS_FLAGS. */ + refs = (flags & BIT(PG_referenced)) ? BIT(0) : 0; + refs += (flags & BIT(PG_workingset)) ? BIT(1) : 0; + refs += ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) << 2; + return refs; } static inline int folio_lru_refs(const struct folio *folio) @@ -236,6 +240,8 @@ static inline void folio_set_lru_refs(struct folio *folio, unsigned int refs) } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); } +void folio_inc_lru_refs(struct folio *folio, unsigned int flags); + static inline int folio_lru_gen(const struct folio *folio) { return lru_get_gen_flags(READ_ONCE(*const_folio_flags(folio, 0))); @@ -243,7 +249,7 @@ static inline int folio_lru_gen(const struct folio *folio) static inline bool lru_gen_is_active(const struct lruvec *lruvec, int gen) { - unsigned long max_seq = lruvec->lrugen.max_seq; + unsigned long max_seq = READ_ONCE(lruvec->lrugen.max_seq); VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); @@ -300,23 +306,24 @@ static inline unsigned long lru_gen_folio_seq(const struct lruvec *lruvec, bool reclaiming) { int gen; + int refs = folio_lru_refs(folio); int type = folio_is_file_lru(folio); const struct lru_gen_folio *lrugen = &lruvec->lrugen; /* - * +-----------------------------------+-----------------------------------+ - * | Accessed through page tables and | Accessed through file descriptors | - * | promoted by folio_update_gen() | and protected by folio_inc_gen() | - * +-----------------------------------+-----------------------------------+ - * | PG_active (set while isolated) | | - * +-----------------+-----------------+-----------------+-----------------+ - * | PG_workingset | PG_referenced | PG_workingset | LRU_REFS_FLAGS | - * +-----------------------------------+-----------------------------------+ - * |<---------- MIN_NR_GENS ---------->| | - * |<---------------------------- MAX_NR_GENS ---------------------------->| + * +------------------------------------------+------------------------------------------+ + * | Accessed through page tables and | Accessed through file descriptors | + * | promoted by folio_inc_lru_refs_walk() | protected by folio_inc_lru_refs/inc_gen | + * +------------------------------------------+------------------------------------------+ + * | PG_active (set at isolation or refault) | | + * +--------------------+---------------------+--------------------+---------------------+ + * | LRU_REFS_MAX | LRU_REFS_WORKINGSET | LRU_REFS_MAX | LRU_REFS_WORKINGSET | + * +------------------------------------------+------------------------------------------+ + * |<-------------- MIN_NR_GENS ------------->| | + * |<----------------------------------- MAX_NR_GENS ----------------------------------->| */ if (folio_test_active(folio)) - gen = MIN_NR_GENS - folio_test_workingset(folio); + gen = MIN_NR_GENS - (refs >= LRU_REFS_WORKINGSET); else if (reclaiming) gen = MAX_NR_GENS; else if ((!folio_is_file_lru(folio) && !folio_test_swapcache(folio)) || @@ -324,7 +331,7 @@ static inline unsigned long lru_gen_folio_seq(const struct lruvec *lruvec, (folio_test_dirty(folio) || folio_test_writeback(folio)))) gen = MIN_NR_GENS; else - gen = MAX_NR_GENS - (folio_test_workingset(folio) || folio_test_referenced(folio)); + gen = MAX_NR_GENS - (refs >= LRU_REFS_WORKINGSET); return max(READ_ONCE(lrugen->max_seq) - gen + 1, READ_ONCE(lrugen->min_seq[type])); } @@ -338,6 +345,7 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, int zone = folio_zonenum(folio); struct lru_gen_folio *lrugen = &lruvec->lrugen; + BUILD_BUG_ON(BIT(LRU_GEN_WIDTH - 1) != MAX_NR_GENS); VM_WARN_ON_ONCE_FOLIO(gen != -1, folio); if (folio_test_unevictable(folio) || !lrugen->enabled) @@ -392,7 +400,6 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, */ static inline void folio_migrate_lru_refs(struct folio *new, const struct folio *old) { - BUILD_BUG_ON(LRU_REFS_MASK & BIT(PG_referenced)); folio_set_lru_refs(new, folio_lru_refs(old)); } #else /* !CONFIG_LRU_GEN */ @@ -422,6 +429,10 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, return false; } +static inline void folio_inc_lru_refs(struct folio *folio, unsigned int flags) +{ +} + static inline void folio_migrate_lru_refs(struct folio *new, const struct folio *old) { if (folio_test_referenced(old)) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index de7d3f8059a5..523783186c57 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -475,57 +475,121 @@ enum lruvec_flags { #define MAX_NR_GENS 4U /* - * Each generation is divided into multiple tiers. A folio accessed N times - * through file descriptors is in tier order_base_2(N). A folio in the first - * tier (N=0,1) is marked by PG_referenced unless it was faulted in through page - * tables or read ahead. A folio in the last tier (MAX_NR_TIERS-1) is marked by - * PG_workingset. A folio in any other tier (1flags. + * Each generation is divided into multiple tiers. A folio's referenced + * count maps to a tier as shown below: * - * In contrast to moving across generations which requires the LRU lock, moving - * across tiers only involves atomic operations on folio->flags and therefore - * has a negligible cost in the buffered access path. In the eviction path, - * comparisons of refaulted/(evicted+protected) from the first tier and the rest - * infer whether folios accessed multiple times through file descriptors are - * statistically hot and thus worth protecting. + * MGLRU (frequency guidance) + * Refs Tier |- Refs: how many times (at least) a folio has been referenced. + * 0 0 |- Mostly cold pages, readahead, etc. [1] + * 1 0 |= LRU_REFS_REFERENCED: Used at least once. [2] + * -WORKINGSET-+|- Pages beyond are workingset and never fall below this floor. [3] + * 2 1<-+|= LRU_REFS_WORKINGSET: Classical workingset, accessed twice, protected. [4] + * 3 2 |- LRU_REFS_PROTECTED: Protected workingset, promoted pages capped at here. [5] + * 4 2 | + * 5 3 |- The tier here is MAX_NR_TIERS - 1 + * 6 3 | + * 7 3 |= LRU_REFS_MAX: Promotion candidate. [6] + * -PROMOTION->-/ * - * MAX_NR_TIERS is set to 4 so that the multi-gen LRU can support twice the - * number of categories of the active/inactive LRU when keeping track of - * accesses through file descriptors. This uses MAX_NR_TIERS-2 spare bits in - * folio->flags, masked by LRU_REFS_MASK. + * Ideally each tier holds folios of similar access patterns: lower tiers + * are less important and evicted faster. A page's reference count and + * tier are capped when it changes generation, preventing it from + * dominating the new generation based on old-generation access history. + * Generation ordering already ensures a newer-gen page is hotter than an + * older-gen one regardless of tier. + * + * Refs tracks accesses from two sources: page table (lazily collected by + * the page table aging walk or rmap eviction lookup) and file descriptors + * (by folio_mark_accessed). Page table accesses are weighted heavier + * because the accessed bit is sticky (undercounts repeated accesses), + * passively collected, and page faults are generally more important as + * userspace does not expect a memory access to block on reclaim. Both + * access types increment refs by one; the result is capped at + * LRU_REFS_PROTECTED on promotion or deferral, or LRU_REFS_MAX otherwise. + * + * 1. Tier is fls(N-1) for N > 1, 0 otherwise. Folios with zero + * accesses (refs == 0) are generally cold, e.g. readahead folios. + * + * Page table access advances a folio by one generation even at the + * lowest refs or tier. Freshly allocated folios start with refs == 0; + * faulted and mapped folios have their page table access bit set, so + * the first page table access check always sets LRU_REFS_REFERENCED and + * moves them one generation forward, driving aging and workingset shift. + * + * 2. Folios accessed once stay on tier 0: one-time usage does not + * qualify for protection. A second access advances the folio, + * aligning with classical LRU's use-twice threshold. A second page + * table access promotes to the latest gen; file access only defers + * eviction from the oldest gen. + * + * 3. Folios accessed at least twice are considered workingset. This + * mostly aligns with classical LRU: at least one I/O is saved by + * keeping them in memory. Folios at or above this level never fall + * below tier 1 (the workingset floor), so tier 0 stays a clean tier + * for cold cache while tier 1 serves as the fallback line for + * actually reused or historically hot folios. + * + * Folios refaulted through a page fault at refs 1 will enter the second + * newest gen, so faulting will be protected better. + * + * 4. Starting from tier 1, PID protection sacrifices lower tiers to + * protect higher tiers by comparing refault rates for long-term + * accuracy, and caps higher refs to this value. Since PID protection + * bypasses page table lookup and clearing, when a further eviction + * attempt occurs after PID loosens, the folio's page table access is + * rechecked and the folio is sent back to LRU_REFS_PROTECTED. This + * also gives folios a fair opportunity to be promoted by file access + * again. + * + * Folios refaulted through a page fault at tier 1 or above are activated + * and enter the newest gen. Non fault page will enter second oldest gen, + * driving aging and workingset shifting. + * + * 5. Pages beyond the ordinary workingset tier form new tiers for the + * PID controller to protect differently. Folios at or above this + * level are capped at LRU_REFS_PROTECTED on promotion or deferral, + * and at LRU_REFS_WORKINGSET under PID protection in the oldest + * generation, where they represent a historical workingset. + * + * 6. Folios that reach LRU_REFS_MAX are advanced to the next generation + * on further access, with refs capped to LRU_REFS_PROTECTED. This + * gives them a fair start for advancement to an even newer generation + * while keeping hot folios distinguishable. + * + * Tiering uses PG_referenced and PG_workingset as the lower two bits, + * and the bits masked by LRU_REFS_MASK as the higher bits, so the refs + * count ranges from 0 to LRU_REFS_MAX. A folio is on the workingset + * tier once accessed at least twice, which is more consistent with the + * classical LRU. + * + * A folio's referenced count never goes backwards except upon gen + * increase as described above, or when explicitly reset by + * lru_gen_clear_refs(). Refault of a reclaimed folio restores + * its referenced count, capped at LRU_REFS_PROTECTED, which aligns with + * promotion. Page table refaults of previous workingset folios send + * them to the latest gen, driving aging faster. + * + * MAX_NR_TIERS is set to 4 so that the multi-gen LRU can support twice + * the number of categories of the active/inactive LRU. */ #define MAX_NR_TIERS 4U #define LRU_TIER_MIN 0U #define LRU_TIER_MAX (MAX_NR_TIERS - 1) +/* Access source flags for folio_inc_lru_refs() */ +#define LRU_REF_MAPPED 0x1U +#define LRU_REF_EXEC 0x2U + +#define LRU_REFS_REFERENCED 0x1 +#define LRU_REFS_WORKINGSET 0x2 +#define LRU_REFS_PROTECTED 0x3 + #ifndef __GENERATING_BOUNDS_H #define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF) #define LRU_REFS_MASK ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF) -#define LRU_REFS_MAX BIT(LRU_REFS_WIDTH) - -/* - * For folios accessed multiple times through file descriptors, - * lru_gen_inc_refs() sets additional bits of LRU_REFS_WIDTH in folio->flags - * after PG_referenced, then PG_workingset after LRU_REFS_WIDTH. After all its - * bits are set, i.e., LRU_REFS_FLAGS|BIT(PG_workingset), a folio is lazily - * promoted into the second oldest generation in the eviction path. And when - * folio_inc_gen() does that, it clears LRU_REFS_FLAGS so that - * lru_gen_inc_refs() can start over. Note that for this case, LRU_REFS_MASK is - * only valid when PG_referenced is set. - * - * For folios accessed multiple times through page tables, folio_update_gen() - * from a page table walk or lru_gen_set_refs() from a rmap walk sets - * PG_referenced after the accessed bit is cleared for the first time. - * Thereafter, those two paths set PG_workingset and promote folios to the - * youngest generation. Like folio_inc_gen(), folio_update_gen() also clears - * PG_referenced. Note that for this case, LRU_REFS_MASK is not used. - * - * For both cases above, after PG_workingset is set on a folio, it remains until - * this folio is either reclaimed, or "deactivated" by lru_gen_clear_refs(). It - * can be set again if lru_gen_test_recent() returns true upon a refault. - */ -#define LRU_REFS_FLAGS (LRU_REFS_MASK | BIT(PG_referenced)) +#define LRU_REFS_FLAGS (LRU_REFS_MASK | BIT(PG_referenced) | BIT(PG_workingset)) +#define LRU_REFS_MAX (BIT(LRU_REFS_WIDTH + 2) - 1) struct lruvec; struct page_vma_mapped_walk; diff --git a/kernel/bounds.c b/kernel/bounds.c index 02b619eb6106..06a034713b5d 100644 --- a/kernel/bounds.c +++ b/kernel/bounds.c @@ -25,7 +25,7 @@ int main(void) DEFINE(SPINLOCK_SIZE, sizeof(spinlock_t)); #ifdef CONFIG_LRU_GEN DEFINE(LRU_GEN_WIDTH, order_base_2(MAX_NR_GENS + 1)); - DEFINE(__LRU_REFS_WIDTH, MAX_NR_TIERS - 2); + DEFINE(__LRU_REFS_WIDTH, MAX_NR_TIERS - 3); #else DEFINE(LRU_GEN_WIDTH, 0); DEFINE(__LRU_REFS_WIDTH, 0); diff --git a/mm/folio.c b/mm/folio.c index 47a437e0f7fd..55cb37366487 100644 --- a/mm/folio.c +++ b/mm/folio.c @@ -273,7 +273,6 @@ static void lru_activate(struct lruvec *lruvec, struct folio *folio) if (folio_test_active(folio) || folio_test_unevictable(folio)) return; - lruvec_del_folio(lruvec, folio); folio_set_active(folio); lruvec_add_folio(lruvec, folio); @@ -352,32 +351,6 @@ static void __lru_cache_activate_folio(struct folio *folio) #ifdef CONFIG_LRU_GEN -static void lru_gen_inc_refs(struct folio *folio) -{ - unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0)); - int refs; - - if (folio_test_unevictable(folio)) - return; - - /* see the comment on LRU_REFS_FLAGS */ - if (!folio_lru_refs(folio)) { - folio_set_lru_refs(folio, 1); - return; - } - - do { - new_flags = old_flags; - refs = lru_get_refs_flags(old_flags); - if (refs == LRU_REFS_MAX) { - if (!folio_test_workingset(folio)) - folio_set_workingset(folio); - return; - } - lru_set_refs_flags(&new_flags, refs + 1); - } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); -} - static bool lru_gen_clear_refs(struct folio *folio) { int gen = folio_lru_gen(folio); @@ -388,7 +361,6 @@ static bool lru_gen_clear_refs(struct folio *folio) return true; folio_set_lru_refs(folio, 0); - folio_clear_workingset(folio); rcu_read_lock(); seq = READ_ONCE(folio_lruvec(folio)->lrugen.min_seq[type]); @@ -399,10 +371,6 @@ static bool lru_gen_clear_refs(struct folio *folio) #else /* !CONFIG_LRU_GEN */ -static void lru_gen_inc_refs(struct folio *folio) -{ -} - static bool lru_gen_clear_refs(struct folio *folio) { return false; @@ -428,7 +396,8 @@ void folio_mark_accessed(struct folio *folio) if (folio_test_dropbehind(folio)) return; if (lru_gen_enabled()) { - lru_gen_inc_refs(folio); + if (!folio_test_unevictable(folio)) + folio_inc_lru_refs(folio, 0); return; } @@ -474,21 +443,6 @@ void folio_add_lru(struct folio *folio) folio_test_unevictable(folio), folio); VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); - /* - * For refaulted workingset folios, set PG_active so they - * can be added to active generations. - * For prefaulted file folios, folio_mark_accessed() sets - * PG_referenced so lru_gen_folio_seq() places them into - * the second oldest generation. - */ - if (lru_gen_enabled() && !folio_test_unevictable(folio) && - lru_gen_in_fault() && !(current->flags & PF_MEMALLOC)) { - if (folio_test_workingset(folio)) - folio_set_active(folio); - else if (!folio_test_referenced(folio)) - folio_mark_accessed(folio); - } - folio_batch_add_and_move(folio, lru_add); } EXPORT_SYMBOL(folio_add_lru); diff --git a/mm/vmscan.c b/mm/vmscan.c index 32890e628240..9be0cd20dc54 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -902,38 +902,175 @@ enum folio_references { }; #ifdef CONFIG_LRU_GEN +/****************************************************************************** + * Referenced count feedback + ******************************************************************************/ + /* - * Only used on a mapped folio in the eviction (rmap walk) path, where promotion - * needs to be done by taking the folio off the LRU list and then adding it back - * with PG_active set. In contrast, the aging (page table walk) path uses - * folio_update_gen(). + * The folio_inc_lru_refs{_*} helpers below collect the referenced info + * (hotness) from other parts, including the page table walker, the rmap walk + * upon eviction, the rmap lookaround, and file descriptors + * (folio_mark_accessed). + * + * Page table accesses escalate a folio in two steps. The first access + * advances it one generation; a second access sends it to the newest + * generation. Executable file folios skip the first step and are promoted + * immediately, as reclaiming them causes IO thrashing. + * + * File descriptor accesses do not promote. They only defer eviction from + * the oldest generation, and only once the folio is a workingset folio + * (LRU_REFS_WORKINGSET), leaving the rest to PID protection. Page table + * accesses are treated more generously because the accessed bit is sticky + * (it under-counts repeated accesses) and because a page fault is more + * costly than file descriptor I/O. + * + * PID protection operates on tier > 0 folios. The one proactive promotion + * outside of it and the page table path is the overflow case where the + * referenced count exceeds LRU_REFS_MAX, which means the folio is hotter + * than everything else in its generation. + * + * Whenever a folio changes generation here its referenced count is capped at + * LRU_REFS_PROTECTED, so it starts at or below the protected tier regardless + * of its old-generation access history. PID protection (folio_inc_gen) caps + * at LRU_REFS_WORKINGSET independently. */ -static bool lru_gen_set_refs(struct folio *folio, const vma_flags_t *vma_flags) -{ - /* see the comment on LRU_REFS_FLAGS */ - if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) { - /* Activate file-backed executable folios after first usage. */ - if (is_exec_file_folio(folio, vma_flags)) { - folio_set_workingset(folio); - folio_set_lru_refs(folio, 0); - return true; + +/* + * Update the folio's lru refs indicator. The caller doesn't need to hold + * the folio lock, isolate the folio, or hold the lruvec lock. Used by both + * cache access (flags == 0) and page table access (LRU_REF_MAPPED, + * optionally with LRU_REF_EXEC). + */ +void folio_inc_lru_refs(struct folio *folio, unsigned int flags) +{ + int max_gen, min_gen; + int type, refs, old_gen, gen; + unsigned long new_flags, old_flags, max_seq; + struct lru_gen_folio *lrugen; + struct lruvec *lruvec = NULL; + + type = folio_is_file_lru(folio); + old_flags = READ_ONCE(*folio_flags(folio, 0)); + do { + new_flags = old_flags; + old_gen = lru_get_gen_flags(old_flags); + refs = lru_get_refs_flags(old_flags) + 1; + gen = old_gen; + if (old_gen < 0) + goto out; + /* + * Lock the lruvec if the folio is on-list. We are already + * doing lazy promotion so in theory we don't need this, + * but for now, concurrent aging would still corrupt the + * size counters. This is a temporary limitation and + * will be lifted very soon, so the lock here is not a + * performance concern. + */ + if (!lruvec) { + lruvec = lruvec_live_lock_irq(folio_lruvec(folio)); + lrugen = &lruvec->lrugen; } + max_seq = READ_ONCE(lrugen->max_seq); + max_gen = lru_gen_from_seq(max_seq); + min_gen = lru_gen_from_seq(READ_ONCE(lrugen->min_seq[type])); + if (old_gen == max_gen) + goto out; - folio_set_lru_refs(folio, 1); - return false; - } + if (flags & (LRU_REF_MAPPED | LRU_REF_EXEC)) { + /* Promote second page table access or executable */ + if (refs > LRU_REFS_REFERENCED || flags & LRU_REF_EXEC) + gen = max_gen; + else + gen = (old_gen + 1) % MAX_NR_GENS; + refs = min(refs, LRU_REFS_PROTECTED); + } else if (refs > LRU_REFS_MAX) { + /* LRU refs counting overflow, bump the gen */ + gen = (old_gen + 1) % MAX_NR_GENS; + refs = LRU_REFS_PROTECTED; + } else if (old_gen == min_gen && refs >= LRU_REFS_WORKINGSET) { + /* Defer eviction of just accessed workingset */ + gen = (old_gen + 1) % MAX_NR_GENS; + refs = min(refs, LRU_REFS_PROTECTED); + } +out: + refs = min(refs, LRU_REFS_MAX); + lru_set_refs_flags(&new_flags, refs); + if (gen != old_gen) + lru_set_gen_flags(&new_flags, gen); + } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); - /* Promote on second access */ - if (folio_lru_refs(folio) > 1) { - folio_set_workingset(folio); - folio_set_lru_refs(folio, 0); - } else { - folio_mark_accessed(folio); - } - return true; + if (gen != old_gen) + lru_gen_update_size(lruvec, folio, old_gen, gen); + if (lruvec) + lruvec_unlock_irq(lruvec); +} + +/* + * Update the folio's lru refs indicator during a page table walk. + * max_seq is stable since this runs inside the aging process. + * + * Returns the old generation and stores the new generation in @new_gen when + * the folio is promoted (to max_gen) or advanced by one generation. + * Returns -1 if no gen change occurred. + */ +static int folio_inc_lru_refs_walk(struct folio *folio, struct lruvec *lruvec, + const vma_flags_t *vma_flags, + int *new_gen, int *type) +{ + unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0)); + unsigned long max_seq = READ_ONCE(lruvec->lrugen.max_seq); + int refs, gen, max_gen, ret; + + max_gen = lru_gen_from_seq(max_seq); + + do { + gen = lru_get_gen_flags(old_flags); + refs = lru_get_refs_flags(old_flags) + 1; + new_flags = old_flags; + + if (gen >= 0 && gen != max_gen) { + ret = gen; + /* Promote second page table access or executable */ + if (refs > LRU_REFS_REFERENCED || is_exec_file_folio(folio, vma_flags)) + *new_gen = max_gen; + else + *new_gen = (gen + 1) % MAX_NR_GENS; + lru_set_gen_flags(&new_flags, *new_gen); + lru_set_refs_flags(&new_flags, min(refs, LRU_REFS_PROTECTED)); + } else { + ret = -1; + lru_set_refs_flags(&new_flags, min(refs, LRU_REFS_MAX)); + } + } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); + + *type = folio_flags_is_file_lru(&old_flags); + return ret; +} + +/* + * Update the lru refs indicator of an isolated folio, only used on + * mapped folios upon the final eviction. + * + * Increments the refs count (capped at LRU_REFS_PROTECTED). Returns true + * if the caller should activate the folio (second access or + * executable), false to keep it in the eviction list. + */ +static bool folio_inc_lru_refs_isolated(struct folio *folio, const vma_flags_t *vma_flags) +{ + unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0)); + int refs; + + do { + new_flags = old_flags; + refs = lru_get_refs_flags(old_flags) + 1; + lru_set_refs_flags(&new_flags, min(refs, LRU_REFS_PROTECTED)); + } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); + + /* Promote second page table access or executable */ + return refs > LRU_REFS_REFERENCED || is_exec_file_folio(folio, vma_flags); } #else -static bool lru_gen_set_refs(struct folio *folio, const vma_flags_t *vma_flags) +static bool folio_inc_lru_refs_isolated(struct folio *folio, const vma_flags_t *vma_flags) { return false; } @@ -968,7 +1105,8 @@ static enum folio_references folio_check_references(struct folio *folio, if (!referenced_ptes) return FOLIOREF_RECLAIM; - return lru_gen_set_refs(folio, &vma_flags) ? FOLIOREF_ACTIVATE : FOLIOREF_KEEP; + return folio_inc_lru_refs_isolated(folio, &vma_flags) ? + FOLIOREF_ACTIVATE : FOLIOREF_KEEP; } referenced_folio = folio_test_clear_referenced(folio); @@ -3344,50 +3482,15 @@ static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv) * the aging ******************************************************************************/ -/* promote pages accessed through page tables */ -static int folio_update_gen(struct folio *folio, int new_gen, int *type, - const vma_flags_t *vma_flags) -{ - unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0)); - int old_gen; - - /* - * See the comment on LRU_REFS_FLAGS, and activate file-backed - * executable folios after first usage to avoid typical IO - * thrashing from reclaiming. - */ - if (!folio_test_referenced(folio) && !folio_test_workingset(folio) && - !is_exec_file_folio(folio, vma_flags)) { - folio_set_lru_refs(folio, 1); - return -1; - } - - do { - old_gen = lru_get_gen_flags(old_flags); - new_flags = old_flags; - - /* lru_gen_del_folio() has isolated this page? */ - if (old_gen < 0) - break; - - lru_set_gen_flags(&new_flags, new_gen); - lru_set_refs_flags(&new_flags, 0); - new_flags |= BIT(PG_workingset); - } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); - - *type = folio_flags_is_file_lru(&old_flags); - return old_gen; -} - static int __folio_inc_gen(struct folio *folio, int old_gen, bool *increased) { unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0)); - int new_gen; + int refs, new_gen; do { new_gen = lru_get_gen_flags(old_flags); - /* folio_update_gen() has promoted this page? */ + /* folio_inc_lru_refs() has promoted this page? */ if (new_gen >= 0 && new_gen != old_gen) { if (increased) *increased = false; @@ -3396,9 +3499,9 @@ static int __folio_inc_gen(struct folio *folio, int old_gen, bool *increased) new_flags = old_flags; new_gen = (old_gen + 1) % MAX_NR_GENS; - + refs = lru_get_refs_flags(old_flags); lru_set_gen_flags(&new_flags, new_gen); - lru_set_refs_flags(&new_flags, 0); + lru_set_refs_flags(&new_flags, min(refs, LRU_REFS_WORKINGSET)); } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); if (increased) @@ -3406,17 +3509,21 @@ static int __folio_inc_gen(struct folio *folio, int old_gen, bool *increased) return new_gen; } -/* protect pages accessed multiple times through file descriptors */ +/* + * Force bump a folio's generation. Used for PID protection or to defer + * eviction of a temporarily unevictable folio. + */ static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio) { + bool gen_increased; int type = folio_is_file_lru(folio); struct lru_gen_folio *lrugen = &lruvec->lrugen; int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); - bool gen_increased; new_gen = __folio_inc_gen(folio, old_gen, &gen_increased); if (gen_increased) lru_gen_update_size(lruvec, folio, old_gen, new_gen); + return new_gen; } @@ -3611,25 +3718,25 @@ static void walk_update_folio(struct lru_gen_mm_walk *walk, struct vm_area_struc struct lruvec *lruvec, struct folio *folio, bool dirty) { int new_gen, old_gen, type; + unsigned int flags = LRU_REF_MAPPED; if (!folio) return; - new_gen = lru_gen_from_seq(READ_ONCE(lruvec->lrugen.max_seq)); - if (dirty && !folio_test_dirty(folio) && !(folio_test_anon(folio) && folio_test_swapbacked(folio) && !folio_test_swapcache(folio))) folio_mark_dirty(folio); if (walk) { - old_gen = folio_update_gen(folio, new_gen, &type, &vma->flags); - if (old_gen >= 0 && old_gen != new_gen) + old_gen = folio_inc_lru_refs_walk(folio, lruvec, &vma->flags, + &new_gen, &type); + if (old_gen >= 0) update_batch_size(walk, folio, old_gen, new_gen, type); - } else if (lru_gen_set_refs(folio, &vma->flags)) { - old_gen = folio_lru_gen(folio); - if (old_gen >= 0 && old_gen != new_gen) - folio_activate(folio); + } else { + if (is_exec_file_folio(folio, &vma->flags)) + flags |= LRU_REF_EXEC; + folio_inc_lru_refs(folio, flags); } } @@ -4036,7 +4143,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness) struct folio *folio = list_entry(pos, struct folio, lru); long nr_pages = folio_nr_pages(folio); int refs = folio_lru_refs(folio); - bool workingset = folio_test_workingset(folio); + int tier = lru_tier_from_refs(refs); bool gen_increased; VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); @@ -4056,13 +4163,8 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness) delta += nr_pages; batch_end = &folio->lru; - /* don't count the workingset being lazily promoted */ - if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) { - int tier = lru_tier_from_refs(refs, workingset); - - WRITE_ONCE(lrugen->protected[hist][type][tier], - lrugen->protected[hist][type][tier] + nr_pages); - } + WRITE_ONCE(lrugen->protected[hist][type][tier], + lrugen->protected[hist][type][tier] + nr_pages); } else { flush_lru_batch(head, &batch_end, target_list); list_move(&folio->lru, &lrugen->folios[new_gen][type][zone]); @@ -4777,8 +4879,7 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c int zone = folio_zonenum(folio); int delta = folio_nr_pages(folio); int refs = folio_lru_refs(folio); - bool workingset = folio_test_workingset(folio); - int tier = lru_tier_from_refs(refs, workingset); + int tier = lru_tier_from_refs(refs); struct lru_gen_folio *lrugen = &lruvec->lrugen; VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio); @@ -4794,17 +4895,15 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c } /* protected */ - if (tier > tier_idx || refs + workingset == BIT(LRU_REFS_WIDTH) + 1) { + if (tier > tier_idx) { + int hist = lru_hist_from_seq(lrugen->min_seq[type]); + gen = folio_inc_gen(lruvec, folio); list_move(&folio->lru, &lrugen->folios[gen][type][zone]); - /* don't count the workingset being lazily promoted */ - if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) { - int hist = lru_hist_from_seq(lrugen->min_seq[type]); + WRITE_ONCE(lrugen->protected[hist][type][tier], + lrugen->protected[hist][type][tier] + delta); - WRITE_ONCE(lrugen->protected[hist][type][tier], - lrugen->protected[hist][type][tier] + delta); - } return true; } @@ -4832,10 +4931,6 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca return false; } - /* see the comment on LRU_REFS_FLAGS */ - if (!folio_test_referenced(folio)) - folio_set_lru_refs(folio, 0); - success = lru_gen_del_folio(lruvec, folio, true); VM_WARN_ON_ONCE_FOLIO(!success, folio); @@ -4923,13 +5018,14 @@ static int get_tier_idx(struct lruvec *lruvec, int type) struct ctrl_pos sp, pv = {}; /* - * To leave a margin for fluctuations, use a larger gain factor (2:3). - * This value is chosen because any other tier would have at least twice - * as many refaults as the first tier. + * To leave a margin for fluctuations, use a larger gain factor (1:2). + * Stop at the first tier whose refault rate is clearly worse than + * that of the cumulative mass of the tiers below it; the PID + * protects the tiers above it. */ - read_ctrl_pos(lruvec, type, LRU_TIER_MIN, LRU_TIER_MIN, 2, &sp); for (tier = LRU_TIER_MIN + 1; tier <= LRU_TIER_MAX; tier++) { - read_ctrl_pos(lruvec, type, tier, tier, 3, &pv); + read_ctrl_pos(lruvec, type, LRU_TIER_MIN, tier - 1, 1, &sp); + read_ctrl_pos(lruvec, type, tier, tier, 2, &pv); if (!positive_ctrl_err(&sp, &pv)) break; } @@ -5067,10 +5163,8 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec, continue; } - /* See the comments on LRU_REFS_FLAGS */ - folio_set_lru_refs(folio, 0); - - /* don't add rejected folios to the oldest generation */ + /* Reset folio's refs before return */ + folio_set_lru_refs(folio, min(folio_lru_refs(folio), LRU_REFS_WORKINGSET)); if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]) folio_set_active(folio); } diff --git a/mm/workingset.c b/mm/workingset.c index 8412f4840ae3..44be539b93d2 100644 --- a/mm/workingset.c +++ b/mm/workingset.c @@ -188,6 +188,12 @@ #define EVICTION_SHIFT_ANON (EVICTION_SHIFT + SWAP_COUNT_SHIFT) #define EVICTION_MASK (~0UL >> EVICTION_SHIFT) #define EVICTION_MASK_ANON (~0UL >> EVICTION_SHIFT_ANON) +/* + * LRU refs uses LRU_REFS_WIDTH + 2 bits, the 2 bits being PG_workingset + * and PG_referenced. The lowest bit is recorded in the workingset field + * of the shadow entry (to reuse pack_shadow()). + */ +#define LRU_REFS_BITS ((LRU_REFS_WIDTH + 2) - 1) /* * Eviction timestamps need to be able to cover the full range of @@ -242,13 +248,12 @@ static void *lru_gen_eviction(struct folio *folio) int type = folio_is_file_lru(folio); int delta = folio_nr_pages(folio); int refs = folio_lru_refs(folio); - bool workingset = folio_test_workingset(folio); - int tier = lru_tier_from_refs(refs, workingset); + int tier = lru_tier_from_refs(refs); struct mem_cgroup *memcg; struct pglist_data *pgdat = folio_pgdat(folio); unsigned short memcg_id; - BUILD_BUG_ON(LRU_GEN_WIDTH + LRU_REFS_WIDTH > + BUILD_BUG_ON(LRU_GEN_WIDTH + LRU_REFS_BITS > BITS_PER_LONG - max(EVICTION_SHIFT, EVICTION_SHIFT_ANON)); rcu_read_lock(); @@ -256,14 +261,14 @@ static void *lru_gen_eviction(struct folio *folio) lruvec = mem_cgroup_lruvec(memcg, pgdat); lrugen = &lruvec->lrugen; min_seq = READ_ONCE(lrugen->min_seq[type]); - token = (min_seq << LRU_REFS_WIDTH) | max(refs - 1, 0); + token = (min_seq << LRU_REFS_BITS) | refs >> 1; hist = lru_hist_from_seq(min_seq); atomic_long_add(delta, &lrugen->evicted[hist][type][tier]); memcg_id = mem_cgroup_private_id(memcg); rcu_read_unlock(); - return pack_shadow(memcg_id, pgdat, token, workingset, type); + return pack_shadow(memcg_id, pgdat, token, refs & 1, type); } /* @@ -284,9 +289,9 @@ static bool lru_gen_test_recent(void *shadow, struct lruvec **lruvec, *lruvec = mem_cgroup_lruvec(memcg, pgdat); max_seq = READ_ONCE((*lruvec)->lrugen.max_seq); - max_seq &= (file ? EVICTION_MASK : EVICTION_MASK_ANON) >> LRU_REFS_WIDTH; + max_seq &= (file ? EVICTION_MASK : EVICTION_MASK_ANON) >> LRU_REFS_BITS; - return abs_diff(max_seq, *token >> LRU_REFS_WIDTH) < MAX_NR_GENS; + return abs_diff(max_seq, *token >> LRU_REFS_BITS) < MAX_NR_GENS; } static void lru_gen_refault(struct folio *folio, void *shadow) @@ -314,22 +319,26 @@ static void lru_gen_refault(struct folio *folio, void *shadow) lrugen = &lruvec->lrugen; hist = lru_hist_from_seq(READ_ONCE(lrugen->min_seq[type])); - refs = (token & (BIT(LRU_REFS_WIDTH) - 1)) + 1; - tier = lru_tier_from_refs(refs, workingset); + refs = ((token & (BIT(LRU_REFS_BITS) - 1)) << 1) + workingset; + tier = lru_tier_from_refs(refs); atomic_long_add(delta, &lrugen->refaulted[hist][type][tier]); - if (workingset) { - /* - * see folio_add_lru(), where folio_set_active() is - * called for workingset folios - */ - if (lru_gen_in_fault()) + /* + * Activate a fault-driven refault folio, which would have been + * promoted had it stayed in memory. + */ + if (refs >= LRU_REFS_REFERENCED) { + if (lru_gen_in_fault()) { + folio_set_active(folio); mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + type, delta); - folio_set_workingset(folio); + } + /* Refault is also promotion, cap the refs like folio_inc_lru_refs */ + folio_set_lru_refs(folio, min(refs, LRU_REFS_PROTECTED)); + } + + if (refs >= LRU_REFS_WORKINGSET) mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + type, delta); - } else - set_mask_bits(&folio->flags.f, LRU_REFS_MASK, (refs - 1UL) << LRU_REFS_PGOFF); unlock: rcu_read_unlock(); } -- 2.55.0 From: Kairui Song Pure code shuffle, no behavior change. Make the folio LRU refs helpers available regardless of CONFIG_LRU_GEN; without MGLRU the refs bits are never set, so they are inert. Move the lru_gen_* helpers into the CONFIG_LRU_GEN section unchanged. This prepares for unifying the API for checking folio referenced and workingset status. folio_migrate_lru_refs() now migrates the complete refs count, including PG_workingset (bit 1 of the encoding), bitwise identical to the direct copy dropped from folio_migrate_flags(). It also gains off-LRU VM_WARN_ON_ONCE checks, compiled out unless CONFIG_DEBUG_VM. lru_set_refs_flags() skips the nonexistent mask bits when LRU_REFS_WIDTH collapses to 0, keeping the generic helpers inert for !CONFIG_LRU_GEN builds. Signed-off-by: Kairui Song --- include/linux/mm_inline.h | 145 +++++++++++++++++++++++----------------------- mm/migrate.c | 2 - 2 files changed, 72 insertions(+), 75 deletions(-) diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 7c8288661edd..5a62b83b4168 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -105,54 +105,6 @@ static __always_inline enum lru_list folio_lru_list(const struct folio *folio) return lru; } -#ifdef CONFIG_LRU_GEN - -static inline bool lru_gen_switching(void) -{ - DECLARE_STATIC_KEY_FALSE(lru_switch); - - return static_branch_unlikely(&lru_switch); -} -#ifdef CONFIG_LRU_GEN_ENABLED -static inline bool lru_gen_enabled(void) -{ - DECLARE_STATIC_KEY_TRUE(lru_gen_caps[NR_LRU_GEN_CAPS]); - - return static_branch_likely(&lru_gen_caps[LRU_GEN_CORE]); -} -#else -static inline bool lru_gen_enabled(void) -{ - DECLARE_STATIC_KEY_FALSE(lru_gen_caps[NR_LRU_GEN_CAPS]); - - return static_branch_unlikely(&lru_gen_caps[LRU_GEN_CORE]); -} -#endif - -static inline bool lru_gen_in_fault(void) -{ - return current->in_lru_fault; -} - -static inline int lru_gen_from_seq(unsigned long seq) -{ - return seq % MAX_NR_GENS; -} - -static inline int lru_hist_from_seq(unsigned long seq) -{ - return seq % NR_HIST_GENS; -} - -static inline int lru_tier_from_refs(unsigned int refs) -{ - BUILD_BUG_ON(fls(LRU_REFS_MAX - 1) > MAX_NR_TIERS - 1); - VM_WARN_ON_ONCE(refs > LRU_REFS_MAX); - if (refs < LRU_REFS_WORKINGSET) - return 0; - return fls(refs - 1); -} - /** * lru_set_gen_flags - Set the LRU generation number to specified folio flags. * @flags: pointer to the folio flags @@ -205,7 +157,8 @@ static inline void lru_set_refs_flags(unsigned long *flags, unsigned int refs) *flags |= BIT(PG_referenced); if (refs & BIT(1)) *flags |= BIT(PG_workingset); - *flags |= ((unsigned long)refs >> 2) << LRU_REFS_PGOFF; + if (LRU_REFS_WIDTH) + *flags |= ((unsigned long)refs >> 2) << LRU_REFS_PGOFF; } /** @@ -240,7 +193,77 @@ static inline void folio_set_lru_refs(struct folio *folio, unsigned int refs) } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); } +#ifdef CONFIG_LRU_GEN void folio_inc_lru_refs(struct folio *folio, unsigned int flags); +#else +static inline void folio_inc_lru_refs(struct folio *folio, unsigned int flags) +{ + /* Should not be called with !CONFIG_LRU_GEN */ + WARN_ON_ONCE(1); +} +#endif + +/** + * folio_migrate_lru_refs - copy the reference state to a new folio + * @new: the destination folio + * @old: the source folio + * + * Transfer the reference state to @new during migration: the MGLRU + * refs count, or PG_referenced and PG_workingset (bits 0-1 of the + * encoding) for the active/inactive LRU. + */ +static inline void folio_migrate_lru_refs(struct folio *new, const struct folio *old) +{ + folio_set_lru_refs(new, folio_lru_refs(old)); +} + +#ifdef CONFIG_LRU_GEN + +static inline bool lru_gen_switching(void) +{ + DECLARE_STATIC_KEY_FALSE(lru_switch); + + return static_branch_unlikely(&lru_switch); +} +#ifdef CONFIG_LRU_GEN_ENABLED +static inline bool lru_gen_enabled(void) +{ + DECLARE_STATIC_KEY_TRUE(lru_gen_caps[NR_LRU_GEN_CAPS]); + + return static_branch_likely(&lru_gen_caps[LRU_GEN_CORE]); +} +#else +static inline bool lru_gen_enabled(void) +{ + DECLARE_STATIC_KEY_FALSE(lru_gen_caps[NR_LRU_GEN_CAPS]); + + return static_branch_unlikely(&lru_gen_caps[LRU_GEN_CORE]); +} +#endif + +static inline bool lru_gen_in_fault(void) +{ + return current->in_lru_fault; +} + +static inline int lru_gen_from_seq(unsigned long seq) +{ + return seq % MAX_NR_GENS; +} + +static inline int lru_hist_from_seq(unsigned long seq) +{ + return seq % NR_HIST_GENS; +} + +static inline int lru_tier_from_refs(unsigned int refs) +{ + BUILD_BUG_ON(fls(LRU_REFS_MAX - 1) > MAX_NR_TIERS - 1); + VM_WARN_ON_ONCE(refs > LRU_REFS_MAX); + if (refs < LRU_REFS_WORKINGSET) + return 0; + return fls(refs - 1); +} static inline int folio_lru_gen(const struct folio *folio) { @@ -388,20 +411,6 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, return true; } - -/** - * folio_migrate_lru_refs - copy the reference state to a new folio - * @new: the destination folio - * @old: the source folio - * - * Transfer the reference state to @new during migration: the MGLRU - * refs count, including PG_referenced, or just PG_referenced for the - * active/inactive LRU. - */ -static inline void folio_migrate_lru_refs(struct folio *new, const struct folio *old) -{ - folio_set_lru_refs(new, folio_lru_refs(old)); -} #else /* !CONFIG_LRU_GEN */ static inline bool lru_gen_enabled(void) @@ -428,16 +437,6 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, { return false; } - -static inline void folio_inc_lru_refs(struct folio *folio, unsigned int flags) -{ -} - -static inline void folio_migrate_lru_refs(struct folio *new, const struct folio *old) -{ - if (folio_test_referenced(old)) - folio_set_referenced(new); -} #endif /* CONFIG_LRU_GEN */ static __always_inline diff --git a/mm/migrate.c b/mm/migrate.c index a369d0c95c38..b1ccffc4e7cd 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -786,8 +786,6 @@ void folio_migrate_flags(struct folio *newfolio, struct folio *folio) folio_set_active(newfolio); } else if (folio_test_clear_unevictable(folio)) folio_set_unevictable(newfolio); - if (folio_test_workingset(folio)) - folio_set_workingset(newfolio); if (folio_test_checked(folio)) folio_set_checked(newfolio); /* -- 2.55.0 From: Kairui Song Pure code shuffle, no behavior change. lru_gen_update_size() now only updates the per-generation counters plus the promotion move; the addition and deletion active/inactive accounting move to lru_gen_add_folio() and lru_gen_del_folio(), which know the generation and can apply the active-window test directly. The unevictable case is untouched: unevictable folios never enter lru_gen_add_folio() and keep being accounted by lruvec_add_folio() as before. Signed-off-by: Kairui Song --- include/linux/mm_inline.h | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 5a62b83b4168..8700563dade8 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -298,21 +298,9 @@ static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *foli if (new_gen >= 0) atomic_long_add(delta, &lrugen->nr_pages[new_gen][type][zone]); - /* addition */ - if (old_gen < 0) { - if (lru_gen_is_active(lruvec, new_gen)) - lru += LRU_ACTIVE; - __update_lru_size(lruvec, lru, zone, delta); + /* return now if not a promotion */ + if (old_gen < 0 || new_gen < 0) return; - } - - /* deletion */ - if (new_gen < 0) { - if (lru_gen_is_active(lruvec, old_gen)) - lru += LRU_ACTIVE; - __update_lru_size(lruvec, lru, zone, -delta); - return; - } /* promotion */ if (!lru_gen_is_active(lruvec, old_gen) && lru_gen_is_active(lruvec, new_gen)) { @@ -366,6 +354,8 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, int gen = folio_lru_gen(folio); int type = folio_is_file_lru(folio); int zone = folio_zonenum(folio); + int delta = folio_nr_pages(folio); + enum lru_list lru = type * LRU_INACTIVE_FILE; struct lru_gen_folio *lrugen = &lruvec->lrugen; BUILD_BUG_ON(BIT(LRU_GEN_WIDTH - 1) != MAX_NR_GENS); @@ -381,6 +371,10 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, set_mask_bits(folio_flags(folio, 0), LRU_GEN_MASK | BIT(PG_active), flags); lru_gen_update_size(lruvec, folio, -1, gen); + if (lru_gen_is_active(lruvec, gen)) + lru += LRU_ACTIVE; + __update_lru_size(lruvec, lru, zone, delta); + /* for folio_rotate_reclaimable() */ if (reclaiming) list_add_tail(&folio->lru, &lrugen->folios[gen][type][zone]); @@ -394,6 +388,9 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, { unsigned long flags; int gen = folio_lru_gen(folio); + int zone = folio_zonenum(folio); + int delta = folio_nr_pages(folio); + enum lru_list lru = folio_is_file_lru(folio) * LRU_INACTIVE_FILE; if (gen < 0) return false; @@ -407,6 +404,9 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; lru_gen_update_size(lruvec, folio, gen, -1); + if (lru_gen_is_active(lruvec, gen)) + lru += LRU_ACTIVE; + __update_lru_size(lruvec, lru, zone, -delta); list_del(&folio->lru); return true; -- 2.55.0 From: Kairui Song Introduce folio_inc_lru_refs_fast(): raise refs from 0 to LRU_REFS_REFERENCED via cmpxchg, or just set PG_referenced for the classical LRU as before. No counter update, promotion, and guarteens no other complex operations in following commits. Convert the gup fast-path sites. No behavior change for classical LRU. Signed-off-by: Kairui Song --- include/linux/mm_inline.h | 25 +++++++++++++++++++++++++ mm/gup.c | 6 +++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 8700563dade8..23ddd5d78571 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -439,6 +439,31 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, } #endif /* CONFIG_LRU_GEN */ +/** + * folio_inc_lru_refs_fast - Bump folio refs count without promotion. + * @folio: the folio + * + * Raise refs from 0 to LRU_REFS_REFERENCED and leave hotter folios + * untouched. For the classical LRU, a plain PG_referenced set. + */ +static __always_inline void folio_inc_lru_refs_fast(struct folio *folio) +{ + unsigned long new_flags, old_flags; + + if (!lru_gen_enabled()) { + folio_set_referenced(folio); + return; + } + + old_flags = READ_ONCE(*folio_flags(folio, 0)); + do { + if (lru_get_refs_flags(old_flags)) + break; + new_flags = old_flags; + lru_set_refs_flags(&new_flags, LRU_REFS_REFERENCED); + } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); +} + static __always_inline void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio) { diff --git a/mm/gup.c b/mm/gup.c index e49f020f721d..cbf609dc8866 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -2899,7 +2899,7 @@ static unsigned long gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, gup_put_folio(folio, 1, flags); goto pte_unmap; } - folio_set_referenced(folio); + folio_inc_lru_refs_fast(folio); pages[nr_pages] = page; nr_pages++; } while (ptep++, addr += PAGE_SIZE, addr != end); @@ -2968,7 +2968,7 @@ static unsigned long gup_fast_pmd_leaf(pmd_t orig, pmd_t *pmdp, for (i = 0; i < nr_pages; i++) *(pages++) = page++; - folio_set_referenced(folio); + folio_inc_lru_refs_fast(folio); return nr_pages; } @@ -3010,7 +3010,7 @@ static unsigned long gup_fast_pud_leaf(pud_t orig, pud_t *pudp, for (i = 0; i < nr_pages; i++) *(pages++) = page++; - folio_set_referenced(folio); + folio_inc_lru_refs_fast(folio); return nr_pages; } -- 2.55.0 From: Kairui Song For MGLRU, switch smap to use the folio refs count API so smap will report all folio with referenced count >= 1 as "Referenced". Current smap checking PG_referenced is causing folios to flick between referenced and not-reference status, because for both MGLRU and active/inactive LRU, PG_referenced may got cleared on second access. (Increase of LRU referenced times count for MGLRU, and movig to active list active/inactive all clears that bit). After this, we will have a more reliable and useful reading for MGLRU after the FG change. The behavior is basically identical to what we had before the series. And there is no behavior change for classical LRU. Signed-off-by: Kairui Song --- fs/proc/task_mmu.c | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index e671b4fd8ded..9cc47efbaa51 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -880,6 +880,22 @@ static void smaps_page_accumulate(struct mem_size_stats *mss, } } +static bool smap_check_folio_referenced(struct folio *folio) +{ + if (lru_gen_enabled()) + return folio_lru_refs(folio); + else + return folio_test_referenced(folio); +} + +static void smap_clear_folio_referenced(struct folio *folio) +{ + if (lru_gen_enabled()) + folio_set_lru_refs(folio, 0); + else + folio_clear_referenced(folio); +} + static void smaps_account(struct mem_size_stats *mss, struct page *page, bool compound, bool young, bool dirty, bool locked, bool present) @@ -906,7 +922,7 @@ static void smaps_account(struct mem_size_stats *mss, struct page *page, mss->resident += size; /* Accumulate the size in pages that have been accessed. */ - if (young || folio_test_young(folio) || folio_test_referenced(folio)) + if (young || folio_test_young(folio) || smap_check_folio_referenced(folio)) mss->referenced += size; /* @@ -1709,7 +1725,7 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, /* Clear accessed and referenced bits. */ pmdp_test_and_clear_young(vma, addr, pmd); folio_test_clear_young(folio); - folio_clear_referenced(folio); + smap_clear_folio_referenced(folio); out: spin_unlock(ptl); return 0; @@ -1738,7 +1754,7 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, /* Clear accessed and referenced bits. */ ptep_test_and_clear_young(vma, addr, pte); folio_test_clear_young(folio); - folio_clear_referenced(folio); + smap_clear_folio_referenced(folio); } pte_unmap_unlock(pte - 1, ptl); cond_resched(); -- 2.55.0 From: Kairui Song For the active/inactive LRU, madvise clears PG_referenced so that one more access is not enough to reactivate a folio, and keeps PG_workingset on a folio demoted out of the active list so its refault is still accounted as a workingset refault (PSI). MGLRU keeps that history in the folio's refs count instead, which now lives in PG_referenced, PG_workingset and LRU_REFS_MASK, so writing to those bits directly corrupts the count. The two hints want different things: - MADV_COLD resets the count in folio_deactivate(), which also moves the folio to the oldest generation. - MADV_PAGEOUT isolates the folio and hands it to reclaim_pages(), which ignores the count, so keep it and let the eviction shadow record the folio's real hotness. PG_young is idle-page-tracking state, not a reclaim input, so it is still cleared for both LRUs. Signed-off-by: Kairui Song --- mm/madvise.c | 48 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/mm/madvise.c b/mm/madvise.c index f75a9d139980..cdd1bebf2485 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -361,6 +361,38 @@ static inline int madvise_folio_pte_batch(unsigned long addr, unsigned long end, FPB_MERGE_YOUNG_DIRTY); } +/* + * Drop the recency information madvise() is expected to drop before + * deactivating or reclaiming a folio. + */ +static void madvise_cold_or_pageout_prep_folio(struct folio *folio) +{ + /* + * VM couldn't reclaim the folio unless we clear PG_young. + * As a side effect, it makes confuse idle-page tracking + * because they will miss recent referenced history. + */ + folio_test_clear_young(folio); + + /* + * MGLRU keeps this history in the refs count. MADV_COLD resets + * it in folio_deactivate(), MADV_PAGEOUT hands the folio to + * reclaim_pages(), which ignores the count, so keep it, and let + * the eviction shadow record the folio's real hotness. + */ + if (lru_gen_enabled()) + return; + + /* + * For the active/inactive LRU, a folio demoted out of the active + * list should have PG_workingset so its refault is still accounted + * as a workingset refault. + */ + folio_clear_referenced(folio); + if (folio_test_active(folio)) + folio_set_workingset(folio); +} + static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, struct mm_walk *walk) @@ -438,10 +470,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, tlb_remove_pmd_tlb_entry(tlb, pmd, addr); } - folio_clear_referenced(folio); - folio_test_clear_young(folio); - if (folio_test_active(folio)) - folio_set_workingset(folio); + madvise_cold_or_pageout_prep_folio(folio); if (pageout) { if (folio_isolate_lru(folio)) { if (folio_test_unevictable(folio)) @@ -547,16 +576,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, tlb_remove_tlb_entries(tlb, pte, nr, addr); } - /* - * We are deactivating a folio for accelerating reclaiming. - * VM couldn't reclaim the folio unless we clear PG_young. - * As a side effect, it makes confuse idle-page tracking - * because they will miss recent referenced history. - */ - folio_clear_referenced(folio); - folio_test_clear_young(folio); - if (folio_test_active(folio)) - folio_set_workingset(folio); + madvise_cold_or_pageout_prep_folio(folio); if (pageout) { if (folio_isolate_lru(folio)) { if (folio_test_unevictable(folio)) -- 2.55.0 From: Kairui Song damon_pa_pageout() cleared PG_referenced directly, which under MGLRU may corrupts the folio's refs count in later commits. That flag is now bit 0 of the count (LRU_REFS_FLAGS). Reset the count instead under MGLRU. Signed-off-by: Kairui Song --- mm/damon/paddr.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 79195026b903..38d3ff7019e3 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -280,7 +280,14 @@ static unsigned long damon_pa_pageout(struct damon_region *r, else *sz_filter_passed += folio_size(folio) / addr_unit; - folio_clear_referenced(folio); + /* + * DAMON only gets here for regions it measured as cold, + * so the hotness can be, and better be dropped. + */ + if (lru_gen_enabled()) + folio_set_lru_refs(folio, 0); + else + folio_clear_referenced(folio); folio_test_clear_young(folio); if (!folio_isolate_lru(folio)) goto put_folio; -- 2.55.0 From: Kairui Song The behavior of updating the folio's access info isn't consistent for huge mapping splitting or ordinary unmapping. The page table's young flag has to be translated into folio's access info. Right now it only check and set folio's referenced flag, which isn't enough since folio flags update on access have its rules. Ordinary unmapping (zapping) calls folio_mark_accessed(), and it also checks if the VMA has recency to avoid false updates. So first just use the right helper here to be more consistent. Signed-off-by: Kairui Song --- mm/huge_memory.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 30b7c63b0e35..28a9aa4c8b20 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -3109,8 +3109,8 @@ static void __split_huge_pud_locked(struct vm_area_struct *vma, pud_t *pud, if (!folio_test_dirty(folio) && pud_dirty(old_pud)) folio_mark_dirty(folio); - if (!folio_test_referenced(folio) && pud_young(old_pud)) - folio_set_referenced(folio); + if (pud_young(old_pud) && vma_has_recency(vma)) + folio_mark_accessed(folio); folio_remove_rmap_pud(folio, page, vma); add_mm_counter(vma->vm_mm, mm_counter_file(folio), -HPAGE_PUD_NR); @@ -3232,8 +3232,8 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, folio = page_folio(page); if (!folio_test_dirty(folio) && pmd_dirty(old_pmd)) folio_mark_dirty(folio); - if (!folio_test_referenced(folio) && pmd_young(old_pmd)) - folio_set_referenced(folio); + if (pmd_young(old_pmd) && vma_has_recency(vma)) + folio_mark_accessed(folio); folio_remove_rmap_pmd(folio, page, vma); add_mm_counter(mm, mm_counter_file(folio), -HPAGE_PMD_NR); folio_put(folio); -- 2.55.0 From: Kairui Song Currently the active/inactive LRU sizes exported through /proc/vmstat and memory.stat are derived from the generation window: folios in the two newest generations are accounted as active, the rest as inactive. This has always caused many problems: - Unmapped file folios stick to the oldest gens, so active files are always under-reported, unlike classical LRU. - The reading is jumpy: on aging, the second-newest gen, previously active, becomes inactive all at once, causing a huge drop of active folios for no reason from time to time. - The reading is reversed on swapless machines, and the active anon number is an oscillating random number. When swappiness is 0 (no swap or swap is full), MGLRU ages anon by moving the gens directly without walk or touching any folio, to reduce the overhead. So all inactive anon folios will suddenly count as active, or vice versa, which breaks many metric reading components, and the reading itself doesn't make any sense in any way. With frequency guided promotion, hotness is already tracked per folio by its referenced count, so we can do folio level accounting instead: a folio is active once its refs reach LRU_REFS_ACTIVATED (currently LRU_REFS_PROTECTED). On access, folios are accounted to the active part. On forced aging or PID protecting, folios are reset to LRU_REFS_WORKINGSET, dropping them to the inactive part. The reading is now smooth and consistent, and much closer to classical LRU. Note that on swapless machines anon refs are never reset by aging (the shortcut skips the per-folio walk), so active anon grows monotonically, but the reading is stable and still much more meaningful than the previous oscillating random number. The accounting is now per folio and lruvec independent, so no size adjustment is needed when reparenting a dying memcg: the hierarchical stats keep the counts in the parent automatically. Signed-off-by: Kairui Song --- fs/proc/task_mmu.c | 2 +- include/linux/mm_inline.h | 79 ++++++++++-------- include/linux/mmzone.h | 23 ++++-- mm/damon/paddr.c | 2 +- mm/folio.c | 35 +------- mm/vmscan.c | 205 +++++++++++++++++++++++++++++----------------- mm/workingset.c | 2 +- 7 files changed, 195 insertions(+), 153 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 9cc47efbaa51..a490170f2a3a 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -891,7 +891,7 @@ static bool smap_check_folio_referenced(struct folio *folio) static void smap_clear_folio_referenced(struct folio *folio) { if (lru_gen_enabled()) - folio_set_lru_refs(folio, 0); + folio_reset_lru_refs(folio); else folio_clear_referenced(folio); } diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 23ddd5d78571..686294fe81b8 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -183,24 +183,39 @@ static inline int folio_lru_refs(const struct folio *folio) return lru_get_refs_flags(READ_ONCE(*const_folio_flags(folio, 0))); } -static inline void folio_set_lru_refs(struct folio *folio, unsigned int refs) +/** + * __folio_set_lru_refs - Set a folio's LRU refs. + * @folio: the folio + * @refs: the new referenced count (0 .. LRU_REFS_MAX) + * + * Set the folio's LRU refs. The folio must be off the LRU list (e.g., + * isolated), or use folio_inc_lru_refs or folio_reset_lru_refs instead. + */ +static inline void __folio_set_lru_refs(struct folio *folio, unsigned int refs) { unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0)); do { new_flags = old_flags; + VM_WARN_ON_ONCE(lru_get_gen_flags(old_flags) != -1); lru_set_refs_flags(&new_flags, refs); } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); } #ifdef CONFIG_LRU_GEN void folio_inc_lru_refs(struct folio *folio, unsigned int flags); +bool folio_reset_lru_refs(struct folio *folio); #else static inline void folio_inc_lru_refs(struct folio *folio, unsigned int flags) { /* Should not be called with !CONFIG_LRU_GEN */ WARN_ON_ONCE(1); } + +static inline bool folio_reset_lru_refs(struct folio *folio) +{ + return false; +} #endif /** @@ -214,7 +229,9 @@ static inline void folio_inc_lru_refs(struct folio *folio, unsigned int flags) */ static inline void folio_migrate_lru_refs(struct folio *new, const struct folio *old) { - folio_set_lru_refs(new, folio_lru_refs(old)); + VM_WARN_ON_ONCE_FOLIO(folio_test_lru(old), old); + VM_WARN_ON_ONCE_FOLIO(folio_test_lru(new), new); + __folio_set_lru_refs(new, folio_lru_refs(old)); } #ifdef CONFIG_LRU_GEN @@ -265,19 +282,15 @@ static inline int lru_tier_from_refs(unsigned int refs) return fls(refs - 1); } -static inline int folio_lru_gen(const struct folio *folio) +static inline bool lru_refs_is_active(unsigned int refs) { - return lru_get_gen_flags(READ_ONCE(*const_folio_flags(folio, 0))); + VM_WARN_ON_ONCE(refs > LRU_REFS_MAX); + return refs >= LRU_REFS_ACTIVATED; } -static inline bool lru_gen_is_active(const struct lruvec *lruvec, int gen) +static inline int folio_lru_gen(const struct folio *folio) { - unsigned long max_seq = READ_ONCE(lruvec->lrugen.max_seq); - - VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); - - /* see the comment on MIN_NR_GENS */ - return gen == lru_gen_from_seq(max_seq) || gen == lru_gen_from_seq(max_seq - 1); + return lru_get_gen_flags(READ_ONCE(*const_folio_flags(folio, 0))); } static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *folio, @@ -286,7 +299,6 @@ static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *foli int type = folio_is_file_lru(folio); int zone = folio_zonenum(folio); int delta = folio_nr_pages(folio); - enum lru_list lru = type * LRU_INACTIVE_FILE; struct lru_gen_folio *lrugen = &lruvec->lrugen; VM_WARN_ON_ONCE(old_gen != -1 && old_gen >= MAX_NR_GENS); @@ -297,19 +309,6 @@ static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *foli atomic_long_sub(delta, &lrugen->nr_pages[old_gen][type][zone]); if (new_gen >= 0) atomic_long_add(delta, &lrugen->nr_pages[new_gen][type][zone]); - - /* return now if not a promotion */ - if (old_gen < 0 || new_gen < 0) - return; - - /* promotion */ - if (!lru_gen_is_active(lruvec, old_gen) && lru_gen_is_active(lruvec, new_gen)) { - __update_lru_size(lruvec, lru, zone, -delta); - __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, delta); - } - - /* demotion requires isolation, e.g., lru_deactivate_fn() */ - VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) && !lru_gen_is_active(lruvec, new_gen)); } static inline unsigned long lru_gen_folio_seq(const struct lruvec *lruvec, @@ -351,7 +350,7 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, { unsigned long seq; unsigned long flags; - int gen = folio_lru_gen(folio); + int gen, refs; int type = folio_is_file_lru(folio); int zone = folio_zonenum(folio); int delta = folio_nr_pages(folio); @@ -359,7 +358,7 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, struct lru_gen_folio *lrugen = &lruvec->lrugen; BUILD_BUG_ON(BIT(LRU_GEN_WIDTH - 1) != MAX_NR_GENS); - VM_WARN_ON_ONCE_FOLIO(gen != -1, folio); + VM_WARN_ON_ONCE_FOLIO(folio_lru_gen(folio) != -1, folio); if (folio_test_unevictable(folio) || !lrugen->enabled) return false; @@ -368,10 +367,12 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, gen = lru_gen_from_seq(seq); flags = (gen + 1UL) << LRU_GEN_PGOFF; /* see the comment on MIN_NR_GENS about PG_active */ - set_mask_bits(folio_flags(folio, 0), LRU_GEN_MASK | BIT(PG_active), flags); + flags = set_mask_bits(folio_flags(folio, 0), LRU_GEN_MASK | BIT(PG_active), flags); + /* use the refs from the atomic snapshot to avoid raced update */ + refs = lru_get_refs_flags(flags); lru_gen_update_size(lruvec, folio, -1, gen); - if (lru_gen_is_active(lruvec, gen)) + if (lru_refs_is_active(refs)) lru += LRU_ACTIVE; __update_lru_size(lruvec, lru, zone, delta); @@ -387,24 +388,31 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) { unsigned long flags; - int gen = folio_lru_gen(folio); + int gen, refs; int zone = folio_zonenum(folio); int delta = folio_nr_pages(folio); enum lru_list lru = folio_is_file_lru(folio) * LRU_INACTIVE_FILE; + unsigned long max_seq = READ_ONCE(lruvec->lrugen.max_seq); + flags = set_mask_bits(folio_flags(folio, 0), LRU_GEN_MASK, 0); + gen = lru_get_gen_flags(flags); + refs = lru_get_refs_flags(flags); if (gen < 0) return false; VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); - /* for folio_migrate_flags() */ - flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0; - flags = set_mask_bits(folio_flags(folio, 0), LRU_GEN_MASK, flags); - gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; + /* + * See the comment in lru_gen_folio_seq. For migration, compaction, + * or any other isolation of a hot folio, try best to retain its gen + * info. Ideally we would keep the full gen info. + */ + if (!reclaiming && ((max_seq - gen) % MAX_NR_GENS) < MIN_NR_GENS) + folio_set_active(folio); lru_gen_update_size(lruvec, folio, gen, -1); - if (lru_gen_is_active(lruvec, gen)) + if (lru_refs_is_active(refs)) lru += LRU_ACTIVE; __update_lru_size(lruvec, lru, zone, -delta); list_del(&folio->lru); @@ -455,6 +463,7 @@ static __always_inline void folio_inc_lru_refs_fast(struct folio *folio) return; } + BUILD_BUG_ON(LRU_REFS_REFERENCED >= LRU_REFS_ACTIVATED); old_flags = READ_ONCE(*folio_flags(folio, 0)); do { if (lru_get_refs_flags(old_flags)) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 523783186c57..da950f848bb9 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -456,15 +456,15 @@ enum lruvec_flags { * least twice before handing this folio over to the eviction. The first check * clears the accessed bit from the initial fault; the second check makes sure * this folio hasn't been used since then. This process, AKA second chance, - * requires a minimum of two generations, hence MIN_NR_GENS. And to maintain ABI - * compatibility with the active/inactive LRU, e.g., /proc/vmstat, these two - * generations are considered active; the rest of generations, if they exist, - * are considered inactive. See lru_gen_is_active(). + * requires a minimum of two generations, hence MIN_NR_GENS. + * + * Active/inactive is per-folio based on the referenced count; + * see the comment above MAX_NR_TIERS. * * PG_active is always cleared while a folio is on one of lrugen->folios[] so - * that the sliding window needs not to worry about it. And it's set again when - * a folio considered active is isolated for non-reclaiming purposes, e.g., - * migration. See lru_gen_add_folio() and lru_gen_del_folio(). + * that the sliding window needs not to worry about it. It is set on a + * fault-driven refault (see lru_gen_refault), or isolation (see + * lru_gen_del_folio). * * MAX_NR_GENS is set to 4 so that the multi-gen LRU can support twice the * number of categories of the active/inactive LRU when keeping track of @@ -564,11 +564,15 @@ enum lruvec_flags { * * A folio's referenced count never goes backwards except upon gen * increase as described above, or when explicitly reset by - * lru_gen_clear_refs(). Refault of a reclaimed folio restores + * folio_reset_lru_refs(). Refault of a reclaimed folio restores * its referenced count, capped at LRU_REFS_PROTECTED, which aligns with * promotion. Page table refaults of previous workingset folios send * them to the latest gen, driving aging faster. * + * For the active/inactive LRU ABI (/proc/vmstat), a folio is active + * if its referenced count reaches LRU_REFS_ACTIVATED. The threshold + * currently set to LRU_REFS_PROTECTED, which is tier 2. + * * MAX_NR_TIERS is set to 4 so that the multi-gen LRU can support twice * the number of categories of the active/inactive LRU. */ @@ -583,6 +587,7 @@ enum lruvec_flags { #define LRU_REFS_REFERENCED 0x1 #define LRU_REFS_WORKINGSET 0x2 #define LRU_REFS_PROTECTED 0x3 +#define LRU_REFS_ACTIVATED LRU_REFS_PROTECTED #ifndef __GENERATING_BOUNDS_H @@ -691,6 +696,8 @@ struct lru_gen_mm_walk { unsigned long seq; /* the next address within an mm to scan */ unsigned long next_addr; + /* to batch activated pages */ + int nr_activated[ANON_AND_FILE][MAX_NR_ZONES]; /* to batch promoted pages */ int nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; /* to batch the mm stats */ diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 38d3ff7019e3..c5539b6f7695 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -285,7 +285,7 @@ static unsigned long damon_pa_pageout(struct damon_region *r, * so the hotness can be, and better be dropped. */ if (lru_gen_enabled()) - folio_set_lru_refs(folio, 0); + folio_reset_lru_refs(folio); else folio_clear_referenced(folio); folio_test_clear_young(folio); diff --git a/mm/folio.c b/mm/folio.c index 55cb37366487..cd9a49c1df68 100644 --- a/mm/folio.c +++ b/mm/folio.c @@ -349,35 +349,6 @@ static void __lru_cache_activate_folio(struct folio *folio) local_unlock(&cpu_fbatches.lock); } -#ifdef CONFIG_LRU_GEN - -static bool lru_gen_clear_refs(struct folio *folio) -{ - int gen = folio_lru_gen(folio); - int type = folio_is_file_lru(folio); - unsigned long seq; - - if (gen < 0) - return true; - - folio_set_lru_refs(folio, 0); - - rcu_read_lock(); - seq = READ_ONCE(folio_lruvec(folio)->lrugen.min_seq[type]); - rcu_read_unlock(); - /* whether can do without shuffling under the LRU lock */ - return gen == lru_gen_from_seq(seq); -} - -#else /* !CONFIG_LRU_GEN */ - -static bool lru_gen_clear_refs(struct folio *folio) -{ - return false; -} - -#endif /* CONFIG_LRU_GEN */ - /** * folio_mark_accessed - Mark a folio as having seen activity. * @folio: The folio to mark. @@ -554,7 +525,7 @@ static void lru_lazyfree(struct lruvec *lruvec, struct folio *folio) lruvec_del_folio(lruvec, folio); folio_clear_active(folio); if (lru_gen_enabled()) - lru_gen_clear_refs(folio); + __folio_set_lru_refs(folio, 0); else folio_clear_referenced(folio); /* @@ -627,7 +598,7 @@ void deactivate_file_folio(struct folio *folio) if (folio_test_unevictable(folio) || !folio_test_lru(folio)) return; - if (lru_gen_enabled() && lru_gen_clear_refs(folio)) + if (lru_gen_enabled() && !folio_reset_lru_refs(folio)) return; folio_batch_add_and_move(folio, lru_deactivate_file); @@ -646,7 +617,7 @@ void folio_deactivate(struct folio *folio) if (folio_test_unevictable(folio) || !folio_test_lru(folio)) return; - if (lru_gen_enabled() ? lru_gen_clear_refs(folio) : !folio_test_active(folio)) + if (lru_gen_enabled() ? !folio_reset_lru_refs(folio) : !folio_test_active(folio)) return; folio_batch_add_and_move(folio, lru_deactivate); diff --git a/mm/vmscan.c b/mm/vmscan.c index 9be0cd20dc54..240a8747490a 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -944,17 +944,19 @@ enum folio_references { void folio_inc_lru_refs(struct folio *folio, unsigned int flags) { int max_gen, min_gen; - int type, refs, old_gen, gen; + int file, old_refs, refs, old_gen, gen; unsigned long new_flags, old_flags, max_seq; + long nr_pages = folio_nr_pages(folio); struct lru_gen_folio *lrugen; struct lruvec *lruvec = NULL; - type = folio_is_file_lru(folio); old_flags = READ_ONCE(*folio_flags(folio, 0)); do { new_flags = old_flags; old_gen = lru_get_gen_flags(old_flags); - refs = lru_get_refs_flags(old_flags) + 1; + old_refs = lru_get_refs_flags(old_flags); + file = folio_flags_is_file_lru(&old_flags); + refs = old_refs + 1; gen = old_gen; if (old_gen < 0) goto out; @@ -972,7 +974,7 @@ void folio_inc_lru_refs(struct folio *folio, unsigned int flags) } max_seq = READ_ONCE(lrugen->max_seq); max_gen = lru_gen_from_seq(max_seq); - min_gen = lru_gen_from_seq(READ_ONCE(lrugen->min_seq[type])); + min_gen = lru_gen_from_seq(READ_ONCE(lrugen->min_seq[file])); if (old_gen == max_gen) goto out; @@ -1001,50 +1003,102 @@ void folio_inc_lru_refs(struct folio *folio, unsigned int flags) if (gen != old_gen) lru_gen_update_size(lruvec, folio, old_gen, gen); + if (lru_refs_is_active(old_refs) != lru_refs_is_active(refs) && old_gen >= 0) { + enum lru_list lru = file * LRU_INACTIVE_FILE; + + __update_lru_size(lruvec, lru + lru_refs_is_active(old_refs), + folio_zonenum(folio), -nr_pages); + __update_lru_size(lruvec, lru + lru_refs_is_active(refs), + folio_zonenum(folio), nr_pages); + } if (lruvec) lruvec_unlock_irq(lruvec); } +/** + * Reset the folio's lru refs indicator. The caller doesn't need to hold + * the folio lock, isolate the folio, or hold the lruvec lock. + */ +bool folio_reset_lru_refs(struct folio *folio) +{ + int type, gen, refs; + unsigned long seq, new_flags, old_flags; + long nr_pages = folio_nr_pages(folio); + struct lruvec *lruvec; + bool reset = false; + + old_flags = READ_ONCE(*folio_flags(folio, 0)); + do { + new_flags = old_flags; + refs = lru_get_refs_flags(old_flags); + type = folio_flags_is_file_lru(&old_flags); + gen = lru_get_gen_flags(old_flags); + if (!refs) + break; + lru_set_refs_flags(&new_flags, 0); + } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); + + if (gen >= 0) { + lruvec = folio_lruvec_live_get(folio); + + /* If the folio is on list, caller might also want to demote it */ + seq = READ_ONCE(lruvec->lrugen.min_seq[type]); + reset = (gen != lru_gen_from_seq(seq)); + if (lru_refs_is_active(refs)) { + __update_lru_size(lruvec, type * LRU_FILE + LRU_ACTIVE, + folio_zonenum(folio), -nr_pages); + __update_lru_size(lruvec, type * LRU_FILE, + folio_zonenum(folio), nr_pages); + } + + folio_lruvec_live_put(lruvec); + } + + return reset; +} + /* * Update the folio's lru refs indicator during a page table walk. * max_seq is stable since this runs inside the aging process. * - * Returns the old generation and stores the new generation in @new_gen when - * the folio is promoted (to max_gen) or advanced by one generation. - * Returns -1 if no gen change occurred. + * Returns the old generation, or -1 if the folio is off the LRU list. + * Old LRU refs, and updated gen and LRU refs info by the successful + * cmpxchg are all stored by returning arguments. */ static int folio_inc_lru_refs_walk(struct folio *folio, struct lruvec *lruvec, - const vma_flags_t *vma_flags, - int *new_gen, int *type) + const vma_flags_t *vma_flags, int *new_gen, + int *old_refs, int *new_refs, int *type) { unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0)); unsigned long max_seq = READ_ONCE(lruvec->lrugen.max_seq); - int refs, gen, max_gen, ret; + int refs, old_gen, max_gen; max_gen = lru_gen_from_seq(max_seq); - do { - gen = lru_get_gen_flags(old_flags); + old_gen = lru_get_gen_flags(old_flags); refs = lru_get_refs_flags(old_flags) + 1; new_flags = old_flags; - if (gen >= 0 && gen != max_gen) { - ret = gen; + if (old_gen >= 0 && old_gen != max_gen) { + *new_refs = min(refs, LRU_REFS_PROTECTED); /* Promote second page table access or executable */ if (refs > LRU_REFS_REFERENCED || is_exec_file_folio(folio, vma_flags)) *new_gen = max_gen; else - *new_gen = (gen + 1) % MAX_NR_GENS; + *new_gen = (old_gen + 1) % MAX_NR_GENS; lru_set_gen_flags(&new_flags, *new_gen); - lru_set_refs_flags(&new_flags, min(refs, LRU_REFS_PROTECTED)); + lru_set_refs_flags(&new_flags, *new_refs); } else { - ret = -1; - lru_set_refs_flags(&new_flags, min(refs, LRU_REFS_MAX)); + *new_gen = old_gen; + *new_refs = min(refs, LRU_REFS_MAX); + lru_set_refs_flags(&new_flags, *new_refs); } } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); *type = folio_flags_is_file_lru(&old_flags); - return ret; + *old_refs = lru_get_refs_flags(old_flags); + + return old_gen; } /* @@ -3482,10 +3536,11 @@ static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv) * the aging ******************************************************************************/ -static int __folio_inc_gen(struct folio *folio, int old_gen, bool *increased) +static int __folio_inc_gen(struct lruvec *lruvec, struct folio *folio, + int old_gen, bool *increased) { unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0)); - int refs, new_gen; + int file, refs, old_refs, new_gen; do { new_gen = lru_get_gen_flags(old_flags); @@ -3499,11 +3554,24 @@ static int __folio_inc_gen(struct folio *folio, int old_gen, bool *increased) new_flags = old_flags; new_gen = (old_gen + 1) % MAX_NR_GENS; - refs = lru_get_refs_flags(old_flags); + old_refs = lru_get_refs_flags(old_flags); + file = folio_flags_is_file_lru(&old_flags); + refs = min(old_refs, LRU_REFS_WORKINGSET); lru_set_gen_flags(&new_flags, new_gen); - lru_set_refs_flags(&new_flags, min(refs, LRU_REFS_WORKINGSET)); + lru_set_refs_flags(&new_flags, refs); } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); + /* Refs capped below PROTECTED: demote if previously active. */ + if (lru_refs_is_active(old_refs) != lru_refs_is_active(refs)) { + enum lru_list lru = file * LRU_INACTIVE_FILE; + int nr_pages = folio_nr_pages(folio); + + __update_lru_size(lruvec, lru + lru_refs_is_active(old_refs), + folio_zonenum(folio), -nr_pages); + __update_lru_size(lruvec, lru + lru_refs_is_active(refs), + folio_zonenum(folio), nr_pages); + } + if (increased) *increased = true; return new_gen; @@ -3520,7 +3588,7 @@ static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio) struct lru_gen_folio *lrugen = &lruvec->lrugen; int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); - new_gen = __folio_inc_gen(folio, old_gen, &gen_increased); + new_gen = __folio_inc_gen(lruvec, folio, old_gen, &gen_increased); if (gen_increased) lru_gen_update_size(lruvec, folio, old_gen, new_gen); @@ -3528,18 +3596,26 @@ static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio) } static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio, - int old_gen, int new_gen, int type) + int old_gen, int new_gen, int old_refs, int new_refs, int type) { int zone = folio_zonenum(folio); int delta = folio_nr_pages(folio); - VM_WARN_ON_ONCE(old_gen >= MAX_NR_GENS); - VM_WARN_ON_ONCE(new_gen >= MAX_NR_GENS); - - walk->batched++; + /* gen counter update */ + if (old_gen != new_gen) { + VM_WARN_ON_ONCE(old_gen >= MAX_NR_GENS); + VM_WARN_ON_ONCE(new_gen >= MAX_NR_GENS); + walk->batched++; + walk->nr_pages[old_gen][type][zone] -= delta; + walk->nr_pages[new_gen][type][zone] += delta; + } - walk->nr_pages[old_gen][type][zone] -= delta; - walk->nr_pages[new_gen][type][zone] += delta; + /* active/inactive counter update */ + if (lru_refs_is_active(old_refs) != lru_refs_is_active(new_refs)) { + walk->nr_activated[type][zone] += + lru_refs_is_active(new_refs) ? delta : -delta; + walk->batched++; + } } static void reset_batch_size(struct lru_gen_mm_walk *walk) @@ -3551,7 +3627,6 @@ static void reset_batch_size(struct lru_gen_mm_walk *walk) walk->batched = 0; for_each_gen_type_zone(gen, type, zone) { - enum lru_list lru = type * LRU_INACTIVE_FILE; int delta = walk->nr_pages[gen][type][zone]; if (!delta) @@ -3559,10 +3634,21 @@ static void reset_batch_size(struct lru_gen_mm_walk *walk) walk->nr_pages[gen][type][zone] = 0; atomic_long_add(delta, &lrugen->nr_pages[gen][type][zone]); + } - if (lru_gen_is_active(lruvec, gen)) - lru += LRU_ACTIVE; - __update_lru_size(lruvec, lru, zone, delta); + /* apply batched active/inactive updates */ + for (type = 0; type < ANON_AND_FILE; type++) { + for (zone = 0; zone < MAX_NR_ZONES; zone++) { + enum lru_list lru = type * LRU_INACTIVE_FILE; + int delta = walk->nr_activated[type][zone]; + + if (delta) { + __update_lru_size(lruvec, lru + LRU_ACTIVE, + zone, delta); + __update_lru_size(lruvec, lru, zone, -delta); + walk->nr_activated[type][zone] = 0; + } + } } lruvec_unlock_irq(lruvec); @@ -3717,7 +3803,7 @@ static bool suitable_to_scan(int total, int young) static void walk_update_folio(struct lru_gen_mm_walk *walk, struct vm_area_struct *vma, struct lruvec *lruvec, struct folio *folio, bool dirty) { - int new_gen, old_gen, type; + int new_gen, old_gen, old_refs, new_refs, type; unsigned int flags = LRU_REF_MAPPED; if (!folio) @@ -3730,9 +3816,10 @@ static void walk_update_folio(struct lru_gen_mm_walk *walk, struct vm_area_struc if (walk) { old_gen = folio_inc_lru_refs_walk(folio, lruvec, &vma->flags, - &new_gen, &type); + &new_gen, &old_refs, &new_refs, &type); if (old_gen >= 0) - update_batch_size(walk, folio, old_gen, new_gen, type); + update_batch_size(walk, folio, old_gen, new_gen, + old_refs, new_refs, type); } else { if (is_exec_file_folio(folio, &vma->flags)) flags |= LRU_REF_EXEC; @@ -4091,6 +4178,7 @@ static void clear_mm_walk(void) VM_WARN_ON_ONCE(walk && memchr_inv(walk->nr_pages, 0, sizeof(walk->nr_pages))); VM_WARN_ON_ONCE(walk && memchr_inv(walk->mm_stats, 0, sizeof(walk->mm_stats))); + VM_WARN_ON_ONCE(walk && memchr_inv(walk->nr_activated, 0, sizeof(walk->nr_activated))); current->reclaim_state->mm_walk = NULL; @@ -4129,8 +4217,6 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness) goto done; VM_WARN_ON_ONCE(get_nr_gens(lruvec, type) != MAX_NR_GENS); - VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) != - lru_gen_is_active(lruvec, target_gen)); /* prevent cold/hot inversion if the type is evictable */ for (zone = 0; zone < MAX_NR_ZONES; zone++) { struct list_head *target_list = &lrugen->folios[target_gen][type][zone]; @@ -4153,7 +4239,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness) prefetchw_next_lru_folio(folio, head); pos = pos->next; - new_gen = __folio_inc_gen(folio, old_gen, &gen_increased); + new_gen = __folio_inc_gen(lruvec, folio, old_gen, &gen_increased); /* * If gen_increased is false, this is a promotion. Put folios * at the head of the promoted gen. Otherwise, put them at @@ -4240,8 +4326,7 @@ static void try_to_inc_min_seq(struct lruvec *lruvec, int swappiness) static bool inc_max_seq(struct lruvec *lruvec, unsigned long seq, int swappiness) { bool success; - int prev, next; - int type, zone; + int type, next; struct lru_gen_folio *lrugen = &lruvec->lrugen; restart: if (seq < READ_ONCE(lrugen->max_seq)) @@ -4267,32 +4352,10 @@ static bool inc_max_seq(struct lruvec *lruvec, unsigned long seq, int swappiness goto restart; } - /* - * Update the active/inactive LRU sizes for compatibility. Both sides of - * the current max_seq need to be covered, since max_seq+1 can overlap - * with min_seq[LRU_GEN_ANON] if swapping is constrained. And if they do - * overlap, cold/hot inversion happens. - */ - prev = lru_gen_from_seq(lrugen->max_seq - 1); - next = lru_gen_from_seq(lrugen->max_seq + 1); - - for (type = 0; type < ANON_AND_FILE; type++) { - for (zone = 0; zone < MAX_NR_ZONES; zone++) { - enum lru_list lru = type * LRU_INACTIVE_FILE; - long delta = atomic_long_read(&lrugen->nr_pages[prev][type][zone]) - - atomic_long_read(&lrugen->nr_pages[next][type][zone]); - - if (!delta) - continue; - - __update_lru_size(lruvec, lru, zone, delta); - __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, -delta); - } - } - for (type = 0; type < ANON_AND_FILE; type++) reset_ctrl_pos(lruvec, type, false); + next = lru_gen_from_seq(lrugen->max_seq + 1); WRITE_ONCE(lrugen->timestamps[next], jiffies); /* make sure preceding modifications appear */ smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1); @@ -4811,7 +4874,6 @@ static void __lru_gen_reparent_memcg(struct lruvec *child_lruvec, struct lruvec int zone, int type) { struct lru_gen_folio *child_lrugen, *parent_lrugen; - enum lru_list lru = type * LRU_INACTIVE_FILE; int i; child_lrugen = &child_lruvec->lrugen; @@ -4820,8 +4882,6 @@ static void __lru_gen_reparent_memcg(struct lruvec *child_lruvec, struct lruvec for (i = 0; i < get_nr_gens(child_lruvec, type); i++) { int gen = lru_gen_from_seq(child_lrugen->max_seq - i); long nr_pages = atomic_long_read(&child_lrugen->nr_pages[gen][type][zone]); - int child_lru_active = lru_gen_is_active(child_lruvec, gen) ? LRU_ACTIVE : 0; - int parent_lru_active = lru_gen_is_active(parent_lruvec, gen) ? LRU_ACTIVE : 0; /* Assuming that child pages are colder than parent pages */ list_splice_tail_init(&child_lrugen->folios[gen][type][zone], @@ -4829,11 +4889,6 @@ static void __lru_gen_reparent_memcg(struct lruvec *child_lruvec, struct lruvec atomic_long_set(&child_lrugen->nr_pages[gen][type][zone], 0); atomic_long_add(nr_pages, &parent_lrugen->nr_pages[gen][type][zone]); - - if (lru_gen_is_active(child_lruvec, gen) != lru_gen_is_active(parent_lruvec, gen)) { - __update_lru_size(child_lruvec, lru + child_lru_active, zone, -nr_pages); - __update_lru_size(parent_lruvec, lru + parent_lru_active, zone, nr_pages); - } } } @@ -5164,7 +5219,7 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec, } /* Reset folio's refs before return */ - folio_set_lru_refs(folio, min(folio_lru_refs(folio), LRU_REFS_WORKINGSET)); + __folio_set_lru_refs(folio, min(folio_lru_refs(folio), LRU_REFS_WORKINGSET)); if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]) folio_set_active(folio); } diff --git a/mm/workingset.c b/mm/workingset.c index 44be539b93d2..a04d495139c4 100644 --- a/mm/workingset.c +++ b/mm/workingset.c @@ -334,7 +334,7 @@ static void lru_gen_refault(struct folio *folio, void *shadow) mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + type, delta); } /* Refault is also promotion, cap the refs like folio_inc_lru_refs */ - folio_set_lru_refs(folio, min(refs, LRU_REFS_PROTECTED)); + __folio_set_lru_refs(folio, min(refs, LRU_REFS_PROTECTED)); } if (refs >= LRU_REFS_WORKINGSET) -- 2.55.0 From: Kairui Song The lruvec spinlock in folio_inc_lru_refs was only taken to keep concurrent aging from corrupting the size counters. This is no longer needed: the per-generation counters are atomic, and the active/inactive ABI counters are updated per folio and linearized by the CAS on the folio flags, so concurrent aging can no longer corrupt them. The same argument already allowed folio_reset_lru_refs() to drop the lock. Use folio_lruvec_live_get()/folio_lruvec_live_put() to hold the RCU read lock around the lruvec lookup, and drop the spinlock entirely. Also simplify the post-CAS accounting guards: gen is only assigned non-negative values after the old_gen < 0 early exit, so gen != old_gen implies gen >= 0, and lru_gen_update_size() needs no explicit guard. The active/inactive ABI update keeps its gen >= 0 check because an off-LRU folio has no lruvec to update. Signed-off-by: Kairui Song --- include/linux/mm_inline.h | 10 +++++----- mm/vmscan.c | 30 ++++++++++++++++-------------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 686294fe81b8..bca00599dec6 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -293,10 +293,9 @@ static inline int folio_lru_gen(const struct folio *folio) return lru_get_gen_flags(READ_ONCE(*const_folio_flags(folio, 0))); } -static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *folio, - int old_gen, int new_gen) +static inline void lru_gen_update_size(struct lruvec *lruvec, int type, + struct folio *folio, int old_gen, int new_gen) { - int type = folio_is_file_lru(folio); int zone = folio_zonenum(folio); int delta = folio_nr_pages(folio); struct lru_gen_folio *lrugen = &lruvec->lrugen; @@ -371,7 +370,7 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, /* use the refs from the atomic snapshot to avoid raced update */ refs = lru_get_refs_flags(flags); - lru_gen_update_size(lruvec, folio, -1, gen); + lru_gen_update_size(lruvec, type, folio, -1, gen); if (lru_refs_is_active(refs)) lru += LRU_ACTIVE; __update_lru_size(lruvec, lru, zone, delta); @@ -389,6 +388,7 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, { unsigned long flags; int gen, refs; + int type = folio_is_file_lru(folio); int zone = folio_zonenum(folio); int delta = folio_nr_pages(folio); enum lru_list lru = folio_is_file_lru(folio) * LRU_INACTIVE_FILE; @@ -411,7 +411,7 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, if (!reclaiming && ((max_seq - gen) % MAX_NR_GENS) < MIN_NR_GENS) folio_set_active(folio); - lru_gen_update_size(lruvec, folio, gen, -1); + lru_gen_update_size(lruvec, type, folio, gen, -1); if (lru_refs_is_active(refs)) lru += LRU_ACTIVE; __update_lru_size(lruvec, lru, zone, -delta); diff --git a/mm/vmscan.c b/mm/vmscan.c index 240a8747490a..9deeb40fd5f3 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -960,16 +960,8 @@ void folio_inc_lru_refs(struct folio *folio, unsigned int flags) gen = old_gen; if (old_gen < 0) goto out; - /* - * Lock the lruvec if the folio is on-list. We are already - * doing lazy promotion so in theory we don't need this, - * but for now, concurrent aging would still corrupt the - * size counters. This is a temporary limitation and - * will be lifted very soon, so the lock here is not a - * performance concern. - */ if (!lruvec) { - lruvec = lruvec_live_lock_irq(folio_lruvec(folio)); + lruvec = folio_lruvec_live_get(folio); lrugen = &lruvec->lrugen; } max_seq = READ_ONCE(lrugen->max_seq); @@ -1001,9 +993,19 @@ void folio_inc_lru_refs(struct folio *folio, unsigned int flags) lru_set_gen_flags(&new_flags, gen); } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); - if (gen != old_gen) - lru_gen_update_size(lruvec, folio, old_gen, gen); - if (lru_refs_is_active(old_refs) != lru_refs_is_active(refs) && old_gen >= 0) { + if (gen != old_gen) { + lru_gen_update_size(lruvec, file, folio, old_gen, gen); + /* + * Gen can only go forward while on list, so concurrent aging + * is fine, except when multiple aging increase max_seq cross + * the sliding window border causing hotness inversion. In that + * very unlikely case, just activate the folio. + */ + if (unlikely(READ_ONCE(lrugen->max_seq) - max_seq > MIN_NR_GENS)) + folio_activate(folio); + } + + if (lru_refs_is_active(old_refs) != lru_refs_is_active(refs) && gen >= 0) { enum lru_list lru = file * LRU_INACTIVE_FILE; __update_lru_size(lruvec, lru + lru_refs_is_active(old_refs), @@ -1012,7 +1014,7 @@ void folio_inc_lru_refs(struct folio *folio, unsigned int flags) folio_zonenum(folio), nr_pages); } if (lruvec) - lruvec_unlock_irq(lruvec); + folio_lruvec_live_put(lruvec); } /** @@ -3590,7 +3592,7 @@ static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio) new_gen = __folio_inc_gen(lruvec, folio, old_gen, &gen_increased); if (gen_increased) - lru_gen_update_size(lruvec, folio, old_gen, new_gen); + lru_gen_update_size(lruvec, type, folio, old_gen, new_gen); return new_gen; } -- 2.55.0 From: Kairui Song For MGLRU, PG_referenced is only the lowest bit of the folio LRU refs count, so folios with a higher refs count no longer have the bit set. folio_pte_referenced() testing the raw bit misses hot folios at refs >= 2, under-counting the referenced folios of a candidate range and aborting otherwise good collapses with SCAN_LACK_REFERENCED_PAGE. Test the refs count directly when MGLRU is enabled, mirroring the smaps conversion. The classical LRU keeps the plain bit test. Signed-off-by: Kairui Song --- mm/khugepaged.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 1deb74cf28af..a3ce6735bc32 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -597,7 +597,9 @@ static bool folio_pte_referenced(struct folio *folio, struct vm_area_struct *vma, unsigned long addr, pte_t pteval) { /* The folio was referenced previously ... */ - if (folio_test_young(folio) || folio_test_referenced(folio)) + if (folio_test_young(folio)) + return true; + if (lru_gen_enabled() ? folio_lru_refs(folio) : folio_test_referenced(folio)) return true; /* ... or the PTE mapping was recently used */ return pte_young(pteval) || mmu_notifier_test_young(vma->vm_mm, addr); -- 2.55.0 From: Kairui Song folio_test_workingset() currently tests the PG_workingset bit, which is only the second lowest access bit of the folio LRU refs count now. Folios with a higher refs count carry it in the LRU_REFS_MASK field and no longer have PG_workingset set, so the bit test misses them. folio_set_workingset() currently sets the PG_workingset bit with a plain set_bit(), which also over-promotes folios at a higher refs count: e.g. a folio at refs 4 is bumped to refs 6, advancing it toward LRU_REFS_MAX and an unintended generation promotion on the next access. Move the test into mm_inline.h as an inline helper based on folio_lru_refs(), checking refs >= LRU_REFS_WORKINGSET. The set stays a plain PG_workingset bit operation reserved for the classical LRU: under MGLRU the refs count is maintained by folio_inc_lru_refs(), and now triggers a debug WARN when called while MGLRU is fully on (the switching window is exempt, as the classical paths legitimately run alongside MGLRU then). The one PageWorkingset() user in erofs is converted to the folio helper. Under the classical LRU the refs count is not maintained, so the test falls back to the PG_workingset bit test, which is the old behavior. Signed-off-by: Kairui Song --- fs/btrfs/compression.c | 1 + fs/erofs/zdata.c | 3 ++- include/linux/mm_inline.h | 35 +++++++++++++++++++++++++++++++++++ include/linux/page-flags.h | 2 -- mm/filemap.c | 1 + mm/page_io.c | 1 + 6 files changed, 40 insertions(+), 3 deletions(-) diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index c62b5148d5ac..57d24413265e 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c index e1e25ca0d190..e326456214b5 100644 --- a/fs/erofs/zdata.c +++ b/fs/erofs/zdata.c @@ -6,6 +6,7 @@ */ #include "compress.h" #include +#include #include #include @@ -1725,7 +1726,7 @@ static void z_erofs_submit_queue(struct z_erofs_frontend *f, DBG_BUGON(bvec.bv_len < sb->s_blocksize); } - if (unlikely(PageWorkingset(bvec.bv_page)) && + if (unlikely(folio_test_workingset(page_folio(bvec.bv_page))) && !memstall) { psi_memstall_enter(&pflags); memstall = 1; diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index bca00599dec6..1dd853c10a54 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -258,6 +258,24 @@ static inline bool lru_gen_enabled(void) } #endif +/** + * folio_test_workingset - Test if a folio is in the workingset. + * @folio: the folio + * + * A folio is workingset when its LRU refs count reaches + * LRU_REFS_WORKINGSET. Under the classical LRU the refs count never + * goes above it, so this is just testing the PG_workingset bit. + * NOTE: folio_set_workingset() must not be used under MGLRU, as the + * folio refs are tracked by folio_inc_lru_refs(), it triggers a debug + * WARN instead for MGLRU. + * + * Return: true if the folio is workingset. + */ +static __always_inline bool folio_test_workingset(const struct folio *folio) +{ + return folio_lru_refs(folio) >= LRU_REFS_WORKINGSET; +} + static inline bool lru_gen_in_fault(void) { return current->in_lru_fault; @@ -445,6 +463,11 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, { return false; } + +static inline bool folio_test_workingset(const struct folio *folio) +{ + return test_bit(PG_workingset, const_folio_flags(folio, FOLIO_HEAD_PAGE)); +} #endif /* CONFIG_LRU_GEN */ /** @@ -473,6 +496,18 @@ static __always_inline void folio_inc_lru_refs_fast(struct folio *folio) } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); } +/* + * For the classical LRU only: under MGLRU the PG_workingset bit is + * part of the folio refs count maintained by folio_inc_lru_refs(), + * and a raw set would corrupt it. The switching window is exempt + * because the classical paths legitimately run alongside MGLRU then. + */ +static __always_inline void folio_set_workingset(struct folio *folio) +{ + VM_WARN_ON_ONCE(lru_gen_enabled() && !lru_gen_switching()); + set_bit(PG_workingset, folio_flags(folio, FOLIO_HEAD_PAGE)); +} + static __always_inline void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio) { diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 86dd0470da11..c5a8ac0a1599 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -554,8 +554,6 @@ PAGEFLAG(LRU, lru, PF_HEAD) __CLEARPAGEFLAG(LRU, lru, PF_HEAD) FOLIO_FLAG(active, FOLIO_HEAD_PAGE) __FOLIO_CLEAR_FLAG(active, FOLIO_HEAD_PAGE) FOLIO_TEST_CLEAR_FLAG(active, FOLIO_HEAD_PAGE) -PAGEFLAG(Workingset, workingset, PF_HEAD) - TESTCLEARFLAG(Workingset, workingset, PF_HEAD) PAGEFLAG(Checked, checked, PF_NO_COMPOUND) /* Used by some filesystems */ /* Xen */ diff --git a/mm/filemap.c b/mm/filemap.c index 00fd89cf6f55..c7dac67a0081 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include diff --git a/mm/page_io.c b/mm/page_io.c index 1da4ff484f09..0fb87ca042d4 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include -- 2.55.0