From: Ridong Chen lru_gen_add_folio() and lru_gen_add_rejected_folio() are wrappers around __lru_gen_add_folio(), which can only raise the generation picked by lru_gen_folio_seq(), through scanned_seq. There is no way for a caller to lower it, e.g. to put a folio into the oldest generation without also adding it to the tail. Let lru_gen_add_folio() take a range [min_seq, max_seq] that the target generation is capped to, with -1UL meaning no limit on that side, and drop the two wrappers. move_folios_to_lru() passes scanned_seq + 1 as min_seq for rejected folios, as before. Also rename "reclaiming" to "tail". With MGLRU, the tail of the LRU is the tail of the oldest generation, so when tail is true, lru_gen_add_folio() now picks min_seq[type] itself, and lru_gen_folio_seq() no longer needs the argument. This does not change behavior. The old reclaiming branch in lru_gen_folio_seq() also resulted in min_seq[type]. It was only skipped for active folios, and none of the callers that pass true add an active folio. max_seq has no user yet. The next patch uses it in lru_deactivate(). No functional change intended. Suggested-by: Kairui Song Signed-off-by: Ridong Chen --- include/linux/mm_inline.h | 52 +++++++++++++++++---------------------- mm/vmscan.c | 11 +++------ 2 files changed, 26 insertions(+), 37 deletions(-) diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index f6f4bc9a704f..69f69b9d8018 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -299,8 +299,7 @@ static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *foli } static inline unsigned long lru_gen_folio_seq(const struct lruvec *lruvec, - const struct folio *folio, - bool reclaiming) + const struct folio *folio) { int gen; int type = folio_is_file_lru(folio); @@ -320,8 +319,6 @@ static inline unsigned long lru_gen_folio_seq(const struct lruvec *lruvec, */ if (folio_test_active(folio)) gen = MIN_NR_GENS - folio_test_workingset(folio); - else if (reclaiming) - gen = MAX_NR_GENS; else if ((!folio_is_file_lru(folio) && !folio_test_swapcache(folio)) || (folio_test_reclaim(folio) && (folio_test_dirty(folio) || folio_test_writeback(folio)))) @@ -332,8 +329,13 @@ static inline unsigned long lru_gen_folio_seq(const struct lruvec *lruvec, return max(READ_ONCE(lrugen->max_seq) - gen + 1, READ_ONCE(lrugen->min_seq[type])); } -static inline bool __lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, - unsigned long scanned_seq, bool reclaiming) +/* + * Cap the seq to [min_seq, max_seq], -1UL means no limit. With no limit and + * tail set, the folio is added to the tail of the oldest gen. + */ +static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, + unsigned long min_seq, unsigned long max_seq, + bool tail) { unsigned long seq; unsigned long flags; @@ -347,9 +349,14 @@ static inline bool __lru_gen_add_folio(struct lruvec *lruvec, struct folio *foli if (folio_test_unevictable(folio) || !lrugen->enabled) return false; - seq = lru_gen_folio_seq(lruvec, folio, reclaiming); - if (scanned_seq != -1UL) - seq = max(seq, min(scanned_seq + 1, lrugen->max_seq)); + if (tail) + seq = READ_ONCE(lrugen->min_seq[type]); + else + seq = lru_gen_folio_seq(lruvec, folio); + if (min_seq != -1UL) + seq = max(seq, min(min_seq, lrugen->max_seq)); + if (max_seq != -1UL) + seq = min(seq, max_seq); gen = lru_gen_from_seq(seq); flags = (gen + 1UL) << LRU_GEN_PGOFF; /* see the comment on MIN_NR_GENS about PG_active */ @@ -357,7 +364,7 @@ static inline bool __lru_gen_add_folio(struct lruvec *lruvec, struct folio *foli lru_gen_update_size(lruvec, folio, -1, gen); /* for folio_rotate_reclaimable() */ - if (reclaiming) + if (tail) list_add_tail(&folio->lru, &lrugen->folios[gen][type][zone]); else list_add(&folio->lru, &lrugen->folios[gen][type][zone]); @@ -365,17 +372,6 @@ static inline bool __lru_gen_add_folio(struct lruvec *lruvec, struct folio *foli return true; } -static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) -{ - return __lru_gen_add_folio(lruvec, folio, -1UL, reclaiming); -} - -static inline bool lru_gen_add_rejected_folio(struct lruvec *lruvec, struct folio *folio, - unsigned long scanned_seq) -{ - return __lru_gen_add_folio(lruvec, folio, scanned_seq, false); -} - static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) { unsigned long flags; @@ -429,13 +425,9 @@ static inline bool lru_gen_in_fault(void) return false; } -static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) -{ - return false; -} - -static inline bool lru_gen_add_rejected_folio(struct lruvec *lruvec, struct folio *folio, - unsigned long scanned_seq) +static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, + unsigned long min_seq, unsigned long max_seq, + bool tail) { return false; } @@ -459,7 +451,7 @@ void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio) VM_WARN_ON_ONCE_FOLIO(!folio_matches_lruvec(folio, lruvec), folio); - if (lru_gen_add_folio(lruvec, folio, false)) + if (lru_gen_add_folio(lruvec, folio, -1UL, -1UL, false)) return; update_lru_size(lruvec, lru, folio_zonenum(folio), @@ -475,7 +467,7 @@ void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio) VM_WARN_ON_ONCE_FOLIO(!folio_matches_lruvec(folio, lruvec), folio); - if (lru_gen_add_folio(lruvec, folio, true)) + if (lru_gen_add_folio(lruvec, folio, -1UL, -1UL, true)) return; update_lru_size(lruvec, lru, folio_zonenum(folio), diff --git a/mm/vmscan.c b/mm/vmscan.c index bc7579d47f9b..19e06574ea00 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1916,7 +1916,8 @@ static unsigned int move_folios_to_lru(struct list_head *list, unsigned long seq !folio_test_referenced(folio) && folio_ref_count(folio) == folio_expected_ref_count(folio)) lruvec_add_folio_tail(lruvec, folio); - else if (seq == -1UL || !lru_gen_add_rejected_folio(lruvec, folio, seq)) + else if (seq == -1UL || + !lru_gen_add_folio(lruvec, folio, seq + 1, -1UL, false)) lruvec_add_folio(lruvec, folio); nr_pages = folio_nr_pages(folio); @@ -5516,7 +5517,7 @@ static bool fill_evictable(struct lruvec *lruvec) /* * lru_gen_add_folio() uses list_add_tail() rather - * than list_add() when reclaiming is true. Match + * than list_add() when tail is true. Match * its ordering to avoid cold/hot inversion. */ if (active) @@ -5530,11 +5531,7 @@ static bool fill_evictable(struct lruvec *lruvec) VM_WARN_ON_ONCE_FOLIO(folio_lru_gen(folio) != -1, folio); lruvec_del_folio(lruvec, folio); - /* - * Borrow reclaiming=true to place inactive folios in - * the older gens. - */ - success = lru_gen_add_folio(lruvec, folio, !active); + success = lru_gen_add_folio(lruvec, folio, -1UL, -1UL, !active); VM_WARN_ON_ONCE(!success); if (!--remaining) -- 2.34.1