folio_inc_gen() currently updates both the folio's generation and the LRU size accounting. This makes it difficult to batch the LRU size updates when moving multiple folios. Extract the generation update into __folio_inc_gen(), which only updates the folio's generation and reports whether the generation was actually increased. Keep folio_inc_gen() as the wrapper that performs the LRU size accounting when needed. This separates the per-folio generation update from LRU accounting and allows the latter to be batched by subsequent changes. Signed-off-by: Barry Song (Xiaomi) Tested-by: Xueyuan Chen Reviewed-by: Lian Wang --- mm/vmscan.c | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index f11491ee9ed5..54bce2f608ef 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3295,21 +3295,21 @@ static int folio_update_gen(struct folio *folio, int gen, const vma_flags_t *vma return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; } -/* protect pages accessed multiple times through file descriptors */ -static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio) +static int __folio_inc_gen(struct folio *folio, int old_gen, bool *increased) { - int type = folio_is_file_lru(folio); - struct lru_gen_folio *lrugen = &lruvec->lrugen; - int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f); + int new_gen; VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio); do { new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; /* folio_update_gen() has promoted this page? */ - if (new_gen >= 0 && new_gen != old_gen) + if (new_gen >= 0 && new_gen != old_gen) { + if (increased) + *increased = false; return new_gen; + } new_gen = (old_gen + 1) % MAX_NR_GENS; @@ -3317,8 +3317,22 @@ static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio) new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF; } while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags)); - lru_gen_update_size(lruvec, folio, old_gen, new_gen); + if (increased) + *increased = true; + return new_gen; +} + +/* protect pages accessed multiple times through file descriptors */ +static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio) +{ + int type = folio_is_file_lru(folio); + struct lru_gen_folio *lrugen = &lruvec->lrugen; + int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); + bool gen_increased; + new_gen = __folio_inc_gen(folio, old_gen, &gen_increased); + if (gen_increased) + lru_gen_update_size(lruvec, folio, old_gen, new_gen); return new_gen; } -- 2.34.1