From: Ridong Chen lru_gen_add_folio() and lru_gen_add_rejected_folio() are wrappers around __lru_gen_add_folio(), which can only raise the generation picked by lru_gen_folio_seq(), through scanned_seq. There is no way for a caller to lower it, e.g. to put a folio into the oldest generation without also adding it to the tail. Let lru_gen_add_folio() take a range [min_seq, max_seq] that the target generation is capped to, with -1UL meaning no limit on that side, and drop the two wrappers. move_folios_to_lru() passes scanned_seq + 1 as min_seq for rejected folios, as before. Also rename "reclaiming" to "tail". With MGLRU, the tail of the LRU is the tail of the oldest generation, so when tail is true, lru_gen_add_folio() now picks min_seq[type] itself, and lru_gen_folio_seq() no longer needs the argument. This does not change behavior. The old reclaiming branch in lru_gen_folio_seq() also resulted in min_seq[type]. It was only skipped for active folios, and none of the callers that pass true add an active folio. max_seq has no user yet. The next patch uses it in lru_deactivate(). No functional change intended. Suggested-by: Kairui Song Signed-off-by: Ridong Chen --- include/linux/mm_inline.h | 52 +++++++++++++++++---------------------- mm/vmscan.c | 11 +++------ 2 files changed, 26 insertions(+), 37 deletions(-) diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index f6f4bc9a704f..69f69b9d8018 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -299,8 +299,7 @@ static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *foli } static inline unsigned long lru_gen_folio_seq(const struct lruvec *lruvec, - const struct folio *folio, - bool reclaiming) + const struct folio *folio) { int gen; int type = folio_is_file_lru(folio); @@ -320,8 +319,6 @@ static inline unsigned long lru_gen_folio_seq(const struct lruvec *lruvec, */ if (folio_test_active(folio)) gen = MIN_NR_GENS - folio_test_workingset(folio); - else if (reclaiming) - gen = MAX_NR_GENS; else if ((!folio_is_file_lru(folio) && !folio_test_swapcache(folio)) || (folio_test_reclaim(folio) && (folio_test_dirty(folio) || folio_test_writeback(folio)))) @@ -332,8 +329,13 @@ static inline unsigned long lru_gen_folio_seq(const struct lruvec *lruvec, return max(READ_ONCE(lrugen->max_seq) - gen + 1, READ_ONCE(lrugen->min_seq[type])); } -static inline bool __lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, - unsigned long scanned_seq, bool reclaiming) +/* + * Cap the seq to [min_seq, max_seq], -1UL means no limit. With no limit and + * tail set, the folio is added to the tail of the oldest gen. + */ +static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, + unsigned long min_seq, unsigned long max_seq, + bool tail) { unsigned long seq; unsigned long flags; @@ -347,9 +349,14 @@ static inline bool __lru_gen_add_folio(struct lruvec *lruvec, struct folio *foli if (folio_test_unevictable(folio) || !lrugen->enabled) return false; - seq = lru_gen_folio_seq(lruvec, folio, reclaiming); - if (scanned_seq != -1UL) - seq = max(seq, min(scanned_seq + 1, lrugen->max_seq)); + if (tail) + seq = READ_ONCE(lrugen->min_seq[type]); + else + seq = lru_gen_folio_seq(lruvec, folio); + if (min_seq != -1UL) + seq = max(seq, min(min_seq, lrugen->max_seq)); + if (max_seq != -1UL) + seq = min(seq, max_seq); gen = lru_gen_from_seq(seq); flags = (gen + 1UL) << LRU_GEN_PGOFF; /* see the comment on MIN_NR_GENS about PG_active */ @@ -357,7 +364,7 @@ static inline bool __lru_gen_add_folio(struct lruvec *lruvec, struct folio *foli lru_gen_update_size(lruvec, folio, -1, gen); /* for folio_rotate_reclaimable() */ - if (reclaiming) + if (tail) list_add_tail(&folio->lru, &lrugen->folios[gen][type][zone]); else list_add(&folio->lru, &lrugen->folios[gen][type][zone]); @@ -365,17 +372,6 @@ static inline bool __lru_gen_add_folio(struct lruvec *lruvec, struct folio *foli return true; } -static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) -{ - return __lru_gen_add_folio(lruvec, folio, -1UL, reclaiming); -} - -static inline bool lru_gen_add_rejected_folio(struct lruvec *lruvec, struct folio *folio, - unsigned long scanned_seq) -{ - return __lru_gen_add_folio(lruvec, folio, scanned_seq, false); -} - static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) { unsigned long flags; @@ -429,13 +425,9 @@ static inline bool lru_gen_in_fault(void) return false; } -static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) -{ - return false; -} - -static inline bool lru_gen_add_rejected_folio(struct lruvec *lruvec, struct folio *folio, - unsigned long scanned_seq) +static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, + unsigned long min_seq, unsigned long max_seq, + bool tail) { return false; } @@ -459,7 +451,7 @@ void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio) VM_WARN_ON_ONCE_FOLIO(!folio_matches_lruvec(folio, lruvec), folio); - if (lru_gen_add_folio(lruvec, folio, false)) + if (lru_gen_add_folio(lruvec, folio, -1UL, -1UL, false)) return; update_lru_size(lruvec, lru, folio_zonenum(folio), @@ -475,7 +467,7 @@ void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio) VM_WARN_ON_ONCE_FOLIO(!folio_matches_lruvec(folio, lruvec), folio); - if (lru_gen_add_folio(lruvec, folio, true)) + if (lru_gen_add_folio(lruvec, folio, -1UL, -1UL, true)) return; update_lru_size(lruvec, lru, folio_zonenum(folio), diff --git a/mm/vmscan.c b/mm/vmscan.c index bc7579d47f9b..19e06574ea00 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1916,7 +1916,8 @@ static unsigned int move_folios_to_lru(struct list_head *list, unsigned long seq !folio_test_referenced(folio) && folio_ref_count(folio) == folio_expected_ref_count(folio)) lruvec_add_folio_tail(lruvec, folio); - else if (seq == -1UL || !lru_gen_add_rejected_folio(lruvec, folio, seq)) + else if (seq == -1UL || + !lru_gen_add_folio(lruvec, folio, seq + 1, -1UL, false)) lruvec_add_folio(lruvec, folio); nr_pages = folio_nr_pages(folio); @@ -5516,7 +5517,7 @@ static bool fill_evictable(struct lruvec *lruvec) /* * lru_gen_add_folio() uses list_add_tail() rather - * than list_add() when reclaiming is true. Match + * than list_add() when tail is true. Match * its ordering to avoid cold/hot inversion. */ if (active) @@ -5530,11 +5531,7 @@ static bool fill_evictable(struct lruvec *lruvec) VM_WARN_ON_ONCE_FOLIO(folio_lru_gen(folio) != -1, folio); lruvec_del_folio(lruvec, folio); - /* - * Borrow reclaiming=true to place inactive folios in - * the older gens. - */ - success = lru_gen_add_folio(lruvec, folio, !active); + success = lru_gen_add_folio(lruvec, folio, -1UL, -1UL, !active); VM_WARN_ON_ONCE(!success); if (!--remaining) -- 2.34.1 From: Ridong Chen folio_deactivate() is used by MADV_COLD and DAMOS_LRU_DEPRIO to move a folio towards eviction. lru_deactivate() does that by deleting the folio from its LRU list, clearing PG_active and PG_referenced, and adding it back with lruvec_add_folio(). With MGLRU, lru_gen_folio_seq() places a swap-backed folio that is not in the swap cache, i.e. an anon or shmem folio, into max_seq - 1, regardless of its refs. That generation is active per lru_gen_is_active(), so such a folio never reaches an inactive generation, and a folio that has aged into min_seq + 1 is moved to a younger generation instead. PGDEACTIVATE is still counted. Other folios are placed into the oldest generation with their refs cleared, except the ones that have PG_reclaim set and are dirty or under writeback, which also go to max_seq - 1. On a 4-generation memcg, 64MB of aged anon memory in min_seq + 1 ends up in max_seq - 1 after MADV_COLD, while 64MB of file memory in the same generation ends up in min_seq. As a result, MADV_COLD can make anon memory less likely to be reclaimed. It can be reproduced with a single process in a memcg with swap enabled: 1) map two 64MB anon regions A and B and fault them in, A first; 2) age the memcg twice through /sys/kernel/debug/lru_gen, so that both regions are in the same generation, min_seq + 1; 3) apply MADV_COLD to B only, A is the control; 4) reclaim the size of one region from anon only: echo "64M swappiness=max" > memory.reclaim; 5) count the swapped out pages of A and B from /proc/self/pagemap. B is expected to be swapped out instead of A. Three runs on mm-new: A swapped out B (MADV_COLD) swapped out before 16384 pages 95 ~ 151 pages after 29 ~ 126 pages 16384 pages Cap the generation to min_seq[type] for MGLRU, so that the folio is added to the head of the oldest generation. This matches the classic LRU, which adds a deactivated folio to the head of the inactive list, and is where most file folios already go. Folios that have PG_reclaim set and are dirty or under writeback are now placed there too, as the classic LRU puts them on the inactive list. min_seq only advances under the LRU lock, which is held here. Fixes: 081488051d28 ("mm/mglru: fix underprotected page cache") Suggested-by: Barry Song Signed-off-by: Ridong Chen --- The reproducer, run as root in a memcg with swap enabled and debugfs mounted, e.g. "mkdir /sys/fs/cgroup/t; ./reclaim-order /sys/fs/cgroup/t": // SPDX-License-Identifier: GPL-2.0 /* * Two 64MB anon regions, A (hot) and B (MADV_COLD). Reclaim 64MB via * memory.reclaim (swappiness=max -> anon only). Report how much of each * region was swapped out (mincore can't see swap; use /proc/self/pagemap). */ #define _GNU_SOURCE #include #include #include #include #include #include #include #include static unsigned long long memcg_id; static const char *cg_name; /* force one aging step on node 0 of our memcg via debugfs */ static void aging(void) { FILE *f = fopen("/sys/kernel/debug/lru_gen", "r"); char line[512], cmd[128]; unsigned long max = 0, s, x1, x2, x3; int in = 0, fd, len; while (fgets(line, sizeof(line), f)) { unsigned long long id; char path[256]; if (sscanf(line, "memcg %llu %255s", &id, path) == 2) { in = !strcmp(path, cg_name); if (in) memcg_id = id; continue; } if (in && sscanf(line, " %lu %lu %lu %lu", &s, &x1, &x2, &x3) == 4) max = s; } fclose(f); len = snprintf(cmd, sizeof(cmd), "+ %llu 0 %lu 200 1\n", memcg_id, max); fd = open("/sys/kernel/debug/lru_gen", O_WRONLY); if (write(fd, cmd, len) != len) perror("aging"); close(fd); } static long swapped(char *p, size_t sz) { int fd = open("/proc/self/pagemap", O_RDONLY); long n = 0; for (size_t off = 0; off < sz; off += 4096) { uint64_t e; pread(fd, &e, 8, ((uintptr_t)(p + off) / 4096) * 8); if (e & (1ULL << 62)) n++; } close(fd); return n; } int main(int argc, char **argv) { const char *cg = argv[1]; cg_name = strrchr(cg, '/'); size_t sz = 64UL << 20; char buf[256], *a, *b, *d; cpu_set_t set; int fd; CPU_ZERO(&set); CPU_SET(0, &set); sched_setaffinity(0, sizeof(set), &set); snprintf(buf, sizeof(buf), "%s/cgroup.procs", cg); fd = open(buf, O_WRONLY); dprintf(fd, "%d\n", getpid()); close(fd); a = mmap(NULL, sz, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); b = mmap(NULL, sz, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); d = mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); madvise(a, sz, MADV_NOHUGEPAGE); madvise(b, sz, MADV_NOHUGEPAGE); /* A is faulted in first, so without MADV_COLD it is the older one */ for (size_t i = 0; i < sz; i += 4096) a[i] = 1; for (size_t i = 0; i < sz; i += 4096) b[i] = 1; d[0] = 1; madvise(d, 4096, MADV_COLD); /* drain lru_add */ /* two agings: clear young bits of both A and B, both end at min_seq+1 */ aging(); aging(); madvise(b, sz, MADV_COLD); madvise(d, 4096, MADV_COLD); snprintf(buf, sizeof(buf), "%s/memory.reclaim", cg); fd = open(buf, O_WRONLY); if (dprintf(fd, "64M swappiness=max\n") < 0) perror("memory.reclaim"); close(fd); printf("hot A swapped: %5ld pages cold B swapped: %5ld pages (of %zu)\n", swapped(a, sz), swapped(b, sz), sz >> 12); return 0; } mm/folio.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/mm/folio.c b/mm/folio.c index 7a3394e7c5dc..1bfd746d7f07 100644 --- a/mm/folio.c +++ b/mm/folio.c @@ -423,6 +423,14 @@ static bool lru_gen_clear_refs(struct folio *folio) return gen == lru_gen_from_seq(seq); } +/* add the folio to the head of the oldest generation */ +static bool lru_gen_add_deactivated_folio(struct lruvec *lruvec, struct folio *folio) +{ + int type = folio_is_file_lru(folio); + + return lru_gen_add_folio(lruvec, folio, -1UL, lruvec->lrugen.min_seq[type], false); +} + #else /* !CONFIG_LRU_GEN */ static void lru_gen_inc_refs(struct folio *folio) @@ -434,6 +442,11 @@ static bool lru_gen_clear_refs(struct folio *folio) return false; } +static bool lru_gen_add_deactivated_folio(struct lruvec *lruvec, struct folio *folio) +{ + return false; +} + #endif /* CONFIG_LRU_GEN */ /** @@ -609,7 +622,8 @@ static void lru_deactivate(struct lruvec *lruvec, struct folio *folio) lruvec_del_folio(lruvec, folio); folio_clear_active(folio); folio_clear_referenced(folio); - lruvec_add_folio(lruvec, folio); + if (!lru_gen_add_deactivated_folio(lruvec, folio)) + lruvec_add_folio(lruvec, folio); __count_vm_events(PGDEACTIVATE, nr_pages); count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_pages); -- 2.34.1