From: Ridong Chen folio_deactivate() is used by MADV_COLD and DAMOS_LRU_DEPRIO to move a folio towards eviction. lru_deactivate() does that by deleting the folio from its LRU list, clearing PG_active and PG_referenced, and adding it back with lruvec_add_folio(). With MGLRU, lru_gen_folio_seq() places a swap-backed folio that is not in the swap cache, i.e. an anon or shmem folio, into max_seq - 1, regardless of its refs. That generation is active per lru_gen_is_active(), so such a folio never reaches an inactive generation, and a folio that has aged into min_seq + 1 is moved to a younger generation instead. PGDEACTIVATE is still counted. Other folios are placed into the oldest generation with their refs cleared, except the ones that have PG_reclaim set and are dirty or under writeback, which also go to max_seq - 1. On a 4-generation memcg, 64MB of aged anon memory in min_seq + 1 ends up in max_seq - 1 after MADV_COLD, while 64MB of file memory in the same generation ends up in min_seq. As a result, MADV_COLD can make anon memory less likely to be reclaimed. It can be reproduced with a single process in a memcg with swap enabled: 1) map two 64MB anon regions A and B and fault them in, A first; 2) age the memcg twice through /sys/kernel/debug/lru_gen, so that both regions are in the same generation, min_seq + 1; 3) apply MADV_COLD to B only, A is the control; 4) reclaim the size of one region from anon only: echo "64M swappiness=max" > memory.reclaim; 5) count the swapped out pages of A and B from /proc/self/pagemap. B is expected to be swapped out instead of A. Three runs on mm-new: A swapped out B (MADV_COLD) swapped out before 16384 pages 95 ~ 151 pages after 29 ~ 126 pages 16384 pages Cap the generation to min_seq[type] for MGLRU, so that the folio is added to the head of the oldest generation. This matches the classic LRU, which adds a deactivated folio to the head of the inactive list, and is where most file folios already go. Folios that have PG_reclaim set and are dirty or under writeback are now placed there too, as the classic LRU puts them on the inactive list. min_seq only advances under the LRU lock, which is held here. Fixes: 081488051d28 ("mm/mglru: fix underprotected page cache") Suggested-by: Barry Song Signed-off-by: Ridong Chen --- The reproducer, run as root in a memcg with swap enabled and debugfs mounted, e.g. "mkdir /sys/fs/cgroup/t; ./reclaim-order /sys/fs/cgroup/t": // SPDX-License-Identifier: GPL-2.0 /* * Two 64MB anon regions, A (hot) and B (MADV_COLD). Reclaim 64MB via * memory.reclaim (swappiness=max -> anon only). Report how much of each * region was swapped out (mincore can't see swap; use /proc/self/pagemap). */ #define _GNU_SOURCE #include #include #include #include #include #include #include #include static unsigned long long memcg_id; static const char *cg_name; /* force one aging step on node 0 of our memcg via debugfs */ static void aging(void) { FILE *f = fopen("/sys/kernel/debug/lru_gen", "r"); char line[512], cmd[128]; unsigned long max = 0, s, x1, x2, x3; int in = 0, fd, len; while (fgets(line, sizeof(line), f)) { unsigned long long id; char path[256]; if (sscanf(line, "memcg %llu %255s", &id, path) == 2) { in = !strcmp(path, cg_name); if (in) memcg_id = id; continue; } if (in && sscanf(line, " %lu %lu %lu %lu", &s, &x1, &x2, &x3) == 4) max = s; } fclose(f); len = snprintf(cmd, sizeof(cmd), "+ %llu 0 %lu 200 1\n", memcg_id, max); fd = open("/sys/kernel/debug/lru_gen", O_WRONLY); if (write(fd, cmd, len) != len) perror("aging"); close(fd); } static long swapped(char *p, size_t sz) { int fd = open("/proc/self/pagemap", O_RDONLY); long n = 0; for (size_t off = 0; off < sz; off += 4096) { uint64_t e; pread(fd, &e, 8, ((uintptr_t)(p + off) / 4096) * 8); if (e & (1ULL << 62)) n++; } close(fd); return n; } int main(int argc, char **argv) { const char *cg = argv[1]; cg_name = strrchr(cg, '/'); size_t sz = 64UL << 20; char buf[256], *a, *b, *d; cpu_set_t set; int fd; CPU_ZERO(&set); CPU_SET(0, &set); sched_setaffinity(0, sizeof(set), &set); snprintf(buf, sizeof(buf), "%s/cgroup.procs", cg); fd = open(buf, O_WRONLY); dprintf(fd, "%d\n", getpid()); close(fd); a = mmap(NULL, sz, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); b = mmap(NULL, sz, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); d = mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); madvise(a, sz, MADV_NOHUGEPAGE); madvise(b, sz, MADV_NOHUGEPAGE); /* A is faulted in first, so without MADV_COLD it is the older one */ for (size_t i = 0; i < sz; i += 4096) a[i] = 1; for (size_t i = 0; i < sz; i += 4096) b[i] = 1; d[0] = 1; madvise(d, 4096, MADV_COLD); /* drain lru_add */ /* two agings: clear young bits of both A and B, both end at min_seq+1 */ aging(); aging(); madvise(b, sz, MADV_COLD); madvise(d, 4096, MADV_COLD); snprintf(buf, sizeof(buf), "%s/memory.reclaim", cg); fd = open(buf, O_WRONLY); if (dprintf(fd, "64M swappiness=max\n") < 0) perror("memory.reclaim"); close(fd); printf("hot A swapped: %5ld pages cold B swapped: %5ld pages (of %zu)\n", swapped(a, sz), swapped(b, sz), sz >> 12); return 0; } mm/folio.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/mm/folio.c b/mm/folio.c index 7a3394e7c5dc..1bfd746d7f07 100644 --- a/mm/folio.c +++ b/mm/folio.c @@ -423,6 +423,14 @@ static bool lru_gen_clear_refs(struct folio *folio) return gen == lru_gen_from_seq(seq); } +/* add the folio to the head of the oldest generation */ +static bool lru_gen_add_deactivated_folio(struct lruvec *lruvec, struct folio *folio) +{ + int type = folio_is_file_lru(folio); + + return lru_gen_add_folio(lruvec, folio, -1UL, lruvec->lrugen.min_seq[type], false); +} + #else /* !CONFIG_LRU_GEN */ static void lru_gen_inc_refs(struct folio *folio) @@ -434,6 +442,11 @@ static bool lru_gen_clear_refs(struct folio *folio) return false; } +static bool lru_gen_add_deactivated_folio(struct lruvec *lruvec, struct folio *folio) +{ + return false; +} + #endif /* CONFIG_LRU_GEN */ /** @@ -609,7 +622,8 @@ static void lru_deactivate(struct lruvec *lruvec, struct folio *folio) lruvec_del_folio(lruvec, folio); folio_clear_active(folio); folio_clear_referenced(folio); - lruvec_add_folio(lruvec, folio); + if (!lru_gen_add_deactivated_folio(lruvec, folio)) + lruvec_add_folio(lruvec, folio); __count_vm_events(PGDEACTIVATE, nr_pages); count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_pages); -- 2.34.1