From: Baoquan He The aging walks into every PUD and runs the PMD-level Bloom filter on each PMD. Add a coarser PUD-level filter one level up: - walk_pmd_range() reports whether the subtree held anything young for this lruvec. That is the same condition which already carries a PMD over into the next generation: a young PMD entry, a huge page owned by this lruvec, or PTEs with enough young ones to pass suitable_to_scan(). - walk_pud_range() records that in the PUD filter, and - on subsequent generations, skips the whole PUD subtree when the filter says it had nothing young last generation. Use (mm, PUD-aligned address) as the filter key rather than the PUD page table pointer. It is a stable identity for the range being walked, is available without walking the page tables in lru_gen_look_around(), and does not depend on the lifetime of a page table page. The double-buffered filter flips with each new iteration, so newly hot or migrated-in pages are re-checked promptly rather than suppressed indefinitely. force_scan walks bypass the PUD test, so manual aging and newly added mms always rescan and re-populate the filter. To keep hot regions marked, also report the covering PUD from the rmap feedback path (lru_gen_look_around()), so regions whose hotness is only observed by eviction will be re-scanned. Suggested-by: Kairui Song Signed-off-by: Baoquan He Signed-off-by: Baoquan He --- mm/vmscan.c | 45 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 9dbb3f477d7d..d12934f2fbb3 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3133,8 +3133,10 @@ static bool iterate_mm_list(struct lru_gen_mm_walk *walk, struct mm_struct **ite spin_unlock(&mm_list->lock); - if (mm && first) + if (mm && first) { reset_bloom_filter(mm_state, BLOOM_FILTER_PMD, walk->seq + 1); + reset_bloom_filter(mm_state, BLOOM_FILTER_PUD, walk->seq + 1); + } if (*iter) mmdrop(*iter); @@ -3738,10 +3740,11 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area *first = -1; } -static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, +static bool walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, struct mm_walk *args) { int i; + bool young = false; pmd_t *pmd; unsigned long next; unsigned long addr; @@ -3778,8 +3781,10 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, walk->mm_stats[MM_LEAF_TOTAL]++; - if (pfn != -1) + if (pfn != -1) { walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first); + young = true; + } continue; } @@ -3789,6 +3794,7 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, continue; walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first); + young = true; } if (!walk->force_scan && @@ -3801,6 +3807,7 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, continue; walk->mm_stats[MM_NONLEAF_ADDED]++; + young = true; /* carry over to the next generation */ update_bloom_filter(mm_state, BLOOM_FILTER_PMD, walk->seq + 1, pmd + i); @@ -3810,6 +3817,8 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end)) goto restart; + + return young; } static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, @@ -3820,6 +3829,7 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, unsigned long addr; unsigned long next; struct lru_gen_mm_walk *walk = args->private; + struct lru_gen_mm_state *mm_state = get_mm_state(walk->lruvec); VM_WARN_ON_ONCE(p4d_leaf(*p4d)); @@ -3827,13 +3837,29 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, restart: for (i = pud_index(start), addr = start; addr != end; i++, addr = next) { pud_t val = pudp_get(pud + i); + void *pud_key; next = pud_addr_end(addr, end); if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val))) continue; - walk_pmd_range(pud + i, addr, next, args); + /* + * Use (mm, PUD-aligned address) as the filter key instead of + * the page table pointer. It is stable across the aging walk + * and the rmap feedback path, and avoids walking the page + * tables in lru_gen_look_around(). + */ + pud_key = (void *)((unsigned long)args->mm ^ (addr & PUD_MASK)); + + /* Skip a subtree that had nothing young for this lruvec last gen */ + if (!walk->force_scan && + !test_bloom_filter(mm_state, BLOOM_FILTER_PUD, walk->seq, pud_key)) + continue; + + if (walk_pmd_range(pud + i, addr, next, args)) + update_bloom_filter(mm_state, BLOOM_FILTER_PUD, walk->seq + 1, + pud_key); if (need_resched() || walk->batched >= MAX_LRU_BATCH) { end = (addr | ~PUD_MASK) + 1; @@ -4437,9 +4463,14 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) lazy_mmu_mode_disable(); /* feedback from rmap walkers to page table walkers */ - if (mm_state && suitable_to_scan(i, young)) - update_bloom_filter(mm_state, BLOOM_FILTER_PMD, - READ_ONCE(lruvec->lrugen.max_seq), pvmw->pmd); + if (mm_state && suitable_to_scan(i, young)) { + unsigned long max_seq = READ_ONCE(lruvec->lrugen.max_seq); + void *pud_key = (void *)((unsigned long)vma->vm_mm ^ + (pvmw->address & PUD_MASK)); + + update_bloom_filter(mm_state, BLOOM_FILTER_PMD, max_seq, pvmw->pmd); + update_bloom_filter(mm_state, BLOOM_FILTER_PUD, max_seq, pud_key); + } mem_cgroup_put(memcg); -- 2.52.0