From: Baoquan He Generalize the Bloom filter state and helpers so that another filter level can reuse the same hash, double-buffering and reset logic. Replace the PMD-level filter array with a two-dimensional filters array indexed by level, and add BLOOM_FILTER_PUD and BLOOM_FILTER_PMD. The PUD-level filters operate at PUD granularity and generalize the suppression one level up: when all PMDs under a PUD fail the PMD-level filter, the whole PMD iteration can be skipped. Cross-node empty walks are the most visible case: a foreign mm's PUDs have no young entries for this lruvec, so the entire PMD iteration is pure waste. The existing PMD-level code now goes through the generic test/update/reset_bloom_filter() helpers. No behavior change yet; the PUD-level filters are populated and used by the next patch. Signed-off-by: Baoquan He Signed-off-by: Baoquan He --- include/linux/mmzone.h | 10 ++++++-- mm/vmscan.c | 58 +++++++++++++++++++++++++++++------------- 2 files changed, 49 insertions(+), 19 deletions(-) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index a49efd9b74ed..1f086ef3a00e 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -589,6 +589,12 @@ enum { /* double-buffering Bloom filters */ #define NR_BLOOM_FILTERS 2 +enum bloom_filter_level { + BLOOM_FILTER_PUD, + BLOOM_FILTER_PMD, + BLOOM_FILTER_LEVELS, +}; + struct lru_gen_mm_state { /* synced with max_seq after each iteration */ unsigned long seq; @@ -596,8 +602,8 @@ struct lru_gen_mm_state { struct list_head *head; /* where the last iteration ended before */ struct list_head *tail; - /* Bloom filters flip after each iteration */ - unsigned long *filters[NR_BLOOM_FILTERS]; + /* Bloom filters flip after each iteration, indexed by level */ + unsigned long *filters[BLOOM_FILTER_LEVELS][NR_BLOOM_FILTERS]; /* the mm stats for debugging */ unsigned long stats[NR_HIST_GENS][NR_MM_STATS]; }; diff --git a/mm/vmscan.c b/mm/vmscan.c index 98037a562ab6..9dbb3f477d7d 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -2798,6 +2798,10 @@ static bool __maybe_unused seq_is_valid(struct lruvec *lruvec) * walk_pmd_range(); the eviction also report them when walking the rmap * in lru_gen_look_around(). * + * A second, coarser pair of filters sits one level up. The filters are + * indexed by level: BLOOM_FILTER_PMD for the PMD-level filters above, and + * BLOOM_FILTER_PUD for the PUD-level pair. + * * For future optimizations: * 1. It's not necessary to keep both filters all the time. The spare one can be * freed after the RCU grace period and reallocated if needed again. @@ -2823,14 +2827,13 @@ static void get_item_key(void *item, int *key) key[1] = hash >> BLOOM_FILTER_SHIFT; } -static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq, - void *item) +static bool __test_bloom_filter(unsigned long **filters, unsigned long seq, void *item) { int key[2]; unsigned long *filter; int gen = filter_gen_from_seq(seq); - filter = READ_ONCE(mm_state->filters[gen]); + filter = READ_ONCE(filters[gen]); if (!filter) return true; @@ -2839,14 +2842,13 @@ static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long s return test_bit(key[0], filter) && test_bit(key[1], filter); } -static void update_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq, - void *item) +static void __update_bloom_filter(unsigned long **filters, unsigned long seq, void *item) { int key[2]; unsigned long *filter; int gen = filter_gen_from_seq(seq); - filter = READ_ONCE(mm_state->filters[gen]); + filter = READ_ONCE(filters[gen]); if (!filter) return; @@ -2858,12 +2860,12 @@ static void update_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long set_bit(key[1], filter); } -static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq) +static void __reset_bloom_filter(unsigned long **filters, unsigned long seq) { unsigned long *filter; int gen = filter_gen_from_seq(seq); - filter = mm_state->filters[gen]; + filter = filters[gen]; if (filter) { bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT)); return; @@ -2871,7 +2873,25 @@ static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); - WRITE_ONCE(mm_state->filters[gen], filter); + WRITE_ONCE(filters[gen], filter); +} + +static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, int level, + unsigned long seq, void *item) +{ + return __test_bloom_filter(mm_state->filters[level], seq, item); +} + +static void update_bloom_filter(struct lru_gen_mm_state *mm_state, int level, + unsigned long seq, void *item) +{ + __update_bloom_filter(mm_state->filters[level], seq, item); +} + +static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, int level, + unsigned long seq) +{ + __reset_bloom_filter(mm_state->filters[level], seq); } /****************************************************************************** @@ -3114,7 +3134,7 @@ static bool iterate_mm_list(struct lru_gen_mm_walk *walk, struct mm_struct **ite spin_unlock(&mm_list->lock); if (mm && first) - reset_bloom_filter(mm_state, walk->seq + 1); + reset_bloom_filter(mm_state, BLOOM_FILTER_PMD, walk->seq + 1); if (*iter) mmdrop(*iter); @@ -3771,7 +3791,8 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first); } - if (!walk->force_scan && !test_bloom_filter(mm_state, walk->seq, pmd + i)) + if (!walk->force_scan && + !test_bloom_filter(mm_state, BLOOM_FILTER_PMD, walk->seq, pmd + i)) continue; walk->mm_stats[MM_NONLEAF_FOUND]++; @@ -3782,7 +3803,7 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, walk->mm_stats[MM_NONLEAF_ADDED]++; /* carry over to the next generation */ - update_bloom_filter(mm_state, walk->seq + 1, pmd + i); + update_bloom_filter(mm_state, BLOOM_FILTER_PMD, walk->seq + 1, pmd + i); } walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first); @@ -4417,8 +4438,8 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) /* feedback from rmap walkers to page table walkers */ if (mm_state && suitable_to_scan(i, young)) - update_bloom_filter(mm_state, READ_ONCE(lruvec->lrugen.max_seq), - pvmw->pmd); + update_bloom_filter(mm_state, BLOOM_FILTER_PMD, + READ_ONCE(lruvec->lrugen.max_seq), pvmw->pmd); mem_cgroup_put(memcg); @@ -5981,6 +6002,7 @@ void lru_gen_init_memcg(struct mem_cgroup *memcg) void lru_gen_exit_memcg(struct mem_cgroup *memcg) { int i; + int level; int nid; struct lru_gen_mm_list *mm_list = get_mm_list(memcg); @@ -5998,9 +6020,11 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg) if (!mm_state) continue; - for (i = 0; i < NR_BLOOM_FILTERS; i++) { - bitmap_free(mm_state->filters[i]); - mm_state->filters[i] = NULL; + for (level = 0; level < BLOOM_FILTER_LEVELS; level++) { + for (i = 0; i < NR_BLOOM_FILTERS; i++) { + bitmap_free(mm_state->filters[level][i]); + mm_state->filters[level][i] = NULL; + } } } } -- 2.52.0