From: Baoquan He Generalize the Bloom filter state and helpers so that another filter level can reuse the same hash, double-buffering and reset logic. Replace the PMD-level filter array with a two-dimensional filters array indexed by level, and add BLOOM_FILTER_PUD and BLOOM_FILTER_PMD. The PUD-level filters operate at PUD granularity and generalize the suppression one level up: when all PMDs under a PUD fail the PMD-level filter, the whole PMD iteration can be skipped. Cross-node empty walks are the most visible case: a foreign mm's PUDs have no young entries for this lruvec, so the entire PMD iteration is pure waste. test_bloom_filter(), update_bloom_filter() and reset_bloom_filter() take the level now and use mm_state->filters[level][gen] directly. No behavior change yet. No PUD-level bitmap is allocated in this patch; the next patch fills and uses them for the first time. Reviewed-by: Kairui Song Signed-off-by: Baoquan He --- include/linux/mmzone.h | 10 ++++++++-- mm/vmscan.c | 39 ++++++++++++++++++++++----------------- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 902f4877fd0c..a3a13a77ca8e 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -589,6 +589,12 @@ enum { /* double-buffering Bloom filters */ #define NR_BLOOM_FILTERS 2 +enum bloom_filter_level { + BLOOM_FILTER_PUD, + BLOOM_FILTER_PMD, + BLOOM_FILTER_LEVELS, +}; + struct lru_gen_mm_state { /* synced with max_seq after each iteration */ unsigned long seq; @@ -596,8 +602,8 @@ struct lru_gen_mm_state { struct list_head *head; /* where the last iteration ended before */ struct list_head *tail; - /* Bloom filters flip after each iteration */ - unsigned long *filters[NR_BLOOM_FILTERS]; + /* Bloom filters flip after each iteration, indexed by level */ + unsigned long *filters[BLOOM_FILTER_LEVELS][NR_BLOOM_FILTERS]; /* the mm stats for debugging */ unsigned long stats[NR_HIST_GENS][NR_MM_STATS]; }; diff --git a/mm/vmscan.c b/mm/vmscan.c index 36e0946bf2cc..a4c64a190391 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -2819,14 +2819,14 @@ static void get_item_key(void *item, int *key) key[1] = hash >> BLOOM_FILTER_SHIFT; } -static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq, - void *item) +static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, int level, + unsigned long seq, void *item) { int key[2]; unsigned long *filter; int gen = filter_gen_from_seq(seq); - filter = READ_ONCE(mm_state->filters[gen]); + filter = READ_ONCE(mm_state->filters[level][gen]); if (!filter) return true; @@ -2835,14 +2835,14 @@ static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long s return test_bit(key[0], filter) && test_bit(key[1], filter); } -static void update_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq, - void *item) +static void update_bloom_filter(struct lru_gen_mm_state *mm_state, int level, + unsigned long seq, void *item) { int key[2]; unsigned long *filter; int gen = filter_gen_from_seq(seq); - filter = READ_ONCE(mm_state->filters[gen]); + filter = READ_ONCE(mm_state->filters[level][gen]); if (!filter) return; @@ -2854,12 +2854,13 @@ static void update_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long set_bit(key[1], filter); } -static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq) +static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, int level, + unsigned long seq) { unsigned long *filter; int gen = filter_gen_from_seq(seq); - filter = mm_state->filters[gen]; + filter = mm_state->filters[level][gen]; if (filter) { bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT)); return; @@ -2867,7 +2868,7 @@ static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); - WRITE_ONCE(mm_state->filters[gen], filter); + WRITE_ONCE(mm_state->filters[level][gen], filter); } /****************************************************************************** @@ -3110,7 +3111,7 @@ static bool iterate_mm_list(struct lru_gen_mm_walk *walk, struct mm_struct **ite spin_unlock(&mm_list->lock); if (mm && first) - reset_bloom_filter(mm_state, walk->seq + 1); + reset_bloom_filter(mm_state, BLOOM_FILTER_PMD, walk->seq + 1); if (*iter) mmdrop(*iter); @@ -3766,7 +3767,8 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first); } - if (!walk->force_scan && !test_bloom_filter(mm_state, walk->seq, pmd + i)) + if (!walk->force_scan && + !test_bloom_filter(mm_state, BLOOM_FILTER_PMD, walk->seq, pmd + i)) continue; walk->mm_stats[MM_NONLEAF_FOUND]++; @@ -3777,7 +3779,7 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, walk->mm_stats[MM_NONLEAF_ADDED]++; /* carry over to the next generation */ - update_bloom_filter(mm_state, walk->seq + 1, pmd + i); + update_bloom_filter(mm_state, BLOOM_FILTER_PMD, walk->seq + 1, pmd + i); } walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first); @@ -4436,8 +4438,8 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) /* feedback from rmap walkers to page table walkers */ if (mm_state && suitable_to_scan(i, young)) - update_bloom_filter(mm_state, READ_ONCE(lruvec->lrugen.max_seq), - pvmw->pmd); + update_bloom_filter(mm_state, BLOOM_FILTER_PMD, + READ_ONCE(lruvec->lrugen.max_seq), pvmw->pmd); mem_cgroup_put(memcg); @@ -6153,6 +6155,7 @@ void lru_gen_init_memcg(struct mem_cgroup *memcg) void lru_gen_exit_memcg(struct mem_cgroup *memcg) { int i; + int level; int nid; struct lru_gen_mm_list *mm_list = get_mm_list(memcg); @@ -6170,9 +6173,11 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg) if (!mm_state) continue; - for (i = 0; i < NR_BLOOM_FILTERS; i++) { - bitmap_free(mm_state->filters[i]); - mm_state->filters[i] = NULL; + for (level = 0; level < BLOOM_FILTER_LEVELS; level++) { + for (i = 0; i < NR_BLOOM_FILTERS; i++) { + bitmap_free(mm_state->filters[level][i]); + mm_state->filters[level][i] = NULL; + } } } } -- 2.52.0