Rename the existing PMD-level filter array (filters -> pmd_filters) and add a second, coarser pair of PUD-level Bloom filters (pud_filters). The PUL filsters operate at 1GB (PUD) granularity, whose 512 PMDs would all fail the PMD-level filter - a general suppression of unnecessary page table walks. Cross-node empty walks are the most visible case: a foreign mm's PUDs have no young entries for this lruvec, so the entire PMD iteration is pure waste. Generalize the Bloom filter helpers (test/update/reset_bloom_filter -> __test/update/reset_bloom_filter) to operate on a generic filters array, and add thin wrappers (test/update/reset_pmd_bloom_filter) that pass the PMD-level array, so the PUD-level pair can reuse the same hash, double-buffering and reset. No behavior change yet; the PUD filters are populated and used by later patches. Signed-off-by: Baoquan He --- include/linux/mmzone.h | 6 ++++-- mm/vmscan.c | 45 ++++++++++++++++++++++++++++-------------- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index f75daa6cd02d..faf28d35065d 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -612,8 +612,10 @@ struct lru_gen_mm_state { struct list_head *head; /* where the last iteration ended before */ struct list_head *tail; - /* Bloom filters flip after each iteration */ - unsigned long *filters[NR_BLOOM_FILTERS]; + /* PMD-level Bloom filters flip after each iteration */ + unsigned long *pmd_filters[NR_BLOOM_FILTERS]; + /* PUD-level Bloom filters flip after each iteration */ + unsigned long *pud_filters[NR_BLOOM_FILTERS]; /* the mm stats for debugging */ unsigned long stats[NR_HIST_GENS][NR_MM_STATS]; }; diff --git a/mm/vmscan.c b/mm/vmscan.c index 55d43ab54459..cd09c5b8af8c 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -2842,14 +2842,13 @@ static void get_item_key(void *item, int *key) key[1] = hash >> BLOOM_FILTER_SHIFT; } -static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq, - void *item) +static bool __test_bloom_filter(unsigned long **filters, unsigned long seq, void *item) { int key[2]; unsigned long *filter; int gen = filter_gen_from_seq(seq); - filter = READ_ONCE(mm_state->filters[gen]); + filter = READ_ONCE(filters[gen]); if (!filter) return true; @@ -2858,14 +2857,13 @@ static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long s return test_bit(key[0], filter) && test_bit(key[1], filter); } -static void update_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq, - void *item) +static void __update_bloom_filter(unsigned long **filters, unsigned long seq, void *item) { int key[2]; unsigned long *filter; int gen = filter_gen_from_seq(seq); - filter = READ_ONCE(mm_state->filters[gen]); + filter = READ_ONCE(filters[gen]); if (!filter) return; @@ -2877,12 +2875,12 @@ static void update_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long set_bit(key[1], filter); } -static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq) +static void __reset_bloom_filter(unsigned long **filters, unsigned long seq) { unsigned long *filter; int gen = filter_gen_from_seq(seq); - filter = mm_state->filters[gen]; + filter = filters[gen]; if (filter) { bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT)); return; @@ -2890,7 +2888,24 @@ static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); - WRITE_ONCE(mm_state->filters[gen], filter); + WRITE_ONCE(filters[gen], filter); +} + +static bool test_pmd_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq, + void *item) +{ + return __test_bloom_filter(mm_state->pmd_filters, seq, item); +} + +static void update_pmd_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq, + void *item) +{ + __update_bloom_filter(mm_state->pmd_filters, seq, item); +} + +static void reset_pmd_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq) +{ + __reset_bloom_filter(mm_state->pmd_filters, seq); } /****************************************************************************** @@ -3133,7 +3148,7 @@ static bool iterate_mm_list(struct lru_gen_mm_walk *walk, struct mm_struct **ite spin_unlock(&mm_list->lock); if (mm && first) - reset_bloom_filter(mm_state, walk->seq + 1); + reset_pmd_bloom_filter(mm_state, walk->seq + 1); if (*iter) mmdrop(*iter); @@ -3771,7 +3786,7 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first); } - if (!walk->force_scan && !test_bloom_filter(mm_state, walk->seq, pmd + i)) + if (!walk->force_scan && !test_pmd_bloom_filter(mm_state, walk->seq, pmd + i)) continue; walk->mm_stats[MM_NONLEAF_FOUND]++; @@ -3782,7 +3797,7 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, walk->mm_stats[MM_NONLEAF_ADDED]++; /* carry over to the next generation */ - update_bloom_filter(mm_state, walk->seq + 1, pmd + i); + update_pmd_bloom_filter(mm_state, walk->seq + 1, pmd + i); } walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first); @@ -4383,7 +4398,7 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) /* feedback from rmap walkers to page table walkers */ if (mm_state && suitable_to_scan(i, young)) - update_bloom_filter(mm_state, max_seq, pvmw->pmd); + update_pmd_bloom_filter(mm_state, max_seq, pvmw->pmd); mem_cgroup_put(memcg); @@ -5946,8 +5961,8 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg) continue; for (i = 0; i < NR_BLOOM_FILTERS; i++) { - bitmap_free(mm_state->filters[i]); - mm_state->filters[i] = NULL; + bitmap_free(mm_state->pmd_filters[i]); + mm_state->pmd_filters[i] = NULL; } } } -- 2.54.0