Add infrastructure to quantify cross-node empty page table walks in the MGLRU aging path. These are walks that traverse an mm's page tables but find no folio belonging to the current lruvec (node+memcg) - pure waste caused by lru_gen_use_mm() setting mm->lru_gen.bitmap to -1 (all nodes) at context switch. New per-walk counters (accumulated in mm_state->stats[]): MM_LEAF_ELIGIBLE - folios belonging to this lruvec MM_WALK_TOTAL - page-table walks completed MM_WALK_EMPTY - walks that found no eligible folio MM_LEAF_TOTAL_EMPTY - leaf entries scanned by empty walks A new tracepoint, mm_vmscan_lru_gen_walk(nid, seq, leaf_total, leaf_eligible, empty), fires after each walk for live monitoring. The debugfs lru_gen output format is updated ("TYFA" -> "TYFALWEE") to display the new fields. This series uses these counters to quantify the cross-node empty walk cost and to validate the suppression implemented by the follow-up patches. Signed-off-by: Baoquan He --- include/linux/mmzone.h | 4 ++++ include/trace/events/vmscan.h | 28 ++++++++++++++++++++++++++++ mm/vmscan.c | 30 ++++++++++++++++++++++++++---- 3 files changed, 58 insertions(+), 4 deletions(-) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index a26c8b855222..34938d4f8e23 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -595,6 +595,10 @@ enum { MM_LEAF_YOUNG, /* young leaf entries */ MM_NONLEAF_FOUND, /* non-leaf entries found in Bloom filters */ MM_NONLEAF_ADDED, /* non-leaf entries added to Bloom filters */ + MM_LEAF_ELIGIBLE, /* folios belonging to this lruvec (node+memcg) */ + MM_WALK_TOTAL, /* page-table walks completed */ + MM_WALK_EMPTY, /* walks that found no eligible folio */ + MM_LEAF_TOTAL_EMPTY, /* leaf entries scanned by empty walks */ NR_MM_STATS }; diff --git a/include/trace/events/vmscan.h b/include/trace/events/vmscan.h index b4bf7b8def1f..c7c2034715b6 100644 --- a/include/trace/events/vmscan.h +++ b/include/trace/events/vmscan.h @@ -659,6 +659,34 @@ TRACE_EVENT(mm_vmscan_kswapd_clear_hopeless, __entry->nid, __print_symbolic(__entry->reason, kswapd_clear_hopeless_reason_ops)) ); +TRACE_EVENT(mm_vmscan_lru_gen_walk, + + TP_PROTO(int nid, unsigned long seq, int leaf_total, + int leaf_eligible, bool empty), + + TP_ARGS(nid, seq, leaf_total, leaf_eligible, empty), + + TP_STRUCT__entry( + __field(int, nid) + __field(unsigned long, seq) + __field(int, leaf_total) + __field(int, leaf_eligible) + __field(bool, empty) + ), + + TP_fast_assign( + __entry->nid = nid; + __entry->seq = seq; + __entry->leaf_total = leaf_total; + __entry->leaf_eligible = leaf_eligible; + __entry->empty = empty; + ), + + TP_printk("nid=%d seq=%lu leaf_total=%d leaf_eligible=%d empty=%d", + __entry->nid, __entry->seq, __entry->leaf_total, + __entry->leaf_eligible, __entry->empty) +); + #endif /* _TRACE_VMSCAN_H */ /* This part must be outside protection */ diff --git a/mm/vmscan.c b/mm/vmscan.c index 17d2b793cbfc..83de2b147919 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3587,6 +3587,8 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, if (!folio) continue; + walk->mm_stats[MM_LEAF_ELIGIBLE]++; + if (folio_test_large(folio)) { const unsigned int max_nr = (end - addr) >> PAGE_SHIFT; @@ -3687,6 +3689,8 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area if (!folio) goto next; + walk->mm_stats[MM_LEAF_ELIGIBLE]++; + if (!pmdp_test_and_clear_young_notify(vma, addr, pmd + i)) goto next; @@ -4108,8 +4112,26 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long seq, do { success = iterate_mm_list(walk, &mm); - if (mm) + if (mm) { + bool empty = false; + walk_mm(mm, walk); + /* A walk that traversed page tables but found no folio + * belonging to this lruvec (node+memcg) is pure waste. */ + if (walk->mm_stats[MM_LEAF_TOTAL]) { + walk->mm_stats[MM_WALK_TOTAL]++; + if (walk->mm_stats[MM_LEAF_ELIGIBLE] == 0) { + walk->mm_stats[MM_WALK_EMPTY]++; + walk->mm_stats[MM_LEAF_TOTAL_EMPTY] += + walk->mm_stats[MM_LEAF_TOTAL]; + empty = true; + } + } + trace_mm_vmscan_lru_gen_walk( + lruvec_pgdat(lruvec)->node_id, walk->seq, + walk->mm_stats[MM_LEAF_TOTAL], + walk->mm_stats[MM_LEAF_ELIGIBLE], empty); + } } while (mm); done: if (success) { @@ -5588,14 +5610,14 @@ static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec, seq_puts(m, " "); for (i = 0; i < NR_MM_STATS; i++) { - const char *s = "xxxx"; + const char *s = "xxxxxxxx"; unsigned long n = 0; if (seq == max_seq && NR_HIST_GENS == 1) { - s = "TYFA"; + s = "TYFALWEE"; n = READ_ONCE(mm_state->stats[hist][i]); } else if (seq != max_seq && NR_HIST_GENS > 1) { - s = "tyfa"; + s = "tyfalwee"; n = READ_ONCE(mm_state->stats[hist][i]); } -- 2.54.0