Production incidents caused by bursts of high-order allocations all entering direct compaction are currently hard to attribute from /proc/vmstat: pgalloc_* has no order breakdown, and compact_stall does not say which order stalled. Tracepoints can recover this on a single machine, but they are impractical as an always-on fleet-wide monitoring source, which is what is needed to correlate latency regressions with allocation behavior after the fact. Add per-order event counters to /proc/vmstat, covering only the allocation slow path, so the page allocator fast path is not touched at all: - pgalloc_slowpath_orderN: entries into __alloc_pages_slowpath(), counted once per allocation, before the restart loop - pgalloc_fail_orderN: allocations that returned NULL to the caller (including a successful allocation freed by memcg charge failure) - compact_stall_orderN / compact_success_orderN: per-order split of the existing direct compaction counters, order 0 is omitted since direct compaction is never entered for it All new counters are purely additive: the existing keys are untouched and compact_stall == sum of compact_stall_orderN. alloc_pages_nolock() is deliberately not counted: it is opportunistic, never enters the slow path, and its NULL returns are expected rather than failures. Counter names are generated for any MAX_PAGE_ORDER the arch Kconfig ranges allow (10..13), a static_assert catches larger values. A per-order split of PGALLOC itself was proposed in 2017 but stalled over fast path overhead concerns, restricting the counters to the slow path avoids that overhead entirely while still capturing the allocations that cause latency. Link: https://lore.kernel.org/all/1499346271-15653-1-git-send-email-guro@fb.com/ Signed-off-by: Daniil Tatianin --- include/linux/vm_event_item.h | 19 +++++++++++++ mm/page_alloc.c | 7 +++++ mm/vmstat.c | 52 +++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+) diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h index 03fe95f5a020..724561b7e800 100644 --- a/include/linux/vm_event_item.h +++ b/include/linux/vm_event_item.h @@ -175,6 +175,25 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, KSTACK_REST, #endif #endif /* CONFIG_DEBUG_STACK_USAGE */ + /* + * Per-order allocation statistics: each *_FIRST..*_LAST range + * is indexed by allocation order. + */ + PGALLOC_SLOWPATH_ORDER_FIRST, + PGALLOC_SLOWPATH_ORDER_LAST = + PGALLOC_SLOWPATH_ORDER_FIRST + MAX_PAGE_ORDER, + PGALLOC_FAIL_ORDER_FIRST, + PGALLOC_FAIL_ORDER_LAST = + PGALLOC_FAIL_ORDER_FIRST + MAX_PAGE_ORDER, +#ifdef CONFIG_COMPACTION + /* Direct compaction is never entered for order 0 */ + COMPACTSTALL_ORDER_FIRST, + COMPACTSTALL_ORDER_LAST = + COMPACTSTALL_ORDER_FIRST + MAX_PAGE_ORDER - 1, + COMPACTSUCCESS_ORDER_FIRST, + COMPACTSUCCESS_ORDER_LAST = + COMPACTSUCCESS_ORDER_FIRST + MAX_PAGE_ORDER - 1, +#endif NR_VM_EVENT_ITEMS }; diff --git a/mm/page_alloc.c b/mm/page_alloc.c index ee902a468c2f..2a2b14f3b516 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -4169,6 +4169,7 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, * count a compaction stall */ count_vm_event(COMPACTSTALL); + count_vm_event(COMPACTSTALL_ORDER_FIRST + order - 1); /* Prep a captured page if available */ if (page) @@ -4184,6 +4185,7 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, zone->compact_blockskip_flush = false; compaction_defer_reset(zone, order, true); count_vm_event(COMPACTSUCCESS); + count_vm_event(COMPACTSUCCESS_ORDER_FIRST + order - 1); return page; } @@ -4757,6 +4759,8 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, WARN_ON_ONCE(current->flags & PF_MEMALLOC); } + count_vm_event(PGALLOC_SLOWPATH_ORDER_FIRST + order); + restart: compaction_retries = 0; no_progress_loops = 0; @@ -5323,6 +5327,9 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, page = NULL; } + if (unlikely(!page)) + count_vm_event(PGALLOC_FAIL_ORDER_FIRST + order); + trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype); kmsan_alloc_page(page, order, alloc_gfp); diff --git a/mm/vmstat.c b/mm/vmstat.c index f534972f517d..08596edc53ca 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1184,6 +1184,50 @@ int fragmentation_index(struct zone *zone, unsigned int order) [xx##_MOVABLE] = yy "_movable", \ TEXT_FOR_DEVICE(xx, yy) +#if MAX_PAGE_ORDER >= 11 +#define TEXT_FOR_ORDER_11(xx, yy) [I((xx) + 11)] = yy "11", +#else +#define TEXT_FOR_ORDER_11(xx, yy) +#endif + +#if MAX_PAGE_ORDER >= 12 +#define TEXT_FOR_ORDER_12(xx, yy) [I((xx) + 12)] = yy "12", +#else +#define TEXT_FOR_ORDER_12(xx, yy) +#endif + +#if MAX_PAGE_ORDER >= 13 +#define TEXT_FOR_ORDER_13(xx, yy) [I((xx) + 13)] = yy "13", +#else +#define TEXT_FOR_ORDER_13(xx, yy) +#endif + +static_assert(MAX_PAGE_ORDER <= 13, + "extend TEXT_FOR_ORDER_* for this MAX_PAGE_ORDER"); + +/* + * (xx) + n must resolve to the vm_event_item for order n, so ranges that + * start at order 1 pass their *_ORDER_FIRST item minus one. + */ +#define TEXTS_FOR_NONZERO_ORDERS(xx, yy) \ + [I((xx) + 1)] = yy "1", \ + [I((xx) + 2)] = yy "2", \ + [I((xx) + 3)] = yy "3", \ + [I((xx) + 4)] = yy "4", \ + [I((xx) + 5)] = yy "5", \ + [I((xx) + 6)] = yy "6", \ + [I((xx) + 7)] = yy "7", \ + [I((xx) + 8)] = yy "8", \ + [I((xx) + 9)] = yy "9", \ + [I((xx) + 10)] = yy "10", \ + TEXT_FOR_ORDER_11(xx, yy) \ + TEXT_FOR_ORDER_12(xx, yy) \ + TEXT_FOR_ORDER_13(xx, yy) + +#define TEXTS_FOR_ORDERS(xx, yy) \ + [I(xx)] = yy "0", \ + TEXTS_FOR_NONZERO_ORDERS(xx, yy) + const char * const vmstat_text[] = { /* enum zone_stat_item counters */ #define I(x) (x) @@ -1488,6 +1532,14 @@ const char * const vmstat_text[] = { #if THREAD_SIZE > 65536 [I(KSTACK_REST)] = "kstack_rest", #endif +#endif + TEXTS_FOR_ORDERS(PGALLOC_SLOWPATH_ORDER_FIRST, "pgalloc_slowpath_order") + TEXTS_FOR_ORDERS(PGALLOC_FAIL_ORDER_FIRST, "pgalloc_fail_order") +#ifdef CONFIG_COMPACTION + TEXTS_FOR_NONZERO_ORDERS(COMPACTSTALL_ORDER_FIRST - 1, + "compact_stall_order") + TEXTS_FOR_NONZERO_ORDERS(COMPACTSUCCESS_ORDER_FIRST - 1, + "compact_success_order") #endif #undef I #endif /* CONFIG_VM_EVENT_COUNTERS */