When automatic NUMA balancing runs in memory-tiering-only mode, NUMA hint faults identify slow-memory pages that are worth promoting. They are not normal task-placement feedback. Without separate tiering feedback, successful tiering promotions are fed to update_task_scan_period() as private remote faults. That leaves local == 0 and shared == 0, so the function hits its early slow-down path and doubles p->numa_scan_period. Repeated promotion feedback can therefore drive the scan period towards the maximum. Track memory-tiering feedback separately from normal task-placement stats. Successful free-space bypass promotions are not counted as tiering scan feedback because they did not pass the hotness and rate-limit checks. Cold, hot, and throttled feedback is generated only by the hotness/rate-limit path, while migration failures slow scanning like the normal placement heuristic does. Use the same 70% threshold style as the normal scan-period heuristic. A hot-dominated window speeds scanning up, a cold-dominated window slows it down, and a throttled ratio of at least 30% also slows it down. If a window has no tiering feedback, for example because promotions used the free-space bypass, double the scan period, following the same no-useful-signal slow-down direction as normal scan-period adjustment. Evaluate the tiering window when a scan sequence completes as well as when tiering faults are reported. This lets an empty window back off the scan period after qualifying tiering feedback stops. On a CXL system with node0 as DRAM and node3 as slow-tier memory, a STREAM Triad-derived OpenMP workload was used to model a bandwidth-sensitive working-set shift. It first accessed one 100G range, then switched to another 100G range. The workload was first-touch placed on node3 and then run from node0 with memory-tiering-only mode enabled. With the free-space bypass disabled, so that promotions have to pass the hotness and rate-limit checks, a kernel with only the bypass control was used as the no-feedback baseline. Tiering scan-period feedback improved the transition window after the active range switch: no feedback tiering feedback first 80s after switch 122.4 GB/s 129.3 GB/s (+5.7%) 200-240s after switch 139.9 GB/s 140.6 GB/s (+0.5%) Signed-off-by: Li Zhe --- include/linux/sched.h | 17 ++++ include/linux/sched/numa_balancing.h | 9 +++ kernel/sched/fair.c | 113 ++++++++++++++++++++++++++- 3 files changed, 137 insertions(+), 2 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index 1c93da75226e..c4959747374b 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -833,6 +833,16 @@ struct task_ipi_mask { struct task_ipi_mask { }; #endif +#ifdef CONFIG_NUMA_BALANCING +enum numa_tiering_fault_stats { + NUMA_TIERING_FAULT_COLD, + NUMA_TIERING_FAULT_HOT, + NUMA_TIERING_FAULT_THROTTLED, + NUMA_TIERING_FAULT_MIGRATE_FAIL, + NR_NUMA_TIERING_FAULT_STATS +}; +#endif + struct task_struct { #ifdef CONFIG_THREAD_INFO_IN_TASK /* @@ -1433,6 +1443,13 @@ struct task_struct { */ unsigned long numa_faults_locality[3]; + /* + * Memory tiering faults are tracked separately from normal NUMA + * placement statistics. In tiering-only mode, slow-memory hint faults + * are used to decide promotion, not task placement. + */ + unsigned long numa_tiering_faults[NR_NUMA_TIERING_FAULT_STATS]; + unsigned long numa_pages_migrated; #endif /* CONFIG_NUMA_BALANCING */ diff --git a/include/linux/sched/numa_balancing.h b/include/linux/sched/numa_balancing.h index 200f50075d5c..12f31ed56d66 100644 --- a/include/linux/sched/numa_balancing.h +++ b/include/linux/sched/numa_balancing.h @@ -8,6 +8,7 @@ */ #include +#include #define TNF_MIGRATED 0x01 #define TNF_NO_GROUP 0x02 @@ -16,6 +17,14 @@ #define TNF_MIGRATE_FAIL 0x10 #define TNF_ACCESS_TIME 0x20 +static inline bool numa_balancing_tiering_only(void) +{ + int mode = sysctl_numa_balancing_mode; + + return (mode & NUMA_BALANCING_MEMORY_TIERING) && + !(mode & NUMA_BALANCING_NORMAL); +} + enum numa_vmaskip_reason { NUMAB_SKIP_UNSUITABLE, NUMAB_SKIP_SHARED_RO, diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 136207b62d87..1d9efc2b7427 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -2760,6 +2760,32 @@ static inline bool cpupid_valid(int cpupid) return cpupid_to_cpu(cpupid) < nr_cpu_ids; } +static void task_numa_tiering_placement(struct task_struct *p); + +static bool task_numa_tiering_update(struct task_struct *p) +{ + if (!numa_balancing_tiering_only() || !p->mm) + return false; + + task_numa_tiering_placement(p); + return true; +} + +static void task_numa_tiering_no_feedback(struct task_struct *p) +{ + task_numa_tiering_update(p); +} + +static void task_numa_tiering_fault(struct task_struct *p, + enum numa_tiering_fault_stats stat, + unsigned long pages) +{ + if (!task_numa_tiering_update(p)) + return; + + p->numa_tiering_faults[stat] += pages; +} + /* * For memory tiering mode, if the target fast memory node has enough free * pages, recently accessed slow memory pages can bypass the normal hotness @@ -2887,6 +2913,7 @@ bool should_numa_migrate_memory(struct task_struct *p, struct folio *folio, pgdat = NODE_DATA(dst_nid); if (numa_balancing_promote_free_space_bypass && pgdat_free_space_enough(pgdat)) { + task_numa_tiering_no_feedback(p); /* workload changed, reset hot threshold */ pgdat->nbp_threshold = 0; mod_node_page_state(pgdat, PGPROMOTE_CANDIDATE_NRL, nr); @@ -2899,10 +2926,18 @@ bool should_numa_migrate_memory(struct task_struct *p, struct folio *folio, th = pgdat->nbp_threshold ? : def_th; latency = numa_hint_fault_latency(folio); - if (latency >= th) + if (latency >= th) { + task_numa_tiering_fault(p, NUMA_TIERING_FAULT_COLD, nr); + return false; + } + + if (numa_promotion_rate_limit(pgdat, rate_limit, nr)) { + task_numa_tiering_fault(p, NUMA_TIERING_FAULT_THROTTLED, nr); return false; + } - return !numa_promotion_rate_limit(pgdat, rate_limit, nr); + task_numa_tiering_fault(p, NUMA_TIERING_FAULT_HOT, nr); + return true; } this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid); @@ -3708,6 +3743,48 @@ static void update_task_scan_period(struct task_struct *p, sizeof(p->numa_faults_locality)); } +/* + * Memory tiering uses NUMA hint faults to identify recently accessed slow + * memory that should be promoted. In tiering-only mode, those faults are not + * normal task-placement feedback, so use tiering-specific outcomes to adjust + * the scan period. + */ +static void update_task_tiering_scan_period(struct task_struct *p) +{ + unsigned long cold, hot, throttled, failed; + unsigned long total; + s64 period, period_slot; + s64 diff = 0; + + cold = p->numa_tiering_faults[NUMA_TIERING_FAULT_COLD]; + hot = p->numa_tiering_faults[NUMA_TIERING_FAULT_HOT]; + throttled = p->numa_tiering_faults[NUMA_TIERING_FAULT_THROTTLED]; + failed = p->numa_tiering_faults[NUMA_TIERING_FAULT_MIGRATE_FAIL]; + total = cold + hot + throttled + failed; + + period = p->numa_scan_period; + period_slot = DIV_ROUND_UP_ULL(period, NUMA_PERIOD_SLOTS); + + if (!total) { + diff = period; + } else if (failed) { + diff = period_slot; + } else if (throttled * NUMA_PERIOD_SLOTS >= + total * (NUMA_PERIOD_SLOTS - NUMA_PERIOD_THRESHOLD)) { + diff = period_slot; + } else if (cold * NUMA_PERIOD_SLOTS >= + total * NUMA_PERIOD_THRESHOLD) { + diff = period_slot; + } else if (hot * NUMA_PERIOD_SLOTS >= + total * NUMA_PERIOD_THRESHOLD) { + diff = -period_slot; + } + + p->numa_scan_period = clamp_t(s64, period + diff, + task_scan_min(p), task_scan_max(p)); + memset(p->numa_tiering_faults, 0, sizeof(p->numa_tiering_faults)); +} + /* * Get the fraction of time the task has been running since the last * NUMA placement cycle. The scheduler keeps similar statistics, but @@ -3974,6 +4051,22 @@ static void task_numa_placement(struct task_struct *p) update_task_scan_period(p, fault_types[0], fault_types[1]); } +static void task_numa_tiering_placement(struct task_struct *p) +{ + int seq; + + if (!p->mm) + return; + + seq = READ_ONCE(p->mm->numa_scan_seq); + if (p->numa_scan_seq == seq) + return; + p->numa_scan_seq = seq; + p->numa_scan_period_max = task_scan_max(p); + + update_task_tiering_scan_period(p); +} + static inline int get_numa_group(struct numa_group *grp) { return refcount_inc_not_zero(&grp->refcount); @@ -4105,6 +4198,8 @@ void task_numa_free(struct task_struct *p, bool final) unsigned long flags; int i; + memset(p->numa_tiering_faults, 0, sizeof(p->numa_tiering_faults)); + if (!numa_faults) return; @@ -4158,6 +4253,11 @@ void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags) if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL)) { if (migrated) p->numa_pages_migrated += pages; + if (numa_balancing_tiering_only() && + (flags & TNF_MIGRATE_FAIL)) + task_numa_tiering_fault(p, + NUMA_TIERING_FAULT_MIGRATE_FAIL, + pages); return; } } @@ -4529,6 +4629,14 @@ static void task_numa_work(struct callback_head *work) reset_ptenuma_scan(p); mmap_read_unlock(mm); + /* + * Tiering-only placement feedback is still scan-window based. Consume + * the window when a scan sequence completes, so an empty tiering + * window can back off after qualifying tiering faults stop. + */ + if (numa_balancing_tiering_only()) + task_numa_tiering_placement(p); + /* * Make sure tasks use at least 32x as much time to run other code * than they used here, to limit NUMA PTE scanning overhead to 3% max. @@ -4562,6 +4670,7 @@ void init_numa_balancing(u64 clone_flags, struct task_struct *p) p->numa_faults = NULL; p->numa_pages_migrated = 0; p->total_numa_faults = 0; + memset(p->numa_tiering_faults, 0, sizeof(p->numa_tiering_faults)); RCU_INIT_POINTER(p->numa_group, NULL); p->last_task_numa_placement = 0; p->last_sum_exec_runtime = 0; -- 2.20.1