Add a new DAMOS_MTHP_SPLIT action to split a large folio to the specified target_order, and a hot_threshold parameter to control split decisions based on sub-page access heatmap. target_order: For MTHP_SPLIT, valid range is 2..HPAGE_PMD_ORDER-1, allowing splits to e.g. order-2 (16KB) or order-3 (32KB) mTHP. An invalid value (0 or >= PMD_ORDER) defaults to order-2; 0 would mean "split to base page" which defeats the purpose of mTHP split. hot_threshold: Minimum percentage (0-100) of hot subpages required to preserve a THP. THPs with hot_fraction >= hot_threshold are kept intact; below it, the THP is split to target_order. Default is 30%, based on ARM SPE profiling on Kunpeng 920 which showed: - 97% of THPs have <10% hot subpages (clearly cold, split) - 1-2% have 10-30% (borderline, tunable) - <1% have >30% (genuinely hot, preserve) The 30% default catches genuinely hot THPs while splitting the vast majority of cold THPs. Exposed as sysfs attribute for per-scheme tuning (e.g. lower for memory-pressure scenarios, higher for latency-sensitive workloads). sysfs interface: /sys/kernel/mm/damon/admin/kdamonds/.../schemes/0/hot_threshold The actual split implementation follows in subsequent patches. Co-developed-by: Kunwu Chan Signed-off-by: Kunwu Chan Signed-off-by: Wang Lian --- include/linux/damon.h | 17 ++++++++++++-- mm/damon/sysfs-schemes.c | 51 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 5a0587556573..982057bbce3b 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -121,6 +121,7 @@ struct damon_target { * @DAMOS_HUGEPAGE: Call ``madvise()`` for the region with MADV_HUGEPAGE. * @DAMOS_NOHUGEPAGE: Call ``madvise()`` for the region with MADV_NOHUGEPAGE. * @DAMOS_COLLAPSE: Call ``madvise()`` for the region with MADV_COLLAPSE. + * @DAMOS_MTHP_SPLIT: Split large folios to the target mTHP order. * @DAMOS_LRU_PRIO: Prioritize the region on its LRU lists. * @DAMOS_LRU_DEPRIO: Deprioritize the region on its LRU lists. * @DAMOS_MIGRATE_HOT: Migrate the regions prioritizing warmer regions. @@ -141,6 +142,7 @@ enum damos_action { DAMOS_HUGEPAGE, DAMOS_NOHUGEPAGE, DAMOS_COLLAPSE, + DAMOS_MTHP_SPLIT, DAMOS_LRU_PRIO, DAMOS_LRU_DEPRIO, DAMOS_MIGRATE_HOT, @@ -573,10 +575,21 @@ struct damos { struct damos_access_pattern pattern; enum damos_action action; /* - * @target_order: target order for mTHP actions (DAMOS_COLLAPSE). - * 0 means system default (PMD order). Valid: 0, 2..HPAGE_PMD_ORDER. + * @target_order: target mTHP order for DAMOS_COLLAPSE and + * DAMOS_MTHP_SPLIT. For COLLAPSE, 0 means PMD order default, + * valid values: 0, 2..HPAGE_PMD_ORDER. For MTHP_SPLIT, + * valid values: 2..HPAGE_PMD_ORDER-1; 0 and HPAGE_PMD_ORDER + * are rejected at scheme creation time (defaulting to 2). */ unsigned int target_order; + /* + * @hot_threshold: minimum hot subpage percentage (0-100) to + * preserve a THP during DAMOS_MTHP_SPLIT. A THP with + * hot_fraction >= hot_threshold is kept intact; below it, the + * THP is split to @target_order. Default 30 based on SPE + * profiling showing 97% of THPs have <10% hot subpages. + */ + unsigned int hot_threshold; unsigned long apply_interval_us; /* private: internal use only */ /* diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 735970717048..823f1ca9bd90 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -2260,6 +2260,7 @@ struct damon_sysfs_scheme { struct damon_sysfs_scheme_regions *tried_regions; int target_nid; unsigned int target_order; + unsigned int hot_threshold; struct damos_sysfs_dests *dests; }; @@ -2293,6 +2294,10 @@ static struct damos_sysfs_action_name damos_sysfs_action_names[] = { .action = DAMOS_COLLAPSE, .name = "collapse", }, + { + .action = DAMOS_MTHP_SPLIT, + .name = "mthp_split", + }, { .action = DAMOS_LRU_PRIO, .name = "lru_prio", @@ -2673,6 +2678,34 @@ static ssize_t target_order_store(struct kobject *kobj, return count; } +static ssize_t hot_threshold_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct damon_sysfs_scheme *scheme = container_of(kobj, + struct damon_sysfs_scheme, kobj); + + return sysfs_emit(buf, "%u\n", scheme->hot_threshold); +} + +static ssize_t hot_threshold_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + struct damon_sysfs_scheme *scheme = container_of(kobj, + struct damon_sysfs_scheme, kobj); + unsigned int val; + int err; + + err = kstrtouint(buf, 0, &val); + if (err) + return err; + + if (val > 100) + return -EINVAL; + + scheme->hot_threshold = val; + return count; +} + static void damon_sysfs_scheme_release(struct kobject *kobj) { kfree(container_of(kobj, struct damon_sysfs_scheme, kobj)); @@ -2690,11 +2723,15 @@ static struct kobj_attribute damon_sysfs_scheme_target_nid_attr = static struct kobj_attribute damon_sysfs_scheme_target_order_attr = __ATTR_RW_MODE(target_order, 0600); +static struct kobj_attribute damon_sysfs_scheme_hot_threshold_attr = + __ATTR_RW_MODE(hot_threshold, 0600); + static struct attribute *damon_sysfs_scheme_attrs[] = { &damon_sysfs_scheme_action_attr.attr, &damon_sysfs_scheme_apply_interval_us_attr.attr, &damon_sysfs_scheme_target_nid_attr.attr, &damon_sysfs_scheme_target_order_attr.attr, + &damon_sysfs_scheme_hot_threshold_attr.attr, NULL, }; ATTRIBUTE_GROUPS(damon_sysfs_scheme); @@ -3048,8 +3085,22 @@ static struct damos *damon_sysfs_mk_scheme( HPAGE_PMD_ORDER, HPAGE_PMD_ORDER); sysfs_scheme->target_order = 0; } + if (sysfs_scheme->action == DAMOS_MTHP_SPLIT && + (sysfs_scheme->target_order == 0 || + sysfs_scheme->target_order >= HPAGE_PMD_ORDER)) { + pr_warn("DAMON mthp_split: target_order %u invalid, need 2..%u. Defaulting to 2.\n", + sysfs_scheme->target_order, + HPAGE_PMD_ORDER - 1); + sysfs_scheme->target_order = 2; + } scheme->target_order = sysfs_scheme->target_order; + if (sysfs_scheme->action == DAMOS_MTHP_SPLIT) { + if (sysfs_scheme->hot_threshold == 0) + sysfs_scheme->hot_threshold = 30; + scheme->hot_threshold = sysfs_scheme->hot_threshold; + } + err = damos_sysfs_add_quota_score(sysfs_quotas->goals, &scheme->quota); if (err) { damon_destroy_scheme(scheme); -- 2.50.1 (Apple Git-155)