From: Wang Lian DAMOS_COLLAPSE currently collapses into PMD-size THP only. Add a target_order field to express per-order mTHP collapse intent. Zero means system default (PMD order, same as current behavior). Valid values are 0 and 2..HPAGE_PMD_ORDER. Wire up the sysfs interface: a per-scheme rw file "target_order". Validate at store time that the value is in range, and warn at scheme creation time if DAMOS_COLLAPSE is used with an unsupported non-PMD order, resetting to 0. The actual mTHP application via the khugepaged wrapper will be added in subsequent patches. Co-developed-by: Kunwu Chan Signed-off-by: Kunwu Chan Signed-off-by: Wang Lian Signed-off-by: Wang Lian --- include/linux/damon.h | 5 +++++ mm/damon/sysfs-schemes.c | 45 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/include/linux/damon.h b/include/linux/damon.h index 6f7edb3590ef..5a0587556573 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -572,6 +572,11 @@ struct damos_migrate_dests { struct damos { struct damos_access_pattern pattern; enum damos_action action; + /* + * @target_order: target order for mTHP actions (DAMOS_COLLAPSE). + * 0 means system default (PMD order). Valid: 0, 2..HPAGE_PMD_ORDER. + */ + unsigned int target_order; unsigned long apply_interval_us; /* private: internal use only */ /* diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 329cfd0bbe9f..735970717048 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -6,7 +6,9 @@ */ #include +#include #include +#include #include "sysfs-common.h" @@ -2257,6 +2259,7 @@ struct damon_sysfs_scheme { struct damon_sysfs_stats *stats; struct damon_sysfs_scheme_regions *tried_regions; int target_nid; + unsigned int target_order; struct damos_sysfs_dests *dests; }; @@ -2642,6 +2645,34 @@ static ssize_t target_nid_store(struct kobject *kobj, return err ? err : count; } +static ssize_t target_order_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct damon_sysfs_scheme *scheme = container_of(kobj, + struct damon_sysfs_scheme, kobj); + + return sysfs_emit(buf, "%u\n", scheme->target_order); +} + +static ssize_t target_order_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + struct damon_sysfs_scheme *scheme = container_of(kobj, + struct damon_sysfs_scheme, kobj); + unsigned int val; + int err; + + err = kstrtouint(buf, 0, &val); + if (err) + return err; + + if (val != 0 && (val < 2 || val > HPAGE_PMD_ORDER)) + return -EINVAL; + + scheme->target_order = val; + return count; +} + static void damon_sysfs_scheme_release(struct kobject *kobj) { kfree(container_of(kobj, struct damon_sysfs_scheme, kobj)); @@ -2656,10 +2687,14 @@ static struct kobj_attribute damon_sysfs_scheme_apply_interval_us_attr = static struct kobj_attribute damon_sysfs_scheme_target_nid_attr = __ATTR_RW_MODE(target_nid, 0600); +static struct kobj_attribute damon_sysfs_scheme_target_order_attr = + __ATTR_RW_MODE(target_order, 0600); + static struct attribute *damon_sysfs_scheme_attrs[] = { &damon_sysfs_scheme_action_attr.attr, &damon_sysfs_scheme_apply_interval_us_attr.attr, &damon_sysfs_scheme_target_nid_attr.attr, + &damon_sysfs_scheme_target_order_attr.attr, NULL, }; ATTRIBUTE_GROUPS(damon_sysfs_scheme); @@ -3005,6 +3040,16 @@ static struct damos *damon_sysfs_mk_scheme( if (!scheme) return NULL; + if (sysfs_scheme->action == DAMOS_COLLAPSE && + sysfs_scheme->target_order != 0 && + sysfs_scheme->target_order != HPAGE_PMD_ORDER) { + pr_warn("DAMON collapse: target_order %u not supported, only PMD order (%u) is available. Use 0 or %u.\n", + sysfs_scheme->target_order, + HPAGE_PMD_ORDER, HPAGE_PMD_ORDER); + sysfs_scheme->target_order = 0; + } + scheme->target_order = sysfs_scheme->target_order; + err = damos_sysfs_add_quota_score(sysfs_quotas->goals, &scheme->quota); if (err) { damon_destroy_scheme(scheme); -- 2.50.1 (Apple Git-155) From: Wang Lian Export a thin wrapper around collapse_huge_page() that allows external subsystems such as DAMON to trigger THP collapse on a target address range. Currently restricted to PMD order (HPAGE_PMD_ORDER), since collapse_huge_page() does not yet support arbitrary mTHP orders. The restriction can be relaxed when khugepaged gains mTHP support. The caller must hold a reference to @mm. Do not hold mmap lock: collapse_huge_page() acquires mmap_read_lock for validation, releases it, then acquires mmap_write_lock for the actual collapse. Holding an outer mmap_read_lock would cause a self-deadlock when the same thread attempts the inner mmap_write_lock. Co-developed-by: Kunwu Chan Signed-off-by: Kunwu Chan Signed-off-by: Wang Lian Signed-off-by: Wang Lian --- include/linux/khugepaged.h | 3 +++ mm/khugepaged.c | 39 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/include/linux/khugepaged.h b/include/linux/khugepaged.h index d7a9053ff4fe..6fb8a6857790 100644 --- a/include/linux/khugepaged.h +++ b/include/linux/khugepaged.h @@ -20,6 +20,9 @@ extern bool current_is_khugepaged(void); void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr, bool install_pmd); +int damon_collapse_folio_range(struct mm_struct *mm, unsigned long start_addr, + unsigned int target_order); + static inline void khugepaged_fork(struct mm_struct *mm, struct mm_struct *oldmm) { if (mm_flags_test(MMF_VM_HUGEPAGE, oldmm)) diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 617bca76db49..0387841ba2e7 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -3272,3 +3272,42 @@ int madvise_collapse(struct vm_area_struct *vma, unsigned long start, return thps == ((hend - hstart) >> HPAGE_PMD_SHIFT) ? 0 : madvise_collapse_errno(last_fail); } + +/** + * damon_collapse_folio_range() - Collapse base pages in range into a THP + * @mm: mm_struct of the target process + * @start_addr: start address (must be order-aligned) + * @target_order: page order of the collapse result (currently only + * HPAGE_PMD_ORDER is supported) + * + * Thin wrapper around collapse_huge_page() for external callers such as + * DAMON. The caller must hold a reference to @mm. Do not hold mmap + * lock: collapse_huge_page() acquires mmap_read_lock for validation, + * releases it, then acquires mmap_write_lock for the collapse. Holding + * an outer mmap_read_lock would self-deadlock. + * + * Return: 0 on success, -EINVAL on bad arguments, negative error from + * madvise_collapse_errno() otherwise. + */ +int damon_collapse_folio_range(struct mm_struct *mm, unsigned long start_addr, + unsigned int target_order) +{ + struct collapse_control cc = { + .is_khugepaged = false, + }; + enum scan_result result; + + if (target_order != HPAGE_PMD_ORDER) { + pr_warn_once("%s: only PMD order (%u) is supported, got %u\n", + __func__, HPAGE_PMD_ORDER, target_order); + return -EINVAL; + } + if (start_addr & ((PAGE_SIZE << target_order) - 1)) + return -EINVAL; + + result = collapse_huge_page(mm, start_addr, 1, 0, &cc, target_order); + if (result == SCAN_SUCCEED) + return 0; + return madvise_collapse_errno(result); +} +EXPORT_SYMBOL_GPL(damon_collapse_folio_range); -- 2.50.1 (Apple Git-155) From: Wang Lian When target_order is set (non-zero), the DAMOS_COLLAPSE handler now calls damon_collapse_folio_range() to collapse pages into the requested mTHP size, iterating over the target region in order-aligned chunks. When target_order is 0 (default), the existing madvise(MADV_COLLAPSE) path is used, preserving backwards compatibility. Region boundaries are expanded outward to the covering aligned range (ALIGN_DOWN start, ALIGN end) so that collapse works even after kdamond_split_regions reduces region sizes below the chunk size. collapse_huge_page() internally validates VMA bounds, so expanding beyond the original region is safe. No external mmap lock is held: collapse_huge_page() acquires mmap_read_lock internally for validation, releases it, then acquires mmap_write_lock for the actual collapse. Holding an outer mmap_read_lock would cause a self-deadlock when the same thread attempts the inner mmap_write_lock. Co-developed-by: Kunwu Chan Signed-off-by: Kunwu Chan Signed-off-by: Wang Lian Signed-off-by: Wang Lian --- mm/damon/vaddr.c | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index d27147603564..2a3757c13bf0 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -14,6 +14,7 @@ #include #include #include +#include #include "../internal.h" #include "ops-common.h" @@ -899,6 +900,40 @@ static unsigned long damos_va_stat(struct damon_target *target, return 0; } +static unsigned long damos_va_collapse(struct damon_target *target, + struct damon_region *r, struct damos *s, + unsigned long *sz_filter_passed) +{ + unsigned long addr, end, chunk_sz; + unsigned int target_order = s->target_order; + unsigned long applied = 0; + struct mm_struct *mm; + int ret; + + if (target_order < 2 || target_order > HPAGE_PMD_ORDER) + return 0; + + chunk_sz = PAGE_SIZE << target_order; + addr = ALIGN_DOWN(r->ar.start, chunk_sz); + end = ALIGN(r->ar.end, chunk_sz); + + mm = damon_get_mm(target); + if (!mm) + return 0; + + while (addr < end) { + ret = damon_collapse_folio_range(mm, addr, target_order); + if (!ret) + applied += chunk_sz; + *sz_filter_passed += chunk_sz; + addr += chunk_sz; + cond_resched(); + } + + mmput(mm); + return applied; +} + static unsigned long damon_va_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, struct damon_region *r, struct damos *scheme, unsigned long *sz_filter_passed) @@ -922,6 +957,9 @@ static unsigned long damon_va_apply_scheme(struct damon_ctx *ctx, madv_action = MADV_NOHUGEPAGE; break; case DAMOS_COLLAPSE: + if (scheme->target_order) + return damos_va_collapse(t, r, scheme, + sz_filter_passed); madv_action = MADV_COLLAPSE; break; case DAMOS_MIGRATE_HOT: -- 2.50.1 (Apple Git-155) From: Wang Lian Add DAMOS_SPLIT to the damos_action enum for splitting large folios into smaller mTHP-order folios. Add a target_order field to struct damos to specify the desired split order. Expose the action as "split" through the DAMON sysfs interface with target_order validation (must be 2..HPAGE_PMD_ORDER-1). Signed-off-by: Wang Lian Signed-off-by: Wang Lian --- include/linux/damon.h | 9 +++++++-- mm/damon/sysfs-schemes.c | 12 ++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 5a0587556573..30cf4afb212c 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -121,6 +121,7 @@ struct damon_target { * @DAMOS_HUGEPAGE: Call ``madvise()`` for the region with MADV_HUGEPAGE. * @DAMOS_NOHUGEPAGE: Call ``madvise()`` for the region with MADV_NOHUGEPAGE. * @DAMOS_COLLAPSE: Call ``madvise()`` for the region with MADV_COLLAPSE. + * @DAMOS_SPLIT: Split large folios to the target mTHP order. * @DAMOS_LRU_PRIO: Prioritize the region on its LRU lists. * @DAMOS_LRU_DEPRIO: Deprioritize the region on its LRU lists. * @DAMOS_MIGRATE_HOT: Migrate the regions prioritizing warmer regions. @@ -141,6 +142,7 @@ enum damos_action { DAMOS_HUGEPAGE, DAMOS_NOHUGEPAGE, DAMOS_COLLAPSE, + DAMOS_SPLIT, DAMOS_LRU_PRIO, DAMOS_LRU_DEPRIO, DAMOS_MIGRATE_HOT, @@ -573,8 +575,11 @@ struct damos { struct damos_access_pattern pattern; enum damos_action action; /* - * @target_order: target order for mTHP actions (DAMOS_COLLAPSE). - * 0 means system default (PMD order). Valid: 0, 2..HPAGE_PMD_ORDER. + * @target_order: target mTHP order for DAMOS_COLLAPSE and + * DAMOS_SPLIT. For COLLAPSE, 0 means PMD order default, + * valid values: 0, 2..HPAGE_PMD_ORDER. For SPLIT, + * valid values: 2..HPAGE_PMD_ORDER-1; 0 and HPAGE_PMD_ORDER + * are rejected at scheme creation time (defaulting to 2). */ unsigned int target_order; unsigned long apply_interval_us; diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 735970717048..547252fc8a20 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -2293,6 +2293,10 @@ static struct damos_sysfs_action_name damos_sysfs_action_names[] = { .action = DAMOS_COLLAPSE, .name = "collapse", }, + { + .action = DAMOS_SPLIT, + .name = "split", + }, { .action = DAMOS_LRU_PRIO, .name = "lru_prio", @@ -3048,6 +3052,14 @@ static struct damos *damon_sysfs_mk_scheme( HPAGE_PMD_ORDER, HPAGE_PMD_ORDER); sysfs_scheme->target_order = 0; } + if (sysfs_scheme->action == DAMOS_SPLIT && + (sysfs_scheme->target_order == 0 || + sysfs_scheme->target_order >= HPAGE_PMD_ORDER)) { + pr_warn("DAMON split: target_order %u invalid, need 2..%u. Defaulting to 2.\n", + sysfs_scheme->target_order, + HPAGE_PMD_ORDER - 1); + sysfs_scheme->target_order = 2; + } scheme->target_order = sysfs_scheme->target_order; err = damos_sysfs_add_quota_score(sysfs_quotas->goals, &scheme->quota); -- 2.50.1 (Apple Git-155) From: Wang Lian Implement the vaddr operations layer handler for DAMOS_SPLIT. For each folio in the target region that is larger than the scheme's target_order, split it via split_folio_to_order(). This supports both anonymous and file-backed (e.g. tmpfs/shmem) folios, covering KVM guest memory backed by THP tmpfs. Signed-off-by: Wang Lian Signed-off-by: Wang Lian --- mm/damon/vaddr.c | 68 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 2a3757c13bf0..3f2061b29ed8 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -934,6 +934,71 @@ static unsigned long damos_va_collapse(struct damon_target *target, return applied; } +static unsigned long damos_va_split(struct damon_target *target, + struct damon_region *r, struct damos *s, + unsigned long *sz_filter_passed) +{ + unsigned long addr, end, chunk_sz; + unsigned int target_order = s->target_order; + unsigned long applied = 0; + struct mm_struct *mm; + struct vm_area_struct *vma; + struct folio *folio; + struct folio_walk fw; + + mm = damon_get_mm(target); + if (!mm) + return 0; + + chunk_sz = PAGE_SIZE << HPAGE_PMD_ORDER; + addr = ALIGN_DOWN(r->ar.start, chunk_sz); + end = ALIGN(r->ar.end, chunk_sz); + + while (addr < end) { + mmap_read_lock(mm); + vma = find_vma(mm, addr); + /* + * split_folio_to_order() supports both anon and shmem + * folios, so we accept any VMA that has a folio at @addr. + * This covers important use cases like tmpfs THP-backed + * KVM guest memory where cold and hot pages are bundled + * together in a single PMD THP. + */ + if (!vma || addr < vma->vm_start) + goto unlock; + + folio = folio_walk_start(&fw, vma, addr, 0); + if (!folio) + goto unlock; + + if (folio_order(folio) > target_order) { + if (!folio_trylock(folio)) { + folio_walk_end(&fw, vma); + goto unlock; + } + folio_get(folio); + folio_walk_end(&fw, vma); + + if (!split_folio_to_order(folio, target_order)) + applied += chunk_sz; + + folio_unlock(folio); + folio_put(folio); + } else { + folio_walk_end(&fw, vma); + } + +unlock: + *sz_filter_passed += chunk_sz; + addr += chunk_sz; + mmap_read_unlock(mm); + cond_resched(); + } + + mmput(mm); + return applied; +} + static unsigned long damon_va_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, struct damon_region *r, struct damos *scheme, unsigned long *sz_filter_passed) @@ -967,6 +1032,9 @@ static unsigned long damon_va_apply_scheme(struct damon_ctx *ctx, return damos_va_migrate(t, r, scheme, sz_filter_passed); case DAMOS_STAT: return damos_va_stat(t, r, scheme, sz_filter_passed); + case DAMOS_SPLIT: + return damos_va_split(t, r, scheme, + sz_filter_passed); default: /* * DAMOS actions that are not yet supported by 'vaddr'. -- 2.50.1 (Apple Git-155)