From: SeongJae Park In some cases, some of the real address that handled by the underlying operations set cannot be handled by DAMON since it uses only 'unsinged long' as the address type. Using DAMON for physical address space monitoring of 32 bit ARM devices with large physical address extension (LPAE) is one example[1]. Add a parameter name 'addr_unit' to core layer to help such cases. DAMON core API callers can set it as the scale factor that will be used by the operations set for translating the core layer's addresses to the real address by multiplying the parameter value to the core layer address. Support of the parameter is up to each operations set layer. The support from the physical address space operations set (paddr) will be added with following commits. [1] https://lore.kernel.org/20250408075553.959388-1-zuoze1@huawei.com Signed-off-by: SeongJae Park --- include/linux/damon.h | 3 ++- mm/damon/core.c | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index f13664c62ddd..b85c6c669cd0 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -746,7 +746,7 @@ struct damon_attrs { * Accesses to other fields must be protected by themselves. * * @ops: Set of monitoring operations for given use cases. - * + * @addr_unit: Scale factor for core to ops address conversion. * @adaptive_targets: Head of monitoring targets (&damon_target) list. * @schemes: Head of schemes (&damos) list. */ @@ -788,6 +788,7 @@ struct damon_ctx { struct mutex kdamond_lock; struct damon_operations ops; + unsigned long addr_unit; struct list_head adaptive_targets; struct list_head schemes; diff --git a/mm/damon/core.c b/mm/damon/core.c index 52a48c9316bc..1a8d3009d606 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -544,6 +544,8 @@ struct damon_ctx *damon_new_ctx(void) ctx->attrs.min_nr_regions = 10; ctx->attrs.max_nr_regions = 1000; + ctx->addr_unit = 1; + INIT_LIST_HEAD(&ctx->adaptive_targets); INIT_LIST_HEAD(&ctx->schemes); @@ -1213,6 +1215,7 @@ int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src) if (err) return err; dst->ops = src->ops; + dst->addr_unit = src->addr_unit ? : 1; return 0; } -- 2.34.1 From: SeongJae Park Add support of addr_unit paramer for access monitoing operations of paddr. Signed-off-by: SeongJae Park --- mm/damon/paddr.c | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 53a55c5114fb..345a0ae63b19 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -18,7 +18,13 @@ #include "../internal.h" #include "ops-common.h" -static void damon_pa_mkold(unsigned long paddr) +static phys_addr_t damon_pa_phys_addr( + unsigned long addr, unsigned long addr_unit) +{ + return (phys_addr_t)addr * addr_unit; +} + +static void damon_pa_mkold(phys_addr_t paddr) { struct folio *folio = damon_get_folio(PHYS_PFN(paddr)); @@ -29,11 +35,12 @@ static void damon_pa_mkold(unsigned long paddr) folio_put(folio); } -static void __damon_pa_prepare_access_check(struct damon_region *r) +static void __damon_pa_prepare_access_check(struct damon_region *r, + unsigned long addr_unit) { r->sampling_addr = damon_rand(r->ar.start, r->ar.end); - damon_pa_mkold(r->sampling_addr); + damon_pa_mkold(damon_pa_phys_addr(r->sampling_addr, addr_unit)); } static void damon_pa_prepare_access_checks(struct damon_ctx *ctx) @@ -43,11 +50,11 @@ static void damon_pa_prepare_access_checks(struct damon_ctx *ctx) damon_for_each_target(t, ctx) { damon_for_each_region(r, t) - __damon_pa_prepare_access_check(r); + __damon_pa_prepare_access_check(r, ctx->addr_unit); } } -static bool damon_pa_young(unsigned long paddr, unsigned long *folio_sz) +static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz) { struct folio *folio = damon_get_folio(PHYS_PFN(paddr)); bool accessed; @@ -62,23 +69,25 @@ static bool damon_pa_young(unsigned long paddr, unsigned long *folio_sz) } static void __damon_pa_check_access(struct damon_region *r, - struct damon_attrs *attrs) + struct damon_attrs *attrs, unsigned long addr_unit) { - static unsigned long last_addr; + static phys_addr_t last_addr; static unsigned long last_folio_sz = PAGE_SIZE; static bool last_accessed; + phys_addr_t sampling_addr = damon_pa_phys_addr( + r->sampling_addr, addr_unit); /* If the region is in the last checked page, reuse the result */ if (ALIGN_DOWN(last_addr, last_folio_sz) == - ALIGN_DOWN(r->sampling_addr, last_folio_sz)) { + ALIGN_DOWN(sampling_addr, last_folio_sz)) { damon_update_region_access_rate(r, last_accessed, attrs); return; } - last_accessed = damon_pa_young(r->sampling_addr, &last_folio_sz); + last_accessed = damon_pa_young(sampling_addr, &last_folio_sz); damon_update_region_access_rate(r, last_accessed, attrs); - last_addr = r->sampling_addr; + last_addr = sampling_addr; } static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx) @@ -89,7 +98,8 @@ static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx) damon_for_each_target(t, ctx) { damon_for_each_region(r, t) { - __damon_pa_check_access(r, &ctx->attrs); + __damon_pa_check_access( + r, &ctx->attrs, ctx->addr_unit); max_nr_accesses = max(r->nr_accesses, max_nr_accesses); } } -- 2.34.1 From: SeongJae Park Add support of addr_unit for DAMOS_PAGEOUT action handling from the DAMOS operation implementation for the physical address space. Signed-off-by: SeongJae Park --- mm/damon/paddr.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 345a0ae63b19..b548813a0472 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -135,10 +135,12 @@ static bool damon_pa_invalid_damos_folio(struct folio *folio, struct damos *s) return false; } -static unsigned long damon_pa_pageout(struct damon_region *r, struct damos *s, +static unsigned long damon_pa_pageout(struct damon_region *r, + unsigned long addr_unit, struct damos *s, unsigned long *sz_filter_passed) { - unsigned long addr, applied; + phys_addr_t addr; + unsigned long applied; LIST_HEAD(folio_list); bool install_young_filter = true; struct damos_filter *filter; @@ -159,8 +161,8 @@ static unsigned long damon_pa_pageout(struct damon_region *r, struct damos *s, damos_add_filter(s, filter); } - addr = r->ar.start; - while (addr < r->ar.end) { + addr = damon_pa_phys_addr(r->ar.start, addr_unit); + while (addr < damon_pa_phys_addr(r->ar.end, addr_unit)) { folio = damon_get_folio(PHYS_PFN(addr)); if (damon_pa_invalid_damos_folio(folio, s)) { addr += PAGE_SIZE; @@ -311,9 +313,11 @@ static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, struct damon_region *r, struct damos *scheme, unsigned long *sz_filter_passed) { + unsigned long aunit = ctx->addr_unit; + switch (scheme->action) { case DAMOS_PAGEOUT: - return damon_pa_pageout(r, scheme, sz_filter_passed); + return damon_pa_pageout(r, aunit, scheme, sz_filter_passed); case DAMOS_LRU_PRIO: return damon_pa_mark_accessed(r, scheme, sz_filter_passed); case DAMOS_LRU_DEPRIO: -- 2.34.1 From: SeongJae Park Add support of addr_unit for DAMOS_LRU_PRIO and DAMOS_LRU_DEPRIO action handling from the DAMOS operation implementation for the physical address space. Signed-off-by: SeongJae Park --- mm/damon/paddr.c | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index b548813a0472..b8a7f462967b 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -195,14 +195,16 @@ static unsigned long damon_pa_pageout(struct damon_region *r, } static inline unsigned long damon_pa_mark_accessed_or_deactivate( - struct damon_region *r, struct damos *s, bool mark_accessed, + struct damon_region *r, unsigned long addr_unit, + struct damos *s, bool mark_accessed, unsigned long *sz_filter_passed) { - unsigned long addr, applied = 0; + phys_addr_t addr; + unsigned long applied = 0; struct folio *folio; - addr = r->ar.start; - while (addr < r->ar.end) { + addr = damon_pa_phys_addr(r->ar.start, addr_unit); + while (addr < damon_pa_phys_addr(r->ar.end, addr_unit)) { folio = damon_get_folio(PHYS_PFN(addr)); if (damon_pa_invalid_damos_folio(folio, s)) { addr += PAGE_SIZE; @@ -228,16 +230,18 @@ static inline unsigned long damon_pa_mark_accessed_or_deactivate( } static unsigned long damon_pa_mark_accessed(struct damon_region *r, - struct damos *s, unsigned long *sz_filter_passed) + unsigned long addr_unit, struct damos *s, + unsigned long *sz_filter_passed) { - return damon_pa_mark_accessed_or_deactivate(r, s, true, + return damon_pa_mark_accessed_or_deactivate(r, addr_unit, s, true, sz_filter_passed); } static unsigned long damon_pa_deactivate_pages(struct damon_region *r, - struct damos *s, unsigned long *sz_filter_passed) + unsigned long addr_unit, struct damos *s, + unsigned long *sz_filter_passed) { - return damon_pa_mark_accessed_or_deactivate(r, s, false, + return damon_pa_mark_accessed_or_deactivate(r, addr_unit, s, false, sz_filter_passed); } @@ -319,9 +323,11 @@ static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx, case DAMOS_PAGEOUT: return damon_pa_pageout(r, aunit, scheme, sz_filter_passed); case DAMOS_LRU_PRIO: - return damon_pa_mark_accessed(r, scheme, sz_filter_passed); + return damon_pa_mark_accessed(r, aunit, scheme, + sz_filter_passed); case DAMOS_LRU_DEPRIO: - return damon_pa_deactivate_pages(r, scheme, sz_filter_passed); + return damon_pa_deactivate_pages(r, aunit, scheme, + sz_filter_passed); case DAMOS_MIGRATE_HOT: case DAMOS_MIGRATE_COLD: return damon_pa_migrate(r, scheme, sz_filter_passed); -- 2.34.1 From: SeongJae Park Add support of addr_unit for DAMOS_MIGRATE_HOT and DAMOS_MIGRATE_COLD action handling from the DAMOS operation implementation for the physical address space. Signed-off-by: SeongJae Park --- mm/damon/paddr.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index b8a7f462967b..76e1ee82b441 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -245,15 +245,17 @@ static unsigned long damon_pa_deactivate_pages(struct damon_region *r, sz_filter_passed); } -static unsigned long damon_pa_migrate(struct damon_region *r, struct damos *s, +static unsigned long damon_pa_migrate(struct damon_region *r, + unsigned long addr_unit, struct damos *s, unsigned long *sz_filter_passed) { - unsigned long addr, applied; + phys_addr_t addr; + unsigned long applied; LIST_HEAD(folio_list); struct folio *folio; - addr = r->ar.start; - while (addr < r->ar.end) { + addr = damon_pa_phys_addr(r->ar.start, addr_unit); + while (addr < damon_pa_phys_addr(r->ar.end, addr_unit)) { folio = damon_get_folio(PHYS_PFN(addr)); if (damon_pa_invalid_damos_folio(folio, s)) { addr += PAGE_SIZE; @@ -330,7 +332,7 @@ static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx, sz_filter_passed); case DAMOS_MIGRATE_HOT: case DAMOS_MIGRATE_COLD: - return damon_pa_migrate(r, scheme, sz_filter_passed); + return damon_pa_migrate(r, aunit, scheme, sz_filter_passed); case DAMOS_STAT: return damon_pa_stat(r, scheme, sz_filter_passed); default: -- 2.34.1 From: SeongJae Park Add support of addr_unit for DAMOS_STAT action handling from the DAMOS operation implementation for the physical address space. Signed-off-by: SeongJae Park --- mm/damon/paddr.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 76e1ee82b441..530bc9d3ce3b 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -289,17 +289,18 @@ static bool damon_pa_scheme_has_filter(struct damos *s) return false; } -static unsigned long damon_pa_stat(struct damon_region *r, struct damos *s, +static unsigned long damon_pa_stat(struct damon_region *r, + unsigned long addr_unit, struct damos *s, unsigned long *sz_filter_passed) { - unsigned long addr; + phys_addr_t addr; struct folio *folio; if (!damon_pa_scheme_has_filter(s)) return 0; - addr = r->ar.start; - while (addr < r->ar.end) { + addr = damon_pa_phys_addr(r->ar.start, addr_unit); + while (addr < damon_pa_phys_addr(r->ar.end, addr_unit)) { folio = damon_get_folio(PHYS_PFN(addr)); if (damon_pa_invalid_damos_folio(folio, s)) { addr += PAGE_SIZE; @@ -334,7 +335,7 @@ static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx, case DAMOS_MIGRATE_COLD: return damon_pa_migrate(r, aunit, scheme, sz_filter_passed); case DAMOS_STAT: - return damon_pa_stat(r, scheme, sz_filter_passed); + return damon_pa_stat(r, aunit, scheme, sz_filter_passed); default: /* DAMOS actions that not yet supported by 'paddr'. */ break; -- 2.34.1 From: SeongJae Park Only DAMON kernel API callers can use addr_unit parameter. Implement a sysfs file to let DAMON sysfs ABI users use it. Signed-off-by: SeongJae Park --- mm/damon/sysfs.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 6d2b0dab50cb..bea782b0a711 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -834,6 +834,7 @@ static const struct damon_sysfs_ops_name damon_sysfs_ops_names[] = { struct damon_sysfs_context { struct kobject kobj; enum damon_ops_id ops_id; + unsigned long addr_unit; struct damon_sysfs_attrs *attrs; struct damon_sysfs_targets *targets; struct damon_sysfs_schemes *schemes; @@ -849,6 +850,7 @@ static struct damon_sysfs_context *damon_sysfs_context_alloc( return NULL; context->kobj = (struct kobject){}; context->ops_id = ops_id; + context->addr_unit = 1; return context; } @@ -997,6 +999,25 @@ static ssize_t operations_store(struct kobject *kobj, return -EINVAL; } +static ssize_t addr_unit_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct damon_sysfs_context *context = container_of(kobj, + struct damon_sysfs_context, kobj); + + return sysfs_emit(buf, "%lu\n", context->addr_unit); +} + +static ssize_t addr_unit_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + struct damon_sysfs_context *context = container_of(kobj, + struct damon_sysfs_context, kobj); + int err = kstrtoul(buf, 0, &context->addr_unit); + + return err ? err : count; +} + static void damon_sysfs_context_release(struct kobject *kobj) { kfree(container_of(kobj, struct damon_sysfs_context, kobj)); @@ -1008,9 +1029,13 @@ static struct kobj_attribute damon_sysfs_context_avail_operations_attr = static struct kobj_attribute damon_sysfs_context_operations_attr = __ATTR_RW_MODE(operations, 0600); +static struct kobj_attribute damon_sysfs_context_addr_unit_attr = + __ATTR_RW_MODE(addr_unit, 0600); + static struct attribute *damon_sysfs_context_attrs[] = { &damon_sysfs_context_avail_operations_attr.attr, &damon_sysfs_context_operations_attr.attr, + &damon_sysfs_context_addr_unit_attr.attr, NULL, }; ATTRIBUTE_GROUPS(damon_sysfs_context); @@ -1397,6 +1422,7 @@ static int damon_sysfs_apply_inputs(struct damon_ctx *ctx, err = damon_select_ops(ctx, sys_ctx->ops_id); if (err) return err; + ctx->addr_unit = sys_ctx->addr_unit; err = damon_sysfs_set_attrs(ctx, sys_ctx->attrs); if (err) return err; -- 2.34.1 From: SeongJae Park Add 'addr_unit' parameter description on DAMON design document. Signed-off-by: SeongJae Park --- Documentation/mm/damon/design.rst | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst index 03f8137256f5..cffa320ead88 100644 --- a/Documentation/mm/damon/design.rst +++ b/Documentation/mm/damon/design.rst @@ -67,7 +67,7 @@ processes, NUMA nodes, files, and backing memory devices would be supportable. Also, if some architectures or devices support special optimized access check features, those will be easily configurable. -DAMON currently provides below three operation sets. Below two subsections +DAMON currently provides below three operation sets. Below three subsections describe how those work. - vaddr: Monitor virtual address spaces of specific processes @@ -135,6 +135,18 @@ the interference is the responsibility of sysadmins. However, it solves the conflict with the reclaim logic using ``PG_idle`` and ``PG_young`` page flags, as Idle page tracking does. +Address Unit +------------ + +DAMON core layer uses ``unsinged long`` type for monitoring target address +ranges. In some cases, the address space for a given operations set could be +too large to be handled with the type. ARM (32-bit) with large physical +address extension is an example. For such cases, a per-operations set +parameter called ``address unit`` is provided. It represents the scale factor +that need to be multiplied to the core layer's address for calculating real +address on the given address space. Support of ``address unit`` parameter is +up to each operations set implementation. ``paddr`` is the only operations set +implementation that supports the parameter. .. _damon_core_logic: -- 2.34.1 From: SeongJae Park Document addr_unit DAMON sysfs file on DAMON usage document. Signed-off-by: SeongJae Park --- Documentation/admin-guide/mm/damon/usage.rst | 11 +++++++---- Documentation/mm/damon/design.rst | 2 ++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Documentation/admin-guide/mm/damon/usage.rst b/Documentation/admin-guide/mm/damon/usage.rst index ff3a2dda1f02..2cae60b6f3ca 100644 --- a/Documentation/admin-guide/mm/damon/usage.rst +++ b/Documentation/admin-guide/mm/damon/usage.rst @@ -61,7 +61,7 @@ comma (","). │ :ref:`kdamonds `/nr_kdamonds │ │ :ref:`0 `/state,pid,refresh_ms │ │ │ :ref:`contexts `/nr_contexts - │ │ │ │ :ref:`0 `/avail_operations,operations + │ │ │ │ :ref:`0 `/avail_operations,operations,addr_unit │ │ │ │ │ :ref:`monitoring_attrs `/ │ │ │ │ │ │ intervals/sample_us,aggr_us,update_us │ │ │ │ │ │ │ intervals_goal/access_bp,aggrs,min_sample_us,max_sample_us @@ -188,9 +188,9 @@ details). At the moment, only one context per kdamond is supported, so only contexts// ------------- -In each context directory, two files (``avail_operations`` and ``operations``) -and three directories (``monitoring_attrs``, ``targets``, and ``schemes``) -exist. +In each context directory, three files (``avail_operations``, ``operations`` +and ``addr_unit``) and three directories (``monitoring_attrs``, ``targets``, +and ``schemes``) exist. DAMON supports multiple types of :ref:`monitoring operations `, including those for virtual address @@ -205,6 +205,9 @@ You can set and get what type of monitoring operations DAMON will use for the context by writing one of the keywords listed in ``avail_operations`` file and reading from the ``operations`` file. +``addr_unit`` file is for setting and getting the :ref:`address unit +` parameter of the operations set. + .. _sysfs_monitoring_attrs: contexts//monitoring_attrs/ diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst index cffa320ead88..fd183cd09ef2 100644 --- a/Documentation/mm/damon/design.rst +++ b/Documentation/mm/damon/design.rst @@ -135,6 +135,8 @@ the interference is the responsibility of sysadmins. However, it solves the conflict with the reclaim logic using ``PG_idle`` and ``PG_young`` page flags, as Idle page tracking does. +.. _damon_design_addr_unit: + Address Unit ------------ -- 2.34.1 From: SeongJae Park Document addr_unit DAMON sysfs file on DAMON ABI document. Signed-off-by: SeongJae Park --- Documentation/ABI/testing/sysfs-kernel-mm-damon | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-damon b/Documentation/ABI/testing/sysfs-kernel-mm-damon index 6791d879759e..cf4d66bd119d 100644 --- a/Documentation/ABI/testing/sysfs-kernel-mm-damon +++ b/Documentation/ABI/testing/sysfs-kernel-mm-damon @@ -77,6 +77,13 @@ Description: Writing a keyword for a monitoring operations set ('vaddr' for Note that only the operations sets that listed in 'avail_operations' file are valid inputs. +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//addr_unit +Date: Apr 2025 +Contact: SeongJae Park +Description: Writing an integer to this file sets the 'address unit' + parameter of the given operations set of the context. Reading + the file returns the last-written 'address unit' value. + What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/sample_us Date: Mar 2022 Contact: SeongJae Park -- 2.34.1 In module DAMON_RECLAIM and DAMON_LRU_SORT, the damon_ctx is independent of the core, necessitating dedicated addr_unit integration for these features. Additionally, if the input monitor_region_start and monitor_region_end are both 0 while addr_unit is set to a non-zero valuethe default system RAM range should be divided by addr_unit. Signed-off-by: Quanmin Yan --- include/linux/damon.h | 4 +++- mm/damon/core.c | 14 ++++++++++---- mm/damon/lru_sort.c | 16 +++++++++++++--- mm/damon/modules-common.c | 5 ++++- mm/damon/modules-common.h | 2 +- mm/damon/reclaim.c | 16 +++++++++++++--- mm/damon/stat.c | 2 +- 7 files changed, 45 insertions(+), 14 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index b85c6c669cd0..1b7b4cf1a3c5 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -942,7 +942,9 @@ int damon_call(struct damon_ctx *ctx, struct damon_call_control *control); int damos_walk(struct damon_ctx *ctx, struct damos_walk_control *control); int damon_set_region_biggest_system_ram_default(struct damon_target *t, - unsigned long *start, unsigned long *end); + unsigned long *start, + unsigned long *end, + unsigned long addr_unit); #endif /* CONFIG_DAMON */ diff --git a/mm/damon/core.c b/mm/damon/core.c index 1a8d3009d606..803c30f64b94 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -2714,6 +2714,7 @@ static bool damon_find_biggest_system_ram(unsigned long *start, * @t: The monitoring target to set the region. * @start: The pointer to the start address of the region. * @end: The pointer to the end address of the region. + * @addr_unit: Scale factor for core to ops address conversion. * * This function sets the region of @t as requested by @start and @end. If the * values of @start and @end are zero, however, this function finds the biggest @@ -2724,16 +2725,21 @@ static bool damon_find_biggest_system_ram(unsigned long *start, * Return: 0 on success, negative error code otherwise. */ int damon_set_region_biggest_system_ram_default(struct damon_target *t, - unsigned long *start, unsigned long *end) + unsigned long *start, + unsigned long *end, + unsigned long addr_unit) { struct damon_addr_range addr_range; if (*start > *end) return -EINVAL; - if (!*start && !*end && - !damon_find_biggest_system_ram(start, end)) - return -EINVAL; + if (!*start && !*end) { + if (!damon_find_biggest_system_ram(start, end) || !addr_unit) + return -EINVAL; + *start /= addr_unit; + *end /= addr_unit; + } addr_range.start = *start; addr_range.end = *end; diff --git a/mm/damon/lru_sort.c b/mm/damon/lru_sort.c index 151a9de5ad8b..8107d08c5e4b 100644 --- a/mm/damon/lru_sort.c +++ b/mm/damon/lru_sort.c @@ -111,6 +111,14 @@ module_param(monitor_region_start, ulong, 0600); static unsigned long monitor_region_end __read_mostly; module_param(monitor_region_end, ulong, 0600); +/* + * Scale factor for DAMON_LRU_SORT to ops address conversion. + * + * This parameter is used to convert to the actual physical address. + */ +static unsigned long addr_unit __read_mostly = 1; +module_param(addr_unit, ulong, 0600); + /* * PID of the DAMON thread * @@ -194,7 +202,8 @@ static int damon_lru_sort_apply_parameters(void) unsigned int hot_thres, cold_thres; int err; - err = damon_modules_new_paddr_ctx_target(¶m_ctx, ¶m_target); + err = damon_modules_new_paddr_ctx_target(¶m_ctx, ¶m_target, + addr_unit); if (err) return err; @@ -221,7 +230,8 @@ static int damon_lru_sort_apply_parameters(void) err = damon_set_region_biggest_system_ram_default(param_target, &monitor_region_start, - &monitor_region_end); + &monitor_region_end, + addr_unit); if (err) goto out; err = damon_commit_ctx(ctx, param_ctx); @@ -323,7 +333,7 @@ MODULE_PARM_DESC(enabled, static int __init damon_lru_sort_init(void) { - int err = damon_modules_new_paddr_ctx_target(&ctx, &target); + int err = damon_modules_new_paddr_ctx_target(&ctx, &target, 1); if (err) goto out; diff --git a/mm/damon/modules-common.c b/mm/damon/modules-common.c index 86d58f8c4f63..613b7cc99368 100644 --- a/mm/damon/modules-common.c +++ b/mm/damon/modules-common.c @@ -13,9 +13,10 @@ * Allocate, set, and return a DAMON context for the physical address space. * @ctxp: Pointer to save the point to the newly created context * @targetp: Pointer to save the point to the newly created target + * @addr_unit: Scale factor for modules to ops address conversion. */ int damon_modules_new_paddr_ctx_target(struct damon_ctx **ctxp, - struct damon_target **targetp) + struct damon_target **targetp, unsigned long addr_unit) { struct damon_ctx *ctx; struct damon_target *target; @@ -24,6 +25,8 @@ int damon_modules_new_paddr_ctx_target(struct damon_ctx **ctxp, if (!ctx) return -ENOMEM; + ctx->addr_unit = addr_unit; + if (damon_select_ops(ctx, DAMON_OPS_PADDR)) { damon_destroy_ctx(ctx); return -EINVAL; diff --git a/mm/damon/modules-common.h b/mm/damon/modules-common.h index f103ad556368..c7048a449321 100644 --- a/mm/damon/modules-common.h +++ b/mm/damon/modules-common.h @@ -46,4 +46,4 @@ 0400); int damon_modules_new_paddr_ctx_target(struct damon_ctx **ctxp, - struct damon_target **targetp); + struct damon_target **targetp, unsigned long addr_unit); diff --git a/mm/damon/reclaim.c b/mm/damon/reclaim.c index 3c71b4596676..0e11b121d693 100644 --- a/mm/damon/reclaim.c +++ b/mm/damon/reclaim.c @@ -128,6 +128,14 @@ module_param(monitor_region_start, ulong, 0600); static unsigned long monitor_region_end __read_mostly; module_param(monitor_region_end, ulong, 0600); +/* + * Scale factor for DAMON_RECLAIM to ops address conversion. + * + * This parameter is used to convert to the actual physical address. + */ +static unsigned long addr_unit __read_mostly = 1; +module_param(addr_unit, ulong, 0600); + /* * Skip anonymous pages reclamation. * @@ -190,7 +198,8 @@ static int damon_reclaim_apply_parameters(void) struct damos_filter *filter; int err; - err = damon_modules_new_paddr_ctx_target(¶m_ctx, ¶m_target); + err = damon_modules_new_paddr_ctx_target(¶m_ctx, ¶m_target, + addr_unit); if (err) return err; @@ -229,7 +238,8 @@ static int damon_reclaim_apply_parameters(void) err = damon_set_region_biggest_system_ram_default(param_target, &monitor_region_start, - &monitor_region_end); + &monitor_region_end, + addr_unit); if (err) goto out; err = damon_commit_ctx(ctx, param_ctx); @@ -327,7 +337,7 @@ MODULE_PARM_DESC(enabled, static int __init damon_reclaim_init(void) { - int err = damon_modules_new_paddr_ctx_target(&ctx, &target); + int err = damon_modules_new_paddr_ctx_target(&ctx, &target, 1); if (err) goto out; diff --git a/mm/damon/stat.c b/mm/damon/stat.c index 87bcd8866d4b..ae7377e7409f 100644 --- a/mm/damon/stat.c +++ b/mm/damon/stat.c @@ -181,7 +181,7 @@ static struct damon_ctx *damon_stat_build_ctx(void) if (!target) goto free_out; damon_add_target(ctx, target); - if (damon_set_region_biggest_system_ram_default(target, &start, &end)) + if (damon_set_region_biggest_system_ram_default(target, &start, &end, ctx->addr_unit)) goto free_out; return ctx; free_out: -- 2.34.1 Adopting addr_unit would make DAMON_MINREGION 'addr_unit * 4096' bytes and cause data alignment issues[1]. Add damon_ctx->min_region to change DAMON_MIN_REGION from a global macro value to per-context variable, let target inherit the min_region from its associated ctx to avoid excessive passing of ctx. [1] https://lore.kernel.org/all/527714dd-0e33-43ab-bbbd-d89670ba79e7@huawei.com Signed-off-by: Quanmin Yan --- include/linux/damon.h | 7 ++++++- mm/damon/core.c | 41 +++++++++++++++++++++++++++-------------- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 1b7b4cf1a3c5..aa045dcb5b5d 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -88,6 +88,7 @@ struct damon_region { /** * struct damon_target - Represents a monitoring target. * @pid: The PID of the virtual address space to monitor. + * @min_region: Minimum Region Size. * @nr_regions: Number of monitoring target regions of this target. * @regions_list: Head of the monitoring target regions of this target. * @list: List head for siblings. @@ -95,10 +96,12 @@ struct damon_region { * Each monitoring context could have multiple targets. For example, a context * for virtual memory address spaces could have multiple target processes. The * @pid should be set for appropriate &struct damon_operations including the - * virtual address spaces monitoring operations. + * virtual address spaces monitoring operations. The @min_region Keeps consistent + * with the associated monitoring context. */ struct damon_target { struct pid *pid; + unsigned long min_region; unsigned int nr_regions; struct list_head regions_list; struct list_head list; @@ -747,6 +750,7 @@ struct damon_attrs { * * @ops: Set of monitoring operations for given use cases. * @addr_unit: Scale factor for core to ops address conversion. + * @min_region: Minimum Region Size. * @adaptive_targets: Head of monitoring targets (&damon_target) list. * @schemes: Head of schemes (&damos) list. */ @@ -789,6 +793,7 @@ struct damon_ctx { struct damon_operations ops; unsigned long addr_unit; + unsigned long min_region; struct list_head adaptive_targets; struct list_head schemes; diff --git a/mm/damon/core.c b/mm/damon/core.c index 803c30f64b94..b162aa1156fc 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -245,16 +245,16 @@ int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, /* no region intersects with this range */ newr = damon_new_region( ALIGN_DOWN(range->start, - DAMON_MIN_REGION), - ALIGN(range->end, DAMON_MIN_REGION)); + t->min_region), + ALIGN(range->end, t->min_region)); if (!newr) return -ENOMEM; damon_insert_region(newr, damon_prev_region(r), r, t); } else { /* resize intersecting regions to fit in this range */ first->ar.start = ALIGN_DOWN(range->start, - DAMON_MIN_REGION); - last->ar.end = ALIGN(range->end, DAMON_MIN_REGION); + t->min_region); + last->ar.end = ALIGN(range->end, t->min_region); /* fill possible holes in the range */ err = damon_fill_regions_holes(first, last, t); @@ -472,6 +472,7 @@ struct damon_target *damon_new_target(void) t->pid = NULL; t->nr_regions = 0; + t->min_region = DAMON_MIN_REGION; INIT_LIST_HEAD(&t->regions_list); INIT_LIST_HEAD(&t->list); @@ -480,6 +481,7 @@ struct damon_target *damon_new_target(void) void damon_add_target(struct damon_ctx *ctx, struct damon_target *t) { + t->min_region = ctx->min_region; list_add_tail(&t->list, &ctx->adaptive_targets); } @@ -545,6 +547,7 @@ struct damon_ctx *damon_new_ctx(void) ctx->attrs.max_nr_regions = 1000; ctx->addr_unit = 1; + ctx->min_region = DAMON_MIN_REGION; INIT_LIST_HEAD(&ctx->adaptive_targets); INIT_LIST_HEAD(&ctx->schemes); @@ -1181,6 +1184,14 @@ static int damon_commit_targets( return 0; } +static void damon_sync_target_min_region(struct damon_ctx *ctx) +{ + struct damon_target *t; + + damon_for_each_target(t, ctx) + t->min_region = ctx->min_region; +} + /** * damon_commit_ctx() - Commit parameters of a DAMON context to another. * @dst: The commit destination DAMON context. @@ -1216,6 +1227,8 @@ int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src) return err; dst->ops = src->ops; dst->addr_unit = src->addr_unit ? : 1; + dst->min_region = max(DAMON_MIN_REGION / dst->addr_unit, 1); + damon_sync_target_min_region(dst); return 0; } @@ -1248,8 +1261,8 @@ static unsigned long damon_region_sz_limit(struct damon_ctx *ctx) if (ctx->attrs.min_nr_regions) sz /= ctx->attrs.min_nr_regions; - if (sz < DAMON_MIN_REGION) - sz = DAMON_MIN_REGION; + if (sz < ctx->min_region) + sz = ctx->min_region; return sz; } @@ -1632,11 +1645,11 @@ static bool damos_skip_charged_region(struct damon_target *t, if (quota->charge_addr_from && r->ar.start < quota->charge_addr_from) { sz_to_skip = ALIGN_DOWN(quota->charge_addr_from - - r->ar.start, DAMON_MIN_REGION); + r->ar.start, t->min_region); if (!sz_to_skip) { - if (damon_sz_region(r) <= DAMON_MIN_REGION) + if (damon_sz_region(r) <= t->min_region) return true; - sz_to_skip = DAMON_MIN_REGION; + sz_to_skip = t->min_region; } damon_split_region_at(t, r, sz_to_skip); r = damon_next_region(r); @@ -1678,8 +1691,8 @@ static bool damos_filter_match(struct damon_ctx *ctx, struct damon_target *t, matched = target_idx == filter->target_idx; break; case DAMOS_FILTER_TYPE_ADDR: - start = ALIGN_DOWN(filter->addr_range.start, DAMON_MIN_REGION); - end = ALIGN_DOWN(filter->addr_range.end, DAMON_MIN_REGION); + start = ALIGN_DOWN(filter->addr_range.start, t->min_region); + end = ALIGN_DOWN(filter->addr_range.end, t->min_region); /* inside the range */ if (start <= r->ar.start && r->ar.end <= end) { @@ -1850,7 +1863,7 @@ static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t, if (c->ops.apply_scheme) { if (quota->esz && quota->charged_sz + sz > quota->esz) { sz = ALIGN_DOWN(quota->esz - quota->charged_sz, - DAMON_MIN_REGION); + c->min_region); if (!sz) goto update_stat; damon_split_region_at(t, r, sz); @@ -2302,13 +2315,13 @@ static void damon_split_regions_of(struct damon_target *t, int nr_subs) sz_region = damon_sz_region(r); for (i = 0; i < nr_subs - 1 && - sz_region > 2 * DAMON_MIN_REGION; i++) { + sz_region > 2 * t->min_region; i++) { /* * Randomly select size of left sub-region to be at * least 10 percent and at most 90% of original region */ sz_sub = ALIGN_DOWN(damon_rand(1, 10) * - sz_region / 10, DAMON_MIN_REGION); + sz_region / 10, t->min_region); /* Do not allow blank region */ if (sz_sub == 0 || sz_sub >= sz_region) continue; -- 2.34.1 By calling damon_sysfs_turn_damon_on(), the execution of damon_commit_ctx() can be bypassed. Therefore, it is necessary to prevent ctx->addr_unit from being set to 0 in damon_sysfs_apply_inputs() and update min_region to avoid potential issues. Signed-off-by: Quanmin Yan --- mm/damon/sysfs.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index bea782b0a711..122824776c1d 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -1422,7 +1422,8 @@ static int damon_sysfs_apply_inputs(struct damon_ctx *ctx, err = damon_select_ops(ctx, sys_ctx->ops_id); if (err) return err; - ctx->addr_unit = sys_ctx->addr_unit; + ctx->addr_unit = sys_ctx->addr_unit ? : 1; + ctx->min_region = max(DAMON_MIN_REGION / ctx->addr_unit, 1); err = damon_sysfs_set_attrs(ctx, sys_ctx->attrs); if (err) return err; -- 2.34.1 After introducing ctx->addr_unit, the unit of sz might not be in bytes. However, sz_applied is returned in bytes after processing by paddr. To maintain external consistency, sz is converted to byte units when updating the state. Signed-off-by: Quanmin Yan --- mm/damon/core.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index b162aa1156fc..bc764f9dc5c5 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1889,7 +1889,9 @@ static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t, r->age = 0; update_stat: - damos_update_stat(s, sz, sz_applied, sz_ops_filter_passed); + damos_update_stat(s, + sz * (c->ops.id == DAMON_OPS_PADDR ? c->addr_unit : 1), + sz_applied, sz_ops_filter_passed); } static void damon_do_apply_schemes(struct damon_ctx *c, -- 2.34.1 For 32-bit systems, damos_stat now uses unsigned long long for byte statistics data to avoid integer overflow risks inherent in the previous design. Signed-off-by: Quanmin Yan --- include/linux/damon.h | 6 +++--- mm/damon/modules-common.h | 4 ++-- mm/damon/sysfs-schemes.c | 12 ++++++------ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index aa045dcb5b5d..d85850cf06c5 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -333,10 +333,10 @@ struct damos_watermarks { */ struct damos_stat { unsigned long nr_tried; - unsigned long sz_tried; + unsigned long long sz_tried; unsigned long nr_applied; - unsigned long sz_applied; - unsigned long sz_ops_filter_passed; + unsigned long long sz_applied; + unsigned long long sz_ops_filter_passed; unsigned long qt_exceeds; }; diff --git a/mm/damon/modules-common.h b/mm/damon/modules-common.h index c7048a449321..ae45d0eb960e 100644 --- a/mm/damon/modules-common.h +++ b/mm/damon/modules-common.h @@ -36,11 +36,11 @@ #define DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(stat, try_name, \ succ_name, qt_exceed_name) \ module_param_named(nr_##try_name, stat.nr_tried, ulong, 0400); \ - module_param_named(bytes_##try_name, stat.sz_tried, ulong, \ + module_param_named(bytes_##try_name, stat.sz_tried, ullong, \ 0400); \ module_param_named(nr_##succ_name, stat.nr_applied, ulong, \ 0400); \ - module_param_named(bytes_##succ_name, stat.sz_applied, ulong, \ + module_param_named(bytes_##succ_name, stat.sz_applied, ullong, \ 0400); \ module_param_named(nr_##qt_exceed_name, stat.qt_exceeds, ulong, \ 0400); diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 74056bcd6a2c..3c4882549a28 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -199,10 +199,10 @@ static const struct kobj_type damon_sysfs_scheme_regions_ktype = { struct damon_sysfs_stats { struct kobject kobj; unsigned long nr_tried; - unsigned long sz_tried; + unsigned long long sz_tried; unsigned long nr_applied; - unsigned long sz_applied; - unsigned long sz_ops_filter_passed; + unsigned long long sz_applied; + unsigned long long sz_ops_filter_passed; unsigned long qt_exceeds; }; @@ -226,7 +226,7 @@ static ssize_t sz_tried_show(struct kobject *kobj, struct kobj_attribute *attr, struct damon_sysfs_stats *stats = container_of(kobj, struct damon_sysfs_stats, kobj); - return sysfs_emit(buf, "%lu\n", stats->sz_tried); + return sysfs_emit(buf, "%llu\n", stats->sz_tried); } static ssize_t nr_applied_show(struct kobject *kobj, @@ -244,7 +244,7 @@ static ssize_t sz_applied_show(struct kobject *kobj, struct damon_sysfs_stats *stats = container_of(kobj, struct damon_sysfs_stats, kobj); - return sysfs_emit(buf, "%lu\n", stats->sz_applied); + return sysfs_emit(buf, "%llu\n", stats->sz_applied); } static ssize_t sz_ops_filter_passed_show(struct kobject *kobj, @@ -253,7 +253,7 @@ static ssize_t sz_ops_filter_passed_show(struct kobject *kobj, struct damon_sysfs_stats *stats = container_of(kobj, struct damon_sysfs_stats, kobj); - return sysfs_emit(buf, "%lu\n", stats->sz_ops_filter_passed); + return sysfs_emit(buf, "%llu\n", stats->sz_ops_filter_passed); } static ssize_t qt_exceeds_show(struct kobject *kobj, -- 2.34.1 In the original quota enforcement implementation, the traffic calculation multiplied A by 1000000 due to time unit conversion, making it highly prone to overflow on 32-bit systems: damos_set_effective_quota if (quota->total_charged_ns) throughput = quota->total_charged_sz * 1000000 / quota->total_charged_ns; Requiring total_charged_sz to be less than 4GB/1000000 is unreasonable. Additionally, when overflow occurs and causes quota->esz to become extremely small, the subsequent damos_apply_scheme logic permanently sets sz to 0, while quota stop updating, ultimately leading to complete functional failure: damos_apply_scheme if (quota->esz && quota->charged_sz + sz > quota->esz) sz = ALIGN_DOWN(quota->esz - quota->charged_sz, DAMON_MIN_REGION); Total charged stats use the unsigned long long data type to reduce overflow risk, with data reset capability after overflow occurs. Signed-off-by: Quanmin Yan --- include/linux/damon.h | 4 ++-- mm/damon/core.c | 18 ++++++++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index d85850cf06c5..45aab331dfb7 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -247,8 +247,8 @@ struct damos_quota { /* private: */ /* For throughput estimation */ - unsigned long total_charged_sz; - unsigned long total_charged_ns; + unsigned long long total_charged_sz; + unsigned long long total_charged_ns; /* For charging the quota */ unsigned long charged_sz; diff --git a/mm/damon/core.c b/mm/damon/core.c index bc764f9dc5c5..5e05fdd91c12 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -15,6 +15,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include @@ -2059,8 +2060,8 @@ static unsigned long damos_quota_score(struct damos_quota *quota) */ static void damos_set_effective_quota(struct damos_quota *quota) { - unsigned long throughput; - unsigned long esz = ULONG_MAX; + unsigned long long throughput; + unsigned long long esz = ULLONG_MAX; if (!quota->ms && list_empty("a->goals)) { quota->esz = quota->sz; @@ -2077,11 +2078,16 @@ static void damos_set_effective_quota(struct damos_quota *quota) } if (quota->ms) { - if (quota->total_charged_ns) - throughput = quota->total_charged_sz * 1000000 / - quota->total_charged_ns; - else + if (quota->total_charged_ns && + likely(quota->total_charged_sz < ULLONG_MAX / 1000000)) { + throughput = div64_u64(quota->total_charged_sz * 1000000, + quota->total_charged_ns); + } else { throughput = PAGE_SIZE * 1024; + /* Reset the variable when an overflow occurs */ + quota->total_charged_ns = 0; + quota->total_charged_sz = 0; + } esz = min(throughput * quota->ms, esz); } -- 2.34.1