Introduce sysfs interfaces to control and monitor the HugeTLB dynamic cache. Expose the following attributes under: /sys/kernel/mm/hugepages/hugepages-/ - max_cached_huge_pages: Max limit of cached hugepages (global). - nr_cached_hugepages: Current number of cached hugepages. Writing to this file allows manual expansion or contraction of the cache. These interfaces support writing delta values (e.g. "+1" or "-1") in addition to absolute values, allowing orchestrators (e.g. borglet) to dynamically scale the cache without racing against the kernel shrinker. Also expose node-specific versions under node sysfs if CONFIG_NUMA. Implement adjust_cached_huge_pages to handle the actual expansion (allocating fresh hugepages to cache) and contraction (freeing cached hugepages to buddy). For expansion and contraction correctness: - Global expansion interleaves allocations across allowed nodes. - Node-specific expansion enforces the global cache limit ceiling and overcommit limits, uses __GFP_THISNODE to prevent fallback, and accounts to the actual allocated node using folio_nid(). - Contraction uses LRU (Least Recently Used) policy (taking from the head of the cache list) to discard cold pages first. Signed-off-by: Sourav Panda --- mm/hugetlb.c | 230 ++++++++++++++++++++++++++++++++++++++++++ mm/hugetlb_internal.h | 9 ++ mm/hugetlb_sysfs.c | 158 +++++++++++++++++++++++++++++ 3 files changed, 397 insertions(+) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index d00aa67b8e13..f18a3123cbcb 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -4171,6 +4171,236 @@ long demote_pool_huge_page(struct hstate *src, nodemask_t *nodes_allowed, return -EBUSY; } +#ifdef CONFIG_HUGETLB_CACHE +static int adjust_cached_huge_pages(struct hstate *h, long count, bool is_delta, int nid, + nodemask_t *nodes_allowed, bool update_limit) +{ + struct folio *folio; + int err = 0; + + mutex_lock(&h->resize_lock); + + if (is_delta) { + long current_val; + + if (nid == NUMA_NO_NODE) + current_val = update_limit ? h->max_cached_huge_pages : + h->nr_cached_hugepages; + else + current_val = update_limit ? h->max_cached_huge_pages_node[nid] : + h->nr_cached_hugepages_node[nid]; + + current_val += count; + if (current_val < 0) + current_val = 0; + count = current_val; + } + + if (nid == NUMA_NO_NODE) { + int temp_next_node = first_node(*nodes_allowed); + + if (update_limit) { + h->max_cached_huge_pages = count; + } else if (count > h->max_cached_huge_pages) { + err = -EINVAL; + goto out; + } + + /* Expansion */ + while (!update_limit) { + unsigned long curr; + + spin_lock_irq(&hugetlb_lock); + curr = h->nr_cached_hugepages; + spin_unlock_irq(&hugetlb_lock); + + if (curr >= count) + break; + + folio = alloc_pool_huge_folio(h, nodes_allowed, + NULL, &temp_next_node); + if (!folio) { + err = -ENOMEM; + break; + } + + spin_lock_irq(&hugetlb_lock); + if (h->nr_cached_hugepages >= count) { + spin_unlock_irq(&hugetlb_lock); + update_and_free_hugetlb_folio(h, folio, false); + break; + } + + account_new_hugetlb_folio(h, folio); + + hugetlb_cache_add(h, folio, false); + spin_unlock_irq(&hugetlb_lock); + cond_resched(); + } + + /* Contraction */ + while (1) { + unsigned long curr; + struct folio *folio = NULL; + + spin_lock_irq(&hugetlb_lock); + curr = h->nr_cached_hugepages; + if (curr <= count) { + spin_unlock_irq(&hugetlb_lock); + break; + } + + int node; + + for_each_node_mask(node, *nodes_allowed) { + if (!list_empty(&h->hugepage_cache_lists[node])) { + folio = list_first_entry(&h->hugepage_cache_lists[node], + struct folio, lru); + break; + } + } + + if (!folio) { + spin_unlock_irq(&hugetlb_lock); + break; + } + + remove_hugetlb_folio(h, folio, false); + spin_unlock_irq(&hugetlb_lock); + + update_and_free_hugetlb_folio(h, folio, false); + cond_resched(); + } + } else { + if (update_limit) { + h->max_cached_huge_pages_node[nid] = count; + } else if (count > h->max_cached_huge_pages_node[nid]) { + err = -EINVAL; + goto out; + } + + /* Node expansion */ + while (!update_limit) { + unsigned long curr; + gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE; + + spin_lock_irq(&hugetlb_lock); + curr = h->nr_cached_hugepages_node[nid]; + if (h->nr_cached_hugepages >= h->max_cached_huge_pages) { + spin_unlock_irq(&hugetlb_lock); + err = -ENOSPC; + break; + } + spin_unlock_irq(&hugetlb_lock); + + if (curr >= count) + break; + + folio = alloc_fresh_hugetlb_folio(h, gfp_mask, + nid, nodes_allowed); + if (!folio) { + err = -ENOMEM; + break; + } + + spin_lock_irq(&hugetlb_lock); + if (h->nr_cached_hugepages_node[nid] >= count) { + spin_unlock_irq(&hugetlb_lock); + update_and_free_hugetlb_folio(h, folio, false); + break; + } + + account_new_hugetlb_folio(h, folio); + + hugetlb_cache_add(h, folio, false); + spin_unlock_irq(&hugetlb_lock); + cond_resched(); + } + + /* Node contraction */ + while (1) { + unsigned long curr; + struct folio *folio = NULL; + + spin_lock_irq(&hugetlb_lock); + curr = h->nr_cached_hugepages_node[nid]; + if (curr <= count) { + spin_unlock_irq(&hugetlb_lock); + break; + } + + if (!list_empty(&h->hugepage_cache_lists[nid])) + folio = list_first_entry(&h->hugepage_cache_lists[nid], + struct folio, lru); + + if (!folio) { + spin_unlock_irq(&hugetlb_lock); + break; + } + + remove_hugetlb_folio(h, folio, false); + spin_unlock_irq(&hugetlb_lock); + + update_and_free_hugetlb_folio(h, folio, false); + cond_resched(); + } + } + +out: + mutex_unlock(&h->resize_lock); + return err; +} + +ssize_t __nr_cached_hugepages_store_common(bool obey_mempolicy, + struct hstate *h, int nid, + long count, bool is_delta, size_t len) +{ + int err; + nodemask_t nodes_allowed, *n_mask; + + if (hstate_is_gigantic_no_runtime(h)) + return -EINVAL; + + if (nid == NUMA_NO_NODE) { + if (!(obey_mempolicy && + init_nodemask_of_mempolicy(&nodes_allowed))) + n_mask = &node_states[N_MEMORY]; + else + n_mask = &nodes_allowed; + } else { + init_nodemask_of_node(&nodes_allowed, nid); + n_mask = &nodes_allowed; + } + + err = adjust_cached_huge_pages(h, count, is_delta, nid, n_mask, false); + + return err ? err : len; +} + +ssize_t __max_cached_huge_pages_store_common(bool obey_mempolicy, + struct hstate *h, int nid, + long count, bool is_delta, size_t len) +{ + int err; + nodemask_t nodes_allowed, *n_mask; + + if (hstate_is_gigantic_no_runtime(h)) + return -EINVAL; + + if (nid == NUMA_NO_NODE) { + if (!(obey_mempolicy && + init_nodemask_of_mempolicy(&nodes_allowed))) + n_mask = &node_states[N_MEMORY]; + else + n_mask = &nodes_allowed; + } else { + init_nodemask_of_node(&nodes_allowed, nid); + n_mask = &nodes_allowed; + } + + err = adjust_cached_huge_pages(h, count, is_delta, nid, n_mask, true); + + return err ? err : len; +} +#endif /* CONFIG_HUGETLB_CACHE */ + ssize_t __nr_hugepages_store_common(bool obey_mempolicy, struct hstate *h, int nid, unsigned long count, size_t len) diff --git a/mm/hugetlb_internal.h b/mm/hugetlb_internal.h index 1d2f870deccf..16ed9ac9dc05 100644 --- a/mm/hugetlb_internal.h +++ b/mm/hugetlb_internal.h @@ -114,4 +114,13 @@ extern void hugetlb_sysctl_init(void); static inline void hugetlb_sysctl_init(void) { } #endif +#ifdef CONFIG_HUGETLB_CACHE +ssize_t __nr_cached_hugepages_store_common(bool obey_mempolicy, + struct hstate *h, int nid, + long count, bool is_delta, size_t len); +ssize_t __max_cached_huge_pages_store_common(bool obey_mempolicy, + struct hstate *h, int nid, + long count, bool is_delta, size_t len); +#endif + #endif /* _LINUX_HUGETLB_INTERNAL_H */ diff --git a/mm/hugetlb_sysfs.c b/mm/hugetlb_sysfs.c index 79ece91406bf..682a3805e6ff 100644 --- a/mm/hugetlb_sysfs.c +++ b/mm/hugetlb_sysfs.c @@ -277,6 +277,152 @@ static ssize_t demote_size_store(struct kobject *kobj, } HSTATE_ATTR(demote_size); +#ifdef CONFIG_HUGETLB_CACHE +static ssize_t nr_cached_hugepages_show_common(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct hstate *h; + unsigned long nr_cached; + int nid; + + h = kobj_to_hstate(kobj, &nid); + if (nid == NUMA_NO_NODE) + nr_cached = h->nr_cached_hugepages; + else + nr_cached = h->nr_cached_hugepages_node[nid]; + + return sysfs_emit(buf, "%lu\n", nr_cached); +} + +static ssize_t nr_cached_hugepages_store_common(bool obey_mempolicy, + struct kobject *kobj, const char *buf, + size_t len) +{ + struct hstate *h; + long count; + bool is_delta = false; + int nid; + int err; + const char *p = skip_spaces(buf); + + if (*p == '+' || *p == '-') { + is_delta = true; + err = kstrtol(p, 10, &count); + } else { + unsigned long ucount; + + err = kstrtoul(p, 10, &ucount); + count = (long)ucount; + } + if (err) + return err; + + h = kobj_to_hstate(kobj, &nid); + return __nr_cached_hugepages_store_common(obey_mempolicy, h, nid, count, is_delta, len); +} + +static ssize_t nr_cached_hugepages_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return nr_cached_hugepages_show_common(kobj, attr, buf); +} + +static ssize_t nr_cached_hugepages_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t len) +{ + return nr_cached_hugepages_store_common(false, kobj, buf, len); +} +HSTATE_ATTR(nr_cached_hugepages); + +#ifdef CONFIG_NUMA +static ssize_t nr_cached_hugepages_mempolicy_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + return nr_cached_hugepages_show_common(kobj, attr, buf); +} + +static ssize_t nr_cached_hugepages_mempolicy_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t len) +{ + return nr_cached_hugepages_store_common(true, kobj, buf, len); +} +HSTATE_ATTR(nr_cached_hugepages_mempolicy); +#endif + +static ssize_t max_cached_huge_pages_show_common(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct hstate *h; + unsigned long max_cached; + int nid; + + h = kobj_to_hstate(kobj, &nid); + if (nid == NUMA_NO_NODE) + max_cached = h->max_cached_huge_pages; + else + max_cached = h->max_cached_huge_pages_node[nid]; + + return sysfs_emit(buf, "%lu\n", max_cached); +} + +static ssize_t max_cached_huge_pages_store_common(bool obey_mempolicy, + struct kobject *kobj, const char *buf, + size_t len) +{ + struct hstate *h; + long count; + bool is_delta = false; + int nid; + int err; + const char *p = skip_spaces(buf); + + if (*p == '+' || *p == '-') { + is_delta = true; + err = kstrtol(p, 10, &count); + } else { + unsigned long ucount; + + err = kstrtoul(p, 10, &ucount); + count = (long)ucount; + } + if (err) + return err; + + h = kobj_to_hstate(kobj, &nid); + return __max_cached_huge_pages_store_common(obey_mempolicy, h, nid, count, is_delta, len); +} + +static ssize_t max_cached_huge_pages_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return max_cached_huge_pages_show_common(kobj, attr, buf); +} + +static ssize_t max_cached_huge_pages_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t len) +{ + return max_cached_huge_pages_store_common(false, kobj, buf, len); +} +HSTATE_ATTR(max_cached_huge_pages); + +#ifdef CONFIG_NUMA +static ssize_t max_cached_huge_pages_mempolicy_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + return max_cached_huge_pages_show_common(kobj, attr, buf); +} + +static ssize_t max_cached_huge_pages_mempolicy_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t len) +{ + return max_cached_huge_pages_store_common(true, kobj, buf, len); +} +HSTATE_ATTR(max_cached_huge_pages_mempolicy); +#endif +#endif /* CONFIG_HUGETLB_CACHE */ + static struct attribute *hstate_attrs[] = { &nr_hugepages_attr.attr, &nr_overcommit_hugepages_attr.attr, @@ -285,6 +431,14 @@ static struct attribute *hstate_attrs[] = { &surplus_hugepages_attr.attr, #ifdef CONFIG_NUMA &nr_hugepages_mempolicy_attr.attr, +#endif +#ifdef CONFIG_HUGETLB_CACHE + &nr_cached_hugepages_attr.attr, + &max_cached_huge_pages_attr.attr, +#ifdef CONFIG_NUMA + &nr_cached_hugepages_mempolicy_attr.attr, + &max_cached_huge_pages_mempolicy_attr.attr, +#endif #endif NULL, }; @@ -359,6 +513,10 @@ static struct attribute *per_node_hstate_attrs[] = { &nr_hugepages_attr.attr, &free_hugepages_attr.attr, &surplus_hugepages_attr.attr, +#ifdef CONFIG_HUGETLB_CACHE + &nr_cached_hugepages_attr.attr, + &max_cached_huge_pages_attr.attr, +#endif NULL, }; -- 2.55.0.rc0.799.gd6f94ed593-goog