Introduce a "tiered_limits" option for the cgroup.memory= kernel
commandline parameter to enable tier-proportional scaling and
enforcing of the memory cgroup controller limits
memory.{min, low, high}. Since mem_cgroup_tiered_limits() will
become a hotpath in the later commits to gate charging, demotion,
and promotion decisions, use a static key so that cgroups not using
tier-aware-memcg limits has minimal overhead.

Enable it by adding to the kernel command line:
	cgroup.memory=tiered_limits

The option is boot-time only, since flipping the bit at runtime could
leave charges uncharged in the future, or uncharges for folios that
were never charged.

This feature is incompatible with cgroup v1, and wil raise a single
warning statement if a system booted with tiered limits mounts a
legacy cgroup:

[XXX] cgroup.memory=tiered_limits should not be enabled with cgroupv1

Signed-off-by: Joshua Hahn <joshua.hahnjy@gmail.com>
---
 include/linux/memcontrol.h | 18 ++++++++++++++++++
 mm/memcontrol.c            | 19 +++++++++++++++++++
 2 files changed, 37 insertions(+)

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 2118d5b33d051..dce03df7eae05 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -530,6 +530,19 @@ static inline bool mem_cgroup_disabled(void)
 	return !cgroup_subsys_enabled(memory_cgrp_subsys);
 }
 
+#ifdef CONFIG_NUMA
+DECLARE_STATIC_KEY_FALSE(memcg_tiered_limits_key);
+static inline bool mem_cgroup_tiered_limits(void)
+{
+	return static_branch_unlikely(&memcg_tiered_limits_key);
+}
+#else
+static inline bool mem_cgroup_tiered_limits(void)
+{
+	return false;
+}
+#endif
+
 static inline void mem_cgroup_protection(struct mem_cgroup *root,
 					 struct mem_cgroup *memcg,
 					 unsigned long *min,
@@ -1083,6 +1096,11 @@ static inline bool mem_cgroup_disabled(void)
 	return true;
 }
 
+static inline bool mem_cgroup_tiered_limits(void)
+{
+	return false;
+}
+
 static inline void memcg_memory_event(struct mem_cgroup *memcg,
 				      enum memcg_memory_event event)
 {
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 29330f5f9d4eb..cefe33b5fd285 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -320,6 +320,13 @@ EXPORT_SYMBOL(memcg_kmem_online_key);
 DEFINE_STATIC_KEY_FALSE(memcg_bpf_enabled_key);
 EXPORT_SYMBOL(memcg_bpf_enabled_key);
 
+#ifdef CONFIG_NUMA
+DEFINE_STATIC_KEY_FALSE(memcg_tiered_limits_key);
+
+/* Tier-proportional scaling of memory controller limits enabled? */
+static bool cgroup_memory_tiered_limits __ro_after_init;
+#endif
+
 /**
  * get_mem_cgroup_css_from_folio - acquire a css of the memcg associated with a folio
  * @folio: folio of interest
@@ -4202,6 +4209,9 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
 	struct mem_cgroup *memcg, *old_memcg;
 	bool memcg_on_dfl = cgroup_subsys_on_dfl(memory_cgrp_subsys);
 
+	if (mem_cgroup_tiered_limits() && !memcg_on_dfl)
+		pr_warn_once("cgroup.memory=tiered_limits should not be enabled with cgroupv1\n");
+
 	old_memcg = set_active_memcg(parent);
 	memcg = mem_cgroup_alloc(parent);
 	set_active_memcg(old_memcg);
@@ -5584,6 +5594,10 @@ static int __init cgroup_memory(char *s)
 			cgroup_memory_nokmem = true;
 		if (!strcmp(token, "nobpf"))
 			cgroup_memory_nobpf = true;
+#ifdef CONFIG_NUMA
+		if (!strcmp(token, "tiered_limits"))
+			cgroup_memory_tiered_limits = true;
+#endif
 	}
 	return 1;
 }
@@ -5630,6 +5644,11 @@ int __init mem_cgroup_init(void)
 	memcg_pn_cachep = KMEM_CACHE(mem_cgroup_per_node,
 				     SLAB_PANIC | SLAB_HWCACHE_ALIGN);
 
+#ifdef CONFIG_NUMA
+	if (cgroup_memory_tiered_limits)
+		static_branch_enable(&memcg_tiered_limits_key);
+#endif
+
 	return 0;
 }
 
-- 
2.53.0-Meta