From: Hui Zhu mem_cgroup_below_low() and mem_cgroup_below_min() decide whether reclaim may touch a cgroup by comparing the effective threshold against usage. The comparison is fixed, so a workload that wants protection on other terms has no way to ask for it. Add low_policy and min_policy to bpf_memcg_ops. A program gets a read-only snapshot of the check: the memcg, the reclaim target, the threshold and the usage. It returns a request, and there is one so far: BPF_MEMCG_PROT_BELOW treats the cgroup as below its threshold, so reclaim skips it. The kernel compares first and asks BPF only when it found the cgroup unprotected. A check runs the policies of the cgroup and of every ancestor and ORs what they return, so a policy can only add protection, never take it away. Reclaim then treats the cgroup as below its threshold throughout, which includes counting it in memory.events. Both dispatchers share one walk setup and teardown, factored out of the high dispatcher. The setup carries the task_struct::in_bpf_memcg re-entry guard and a cgroup_on_dfl() test: a v1 cgroup has no effective array and an uninitialised cgrp->bpf.refcnt. The memory.high debt is v2-only, so a v1 memcg never reached the high dispatcher and no reachable path of it changes, but it does reach the protection checks: emin and elow stay 0 and reclaim still asks. Guard the dispatcher declarations with CONFIG_BPF_SYSCALL and add stubs so every supported configuration builds. mm/bpf_memcontrol.c is built only with struct_ops support, while CONFIG_MEMCG does not select it, which already left bpf_memcg_high_policy() undefined at link time. Signed-off-by: Hui Zhu --- include/linux/bpf_memcontrol.h | 104 ++++++++++++++++++++- include/linux/memcontrol.h | 23 ++++- mm/bpf_memcontrol.c | 162 ++++++++++++++++++++++++++------- 3 files changed, 247 insertions(+), 42 deletions(-) diff --git a/include/linux/bpf_memcontrol.h b/include/linux/bpf_memcontrol.h index 76ea5d1c3d32..53222e57ae4a 100644 --- a/include/linux/bpf_memcontrol.h +++ b/include/linux/bpf_memcontrol.h @@ -2,9 +2,9 @@ /* * BPF policy hooks for the memory controller. * - * A bpf_memcg_ops is attached to a cgroup. A charge runs the policies of - * that cgroup and of every ancestor, and the kernel combines what they - * return. BPF only picks between things the kernel already does. + * A bpf_memcg_ops is attached to a cgroup. A hook runs the policies of that + * cgroup and of every ancestor, and the kernel combines what they return. + * BPF only picks between things the kernel already does. */ #ifndef _LINUX_BPF_MEMCONTROL_H #define _LINUX_BPF_MEMCONTROL_H @@ -30,6 +30,22 @@ enum bpf_memcg_high_request { #define BPF_MEMCG_HIGH_VALID_MASK BPF_MEMCG_HIGH_DEFER_INLINE +/* + * What a policy can ask for when the kernel checks memory.low or memory.min + * protection. The kernel ORs them, so one policy cannot undo another. + */ +enum bpf_memcg_prot_request { + BPF_MEMCG_PROT_NO_OPINION = 0, + /* + * Treat the cgroup as below its threshold, so reclaim skips it. This + * only adds protection: a policy cannot expose a cgroup the kernel + * already considers below its threshold. + */ + BPF_MEMCG_PROT_BELOW = 1U << 0, +}; + +#define BPF_MEMCG_PROT_VALID_MASK BPF_MEMCG_PROT_BELOW + /* Read-only snapshot. Only values the caller already has. */ struct bpf_memcg_ctx { struct mem_cgroup *memcg; /* charged memcg */ @@ -41,6 +57,21 @@ struct bpf_memcg_ctx { u32 gfp_flags; }; +/* + * Read-only snapshot of a protection check. @threshold is the effective + * memory.low for low_policy and the effective memory.min for min_policy, and + * the kernel has already found it below @usage. + */ +struct bpf_memcg_prot_ctx { + struct mem_cgroup *memcg; /* cgroup checked */ + struct mem_cgroup *target; /* NULL for global reclaim */ + struct task_struct *task; /* current */ + u64 cgroup_id; + u64 target_cgroup_id; /* 0 if none */ + u64 threshold; /* elow or emin, pages */ + u64 usage; /* pages */ +}; + struct bpf_memcg_ops { /** * high_policy - say where memory.high should be enforced @@ -50,8 +81,35 @@ struct bpf_memcg_ops { * are dropped. */ u32 (*high_policy)(const struct bpf_memcg_ctx *ctx); + + /** + * low_policy - say whether a cgroup is below its memory.low + * @ctx: snapshot of the protection check + * + * Only runs once the kernel has found @ctx->usage above + * @ctx->threshold, so a policy can only add protection. Reclaim then + * treats the cgroup as below memory.low throughout, which includes + * counting it in memory.events. + * + * Return: bits from enum bpf_memcg_prot_request, or 0. Other bits + * are dropped. + */ + u32 (*low_policy)(const struct bpf_memcg_prot_ctx *ctx); + + /** + * min_policy - say whether a cgroup is below its memory.min + * @ctx: snapshot of the protection check + * + * Same as low_policy, for the hard memory.min protection. + * + * Return: bits from enum bpf_memcg_prot_request, or 0. Other bits + * are dropped. + */ + u32 (*min_policy)(const struct bpf_memcg_prot_ctx *ctx); }; +#if IS_ENABLED(CONFIG_BPF_SYSCALL) + /* * Run every high_policy on @memcg's cgroup and its ancestors, and return the * combined request for the caller to act on. @@ -63,4 +121,44 @@ struct bpf_memcg_ops { u32 bpf_memcg_high_policy(struct mem_cgroup *memcg, struct mem_cgroup *over_limit, gfp_t gfp_mask); +/* + * Run every low_policy or min_policy on @memcg's cgroup and its ancestors, + * and return the combined request for the caller to act on. + * + * @memcg: the memcg whose protection is being checked, never the root + * @target: the reclaim root, or NULL for global reclaim + * @elow/@emin: the effective threshold, in pages + * @usage: the current usage, in pages + */ +u32 bpf_memcg_low_policy(struct mem_cgroup *memcg, struct mem_cgroup *target, + unsigned long elow, unsigned long usage); +u32 bpf_memcg_min_policy(struct mem_cgroup *memcg, struct mem_cgroup *target, + unsigned long emin, unsigned long usage); + +#else /* CONFIG_BPF_SYSCALL */ + +/* mm/bpf_memcontrol.c is only built with struct_ops support. */ +static inline u32 bpf_memcg_high_policy(struct mem_cgroup *memcg, + struct mem_cgroup *over_limit, + gfp_t gfp_mask) +{ + return BPF_MEMCG_HIGH_NO_OPINION; +} + +static inline u32 bpf_memcg_low_policy(struct mem_cgroup *memcg, + struct mem_cgroup *target, + unsigned long elow, unsigned long usage) +{ + return BPF_MEMCG_PROT_NO_OPINION; +} + +static inline u32 bpf_memcg_min_policy(struct mem_cgroup *memcg, + struct mem_cgroup *target, + unsigned long emin, unsigned long usage) +{ + return BPF_MEMCG_PROT_NO_OPINION; +} + +#endif /* CONFIG_BPF_SYSCALL */ + #endif /* _LINUX_BPF_MEMCONTROL_H */ diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 7d1c0ce189a8..3f937be7b057 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -23,6 +23,7 @@ #include #include #include +#include struct mem_cgroup; struct obj_cgroup; @@ -621,21 +622,35 @@ static inline bool mem_cgroup_unprotected(struct mem_cgroup *target, static inline bool mem_cgroup_below_low(struct mem_cgroup *target, struct mem_cgroup *memcg) { + unsigned long elow, usage; + if (mem_cgroup_unprotected(target, memcg)) return false; - return READ_ONCE(memcg->memory.elow) >= - page_counter_read(&memcg->memory); + elow = READ_ONCE(memcg->memory.elow); + usage = page_counter_read(&memcg->memory); + if (elow >= usage) + return true; + + return bpf_memcg_low_policy(memcg, target, elow, usage) & + BPF_MEMCG_PROT_BELOW; } static inline bool mem_cgroup_below_min(struct mem_cgroup *target, struct mem_cgroup *memcg) { + unsigned long emin, usage; + if (mem_cgroup_unprotected(target, memcg)) return false; - return READ_ONCE(memcg->memory.emin) >= - page_counter_read(&memcg->memory); + emin = READ_ONCE(memcg->memory.emin); + usage = page_counter_read(&memcg->memory); + if (emin >= usage) + return true; + + return bpf_memcg_min_policy(memcg, target, emin, usage) & + BPF_MEMCG_PROT_BELOW; } int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp); diff --git a/mm/bpf_memcontrol.c b/mm/bpf_memcontrol.c index cfd0f1d443c9..999d2012abd4 100644 --- a/mm/bpf_memcontrol.c +++ b/mm/bpf_memcontrol.c @@ -255,8 +255,20 @@ static u32 high_policy_stub(const struct bpf_memcg_ctx *ctx) return BPF_MEMCG_HIGH_NO_OPINION; } +static u32 low_policy_stub(const struct bpf_memcg_prot_ctx *ctx) +{ + return BPF_MEMCG_PROT_NO_OPINION; +} + +static u32 min_policy_stub(const struct bpf_memcg_prot_ctx *ctx) +{ + return BPF_MEMCG_PROT_NO_OPINION; +} + static struct bpf_memcg_ops __bpf_memcg_ops = { .high_policy = high_policy_stub, + .low_policy = low_policy_stub, + .min_policy = min_policy_stub, }; static const struct bpf_func_proto * @@ -264,7 +276,7 @@ bpf_memcg_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { /* * The base set is all a policy needs today, and none of it sleeps. - * Anything added here must be safe from the charge path. + * Anything added here must be safe from the charge and reclaim paths. */ return bpf_base_func_proto(func_id, prog); } @@ -293,7 +305,7 @@ static int bpf_memcg_check_member(const struct btf_type *t, const struct btf_member *member, const struct bpf_prog *prog) { - /* Members run from the charge path, which cannot sleep. */ + /* Members run from the charge and reclaim paths, which cannot sleep. */ if (prog->sleepable) return -EINVAL; @@ -335,6 +347,62 @@ static struct bpf_struct_ops bpf_memcg_ops_desc = { */ }; +/* + * Prepare to run the policies attached to @memcg's cgroup and its ancestors. + * + * On success returns the cgroup with its bpf reference held, inside the RCU + * section the effective array walk needs; the caller finishes with + * bpf_memcg_walk_end(). NULL means no policy can run and the caller keeps + * the request it started with: nothing is attached anywhere, the cgroup is + * on a v1 hierarchy, this task is already inside a policy, or the cgroup is + * going. + */ +static inline struct cgroup *bpf_memcg_walk_begin(struct mem_cgroup *memcg) +{ + struct cgroup *cgrp = memcg->css.cgroup; + + if (!cgroup_bpf_enabled(CGROUP_MEMCG_OPS)) + return NULL; + + /* + * Only the default hierarchy has a cgroup_bpf, and the static key is + * global, so one policy anywhere turns this on for v1 memcgs too. A + * v1 cgroup has no effective array and an uninitialised + * cgrp->bpf.refcnt. + */ + if (!cgroup_on_dfl(cgrp)) + return NULL; + + /* + * A program can allocate and re-enter the charge path, and reclaim + * with it. Skip the nested call. This guards the callbacks only. + */ + if (current->in_bpf_memcg) + return NULL; + current->in_bpf_memcg = 1; + + rcu_read_lock_dont_migrate(); + + /* + * A memcg outlives its cgroup while it has charges, and + * cgroup_bpf_release() frees the arrays when the cgroup goes. + */ + if (!cgroup_bpf_tryget_live(cgrp)) { + rcu_read_unlock_migrate(); + current->in_bpf_memcg = 0; + return NULL; + } + + return cgrp; +} + +static inline void bpf_memcg_walk_end(struct cgroup *cgrp) +{ + cgroup_bpf_put(cgrp); + rcu_read_unlock_migrate(); + current->in_bpf_memcg = 0; +} + static void bpf_memcg_ctx_init(struct bpf_memcg_ctx *ctx, struct mem_cgroup *memcg, struct mem_cgroup *over_limit, gfp_t gfp_mask) @@ -358,36 +426,9 @@ u32 bpf_memcg_high_policy(struct mem_cgroup *memcg, u32 acc = BPF_MEMCG_HIGH_NO_OPINION; struct cgroup *cgrp; - if (!cgroup_bpf_enabled(CGROUP_MEMCG_OPS)) - return acc; - - /* - * Only the default hierarchy has a cgroup_bpf, and the static key is - * global, so one policy anywhere turns this on for v1 memcgs too. A - * v1 memcg still cannot get here, because memory.high and swap.high - * are both v2-only and so it never builds the debt that leads to this - * call. A hook on a path v1 can reach needs its own cgroup_on_dfl() - * test: a v1 cgroup has no effective array and an uninitialised - * cgrp->bpf.refcnt. - */ - cgrp = memcg->css.cgroup; - - /* - * A program can allocate and re-enter the charge path. Skip the - * nested call. This guards the callbacks only. - */ - if (current->in_bpf_memcg) + cgrp = bpf_memcg_walk_begin(memcg); + if (!cgrp) return acc; - current->in_bpf_memcg = 1; - - rcu_read_lock_dont_migrate(); - - /* - * A memcg outlives its cgroup while it has charges, and - * cgroup_bpf_release() frees the arrays when the cgroup goes. - */ - if (!cgroup_bpf_tryget_live(cgrp)) - goto out; bpf_memcg_ctx_init(&ctx, memcg, over_limit, gfp_mask); @@ -397,15 +438,66 @@ u32 bpf_memcg_high_policy(struct mem_cgroup *memcg, BPF_MEMCG_HIGH_VALID_MASK; } - cgroup_bpf_put(cgrp); -out: - rcu_read_unlock_migrate(); + bpf_memcg_walk_end(cgrp); - current->in_bpf_memcg = 0; + return acc; +} + +/* + * Run every low_policy (@is_min false) or min_policy (@is_min true) on + * @memcg's cgroup and its ancestors, and return the combined request. + */ +static u32 bpf_memcg_prot_policy(struct mem_cgroup *memcg, + struct mem_cgroup *target, + unsigned long threshold, unsigned long usage, + bool is_min) +{ + const struct bpf_prog_array_item *item; + const struct bpf_memcg_ops *ops; + struct bpf_memcg_prot_ctx ctx; + u32 acc = BPF_MEMCG_PROT_NO_OPINION; + struct cgroup *cgrp; + + cgrp = bpf_memcg_walk_begin(memcg); + if (!cgrp) + return acc; + + ctx = (struct bpf_memcg_prot_ctx) { + .memcg = memcg, + .target = target, + .task = current, + .cgroup_id = cgroup_id(cgrp), + .target_cgroup_id = target ? + cgroup_id(target->css.cgroup) : 0, + .threshold = threshold, + .usage = usage, + }; + + bpf_cgroup_struct_ops_foreach(ops, item, cgrp, CGROUP_MEMCG_OPS) { + u32 (*policy)(const struct bpf_memcg_prot_ctx *ctx); + + policy = is_min ? ops->min_policy : ops->low_policy; + if (policy) + acc |= policy(&ctx) & BPF_MEMCG_PROT_VALID_MASK; + } + + bpf_memcg_walk_end(cgrp); return acc; } +u32 bpf_memcg_low_policy(struct mem_cgroup *memcg, struct mem_cgroup *target, + unsigned long elow, unsigned long usage) +{ + return bpf_memcg_prot_policy(memcg, target, elow, usage, false); +} + +u32 bpf_memcg_min_policy(struct mem_cgroup *memcg, struct mem_cgroup *target, + unsigned long emin, unsigned long usage) +{ + return bpf_memcg_prot_policy(memcg, target, emin, usage, true); +} + static int __init bpf_memcg_ops_register(void) { /* -- 2.43.0