A memory cgroup allocates a shrinker_info_unit for every SHRINKER_UNIT_BITS ids up to shrinker_nr_max on every NUMA node when it comes online, whether or not it will ever hold objects of those shrinkers. Every superblock registers a memcg-aware shrinker, so on a host with many mounts each new cgroup pays for all of them while its tasks touch a few. A map bit is only ever set on a memcg that holds a list_lru list of that shrinker. So the point where a memcg gets such a list is the one place a unit can be installed ahead of its first use, and that is where it is installed. The rule the code now relies on: A non-root memcg that has a list on a list_lru has the unit of that list's shrinker, on every node. It holds because: - the unit is installed before the list is published; - a unit lives until the memcg is freed. So set_shrinker_bit() always finds the unit. A reader that finds none knows the memcg never held objects of that shrinker: there is nothing to walk and no deferred work. There is one case where a memcg has no list: with cgroup.memory=nokmem a list_lru is never per-memcg, but a SHRINKER_MEMCG_AWARE shrinker that also has SHRINKER_NONSLAB still gets an id. If such a shrinker keeps lists of its own and sets map bits itself, it has to call memcg_shrinker_unit_alloc() first. No shrinker in the tree does. The install runs with the gfp of the list allocation, possibly atomic, so it cannot take shrinker_mutex. A spinlock orders it against the info replacement in expand_one_shrinker_info() instead, so a unit cannot be installed into an info that is being copied out of. The root memcg keeps all its units: global reclaim keeps the deferred work of memcg-aware shrinkers there, and root has no list allocation to install them from. The units of a memcg now scale with the number of shrinkers it and its descendants hold objects of, not with shrinker_nr_max. Assisted-by: LLM Signed-off-by: Konstantin Khorenko --- include/linux/memcontrol.h | 2 + mm/list_lru.c | 6 ++ mm/shrinker.c | 159 ++++++++++++++++++++++++++++++++++++--------- 3 files changed, 138 insertions(+), 29 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 7d1c0ce189a88..a14f43c3a1b01 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -1676,6 +1676,8 @@ static inline u64 mem_cgroup_get_socket_pressure(struct mem_cgroup *memcg) #endif int alloc_shrinker_info(struct mem_cgroup *memcg); +int memcg_shrinker_unit_alloc(struct mem_cgroup *memcg, int shrinker_id, + gfp_t gfp); void free_shrinker_info(struct mem_cgroup *memcg); void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id); void reparent_shrinker_deferred(struct mem_cgroup *memcg); diff --git a/mm/list_lru.c b/mm/list_lru.c index 36662d02ff963..7247b7c081b4c 100644 --- a/mm/list_lru.c +++ b/mm/list_lru.c @@ -597,6 +597,12 @@ static int __memcg_list_lru_alloc(struct mem_cgroup *memcg, parent = parent_mem_cgroup(pos); } + if (!css_is_dying(&pos->css) && + memcg_shrinker_unit_alloc(pos, lru_shrinker_id(lru), gfp)) { + kfree(mlru); + return -ENOMEM; + } + if (!mlru) { mlru = memcg_init_list_lru_one(lru, gfp); if (!mlru) diff --git a/mm/shrinker.c b/mm/shrinker.c index a9418072a509e..79f48fb7fa261 100644 --- a/mm/shrinker.c +++ b/mm/shrinker.c @@ -13,6 +13,14 @@ DEFINE_MUTEX(shrinker_mutex); #ifdef CONFIG_MEMCG static int shrinker_nr_max; +/* + * Orders installing a unit against the pointer copy in + * expand_one_shrinker_info(). Installs run from the list_lru allocation + * path with the caller's gfp and cannot wait for shrinker_mutex; they can + * nest in the i_pages lock, which is taken from interrupts. + */ +static DEFINE_SPINLOCK(shrinker_unit_lock); + static inline int shrinker_unit_size(int nr_items) { return (DIV_ROUND_UP(nr_items, SHRINKER_UNIT_BITS) * sizeof(struct shrinker_info_unit *)); @@ -30,9 +38,6 @@ static inline void shrinker_unit_free(struct shrinker_info *info, int start) nr = DIV_ROUND_UP(info->map_nr_max, SHRINKER_UNIT_BITS); for (i = start; i < nr; i++) { - if (!unit[i]) - break; - kfree(unit[i]); unit[i] = NULL; } @@ -96,7 +101,13 @@ int alloc_shrinker_info(struct mem_cgroup *memcg) if (!info) goto err; info->map_nr_max = shrinker_nr_max; - if (shrinker_unit_alloc(info, NULL, nid)) { + /* + * Global reclaim keeps the deferred work of memcg-aware + * shrinkers in the root's units, and root has no list_lru + * allocation to install them from. + */ + if (mem_cgroup_is_root(memcg) && + shrinker_unit_alloc(info, NULL, nid)) { kvfree(info); goto err; } @@ -124,6 +135,7 @@ static int expand_one_shrinker_info(struct mem_cgroup *memcg, int new_size, { struct shrinker_info *new, *old; struct mem_cgroup_per_node *pn; + unsigned long flags; int nid; for_each_node(nid) { @@ -143,13 +155,20 @@ static int expand_one_shrinker_info(struct mem_cgroup *memcg, int new_size, new->map_nr_max = new_nr_max; - memcpy(new->unit, old->unit, old_size); - if (shrinker_unit_alloc(new, old, nid)) { + if (mem_cgroup_is_root(memcg) && + shrinker_unit_alloc(new, old, nid)) { kvfree(new); return -ENOMEM; } + /* + * A unit installed meanwhile lands in old before the copy or + * in new after the publish. + */ + spin_lock_irqsave(&shrinker_unit_lock, flags); + memcpy(new->unit, old->unit, old_size); rcu_assign_pointer(pn->shrinker_info, new); + spin_unlock_irqrestore(&shrinker_unit_lock, flags); kvfree_rcu(old, rcu); } @@ -204,21 +223,26 @@ static inline int calc_shrinker_id(int index, int offset) void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id) { - if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) { - struct shrinker_info *info; + struct shrinker_info_unit *unit; + struct shrinker_info *info; - rcu_read_lock(); - info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info); - if (!WARN_ON_ONCE(shrinker_id >= info->map_nr_max)) { - struct shrinker_info_unit *unit; - - unit = info->unit[shrinker_id_to_index(shrinker_id)]; - /* Pairs with smp mb in shrink_slab() */ - smp_mb__before_atomic(); - set_bit(shrinker_id_to_offset(shrinker_id), unit->map); - } - rcu_read_unlock(); - } + if (shrinker_id < 0 || !memcg || mem_cgroup_is_root(memcg)) + return; + + rcu_read_lock(); + info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info); + if (WARN_ON_ONCE(shrinker_id >= info->map_nr_max)) + goto unlock; + + unit = READ_ONCE(info->unit[shrinker_id_to_index(shrinker_id)]); + if (WARN_ON_ONCE(!unit)) + goto unlock; + + /* Pairs with smp mb in shrink_slab() */ + smp_mb__before_atomic(); + set_bit(shrinker_id_to_offset(shrinker_id), unit->map); +unlock: + rcu_read_unlock(); } static DEFINE_IDR(shrinker_idr); @@ -263,14 +287,16 @@ static void shrinker_memcg_remove(struct shrinker *shrinker) static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker, struct mem_cgroup *memcg) { + int offset = shrinker_id_to_offset(shrinker->id); struct shrinker_info *info; struct shrinker_info_unit *unit; long nr_deferred; rcu_read_lock(); info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info); - unit = info->unit[shrinker_id_to_index(shrinker->id)]; - nr_deferred = atomic_long_xchg(&unit->nr_deferred[shrinker_id_to_offset(shrinker->id)], 0); + unit = READ_ONCE(info->unit[shrinker_id_to_index(shrinker->id)]); + /* No unit: this memcg never held objects of this shrinker. */ + nr_deferred = unit ? atomic_long_xchg(&unit->nr_deferred[offset], 0) : 0; rcu_read_unlock(); return nr_deferred; @@ -279,20 +305,83 @@ static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker, static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker, struct mem_cgroup *memcg) { + int offset = shrinker_id_to_offset(shrinker->id); struct shrinker_info *info; struct shrinker_info_unit *unit; long nr_deferred; rcu_read_lock(); info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info); - unit = info->unit[shrinker_id_to_index(shrinker->id)]; - nr_deferred = - atomic_long_add_return(nr, &unit->nr_deferred[shrinker_id_to_offset(shrinker->id)]); + unit = READ_ONCE(info->unit[shrinker_id_to_index(shrinker->id)]); + nr_deferred = unit ? atomic_long_add_return(nr, &unit->nr_deferred[offset]) : 0; rcu_read_unlock(); return nr_deferred; } +static int shrinker_unit_install(struct mem_cgroup *memcg, int nid, int index, + gfp_t gfp) +{ + struct shrinker_info_unit *unit; + struct shrinker_info *info; + unsigned long flags; + + rcu_read_lock(); + info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info); + /* Online memcgs have an info, and ids are expanded into it first. */ + if (WARN_ON_ONCE(!info || + index >= shrinker_id_to_index(info->map_nr_max)) || + READ_ONCE(info->unit[index])) { + rcu_read_unlock(); + return 0; + } + rcu_read_unlock(); + + unit = kzalloc_node(sizeof(*unit), gfp, nid); + if (!unit) + return -ENOMEM; + + spin_lock_irqsave(&shrinker_unit_lock, flags); + /* memcg is pinned: only expand replaces info, and it holds the lock */ + info = rcu_dereference_protected(memcg->nodeinfo[nid]->shrinker_info, + lockdep_is_held(&shrinker_unit_lock)); + if (!info->unit[index]) { + /* Walkers read the pointer locklessly: publish a zeroed unit. */ + smp_store_release(&info->unit[index], unit); + unit = NULL; + } + spin_unlock_irqrestore(&shrinker_unit_lock, flags); + + kfree(unit); + return 0; +} + +/* + * Called before @memcg gets a list on a list_lru of shrinker @shrinker_id, + * from the path that allocates the list. A map bit is only ever set on a + * memcg that has the list, so set_shrinker_bit() always finds the unit. + * A shrinker that sets map bits without a list_lru, such as a NONSLAB + * one under cgroup.memory=nokmem, must call this first. + */ +int memcg_shrinker_unit_alloc(struct mem_cgroup *memcg, int shrinker_id, + gfp_t gfp) +{ + int nid, ret; + + if (shrinker_id < 0 || mem_cgroup_is_root(memcg)) + return 0; + + for_each_node(nid) { + ret = shrinker_unit_install(memcg, nid, + shrinker_id_to_index(shrinker_id), + gfp); + if (ret) + return ret; + } + + return 0; +} + void reparent_shrinker_deferred(struct mem_cgroup *memcg) { int nid, index, offset; @@ -307,8 +396,17 @@ void reparent_shrinker_deferred(struct mem_cgroup *memcg) child_info = shrinker_info_protected(memcg, nid); parent_info = shrinker_info_protected(parent, nid); for (index = 0; index < shrinker_id_to_index(child_info->map_nr_max); index++) { - child_unit = child_info->unit[index]; - parent_unit = parent_info->unit[index]; + child_unit = READ_ONCE(child_info->unit[index]); + if (!child_unit) + continue; + + if (!READ_ONCE(parent_info->unit[index])) + shrinker_unit_install(parent, nid, index, GFP_KERNEL); + parent_unit = READ_ONCE(parent_info->unit[index]); + /* Drop the debt rather than fail the cgroup teardown. */ + if (!parent_unit) + continue; + for (offset = 0; offset < SHRINKER_UNIT_BITS; offset++) { nr = atomic_long_read(&child_unit->nr_deferred[offset]); atomic_long_add(nr, &parent_unit->nr_deferred[offset]); @@ -529,10 +627,13 @@ static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid, if (index < shrinker_id_to_index(info->map_nr_max)) { struct shrinker_info_unit *unit; - unit = info->unit[index]; + unit = READ_ONCE(info->unit[index]); rcu_read_unlock(); + if (!unit) + goto next; + for_each_set_bit(offset, unit->map, SHRINKER_UNIT_BITS) { struct shrink_control sc = { .gfp_mask = gfp_mask, @@ -587,7 +688,7 @@ static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid, freed += ret; shrinker_put(shrinker); } - +next: index++; goto again; } -- 2.43.0