From: Kairui Song To enable moving file pages in folio_mark_accessed directly and lazily for MGLRU, allow updating the LRU statistic atomically without holding a lock. It may cause temporary counter underflow, which should be fine as we still follow final consistency of the counter, and it only serves as a factor for calculating the reclaim budget in vmscan. A little inaccuracy has no visible effect. Signed-off-by: Kairui Song --- include/linux/memcontrol.h | 6 +++--- include/linux/mm_inline.h | 3 +-- mm/memcontrol.c | 6 +++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 46bf724cae7a..2daf2be5d1e5 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -106,7 +106,7 @@ struct mem_cgroup_per_node { /* Written on every LRU update and on every reclaim iteration. */ __cacheline_group_begin_aligned(memcg_pn_write_hot); - long lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS]; + atomic_long_t lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS]; struct mem_cgroup_reclaim_iter iter; #ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC /* slab stats for nmi context */ @@ -926,8 +926,8 @@ unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec, struct mem_cgroup_per_node *mz; mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); - val = READ_ONCE(mz->lru_zone_size[zone_idx][lru]); - if (WARN_ON_ONCE(val < 0)) + val = atomic_long_read(&mz->lru_zone_size[zone_idx][lru]); + if (val < 0) return 0; return val; diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index ab69b9930893..597f013c8e04 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -47,11 +47,10 @@ static __always_inline void __update_lru_size(struct lruvec *lruvec, { struct pglist_data *pgdat = lruvec_pgdat(lruvec); - lockdep_assert_held(&lruvec->lru_lock); WARN_ON_ONCE(nr_pages != (int)nr_pages); mod_lruvec_state(lruvec, NR_LRU_BASE + lru, nr_pages); - __mod_zone_page_state(&pgdat->node_zones[zid], + mod_zone_page_state(&pgdat->node_zones[zid], NR_ZONE_LRU_BASE + lru, nr_pages); } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 1460cba53588..3cc14d1932f0 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1544,8 +1544,8 @@ struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio, * @zid: zone id of the accounted pages * @nr_pages: positive when adding or negative when removing * - * This function must be called under lru_lock, just before a page is added - * to or just after a page is removed from an lru list. + * This function must be called when a page is added to or removed from + * an lru list. Caller need to protect the lruvec from being freed. */ void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, int zid, long nr_pages) @@ -1556,7 +1556,7 @@ void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, return; mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); - mz->lru_zone_size[zid][lru] += nr_pages; + atomic_long_add(nr_pages, &mz->lru_zone_size[zid][lru]); } /** -- 2.55.0