Refactor try_charge_memcg by flattening the nested memsw/memory page_counter operations to separate the logic between the two. When page_counter_try_charge is made stock-aware, this flattening makes the control flow easier to follow since each page counter now has its own success/failure paths. No functional changes intended. Acked-by: Shakeel Butt Reviewed-by: Harry Yoo (Meta) Signed-off-by: Joshua Hahn --- mm/memcontrol.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index ffbcea28a608c..66f38efb800ab 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2811,18 +2811,21 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask, batch = nr_pages; reclaim_options = MEMCG_RECLAIM_MAY_SWAP; - if (!do_memsw_account() || - page_counter_try_charge(&memcg->memsw, batch, &counter)) { - if (page_counter_try_charge(&memcg->memory, batch, &counter)) - goto done_restock; - if (do_memsw_account()) - page_counter_uncharge(&memcg->memsw, batch); - mem_over_limit = mem_cgroup_from_counter(counter, memory); - } else { + if (do_memsw_account() && + !page_counter_try_charge(&memcg->memsw, batch, &counter)) { mem_over_limit = mem_cgroup_from_counter(counter, memsw); reclaim_options &= ~MEMCG_RECLAIM_MAY_SWAP; + goto reclaim; } + if (page_counter_try_charge(&memcg->memory, batch, &counter)) + goto done_restock; + + if (do_memsw_account()) + page_counter_uncharge(&memcg->memsw, batch); + mem_over_limit = mem_cgroup_from_counter(counter, memory); + +reclaim: if (batch > nr_pages) { batch = nr_pages; goto retry; -- 2.53.0-Meta Introduce a copy of the seven-slot per-CPU stock representation from memcg in the page_counter layer. struct page_counter_stock_pcp preserves everything from struct memcg_stock_pcp, but adds a new backpointer to the base of the percpu stock, since there will be multiple percpu stock base pointers later in the series (one for memory, one for memsw). struct page_counter also gets a pointer to the base of the percpu stock, as well as a struct cgroup_subsys_state pointer to pin its owning CSS as long as the cached counter remains reachable. Let's also copy over the drain, refill, and "flush required" functions, preserving all behaviors from the memcg equivalent. In this patch they are not connected. No functional changes intended. Suggested-by: Johannes Weiner Signed-off-by: Joshua Hahn --- include/linux/page_counter.h | 31 ++++++++ mm/page_counter.c | 142 +++++++++++++++++++++++++++++++++++ 2 files changed, 173 insertions(+) diff --git a/include/linux/page_counter.h b/include/linux/page_counter.h index 2baf7a2b29b2e..cf627c5090093 100644 --- a/include/linux/page_counter.h +++ b/include/linux/page_counter.h @@ -5,8 +5,30 @@ #include #include #include +#include +#include +#include #include +/* + * The value of NR_PAGE_COUNTER_STOCK is selected to keep the cached counters + * and their nr_pages in a single cacheline. This may change in the future. + */ +#define NR_PAGE_COUNTER_STOCK 7 +#define PAGE_COUNTER_STOCK_BATCH 64UL +struct cgroup_subsys_state; +struct page_counter; +struct page_counter_stock_pcp { + local_trylock_t lock; + u8 nr_pages[NR_PAGE_COUNTER_STOCK]; + struct page_counter *cached[NR_PAGE_COUNTER_STOCK]; + + struct page_counter_stock_pcp __percpu *base; + struct work_struct work; + unsigned long flags; + u8 drain_idx; +}; + struct page_counter { /* * Make sure 'usage' does not share cacheline with any other field in @@ -41,6 +63,8 @@ struct page_counter { unsigned long high; unsigned long max; struct page_counter *parent; + struct page_counter_stock_pcp __percpu *stock; + struct cgroup_subsys_state *stock_css; } ____cacheline_internodealigned_in_smp; #if BITS_PER_LONG == 32 @@ -61,6 +85,8 @@ static inline void page_counter_init(struct page_counter *counter, counter->parent = parent; counter->protection_support = protection_support; counter->track_failcnt = false; + counter->stock = NULL; + counter->stock_css = NULL; } static inline unsigned long page_counter_read(const struct page_counter *counter) @@ -74,6 +100,11 @@ void page_counter_charge(struct page_counter *counter, unsigned long nr_pages); bool page_counter_try_charge(struct page_counter *counter, unsigned long nr_pages, struct page_counter **fail); +void page_counter_refill_stock(struct page_counter *counter, + unsigned long nr_pages); +void page_counter_drain_stock_fully(struct page_counter_stock_pcp *stock); +bool page_counter_stock_flush_required(struct page_counter_stock_pcp *stock, + struct cgroup_subsys_state *root_css); void page_counter_uncharge(struct page_counter *counter, unsigned long nr_pages); void page_counter_set_min(struct page_counter *counter, unsigned long nr_pages); void page_counter_set_low(struct page_counter *counter, unsigned long nr_pages); diff --git a/mm/page_counter.c b/mm/page_counter.c index 5293460831f80..70d76689e3212 100644 --- a/mm/page_counter.c +++ b/mm/page_counter.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -14,6 +15,14 @@ #include #include +/* + * Watermarks for a charge stock slot, in the spirit of pcp->high and + * pcp->batch: PAGE_COUNTER_STOCK_HIGH is the high watermark at which a slot is + * trimmed down to PAGE_COUNTER_STOCK_LOW rather than emptied. + */ +#define PAGE_COUNTER_STOCK_LOW (PAGE_COUNTER_STOCK_BATCH / 2) +#define PAGE_COUNTER_STOCK_HIGH PAGE_COUNTER_STOCK_BATCH + static bool track_protection(struct page_counter *c) { return c->protection_support; @@ -127,6 +136,24 @@ void page_counter_charge(struct page_counter *counter, unsigned long nr_pages) } } +static void page_counter_drain_stock(struct page_counter_stock_pcp *stock, + int i) +{ + struct page_counter *counter = stock->cached[i]; + u8 nr_pages; + + if (!counter) + return; + + nr_pages = stock->nr_pages[i]; + if (nr_pages) { + page_counter_uncharge(counter, nr_pages); + WRITE_ONCE(stock->nr_pages[i], 0); + } + WRITE_ONCE(stock->cached[i], NULL); + css_put(counter->stock_css); +} + /** * page_counter_try_charge - try to hierarchically charge pages * @counter: counter @@ -192,6 +219,121 @@ bool page_counter_try_charge(struct page_counter *counter, return false; } +void page_counter_drain_stock_fully(struct page_counter_stock_pcp *stock) +{ + int i; + + for (i = 0; i < NR_PAGE_COUNTER_STOCK; i++) + page_counter_drain_stock(stock, i); +} + +bool page_counter_stock_flush_required(struct page_counter_stock_pcp *stock, + struct cgroup_subsys_state *root_css) +{ + struct cgroup_subsys_state *css; + struct page_counter *counter; + bool flush = false; + int i; + + rcu_read_lock(); + for (i = 0; i < NR_PAGE_COUNTER_STOCK; i++) { + counter = READ_ONCE(stock->cached[i]); + if (!counter) + continue; + css = READ_ONCE(counter->stock_css); + + if (READ_ONCE(stock->nr_pages[i]) && + cgroup_is_descendant(css->cgroup, root_css->cgroup)) { + flush = true; + break; + } + } + rcu_read_unlock(); + return flush; +} + +/** + * page_counter_refill_stock - return pages to a counter's stock + * @counter: counter to return pages to + * @nr_pages: number of pages to return + * + * If the stock cannot accept the pages, uncharge them from the hierarchy. + */ +void page_counter_refill_stock(struct page_counter *counter, + unsigned long nr_pages) +{ + struct page_counter_stock_pcp __percpu *stock = READ_ONCE(counter->stock); + struct page_counter_stock_pcp *pcp_stock; + struct page_counter *cached; + unsigned int stock_pages; + int empty_slot = -1; + int i; + + /* + * nr_pages[] is a u8 and a slot is capped at PAGE_COUNTER_STOCK_HIGH. + * Raising PAGE_COUNTER_STOCK_BATCH beyond 127 would need careful + * handling of nr_pages[] in struct page_counter_stock_pcp. + */ + BUILD_BUG_ON(PAGE_COUNTER_STOCK_BATCH > S8_MAX); + BUILD_BUG_ON(PAGE_COUNTER_STOCK_HIGH > U8_MAX); + + if (unlikely(!stock || nr_pages > PAGE_COUNTER_STOCK_BATCH || + !local_trylock(&stock->lock))) { + /* + * For a larger-than-batch refill or an unlikely failure to lock + * the per-CPU stock, uncharge the hierarchy directly. + */ + page_counter_uncharge(counter, nr_pages); + return; + } + + pcp_stock = this_cpu_ptr(stock); + for (i = 0; i < NR_PAGE_COUNTER_STOCK; i++) { + cached = pcp_stock->cached[i]; + if (!cached && empty_slot == -1) + empty_slot = i; + if (counter != cached) + continue; + + stock_pages = pcp_stock->nr_pages[i] + nr_pages; + if (unlikely(stock_pages > PAGE_COUNTER_STOCK_HIGH)) { + WRITE_ONCE(pcp_stock->nr_pages[i], PAGE_COUNTER_STOCK_LOW); + stock_pages -= PAGE_COUNTER_STOCK_LOW; + } else { + WRITE_ONCE(pcp_stock->nr_pages[i], stock_pages); + stock_pages = 0; + } + local_unlock(&stock->lock); + + if (unlikely(stock_pages)) + page_counter_uncharge(counter, stock_pages); + return; + } + + cached = NULL; + stock_pages = 0; + + i = empty_slot; + if (i == -1) { + i = pcp_stock->drain_idx++; + if (pcp_stock->drain_idx == NR_PAGE_COUNTER_STOCK) + pcp_stock->drain_idx = 0; + + cached = pcp_stock->cached[i]; + stock_pages = pcp_stock->nr_pages[i]; + } + css_get(counter->stock_css); + WRITE_ONCE(pcp_stock->cached[i], counter); + WRITE_ONCE(pcp_stock->nr_pages[i], nr_pages); + local_unlock(&stock->lock); + + if (unlikely(cached)) { + if (stock_pages) + page_counter_uncharge(cached, stock_pages); + css_put(cached->stock_css); + } +} + /** * page_counter_uncharge - hierarchically uncharge pages * @counter: counter -- 2.53.0-Meta Make page_counter_try_charge() consume stock transparently to callers while preserving the same semantics as try_charge_memcg's greedy charge attempt, refill, and !allow_spinning special case. page_counter_try_charge gets two new parameters, may_batch and nr_charged. may_batch is intended to preserve the behavior of !allow_spinning scenarios in try_charge_memcg, where the goal is to charge as quickly as possible, without evicting other stock slots or making a greedy charge to refill the stock. nr_charged is used to reflect the size of the successful hierarchy charge, preserving the existing batch-based memory.high accounting for current->memcg_nr_pages_over_high. As with refill_stock() today, the full charge is reported even if the excess is later uncharged because the stock could not accept it. This only affects memory.high throttling heuristics and never fails a charge. So this preserves the existing accounting heuristic. Accounting for credits returned by refill would be a separate behavioral change which may be addressed in a future follow-up. As of this patch, no page_counter has stock yet, so there are no functional changes intended. Signed-off-by: Joshua Hahn --- include/linux/page_counter.h | 4 +- kernel/cgroup/dmem.c | 2 +- mm/hugetlb_cgroup.c | 2 +- mm/memcontrol-v1.c | 3 +- mm/memcontrol.c | 8 ++-- mm/page_counter.c | 85 +++++++++++++++++++++++++++++++----- 6 files changed, 86 insertions(+), 18 deletions(-) diff --git a/include/linux/page_counter.h b/include/linux/page_counter.h index cf627c5090093..58532259e7f59 100644 --- a/include/linux/page_counter.h +++ b/include/linux/page_counter.h @@ -98,8 +98,8 @@ long page_counter_margin(const struct page_counter *counter); void page_counter_cancel(struct page_counter *counter, unsigned long nr_pages); void page_counter_charge(struct page_counter *counter, unsigned long nr_pages); bool page_counter_try_charge(struct page_counter *counter, - unsigned long nr_pages, - struct page_counter **fail); + unsigned long nr_pages, struct page_counter **fail, + bool may_batch, unsigned long *nr_charged); void page_counter_refill_stock(struct page_counter *counter, unsigned long nr_pages); void page_counter_drain_stock_fully(struct page_counter_stock_pcp *stock); diff --git a/kernel/cgroup/dmem.c b/kernel/cgroup/dmem.c index d49ed5d4a9237..dd325e9413fb5 100644 --- a/kernel/cgroup/dmem.c +++ b/kernel/cgroup/dmem.c @@ -736,7 +736,7 @@ int dmem_cgroup_try_charge(struct dmem_cgroup_region *region, u64 size, goto err; } - if (!page_counter_try_charge(&pool->cnt, size, &fail)) { + if (!page_counter_try_charge(&pool->cnt, size, &fail, false, NULL)) { if (ret_limit_pool) { *ret_limit_pool = container_of(fail, struct dmem_cgroup_pool_state, cnt); css_get(&(*ret_limit_pool)->cs->css); diff --git a/mm/hugetlb_cgroup.c b/mm/hugetlb_cgroup.c index 3fb41311e4c7d..bbc2dd3fa3c97 100644 --- a/mm/hugetlb_cgroup.c +++ b/mm/hugetlb_cgroup.c @@ -304,7 +304,7 @@ static int __hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages, if (!page_counter_try_charge( __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, rsvd), - nr_pages, &counter)) { + nr_pages, &counter, false, NULL)) { ret = -ENOMEM; hugetlb_event(h_cg, idx, HUGETLB_MAX); css_put(&h_cg->css); diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c index b7f2868885071..a146e54c6f9f7 100644 --- a/mm/memcontrol-v1.c +++ b/mm/memcontrol-v1.c @@ -2193,7 +2193,8 @@ bool memcg1_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages, { struct page_counter *fail; - if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) { + if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail, + false, NULL)) { memcg->tcpmem_pressure = 0; return true; } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 66f38efb800ab..311bc2fa644a3 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2812,13 +2812,14 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask, reclaim_options = MEMCG_RECLAIM_MAY_SWAP; if (do_memsw_account() && - !page_counter_try_charge(&memcg->memsw, batch, &counter)) { + !page_counter_try_charge(&memcg->memsw, batch, &counter, false, + NULL)) { mem_over_limit = mem_cgroup_from_counter(counter, memsw); reclaim_options &= ~MEMCG_RECLAIM_MAY_SWAP; goto reclaim; } - if (page_counter_try_charge(&memcg->memory, batch, &counter)) + if (page_counter_try_charge(&memcg->memory, batch, &counter, false, NULL)) goto done_restock; if (do_memsw_account()) @@ -6051,7 +6052,8 @@ int __mem_cgroup_try_charge_swap(struct folio *folio) private_id = mem_cgroup_private_id_get(memcg, nr_pages); if (!mem_cgroup_private_id_is_root(private_id) && - !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) { + !page_counter_try_charge(&memcg->swap, nr_pages, &counter, + false, NULL)) { memcg_memory_event(memcg, MEMCG_SWAP_MAX); memcg_memory_event(memcg, MEMCG_SWAP_FAIL); mem_cgroup_private_id_put(private_id, nr_pages); diff --git a/mm/page_counter.c b/mm/page_counter.c index 70d76689e3212..e1feb2bd1e225 100644 --- a/mm/page_counter.c +++ b/mm/page_counter.c @@ -154,23 +154,79 @@ static void page_counter_drain_stock(struct page_counter_stock_pcp *stock, css_put(counter->stock_css); } +/* + * Consume the cached charge if enough nr_pages are present, otherwise return + * failure. Also return failure for charge requests larger than + * PAGE_COUNTER_STOCK_BATCH or if the local lock is already taken. + */ +static bool page_counter_consume_stock(struct page_counter *counter, + unsigned long nr_pages) +{ + struct page_counter_stock_pcp __percpu *stock = READ_ONCE(counter->stock); + struct page_counter_stock_pcp *pcp_stock; + u8 stock_pages; + bool ret = false; + int i; + + if (!stock || unlikely(nr_pages > PAGE_COUNTER_STOCK_BATCH || + !local_trylock(&stock->lock))) + return false; + + pcp_stock = this_cpu_ptr(stock); + for (i = 0; i < NR_PAGE_COUNTER_STOCK; i++) { + struct page_counter *cached = pcp_stock->cached[i]; + + if (counter != cached) + continue; + + stock_pages = pcp_stock->nr_pages[i]; + if (unlikely(stock_pages < nr_pages)) + break; + + stock_pages -= nr_pages; + WRITE_ONCE(pcp_stock->nr_pages[i], stock_pages); + if (unlikely(!stock_pages)) { + WRITE_ONCE(pcp_stock->cached[i], NULL); + css_put(counter->stock_css); + } + ret = true; + break; + } + local_unlock(&stock->lock); + + return ret; +} + /** - * page_counter_try_charge - try to hierarchically charge pages + * page_counter_try_charge - try to hierarchically charge pages using stock * @counter: counter - * @nr_pages: number of pages to charge - * @fail: points first counter to hit its limit, if any + * @nr_pages: number of pages requested + * @fail: points to the first counter to hit its limit, if any + * @may_batch: whether a stock miss may trigger a batch charge + * @nr_charged: optional; set to the hierarchy charge size on success * - * Returns %true on success, or %false and @fail if the counter or one - * of its ancestors has hit its configured limit. + * Return: %true if the request was satisfied. A failed batch charge may update + * @fail before an exact retry succeeds. */ bool page_counter_try_charge(struct page_counter *counter, - unsigned long nr_pages, - struct page_counter **fail) + unsigned long nr_pages, struct page_counter **fail, + bool may_batch, unsigned long *nr_charged) { + unsigned long charge = nr_pages; struct page_counter *c; bool protection = track_protection(counter); bool track_failcnt = counter->track_failcnt; + if (READ_ONCE(counter->stock) && may_batch) + charge = max(nr_pages, PAGE_COUNTER_STOCK_BATCH); + +retry: + if (page_counter_consume_stock(counter, nr_pages)) { + if (nr_charged) + *nr_charged = 0; + return true; + } + for (c = counter; c; c = c->parent) { long new; /* @@ -187,9 +243,9 @@ bool page_counter_try_charge(struct page_counter *counter, * we either see the new limit or the setter sees the * counter has changed and retries. */ - new = atomic_long_add_return(nr_pages, &c->usage); + new = atomic_long_add_return(charge, &c->usage); if (new > c->max) { - atomic_long_sub(nr_pages, &c->usage); + atomic_long_sub(charge, &c->usage); /* * This is racy, but we can live with some * inaccuracy in the failcnt which is only used @@ -210,11 +266,20 @@ bool page_counter_try_charge(struct page_counter *counter, WRITE_ONCE(c->watermark, new); } } + if (charge > nr_pages) + page_counter_refill_stock(counter, charge - nr_pages); + if (nr_charged) + *nr_charged = charge; return true; failed: for (c = counter; c != *fail; c = c->parent) - page_counter_cancel(c, nr_pages); + page_counter_cancel(c, charge); + + if (charge > nr_pages) { + charge = nr_pages; + goto retry; + } return false; } -- 2.53.0-Meta PF_MEMALLOC charges are currently forced right after consume_stock(), before trying the limits, so that they skip memory.high and the high_work worker can't queue itself from its own reclaim. A later patch in this series moves the memory stock into the page counter, so that the stock is consumed inside page_counter_try_charge(). That leaves no place between consuming the stock and trying the limits for the PF_MEMALLOC check. Prepare for this by checking PF_MEMALLOC on both outcomes of the page counter charge instead: skip memory.high on success, and force the charge on failure. These charges still do not batch, so the resulting usage is the same. The differences are that a PF_MEMALLOC charge that hits a limit is counted in the cgroup-v1 failcnt again, and its MEMCG_MAX event goes to the cgroup whose counter failed, as for any other charge. Suggested-by: Shakeel Butt Signed-off-by: Joshua Hahn --- mm/memcontrol.c | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 311bc2fa644a3..75b915febb06b 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2791,23 +2791,8 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask, if (consume_stock(memcg, nr_pages)) return ret; - /* - * Reclaim may need memory to free memory. Let it go over the - * limits for a while rather than fail or OOM kill. Doing this - * before trying the limits also skips memory.high, so the - * high_work worker can't queue itself from its own reclaim. - */ - if (unlikely(current->flags & PF_MEMALLOC)) { - /* Find the cgroup this charge puts over its limit, if any. */ - for (mem_over_limit = memcg; mem_over_limit; - mem_over_limit = parent_mem_cgroup(mem_over_limit)) - if (mem_cgroup_margin(mem_over_limit) < nr_pages) - break; - goto force; - } - - if (!allow_spinning) - /* Avoid the refill and flush of the older stock */ + /* Charges from reclaim or non-spinning contexts don't batch. */ + if (!allow_spinning || (current->flags & PF_MEMALLOC)) batch = nr_pages; reclaim_options = MEMCG_RECLAIM_MAY_SWAP; @@ -2832,6 +2817,13 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask, goto retry; } + /* + * Reclaim may need memory to free memory. Let it go over the + * limits for a while rather than fail or OOM kill. + */ + if (unlikely(current->flags & PF_MEMALLOC)) + goto force; + if (unlikely(task_in_memcg_oom(current))) goto nomem; @@ -2927,9 +2919,9 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask, out: /* * Don't forget to raise a MEMCG_MAX event for forced or rejected - * requests. mem_over_limit is NULL if a PF_MEMALLOC charge fits. + * requests. */ - if (!raised_max_event && mem_over_limit) + if (!raised_max_event) __memcg_memory_event(mem_over_limit, MEMCG_MAX, allow_spinning); return ret; @@ -2938,6 +2930,13 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask, if (batch > nr_pages) refill_stock(memcg, batch - nr_pages); + /* + * Skip memory.high for charges from reclaim, so the high_work + * worker can't queue itself from its own reclaim. + */ + if (unlikely(current->flags & PF_MEMALLOC)) + return ret; + memcg_enforce_high(memcg, batch, gfp_mask); return ret; } -- 2.53.0-Meta Transition memcg to use the page_counter_stock for the memory page_counter instead of relying on a memcg-wide stock. One aspect that remains non-transparent to memcg is the uncharge path. This is intentional, as the caller is responsible for managing the batching. Cacheable releases refill the stock, while already-batched frees (i.e. uncharge_gather), rollbacks, and accounting transfers uncharge the hierarchy directly. The refill helper itself already falls back to a raw uncharge when the stock cannot accept the pages anyways. Because the memory and memsw counters no longer share one stock, their raw values can temporarily diverge, and the raw memory counter can exceed the raw memsw counter. Preserve the legacy user-visible memory <= memory+swap relationship by reporting the smaller of the two raw values for memory.usage_in_bytes. We accept this fuzziness over breaking the invariant, as usage_in_bytes has always been a fuzzy value due to stock charges being included. Moreover, the raw counters are upper bounds on charged memory, so the reported value stays within that fuzz. One final nuance is that with divergent charges now, it is possible for memory to fail first even when memory and memsw have equal limits. When this happens, make sure reclaim doesn't swap, as if memsw failed. Factor out this logic into memcg_charge_reclaim_options() for clarity. With this transition, remove all memcg code that is no longer used. After all of this, there should be no functional change for cgroup v2 users, and the cgroup-v1 equal-limit no-swap behavior is preserved. All v2 behaviors from memcg are preserved, just moved from memcg to page_counter code, so that future work can introduce additional page_counters without removing the fast path. Explicitly, the preserved behaviors are: - 7-slot stock - drain policy works locally and remotely through the memcg_wq - check whether a stock requires flushing before taking action - exact charging for non-spinning callers - report hierarchy growth for memory.high overage accounting As of this patch, this leaves memsw un-stocked and always taking the slow path (raw hierarchy charge). The next patch will make memsw stocked, which will close the fast path gap for legacy cgroup users. Suggested-by: Johannes Weiner Signed-off-by: Joshua Hahn --- mm/memcontrol-v1.c | 14 +- mm/memcontrol.c | 322 ++++++++++----------------------------------- 2 files changed, 84 insertions(+), 252 deletions(-) diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c index a146e54c6f9f7..2f29315b69fc3 100644 --- a/mm/memcontrol-v1.c +++ b/mm/memcontrol-v1.c @@ -122,10 +122,18 @@ static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap) if (swap) val += total_swap_pages - get_nr_swap_pages(); } else { - if (!swap) - val = page_counter_read(&memcg->memory); - else + if (!swap) { + /* + * Both counters remain upper bounds on charged memory, + * but cached charges can make their raw values drift. + * Use the smaller value to preserve the user-visible + * memory <= memsw invariant. + */ + val = min(page_counter_read(&memcg->memory), + page_counter_read(&memcg->memsw)); + } else { val = page_counter_read(&memcg->memsw); + } } return val; } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 75b915febb06b..d03d64969548d 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2050,33 +2050,11 @@ void mem_cgroup_print_oom_group(struct mem_cgroup *memcg) pr_cont(" are going to be killed due to memory.oom.group set\n"); } -/* - * The value of NR_MEMCG_STOCK is selected to keep the cached memcgs and their - * nr_pages in a single cacheline. This may change in future. - */ -#define NR_MEMCG_STOCK 7 - -/* - * Watermarks for a charge stock slot, in the spirit of pcp->high and - * pcp->batch: MEMCG_STOCK_HIGH is the high watermark at which a slot is - * trimmed, and it is trimmed down to MEMCG_STOCK_LOW rather than emptied. - */ -#define MEMCG_STOCK_LOW (MEMCG_CHARGE_BATCH / 2) -#define MEMCG_STOCK_HIGH (MEMCG_CHARGE_BATCH) - #define FLUSHING_CACHED_CHARGE 0 -struct memcg_stock_pcp { - local_trylock_t lock; - uint8_t nr_pages[NR_MEMCG_STOCK]; - struct mem_cgroup *cached[NR_MEMCG_STOCK]; - - struct work_struct work; - unsigned long flags; - uint8_t drain_idx; -}; -static DEFINE_PER_CPU_ALIGNED(struct memcg_stock_pcp, memcg_stock) = { +static DEFINE_PER_CPU_ALIGNED(struct page_counter_stock_pcp, memory_stock) = { .lock = INIT_LOCAL_TRYLOCK(lock), + .base = &memory_stock, }; /* @@ -2126,54 +2104,6 @@ static void drain_obj_stock(struct obj_stock_pcp *stock); static bool obj_stock_flush_required(struct obj_stock_pcp *stock, struct mem_cgroup *root_memcg); -/** - * consume_stock: Try to consume stocked charge on this cpu. - * @memcg: memcg to consume from. - * @nr_pages: how many pages to charge. - * - * Consume the cached charge if enough nr_pages are present otherwise return - * failure. Also return failure for charge request larger than - * MEMCG_CHARGE_BATCH or if the local lock is already taken. - * - * returns true if successful, false otherwise. - */ -static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) -{ - struct memcg_stock_pcp *stock; - uint8_t stock_pages; - bool ret = false; - int i; - - if (unlikely(nr_pages > MEMCG_CHARGE_BATCH || - !local_trylock(&memcg_stock.lock))) - return ret; - - stock = this_cpu_ptr(&memcg_stock); - - for (i = 0; i < NR_MEMCG_STOCK; ++i) { - if (memcg != stock->cached[i]) - continue; - - stock_pages = stock->nr_pages[i]; - if (unlikely(stock_pages < nr_pages)) - break; - - stock_pages -= nr_pages; - WRITE_ONCE(stock->nr_pages[i], stock_pages); - if (unlikely(!stock_pages)) { - WRITE_ONCE(stock->cached[i], NULL); - css_put(&memcg->css); - } - - ret = true; - break; - } - - local_unlock(&memcg_stock.lock); - - return ret; -} - static void memcg_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages) { page_counter_uncharge(&memcg->memory, nr_pages); @@ -2181,49 +2111,22 @@ static void memcg_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages) page_counter_uncharge(&memcg->memsw, nr_pages); } -/* - * Returns stocks cached in percpu and reset cached information. - */ -static void drain_stock(struct memcg_stock_pcp *stock, int i) -{ - struct mem_cgroup *old = stock->cached[i]; - uint8_t stock_pages; - - if (!old) - return; - - stock_pages = stock->nr_pages[i]; - if (stock_pages) { - memcg_uncharge(old, stock_pages); - WRITE_ONCE(stock->nr_pages[i], 0); - } - - WRITE_ONCE(stock->cached[i], NULL); - css_put(&old->css); -} - -static void drain_stock_fully(struct memcg_stock_pcp *stock) -{ - int i; - - for (i = 0; i < NR_MEMCG_STOCK; ++i) - drain_stock(stock, i); -} - -static void drain_local_memcg_stock(struct work_struct *dummy) +static void drain_local_stock(struct work_struct *work) { - struct memcg_stock_pcp *stock; + struct page_counter_stock_pcp *pcp_stock; + struct page_counter_stock_pcp __percpu *stock; if (WARN_ONCE(!in_task(), "drain in non-task context")) return; - local_lock(&memcg_stock.lock); + stock = container_of(work, struct page_counter_stock_pcp, work)->base; + local_lock(&stock->lock); - stock = this_cpu_ptr(&memcg_stock); - drain_stock_fully(stock); - clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags); + pcp_stock = this_cpu_ptr(stock); + page_counter_drain_stock_fully(pcp_stock); + clear_bit(FLUSHING_CACHED_CHARGE, &pcp_stock->flags); - local_unlock(&memcg_stock.lock); + local_unlock(&stock->lock); } static void drain_local_obj_stock(struct work_struct *dummy) @@ -2242,109 +2145,6 @@ static void drain_local_obj_stock(struct work_struct *dummy) local_unlock(&obj_stock.lock); } -static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) -{ - struct memcg_stock_pcp *stock; - struct mem_cgroup *cached; - unsigned int stock_pages; - int empty_slot = -1; - int i; - - /* - * nr_pages[] is a uint8_t and a slot's count is capped at - * MEMCG_STOCK_HIGH. Raising MEMCG_CHARGE_BATCH beyond 127 would need - * more careful handling of nr_pages[] in struct memcg_stock_pcp. - */ - BUILD_BUG_ON(MEMCG_CHARGE_BATCH > S8_MAX); - BUILD_BUG_ON(MEMCG_STOCK_HIGH > U8_MAX); - - VM_WARN_ON_ONCE(mem_cgroup_is_root(memcg)); - - if (unlikely(nr_pages > MEMCG_CHARGE_BATCH || - !local_trylock(&memcg_stock.lock))) { - /* - * In case of larger than batch refill or unlikely failure to - * lock the percpu memcg_stock.lock, uncharge memcg directly. - */ - memcg_uncharge(memcg, nr_pages); - return; - } - - stock = this_cpu_ptr(&memcg_stock); - for (i = 0; i < NR_MEMCG_STOCK; ++i) { - cached = stock->cached[i]; - if (!cached && empty_slot == -1) - empty_slot = i; - - if (memcg != cached) - continue; - - stock_pages = stock->nr_pages[i] + nr_pages; - if (unlikely(stock_pages > MEMCG_STOCK_HIGH)) { - WRITE_ONCE(stock->nr_pages[i], MEMCG_STOCK_LOW); - stock_pages -= MEMCG_STOCK_LOW; - } else { - WRITE_ONCE(stock->nr_pages[i], stock_pages); - stock_pages = 0; - } - - local_unlock(&memcg_stock.lock); - - if (unlikely(stock_pages)) - memcg_uncharge(memcg, stock_pages); - return; - } - - cached = NULL; - stock_pages = 0; - - i = empty_slot; - if (i == -1) { - i = stock->drain_idx++; - if (stock->drain_idx == NR_MEMCG_STOCK) - stock->drain_idx = 0; - - cached = stock->cached[i]; - stock_pages = stock->nr_pages[i]; - } - - css_get(&memcg->css); - WRITE_ONCE(stock->cached[i], memcg); - WRITE_ONCE(stock->nr_pages[i], nr_pages); - - local_unlock(&memcg_stock.lock); - - if (unlikely(cached)) { - if (stock_pages) - memcg_uncharge(cached, stock_pages); - - css_put(&cached->css); - } -} - -static bool is_memcg_drain_needed(struct memcg_stock_pcp *stock, - struct mem_cgroup *root_memcg) -{ - struct mem_cgroup *memcg; - bool flush = false; - int i; - - rcu_read_lock(); - for (i = 0; i < NR_MEMCG_STOCK; ++i) { - memcg = READ_ONCE(stock->cached[i]); - if (!memcg) - continue; - - if (READ_ONCE(stock->nr_pages[i]) && - mem_cgroup_is_descendant(memcg, root_memcg)) { - flush = true; - break; - } - } - rcu_read_unlock(); - return flush; -} - static bool schedule_drain_work(int cpu, struct work_struct *work) { /* @@ -2376,23 +2176,25 @@ void drain_all_stock(struct mem_cgroup *root_memcg) * Notify other cpus that system-wide "drain" is running * We do not care about races with the cpu hotplug because cpu down * as well as workers from this path always operate on the local - * per-cpu data. CPU up doesn't touch memcg_stock at all. + * per-cpu data. CPU up doesn't touch the stocks at all. */ migrate_disable(); curcpu = smp_processor_id(); for_each_online_cpu(cpu) { - struct memcg_stock_pcp *memcg_st = &per_cpu(memcg_stock, cpu); + struct page_counter_stock_pcp *memory_st = + per_cpu_ptr(&memory_stock, cpu); struct obj_stock_pcp *obj_st = &per_cpu(obj_stock, cpu); - if (!test_bit(FLUSHING_CACHED_CHARGE, &memcg_st->flags) && - is_memcg_drain_needed(memcg_st, root_memcg) && + if (!test_bit(FLUSHING_CACHED_CHARGE, &memory_st->flags) && + page_counter_stock_flush_required(memory_st, + &root_memcg->css) && !test_and_set_bit(FLUSHING_CACHED_CHARGE, - &memcg_st->flags)) { + &memory_st->flags)) { if (cpu == curcpu) - drain_local_memcg_stock(&memcg_st->work); - else if (!schedule_drain_work(cpu, &memcg_st->work)) + drain_local_stock(&memory_st->work); + else if (!schedule_drain_work(cpu, &memory_st->work)) clear_bit(FLUSHING_CACHED_CHARGE, - &memcg_st->flags); + &memory_st->flags); } if (!test_bit(FLUSHING_CACHED_CHARGE, &obj_st->flags) && @@ -2412,12 +2214,13 @@ void drain_all_stock(struct mem_cgroup *root_memcg) static int memcg_hotplug_cpu_dead(unsigned int cpu) { - struct memcg_stock_pcp *memcg_st = &per_cpu(memcg_stock, cpu); + struct page_counter_stock_pcp *memory_st = + per_cpu_ptr(&memory_stock, cpu); struct obj_stock_pcp *obj_st = &per_cpu(obj_stock, cpu); /* no need for the local lock */ drain_obj_stock(obj_st); - drain_stock_fully(memcg_st); + page_counter_drain_stock_fully(memory_st); /* * A drain work queued before the CPU went away is executed by an @@ -2425,7 +2228,7 @@ static int memcg_hotplug_cpu_dead(unsigned int cpu) * clear the flags here to make these stocks drainable again once * the CPU comes back online. */ - clear_bit(FLUSHING_CACHED_CHARGE, &memcg_st->flags); + clear_bit(FLUSHING_CACHED_CHARGE, &memory_st->flags); clear_bit(FLUSHING_CACHED_CHARGE, &obj_st->flags); return 0; @@ -2771,13 +2574,33 @@ static void memcg_enforce_high(struct mem_cgroup *memcg, unsigned int nr_pages, __mem_cgroup_handle_over_high(gfp_mask); } +static unsigned int memcg_charge_reclaim_options(struct mem_cgroup *memcg, + struct page_counter *counter) +{ + if (!do_memsw_account()) + return MEMCG_RECLAIM_MAY_SWAP; + + if (counter == &memcg->memsw) + return 0; + + /* + * With separate stocks, raw memory can exceed raw memsw, so memory can + * fail first even when equal limits leave no room for swap. Don't swap + * in that case, as if memsw had failed. + */ + if (READ_ONCE(memcg->memory.max) >= READ_ONCE(memcg->memsw.max)) + return 0; + + return MEMCG_RECLAIM_MAY_SWAP; +} + static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask, unsigned int nr_pages) { - unsigned int batch = max(MEMCG_CHARGE_BATCH, nr_pages); int nr_retries = MAX_RECLAIM_RETRIES; struct mem_cgroup *mem_over_limit; struct page_counter *counter; + unsigned long nr_charged; unsigned long nr_reclaimed; bool passed_oom = false; unsigned int reclaim_options; @@ -2785,37 +2608,31 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask, bool raised_max_event = false; unsigned long pflags; bool allow_spinning = gfpflags_allow_spinning(gfp_mask); + /* Charges from reclaim or non-spinning contexts don't batch. */ + bool may_batch = allow_spinning && !(current->flags & PF_MEMALLOC); int ret = 0; retry: - if (consume_stock(memcg, nr_pages)) - return ret; - - /* Charges from reclaim or non-spinning contexts don't batch. */ - if (!allow_spinning || (current->flags & PF_MEMALLOC)) - batch = nr_pages; - - reclaim_options = MEMCG_RECLAIM_MAY_SWAP; if (do_memsw_account() && - !page_counter_try_charge(&memcg->memsw, batch, &counter, false, + !page_counter_try_charge(&memcg->memsw, nr_pages, &counter, false, NULL)) { mem_over_limit = mem_cgroup_from_counter(counter, memsw); - reclaim_options &= ~MEMCG_RECLAIM_MAY_SWAP; goto reclaim; } - if (page_counter_try_charge(&memcg->memory, batch, &counter, false, NULL)) - goto done_restock; + if (page_counter_try_charge(&memcg->memory, nr_pages, &counter, + may_batch, &nr_charged)) + goto check_high; if (do_memsw_account()) - page_counter_uncharge(&memcg->memsw, batch); + page_counter_uncharge(&memcg->memsw, nr_pages); mem_over_limit = mem_cgroup_from_counter(counter, memory); reclaim: - if (batch > nr_pages) { - batch = nr_pages; - goto retry; - } + reclaim_options = memcg_charge_reclaim_options(mem_over_limit, counter); + + /* Do not retry speculative batch charges after the first miss. */ + may_batch = false; /* * Reclaim may need memory to free memory. Let it go over the @@ -2926,9 +2743,9 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask, return ret; -done_restock: - if (batch > nr_pages) - refill_stock(memcg, batch - nr_pages); +check_high: + if (!nr_charged) + return ret; /* * Skip memory.high for charges from reclaim, so the high_work @@ -2937,7 +2754,7 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask, if (unlikely(current->flags & PF_MEMALLOC)) return ret; - memcg_enforce_high(memcg, batch, gfp_mask); + memcg_enforce_high(memcg, nr_charged, gfp_mask); return ret; } @@ -3226,8 +3043,11 @@ static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg, account_kmem_nmi_safe(memcg, -nr_pages); memcg1_account_kmem(memcg, -nr_pages); - if (!mem_cgroup_is_root(memcg)) - refill_stock(memcg, nr_pages); + if (!mem_cgroup_is_root(memcg)) { + page_counter_refill_stock(&memcg->memory, nr_pages); + if (do_memsw_account()) + page_counter_uncharge(&memcg->memsw, nr_pages); + } css_put(&memcg->css); } @@ -4374,6 +4194,8 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX); if (parent) { page_counter_init(&memcg->memory, &parent->memory, memcg_on_dfl); + memcg->memory.stock = &memory_stock; + memcg->memory.stock_css = &memcg->css; page_counter_init(&memcg->swap, &parent->swap, false); #ifdef CONFIG_MEMCG_V1 WRITE_ONCE(memcg->swappiness, mem_cgroup_swappiness(parent)); @@ -5839,7 +5661,7 @@ void mem_cgroup_sk_uncharge(const struct sock *sk, unsigned int nr_pages) mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages); - refill_stock(memcg, nr_pages); + page_counter_refill_stock(&memcg->memory, nr_pages); } void mem_cgroup_flush_workqueue(void) @@ -5987,6 +5809,8 @@ int __init mem_cgroup_init(void) * exceed S32_MAX / PAGE_SIZE. */ BUILD_BUG_ON(MEMCG_CHARGE_BATCH > S32_MAX / PAGE_SIZE); + /* Batched page-counter charges feed memcg's memory.high accounting. */ + BUILD_BUG_ON(MEMCG_CHARGE_BATCH != PAGE_COUNTER_STOCK_BATCH); memcg_struct_check(); @@ -5997,8 +5821,8 @@ int __init mem_cgroup_init(void) WARN_ON(!memcg_wq); for_each_possible_cpu(cpu) { - INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work, - drain_local_memcg_stock); + INIT_WORK(&per_cpu_ptr(&memory_stock, cpu)->work, + drain_local_stock); INIT_WORK(&per_cpu_ptr(&obj_stock, cpu)->work, drain_local_obj_stock); } -- 2.53.0-Meta Attach legacy memsw page counters to their own seven-slot per-CPU stock. Charge, refill, drain, and hotplug handling now operate on the memory and memsw banks independently. Factor the common drain scheduling into schedule_stock_drain() now that both stocks use it. If the memory charge fails, roll back only nr_pages from memsw. Any batch surplus the memsw charge already placed in the memsw stock stays cached, like any other stock slot. The separate banks can hit, contend, evict, and drain independently, so their raw counters can temporarily drift by their cached amounts. The previous patch preserves the user-visible cgroup-v1 memory <= memory + swap invariant by reporting memory.usage_in_bytes as the smaller raw value. Signed-off-by: Joshua Hahn --- mm/memcontrol.c | 64 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 16 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index d03d64969548d..aac987ebaab6f 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2057,6 +2057,13 @@ static DEFINE_PER_CPU_ALIGNED(struct page_counter_stock_pcp, memory_stock) = { .base = &memory_stock, }; +#ifdef CONFIG_MEMCG_V1 +static DEFINE_PER_CPU_ALIGNED(struct page_counter_stock_pcp, memsw_stock) = { + .lock = INIT_LOCAL_TRYLOCK(lock), + .base = &memsw_stock, +}; +#endif + /* * NR_OBJ_STOCK is sized so the entire hot path of obj_stock_pcp * (lock, accounting metadata, nr_bytes[] and cached[]) fits within a @@ -2161,12 +2168,30 @@ static bool schedule_drain_work(int cpu, struct work_struct *work) return true; } +static void schedule_stock_drain(struct page_counter_stock_pcp __percpu *stock, + struct cgroup_subsys_state *root_css, + int cpu, int curcpu) +{ + struct page_counter_stock_pcp *pcp_stock = per_cpu_ptr(stock, cpu); + + if (test_bit(FLUSHING_CACHED_CHARGE, &pcp_stock->flags) || + !page_counter_stock_flush_required(pcp_stock, root_css) || + test_and_set_bit(FLUSHING_CACHED_CHARGE, &pcp_stock->flags)) + return; + + if (cpu == curcpu) + drain_local_stock(&pcp_stock->work); + else if (!schedule_drain_work(cpu, &pcp_stock->work)) + clear_bit(FLUSHING_CACHED_CHARGE, &pcp_stock->flags); +} + /* * Drains all per-CPU charge caches for given root_memcg resp. subtree * of the hierarchy under it. */ void drain_all_stock(struct mem_cgroup *root_memcg) { + struct cgroup_subsys_state *root_css = &root_memcg->css; int cpu, curcpu; /* If someone's already draining, avoid adding running more workers. */ @@ -2181,21 +2206,13 @@ void drain_all_stock(struct mem_cgroup *root_memcg) migrate_disable(); curcpu = smp_processor_id(); for_each_online_cpu(cpu) { - struct page_counter_stock_pcp *memory_st = - per_cpu_ptr(&memory_stock, cpu); struct obj_stock_pcp *obj_st = &per_cpu(obj_stock, cpu); - if (!test_bit(FLUSHING_CACHED_CHARGE, &memory_st->flags) && - page_counter_stock_flush_required(memory_st, - &root_memcg->css) && - !test_and_set_bit(FLUSHING_CACHED_CHARGE, - &memory_st->flags)) { - if (cpu == curcpu) - drain_local_stock(&memory_st->work); - else if (!schedule_drain_work(cpu, &memory_st->work)) - clear_bit(FLUSHING_CACHED_CHARGE, - &memory_st->flags); - } + schedule_stock_drain(&memory_stock, root_css, cpu, curcpu); +#ifdef CONFIG_MEMCG_V1 + if (do_memsw_account()) + schedule_stock_drain(&memsw_stock, root_css, cpu, curcpu); +#endif if (!test_bit(FLUSHING_CACHED_CHARGE, &obj_st->flags) && obj_stock_flush_required(obj_st, root_memcg) && @@ -2221,6 +2238,9 @@ static int memcg_hotplug_cpu_dead(unsigned int cpu) /* no need for the local lock */ drain_obj_stock(obj_st); page_counter_drain_stock_fully(memory_st); +#ifdef CONFIG_MEMCG_V1 + page_counter_drain_stock_fully(per_cpu_ptr(&memsw_stock, cpu)); +#endif /* * A drain work queued before the CPU went away is executed by an @@ -2229,6 +2249,10 @@ static int memcg_hotplug_cpu_dead(unsigned int cpu) * the CPU comes back online. */ clear_bit(FLUSHING_CACHED_CHARGE, &memory_st->flags); +#ifdef CONFIG_MEMCG_V1 + clear_bit(FLUSHING_CACHED_CHARGE, + &per_cpu_ptr(&memsw_stock, cpu)->flags); +#endif clear_bit(FLUSHING_CACHED_CHARGE, &obj_st->flags); return 0; @@ -2614,8 +2638,8 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask, retry: if (do_memsw_account() && - !page_counter_try_charge(&memcg->memsw, nr_pages, &counter, false, - NULL)) { + !page_counter_try_charge(&memcg->memsw, nr_pages, &counter, + may_batch, NULL)) { mem_over_limit = mem_cgroup_from_counter(counter, memsw); goto reclaim; } @@ -3046,7 +3070,7 @@ static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg, if (!mem_cgroup_is_root(memcg)) { page_counter_refill_stock(&memcg->memory, nr_pages); if (do_memsw_account()) - page_counter_uncharge(&memcg->memsw, nr_pages); + page_counter_refill_stock(&memcg->memsw, nr_pages); } css_put(&memcg->css); @@ -4198,6 +4222,10 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) memcg->memory.stock_css = &memcg->css; page_counter_init(&memcg->swap, &parent->swap, false); #ifdef CONFIG_MEMCG_V1 + if (!memcg_on_dfl) { + memcg->memsw.stock = &memsw_stock; + memcg->memsw.stock_css = &memcg->css; + } WRITE_ONCE(memcg->swappiness, mem_cgroup_swappiness(parent)); memcg->memory.track_failcnt = !memcg_on_dfl; memcg->memsw.track_failcnt = !memcg_on_dfl; @@ -5823,6 +5851,10 @@ int __init mem_cgroup_init(void) for_each_possible_cpu(cpu) { INIT_WORK(&per_cpu_ptr(&memory_stock, cpu)->work, drain_local_stock); +#ifdef CONFIG_MEMCG_V1 + INIT_WORK(&per_cpu_ptr(&memsw_stock, cpu)->work, + drain_local_stock); +#endif INIT_WORK(&per_cpu_ptr(&obj_stock, cpu)->work, drain_local_obj_stock); } -- 2.53.0-Meta In the old memcg_stock world, non-leaf cgroups in cgroup v2 were never directly charged (except for reparented objcgs), since processes can only live in leaf cgroups. This meant that stocking was exclusive to leaf cgroups. In the new page_counter_stock world, stock belongs to each page_counter. To get parity with vanilla stock, prevent stock from accumulating in page_counters once they have children. This makes sure that stock does not get stranded in the hierarchy, unable to be used by processes that actually charge. When a parent becomes a leaf again, re-enable stock. Prevent stock from accumulating in non-leaf page_counters in three ways: 1. Clear the parent's memory stock pointer to make sure the stock cannot get refilled in the future. 2. Schedule drains for non-isolated CPUs that stock the new parent. 3. CPUs that are not drained due to either isolation or racing with a refill are cleaned up lazily. The next local stock miss on that CPU scans for any stock that has been disabled, and will drain its stock. Note that this happens under the local stock lock, but this is rare. Signed-off-by: Joshua Hahn --- mm/memcontrol.c | 28 ++++++++++++++++++++++++++++ mm/page_counter.c | 8 +++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index aac987ebaab6f..a7549fd06d93e 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2185,6 +2185,22 @@ static void schedule_stock_drain(struct page_counter_stock_pcp __percpu *stock, clear_bit(FLUSHING_CACHED_CHARGE, &pcp_stock->flags); } +static void mem_cgroup_disable_stock(struct mem_cgroup *memcg) +{ + struct page_counter_stock_pcp __percpu *stock; + int cpu, curcpu; + + stock = xchg(&memcg->memory.stock, NULL); + if (!stock) + return; + + migrate_disable(); + curcpu = smp_processor_id(); + for_each_online_cpu(cpu) + schedule_stock_drain(stock, &memcg->css, cpu, curcpu); + migrate_enable(); +} + /* * Drains all per-CPU charge caches for given root_memcg resp. subtree * of the hierarchy under it. @@ -4308,6 +4324,9 @@ static int mem_cgroup_css_online(struct cgroup_subsys_state *css) */ xa_store(&mem_cgroup_private_ids, memcg->private_id, memcg->private_id_objcg, GFP_KERNEL); + if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && css->parent) + mem_cgroup_disable_stock(parent_mem_cgroup(memcg)); + return 0; free_objcg: for_each_node(nid) { @@ -4360,6 +4379,15 @@ static void mem_cgroup_css_offline(struct cgroup_subsys_state *css) drain_all_stock(memcg); mem_cgroup_private_id_kill(memcg); + + if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) { + struct mem_cgroup *parent = parent_mem_cgroup(memcg); + + if (parent && !css_is_dying(&parent->css) && + atomic_read(&parent->css.online_cnt) == 2 && + !mem_cgroup_is_root(parent)) + WRITE_ONCE(parent->memory.stock, &memory_stock); + } } static void mem_cgroup_css_released(struct cgroup_subsys_state *css) diff --git a/mm/page_counter.c b/mm/page_counter.c index e1feb2bd1e225..7a3b7cea0ff94 100644 --- a/mm/page_counter.c +++ b/mm/page_counter.c @@ -158,6 +158,7 @@ static void page_counter_drain_stock(struct page_counter_stock_pcp *stock, * Consume the cached charge if enough nr_pages are present, otherwise return * failure. Also return failure for charge requests larger than * PAGE_COUNTER_STOCK_BATCH or if the local lock is already taken. + * Drain disabled cached counters while looking up the requested counter. */ static bool page_counter_consume_stock(struct page_counter *counter, unsigned long nr_pages) @@ -176,12 +177,17 @@ static bool page_counter_consume_stock(struct page_counter *counter, for (i = 0; i < NR_PAGE_COUNTER_STOCK; i++) { struct page_counter *cached = pcp_stock->cached[i]; + /* Drain any stock that is disabled but hasn't drained yet */ + if (cached && !READ_ONCE(cached->stock)) { + page_counter_drain_stock(pcp_stock, i); + continue; + } if (counter != cached) continue; stock_pages = pcp_stock->nr_pages[i]; if (unlikely(stock_pages < nr_pages)) - break; + continue; stock_pages -= nr_pages; WRITE_ONCE(pcp_stock->nr_pages[i], stock_pages); -- 2.53.0-Meta