Make page_counter_try_charge() consume stock transparently to callers while preserving the same semantics as try_charge_memcg's greedy charge attempt, refill, and !allow_spinning special case. page_counter_try_charge gets two new parameters, may_batch and nr_charged. may_batch is intended to preserve the behavior of !allow_spinning scenarios in try_charge_memcg, where the goal is to charge as quickly as possible, without evicting other stock slots or making a greedy charge to refill the stock. nr_charged is used to reflect the size of the successful hierarchy charge, preserving the existing batch-based memory.high accounting for current->memcg_nr_pages_over_high. As of this patch, no page_counter has stock yet, so there are no functional changes intended. Signed-off-by: Joshua Hahn --- include/linux/page_counter.h | 4 +- kernel/cgroup/dmem.c | 2 +- mm/hugetlb_cgroup.c | 2 +- mm/memcontrol-v1.c | 3 +- mm/memcontrol.c | 8 ++-- mm/page_counter.c | 82 +++++++++++++++++++++++++++++++----- 6 files changed, 83 insertions(+), 18 deletions(-) diff --git a/include/linux/page_counter.h b/include/linux/page_counter.h index 9cb5612fe190f..a5a5a789d002c 100644 --- a/include/linux/page_counter.h +++ b/include/linux/page_counter.h @@ -98,8 +98,8 @@ long page_counter_margin(struct page_counter *counter); void page_counter_cancel(struct page_counter *counter, unsigned long nr_pages); void page_counter_charge(struct page_counter *counter, unsigned long nr_pages); bool page_counter_try_charge(struct page_counter *counter, - unsigned long nr_pages, - struct page_counter **fail); + unsigned long nr_pages, struct page_counter **fail, + bool may_batch, unsigned long *nr_charged); void page_counter_refill_stock(struct page_counter *counter, unsigned long nr_pages); void page_counter_drain_stock_fully(struct page_counter_stock_pcp *stock); diff --git a/kernel/cgroup/dmem.c b/kernel/cgroup/dmem.c index 4683f3d680226..569307aa4bd62 100644 --- a/kernel/cgroup/dmem.c +++ b/kernel/cgroup/dmem.c @@ -736,7 +736,7 @@ int dmem_cgroup_try_charge(struct dmem_cgroup_region *region, u64 size, goto err; } - if (!page_counter_try_charge(&pool->cnt, size, &fail)) { + if (!page_counter_try_charge(&pool->cnt, size, &fail, false, NULL)) { if (ret_limit_pool) { *ret_limit_pool = container_of(fail, struct dmem_cgroup_pool_state, cnt); css_get(&(*ret_limit_pool)->cs->css); diff --git a/mm/hugetlb_cgroup.c b/mm/hugetlb_cgroup.c index ecb6e0b7819a0..5b8d9f0e25535 100644 --- a/mm/hugetlb_cgroup.c +++ b/mm/hugetlb_cgroup.c @@ -274,7 +274,7 @@ static int __hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages, if (!page_counter_try_charge( __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, rsvd), - nr_pages, &counter)) { + nr_pages, &counter, false, NULL)) { ret = -ENOMEM; hugetlb_event(h_cg, idx, HUGETLB_MAX); css_put(&h_cg->css); diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c index bf2c7d53b01b1..aba9e3b851235 100644 --- a/mm/memcontrol-v1.c +++ b/mm/memcontrol-v1.c @@ -2194,7 +2194,8 @@ bool memcg1_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages, { struct page_counter *fail; - if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) { + if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail, + false, NULL)) { memcg->tcpmem_pressure = 0; return true; } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 48c475909e6bb..04ab7355c6d2d 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2708,13 +2708,14 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask, reclaim_options = MEMCG_RECLAIM_MAY_SWAP; if (do_memsw_account() && - !page_counter_try_charge(&memcg->memsw, batch, &counter)) { + !page_counter_try_charge(&memcg->memsw, batch, &counter, false, + NULL)) { mem_over_limit = mem_cgroup_from_counter(counter, memsw); reclaim_options &= ~MEMCG_RECLAIM_MAY_SWAP; goto reclaim; } - if (page_counter_try_charge(&memcg->memory, batch, &counter)) + if (page_counter_try_charge(&memcg->memory, batch, &counter, false, NULL)) goto done_restock; if (do_memsw_account()) @@ -5965,7 +5966,8 @@ int __mem_cgroup_try_charge_swap(struct folio *folio) rcu_read_unlock(); if (!mem_cgroup_is_root(memcg) && - !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) { + !page_counter_try_charge(&memcg->swap, nr_pages, &counter, false, + NULL)) { memcg_memory_event(memcg, MEMCG_SWAP_MAX); memcg_memory_event(memcg, MEMCG_SWAP_FAIL); mem_cgroup_private_id_put(memcg, nr_pages); diff --git a/mm/page_counter.c b/mm/page_counter.c index 480a447bd7265..3cd8601961673 100644 --- a/mm/page_counter.c +++ b/mm/page_counter.c @@ -136,23 +136,76 @@ void page_counter_charge(struct page_counter *counter, unsigned long nr_pages) } } +/* + * Consume the cached charge if enough nr_pages are present, otherwise return + * failure. Also return failure for charge requests larger than + * PAGE_COUNTER_STOCK_BATCH or if the local lock is already taken. + */ +static bool page_counter_consume_stock(struct page_counter *counter, + unsigned long nr_pages) +{ + struct page_counter_stock_pcp __percpu *stock = counter->stock; + struct page_counter_stock_pcp *pcp_stock; + u8 stock_pages; + bool ret = false; + int i; + + if (nr_pages > PAGE_COUNTER_STOCK_BATCH || + !local_trylock(&stock->lock)) + return false; + + pcp_stock = this_cpu_ptr(stock); + for (i = 0; i < NR_PAGE_COUNTER_STOCK; i++) { + if (counter != READ_ONCE(pcp_stock->cached[i])) + continue; + + stock_pages = READ_ONCE(pcp_stock->nr_pages[i]); + if (stock_pages >= nr_pages) { + stock_pages -= nr_pages; + WRITE_ONCE(pcp_stock->nr_pages[i], stock_pages); + if (!stock_pages) { + css_put(counter->stock_css); + WRITE_ONCE(pcp_stock->cached[i], NULL); + } + ret = true; + } + break; + } + local_unlock(&stock->lock); + + return ret; +} + /** - * page_counter_try_charge - try to hierarchically charge pages + * page_counter_try_charge - try to hierarchically charge pages using stock * @counter: counter - * @nr_pages: number of pages to charge - * @fail: points first counter to hit its limit, if any + * @nr_pages: number of pages requested + * @fail: points to the first counter to hit its limit, if any + * @may_batch: whether a stock miss may trigger a batch charge + * @nr_charged: optional; set to the hierarchy charge size on success * - * Returns %true on success, or %false and @fail if the counter or one - * of its ancestors has hit its configured limit. + * Return: %true if the request was satisfied. A failed batch charge may update + * @fail before an exact retry succeeds. */ bool page_counter_try_charge(struct page_counter *counter, - unsigned long nr_pages, - struct page_counter **fail) + unsigned long nr_pages, struct page_counter **fail, + bool may_batch, unsigned long *nr_charged) { + unsigned long charge = nr_pages; struct page_counter *c; bool protection = track_protection(counter); bool track_failcnt = counter->track_failcnt; + if (counter->stock && may_batch) + charge = max(nr_pages, PAGE_COUNTER_STOCK_BATCH); + +retry: + if (counter->stock && page_counter_consume_stock(counter, nr_pages)) { + if (nr_charged) + *nr_charged = 0; + return true; + } + for (c = counter; c; c = c->parent) { long new; /* @@ -169,9 +222,9 @@ bool page_counter_try_charge(struct page_counter *counter, * we either see the new limit or the setter sees the * counter has changed and retries. */ - new = atomic_long_add_return(nr_pages, &c->usage); + new = atomic_long_add_return(charge, &c->usage); if (new > c->max) { - atomic_long_sub(nr_pages, &c->usage); + atomic_long_sub(charge, &c->usage); /* * This is racy, but we can live with some * inaccuracy in the failcnt which is only used @@ -192,11 +245,20 @@ bool page_counter_try_charge(struct page_counter *counter, WRITE_ONCE(c->watermark, new); } } + if (charge > nr_pages) + page_counter_refill_stock(counter, charge - nr_pages); + if (nr_charged) + *nr_charged = charge; return true; failed: for (c = counter; c != *fail; c = c->parent) - page_counter_cancel(c, nr_pages); + page_counter_cancel(c, charge); + + if (charge > nr_pages) { + charge = nr_pages; + goto retry; + } return false; } -- 2.53.0-Meta