In consume_stock() and refill_stock(), each per-CPU stock slot holds up to MEMCG_STOCK_HIGH (64) pages, so depleting a slot in consume_stock() (!stock_pages) or trimming a slot in refill_stock() (stock_pages > MEMCG_STOCK_HIGH) happens at most once per batch. Similarly, requests larger than MEMCG_CHARGE_BATCH, local_trylock() failure, and insufficient cached pages (stock_pages < nr_pages) are all slow paths. Without unlikely() annotations (when building without FDO, or when an FDO profile does not cover a caller), Clang 23 treats these branches as likely and inlines css_put() in consume_stock() and memcg_uncharge() in refill_stock() right into the middle of the hot path. Consequently, the fast path has to jump over the inlined slow path and jump again to the function epilogue after local_unlock(). Let's annotate the slow paths in consume_stock() and refill_stock() with unlikely() so that the compiler keeps css_put() and memcg_uncharge() out-of-line at the cold tail and lets local_unlock() and the function epilogue fall through on the fast path: Before (refill_stock): stock_pages = stock->nr_pages[i] + nr_pages; 7ab2: 43 0f b6 44 26 01 movzbl 0x1(%r14,%r12), %eax 7ab8: 01 c6 addl %eax, %esi if (stock_pages > MEMCG_STOCK_HIGH) { 7aba: 83 fe 41 cmpl $0x41, %esi 7abd: 72 1d jb 0x7adc memcg_uncharge(memcg, stock_pages - MEMCG_STOCK_LOW); # inlined 7abf: 41 89 f7 movl %esi, %r15d 7ac2: 41 83 c7 e0 addl $-0x20, %r15d 7ac6: 48 8d bb 00 01 00 00 leaq 0x100(%rbx), %rdi 7acd: 4c 89 fe movq %r15, %rsi 7ad0: e8 00 00 00 00 callq 7ad5: be 20 00 00 00 movl $0x20, %esi 7ada: eb 41 jmp 0x7b1d WRITE_ONCE(stock->nr_pages[i], stock_pages); 7adc: 43 88 74 26 01 movb %sil, 0x1(%r14,%r12) local_unlock(&memcg_stock.lock); 7ae1: 65 48 8b 05 00 00 00 00 movq %gs:(%rip), %rax 7ae9: c6 80 00 00 00 00 00 movb $0x0, (%rax) 7af0: e9 03 fe ff ff jmp 0x78f8 After (refill_stock): stock_pages = stock->nr_pages[i] + nr_pages; 7b76: 42 0f b6 44 33 01 movzbl 0x1(%rbx,%r14), %eax 7b7c: 01 c6 addl %eax, %esi if (unlikely(stock_pages > MEMCG_STOCK_HIGH)) { 7b7e: 83 fe 41 cmpl $0x41, %esi 7b81: 73 20 jae 0x7ba3 WRITE_ONCE(stock->nr_pages[i], stock_pages); 7b83: 42 88 74 33 01 movb %sil, 0x1(%rbx,%r14) local_unlock(&memcg_stock.lock); 7b88: 65 48 8b 05 00 00 00 00 movq %gs:(%rip), %rax 7b90: c6 80 00 00 00 00 00 movb $0x0, (%rax) } 7b97: 5b popq %rbx 7b98: 41 5e popq %r14 7b9a: 41 5f popq %r15 7b9c: 5d popq %rbp 7b9d: 2e e9 00 00 00 00 jmp <__x86_return_thunk> memcg_uncharge(memcg, stock_pages - MEMCG_STOCK_LOW); 7ba3: 83 c6 e0 addl $-0x20, %esi 7ba6: e8 15 f2 ff ff callq 0x6dc0 7bab: be 20 00 00 00 movl $0x20, %esi 7bb0: eb d1 jmp 0x7b83 Signed-off-by: Kuniyuki Iwashima --- v3: Mention Clang 23 and restore asm output --- mm/memcontrol.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 6666acc9cee8..a04b44656b38 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2157,8 +2157,8 @@ static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) bool ret = false; int i; - if (nr_pages > MEMCG_CHARGE_BATCH || - !local_trylock(&memcg_stock.lock)) + if (unlikely(nr_pages > MEMCG_CHARGE_BATCH || + !local_trylock(&memcg_stock.lock))) return ret; stock = this_cpu_ptr(&memcg_stock); @@ -2168,12 +2168,12 @@ static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) continue; stock_pages = stock->nr_pages[i]; - if (stock_pages < nr_pages) + if (unlikely(stock_pages < nr_pages)) break; stock_pages -= nr_pages; WRITE_ONCE(stock->nr_pages[i], stock_pages); - if (!stock_pages) { + if (unlikely(!stock_pages)) { WRITE_ONCE(stock->cached[i], NULL); css_put(&memcg->css); } @@ -2273,8 +2273,8 @@ static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) VM_WARN_ON_ONCE(mem_cgroup_is_root(memcg)); - if (nr_pages > MEMCG_CHARGE_BATCH || - !local_trylock(&memcg_stock.lock)) { + if (unlikely(nr_pages > MEMCG_CHARGE_BATCH || + !local_trylock(&memcg_stock.lock))) { /* * In case of larger than batch refill or unlikely failure to * lock the percpu memcg_stock.lock, uncharge memcg directly. @@ -2293,7 +2293,7 @@ static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) continue; stock_pages = stock->nr_pages[i] + nr_pages; - if (stock_pages > MEMCG_STOCK_HIGH) { + if (unlikely(stock_pages > MEMCG_STOCK_HIGH)) { memcg_uncharge(memcg, stock_pages - MEMCG_STOCK_LOW); stock_pages = MEMCG_STOCK_LOW; -- 2.56.0.317.g454f8302e1-goog