In consume_stock() and refill_stock(), each per-CPU stock slot holds up to MEMCG_CHARGE_BATCH (64) pages, so depleting a slot in consume_stock() (!stock_pages) or overflowing a slot in refill_stock() (stock_pages > MEMCG_CHARGE_BATCH) happens at most once per batch. Similarly, requests larger than MEMCG_CHARGE_BATCH, local_trylock() failure, and insufficient cached pages (stock_pages < nr_pages) are all slow paths. Without unlikely() annotations (when building without FDO, or when an FDO profile does not cover a caller), the compiler treats these branches as likely and inlines css_put() in consume_stock() and drain_stock() in refill_stock() right into the middle of the hot path. Consequently, the fast path has to jump over the inlined slow path and jump again to the function epilogue after local_unlock(). Let's annotate the slow paths in consume_stock() and refill_stock() with unlikely() so that the compiler keeps css_put() and drain_stock() out-of-line at the cold tail and lets local_unlock() and the function epilogue fall through on the fast path: Before (refill_stock): stock_pages = stock->nr_pages[i] + nr_pages; b242: 43 02 74 3c 01 addb 0x1(%r12,%r15), %sil WRITE_ONCE(stock->nr_pages[i], stock_pages); b247: 43 88 74 3c 01 movb %sil, 0x1(%r12,%r15) if (stock_pages > MEMCG_CHARGE_BATCH) b24c: 40 80 fe 41 cmpb $0x41, %sil b250: 72 4f jb 0xb2a1 struct mem_cgroup *old = stock->cached[i]; # drain_stock() inlined b252: 4b 8b 5c fc 08 movq 0x8(%r12,%r15,8), %rbx ... b276: e8 00 00 00 00 callq ... local_unlock(&memcg_stock.lock); b2a1: 65 48 8b 05 00 00 00 00 movq %gs:(%rip), %rax b2a9: c6 80 00 00 00 00 00 movb $0x0, (%rax) b2b0: e9 93 fd ff ff jmp 0xb048 After (refill_stock): stock_pages = stock->nr_pages[i] + nr_pages; b278: 40 02 74 03 01 addb 0x1(%rbx,%rax), %sil WRITE_ONCE(stock->nr_pages[i], stock_pages); b27d: 40 88 74 03 01 movb %sil, 0x1(%rbx,%rax) if (unlikely(stock_pages > MEMCG_CHARGE_BATCH)) b282: 40 80 fe 41 cmpb $0x41, %sil b286: 73 23 jae 0xb2ab local_unlock(&memcg_stock.lock); b288: 65 48 8b 05 00 00 00 00 movq %gs:(%rip), %rax b290: c6 80 00 00 00 00 00 movb $0x0, (%rax) } b297: 48 83 c4 08 addq $0x8, %rsp b29b: 5b popq %rbx b29c: 41 5c popq %r12 b29e: 41 5d popq %r13 b2a0: 41 5e popq %r14 b2a2: 41 5f popq %r15 b2a4: 5d popq %rbp b2a5: 2e e9 00 00 00 00 jmp <__x86_return_thunk> drain_stock(stock, i); b2ab: 48 89 df movq %rbx, %rdi b2ae: 89 c6 movl %eax, %esi b2b0: e8 2b 16 00 00 callq 0xc8e0 b2b5: eb d1 jmp 0xb288 Signed-off-by: Kuniyuki Iwashima --- mm/memcontrol.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 7c722a593cb5..cc3d78938b34 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2128,8 +2128,8 @@ static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) bool ret = false; int i; - if (nr_pages > MEMCG_CHARGE_BATCH || - !local_trylock(&memcg_stock.lock)) + if (unlikely(nr_pages > MEMCG_CHARGE_BATCH || + !local_trylock(&memcg_stock.lock))) return ret; stock = this_cpu_ptr(&memcg_stock); @@ -2139,12 +2139,12 @@ static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) continue; stock_pages = stock->nr_pages[i]; - if (stock_pages < nr_pages) + if (unlikely(stock_pages < nr_pages)) break; stock_pages -= nr_pages; WRITE_ONCE(stock->nr_pages[i], stock_pages); - if (!stock_pages) { + if (unlikely(!stock_pages)) { css_put(&memcg->css); WRITE_ONCE(stock->cached[i], NULL); } @@ -2243,8 +2243,8 @@ static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) VM_WARN_ON_ONCE(mem_cgroup_is_root(memcg)); - if (nr_pages > MEMCG_CHARGE_BATCH || - !local_trylock(&memcg_stock.lock)) { + if (unlikely(nr_pages > MEMCG_CHARGE_BATCH || + !local_trylock(&memcg_stock.lock))) { /* * In case of larger than batch refill or unlikely failure to * lock the percpu memcg_stock.lock, uncharge memcg directly. @@ -2264,7 +2264,7 @@ static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) stock_pages = stock->nr_pages[i] + nr_pages; WRITE_ONCE(stock->nr_pages[i], stock_pages); - if (stock_pages > MEMCG_CHARGE_BATCH) + if (unlikely(stock_pages > MEMCG_CHARGE_BATCH)) drain_stock(stock, i); local_unlock(&memcg_stock.lock); -- 2.56.0.212.g129ede20d9-goog