On host w/ AMD EPYC 9B45 128-Core Processor (Zen 5 Turin, 2 sockets, 512 vCPUs), tcp_rr w/ 20000 flows throughput often degrades by ~50%, 32M -> 17M~27M. In the bad case, the number of involuntary context switches was extremely high by 12x (241733 -> 3093367). In perf record, 96% samples had the same pattern, where the thread was interrupted during page_counter_uncharge() by softirq, and later ksoftirqd was woken up: # perf record -C 0-15 -e sched:sched_wakeup -g \ -o /tmp/sched_wakeup.data -- sleep 1 # perf report -i /tmp/sched_wakeup.data --stdio --no-children 96.36% tcp_rr [kernel.kallsyms] [k] wake_up_process | ---wake_up_process | --96.00%--irq_exit_rcu | --95.82%--sysvec_call_function_single asm_sysvec_call_function_single | --95.58%--page_counter_uncharge mem_cgroup_sk_uncharge sock_rfree tcp_recvmsg __x64_sys_recvfrom do_syscall_64 entry_SYSCALL_64_after_hwframe # perf script -i /tmp/sched_wakeup.data | grep -A 20 "comm=ksoftirqd" tcp_rr 101952 [001] 9679.340985: sched:sched_wakeup: comm=ksoftirqd/1 pid=23 prio=120 target_cpu=001 ffffffffb64cd0cc wake_up_process+0xe6c ([kernel.kallsyms]) ffffffffb64cb034 irq_exit_rcu+0x264 ([kernel.kallsyms]) ffffffffb68ad31c sysvec_call_function_single+0xcc ([kernel.kallsyms]) ffffffffb560158a asm_sysvec_call_function_single+0x1a ([kernel.kallsyms]) ffffffffb653c4ce page_counter_uncharge+0x2e ([kernel.kallsyms]) ffffffffb6541b28 mem_cgroup_sk_uncharge+0x3c8 ([kernel.kallsyms]) ffffffffb6561383 sock_rfree+0x83 ([kernel.kallsyms]) ffffffffb6577faf tcp_recvmsg+0x2ff ([kernel.kallsyms]) ffffffffb65604d9 __x64_sys_recvfrom+0x159 ([kernel.kallsyms]) The problem is that when stock_pages exceeds MEMCG_STOCK_HIGH or all slots in struct memcg_stock_pcp are full, __refill_stock() calls memcg_uncharge() or drain_stock() while holding memcg_stock.lock. Because local_trylock() only disables preemption and leaves interrupts and softirqs enabled, an RPS IPI arriving while __refill_stock() stalls on page_counter_uncharge() invokes NET_RX_SOFTIRQ (process_backlog()) on the same CPU while memcg_stock.lock is held. Every incoming packet in process_backlog() then fails local_trylock() in __consume_stock() and falls back to try_charge_memcg() and __refill_stock(), hammering the global page_counter atomic twice per packet, and slow processing finally wakes up ksoftirqd. Let's move memcg_uncharge(), mod_memcg_state(), and css_put() outside local_unlock(&memcg_stock.lock) in __refill_stock(). With the change, tcp_rr w/ 20000 flows gains ~4% more throughput and is stable around the highest value. Signed-off-by: Kuniyuki Iwashima --- mm/memcontrol.c | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index b92e952c076a..87b189aba07d 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2331,13 +2331,13 @@ static __always_inline void __refill_stock(struct mem_cgroup *memcg, stock_pages = stock->nr_pages[i] + nr_pages; if (unlikely(stock_pages > MEMCG_STOCK_HIGH)) { - memcg_uncharge(memcg, - stock_pages - MEMCG_STOCK_LOW); - stock_pages = MEMCG_STOCK_LOW; + WRITE_ONCE(stock->nr_pages[i], MEMCG_STOCK_LOW); + stock_pages -= MEMCG_STOCK_LOW; + } else { + WRITE_ONCE(stock->nr_pages[i], stock_pages); + stock_pages = 0; } - WRITE_ONCE(stock->nr_pages[i], stock_pages); - if (nr_sock) { stock_sock = stock->nr_sock[i] - nr_sock; @@ -2353,16 +2353,25 @@ static __always_inline void __refill_stock(struct mem_cgroup *memcg, if (unlikely(stock_sock)) mod_memcg_state(memcg, MEMCG_SOCK, stock_sock); + if (unlikely(stock_pages)) + memcg_uncharge(memcg, stock_pages); return; } + cached = NULL; + stock_pages = 0; + i = empty_slot; if (i == -1) { i = stock->drain_idx++; if (stock->drain_idx == NR_MEMCG_STOCK) stock->drain_idx = 0; - drain_stock(stock, i); + + cached = stock->cached[i]; + stock_pages = stock->nr_pages[i]; + stock_sock = stock->nr_sock[i]; + WRITE_ONCE(stock->nr_sock[i], 0); } css_get(&memcg->css); @@ -2371,6 +2380,14 @@ static __always_inline void __refill_stock(struct mem_cgroup *memcg, local_unlock(&memcg_stock.lock); + if (unlikely(cached)) { + if (stock_pages) + memcg_uncharge(cached, stock_pages); + if (stock_sock) + mod_memcg_state(cached, MEMCG_SOCK, stock_sock); + css_put(&cached->css); + } + update_sock: if (nr_sock) mod_memcg_state(memcg, MEMCG_SOCK, -nr_sock); -- 2.56.0.313.g2479322e68-goog