On x86-64, dst_entry::tclassid shares the same cache line with dst_entry::__rcuref. Updates to __rcuref cause false sharing with tclassid reads, for example in ip_rcv_finish_core() when CONFIG_IP_ROUTE_CLASSID=y. Memcached perf c2c shows the paths: read: net_rx_action -> process_backlog -> ip_rcv_finish_core -> skb_dst(skb)->tclassid (load) acquire: tcp_write_xmit -> __tcp_transmit_skb -> __dev_queue_xmit -> skb_dst_force -> dst_hold_safe -> rcuref_get release: tcp_rcv_established/tcp_add_backlog -> tcp_cleanup_skb -> skb_dst_drop -> dst_release -> rcuref_put Move tclassid after lwtstate on 64-bit to avoid false sharing with __rcuref. On x86-64, struct dst_entry grows from 136 to 144 bytes. struct rtable and struct rt6_info embed dst_entry, so their sizes grow from 176 to 184 and from 224 to 232 bytes, respectively. However, both struct rtable and struct rt6_info are allocated from slab caches with SLAB_HWCACHE_ALIGN, so the per-object allocation sizes remain unchanged at 192 bytes for rtable and 256 bytes for rt6_info. Memcached perf c2c samples at the class-ID load: +----------+------------+-------------+ | Kernel | Local HITM | Remote HITM | +----------+------------+-------------+ | Baseline | 894 | 680 | | Patched | 15 | 0 | +----------+------------+-------------+ Results with 64 workers per endpoint (baseline 6628e7d4c6b2). Improvement is 100 * (patched average / baseline average - 1). Memcached (5 iterations, average operations/s): +--------------------+--------------------+ | Case | Improvement (%) | +--------------------+--------------------+ | default | +27.20 | | hit | +52.36 | | balanced | -0.52 | | large | +64.68 | +--------------------+--------------------+ Memcached over IPv4 loopback. The default client command is: memtier_benchmark -s 127.0.0.1 -p 11221 -P memcache_text -t 64 -c 1 --pipeline=16 --test-time=60 --ratio=1:100 --data-size=32 --key-maximum=10000000 hit: --key-maximum=1000 balanced: --key-maximum=1000 --ratio=1:1 large: --key-maximum=1000 --data-size=1024 TCP request/reply (5 iterations, average transactions/s): +----------------------+--------------------+ | Case | Improvement (%) | +----------------------+--------------------+ | tcp4-rr | +1.85 | | ipv6-loopback-plain | -0.60 | | ipv6-loopback-expiry | -0.24 | | tcp6-rr | -1.60 | +----------------------+--------------------+ 64 pinned netperf clients per case, with 128-byte requests and replies: tcp4-rr: netperf -t TCP_RR -l 8 -- -r 128,128 ipv6-loopback-plain: netperf -6 -t TCP_RR -l 8 -- -r 128,128 ipv6-loopback-expiry: netperf -6 -t TCP_RR -l 8 -- -r 128,128 tcp6-rr: netperf -6 -t TCP_RR -l 30 -- -r 128,128 Transfer (5 iterations, average received Mbit/s): +-------------+--------------------+ | Case | Improvement (%) | +-------------+--------------------+ | udp4-small | -0.91 | | udp4-frag | +0.41 | | tcp4-stream | +0.92 | | udp6-small | -0.81 | | udp6-frag | +1.37 | | tcp6-stream | +3.22 | | udpxfrm | +4.57 | | tcpxfrm | -1.23 | | geneve0 | +0.49 | | geneve8 | +4.28 | | geneve32 | +1.53 | +-------------+--------------------+ 64 pinned netperf clients per case. Netperf commands: udp4-small: netperf -t UDP_STREAM -l 30 -- -m 512 udp4-frag: netperf -t UDP_STREAM -l 8 -- -m 8K tcp4-stream: netperf -t TCP_STREAM -l 8 -- -m 10K udp6-small: netperf -6 -t UDP_STREAM -l 8 -- -m 512 udp6-frag: netperf -6 -t UDP_STREAM -l 8 -- -m 8K tcp6-stream: netperf -6 -t TCP_STREAM -l 8 -- -m 10K udpxfrm: netperf -t UDP_STREAM -l 8 -- -m 8K tcpxfrm: netperf -t TCP_STREAM -l 8 -- -m 10K geneve0: netperf -t TCP_STREAM -l 8 -- -m 10K geneve8: netperf -t TCP_STREAM -l 8 -- -m 10K geneve32: netperf -t TCP_STREAM -l 30 -- -m 10K Suggested-by: Tim Chen Reviewed-by: Tianyou Li Reviewed-by: Wangyang Guo Signed-off-by: Jiebin Sun --- include/net/dst.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/include/net/dst.h b/include/net/dst.h index dbedfe72e1fd..f01cfb42f97c 100644 --- a/include/net/dst.h +++ b/include/net/dst.h @@ -76,17 +76,17 @@ struct dst_entry { struct rcu_head rcu_head; short error; short __pad; - __u32 tclassid; #ifndef CONFIG_64BIT + __u32 tclassid; struct lwtunnel_state *lwtstate; rcuref_t __rcuref; /* 32-bit offset 64 */ #endif netdevice_tracker dev_tracker; /* - * Used by rtable and rt6_info. Moves lwtstate into the next cache - * line on 64bit so that lwtstate does not cause false sharing with - * __rcuref under contention of __rcuref. This also puts the + * Used by rtable and rt6_info. Moves lwtstate and tclassid into the + * next cache line on 64bit so that they do not cause false sharing + * with __rcuref under contention of __rcuref. This also puts the * frequently accessed members of rtable and rt6_info out of the * __rcuref cache line. */ @@ -94,6 +94,7 @@ struct dst_entry { struct uncached_list *rt_uncached_list; #ifdef CONFIG_64BIT struct lwtunnel_state *lwtstate; + __u32 tclassid; #endif }; -- 2.52.0