When RX HWtstamp is enabled on NIC, skb_hwtstamps(skb)->hwtstamp is set, which sets TCP_SKB_CB(skb)->has_rxtstamp to true. Then, tcp_recvmsg_locked() always calls tcp_update_recv_tstamps() and sets TCP_CMSG_TS to cmsg_flags so that later tcp_recvmsg() calls tcp_recv_timestamp() after release_sock(). If a socket does not have SOCK_RCVTSTAMP in sk->sk_flags or any flags set in sk->sk_tsflags, tcp_recv_timestamp() does nothing. However, it is not free because sk->sk_tsflags shares the same cacheline with sk->sk_forward_alloc, which is always modified under lock_sock()/bh_lock_sock(), causing cacheline bouncing. 0.31 │ 403: test %al,%al 0.02 │ ↓ je 4a0 12.40 │ test $0x2,%bpl ; if (cmsg_flags & TCP_CMSG_TS) │ ↓ je 48f 11.59 │ mov 0x60(%r14),%r15 ; sock_flag(sk, SOCK_TSTAMP_NEW) 12.25 │ mov 0x170(%r14),%ebx ; READ_ONCE(sk->sk_tsflags) 20.70 │ mov 0xd0(%rsp),%rdi ; if (tss->ts[0]) { 0.39 │ test %rdi,%rdi │ ↓ je 44c Also, sk->sk_flags is on the cold cacheline in tcp_recvmsg(). Commit f0e70409b7eb ("net: avoid unconditionally touching sk_tsflags on RX") introduced SOCK_TIMESTAMPING_ANY to avoid such false sharing. Let's check SOCK_RCVTSTAMP and SOCK_TIMESTAMPING_ANY under lock_sock(). We still need to touch sk->sk_flags, but with this change, tcp_rr -T 128 -F 10000 gains ~3% more throughput on AMD EPYC 7B12 Zen 2 Rome (2 sockets, 256 vCPUs). Signed-off-by: Kuniyuki Iwashima --- include/net/sock.h | 7 +++++++ net/ipv4/tcp.c | 12 +++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/include/net/sock.h b/include/net/sock.h index 173b37b2b8bb..ba310182e6cc 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -1063,6 +1063,13 @@ static inline bool sock_flag(const struct sock *sk, enum sock_flags flag) return test_bit(flag, &sk->sk_flags); } +static inline bool sk_want_rx_tstamps(const struct sock *sk) +{ + return READ_ONCE(sk->sk_flags) & + ((1UL << SOCK_RCVTSTAMP) | + (1UL << SOCK_TIMESTAMPING_ANY)); +} + #ifdef CONFIG_NET DECLARE_STATIC_KEY_FALSE(memalloc_socks_key); static inline int sk_memalloc_socks(void) diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index b9de7c302ba8..4166be09b826 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1878,6 +1878,12 @@ void tcp_set_rcvbuf(struct sock *sk, int val) tcp_set_window_clamp(sk, tcp_win_from_space(sk, val)); } +static bool tcp_want_rx_tstamps(struct sock *sk, struct sk_buff *skb) +{ + return TCP_SKB_CB(skb)->has_rxtstamp && + sk_want_rx_tstamps(sk); +} + #ifdef CONFIG_MMU static const struct vm_operations_struct tcp_vm_ops = { }; @@ -2069,7 +2075,7 @@ static int tcp_zc_handle_leftover(struct tcp_zerocopy_receive *zc, offset = *seq - TCP_SKB_CB(skb)->seq; } else { skb = tcp_recv_skb(sk, *seq, &offset); - if (TCP_SKB_CB(skb)->has_rxtstamp) { + if (tcp_want_rx_tstamps(sk, skb)) { tcp_update_recv_tstamps(skb, tss); zc->msg_flags |= TCP_CMSG_TS; } @@ -2280,7 +2286,7 @@ static int tcp_zerocopy_receive(struct sock *sk, if (!skb_frags_readable(skb)) break; - if (TCP_SKB_CB(skb)->has_rxtstamp) { + if (tcp_want_rx_tstamps(sk, skb)) { tcp_update_recv_tstamps(skb, tss); zc->msg_flags |= TCP_CMSG_TS; } @@ -2893,7 +2899,7 @@ static int tcp_recvmsg_locked(struct sock *sk, struct msghdr *msg, size_t len, tcp_fast_path_check(sk); } - if (TCP_SKB_CB(skb)->has_rxtstamp) { + if (tcp_want_rx_tstamps(sk, skb)) { tcp_update_recv_tstamps(skb, tss); *cmsg_flags |= TCP_CMSG_TS; } -- 2.56.0.385.gac671c819a-goog