tcp_v{4,6}_send_check() are only called from tcp_output.c and should be made static so that the compiler does not need to put an out of line copy of them. Remove (struct inet_connection_sock_af_ops) send_check field and use instead @net_header_len. Move @net_header_len close to @queue_xmit for data locality as both are used in TCP tx fast path. $ scripts/bloat-o-meter -t vmlinux.2 vmlinux.3 add/remove: 0/4 grow/shrink: 0/2 up/down: 0/-320 (-320) Function old new delta __tcp_transmit_skb 3075 3072 -3 mptcp_subflow_init 777 763 -14 __pfx_tcp_v6_send_check 16 - -16 __pfx_tcp_v4_send_check 16 - -16 tcp_v6_send_check 135 - -135 tcp_v4_send_check 136 - -136 Total: Before=24893840, After=24893520, chg -0.00% Signed-off-by: Eric Dumazet --- include/net/inet_connection_sock.h | 3 +-- include/net/tcp.h | 3 --- net/ipv4/tcp_ipv4.c | 1 - net/ipv4/tcp_output.c | 8 +++----- net/ipv6/tcp_ipv6.c | 2 -- net/mptcp/subflow.c | 1 - net/tls/tls_device_fallback.c | 3 --- 7 files changed, 4 insertions(+), 17 deletions(-) diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h index ecb362025c4e5183ec78aef4b45c249da87c19ea..bbc9355871c767b51e3d1a4d2436022a8556416c 100644 --- a/include/net/inet_connection_sock.h +++ b/include/net/inet_connection_sock.h @@ -34,7 +34,7 @@ struct tcp_congestion_ops; */ struct inet_connection_sock_af_ops { int (*queue_xmit)(struct sock *sk, struct sk_buff *skb, struct flowi *fl); - void (*send_check)(struct sock *sk, struct sk_buff *skb); + u16 net_header_len; int (*rebuild_header)(struct sock *sk); void (*sk_rx_dst_set)(struct sock *sk, const struct sk_buff *skb); int (*conn_request)(struct sock *sk, struct sk_buff *skb); @@ -43,7 +43,6 @@ struct inet_connection_sock_af_ops { struct dst_entry *dst, struct request_sock *req_unhash, bool *own_req); - u16 net_header_len; int (*setsockopt)(struct sock *sk, int level, int optname, sockptr_t optval, unsigned int optlen); int (*getsockopt)(struct sock *sk, int level, int optname, diff --git a/include/net/tcp.h b/include/net/tcp.h index b9bebf5df4aa6ecf6f421400fb93cefb4104e5aa..339378be8276af2d293f7f52d1aa506e369c5f8d 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -531,7 +531,6 @@ u16 tcp_get_syncookie_mss(struct request_sock_ops *rsk_ops, * TCP v4 functions exported for the inet6 API */ -void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb); void tcp_v4_mtu_reduced(struct sock *sk); void tcp_req_err(struct sock *sk, u32 seq, bool abort); void tcp_ld_RTO_revert(struct sock *sk, u32 seq); @@ -1124,8 +1123,6 @@ static inline int tcp_v6_sdif(const struct sk_buff *skb) extern const struct inet_connection_sock_af_ops ipv6_specific; -void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb); - INDIRECT_CALLABLE_DECLARE(int tcp_v6_rcv(struct sk_buff *skb)); void tcp_v6_early_demux(struct sk_buff *skb); diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 7537adf0846f4fe7e0a1dd1cdec559a982329cc1..c7dcacf418380a70d37e3c6005c2202495c74f8e 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -2413,7 +2413,6 @@ EXPORT_IPV6_MOD(inet_sk_rx_dst_set); const struct inet_connection_sock_af_ops ipv4_specific = { .queue_xmit = ip_queue_xmit, - .send_check = tcp_v4_send_check, .rebuild_header = inet_sk_rebuild_header, .sk_rx_dst_set = inet_sk_rx_dst_set, .conn_request = tcp_v4_conn_request, diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index 8c1731b544f42cfa1e7629dde128e969b60d2343..7748cee03e87d74b1bf40e60c6458717c6225a36 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -1479,22 +1479,20 @@ INDIRECT_CALLABLE_DECLARE(int ip_queue_xmit(struct sock *sk, struct sk_buff *skb INDIRECT_CALLABLE_DECLARE(int inet6_csk_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl)); /* This routine computes an IPv4 TCP checksum. */ -void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb) +static void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb) { const struct inet_sock *inet = inet_sk(sk); __tcp_v4_send_check(skb, inet->inet_saddr, inet->inet_daddr); } -EXPORT_IPV6_MOD(tcp_v4_send_check); #if IS_ENABLED(CONFIG_IPV6) #include -void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb) +static void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb) { __tcp_v6_send_check(skb, &sk->sk_v6_rcv_saddr, &sk->sk_v6_daddr); } -EXPORT_IPV6_MOD(tcp_v6_send_check); #endif /* This routine actually transmits TCP packets queued in by @@ -1659,7 +1657,7 @@ static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, bpf_skops_write_hdr_opt(sk, skb, NULL, NULL, 0, &opts); #if IS_ENABLED(CONFIG_IPV6) - if (likely(icsk->icsk_af_ops->send_check == tcp_v6_send_check)) + if (likely(icsk->icsk_af_ops->net_header_len == sizeof(struct ipv6hdr))) tcp_v6_send_check(sk, skb); else #else diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index ca01b11c0ac959e4ee18dd388c3d17e8a4c88761..11a055f4863a383df5105303cb181e11614b262d 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -2017,7 +2017,6 @@ static struct timewait_sock_ops tcp6_timewait_sock_ops = { const struct inet_connection_sock_af_ops ipv6_specific = { .queue_xmit = inet6_csk_xmit, - .send_check = tcp_v6_send_check, .rebuild_header = inet6_sk_rebuild_header, .sk_rx_dst_set = inet6_sk_rx_dst_set, .conn_request = tcp_v6_conn_request, @@ -2049,7 +2048,6 @@ static const struct tcp_sock_af_ops tcp_sock_ipv6_specific = { */ static const struct inet_connection_sock_af_ops ipv6_mapped = { .queue_xmit = ip_queue_xmit, - .send_check = tcp_v4_send_check, .rebuild_header = inet_sk_rebuild_header, .sk_rx_dst_set = inet_sk_rx_dst_set, .conn_request = tcp_v6_conn_request, diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c index 96d54cb2cd93f447e0e04c53d8eaceaa7de3e9c8..e9f91ce43ffc1f5b76f9042a3b20fa34e5b0b453 100644 --- a/net/mptcp/subflow.c +++ b/net/mptcp/subflow.c @@ -2192,7 +2192,6 @@ void __init mptcp_subflow_init(void) subflow_v6m_specific = subflow_v6_specific; subflow_v6m_specific.queue_xmit = ipv4_specific.queue_xmit; - subflow_v6m_specific.send_check = ipv4_specific.send_check; subflow_v6m_specific.net_header_len = ipv4_specific.net_header_len; subflow_v6m_specific.mtu_reduced = ipv4_specific.mtu_reduced; subflow_v6m_specific.rebuild_header = subflow_rebuild_header; diff --git a/net/tls/tls_device_fallback.c b/net/tls/tls_device_fallback.c index 03d508a45aaee9218bb2deed542534785f2019d6..de7d86bdd7ec99df5be4215ff79dbd5e86e49934 100644 --- a/net/tls/tls_device_fallback.c +++ b/net/tls/tls_device_fallback.c @@ -149,9 +149,6 @@ static int tls_enc_records(struct aead_request *aead_req, return rc; } -/* Can't use icsk->icsk_af_ops->send_check here because the ip addresses - * might have been changed by NAT. - */ static void update_chksum(struct sk_buff *skb, int headln) { struct tcphdr *th = tcp_hdr(skb); -- 2.53.0.rc1.225.gd81095ad13-goog