ip_md_tunnel_xmit unnecessarily references the dst_entry from the dst_cache when interacting with the cache. Reduce this overhead by avoiding the redundant refcount increments. This is only possible in flows where the cache is used. Otherwise, we fall-back to a referenced dst. This change is safe since ipv4 supports noref xmit under RCU which is already the case for ip_md_tunnel_xmit. Signed-off-by: Marek Mietus --- net/ipv4/ip_tunnel.c | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/net/ipv4/ip_tunnel.c b/net/ipv4/ip_tunnel.c index 6aa045793048..323d085cc377 100644 --- a/net/ipv4/ip_tunnel.c +++ b/net/ipv4/ip_tunnel.c @@ -578,6 +578,7 @@ void ip_md_tunnel_xmit(struct sk_buff *skb, struct net_device *dev, const struct iphdr *inner_iph; struct rtable *rt = NULL; struct flowi4 fl4; + dstref_t dstref; __be16 df = 0; u8 tos, ttl; bool use_cache; @@ -608,20 +609,26 @@ void ip_md_tunnel_xmit(struct sk_buff *skb, struct net_device *dev, goto tx_error; use_cache = ip_tunnel_dst_cache_usable(skb, tun_info); - if (use_cache) - rt = dst_cache_get_ip4(&tun_info->dst_cache, &fl4.saddr); + if (use_cache) { + rt = dst_cache_get_ip4_rcu(&tun_info->dst_cache, &fl4.saddr); + dstref = dst_to_dstref_noref(&rt->dst); + } if (!rt) { rt = ip_route_output_key(tunnel->net, &fl4); if (IS_ERR(rt)) { DEV_STATS_INC(dev, tx_carrier_errors); goto tx_error; } - if (use_cache) - dst_cache_set_ip4(&tun_info->dst_cache, &rt->dst, - fl4.saddr); + if (use_cache) { + dst_cache_steal_ip4(&tun_info->dst_cache, &rt->dst, + fl4.saddr); + dstref = dst_to_dstref_noref(&rt->dst); + } else { + dstref = dst_to_dstref(&rt->dst); + } } if (rt->dst.dev == dev) { - ip_rt_put(rt); + dstref_drop(dstref); DEV_STATS_INC(dev, collisions); goto tx_error; } @@ -630,7 +637,7 @@ void ip_md_tunnel_xmit(struct sk_buff *skb, struct net_device *dev, df = htons(IP_DF); if (tnl_update_pmtu(dev, skb, rt, df, inner_iph, tunnel_hlen, key->u.ipv4.dst, true)) { - ip_rt_put(rt); + dstref_drop(dstref); goto tx_error; } @@ -647,13 +654,13 @@ void ip_md_tunnel_xmit(struct sk_buff *skb, struct net_device *dev, headroom += LL_RESERVED_SPACE(rt->dst.dev) + rt->dst.header_len; if (skb_cow_head(skb, headroom)) { - ip_rt_put(rt); + dstref_drop(dstref); goto tx_dropped; } ip_tunnel_adj_headroom(dev, headroom); - iptunnel_xmit(NULL, dst_to_dstref(&rt->dst), skb, fl4.saddr, fl4.daddr, proto, tos, ttl, + iptunnel_xmit(NULL, dstref, skb, fl4.saddr, fl4.daddr, proto, tos, ttl, df, !net_eq(tunnel->net, dev_net(dev)), 0); return; tx_error: -- 2.51.0