Introduce sw flowtable acceleration for the TX/RX paths of IPv4 over IPv6 tunnels, relying on the netfilter flowtable infrastructure. The feature can be tested with a forwarding scenario between two NICs (eth0 and eth1), where an IPv4 over IPv6 tunnel is used to reach a remote site via eth1 as the underlay device: ETH0 -- TUN0 <==> ETH1 -- [IP network] -- TUN1 (2001:db8:2::2) [IP configuration] 6: eth0: mtu 1500 qdisc noqueue state UP group default qlen 1000 link/ether 00:00:22:33:11:55 brd ff:ff:ff:ff:ff:ff inet 192.168.0.2/24 scope global eth0 valid_lft forever preferred_lft forever 7: eth1: mtu 1500 qdisc noqueue state UP group default qlen 1000 link/ether 00:11:22:33:11:55 brd ff:ff:ff:ff:ff:ff inet6 2001:db8:2::1/64 scope global nodad valid_lft forever preferred_lft forever 8: tun0@NONE: mtu 1480 qdisc noqueue state UNKNOWN group default qlen 1000 link/tunnel6 2001:db8:2::1 peer 2001:db8:2::2 permaddr ce9c:2940:7dcc:: inet 192.168.100.1/24 scope global tun0 valid_lft forever preferred_lft forever $ ip route show default via 192.168.100.2 dev tun0 192.168.0.0/24 dev eth0 proto kernel scope link src 192.168.0.2 192.168.100.0/24 dev tun0 proto kernel scope link src 192.168.100.1 $ ip -6 route show 2001:db8:2::/64 dev eth1 proto kernel metric 256 pref medium $ nft list ruleset table inet filter { flowtable ft { hook ingress priority filter devices = { eth0, eth1 } } chain forward { type filter hook forward priority filter; policy accept; meta l4proto { tcp, udp } flow add @ft } } When reproducing this scenario using veth interfaces, the following results were observed: - TCP stream received from IPv4 over IPv6 tunnel: - net-next (baseline): ~126 Gbps - net-next + IP4IP6 flowtable support: ~138 Gbps - TCP stream transmitted to IPv4 over IPv6 tunnel: - net-next (baseline): ~127 Gbps - net-next + IP4IP6 flowtable support: ~140 Gbps Signed-off-by: Lorenzo Bianconi --- net/netfilter/nf_flow_table_ip.c | 90 +++++++++++++++++++++++++++++++++------- 1 file changed, 76 insertions(+), 14 deletions(-) diff --git a/net/netfilter/nf_flow_table_ip.c b/net/netfilter/nf_flow_table_ip.c index 902b9b595677..ea2cd9df33f8 100644 --- a/net/netfilter/nf_flow_table_ip.c +++ b/net/netfilter/nf_flow_table_ip.c @@ -856,7 +856,8 @@ static unsigned int nf_flow_queue_xmit(struct net *net, struct sk_buff *skb, return NF_STOLEN; } -static int nf_flow_queue_xmit4(struct sk_buff *skb, +static int nf_flow_queue_xmit4(struct nf_flowtable_ctx *ctx, + struct sk_buff *skb, struct flow_offload_tuple_rhash *tuplehash, const struct nf_hook_state *state) { @@ -869,6 +870,16 @@ static int nf_flow_queue_xmit4(struct sk_buff *skb, struct rtable *rt; if (unlikely(tuplehash->tuple.xmit_type == FLOW_OFFLOAD_XMIT_XFRM)) { + if (tuplehash->tuple.l3proto == AF_INET6) { + struct rt6_info *rt6; + + rt6 = dst_rt6_info(tuplehash->tuple.dst_cache); + memset(skb->cb, 0, sizeof(struct inet6_skb_parm)); + IP6CB(skb)->iif = skb->dev->ifindex; + IP6CB(skb)->flags = IP6SKB_FORWARDED; + return nf_flow_xmit_xfrm(skb, state, &rt6->dst); + } + rt = dst_rtable(tuplehash->tuple.dst_cache); memset(skb->cb, 0, sizeof(struct inet_skb_parm)); IPCB(skb)->iif = skb->dev->ifindex; @@ -886,22 +897,38 @@ static int nf_flow_queue_xmit4(struct sk_buff *skb, return NF_DROP; switch (tuplehash->tuple.xmit_type) { - case FLOW_OFFLOAD_XMIT_NEIGH: - rt = dst_rtable(tuplehash->tuple.dst_cache); + case FLOW_OFFLOAD_XMIT_NEIGH: { + struct dst_entry *dst; + xmit.outdev = dev_get_by_index_rcu(state->net, tuplehash->tuple.ifidx); if (!xmit.outdev) { flow_offload_teardown(flow); return NF_DROP; } - neigh = ip_neigh_gw4(rt->dst.dev, rt_nexthop(rt, ip_addr->ip)); + if (other_tuple->tun.encap_proto ? + other_tuple->tun.encap_proto == AF_INET6 : + ctx->tun.inner_proto == IPPROTO_IPV6) { + struct rt6_info *rt6; + + rt6 = dst_rt6_info(tuplehash->tuple.dst_cache); + neigh = ip_neigh_gw6(rt6->dst.dev, + rt6_nexthop(rt6, &ip_addr->in6)); + dst = &rt6->dst; + } else { + rt = dst_rtable(tuplehash->tuple.dst_cache); + neigh = ip_neigh_gw4(rt->dst.dev, + rt_nexthop(rt, ip_addr->ip)); + dst = &rt->dst; + } if (IS_ERR(neigh)) { flow_offload_teardown(flow); return NF_DROP; } xmit.dest = neigh->ha; skb_dst_drop(skb); - skb_dst_set_noref(skb, &rt->dst); + skb_dst_set_noref(skb, dst); break; + } case FLOW_OFFLOAD_XMIT_DIRECT: xmit.outdev = dev_get_by_index_rcu(state->net, tuplehash->tuple.out.ifidx); if (!xmit.outdev) { @@ -948,7 +975,7 @@ nf_flow_offload_ip_hook(void *priv, struct sk_buff *skb, else if (ret == 0) return NF_ACCEPT; - return nf_flow_queue_xmit4(skb, tuplehash, state); + return nf_flow_queue_xmit4(&ctx, skb, tuplehash, state); } EXPORT_SYMBOL_GPL(nf_flow_offload_ip_hook); @@ -1175,13 +1202,18 @@ nf_flow_offload_ipv6_lookup(struct nf_flowtable_ctx *ctx, { struct flow_offload_tuple tuple = {}; - if (nf_flow_tuple_ipv6(ctx, skb, &tuple) < 0) + if (ctx->tun.inner_proto == IPPROTO_IPIP) { + if (nf_flow_tuple_ip(ctx, skb, &tuple) < 0) + return NULL; + } else if (nf_flow_tuple_ipv6(ctx, skb, &tuple) < 0) { return NULL; + } return flow_offload_lookup(flow_table, &tuple); } -static int nf_flow_queue_xmit6(struct sk_buff *skb, +static int nf_flow_queue_xmit6(struct nf_flowtable_ctx *ctx, + struct sk_buff *skb, struct flow_offload_tuple_rhash *tuplehash, const struct nf_hook_state *state) { @@ -1194,6 +1226,16 @@ static int nf_flow_queue_xmit6(struct sk_buff *skb, struct rt6_info *rt; if (unlikely(tuplehash->tuple.xmit_type == FLOW_OFFLOAD_XMIT_XFRM)) { + if (tuplehash->tuple.l3proto == AF_INET) { + struct rtable *rt4; + + rt4 = dst_rtable(tuplehash->tuple.dst_cache); + memset(skb->cb, 0, sizeof(struct inet_skb_parm)); + IPCB(skb)->iif = skb->dev->ifindex; + IPCB(skb)->flags = IPSKB_FORWARDED; + return nf_flow_xmit_xfrm(skb, state, &rt4->dst); + } + rt = dst_rt6_info(tuplehash->tuple.dst_cache); memset(skb->cb, 0, sizeof(struct inet6_skb_parm)); IP6CB(skb)->iif = skb->dev->ifindex; @@ -1211,22 +1253,38 @@ static int nf_flow_queue_xmit6(struct sk_buff *skb, return NF_DROP; switch (tuplehash->tuple.xmit_type) { - case FLOW_OFFLOAD_XMIT_NEIGH: - rt = dst_rt6_info(tuplehash->tuple.dst_cache); + case FLOW_OFFLOAD_XMIT_NEIGH: { + struct dst_entry *dst; + xmit.outdev = dev_get_by_index_rcu(state->net, tuplehash->tuple.ifidx); if (!xmit.outdev) { flow_offload_teardown(flow); return NF_DROP; } - neigh = ip_neigh_gw6(rt->dst.dev, rt6_nexthop(rt, &ip_addr->in6)); + if (other_tuple->tun.encap_proto ? + other_tuple->tun.encap_proto == AF_INET : + ctx->tun.inner_proto == IPPROTO_IPIP) { + struct rtable *rt4; + + rt4 = dst_rtable(tuplehash->tuple.dst_cache); + neigh = ip_neigh_gw4(rt4->dst.dev, + rt_nexthop(rt4, ip_addr->ip)); + dst = &rt4->dst; + } else { + rt = dst_rt6_info(tuplehash->tuple.dst_cache); + neigh = ip_neigh_gw6(rt->dst.dev, + rt6_nexthop(rt, &ip_addr->in6)); + dst = &rt->dst; + } if (IS_ERR(neigh)) { flow_offload_teardown(flow); return NF_DROP; } xmit.dest = neigh->ha; skb_dst_drop(skb); - skb_dst_set_noref(skb, &rt->dst); + skb_dst_set_noref(skb, dst); break; + } case FLOW_OFFLOAD_XMIT_DIRECT: xmit.outdev = dev_get_by_index_rcu(state->net, tuplehash->tuple.out.ifidx); if (!xmit.outdev) { @@ -1267,12 +1325,16 @@ nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb, if (!tuplehash) return NF_ACCEPT; - ret = nf_flow_offload_ipv6_forward(&ctx, flow_table, tuplehash, skb); + if (ctx.tun.inner_proto == IPPROTO_IPIP) + ret = nf_flow_offload_forward(&ctx, flow_table, tuplehash, skb); + else + ret = nf_flow_offload_ipv6_forward(&ctx, flow_table, tuplehash, + skb); if (ret < 0) return NF_DROP; else if (ret == 0) return NF_ACCEPT; - return nf_flow_queue_xmit6(skb, tuplehash, state); + return nf_flow_queue_xmit6(&ctx, skb, tuplehash, state); } EXPORT_SYMBOL_GPL(nf_flow_offload_ipv6_hook); -- 2.56.0