rt_flush_dev() currently walks every per-CPU uncached route list for each device being removed. This repeatedly examines unrelated routes and makes teardown increasingly expensive as the number of devices grows. Replace each per-CPU list with 64 hash buckets keyed by the route's netdevice. Keep the owning-list pointer in dst_entry so route removal remains unchanged, while device teardown only walks the matching bucket on each CPU. Hash collisions are filtered by the existing device comparison. Signed-off-by: Chris J Arges --- net/ipv4/route.c | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/net/ipv4/route.c b/net/ipv4/route.c index 604cc51dfd9b..3f9bc1ec72cc 100644 --- a/net/ipv4/route.c +++ b/net/ipv4/route.c @@ -74,6 +74,7 @@ #include #include #include +#include #include #include #include @@ -1552,11 +1553,22 @@ struct uncached_list { struct list_head head; }; -static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt_uncached_list); +#define RT_UNCACHED_HASH_BITS 6 +#define RT_UNCACHED_HASH_SIZE BIT(RT_UNCACHED_HASH_BITS) + +struct uncached_table { + struct uncached_list buckets[RT_UNCACHED_HASH_SIZE]; +}; + +static DEFINE_PER_CPU_ALIGNED(struct uncached_table, rt_uncached_table); void rt_add_uncached_list(struct rtable *rt) { - struct uncached_list *ul = raw_cpu_ptr(&rt_uncached_list); + struct uncached_table *table = raw_cpu_ptr(&rt_uncached_table); + struct uncached_list *ul; + + ul = &table->buckets[hash_ptr(dst_dev(&rt->dst), + RT_UNCACHED_HASH_BITS)]; rt->dst.rt_uncached_list = ul; @@ -1588,14 +1600,18 @@ void rt_flush_dev(struct net_device *dev) int cpu; for_each_possible_cpu(cpu) { - struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu); + struct uncached_table *table; + struct uncached_list *ul; + + table = per_cpu_ptr(&rt_uncached_table, cpu); + ul = &table->buckets[hash_ptr(dev, RT_UNCACHED_HASH_BITS)]; if (list_empty(&ul->head)) continue; spin_lock_bh(&ul->lock); list_for_each_entry_safe(rt, safe, &ul->head, dst.rt_uncached) { - if (rt->dst.dev != dev) + if (dst_dev(&rt->dst) != dev) continue; rcu_assign_pointer(rt->dst.dev_rcu, blackhole_netdev); netdev_ref_replace(dev, blackhole_netdev, @@ -3771,10 +3787,16 @@ int __init ip_rt_init(void) ip_tstamps = idents_hash + (ip_idents_mask + 1) * sizeof(*ip_idents); for_each_possible_cpu(cpu) { - struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu); + struct uncached_table *table; + int bucket; + + table = per_cpu_ptr(&rt_uncached_table, cpu); + for (bucket = 0; bucket < RT_UNCACHED_HASH_SIZE; bucket++) { + struct uncached_list *ul = &table->buckets[bucket]; - INIT_LIST_HEAD(&ul->head); - spin_lock_init(&ul->lock); + INIT_LIST_HEAD(&ul->head); + spin_lock_init(&ul->lock); + } } #ifdef CONFIG_IP_ROUTE_CLASSID ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct)); -- 2.43.0 rt6_uncached_list_flush_dev() currently walks every per-CPU uncached route list for each device being removed. Hash uncached routes by their inet6 device so ordinary device teardown only visits the matching bucket on each CPU. ip6_rt_get_dev_rcu() can return loopback or an L3 master while rt6i_idev still refers to the original interface, so such a route must be reachable from either device. Place those routes on a separate per-CPU list that is always visited in addition to the keyed bucket. This avoids growing struct rt6_info while filtering most unrelated routes from ordinary device teardown. Signed-off-by: Chris J Arges --- net/ipv6/route.c | 102 +++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 69 insertions(+), 33 deletions(-) diff --git a/net/ipv6/route.c b/net/ipv6/route.c index 16dfac54a259..860530074027 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -133,11 +134,28 @@ struct uncached_list { struct list_head head; }; -static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt6_uncached_list); +#define RT6_UNCACHED_HASH_BITS 6 +#define RT6_UNCACHED_HASH_SIZE BIT(RT6_UNCACHED_HASH_BITS) + +struct rt6_uncached_table { + struct uncached_list buckets[RT6_UNCACHED_HASH_SIZE]; + /* Routes that must be discoverable through two different devices. */ + struct uncached_list mismatch; +}; + +static DEFINE_PER_CPU_ALIGNED(struct rt6_uncached_table, rt6_uncached_table); void rt6_uncached_list_add(struct rt6_info *rt) { - struct uncached_list *ul = raw_cpu_ptr(&rt6_uncached_list); + struct rt6_uncached_table *table = raw_cpu_ptr(&rt6_uncached_table); + struct net_device *rt_dev = dst_dev(&rt->dst); + struct uncached_list *ul; + + if (rt->rt6i_idev && rt->rt6i_idev->dev != rt_dev) + ul = &table->mismatch; + else + ul = &table->buckets[hash_ptr(rt_dev, + RT6_UNCACHED_HASH_BITS)]; rt->dst.rt_uncached_list = ul; @@ -157,40 +175,50 @@ void rt6_uncached_list_del(struct rt6_info *rt) } } +static void rt6_uncached_list_flush(struct uncached_list *ul, + struct net_device *dev) +{ + struct rt6_info *rt, *safe; + + if (list_empty(&ul->head)) + return; + + spin_lock_bh(&ul->lock); + list_for_each_entry_safe(rt, safe, &ul->head, dst.rt_uncached) { + struct inet6_dev *rt_idev = rt->rt6i_idev; + struct net_device *rt_dev = dst_dev(&rt->dst); + bool handled = false; + + if (rt_idev && rt_idev->dev == dev) { + rt->rt6i_idev = in6_dev_get(blackhole_netdev); + in6_dev_put(rt_idev); + handled = true; + } + + if (rt_dev == dev) { + rt->dst.dev = blackhole_netdev; + netdev_ref_replace(rt_dev, blackhole_netdev, + &rt->dst.dev_tracker, GFP_ATOMIC); + handled = true; + } + if (handled) + list_del_init(&rt->dst.rt_uncached); + } + spin_unlock_bh(&ul->lock); +} + static void rt6_uncached_list_flush_dev(struct net_device *dev) { int cpu; for_each_possible_cpu(cpu) { - struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu); - struct rt6_info *rt, *safe; + struct rt6_uncached_table *table; + struct uncached_list *ul; - if (list_empty(&ul->head)) - continue; - - spin_lock_bh(&ul->lock); - list_for_each_entry_safe(rt, safe, &ul->head, dst.rt_uncached) { - struct inet6_dev *rt_idev = rt->rt6i_idev; - struct net_device *rt_dev = rt->dst.dev; - bool handled = false; - - if (rt_idev && rt_idev->dev == dev) { - rt->rt6i_idev = in6_dev_get(blackhole_netdev); - in6_dev_put(rt_idev); - handled = true; - } - - if (rt_dev == dev) { - rt->dst.dev = blackhole_netdev; - netdev_ref_replace(rt_dev, blackhole_netdev, - &rt->dst.dev_tracker, - GFP_ATOMIC); - handled = true; - } - if (handled) - list_del_init(&rt->dst.rt_uncached); - } - spin_unlock_bh(&ul->lock); + table = per_cpu_ptr(&rt6_uncached_table, cpu); + ul = &table->buckets[hash_ptr(dev, RT6_UNCACHED_HASH_BITS)]; + rt6_uncached_list_flush(ul, dev); + rt6_uncached_list_flush(&table->mismatch, dev); } } @@ -6982,10 +7010,18 @@ int __init ip6_route_init(void) #endif for_each_possible_cpu(cpu) { - struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu); + struct rt6_uncached_table *table; + int bucket; + + table = per_cpu_ptr(&rt6_uncached_table, cpu); + for (bucket = 0; bucket < RT6_UNCACHED_HASH_SIZE; bucket++) { + struct uncached_list *ul = &table->buckets[bucket]; - INIT_LIST_HEAD(&ul->head); - spin_lock_init(&ul->lock); + INIT_LIST_HEAD(&ul->head); + spin_lock_init(&ul->lock); + } + INIT_LIST_HEAD(&table->mismatch.head); + spin_lock_init(&table->mismatch.lock); } out: -- 2.43.0 Local IPv6 routes through a VRF can use the VRF as dst.dev while retaining the VRF member interface in rt6i_idev. Exercise device teardown while such uncached routes are retained by a delayed qdisc. Reuse the existing VRF topology and msg_zerocopy raw-header sender, and verify route creation, qdisc retention, and prompt interface deletion. Signed-off-by: Chris J Arges --- tools/testing/selftests/net/vrf-xfrm-tests.sh | 35 +++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tools/testing/selftests/net/vrf-xfrm-tests.sh b/tools/testing/selftests/net/vrf-xfrm-tests.sh index b64dd891699d..4f409d135a99 100755 --- a/tools/testing/selftests/net/vrf-xfrm-tests.sh +++ b/tools/testing/selftests/net/vrf-xfrm-tests.sh @@ -385,6 +385,37 @@ run_tests() cleanup_xfrm_dev } +test_ipv6_uncached_mismatch() +{ + local sender_pid + local backlog + local rc + + # A local route through a VRF uses the VRF as dst.dev while retaining + # the VRF member interface in rt6i_idev. Raw header sends create uncached + # routes, and netem keeps them referenced while the interface is deleted. + run_cmd_host1 tc qdisc replace dev ${VRF} root netem limit 1 delay 10s + ip -6 -netns "$host1" route add local ${HOST1_6}/128 dev eth0 + ip netns exec "$host1" ./msg_zerocopy -6 \ + -S ${HOST1_6} -D ${HOST1_6} -s 1200 -t 0 raw_hdrincl \ + >/dev/null 2>&1 & + sender_pid=$! + wait "$sender_pid" + rc=$? + log_test $rc 0 "Create uncached IPv6 routes with mismatched devices" + [ $rc -ne 0 ] && return + + backlog=$(ip netns exec "$host1" tc -s qdisc show dev ${VRF}) + if ! echo "$backlog" | grep -Eq 'backlog .* [1-9][0-9]*p'; then + log_test 1 0 "Retain uncached IPv6 routes in VRF qdisc" + return + fi + log_test 0 0 "Retain uncached IPv6 routes in VRF qdisc" + + run_cmd_host1 timeout 2 ip link del eth0 + log_test $? 0 "Flush uncached IPv6 routes with mismatched devices" +} + ################################################################################ # usage @@ -425,6 +456,10 @@ echo echo "netem qdisc on VRF device" run_tests +echo +echo "Uncached IPv6 route with mismatched devices" +test_ipv6_uncached_mismatch + printf "\nTests passed: %3d\n" ${nsuccess} printf "Tests failed: %3d\n" ${nfail} -- 2.43.0