RFS and accelerated RFS are enabled per host and per rx queue, but whether steering helps is a property of the flow. A host that carries a few long-lived flows which benefit from steering alongside a large churn of short-lived flows which arrive on the right CPU already has no way to express that today. Add a boolean socket option, SO_BYPASS_RFS. While set, sock_rps_record_flow() leaves the socket out of rps_sock_flow_table, so get_rps_cpu() finds no match and neither software RFS nor ndo_rx_flow_steer() acts on the flow; it stays on the CPU chosen by RSS or RPS. The bit lives in sock_common next to skc_state, in the space reclaimed by commit 13df9cc6fc0b ("net: Shrink skc_reuse:4 to 2 bits."), so the test shares the cache line the record path already reads and sock_copy() carries it to accepted children: setting it on a listening socket excludes every connection accepted from it, while sockets the same process connects itself remain eligible. Setting the option clears the socket's entry, so a flow recorded earlier stops being steered with the next packet. Closing a bypassing socket leaves the table alone: a close clears its slot without checking who last wrote it, and a churn of bypassing sockets could evict the hints of the steered flows. Both tests sit behind the rfs_needed static branch, so hosts without RFS pay nothing. Signed-off-by: Vitaly Zuevsky --- v2: - fold the documentation into this patch - rename SO_NO_RFS to SO_BYPASS_RFS (Kuniyuki Iwashima) - keep the state in a sock_common bit instead of sk_flags (Kuniyuki) - test it in the rps.h wrappers, next to rfs_is_needed() - clear the table entry when the option is set on a recorded socket - drop the comments and the unlikely() - guard the sockopt with CONFIG_RPS v1: https://lore.kernel.org/netdev/20261031113330.45704-1-vitaly.zuevsky@gcore.com/ Documentation/networking/scaling.rst | 15 +++++++++++++++ arch/alpha/include/uapi/asm/socket.h | 2 ++ arch/mips/include/uapi/asm/socket.h | 2 ++ arch/parisc/include/uapi/asm/socket.h | 2 ++ arch/sparc/include/uapi/asm/socket.h | 2 ++ include/net/rps.h | 4 ++-- include/net/sock.h | 5 ++++- include/uapi/asm-generic/socket.h | 2 ++ net/core/sock.c | 15 +++++++++++++++ 9 files changed, 46 insertions(+), 3 deletions(-) diff --git a/Documentation/networking/scaling.rst b/Documentation/networking/scaling.rst index 6c9836000a8d..b23c87405622 100644 --- a/Documentation/networking/scaling.rst +++ b/Documentation/networking/scaling.rst @@ -428,6 +428,21 @@ queues. So for instance, if rps_sock_flow_entries is set to 131072 and there are 16 configured receive queues, rps_flow_cnt for each queue might be configured as 8192. +Excluding individual sockets +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +RFS is configured per host and per queue, but whether it pays off +depends on the traffic pattern. SO_BYPASS_RFS keeps a socket's flow +out of rps_sock_flow_table, so neither RFS nor accelerated RFS acts on +it: packets stay on the CPU chosen by RSS or RPS, and no filter is +programmed into the NIC. The option is inherited through accept(), so +a listening socket that sets it excludes every connection it accepts +while the sockets the process connects itself remain steered. It may +be set at any time; an entry already recorded is removed. This suits +a host carrying a few long-lived flows worth steering alongside a +churn of short-lived ones for which the cross-CPU enqueue, the wakeup +and the filter update cost more than they save. + Accelerated RFS =============== diff --git a/arch/alpha/include/uapi/asm/socket.h b/arch/alpha/include/uapi/asm/socket.h index 5d3524c26b2b..4df4e1c520fb 100644 --- a/arch/alpha/include/uapi/asm/socket.h +++ b/arch/alpha/include/uapi/asm/socket.h @@ -161,6 +161,8 @@ #define SO_PEERPIDFD_THREAD 87 +#define SO_BYPASS_RFS 88 + #if !defined(__KERNEL__) #if __BITS_PER_LONG == 64 diff --git a/arch/mips/include/uapi/asm/socket.h b/arch/mips/include/uapi/asm/socket.h index 245a43f52fb1..e1e9d34afdb3 100644 --- a/arch/mips/include/uapi/asm/socket.h +++ b/arch/mips/include/uapi/asm/socket.h @@ -172,6 +172,8 @@ #define SO_PEERPIDFD_THREAD 87 +#define SO_BYPASS_RFS 88 + #if !defined(__KERNEL__) #if __BITS_PER_LONG == 64 diff --git a/arch/parisc/include/uapi/asm/socket.h b/arch/parisc/include/uapi/asm/socket.h index f23710e1c671..c38445422129 100644 --- a/arch/parisc/include/uapi/asm/socket.h +++ b/arch/parisc/include/uapi/asm/socket.h @@ -153,6 +153,8 @@ #define SO_PEERPIDFD_THREAD 0x4055 +#define SO_BYPASS_RFS 0x4056 + #if !defined(__KERNEL__) #if __BITS_PER_LONG == 64 diff --git a/arch/sparc/include/uapi/asm/socket.h b/arch/sparc/include/uapi/asm/socket.h index b35b25bdefc2..f0cf0a3d2916 100644 --- a/arch/sparc/include/uapi/asm/socket.h +++ b/arch/sparc/include/uapi/asm/socket.h @@ -154,6 +154,8 @@ #define SO_PEERPIDFD_THREAD 0x0060 +#define SO_BYPASS_RFS 0x0061 + #if !defined(__KERNEL__) diff --git a/include/net/rps.h b/include/net/rps.h index e33c6a2fa8bb..7c62ae607d18 100644 --- a/include/net/rps.h +++ b/include/net/rps.h @@ -149,7 +149,7 @@ static inline void sock_rps_record_flow_hash(__u32 hash) static inline void sock_rps_record_flow(const struct sock *sk) { #ifdef CONFIG_RPS - if (!rfs_is_needed()) + if (!rfs_is_needed() || sk->sk_bypass_rfs) return; _sock_rps_record_flow(sk); @@ -159,7 +159,7 @@ static inline void sock_rps_record_flow(const struct sock *sk) static inline void sock_rps_delete_flow(const struct sock *sk) { #ifdef CONFIG_RPS - if (!rfs_is_needed()) + if (!rfs_is_needed() || sk->sk_bypass_rfs) return; _sock_rps_delete_flow(sk); diff --git a/include/net/sock.h b/include/net/sock.h index 9bba1e793f1e..a52db61775d8 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -124,6 +124,7 @@ typedef __u64 __bitwise __addrpair; * @skc_ipv6only: socket is IPV6 only * @skc_net_refcnt: socket is using net ref counting * @skc_bypass_prot_mem: bypass the per-protocol memory accounting for skb + * @skc_bypass_rfs: %SO_BYPASS_RFS setting * @skc_unused: unused bits * @skc_bound_dev_if: bound device index if != 0 * @skc_bind_node: bind hash linkage for various protocol lookup tables @@ -182,7 +183,8 @@ struct sock_common { unsigned char skc_ipv6only:1; unsigned char skc_net_refcnt:1; unsigned char skc_bypass_prot_mem:1; - unsigned char skc_unused:2; + unsigned char skc_bypass_rfs:1; + unsigned char skc_unused:1; int skc_bound_dev_if; union { struct hlist_node skc_bind_node; @@ -395,6 +397,7 @@ struct sock { #define sk_ipv6only __sk_common.skc_ipv6only #define sk_net_refcnt __sk_common.skc_net_refcnt #define sk_bypass_prot_mem __sk_common.skc_bypass_prot_mem +#define sk_bypass_rfs __sk_common.skc_bypass_rfs #define sk_bound_dev_if __sk_common.skc_bound_dev_if #define sk_bind_node __sk_common.skc_bind_node #define sk_prot __sk_common.skc_prot diff --git a/include/uapi/asm-generic/socket.h b/include/uapi/asm-generic/socket.h index 56fed7ab27ab..6f6987cb8a53 100644 --- a/include/uapi/asm-generic/socket.h +++ b/include/uapi/asm-generic/socket.h @@ -156,6 +156,8 @@ #define SO_PEERPIDFD_THREAD 87 +#define SO_BYPASS_RFS 88 + #if !defined(__KERNEL__) #if __BITS_PER_LONG == 64 || (defined(__x86_64__) && defined(__ILP32__)) diff --git a/net/core/sock.c b/net/core/sock.c index e2401d735c5a..00808217aebf 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -144,6 +144,7 @@ #include #include +#include #include #include #include @@ -1597,6 +1598,14 @@ int sk_setsockopt(struct sock *sk, int level, int optname, reuseport_update_incoming_cpu(sk, val); break; +#ifdef CONFIG_RPS + case SO_BYPASS_RFS: + if (valbool) + sock_rps_delete_flow(sk); + sk->sk_bypass_rfs = valbool; + break; +#endif + case SO_CNX_ADVICE: if (val == 1) dst_negative_advice(sk); @@ -2118,6 +2127,12 @@ int sk_getsockopt(struct sock *sk, int level, int optname, v.val = READ_ONCE(sk->sk_incoming_cpu); break; +#ifdef CONFIG_RPS + case SO_BYPASS_RFS: + v.val = sk->sk_bypass_rfs; + break; +#endif + case SO_MEMINFO: { u32 meminfo[SK_MEMINFO_VARS]; base-commit: fa5196392691051963726787944db872420dfa75 -- 2.34.1