Add a test which checks that the RSS table is at least 4x the max queue count supported by the device. The original RSS spec from Microsoft stated that the RSS indirection table should be 2 to 8 times the CPU count, presumably assuming queue per CPU. If the CPU count is not a power of two, however, a power-of-2 table 2x larger than queue count results in a 33% traffic imbalance. Validate that the indirection table is at least 4x the queue count. This lowers the imbalance to 16% which empirically appears to be more acceptable to memcache-like workloads. Reviewed-by: Willem de Bruijn Signed-off-by: Jakub Kicinski --- v2: - no changes (see patch 2) v1: https://lore.kernel.org/20260130192912.826454-1-kuba@kernel.org CC: shuah@kernel.org CC: linux-kselftest@vger.kernel.org --- .../testing/selftests/drivers/net/hw/Makefile | 1 + .../selftests/drivers/net/hw/rss_drv.py | 88 +++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100755 tools/testing/selftests/drivers/net/hw/rss_drv.py diff --git a/tools/testing/selftests/drivers/net/hw/Makefile b/tools/testing/selftests/drivers/net/hw/Makefile index 9c163ba6feee..a64140333a46 100644 --- a/tools/testing/selftests/drivers/net/hw/Makefile +++ b/tools/testing/selftests/drivers/net/hw/Makefile @@ -35,6 +35,7 @@ TEST_PROGS = \ pp_alloc_fail.py \ rss_api.py \ rss_ctx.py \ + rss_drv.py \ rss_flow_label.py \ rss_input_xfrm.py \ toeplitz.py \ diff --git a/tools/testing/selftests/drivers/net/hw/rss_drv.py b/tools/testing/selftests/drivers/net/hw/rss_drv.py new file mode 100755 index 000000000000..2d1a33189076 --- /dev/null +++ b/tools/testing/selftests/drivers/net/hw/rss_drv.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 + +""" +Driver-related behavior tests for RSS. +""" + +from lib.py import ksft_run, ksft_exit, ksft_ge +from lib.py import ksft_variants, KsftNamedVariant, KsftSkipEx +from lib.py import defer, ethtool +from lib.py import EthtoolFamily, NlError +from lib.py import NetDrvEnv + + +def _is_power_of_two(n): + return n > 0 and (n & (n - 1)) == 0 + + +def _get_rss(cfg, context=0): + return ethtool(f"-x {cfg.ifname} context {context}", json=True)[0] + + +def _test_rss_indir_size(cfg, qcnt, context=0): + """Test that indirection table size is at least 4x queue count.""" + ethtool(f"-L {cfg.ifname} combined {qcnt}") + + rss = _get_rss(cfg, context=context) + indir = rss['rss-indirection-table'] + ksft_ge(len(indir), 4 * qcnt, "Table smaller than 4x") + return len(indir) + + +def _maybe_create_context(cfg, create_context): + """ Either create a context and return its ID or return 0 for main ctx """ + if not create_context: + return 0 + try: + ctx = cfg.ethnl.rss_create_act({'header': {'dev-index': cfg.ifindex}}) + ctx_id = ctx['context'] + defer(cfg.ethnl.rss_delete_act, + {'header': {'dev-index': cfg.ifindex}, 'context': ctx_id}) + except NlError: + raise KsftSkipEx("Device does not support additional RSS contexts") + + return ctx_id + + +@ksft_variants([ + KsftNamedVariant("main", False), + KsftNamedVariant("ctx", True), +]) +def indir_size_4x(cfg, create_context): + """ + Test that the indirection table has at least 4 entries per queue. + Empirically network-heavy workloads like memcache suffer with the 33% + imbalance of a 2x indirection table size. + 4x table translates to a 16% imbalance. + """ + channels = cfg.ethnl.channels_get({'header': {'dev-index': cfg.ifindex}}) + ch_max = channels.get('combined-max', 0) + qcnt = channels['combined-count'] + + if ch_max < 3: + raise KsftSkipEx(f"Not enough queues for the test: max={ch_max}") + + defer(ethtool, f"-L {cfg.ifname} combined {qcnt}") + ethtool(f"-L {cfg.ifname} combined 3") + + ctx_id = _maybe_create_context(cfg, create_context) + + indir_sz = _test_rss_indir_size(cfg, 3, context=ctx_id) + + # Test with max queue count (max - 1 if max is a power of two) + test_max = ch_max - 1 if _is_power_of_two(ch_max) else ch_max + if test_max > 3 and indir_sz < test_max * 4: + _test_rss_indir_size(cfg, test_max, context=ctx_id) + + +def main() -> None: + """ Ksft boiler plate main """ + with NetDrvEnv(__file__) as cfg: + cfg.ethnl = EthtoolFamily() + ksft_run([indir_size_4x], args=(cfg, )) + ksft_exit() + + +if __name__ == "__main__": + main() -- 2.52.0 Spell out the recommendation that the RSS table should be 4x the queue count to avoid traffic imbalance. Include minor rephrasing and removal of the explicit 128 entry example since a 128 entry table is inadequate on modern machines. Signed-off-by: Jakub Kicinski --- v2: - new patch CC: edumazet@google.com --- Documentation/networking/scaling.rst | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Documentation/networking/scaling.rst b/Documentation/networking/scaling.rst index 99b6a61e5e31..0023afa530ec 100644 --- a/Documentation/networking/scaling.rst +++ b/Documentation/networking/scaling.rst @@ -38,11 +38,15 @@ that is not the focus of these techniques. The filter used in RSS is typically a hash function over the network and/or transport layer headers-- for example, a 4-tuple hash over IP addresses and TCP ports of a packet. The most common hardware -implementation of RSS uses a 128-entry indirection table where each entry +implementation of RSS uses an indirection table where each entry stores a queue number. The receive queue for a packet is determined -by masking out the low order seven bits of the computed hash for the -packet (usually a Toeplitz hash), taking this number as a key into the -indirection table and reading the corresponding value. +by indexing the indirection table with the low order bits of the +computed hash for the packet (usually a Toeplitz hash). + +The indirection table helps even out the traffic distribution when queue +count is not a power of two. NICs should provide an indirection table +at least 4 times larger than the queue count. 4x table results in ~16% +imbalance between the queues, which is acceptable for most applications. Some NICs support symmetric RSS hashing where, if the IP (source address, destination address) and TCP/UDP (source port, destination port) tuples -- 2.52.0