Add a test that measures the maximum number of random read and random write IOPS that the NBD driver can achieve. Such a test is useful to detect performance regressions in the NBD driver and to evaluate NBD performance optimizations. Minimize the overhead outside the NBD driver as follows: - Back the NBD export with a 1 GiB null_blk device that is tuned for maximum IOPS: no memory backing, immediate completion, a 4 KiB logical block size, one hardware queue per CPU and a hardware queue depth of 1024. Keep the device small because nbd-server opens the exported device without O_DIRECT and hence all NBD I/O passes through the page cache of the null_blk device. - Select the "none" I/O scheduler for the null_blk device and disable readahead such that each 4 KiB NBD read results in a single 4 KiB read of the null_blk device. - Connect nbd-client and nbd-server over a Unix domain socket if nbd-client supports the -unix option. Fall back to a TCP connection over the loopback interface otherwise. - Use one NBD connection per CPU, with a maximum of 16 connections. - Measure the IOPS with fio using the io_uring I/O engine, the "none" I/O scheduler for the NBD device, direct I/O, 4 KiB requests, a queue depth of 64 and one job per NBD connection. Run a random read pass and a random write pass that each take half of the test time and report the results via the "read iops" and "write iops" performance fields. Note: since nbd-server performs buffered writes, the write IOPS mainly reflect how fast nbd-server can dirty pages in the page cache. Fail the test if configuring the null_blk request queue fails or if nbd-server does not start listening in time. Signed-off-by: Bart Van Assche --- tests/nbd/006 | 159 ++++++++++++++++++++++++++++++++++++++++++++++ tests/nbd/006.out | 2 + 2 files changed, 161 insertions(+) create mode 100755 tests/nbd/006 create mode 100644 tests/nbd/006.out diff --git a/tests/nbd/006 b/tests/nbd/006 new file mode 100755 index 000000000000..cd3e705f4b2f --- /dev/null +++ b/tests/nbd/006 @@ -0,0 +1,159 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-3.0+ +# Copyright (C) 2026 Google LLC +# +# Measure the random read and random write IOPS of an NBD device that is backed +# by a null_blk device. Use a Unix domain socket for the NBD connections if +# supported by nbd-client and nbd-server and TCP otherwise. + +. tests/nbd/rc +. common/null_blk + +DESCRIPTION="measure NBD read and write IOPS with null_blk as backing device" +TIMED=1 + +requires() { + _have_nbd + _have_null_blk + _have_fio +} + +# Whether nbd-client supports connecting over a Unix domain socket. The +# nbd-server "unixsock" configuration option has been introduced in the same +# nbd release as the nbd-client -unix option. +_nbd_have_unix_socket() { + nbd-client --help 2>&1 | grep -q -- '-unix' +} + +test() { + local nullb=nullb1 nullb_dev=/dev/nullb1 nbd_dev=/dev/nbd0 + local sock="${TMPDIR}/nbd.sock" pid_file="${TMPDIR}/nbd.pid" + local -i nr_cpus nr_conns i + local -a connect_args + local use_unix=0 + + echo "Running ${TEST_NAME}" + + nr_cpus=$(nproc) + nr_conns=$((nr_cpus < 16 ? nr_cpus : 16)) + + # Load null_blk without creating any device and next create a single + # null_blk device via configfs that is tuned for maximum IOPS: no + # memory backing, immediate completion and one hardware queue per CPU. + if ! _init_null_blk nr_devices=0; then + echo "Loading null_blk failed" + return 1 + fi + local nullb_params=( + blocksize=4096 + completion_nsec=0 + hw_queue_depth=1024 + irqmode=0 + memory_backed=0 + queue_mode=2 + size=1024 # MiB + submit_queues="$nr_cpus" + power=1 + ) + if ! _configure_null_blk "$nullb" "${nullb_params[@]}"; then + echo "Configuring null_blk failed" + _exit_null_blk + return 1 + fi + # nbd-server opens the exported device without O_DIRECT. Disable + # readahead such that each NBD read causes a single 4 KiB read of the + # null_blk device. + if ! _write_queue_sched none "/sys/block/${nullb}/queue/scheduler" || + ! grep -q '\[none\]' "/sys/block/${nullb}/queue/scheduler" || + ! echo 0 > "/sys/block/${nullb}/queue/read_ahead_kb"; then + echo "Configuring the ${nullb} request queue failed" + _exit_null_blk + return 1 + fi + + _nbd_have_unix_socket && use_unix=1 + + { + echo "[generic]" + echo "allowlist=true" + if ((use_unix)); then + echo "unixsock=${sock}" + fi + echo "[export]" + echo "exportname=${nullb_dev}" + } > "${TMPDIR}/nbd.conf" + cat "${TMPDIR}/nbd.conf" >> "$FULL" + + if ! nbd-server -p "$pid_file" -C "${TMPDIR}/nbd.conf" \ + >> "$FULL" 2>&1; then + echo "Starting nbd-server failed" + _exit_null_blk + return 1 + fi + + if ((use_unix)); then + echo "Using Unix domain socket ${sock}" >> "$FULL" + connect_args=(-unix "$sock") + for ((i = 0; i < 100; i++)); do + [[ -S "$sock" ]] && break + sleep .1 + done + else + echo "Using TCP socket" >> "$FULL" + connect_args=(localhost) + for ((i = 0; i < 100; i++)); do + nbd-client -l localhost >> "$FULL" 2>&1 && break + sleep .1 + done + fi + if ((i >= 100)); then + echo "Timeout while waiting for nbd-server" + kill -SIGTERM "$(cat "$pid_file")" + _exit_null_blk + return 1 + fi + + # Use -connections instead of -C because some nbd-client versions + # do not declare that -C takes an argument and pass a NULL pointer + # to strtol() if -C is used. + if ! nbd-client -L -N export -connections "$nr_conns" -b 4096 \ + "${connect_args[@]}" "$nbd_dev" >> "$FULL" 2>&1 || + ! _wait_for_nbd_connect; then + echo "Connecting to nbd-server failed" + kill -SIGTERM "$(cat "$pid_file")" + _exit_null_blk + return 1 + fi + udevadm settle + + # _run_fio passes --group_reporting=1 and --runtime="$TIMEOUT" to fio. + : "${TIMEOUT:=60}" + _divide_timeout 2 + + local fio_params=( + --bs=4k + --direct=1 + --filename="$nbd_dev" + --iodepth=64 + --ioengine=io_uring + --ioscheduler=none + --norandommap + --numjobs="$nr_conns" + --time_based=1 + ) + local rw + for rw in read write; do + FIO_PERF_FIELDS=("${rw} iops") + _fio_perf --name="nbd-${rw}-iops" --rw="rand${rw}" \ + "${fio_params[@]}" + done + + nbd-client -d "$nbd_dev" >> "$FULL" 2>&1 + _wait_for_nbd_disconnect + kill -SIGTERM "$(cat "$pid_file")" + rm -f "$pid_file" "$sock" + udevadm settle + _exit_null_blk + + echo "Test complete" +} diff --git a/tests/nbd/006.out b/tests/nbd/006.out new file mode 100644 index 000000000000..acc8af5bfb2e --- /dev/null +++ b/tests/nbd/006.out @@ -0,0 +1,2 @@ +Running nbd/006 +Test complete