Add a test that measures the random read and random write IOPS that the NBD driver can achieve, both for a single outstanding request and for the maximum achievable parallelism. Such a test is useful to detect latency and throughput regressions in the NBD driver and to evaluate NBD performance optimizations. Minimize the overhead outside the NBD driver as follows: - Back the NBD export with a 1 GiB null_blk device that is tuned for maximum IOPS: no memory backing, immediate completion, a 4 KiB logical block size, one hardware queue per CPU and a hardware queue depth of 1024. Keep the device small because nbd-server opens the exported device without O_DIRECT and hence all NBD I/O passes through the page cache of the null_blk device. - Select the "none" I/O scheduler for the null_blk device and disable readahead such that each 4 KiB NBD read results in a single 4 KiB read of the null_blk device. - Connect nbd-client and nbd-server over a Unix domain socket if nbd-client supports the -unix option and nbd-server supports the "unixsock" configuration option. Fall back to a TCP connection over the loopback interface otherwise or if the Unix domain socket does not appear. - Use one NBD connection per CPU, with a maximum of 16 connections. - Measure the IOPS with fio using the io_uring I/O engine, the "none" I/O scheduler for the NBD device, direct I/O and 4 KiB requests. Run fio four times, each run taking a quarter of the test time: - Random reads and random writes with a single job and a queue depth of one. These results reflect the NBD round-trip latency. - Random reads and random writes with one job per NBD connection and a queue depth of 64. These results reflect the maximum NBD throughput. Report the results via the "read iops" and "write iops" performance fields, prefixed with "numjobs= iodepth= " to tell the two configurations apart. Note: since nbd-server performs buffered writes, the write IOPS mainly reflect how fast nbd-server can dirty pages in the page cache. Fail the test if configuring the null_blk request queue fails or if nbd-server does not start listening on the TCP port in time. Fall back to TCP instead of failing if the Unix domain socket does not appear in time. Signed-off-by: Bart Van Assche --- tests/nbd/006 | 202 ++++++++++++++++++++++++++++++++++++++++++++++ tests/nbd/006.out | 2 + 2 files changed, 204 insertions(+) create mode 100755 tests/nbd/006 create mode 100644 tests/nbd/006.out diff --git a/tests/nbd/006 b/tests/nbd/006 new file mode 100755 index 000000000000..03bb0fc66bb4 --- /dev/null +++ b/tests/nbd/006 @@ -0,0 +1,202 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-3.0+ +# Copyright (C) 2026 Google LLC +# +# Measure the random read and random write IOPS of an NBD device that is backed +# by a null_blk device. Use a Unix domain socket for the NBD connections if +# supported by nbd-client and nbd-server and TCP otherwise. + +. tests/nbd/rc +. common/null_blk + +DESCRIPTION="measure NBD read and write IOPS with null_blk as backing device" +TIMED=1 + +requires() { + _have_nbd + _have_null_blk + _have_fio +} + +# Whether both nbd-client and nbd-server support Unix domain sockets: +# nbd-client must support the -unix option and nbd-server must recognize the +# "unixsock" configuration option. Since nbd-server does not report which +# configuration options it supports, look for the option name in the +# nbd-server executable. +_nbd_have_unix_socket() { + local server + + nbd-client --help 2>&1 | + grep -qE -- '(^|[^[:alnum:]-])-unix([^[:alnum:]-]|$)' || return 1 + server=$(type -P nbd-server) || return 1 + grep -aqw unixsock "$server" +} + +# Undo those setup steps of test() that have been completed. Uses the local +# variables of test() via dynamic scoping. +cleanup() { + local -i j + local pid + + if ((nbd_connected)); then + nbd-client -d "$nbd_dev" >> "$FULL" 2>&1 + _wait_for_nbd_disconnect + fi + if [[ -s "$pid_file" ]]; then + pid=$(<"$pid_file") + kill -SIGTERM "$pid" + # Wait until nbd-server has exited such that it no longer keeps + # the null_blk device open. + for ((j = 0; j < 100; j++)); do + kill -0 "$pid" 2>/dev/null || break + sleep .1 + done + fi + rm -f "$pid_file" "$sock" + udevadm settle + if ((null_blk_loaded)); then + _exit_null_blk + fi +} + +test() { + local nullb=nullb1 nullb_dev=/dev/nullb1 nbd_dev=/dev/nbd0 + local sock="${TMPDIR}/nbd.sock" pid_file="${TMPDIR}/nbd.pid" + local conf="${TMPDIR}/nbd.conf" + local -i nr_cpus nr_conns i use_unix=0 + local -a connect_args + # Setup progress, used by cleanup(). + local -i null_blk_loaded=0 nbd_connected=0 + + echo "Running ${TEST_NAME}" + + nr_cpus=$(nproc) + nr_conns=$((nr_cpus < 16 ? nr_cpus : 16)) + + # Load null_blk without creating any device and next create a single + # null_blk device via configfs that is tuned for maximum IOPS: no + # memory backing, immediate completion and one hardware queue per CPU. + if ! _init_null_blk nr_devices=0; then + echo "Loading null_blk failed" + return 1 + fi + null_blk_loaded=1 + local nullb_params=( + blocksize=4096 + completion_nsec=0 + hw_queue_depth=1024 + irqmode=0 + memory_backed=0 + queue_mode=2 + size=1024 # MiB + submit_queues="$nr_cpus" + power=1 + ) + if ! _configure_null_blk "$nullb" "${nullb_params[@]}"; then + echo "Configuring null_blk failed" + cleanup + return 1 + fi + # nbd-server opens the exported device without O_DIRECT. Disable + # readahead such that each NBD read causes a single 4 KiB read of the + # null_blk device. + if ! _write_queue_sched none "/sys/block/${nullb}/queue/scheduler" || + ! grep -q '\[none\]' "/sys/block/${nullb}/queue/scheduler" || + ! echo 0 > "/sys/block/${nullb}/queue/read_ahead_kb"; then + echo "Configuring the ${nullb} request queue failed" + cleanup + return 1 + fi + + _nbd_have_unix_socket && use_unix=1 + + { + echo "[generic]" + echo "allowlist=true" + if ((use_unix)); then + echo "unixsock=${sock}" + fi + echo "[export]" + echo "exportname=${nullb_dev}" + } > "$conf" + cat "$conf" >> "$FULL" + + if ! nbd-server -p "$pid_file" -C "$conf" >> "$FULL" 2>&1; then + echo "Starting nbd-server failed" + cleanup + return 1 + fi + + # Wait until nbd-server listens on the Unix domain socket. Fall back to + # TCP if the Unix domain socket does not show up. + if ((use_unix)); then + for ((i = 0; i < 100; i++)); do + [[ -S "$sock" ]] && break + sleep .1 + done + if ((i >= 100)); then + echo "${sock} did not appear; falling back to TCP" >> "$FULL" + use_unix=0 + fi + fi + if ((use_unix)); then + echo "Using Unix domain socket ${sock}" >> "$FULL" + connect_args=(-unix "$sock") + else + echo "Using TCP socket" >> "$FULL" + connect_args=(localhost) + for ((i = 0; i < 100; i++)); do + nbd-client -l localhost >> "$FULL" 2>&1 && break + sleep .1 + done + if ((i >= 100)); then + echo "Timeout while waiting for nbd-server" + cleanup + return 1 + fi + fi + + # Use -connections instead of -C because some nbd-client versions + # do not declare that -C takes an argument and pass a NULL pointer + # to strtol() if -C is used. + nbd_connected=1 + if ! nbd-client -L -N export -connections "$nr_conns" -b 4096 \ + "${connect_args[@]}" "$nbd_dev" >> "$FULL" 2>&1 || + ! _wait_for_nbd_connect; then + echo "Connecting to nbd-server failed" + cleanup + return 1 + fi + udevadm settle + + # Measure the read and write IOPS twice: once with a single job and + # queue depth one and once with the parameters that maximize the IOPS. + # _run_fio passes --group_reporting=1 and --runtime="$TIMEOUT" to fio. + : "${TIMEOUT:=60}" + _divide_timeout 4 + + local fio_params=( + --bs=4k + --direct=1 + --filename="$nbd_dev" + --ioengine=io_uring + --ioscheduler=none + --norandommap + --time_based=1 + ) + local rw numjobs iodepth numjobs_iodepth + for numjobs_iodepth in "1 1" "${nr_conns} 64"; do + read -r numjobs iodepth <<< "$numjobs_iodepth" + FIO_PERF_PREFIX="numjobs=${numjobs} iodepth=${iodepth} " + for rw in read write; do + FIO_PERF_FIELDS=("${rw} iops") + _fio_perf --name="nbd-${rw}-iops" --rw="rand${rw}" \ + --numjobs="$numjobs" --iodepth="$iodepth" \ + "${fio_params[@]}" + done + done + + cleanup + + echo "Test complete" +} diff --git a/tests/nbd/006.out b/tests/nbd/006.out new file mode 100644 index 000000000000..acc8af5bfb2e --- /dev/null +++ b/tests/nbd/006.out @@ -0,0 +1,2 @@ +Running nbd/006 +Test complete