nvme-tcp exposes /sys/class/nvme/nvmeX/queues//io_cpu and managed, so a control plane can choose the CPU that runs each I/O queue's socket work [1]. Test that - io_cpu reads back a written CPU and rejects invalid values - managed is read only and turns 0 once io_cpu is written - an assignment survives a controller reset - the queue directories follow the queue count as attr_qid_max shrinks and grows it - removing the controller while io_cpu is written completes The test skips when the kernel does not have the attribute. [1] https://lore.kernel.org/linux-nvme/20261018010059.83120-1-saravanand@crusoe.ai/ Assisted-by: Claude:claude-opus-5-5 [Claude Code] Signed-off-by: Saravanan D --- This tests the kernel series in the parent thread, which is under review on linux-nvme. tests/nvme/075 | 185 +++++++++++++++++++++++++++++++++++++++++++++ tests/nvme/075.out | 2 + 2 files changed, 187 insertions(+) create mode 100755 tests/nvme/075 create mode 100644 tests/nvme/075.out diff --git a/tests/nvme/075 b/tests/nvme/075 new file mode 100755 index 0000000..0b4bce5 --- /dev/null +++ b/tests/nvme/075 @@ -0,0 +1,185 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-3.0+ +# Copyright (C) 2026 Saravanan D +# +# Test the per-queue io_cpu sysfs attribute of nvme-tcp + +. tests/nvme/rc + +DESCRIPTION="test the per-queue io_cpu sysfs attribute" + +requires() { + _nvme_requires + _have_loop + _require_nvme_trtype tcp + _require_min_cpus 2 +} + +set_conditions() { + _set_nvme_trtype "$@" +} + +# last CPU in the online list, and the first CPU id past the possible ones +last_online_cpu() { + awk -F'[-,]' '{ print $NF }' /sys/devices/system/cpu/online +} + +first_impossible_cpu() { + echo $(( $(awk -F'[-,]' '{ print $NF }' /sys/devices/system/cpu/possible) + 1 )) +} + +nr_queue_dirs() { + local nvmedev="$1" + + find "/sys/class/nvme/${nvmedev}/queues" -mindepth 1 -maxdepth 1 \ + -type d 2>/dev/null | wc -l +} + +expect_attr() { + local file="$1" + local expected="$2" + local value + + value=$(cat "${file}") + if [[ "${value}" != "${expected}" ]]; then + echo "${file}: read ${value}, expected ${expected}" + return 1 + fi + return 0 +} + +test_read_write() { + local qdir="$1" + local cpu="$2" + local bad + + expect_attr "${qdir}/1/managed" 1 + + echo "${cpu}" > "${qdir}/1/io_cpu" + expect_attr "${qdir}/1/io_cpu" "${cpu}" + expect_attr "${qdir}/1/managed" 0 + + for bad in "$(first_impossible_cpu)" -2 abc 0x1; do + if { echo "${bad}" > "${qdir}/1/io_cpu"; } 2>/dev/null; then + echo "io_cpu accepted ${bad}" + fi + done + expect_attr "${qdir}/1/io_cpu" "${cpu}" + + if { echo 1 > "${qdir}/1/managed"; } 2>/dev/null; then + echo "managed is writable" + fi + + echo unbound > "${qdir}/2/io_cpu" + expect_attr "${qdir}/2/io_cpu" -1 + echo -1 > "${qdir}/2/io_cpu" + expect_attr "${qdir}/2/io_cpu" -1 + expect_attr "${qdir}/2/managed" 0 +} + +test_reset() { + local nvmedev="$1" + local qdir="$2" + local cpu="$3" + + echo 1 > "/sys/class/nvme/${nvmedev}/reset_controller" + _nvmf_wait_for_state "${def_subsysnqn}" "live" || return 1 + + expect_attr "${qdir}/1/io_cpu" "${cpu}" + expect_attr "${qdir}/1/managed" 0 + expect_attr "${qdir}/2/io_cpu" -1 + expect_attr "${qdir}/2/managed" 0 +} + +set_qid_max() { + local nvmedev="$1" + local qid_max="$2" + local queue_count_file="/sys/class/nvme-fabrics/ctl/${nvmedev}/queue_count" + local retries=10 + + echo "${qid_max}" > \ + "${NVMET_CFS}/subsystems/${def_subsysnqn}/attr_qid_max" + while [[ "$(cat "${queue_count_file}")" -ne $((qid_max + 1)) ]]; do + if (( retries-- == 0 )); then + echo "queue count did not change to $((qid_max + 1))" + return 1 + fi + sleep 1 + done + _nvmf_wait_for_state "${def_subsysnqn}" "live" || return 1 + + if [[ "$(nr_queue_dirs "${nvmedev}")" -ne "${qid_max}" ]]; then + echo "$(nr_queue_dirs "${nvmedev}") queue directories," \ + "expected ${qid_max}" + return 1 + fi + return 0 +} + +io_cpu_writer() { + local qdir="$1" + local cpu="$2" + local file + + while true; do + for file in "${qdir}"/*/io_cpu; do + { echo "${cpu}" > "${file}"; } 2>/dev/null + { echo unbound > "${file}"; } 2>/dev/null + done + done +} + +test() { + echo "Running ${TEST_NAME}" + + _setup_nvmet + + local cfs_path="${NVMET_CFS}/subsystems/${def_subsysnqn}" + local skipped=false + local nvmedev qdir cpu writer + + _nvmet_target_setup --blkdev file + + _nvme_connect_subsys --keep-alive-tmo 1 --reconnect-delay 2 + _nvmf_wait_for_state "${def_subsysnqn}" "live" + nvmedev=$(_find_nvme_dev "${def_subsysnqn}") + qdir="/sys/class/nvme/${nvmedev}/queues" + cpu=$(last_online_cpu) + + if [[ ! -f "${qdir}/1/io_cpu" ]]; then + SKIP_REASONS+=("missing per-queue io_cpu attribute") + skipped=true + elif [[ "$(nr_queue_dirs "${nvmedev}")" -lt 2 ]]; then + SKIP_REASONS+=("controller has fewer than 2 I/O queues") + skipped=true + else + test_read_write "${qdir}" "${cpu}" + test_reset "${nvmedev}" "${qdir}" "${cpu}" + + if [[ -f "${cfs_path}/attr_qid_max" ]]; then + set_qid_max "${nvmedev}" 1 + expect_attr "${qdir}/1/io_cpu" "${cpu}" + set_qid_max "${nvmedev}" 2 + fi + + # remove the controller while its queue directories are written + io_cpu_writer "${qdir}" "${cpu}" & + writer=$! + sleep 1 + fi + + _nvme_disconnect_subsys + + if [[ -n "${writer}" ]]; then + kill "${writer}" + wait "${writer}" 2>/dev/null + fi + + _nvmet_target_cleanup + + if [[ "${skipped}" = true ]] ; then + return 1 + fi + + echo "Test complete" +} diff --git a/tests/nvme/075.out b/tests/nvme/075.out new file mode 100644 index 0000000..0613dfa --- /dev/null +++ b/tests/nvme/075.out @@ -0,0 +1,2 @@ +Running nvme/075 +Test complete -- 2.55.0