From: Bobby Eshleman Add test cases for autorelease and new edge cases. A happy path check_rx / check_rx_autorelease test is added. The test check_unbind_before_recv/_autorelease tests that after a connection is accepted, but before recv is called, that unbind behaves correctly. The test check_unbind_after_recv/_autorelease tests that after a connection is accepted, but after recv is called, that unbind behaves correctly. To facilitate the unbind tests, ncdevmem is changed to take an "unbind" mode. The unbind modes are defined as the following: UNBIND_MODE_NORMAL: unbind after done with normal traffic UNBIND_MODE_BEFORE_RECV: Unbind before any recvmsg. The socket hasn't become a user yet, so binding->users reaches zero and recvmsg should fail with ENODEV. This validates that sockets can't access devmem after the binding is torn down. UNBIND_MODE_AFTER_RECV: Do one recvmsg first (socket becomes a user), then unbind, then continue receiving. This validates that binding->users keeps the binding alive for sockets that already acquired a reference via recvmsg. ncdevmem is also changed to take an autorelease flag for toggling the autorelease mode. TAP version 13 1..8 ok 1 devmem.check_rx ok 2 devmem.check_rx_autorelease ok 3 devmem.check_unbind_before_recv ok 4 devmem.check_unbind_before_recv_autorelease ok 5 devmem.check_unbind_after_recv ok 6 devmem.check_unbind_after_recv_autorelease ok 7 devmem.check_tx ok 8 devmem.check_tx_chunks Signed-off-by: Bobby Eshleman --- Changes in v10: - add tests for "unbind before/after recv" edge cases Changes in v8: - removed stale/missing tests Changes in v7: - use autorelease netlink - remove sockopt tests --- tools/testing/selftests/drivers/net/hw/devmem.py | 98 ++++++++++++++++++++++- tools/testing/selftests/drivers/net/hw/ncdevmem.c | 68 ++++++++++++++-- 2 files changed, 157 insertions(+), 9 deletions(-) diff --git a/tools/testing/selftests/drivers/net/hw/devmem.py b/tools/testing/selftests/drivers/net/hw/devmem.py index 45c2d49d55b6..0bbfdf19e23d 100755 --- a/tools/testing/selftests/drivers/net/hw/devmem.py +++ b/tools/testing/selftests/drivers/net/hw/devmem.py @@ -25,7 +25,98 @@ def check_rx(cfg) -> None: port = rand_port() socat = f"socat -u - TCP{cfg.addr_ipver}:{cfg.baddr}:{port},bind={cfg.remote_baddr}:{port}" - listen_cmd = f"{cfg.bin_local} -l -f {cfg.ifname} -s {cfg.addr} -p {port} -c {cfg.remote_addr} -v 7" + listen_cmd = f"{cfg.bin_local} -l -f {cfg.ifname} -s {cfg.addr} -p {port} \ + -c {cfg.remote_addr} -v 7 -a 0" + + with bkg(listen_cmd, exit_wait=True) as ncdevmem: + wait_port_listen(port) + cmd(f"yes $(echo -e \x01\x02\x03\x04\x05\x06) | \ + head -c 1K | {socat}", host=cfg.remote, shell=True) + + ksft_eq(ncdevmem.ret, 0) + + +@ksft_disruptive +def check_rx_autorelease(cfg) -> None: + """Test devmem TCP receive with autorelease mode enabled.""" + require_devmem(cfg) + + port = rand_port() + socat = f"socat -u - TCP{cfg.addr_ipver}:{cfg.baddr}:{port},bind={cfg.remote_baddr}:{port}" + listen_cmd = f"{cfg.bin_local} -l -f {cfg.ifname} -s {cfg.addr} -p {port} \ + -c {cfg.remote_addr} -v 7 -a 1" + + with bkg(listen_cmd, exit_wait=True) as ncdevmem: + wait_port_listen(port) + cmd(f"yes $(echo -e \x01\x02\x03\x04\x05\x06) | \ + head -c 1K | {socat}", host=cfg.remote, shell=True) + + ksft_eq(ncdevmem.ret, 0) + + +@ksft_disruptive +def check_unbind_before_recv(cfg) -> None: + """Test dmabuf unbind before socket recv with autorelease disabled.""" + require_devmem(cfg) + + port = rand_port() + socat = f"socat -u - TCP{cfg.addr_ipver}:{cfg.baddr}:{port},bind={cfg.remote_baddr}:{port}" + listen_cmd = f"{cfg.bin_local} -l -f {cfg.ifname} -s {cfg.addr} -p {port} \ + -c {cfg.remote_addr} -v 7 -a 0 -U 1" + + with bkg(listen_cmd, exit_wait=True) as ncdevmem: + wait_port_listen(port) + cmd(f"yes $(echo -e \x01\x02\x03\x04\x05\x06) | \ + head -c 1K | {socat}", host=cfg.remote, shell=True) + + ksft_eq(ncdevmem.ret, 0) + + +@ksft_disruptive +def check_unbind_before_recv_autorelease(cfg) -> None: + """Test dmabuf unbind before socket recv with autorelease enabled.""" + require_devmem(cfg) + + port = rand_port() + socat = f"socat -u - TCP{cfg.addr_ipver}:{cfg.baddr}:{port},bind={cfg.remote_baddr}:{port}" + listen_cmd = f"{cfg.bin_local} -l -f {cfg.ifname} -s {cfg.addr} -p {port} \ + -c {cfg.remote_addr} -v 7 -a 1 -U 1" + + with bkg(listen_cmd, exit_wait=True) as ncdevmem: + wait_port_listen(port) + cmd(f"yes $(echo -e \x01\x02\x03\x04\x05\x06) | \ + head -c 1K | {socat}", host=cfg.remote, shell=True) + + ksft_eq(ncdevmem.ret, 0) + + +@ksft_disruptive +def check_unbind_after_recv(cfg) -> None: + """Test dmabuf unbind after socket recv with autorelease disabled.""" + require_devmem(cfg) + + port = rand_port() + socat = f"socat -u - TCP{cfg.addr_ipver}:{cfg.baddr}:{port},bind={cfg.remote_baddr}:{port}" + listen_cmd = f"{cfg.bin_local} -l -f {cfg.ifname} -s {cfg.addr} -p {port} \ + -c {cfg.remote_addr} -v 7 -a 0 -U 2" + + with bkg(listen_cmd, exit_wait=True) as ncdevmem: + wait_port_listen(port) + cmd(f"yes $(echo -e \x01\x02\x03\x04\x05\x06) | \ + head -c 1K | {socat}", host=cfg.remote, shell=True) + + ksft_eq(ncdevmem.ret, 0) + + +@ksft_disruptive +def check_unbind_after_recv_autorelease(cfg) -> None: + """Test dmabuf unbind after socket recv with autorelease enabled.""" + require_devmem(cfg) + + port = rand_port() + socat = f"socat -u - TCP{cfg.addr_ipver}:{cfg.baddr}:{port},bind={cfg.remote_baddr}:{port}" + listen_cmd = f"{cfg.bin_local} -l -f {cfg.ifname} -s {cfg.addr} -p {port} \ + -c {cfg.remote_addr} -v 7 -a 1 -U 2" with bkg(listen_cmd, exit_wait=True) as ncdevmem: wait_port_listen(port) @@ -68,7 +159,10 @@ def main() -> None: cfg.bin_local = path.abspath(path.dirname(__file__) + "/ncdevmem") cfg.bin_remote = cfg.remote.deploy(cfg.bin_local) - ksft_run([check_rx, check_tx, check_tx_chunks], + ksft_run([check_rx, check_rx_autorelease, + check_unbind_before_recv, check_unbind_before_recv_autorelease, + check_unbind_after_recv, check_unbind_after_recv_autorelease, + check_tx, check_tx_chunks], args=(cfg, )) ksft_exit() diff --git a/tools/testing/selftests/drivers/net/hw/ncdevmem.c b/tools/testing/selftests/drivers/net/hw/ncdevmem.c index 3288ed04ce08..5cbff3c602b2 100644 --- a/tools/testing/selftests/drivers/net/hw/ncdevmem.c +++ b/tools/testing/selftests/drivers/net/hw/ncdevmem.c @@ -85,6 +85,13 @@ #define MAX_IOV 1024 +enum unbind_mode_type { + UNBIND_MODE_NORMAL, + UNBIND_MODE_BEFORE_RECV, + UNBIND_MODE_AFTER_RECV, + UNBIND_MODE_INVAL, +}; + static size_t max_chunk; static char *server_ip; static char *client_ip; @@ -92,6 +99,8 @@ static char *port; static size_t do_validation; static int start_queue = -1; static int num_queues = -1; +static int devmem_autorelease; +static enum unbind_mode_type unbind_mode; static char *ifname; static unsigned int ifindex; static unsigned int dmabuf_id; @@ -679,7 +688,8 @@ static int configure_flow_steering(struct sockaddr_in6 *server_sin) static int bind_rx_queue(unsigned int ifindex, unsigned int dmabuf_fd, struct netdev_queue_id *queues, - unsigned int n_queue_index, struct ynl_sock **ys) + unsigned int n_queue_index, struct ynl_sock **ys, + int autorelease) { struct netdev_bind_rx_req *req = NULL; struct netdev_bind_rx_rsp *rsp = NULL; @@ -695,6 +705,7 @@ static int bind_rx_queue(unsigned int ifindex, unsigned int dmabuf_fd, req = netdev_bind_rx_req_alloc(); netdev_bind_rx_req_set_ifindex(req, ifindex); netdev_bind_rx_req_set_fd(req, dmabuf_fd); + netdev_bind_rx_req_set_autorelease(req, autorelease); __netdev_bind_rx_req_set_queues(req, queues, n_queue_index); rsp = netdev_bind_rx(*ys, req); @@ -872,7 +883,8 @@ static int do_server(struct memory_buffer *mem) goto err_reset_rss; } - if (bind_rx_queue(ifindex, mem->fd, create_queues(), num_queues, &ys)) { + if (bind_rx_queue(ifindex, mem->fd, create_queues(), num_queues, &ys, + devmem_autorelease)) { pr_err("Failed to bind"); goto err_reset_flow_steering; } @@ -922,6 +934,23 @@ static int do_server(struct memory_buffer *mem) fprintf(stderr, "Got connection from %s:%d\n", buffer, ntohs(client_addr.sin6_port)); + if (unbind_mode == UNBIND_MODE_BEFORE_RECV) { + struct pollfd pfd = { + .fd = client_fd, + .events = POLLIN, + }; + + /* Wait for data then unbind (before recvmsg) */ + ret = poll(&pfd, 1, 5000); + if (ret <= 0) { + pr_err("poll failed or timed out waiting for data"); + goto err_close_client; + } + + ynl_sock_destroy(ys); + ys = NULL; + } + while (1) { struct iovec iov = { .iov_base = iobuf, .iov_len = sizeof(iobuf) }; @@ -942,11 +971,19 @@ static int do_server(struct memory_buffer *mem) if (ret < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) continue; if (ret < 0) { + if (unbind_mode == UNBIND_MODE_BEFORE_RECV && + errno == ENODEV) + goto cleanup; + perror("recvmsg"); if (errno == EFAULT) { pr_err("received EFAULT, won't recover"); goto err_close_client; } + if (errno == ENODEV) { + pr_err("unexpected ENODEV"); + goto err_close_client; + } continue; } if (ret == 0) { @@ -1025,6 +1062,11 @@ static int do_server(struct memory_buffer *mem) goto err_close_client; } + if (unbind_mode == UNBIND_MODE_AFTER_RECV && ys) { + ynl_sock_destroy(ys); + ys = NULL; + } + fprintf(stderr, "total_received=%lu\n", total_received); } @@ -1043,7 +1085,8 @@ static int do_server(struct memory_buffer *mem) err_free_tmp: free(tmp_mem); err_unbind: - ynl_sock_destroy(ys); + if (ys) + ynl_sock_destroy(ys); err_reset_flow_steering: reset_flow_steering(); err_reset_rss: @@ -1092,7 +1135,7 @@ int run_devmem_tests(void) goto err_reset_headersplit; } - if (!bind_rx_queue(ifindex, mem->fd, queues, num_queues, &ys)) { + if (!bind_rx_queue(ifindex, mem->fd, queues, num_queues, &ys, 0)) { pr_err("Binding empty queues array should have failed"); goto err_unbind; } @@ -1108,7 +1151,7 @@ int run_devmem_tests(void) goto err_reset_headersplit; } - if (!bind_rx_queue(ifindex, mem->fd, queues, num_queues, &ys)) { + if (!bind_rx_queue(ifindex, mem->fd, queues, num_queues, &ys, 0)) { pr_err("Configure dmabuf with header split off should have failed"); goto err_unbind; } @@ -1124,7 +1167,7 @@ int run_devmem_tests(void) goto err_reset_headersplit; } - if (bind_rx_queue(ifindex, mem->fd, queues, num_queues, &ys)) { + if (bind_rx_queue(ifindex, mem->fd, queues, num_queues, &ys, 0)) { pr_err("Failed to bind"); goto err_reset_headersplit; } @@ -1397,7 +1440,7 @@ int main(int argc, char *argv[]) int is_server = 0, opt; int ret, err = 1; - while ((opt = getopt(argc, argv, "ls:c:p:v:q:t:f:z:")) != -1) { + while ((opt = getopt(argc, argv, "ls:c:p:v:q:t:f:z:a:U:")) != -1) { switch (opt) { case 'l': is_server = 1; @@ -1426,6 +1469,12 @@ int main(int argc, char *argv[]) case 'z': max_chunk = atoi(optarg); break; + case 'a': + devmem_autorelease = atoi(optarg); + break; + case 'U': + unbind_mode = atoi(optarg); + break; case '?': fprintf(stderr, "unknown option: %c\n", optopt); break; @@ -1437,6 +1486,11 @@ int main(int argc, char *argv[]) return 1; } + if (unbind_mode >= UNBIND_MODE_INVAL) { + pr_err("invalid unbind mode %u\n", unbind_mode); + return 1; + } + ifindex = if_nametoindex(ifname); fprintf(stderr, "using ifindex=%u\n", ifindex); -- 2.47.3