Add a new flag, KVM_IOEVENTFD_FLAG_POST_WRITE, when assigning an ioeventfd that results in the value written by the guest to be copied to user-supplied memory instead of being discarded. The goal of this new mechanism is to speed up doorbell writes on NVMe controllers emulated outside of the VMM. Currently, a doorbell write to an NVMe SQ tail doorbell requires returning from ioctl(KVM_RUN) and the VMM communicating the event, along with the doorbell value, to the NVMe controller emulation task. With POST_WRITE, the NVMe emulation task is directly notified of the doorbell write and can find the doorbell value in a known location, without involving VMM. Add tests for this new functionality. LLM (claude-4.6-opus-high) was used mainly for the tests and to a lesser extent for pre-reviewing this patch. Signed-off-by: Thanos Makatos --- Changes since v1 (Sean): - Made test x86-only. - Added explicit padding to struct kvm_ioeventfd. - Renamed ioeventfd_test to ioeventfd_posted_write_test and moved to maintain alphabetical sort. - Dropped post_addr from kvm_assign_ioeventfd_idx. - Expanded comment in kvm_assign_ioeventfd_idx. - Refactored part of ioeventfd_in_range() into ioeventfd_get_val() and reused it. Documentation/virt/kvm/api.rst | 14 +- include/uapi/linux/kvm.h | 7 +- tools/testing/selftests/kvm/Makefile.kvm | 1 + .../kvm/x86/ioeventfd_posted_write_test.c | 623 ++++++++++++++++++ virt/kvm/eventfd.c | 94 ++- virt/kvm/kvm_main.c | 1 + 6 files changed, 719 insertions(+), 21 deletions(-) create mode 100644 tools/testing/selftests/kvm/x86/ioeventfd_posted_write_test.c diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index e95c9cda085d..a5e61ba7eaf5 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -2132,7 +2132,9 @@ provided event instead of triggering an exit. __u32 len; /* 0, 1, 2, 4, or 8 bytes */ __s32 fd; __u32 flags; - __u8 pad[36]; + __u32 pad0; + __u64 post_addr; /* address to write to if POST_WRITE is set */ + __u8 pad[24]; }; For the special case of virtio-ccw devices on s390, the ioevent is matched @@ -2145,6 +2147,7 @@ The following flags are defined:: #define KVM_IOEVENTFD_FLAG_DEASSIGN (1 << kvm_ioeventfd_flag_nr_deassign) #define KVM_IOEVENTFD_FLAG_VIRTIO_CCW_NOTIFY \ (1 << kvm_ioeventfd_flag_nr_virtio_ccw_notify) + #define KVM_IOEVENTFD_FLAG_POST_WRITE (1 << kvm_ioeventfd_flag_nr_post_write) If datamatch flag is set, the event will be signaled only if the written value to the registered address is equal to datamatch in struct kvm_ioeventfd. @@ -2157,6 +2160,15 @@ the kernel will ignore the length of guest write and may get a faster vmexit. The speedup may only apply to specific architectures, but the ioeventfd will work anyway. +With KVM_IOEVENTFD_FLAG_POST_WRITE, the value being written is copied to the +userspace address specified by post_addr, and the eventfd is signaled. The +copy is guaranteed to complete before the eventfd is signaled, so a userspace +reader that wakes on the eventfd will observe the written value. When multiple +vCPUs write to the same ioeventfd concurrently, the value at post_addr reflects +one of the writes. If the copy to post_addr fails (e.g. the memory has been +unmapped), the eventfd is not signaled and the write is reported to userspace +as a regular MMIO/PIO exit. + 4.60 KVM_DIRTY_TLB ------------------ diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 8dff2fc1972e..07ac3142ce17 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -644,6 +644,7 @@ enum { kvm_ioeventfd_flag_nr_deassign, kvm_ioeventfd_flag_nr_virtio_ccw_notify, kvm_ioeventfd_flag_nr_fast_mmio, + kvm_ioeventfd_flag_nr_post_write, kvm_ioeventfd_flag_nr_max, }; @@ -652,6 +653,7 @@ enum { #define KVM_IOEVENTFD_FLAG_DEASSIGN (1 << kvm_ioeventfd_flag_nr_deassign) #define KVM_IOEVENTFD_FLAG_VIRTIO_CCW_NOTIFY \ (1 << kvm_ioeventfd_flag_nr_virtio_ccw_notify) +#define KVM_IOEVENTFD_FLAG_POST_WRITE (1 << kvm_ioeventfd_flag_nr_post_write) #define KVM_IOEVENTFD_VALID_FLAG_MASK ((1 << kvm_ioeventfd_flag_nr_max) - 1) @@ -661,7 +663,9 @@ struct kvm_ioeventfd { __u32 len; /* 1, 2, 4, or 8 bytes; or 0 to ignore length */ __s32 fd; __u32 flags; - __u8 pad[36]; + __u32 pad0; + __u64 post_addr; /* address to write to if POST_WRITE is set */ + __u8 pad[24]; }; #define KVM_X86_DISABLE_EXITS_MWAIT (1 << 0) @@ -1000,6 +1004,7 @@ struct kvm_enable_cap { #define KVM_CAP_PPC_COMPAT_CAPS 250 #define KVM_CAP_ARM_PMU_V3_STRICT 251 #define KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES 252 +#define KVM_CAP_IOEVENTFD_POST_WRITE 253 struct kvm_irq_routing_irqchip { __u32 irqchip; diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 3cb35a6963d5..38860f2f08b0 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -92,6 +92,7 @@ TEST_GEN_PROGS_x86 += x86/hyperv_features TEST_GEN_PROGS_x86 += x86/hyperv_ipi TEST_GEN_PROGS_x86 += x86/hyperv_svm_test TEST_GEN_PROGS_x86 += x86/hyperv_tlb_flush +TEST_GEN_PROGS_x86 += x86/ioeventfd_posted_write_test TEST_GEN_PROGS_x86 += x86/kvm_clock_test TEST_GEN_PROGS_x86 += x86/kvm_pv_test TEST_GEN_PROGS_x86 += x86/kvm_buslock_test diff --git a/tools/testing/selftests/kvm/x86/ioeventfd_posted_write_test.c b/tools/testing/selftests/kvm/x86/ioeventfd_posted_write_test.c new file mode 100644 index 000000000000..0c7072e9d377 --- /dev/null +++ b/tools/testing/selftests/kvm/x86/ioeventfd_posted_write_test.c @@ -0,0 +1,623 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ioeventfd_posted_write_test.c - Tests for KVM_IOEVENTFD_FLAG_POST_WRITE. + * + * Tests that when KVM_IOEVENTFD_FLAG_POST_WRITE is set and the MMIO/PIO + * address is written to, the value is copied to the user-provided address + * and the eventfd is signaled. Also tests negative cases and interactions + * with DATAMATCH. + * + * Copyright Nutanix, 2026 + * + * Author: Thanos Makatos + */ + +#include +#include +#include + +#include "kvm_util.h" +#include "processor.h" +#include "ucall_common.h" + +#define MMIO_GPA (1UL << 30) +#define PIO_PORT 0xe000 +#define TEST_VAL 0xDEADBEEFCAFEBABEULL +#define MATCH_VAL 0x42U +#define NOMATCH_VAL (MATCH_VAL + 1) +#define POISON_VAL 0xFFFFFFFFU + +/* + * Check that the most recent vCPU exit is a ucall (delivered as KVM_EXIT_IO + * on x86) matching @expected_cmd. The caller must have already called + * vcpu_run(). + * + * @expected_cmd: UCALL_SYNC, UCALL_DONE, etc. + * @expected_stage: for UCALL_SYNC, the stage number passed by GUEST_SYNC(). + * Ignored for other ucall types. + * + * Aborts the test on UCALL_ABORT (a guest-side assertion failure). + */ +static void assert_ucall(struct kvm_vcpu *vcpu, uint64_t expected_cmd, + uint64_t expected_stage) +{ + struct ucall uc; + + TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO); + + switch (get_ucall(vcpu, &uc)) { + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + break; + case UCALL_SYNC: + TEST_ASSERT(expected_cmd == UCALL_SYNC, + "got UCALL_SYNC, expected %lu", + expected_cmd); + TEST_ASSERT(uc.args[1] == expected_stage, + "expected stage %lu, got %lu", + expected_stage, uc.args[1]); + break; + case UCALL_DONE: + TEST_ASSERT(expected_cmd == UCALL_DONE, + "got UCALL_DONE, expected %lu", + expected_cmd); + break; + default: + TEST_FAIL("unexpected ucall %lu", uc.cmd); + } +} + +/* + * Verify that KVM_IOEVENTFD rejects invalid POST_WRITE configurations: + * - len=0: the kernel needs a non-zero length to know how many bytes to copy. + * - post_addr=NULL: there is no destination for the copy. + * - post_addr outside the process address space: access_ok() rejects it. + * All three must fail with EINVAL. + */ +static void test_post_write_negative(void) +{ + struct kvm_ioeventfd ioeventfd; + struct kvm_vm *vm; + uint64_t dummy; + int ret; + int fd; + + vm = vm_create_barebones(); + fd = kvm_new_eventfd(); + + /* length cannot be zero */ + ioeventfd = (struct kvm_ioeventfd) { + .addr = MMIO_GPA, + .len = 0, + .fd = fd, + .flags = KVM_IOEVENTFD_FLAG_POST_WRITE, + .post_addr = (u64)&dummy, + }; + ret = __vm_ioctl(vm, KVM_IOEVENTFD, &ioeventfd); + TEST_ASSERT(ret && errno == EINVAL, + "len=0: expected EINVAL, got ret=%d errno=%d", ret, errno); + + /* post_addr cannot be NULL */ + ioeventfd.len = 4; + ioeventfd.post_addr = 0ULL; + ret = __vm_ioctl(vm, KVM_IOEVENTFD, &ioeventfd); + TEST_ASSERT(ret && errno == EINVAL, + "NULL post_addr: expected EINVAL, got ret=%d errno=%d", + ret, errno); + + /* bogus post_addr */ + ioeventfd.post_addr = (u64)0xdeaddeaddeaddeadULL; + ret = __vm_ioctl(vm, KVM_IOEVENTFD, &ioeventfd); + TEST_ASSERT(ret && errno == EINVAL, + "bad post_addr: expected EINVAL, got ret=%d errno=%d", + ret, errno); + + close(fd); + kvm_vm_free(vm); +} + +#define DEFINE_GUEST_WRITE_FN(suffix, type) \ +static void guest_code_w##suffix(void) { \ + *(volatile type *)MMIO_GPA = (type)TEST_VAL; \ + GUEST_DONE(); \ +} + +DEFINE_GUEST_WRITE_FN(1, uint8_t) +DEFINE_GUEST_WRITE_FN(2, uint16_t) +DEFINE_GUEST_WRITE_FN(4, uint32_t) +DEFINE_GUEST_WRITE_FN(8, uint64_t) + +/* + * Verify that ioeventfd_write copies exactly @width bytes to post_addr for + * each supported MMIO write width (1, 2, 4, 8). The guest writes the low + * @width bytes of TEST_VAL; the host checks that exactly those bytes land + * at post_addr and the eventfd is signaled. + */ +static void test_post_write_width(int width, void (*guest_fn)(void)) +{ + uint64_t actual, expected, count; + struct kvm_ioeventfd ioeventfd; + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + int fd, ret; + + /* need to initialize to 0 because the guest writes the low @width bytes */ + actual = 0; + expected = 0; + + vm = vm_create_with_one_vcpu(&vcpu, guest_fn); + virt_map(vm, MMIO_GPA, MMIO_GPA, 1); + + fd = kvm_new_eventfd(); + + ioeventfd = (struct kvm_ioeventfd) { + .addr = MMIO_GPA, + .len = width, + .fd = fd, + .flags = KVM_IOEVENTFD_FLAG_POST_WRITE, + .post_addr = (u64)&actual, + }; + + ret = __vm_ioctl(vm, KVM_IOEVENTFD, &ioeventfd); + TEST_ASSERT(!ret, "KVM_IOEVENTFD failed: %s", strerror(errno)); + + vcpu_run(vcpu); + assert_ucall(vcpu, UCALL_DONE, 0); + + ret = read(fd, &count, sizeof(count)); + TEST_ASSERT(ret == sizeof(count), + "eventfd read failed: ret=%d errno=%d", ret, errno); + + memcpy(&expected, &(uint64_t){TEST_VAL}, width); + TEST_ASSERT_EQ(actual, expected); + + close(fd); + kvm_vm_free(vm); +} + +static void guest_code_datamatch(void) +{ + *(volatile uint32_t *)MMIO_GPA = MATCH_VAL; + GUEST_SYNC(1); + *(volatile uint32_t *)MMIO_GPA = NOMATCH_VAL; + GUEST_SYNC(2); + GUEST_DONE(); +} + +/* + * Test the interaction between DATAMATCH and POST_WRITE. When both flags are + * set, ioeventfd_write should only fire (signal eventfd + copy value) when the + * written value matches datamatch. A non-matching write must leave the eventfd + * unsignaled and post_addr untouched, and fall through to KVM_EXIT_MMIO. + */ +static void test_post_write_datamatch(void) +{ + struct kvm_ioeventfd ioeventfd; + struct kvm_vcpu *vcpu; + struct kvm_run *run; + struct kvm_vm *vm; + struct pollfd pfd; + uint64_t count; + uint32_t actual; + int fd, ret; + + actual = POISON_VAL; + + vm = vm_create_with_one_vcpu(&vcpu, guest_code_datamatch); + virt_map(vm, MMIO_GPA, MMIO_GPA, 1); + run = vcpu->run; + + fd = kvm_new_eventfd(); + pfd = (struct pollfd){ .fd = fd, .events = POLLIN }; + + ioeventfd = (struct kvm_ioeventfd) { + .datamatch = MATCH_VAL, + .addr = MMIO_GPA, + .len = 4, + .fd = fd, + .flags = KVM_IOEVENTFD_FLAG_POST_WRITE | + KVM_IOEVENTFD_FLAG_DATAMATCH, + .post_addr = (u64)&actual, + }; + + ret = __vm_ioctl(vm, KVM_IOEVENTFD, &ioeventfd); + TEST_ASSERT(!ret, "KVM_IOEVENTFD failed: %s", strerror(errno)); + + /* + * Guest writes MATCH_VAL → ioeventfd fires (value copied, eventfd + * signaled), vCPU continues, then GUEST_SYNC(1). + */ + vcpu_run(vcpu); + assert_ucall(vcpu, UCALL_SYNC, 1); + TEST_ASSERT(read(fd, &count, sizeof(count)) == sizeof(count), + "eventfd read failed: errno=%d", errno); + TEST_ASSERT_EQ(actual, MATCH_VAL); + + actual = POISON_VAL; + + /* + * Guest writes NOMATCH_VAL → ioeventfd_in_range() returns false, bus + * returns -EOPNOTSUPP → KVM_EXIT_MMIO to userspace. + */ + vcpu_run(vcpu); + TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_MMIO); + TEST_ASSERT(run->mmio.is_write, "expected MMIO write"); + TEST_ASSERT(run->mmio.phys_addr == MMIO_GPA, + "expected MMIO at 0x%lx, got 0x%llx", + MMIO_GPA, run->mmio.phys_addr); + + /* Re-enter: KVM completes the MMIO, guest runs to GUEST_SYNC(2). */ + vcpu_run(vcpu); + assert_ucall(vcpu, UCALL_SYNC, 2); + + TEST_ASSERT(poll(&pfd, 1, 0) == 0, + "eventfd should not be signaled after non-matching write"); + TEST_ASSERT_EQ(actual, (uint32_t)POISON_VAL); + + vcpu_run(vcpu); + assert_ucall(vcpu, UCALL_DONE, 0); + + close(fd); + kvm_vm_free(vm); +} + +static void guest_code_multi(void) +{ + *(volatile uint32_t *)MMIO_GPA = 0x11111111; + GUEST_SYNC(1); + *(volatile uint32_t *)MMIO_GPA = 0x22222222; + GUEST_SYNC(2); + *(volatile uint32_t *)MMIO_GPA = 0x33333333; + GUEST_SYNC(3); + GUEST_DONE(); +} + +/* + * Verify that post_addr is updated on every MMIO write, not just the first. + * The guest writes three distinct values in sequence; the host checks after + * each one that post_addr holds the latest value and the eventfd is signaled + * each time. + */ +static void test_post_write_multi(void) +{ + static const uint32_t expected[] = { + 0x11111111, 0x22222222, 0x33333333, + }; + struct kvm_ioeventfd ioeventfd; + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + uint64_t count; + uint32_t actual; + int fd, ret, i; + + actual = POISON_VAL; + + vm = vm_create_with_one_vcpu(&vcpu, guest_code_multi); + virt_map(vm, MMIO_GPA, MMIO_GPA, 1); + + fd = kvm_new_eventfd(); + + ioeventfd = (struct kvm_ioeventfd) { + .addr = MMIO_GPA, + .len = 4, + .fd = fd, + .flags = KVM_IOEVENTFD_FLAG_POST_WRITE, + .post_addr = (u64)&actual, + }; + + ret = __vm_ioctl(vm, KVM_IOEVENTFD, &ioeventfd); + TEST_ASSERT(!ret, "KVM_IOEVENTFD failed: %s", strerror(errno)); + + for (i = 0; i < ARRAY_SIZE(expected); i++) { + vcpu_run(vcpu); + assert_ucall(vcpu, UCALL_SYNC, i + 1); + TEST_ASSERT(read(fd, &count, sizeof(count)) == sizeof(count), + "eventfd read failed: errno=%d", errno); + TEST_ASSERT_EQ(actual, expected[i]); + } + + vcpu_run(vcpu); + assert_ucall(vcpu, UCALL_DONE, 0); + + close(fd); + kvm_vm_free(vm); +} + +static void guest_code_multi_nosync(void) +{ + *(volatile uint32_t *)MMIO_GPA = 0x11111111; + *(volatile uint32_t *)MMIO_GPA = 0x22222222; + *(volatile uint32_t *)MMIO_GPA = 0x33333333; + GUEST_DONE(); +} + +/* + * Variant of the multi-write test where the guest performs three consecutive + * MMIO writes with no GUEST_SYNC in between. All three are handled in-kernel + * by ioeventfd before the vCPU exits at GUEST_DONE. Verify that: + * - post_addr reflects the last written value (0x33333333). + * - A single eventfd read() returns a counter of 3 (one signal per write). + */ +static void test_post_write_multi_nosync(void) +{ + struct kvm_ioeventfd ioeventfd; + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + uint64_t count; + uint32_t actual; + int fd, ret; + + actual = POISON_VAL; + + vm = vm_create_with_one_vcpu(&vcpu, guest_code_multi_nosync); + virt_map(vm, MMIO_GPA, MMIO_GPA, 1); + + fd = kvm_new_eventfd(); + + ioeventfd = (struct kvm_ioeventfd) { + .addr = MMIO_GPA, + .len = 4, + .fd = fd, + .flags = KVM_IOEVENTFD_FLAG_POST_WRITE, + .post_addr = (u64)&actual, + }; + + ret = __vm_ioctl(vm, KVM_IOEVENTFD, &ioeventfd); + TEST_ASSERT(!ret, "KVM_IOEVENTFD failed: %s", strerror(errno)); + + vcpu_run(vcpu); + assert_ucall(vcpu, UCALL_DONE, 0); + + ret = read(fd, &count, sizeof(count)); + TEST_ASSERT(ret == sizeof(count), + "eventfd read failed: ret=%d errno=%d", ret, errno); + TEST_ASSERT_EQ(count, (uint64_t)3); + TEST_ASSERT_EQ(actual, (uint32_t)0x33333333); + + close(fd); + kvm_vm_free(vm); +} + +static void guest_code_deassign(void) +{ + *(volatile uint32_t *)MMIO_GPA = MATCH_VAL; + GUEST_SYNC(1); + *(volatile uint32_t *)MMIO_GPA = MATCH_VAL; + GUEST_DONE(); +} + +/* + * Verify that deassigning an ioeventfd with POST_WRITE fully removes it from + * the I/O bus. + */ +static void test_post_write_deassign(void) +{ + struct kvm_ioeventfd ioeventfd; + struct kvm_vcpu *vcpu; + struct kvm_run *run; + struct kvm_vm *vm; + struct pollfd pfd; + uint64_t count; + uint32_t actual; + int fd, ret; + + actual = POISON_VAL; + + vm = vm_create_with_one_vcpu(&vcpu, guest_code_deassign); + virt_map(vm, MMIO_GPA, MMIO_GPA, 1); + run = vcpu->run; + + fd = kvm_new_eventfd(); + pfd = (struct pollfd){ .fd = fd, .events = POLLIN }; + + ioeventfd = (struct kvm_ioeventfd) { + .addr = MMIO_GPA, + .len = 4, + .fd = fd, + .flags = KVM_IOEVENTFD_FLAG_POST_WRITE, + .post_addr = (u64)&actual, + }; + + ret = __vm_ioctl(vm, KVM_IOEVENTFD, &ioeventfd); + TEST_ASSERT(!ret, "KVM_IOEVENTFD assign failed: %s", strerror(errno)); + + /* + * Guest writes MATCH_VAL → ioeventfd fires, then GUEST_SYNC(1). + */ + vcpu_run(vcpu); + assert_ucall(vcpu, UCALL_SYNC, 1); + TEST_ASSERT(read(fd, &count, sizeof(count)) == sizeof(count), + "eventfd read failed: errno=%d", errno); + TEST_ASSERT_EQ(actual, MATCH_VAL); + + /* Deassign the ioeventfd. */ + ioeventfd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN; + ret = __vm_ioctl(vm, KVM_IOEVENTFD, &ioeventfd); + TEST_ASSERT(!ret, "KVM_IOEVENTFD deassign failed: %s", strerror(errno)); + + actual = POISON_VAL; + + /* + * Guest writes MATCH_VAL again → no handler on the bus → + * KVM_EXIT_MMIO to userspace. + */ + vcpu_run(vcpu); + TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_MMIO); + TEST_ASSERT(run->mmio.is_write, "expected MMIO write"); + TEST_ASSERT(run->mmio.phys_addr == MMIO_GPA, + "expected MMIO at 0x%lx, got 0x%llx", + MMIO_GPA, run->mmio.phys_addr); + + /* Re-enter: KVM completes MMIO, guest runs to GUEST_DONE. */ + vcpu_run(vcpu); + assert_ucall(vcpu, UCALL_DONE, 0); + + TEST_ASSERT(poll(&pfd, 1, 0) == 0, + "eventfd should not be signaled after deassign"); + TEST_ASSERT_EQ(actual, (uint32_t)POISON_VAL); + + close(fd); + kvm_vm_free(vm); +} + +static void guest_code_pio(void) +{ + outl(PIO_PORT, (uint32_t)TEST_VAL); + GUEST_DONE(); +} + +/* + * Verify that POST_WRITE works on the PIO bus (KVM_PIO_BUS), not just MMIO. + * The guest does an outl to PIO_PORT; the host checks that the written value + * is copied to post_addr and the eventfd is signaled. + */ +static void test_post_write_pio(void) +{ + struct kvm_ioeventfd ioeventfd; + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + uint64_t count; + uint32_t actual; + int fd, ret; + + actual = POISON_VAL; + + vm = vm_create_with_one_vcpu(&vcpu, guest_code_pio); + + fd = kvm_new_eventfd(); + + ioeventfd = (struct kvm_ioeventfd) { + .addr = PIO_PORT, + .len = 4, + .fd = fd, + .flags = KVM_IOEVENTFD_FLAG_POST_WRITE | + KVM_IOEVENTFD_FLAG_PIO, + .post_addr = (u64)&actual, + }; + + ret = __vm_ioctl(vm, KVM_IOEVENTFD, &ioeventfd); + TEST_ASSERT(!ret, "KVM_IOEVENTFD failed: %s", strerror(errno)); + + vcpu_run(vcpu); + assert_ucall(vcpu, UCALL_DONE, 0); + + ret = read(fd, &count, sizeof(count)); + TEST_ASSERT(ret == sizeof(count), + "eventfd read failed: ret=%d errno=%d", ret, errno); + + TEST_ASSERT_EQ(actual, (uint32_t)TEST_VAL); + + close(fd); + kvm_vm_free(vm); +} + +static void guest_code_pio_datamatch(void) +{ + outl(PIO_PORT, MATCH_VAL); + GUEST_SYNC(1); + outl(PIO_PORT, NOMATCH_VAL); + GUEST_SYNC(2); + GUEST_DONE(); +} + +/* + * Test POST_WRITE + PIO + DATAMATCH together. When all three flags are set, + * the ioeventfd should only fire when the outl value matches datamatch. + * A non-matching outl must fall through to KVM_EXIT_IO (port I/O exit), + * leaving the eventfd unsignaled and post_addr untouched. + */ +static void test_post_write_pio_datamatch(void) +{ + struct kvm_ioeventfd ioeventfd; + struct kvm_vcpu *vcpu; + struct kvm_run *run; + struct kvm_vm *vm; + struct pollfd pfd; + uint64_t count; + uint32_t actual; + int fd, ret; + + actual = POISON_VAL; + + vm = vm_create_with_one_vcpu(&vcpu, guest_code_pio_datamatch); + run = vcpu->run; + + fd = kvm_new_eventfd(); + pfd = (struct pollfd){ .fd = fd, .events = POLLIN }; + + ioeventfd = (struct kvm_ioeventfd) { + .datamatch = MATCH_VAL, + .addr = PIO_PORT, + .len = 4, + .fd = fd, + .flags = KVM_IOEVENTFD_FLAG_POST_WRITE | + KVM_IOEVENTFD_FLAG_PIO | + KVM_IOEVENTFD_FLAG_DATAMATCH, + .post_addr = (u64)&actual, + }; + + ret = __vm_ioctl(vm, KVM_IOEVENTFD, &ioeventfd); + TEST_ASSERT(!ret, "KVM_IOEVENTFD failed: %s", strerror(errno)); + + /* + * Guest does outl MATCH_VAL → ioeventfd fires, then GUEST_SYNC(1). + */ + vcpu_run(vcpu); + assert_ucall(vcpu, UCALL_SYNC, 1); + TEST_ASSERT(read(fd, &count, sizeof(count)) == sizeof(count), + "eventfd read failed: errno=%d", errno); + TEST_ASSERT_EQ(actual, MATCH_VAL); + + actual = POISON_VAL; + + /* + * Guest does outl NOMATCH_VAL → no match → KVM_EXIT_IO. + */ + vcpu_run(vcpu); + TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO); + TEST_ASSERT(run->io.direction == KVM_EXIT_IO_OUT, + "expected PIO write"); + TEST_ASSERT(run->io.port == PIO_PORT, + "expected PIO at 0x%x, got 0x%x", + PIO_PORT, run->io.port); + + /* Re-enter: guest continues to GUEST_SYNC(2). */ + vcpu_run(vcpu); + assert_ucall(vcpu, UCALL_SYNC, 2); + + TEST_ASSERT(poll(&pfd, 1, 0) == 0, + "eventfd should not be signaled after non-matching PIO write"); + TEST_ASSERT_EQ(actual, (uint32_t)POISON_VAL); + + /* GUEST_DONE */ + vcpu_run(vcpu); + assert_ucall(vcpu, UCALL_DONE, 0); + + close(fd); + kvm_vm_free(vm); +} + +int main(void) +{ + TEST_REQUIRE(kvm_check_cap(KVM_CAP_IOEVENTFD_POST_WRITE)); + + test_post_write_negative(); + + test_post_write_width(1, guest_code_w1); + test_post_write_width(2, guest_code_w2); + test_post_write_width(4, guest_code_w4); + test_post_write_width(8, guest_code_w8); + + test_post_write_datamatch(); + + test_post_write_multi(); + + test_post_write_multi_nosync(); + + test_post_write_deassign(); + + test_post_write_pio(); + test_post_write_pio_datamatch(); + + return 0; +} diff --git a/virt/kvm/eventfd.c b/virt/kvm/eventfd.c index f43b8b25bc85..b0f2a6b43e35 100644 --- a/virt/kvm/eventfd.c +++ b/virt/kvm/eventfd.c @@ -743,6 +743,7 @@ struct _ioeventfd { struct kvm_io_device dev; u8 bus_idx; bool wildcard; + void __user *post_addr; }; static inline struct _ioeventfd * @@ -759,11 +760,32 @@ ioeventfd_release(struct _ioeventfd *p) kfree(p); } +#define ioeventfd_get_val(__ptr, __len, __unsupported_len) \ +({ \ + u64 __val; \ + \ + switch (__len) { \ + case 1: \ + __val = get_unaligned((u8 *)__ptr); \ + break; \ + case 2: \ + __val = get_unaligned((u16 *)__ptr); \ + break; \ + case 4: \ + __val = get_unaligned((u32 *)__ptr); \ + break; \ + case 8: \ + __val = get_unaligned((u64 *)__ptr); \ + break; \ + default: \ + goto __unsupported_len; \ + } \ + __val; \ +}) + static bool ioeventfd_in_range(struct _ioeventfd *p, gpa_t addr, int len, const void *val) { - u64 _val; - if (addr != p->addr) /* address must be precise for a hit */ return false; @@ -781,24 +803,10 @@ ioeventfd_in_range(struct _ioeventfd *p, gpa_t addr, int len, const void *val) return true; /* otherwise, we have to actually compare the data */ - switch (len) { - case 1: - _val = get_unaligned((u8 *)val); - break; - case 2: - _val = get_unaligned((u16 *)val); - break; - case 4: - _val = get_unaligned((u32 *)val); - break; - case 8: - _val = get_unaligned((u64 *)val); - break; - default: - return false; - } + return ioeventfd_get_val(val, len, unsupported_len) == p->datamatch; - return _val == p->datamatch; +unsupported_len: + return false; } /* MMIO/PIO writes trigger an event if the addr/val match */ @@ -811,8 +819,36 @@ ioeventfd_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this, gpa_t addr, if (!ioeventfd_in_range(p, addr, len, val)) return -EOPNOTSUPP; + if (p->post_addr) { + u64 _val = ioeventfd_get_val(val, len, unsupported_len); + int r; + + switch (len) { + case 1: + r = put_user(_val, (u8 __user *)p->post_addr); + break; + case 2: + r = put_user(_val, (u16 __user *)p->post_addr); + break; + case 4: + r = put_user(_val, (u32 __user *)p->post_addr); + break; + case 8: + r = put_user(_val, (u64 __user *)p->post_addr); + break; + default: + WARN_ON_ONCE(1); + r = 0; + break; + } + if (r) + return r; + } + eventfd_signal(p->eventfd); return 0; +unsupported_len: + return -EOPNOTSUPP; } /* @@ -872,6 +908,18 @@ static int kvm_assign_ioeventfd_idx(struct kvm *kvm, if (IS_ERR(eventfd)) return PTR_ERR(eventfd); + /* + * Disallow posting to virtual address 0 so that KVM can treat + * a NULL user pointer as "no posted write". + */ + if ((args->flags & KVM_IOEVENTFD_FLAG_POST_WRITE) && + (!args->len || !u64_to_user_ptr(args->post_addr) || + args->post_addr != untagged_addr(args->post_addr) || + !access_ok(u64_to_user_ptr(args->post_addr), args->len))) { + ret = -EINVAL; + goto fail; + } + p = kzalloc_obj(*p, GFP_KERNEL_ACCOUNT); if (!p) { ret = -ENOMEM; @@ -890,6 +938,9 @@ static int kvm_assign_ioeventfd_idx(struct kvm *kvm, else p->wildcard = true; + if (args->flags & KVM_IOEVENTFD_FLAG_POST_WRITE) + p->post_addr = u64_to_user_ptr(args->post_addr); + mutex_lock(&kvm->slots_lock); /* Verify that there isn't a match already */ @@ -941,6 +992,11 @@ kvm_deassign_ioeventfd_idx(struct kvm *kvm, enum kvm_bus bus_idx, mutex_lock(&kvm->slots_lock); list_for_each_entry(p, &kvm->ioeventfds, list) { + /* + * No need to match post_addr, ioeventfd_check_collision + * prevents duplicate registrations that only differ by + * post_addr. + */ if (p->bus_idx != bus_idx || p->eventfd != eventfd || p->addr != args->addr || diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index cf4cb586f140..7c8b013fad77 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -4982,6 +4982,7 @@ static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) case KVM_CAP_IRQFD: #endif case KVM_CAP_IOEVENTFD_ANY_LENGTH: + case KVM_CAP_IOEVENTFD_POST_WRITE: case KVM_CAP_CHECK_EXTENSION_VM: case KVM_CAP_ENABLE_CAP_VM: case KVM_CAP_HALT_POLL: -- 2.47.3