From: Mina Almasry Add vfio_pci_find_mem_bar() to libvfio to locate a BAR (or range within a BAR) of at least min_size bytes that behaves like ordinary memory and can be safely read and written without device-specific side effects. Detection is backed by a table of known devices, starting with QEMU's pci-testdev (1b36:0005) BAR 2 when configured with membar= and membar-backed=on. Suggested-by: David Matlack Cc: David Matlack Cc: Matt Evans Signed-off-by: Mina Almasry Reviewed-by: David Matlack Signed-off-by: David Matlack --- .../lib/include/libvfio/vfio_pci_device.h | 10 +++ .../selftests/vfio/lib/vfio_pci_device.c | 67 +++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h b/tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h index 007adc322c1e..19d458adba1a 100644 --- a/tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h +++ b/tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h @@ -15,6 +15,13 @@ struct vfio_pci_bar { void *vaddr; }; +struct vfio_pci_mem_bar { + struct vfio_pci_bar *bar; + void *vaddr; + u64 offset; + u64 size; +}; + struct vfio_pci_device { const char *bdf; int fd; @@ -162,6 +169,9 @@ static inline bool vfio_pci_device_match(struct vfio_pci_device *device, (device_id == vfio_pci_config_readw(device, PCI_DEVICE_ID)); } +bool vfio_pci_find_mem_bar(struct vfio_pci_device *device, u64 min_size, + struct vfio_pci_mem_bar *mem_bar); + const char *vfio_pci_get_cdev_path(const char *bdf); void vfio_pci_group_setup(struct vfio_pci_device *device, const char *bdf); diff --git a/tools/testing/selftests/vfio/lib/vfio_pci_device.c b/tools/testing/selftests/vfio/lib/vfio_pci_device.c index 4063a0e2b3df..d5308fe61490 100644 --- a/tools/testing/selftests/vfio/lib/vfio_pci_device.c +++ b/tools/testing/selftests/vfio/lib/vfio_pci_device.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -245,6 +246,72 @@ static void vfio_pci_bar_unmap_all(struct vfio_pci_device *device) } } +#define PCI_DEVICE_ID_REDHAT_TEST 0x0005 + +struct vfio_pci_mem_bar_desc { + u16 vendor_id; + u16 device_id; + u8 bar_index; + u64 offset; + u64 size; +}; + +static const struct vfio_pci_mem_bar_desc mem_bar_descs[] = { + /* QEMU pci-testdev with membar=,membar-backed=on */ + { + .vendor_id = PCI_VENDOR_ID_REDHAT, + .device_id = PCI_DEVICE_ID_REDHAT_TEST, + .bar_index = 2, + }, +}; + +bool vfio_pci_find_mem_bar(struct vfio_pci_device *device, u64 min_size, + struct vfio_pci_mem_bar *mem_bar) +{ + const u32 rw_mmap_flags = VFIO_REGION_INFO_FLAG_MMAP | + VFIO_REGION_INFO_FLAG_READ | + VFIO_REGION_INFO_FLAG_WRITE; + size_t i; + + for (i = 0; i < ARRAY_SIZE(mem_bar_descs); i++) { + const struct vfio_pci_mem_bar_desc *desc = &mem_bar_descs[i]; + struct vfio_pci_bar *bar; + u64 size; + + if (!vfio_pci_device_match(device, desc->vendor_id, + desc->device_id)) + continue; + + VFIO_ASSERT_LT(desc->bar_index, PCI_STD_NUM_BARS); + bar = &device->bars[desc->bar_index]; + + if ((bar->info.flags & rw_mmap_flags) != rw_mmap_flags) + continue; + + VFIO_ASSERT_NOT_NULL(bar->vaddr); + + if (bar->info.size <= desc->offset) + continue; + + size = bar->info.size - desc->offset; + if (desc->size) + size = min(size, desc->size); + + if (size < min_size) + continue; + + *mem_bar = (struct vfio_pci_mem_bar){ + .bar = bar, + .vaddr = bar->vaddr + desc->offset, + .offset = desc->offset, + .size = size, + }; + return true; + } + + return false; +} + void vfio_pci_config_access(struct vfio_pci_device *device, bool write, size_t config, size_t size, void *data) { -- 2.56.0.567.gdb80ae8cd5-goog Write and read back test patterns before returning a BAR from vfio_pci_find_mem_bar(), restoring the original contents and PCI_COMMAND_MEMORY state afterward. QEMU's pci-testdev exposes a readable, writable, mmap-capable BAR2 whenever membar= is passed, even if membar-backed=on was omitted and accesses are dropped as unimplemented MMIO. Probing the BAR upfront lets callers skip cleanly instead of failing later DMA or fault checks. Suggested-by: Alex Williamson Signed-off-by: David Matlack --- .../selftests/vfio/lib/vfio_pci_device.c | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tools/testing/selftests/vfio/lib/vfio_pci_device.c b/tools/testing/selftests/vfio/lib/vfio_pci_device.c index d5308fe61490..9840463594ab 100644 --- a/tools/testing/selftests/vfio/lib/vfio_pci_device.c +++ b/tools/testing/selftests/vfio/lib/vfio_pci_device.c @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -265,6 +266,33 @@ static const struct vfio_pci_mem_bar_desc mem_bar_descs[] = { }, }; +static bool vfio_pci_mem_bar_verify(struct vfio_pci_device *device, void *vaddr) +{ + const u64 pattern = 0x55aa55aa55aa55aaULL; + u16 orig_cmd; + u64 orig; + bool ok; + + orig_cmd = vfio_pci_config_readw(device, PCI_COMMAND); + if (!(orig_cmd & PCI_COMMAND_MEMORY)) + vfio_pci_config_writew(device, PCI_COMMAND, + orig_cmd | PCI_COMMAND_MEMORY); + + orig = readq(vaddr); + writeq(pattern, vaddr); + ok = (readq(vaddr) == pattern); + if (ok) { + writeq(~pattern, vaddr); + ok = (readq(vaddr) == ~pattern); + } + writeq(orig, vaddr); + + if (!(orig_cmd & PCI_COMMAND_MEMORY)) + vfio_pci_config_writew(device, PCI_COMMAND, orig_cmd); + + return ok; +} + bool vfio_pci_find_mem_bar(struct vfio_pci_device *device, u64 min_size, struct vfio_pci_mem_bar *mem_bar) { @@ -273,6 +301,8 @@ bool vfio_pci_find_mem_bar(struct vfio_pci_device *device, u64 min_size, VFIO_REGION_INFO_FLAG_WRITE; size_t i; + min_size = max_t(u64, min_size, sizeof(u64)); + for (i = 0; i < ARRAY_SIZE(mem_bar_descs); i++) { const struct vfio_pci_mem_bar_desc *desc = &mem_bar_descs[i]; struct vfio_pci_bar *bar; @@ -300,6 +330,9 @@ bool vfio_pci_find_mem_bar(struct vfio_pci_device *device, u64 min_size, if (size < min_size) continue; + if (!vfio_pci_mem_bar_verify(device, bar->vaddr + desc->offset)) + continue; + *mem_bar = (struct vfio_pci_mem_bar){ .bar = bar, .vaddr = bar->vaddr + desc->offset, -- 2.56.0.567.gdb80ae8cd5-goog Parse whitespace-separated PCI BDFs from VFIO_SELFTESTS_BDF in vfio_selftests_get_bdfs() instead of requiring a single BDF. Multi-device tests such as P2P DMA tests need two or more devices even when invoked through run_kselftest.sh without command-line arguments. Single-device callers using vfio_selftests_get_bdf() continue to receive the first BDF. Signed-off-by: David Matlack --- tools/testing/selftests/vfio/lib/libvfio.c | 53 ++++++++++++++++------ 1 file changed, 39 insertions(+), 14 deletions(-) diff --git a/tools/testing/selftests/vfio/lib/libvfio.c b/tools/testing/selftests/vfio/lib/libvfio.c index 3a3d1ed635c1..448d11442b53 100644 --- a/tools/testing/selftests/vfio/lib/libvfio.c +++ b/tools/testing/selftests/vfio/lib/libvfio.c @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -32,38 +33,62 @@ static char **get_bdfs_cmdline(int *argc, char *argv[], int *nr_bdfs) return *nr_bdfs ? &argv[i] : NULL; } -static char *get_bdf_env(void) +static char **get_bdfs_env(int *nr_bdfs) { - char *bdf; + static char **bdfs; + static int count; + char *env, *tok, *saveptr; + int cap = 0; - bdf = getenv("VFIO_SELFTESTS_BDF"); - if (!bdf) + if (bdfs) { + *nr_bdfs = count; + return bdfs; + } + + env = getenv("VFIO_SELFTESTS_BDF"); + if (!env) return NULL; - VFIO_ASSERT_TRUE(is_bdf(bdf), "Invalid BDF: %s\n", bdf); - return bdf; + env = strdup(env); + VFIO_ASSERT_NOT_NULL(env); + + for (tok = strtok_r(env, " \t\n", &saveptr); tok; + tok = strtok_r(NULL, " \t\n", &saveptr)) { + VFIO_ASSERT_TRUE(is_bdf(tok), "Invalid BDF: %s\n", tok); + if (count == cap) { + cap = cap ? cap * 2 : 4; + bdfs = realloc(bdfs, cap * sizeof(*bdfs)); + VFIO_ASSERT_NOT_NULL(bdfs); + } + bdfs[count++] = tok; + } + + if (!count) { + free(env); + return NULL; + } + + *nr_bdfs = count; + return bdfs; } char **vfio_selftests_get_bdfs(int *argc, char *argv[], int *nr_bdfs) { - static char *env_bdf; char **bdfs; bdfs = get_bdfs_cmdline(argc, argv, nr_bdfs); if (bdfs) return bdfs; - env_bdf = get_bdf_env(); - if (env_bdf) { - *nr_bdfs = 1; - return &env_bdf; - } + bdfs = get_bdfs_env(nr_bdfs); + if (bdfs) + return bdfs; fprintf(stderr, "Unable to determine which device(s) to use, skipping test.\n"); fprintf(stderr, "\n"); - fprintf(stderr, "To pass the device address via environment variable:\n"); + fprintf(stderr, "To pass the device address(es) via environment variable:\n"); fprintf(stderr, "\n"); - fprintf(stderr, " export VFIO_SELFTESTS_BDF=\"segment:bus:device.function\"\n"); + fprintf(stderr, " export VFIO_SELFTESTS_BDF=\"segment:bus:device.function ...\"\n"); fprintf(stderr, " %s [options]\n", argv[0]); fprintf(stderr, "\n"); fprintf(stderr, "To pass the device address(es) via argv:\n"); -- 2.56.0.567.gdb80ae8cd5-goog Add vfio_pci_dmabuf_create() and __vfio_pci_dmabuf_create() to export a VFIO PCI region range as a dma-buf fd via VFIO_DEVICE_FEATURE_DMA_BUF, and preserve -errno on failure in vfio_device_feature_ioctl(). Native iommufd maps device MMIO BARs for peer-to-peer DMA via dma-buf file descriptors rather than VM_PFNMAP virtual addresses. Signed-off-by: David Matlack --- .../lib/include/libvfio/vfio_pci_device.h | 5 +++ .../selftests/vfio/lib/vfio_pci_device.c | 33 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h b/tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h index 19d458adba1a..fd130f680a2d 100644 --- a/tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h +++ b/tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h @@ -172,6 +172,11 @@ static inline bool vfio_pci_device_match(struct vfio_pci_device *device, bool vfio_pci_find_mem_bar(struct vfio_pci_device *device, u64 min_size, struct vfio_pci_mem_bar *mem_bar); +int __vfio_pci_dmabuf_create(struct vfio_pci_device *device, u32 region_index, + u64 offset, u64 length, u32 open_flags); +int vfio_pci_dmabuf_create(struct vfio_pci_device *device, u32 region_index, + u64 offset, u64 length, u32 open_flags); + const char *vfio_pci_get_cdev_path(const char *bdf); void vfio_pci_group_setup(struct vfio_pci_device *device, const char *bdf); diff --git a/tools/testing/selftests/vfio/lib/vfio_pci_device.c b/tools/testing/selftests/vfio/lib/vfio_pci_device.c index 9840463594ab..eca6eb13d7f0 100644 --- a/tools/testing/selftests/vfio/lib/vfio_pci_device.c +++ b/tools/testing/selftests/vfio/lib/vfio_pci_device.c @@ -153,11 +153,44 @@ static int vfio_device_feature_ioctl(int fd, u32 flags, void *data, feature->flags = flags; ret = ioctl(fd, VFIO_DEVICE_FEATURE, feature); + if (ret < 0) + ret = -errno; free(feature); return ret; } +int __vfio_pci_dmabuf_create(struct vfio_pci_device *device, u32 region_index, + u64 offset, u64 length, u32 open_flags) +{ + u8 buf[sizeof(struct vfio_device_feature_dma_buf) + + sizeof(struct vfio_region_dma_range)] = {}; + struct vfio_device_feature_dma_buf *dma_buf = (void *)buf; + + dma_buf->region_index = region_index; + dma_buf->open_flags = open_flags; + dma_buf->nr_ranges = 1; + dma_buf->dma_ranges[0].offset = offset; + dma_buf->dma_ranges[0].length = length; + + return vfio_device_feature_ioctl(device->fd, + VFIO_DEVICE_FEATURE_GET | + VFIO_DEVICE_FEATURE_DMA_BUF, + buf, sizeof(buf)); +} + +int vfio_pci_dmabuf_create(struct vfio_pci_device *device, u32 region_index, + u64 offset, u64 length, u32 open_flags) +{ + int fd; + + fd = __vfio_pci_dmabuf_create(device, region_index, offset, length, + open_flags); + VFIO_ASSERT_GE(fd, 0, "VFIO_DEVICE_FEATURE_DMA_BUF failed: %d\n", fd); + + return fd; +} + static void vfio_device_feature_set(int fd, u16 feature, void *data, size_t data_size) { u32 flags = VFIO_DEVICE_FEATURE_SET | feature; -- 2.56.0.567.gdb80ae8cd5-goog Add iommu_map_file() and __iommu_map_file() to map a file descriptor (such as a memfd or VFIO PCI dma-buf) into an iommufd IOAS via IOMMU_IOAS_MAP_FILE. Also skip regions without a host virtual address in __iommu_hva2iova(), since file-backed DMA mappings do not require one. Signed-off-by: David Matlack --- .../vfio/lib/include/libvfio/iommu.h | 9 ++++++ tools/testing/selftests/vfio/lib/iommu.c | 29 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/tools/testing/selftests/vfio/lib/include/libvfio/iommu.h b/tools/testing/selftests/vfio/lib/include/libvfio/iommu.h index e9a3386a4719..2b39c49e66da 100644 --- a/tools/testing/selftests/vfio/lib/include/libvfio/iommu.h +++ b/tools/testing/selftests/vfio/lib/include/libvfio/iommu.h @@ -42,6 +42,15 @@ static inline void iommu_map(struct iommu *iommu, struct dma_region *region) VFIO_ASSERT_EQ(__iommu_map(iommu, region), 0); } +int __iommu_map_file(struct iommu *iommu, int fd, u64 file_offset, + struct dma_region *region); + +static inline void iommu_map_file(struct iommu *iommu, int fd, u64 file_offset, + struct dma_region *region) +{ + VFIO_ASSERT_EQ(__iommu_map_file(iommu, fd, file_offset, region), 0); +} + int __iommu_unmap(struct iommu *iommu, struct dma_region *region, u64 *unmapped); static inline void iommu_unmap(struct iommu *iommu, struct dma_region *region) diff --git a/tools/testing/selftests/vfio/lib/iommu.c b/tools/testing/selftests/vfio/lib/iommu.c index b6f3c5c84e01..ba602685f892 100644 --- a/tools/testing/selftests/vfio/lib/iommu.c +++ b/tools/testing/selftests/vfio/lib/iommu.c @@ -71,6 +71,9 @@ int __iommu_hva2iova(struct iommu *iommu, void *vaddr, iova_t *iova) struct dma_region *region; list_for_each_entry(region, &iommu->dma_regions, link) { + if (!region->vaddr) + continue; + if (vaddr < region->vaddr) continue; @@ -149,6 +152,32 @@ int __iommu_map(struct iommu *iommu, struct dma_region *region) return 0; } +int __iommu_map_file(struct iommu *iommu, int fd, u64 file_offset, + struct dma_region *region) +{ + struct iommu_ioas_map_file args = { + .size = sizeof(args), + .flags = IOMMU_IOAS_MAP_READABLE | + IOMMU_IOAS_MAP_WRITEABLE | + IOMMU_IOAS_MAP_FIXED_IOVA, + .ioas_id = iommu->ioas_id, + .fd = fd, + .start = file_offset, + .length = region->size, + .iova = region->iova, + }; + + if (!iommu->iommufd) + return -EOPNOTSUPP; + + if (ioctl(iommu->iommufd, IOMMU_IOAS_MAP_FILE, &args)) + return -errno; + + list_add(®ion->link, &iommu->dma_regions); + + return 0; +} + static int __vfio_iommu_unmap(int fd, u64 iova, u64 size, u32 flags, u64 *unmapped) { struct vfio_iommu_type1_dma_unmap args = { -- 2.56.0.567.gdb80ae8cd5-goog Add vfio_pci_find_p2p_pair() to locate an initiator device with memcpy driver support and a distinct target device with a memory BAR of at least min_bar_size bytes from an array of VFIO PCI devices. Factor out vfio_pci_has_p2p_path() as the central place to validate PCI topology and P2P DMA reachability between the two devices in the future. Signed-off-by: David Matlack --- .../lib/include/libvfio/vfio_pci_device.h | 8 ++++ .../selftests/vfio/lib/vfio_pci_device.c | 48 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h b/tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h index fd130f680a2d..9bda93712859 100644 --- a/tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h +++ b/tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h @@ -42,6 +42,12 @@ struct vfio_pci_device { struct vfio_pci_driver driver; }; +struct vfio_pci_p2p_pair { + struct vfio_pci_device *initiator; + struct vfio_pci_device *target; + struct vfio_pci_mem_bar mem_bar; +}; + #define dev_info(_dev, _fmt, ...) printf("%s: " _fmt, (_dev)->bdf, ##__VA_ARGS__) #define dev_err(_dev, _fmt, ...) fprintf(stderr, "%s: " _fmt, (_dev)->bdf, ##__VA_ARGS__) @@ -171,6 +177,8 @@ static inline bool vfio_pci_device_match(struct vfio_pci_device *device, bool vfio_pci_find_mem_bar(struct vfio_pci_device *device, u64 min_size, struct vfio_pci_mem_bar *mem_bar); +bool vfio_pci_find_p2p_pair(struct vfio_pci_device **devices, int nr_devices, + u64 min_bar_size, struct vfio_pci_p2p_pair *pair); int __vfio_pci_dmabuf_create(struct vfio_pci_device *device, u32 region_index, u64 offset, u64 length, u32 open_flags); diff --git a/tools/testing/selftests/vfio/lib/vfio_pci_device.c b/tools/testing/selftests/vfio/lib/vfio_pci_device.c index eca6eb13d7f0..16fc98ed170b 100644 --- a/tools/testing/selftests/vfio/lib/vfio_pci_device.c +++ b/tools/testing/selftests/vfio/lib/vfio_pci_device.c @@ -378,6 +378,54 @@ bool vfio_pci_find_mem_bar(struct vfio_pci_device *device, u64 min_size, return false; } +static bool vfio_pci_has_p2p_path(struct vfio_pci_device *initiator, + struct vfio_pci_device *target) +{ + /* + * TODO: Check PCI topology / ACS / host bridge support to verify that + * P2P DMA transactions from @initiator can reach @target. + */ + return initiator != target; +} + +bool vfio_pci_find_p2p_pair(struct vfio_pci_device **devices, int nr_devices, + u64 min_bar_size, struct vfio_pci_p2p_pair *pair) +{ + struct vfio_pci_mem_bar mem_bar; + int i, j; + + for (i = 0; i < nr_devices; i++) { + struct vfio_pci_device *initiator = devices[i]; + const struct vfio_pci_driver_ops *ops = initiator->driver.ops; + + if (!ops || !ops->memcpy_start || !ops->memcpy_wait) + continue; + + for (j = 0; j < nr_devices; j++) { + struct vfio_pci_device *target = devices[j]; + + if (i == j) + continue; + + if (!vfio_pci_has_p2p_path(initiator, target)) + continue; + + if (!vfio_pci_find_mem_bar(target, min_bar_size, + &mem_bar)) + continue; + + *pair = (struct vfio_pci_p2p_pair){ + .initiator = initiator, + .target = target, + .mem_bar = mem_bar, + }; + return true; + } + } + + return false; +} + void vfio_pci_config_access(struct vfio_pci_device *device, bool write, size_t config, size_t size, void *data) { -- 2.56.0.567.gdb80ae8cd5-goog Add vfio_pci_p2pdma_test to validate peer-to-peer DMA between an initiator device driven by the VFIO selftests driver framework and a target device exposing a memory BAR located via vfio_pci_find_p2p_pair(). Map the target BAR into the IOMMU across all supported IOMMU modes (using VFIO_IOMMU_MAP_DMA on the BAR mmap for type1/type1v2 and VFIO_DEVICE_FEATURE_DMA_BUF + IOMMU_IOAS_MAP_FILE for iommufd) and test RAM-to-BAR writes, BAR-to-RAM reads, BAR-to-BAR copies, RAM-BAR-RAM roundtrips, and iommufd dmabuf revocation when PCI_COMMAND_MEMORY is cleared on the target. Signed-off-by: David Matlack --- tools/testing/selftests/vfio/Makefile | 1 + .../selftests/vfio/vfio_pci_p2pdma_test.c | 435 ++++++++++++++++++ 2 files changed, 436 insertions(+) create mode 100644 tools/testing/selftests/vfio/vfio_pci_p2pdma_test.c diff --git a/tools/testing/selftests/vfio/Makefile b/tools/testing/selftests/vfio/Makefile index 2c32c48db509..b391d22b52ca 100644 --- a/tools/testing/selftests/vfio/Makefile +++ b/tools/testing/selftests/vfio/Makefile @@ -12,6 +12,7 @@ TEST_GEN_PROGS += vfio_iommufd_setup_test TEST_GEN_PROGS += vfio_pci_device_test TEST_GEN_PROGS += vfio_pci_device_init_perf_test TEST_GEN_PROGS += vfio_pci_driver_test +TEST_GEN_PROGS += vfio_pci_p2pdma_test TEST_GEN_PROGS += vfio_pci_sriov_uapi_test TEST_FILES += scripts/cleanup.sh diff --git a/tools/testing/selftests/vfio/vfio_pci_p2pdma_test.c b/tools/testing/selftests/vfio/vfio_pci_p2pdma_test.c new file mode 100644 index 000000000000..43e1537076ad --- /dev/null +++ b/tools/testing/selftests/vfio/vfio_pci_p2pdma_test.c @@ -0,0 +1,435 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +#include "../kselftest_harness.h" + +static char **device_bdfs; +static int nr_bdfs; + +static void set_msi_nonblock(struct vfio_pci_device *dev) +{ + if (dev->driver.ops->send_msi) + fcntl_set_nonblock(dev->msi_eventfds[dev->driver.msi]); +} + +static void assert_no_msi(struct vfio_pci_device *dev) +{ + u64 value; + int fd; + + if (!dev->driver.ops->send_msi) + return; + + fd = dev->msi_eventfds[dev->driver.msi]; + VFIO_ASSERT_EQ(read(fd, &value, sizeof(value)), -1); + VFIO_ASSERT_EQ(errno, EAGAIN); +} + +static void ram_region_setup(struct iommu *iommu, + struct iova_allocator *iova_allocator, + struct dma_region *region, u64 size) +{ + void *vaddr; + + vaddr = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1, 0); + VFIO_ASSERT_NE(vaddr, MAP_FAILED); + + region->vaddr = vaddr; + region->iova = iova_allocator_alloc(iova_allocator, size); + region->size = size; + + iommu_map(iommu, region); +} + +static void ram_region_teardown(struct iommu *iommu, struct dma_region *region) +{ + if (!region->vaddr) + return; + + iommu_unmap(iommu, region); + VFIO_ASSERT_EQ(munmap(region->vaddr, region->size), 0); + region->vaddr = NULL; +} + +static u64 pattern_word(size_t index, u8 seed) +{ + return (u64)seed * 0x0101010101010101ULL ^ + ((u64)index * 0x9e3779b97f4a7c15ULL); +} + +static void fill_ram(void *buf, size_t size, u8 seed) +{ + __le64 *p = buf; + size_t i; + + VFIO_ASSERT_TRUE(IS_ALIGNED(size, sizeof(u64))); + for (i = 0; i < size / sizeof(u64); i++) + p[i] = cpu_to_le64(pattern_word(i, seed)); +} + +static void fill_bar(void *bar, size_t size, u8 seed) +{ + size_t i; + + VFIO_ASSERT_TRUE(IS_ALIGNED(size, sizeof(u64))); + for (i = 0; i < size / sizeof(u64); i++) + writeq(pattern_word(i, seed), bar + i * sizeof(u64)); +} + +static void assert_ram_eq_bar(const void *ram, const void *bar, size_t size) +{ + const __le64 *pr = ram; + size_t i; + + VFIO_ASSERT_TRUE(IS_ALIGNED(size, sizeof(u64))); + for (i = 0; i < size / sizeof(u64); i++) + VFIO_ASSERT_EQ(le64_to_cpu(pr[i]), + readq(bar + i * sizeof(u64)), + "Mismatch at byte offset %zu\n", + i * sizeof(u64)); +} + +static void assert_bar_eq_bar(const void *bar_a, const void *bar_b, + size_t size) +{ + size_t i; + + VFIO_ASSERT_TRUE(IS_ALIGNED(size, sizeof(u64))); + for (i = 0; i < size / sizeof(u64); i++) + VFIO_ASSERT_EQ(readq(bar_a + i * sizeof(u64)), + readq(bar_b + i * sizeof(u64)), + "Mismatch at byte offset %zu\n", + i * sizeof(u64)); +} + +static void assert_no_pattern_leak(const void *bar_pattern, const void *dst, + size_t size) +{ + const u8 *pb = bar_pattern; + const u8 *pd = dst; + size_t offset, chunk; + + for (offset = 0; offset < size; offset += chunk) { + chunk = min_t(size_t, 64, size - offset); + VFIO_ASSERT_NE(memcmp(pb + offset, pd + offset, chunk), 0, + "Leaked BAR data at byte offset %zu\n", offset); + } +} + +FIXTURE(vfio_pci_p2pdma_test) { + struct iommu *iommu; + struct iova_allocator *iova_allocator; + struct vfio_pci_device **devices; + int nr_open; + + struct vfio_pci_device *initiator; + struct vfio_pci_device *target; + struct vfio_pci_mem_bar mem_bar; + + struct dma_region ram_region; + struct dma_region bar_region; + bool bar_mapped; + int dmabuf_fd; + + u64 size; + void *ram_src; + void *ram_dst; + iova_t ram_src_iova; + iova_t ram_dst_iova; + iova_t bar_iova; +}; + +FIXTURE_VARIANT(vfio_pci_p2pdma_test) { + const char *iommu_mode; +}; + +#define FIXTURE_VARIANT_ADD_IOMMU_MODE(_iommu_mode) \ +FIXTURE_VARIANT_ADD(vfio_pci_p2pdma_test, _iommu_mode) { \ + .iommu_mode = #_iommu_mode, \ +} + +FIXTURE_VARIANT_ADD_ALL_IOMMU_MODES(); + +#undef FIXTURE_VARIANT_ADD_IOMMU_MODE + +FIXTURE_SETUP(vfio_pci_p2pdma_test) +{ + const u64 page_size = getpagesize(); + const u64 min_bar_size = max_t(u64, 2 * page_size, SZ_128K); + struct vfio_pci_p2p_pair pair; + struct vfio_pci_driver *driver; + u64 bar_map_size; + int i, rc; + + self->dmabuf_fd = -1; + + self->iommu = iommu_init(variant->iommu_mode); + self->devices = calloc_assert(nr_bdfs, sizeof(*self->devices)); + + for (i = 0; i < nr_bdfs; i++) { + self->devices[i] = vfio_pci_device_init(device_bdfs[i], + self->iommu); + self->nr_open++; + } + + if (!vfio_pci_find_p2p_pair(self->devices, self->nr_open, min_bar_size, + &pair)) + SKIP(return, + "No (initiator with driver, target with >= %lluB memory BAR) pair found", + (unsigned long long)min_bar_size); + + self->initiator = pair.initiator; + self->target = pair.target; + self->mem_bar = pair.mem_bar; + + printf("P2P pair: initiator %s (%s) -> target %s BAR%u (size 0x%lx)\n", + self->initiator->bdf, self->initiator->driver.ops->name, + self->target->bdf, self->mem_bar.bar->info.index, + (unsigned long)self->mem_bar.size); + + vfio_pci_cmd_set(self->target, PCI_COMMAND_MEMORY); + + self->iova_allocator = iova_allocator_init(self->iommu); + + driver = &self->initiator->driver; + ram_region_setup(self->iommu, self->iova_allocator, &driver->region, + driver->region.size); + ram_region_setup(self->iommu, self->iova_allocator, &self->ram_region, + SZ_16M); + + bar_map_size = min_t(u64, round_down(self->mem_bar.size, page_size), + SZ_16M); + self->bar_region.vaddr = self->mem_bar.vaddr; + self->bar_region.size = bar_map_size; + self->bar_region.iova = iova_allocator_alloc(self->iova_allocator, + bar_map_size); + + if (self->iommu->iommufd) { + u32 bar_idx = self->mem_bar.bar->info.index; + + self->dmabuf_fd = __vfio_pci_dmabuf_create(self->target, + bar_idx, + self->mem_bar.offset, + bar_map_size, + O_RDWR | O_CLOEXEC); + if (self->dmabuf_fd == -EOPNOTSUPP) + SKIP(return, "Kernel does not support VFIO_DEVICE_FEATURE_DMA_BUF"); + ASSERT_GE(self->dmabuf_fd, 0); + + iommu_map_file(self->iommu, self->dmabuf_fd, 0, + &self->bar_region); + } else { + rc = __iommu_map(self->iommu, &self->bar_region); + if (rc && !strncmp(variant->iommu_mode, "iommufd_compat_", 15)) + SKIP(return, + "IOMMU mode %s does not support mapping MMIO BARs (rc=%d)", + variant->iommu_mode, rc); + ASSERT_EQ(0, rc); + } + self->bar_mapped = true; + + vfio_pci_driver_init(self->initiator); + + self->size = min_t(u64, driver->max_memcpy_size * 4, + min_t(u64, self->ram_region.size / 2, + bar_map_size / 2)); + + self->ram_src = self->ram_region.vaddr; + self->ram_dst = self->ram_src + self->size; + + self->ram_src_iova = to_iova(self->initiator, self->ram_src); + self->ram_dst_iova = to_iova(self->initiator, self->ram_dst); + self->bar_iova = to_iova(self->initiator, self->mem_bar.vaddr); +} + +FIXTURE_TEARDOWN(vfio_pci_p2pdma_test) +{ + int i; + + if (self->initiator && self->initiator->driver.initialized) + vfio_pci_driver_remove(self->initiator); + + if (self->bar_mapped) + iommu_unmap(self->iommu, &self->bar_region); + + if (self->dmabuf_fd >= 0) + VFIO_ASSERT_EQ(close(self->dmabuf_fd), 0); + + if (self->initiator) + ram_region_teardown(self->iommu, + &self->initiator->driver.region); + ram_region_teardown(self->iommu, &self->ram_region); + + if (self->iova_allocator) + iova_allocator_cleanup(self->iova_allocator); + + for (i = 0; i < self->nr_open; i++) + vfio_pci_device_cleanup(self->devices[i]); + free(self->devices); + + iommu_cleanup(self->iommu); +} + +TEST_F(vfio_pci_p2pdma_test, ram_to_bar) +{ + set_msi_nonblock(self->initiator); + + fill_ram(self->ram_src, self->size, 0x11); + fill_bar(self->mem_bar.vaddr, self->size, 0xee); + + ASSERT_EQ(0, vfio_pci_driver_memcpy(self->initiator, + self->ram_src_iova, + self->bar_iova, + self->size)); + + assert_ram_eq_bar(self->ram_src, self->mem_bar.vaddr, self->size); + assert_no_msi(self->initiator); +} + +TEST_F(vfio_pci_p2pdma_test, bar_to_ram) +{ + set_msi_nonblock(self->initiator); + + fill_bar(self->mem_bar.vaddr, self->size, 0x22); + memset(self->ram_dst, 0, self->size); + + ASSERT_EQ(0, vfio_pci_driver_memcpy(self->initiator, + self->bar_iova, + self->ram_dst_iova, + self->size)); + + assert_ram_eq_bar(self->ram_dst, self->mem_bar.vaddr, self->size); + assert_no_msi(self->initiator); +} + +TEST_F(vfio_pci_p2pdma_test, bar_to_bar) +{ + void *bar_src = self->mem_bar.vaddr; + void *bar_dst = self->mem_bar.vaddr + self->size; + + set_msi_nonblock(self->initiator); + + fill_bar(bar_src, self->size, 0x33); + fill_bar(bar_dst, self->size, 0xcc); + + ASSERT_EQ(0, vfio_pci_driver_memcpy(self->initiator, + self->bar_iova, + self->bar_iova + self->size, + self->size)); + + assert_bar_eq_bar(bar_src, bar_dst, self->size); + assert_no_msi(self->initiator); +} + +TEST_F(vfio_pci_p2pdma_test, roundtrip) +{ + set_msi_nonblock(self->initiator); + + fill_ram(self->ram_src, self->size, 0x44); + fill_bar(self->mem_bar.vaddr, self->size, 0x00); + memset(self->ram_dst, 0xff, self->size); + + ASSERT_EQ(0, vfio_pci_driver_memcpy(self->initiator, + self->ram_src_iova, + self->bar_iova, + self->size)); + ASSERT_EQ(0, vfio_pci_driver_memcpy(self->initiator, + self->bar_iova, + self->ram_dst_iova, + self->size)); + + ASSERT_EQ(0, memcmp(self->ram_src, self->ram_dst, self->size)); + assert_no_msi(self->initiator); +} + +TEST_F(vfio_pci_p2pdma_test, dmabuf_revoke) +{ + if (self->dmabuf_fd < 0) + SKIP(return, "Test requires dmabuf-backed IOMMU mapping"); + + set_msi_nonblock(self->initiator); + + fill_bar(self->mem_bar.vaddr, self->size, 0x55); + fill_ram(self->ram_src, self->size, 0xaa); + + /* + * Clearing PCI_COMMAND_MEMORY on the target revokes the dmabuf and + * tears down its IOMMU mapping in place. P2P DMA must not reach the + * BAR, and remapping the revoked dmabuf must fail with -ENODEV until + * PCI_COMMAND_MEMORY is re-enabled. + */ + vfio_pci_cmd_clear(self->target, PCI_COMMAND_MEMORY); + + /* + * Ignore the return values: DMA writes are posted, and not all + * devices report accesses to unmapped IOVAs as errors on DMA reads + * (e.g. QEMU's emulated igb completes reads from unmapped IOVAs with + * zeros). Verify isolation via the destination buffers instead. + */ + fill_ram(self->ram_dst, self->size, 0x33); + vfio_pci_driver_memcpy(self->initiator, self->ram_src_iova, + self->bar_iova, self->size); + vfio_pci_driver_memcpy(self->initiator, self->bar_iova, + self->ram_dst_iova, self->size); + assert_no_msi(self->initiator); + + iommu_unmap(self->iommu, &self->bar_region); + self->bar_mapped = false; + + ASSERT_EQ(-ENODEV, __iommu_map_file(self->iommu, self->dmabuf_fd, 0, + &self->bar_region)); + + vfio_pci_cmd_set(self->target, PCI_COMMAND_MEMORY); + + /* + * Revoked P2P writes must not have modified the BAR, and revoked P2P + * reads must not have leaked BAR contents into RAM. + */ + fill_ram(self->ram_src, self->size, 0x55); + assert_ram_eq_bar(self->ram_src, self->mem_bar.vaddr, self->size); + assert_no_pattern_leak(self->ram_src, self->ram_dst, self->size); + + /* Once memory decode is re-enabled, the dmabuf can be mapped again. */ + iommu_map_file(self->iommu, self->dmabuf_fd, 0, &self->bar_region); + self->bar_mapped = true; + + memset(self->ram_dst, 0, self->size); + ASSERT_EQ(0, vfio_pci_driver_memcpy(self->initiator, + self->bar_iova, + self->ram_dst_iova, + self->size)); + ASSERT_EQ(0, memcmp(self->ram_src, self->ram_dst, self->size)); + + fill_ram(self->ram_src, self->size, 0x77); + ASSERT_EQ(0, vfio_pci_driver_memcpy(self->initiator, + self->ram_src_iova, + self->bar_iova, + self->size)); + assert_ram_eq_bar(self->ram_src, self->mem_bar.vaddr, self->size); + assert_no_msi(self->initiator); +} + +int main(int argc, char *argv[]) +{ + device_bdfs = vfio_selftests_get_bdfs(&argc, argv, &nr_bdfs); + if (nr_bdfs < 2) { + fprintf(stderr, + "P2P DMA test requires at least 2 devices (got %d), skipping.\n", + nr_bdfs); + return KSFT_SKIP; + } + + return test_harness_run(argc, argv); +} -- 2.56.0.567.gdb80ae8cd5-goog