Support tests that want to add multiple devices to the same container/iommufd by decoupling struct vfio_pci_device from struct iommu. For backwards compatibility with existing tests, and to keep single-device tests simple, vfio_pci_device_init() and vfio_pci_device_cleanup() remain unchanged. Multi-devices tests can now put multiple devices in the same container/iommufd like so: iommu = iommu_init(iommu_mode); device1 = __vfio_pci_device_init(bdf1, iommu); device2 = __vfio_pci_device_init(bdf2, iommu); device3 = __vfio_pci_device_init(bdf3, iommu); ... __vfio_pci_device_cleanup(device3); __vfio_pci_device_cleanup(device2); __vfio_pci_device_cleanup(device1); iommu_cleanup(iommu); Signed-off-by: David Matlack --- .../selftests/vfio/lib/include/vfio_util.h | 7 ++ .../selftests/vfio/lib/vfio_pci_device.c | 107 ++++++++++++------ 2 files changed, 80 insertions(+), 34 deletions(-) diff --git a/tools/testing/selftests/vfio/lib/include/vfio_util.h b/tools/testing/selftests/vfio/lib/include/vfio_util.h index 14cd0bec45c0..8a01bcaa3ee8 100644 --- a/tools/testing/selftests/vfio/lib/include/vfio_util.h +++ b/tools/testing/selftests/vfio/lib/include/vfio_util.h @@ -206,8 +206,15 @@ const char *vfio_pci_get_cdev_path(const char *bdf); extern const char *default_iommu_mode; +struct iommu *iommu_init(const char *iommu_mode); +void iommu_cleanup(struct iommu *iommu); + +struct vfio_pci_device *__vfio_pci_device_init(const char *bdf, struct iommu *iommu); struct vfio_pci_device *vfio_pci_device_init(const char *bdf, const char *iommu_mode); + +void __vfio_pci_device_cleanup(struct vfio_pci_device *device); void vfio_pci_device_cleanup(struct vfio_pci_device *device); + void vfio_pci_device_reset(struct vfio_pci_device *device); void vfio_pci_dma_map(struct vfio_pci_device *device, diff --git a/tools/testing/selftests/vfio/lib/vfio_pci_device.c b/tools/testing/selftests/vfio/lib/vfio_pci_device.c index e4596a570422..de3a8d4d74f0 100644 --- a/tools/testing/selftests/vfio/lib/vfio_pci_device.c +++ b/tools/testing/selftests/vfio/lib/vfio_pci_device.c @@ -330,23 +330,21 @@ static void vfio_pci_group_setup(struct vfio_pci_device *device, const char *bdf static void vfio_pci_container_setup(struct vfio_pci_device *device, const char *bdf) { - unsigned long iommu_type = device->iommu->mode->iommu_type; - const char *path = device->iommu->mode->container_path; - int version; + struct iommu *iommu = device->iommu; + unsigned long iommu_type = iommu->mode->iommu_type; int ret; - device->iommu->container_fd = open(path, O_RDWR); - VFIO_ASSERT_GE(device->iommu->container_fd, 0, "open(%s) failed\n", path); - - version = ioctl(device->iommu->container_fd, VFIO_GET_API_VERSION); - VFIO_ASSERT_EQ(version, VFIO_API_VERSION, "Unsupported version: %d\n", version); - vfio_pci_group_setup(device, bdf); - ret = ioctl(device->iommu->container_fd, VFIO_CHECK_EXTENSION, iommu_type); + ret = ioctl(iommu->container_fd, VFIO_CHECK_EXTENSION, iommu_type); VFIO_ASSERT_GT(ret, 0, "VFIO IOMMU type %lu not supported\n", iommu_type); - ioctl_assert(device->iommu->container_fd, VFIO_SET_IOMMU, (void *)iommu_type); + /* + * Allow multiple threads to race to set the IOMMU type on the + * container. The first will succeed and the rest should fail + * because the IOMMU type is already set. + */ + (void)ioctl(iommu->container_fd, VFIO_SET_IOMMU, (void *)iommu_type); device->fd = ioctl(device->group_fd, VFIO_GROUP_GET_DEVICE_FD, bdf); VFIO_ASSERT_GE(device->fd, 0); @@ -494,32 +492,53 @@ static void vfio_pci_iommufd_setup(struct vfio_pci_device *device, const char *b VFIO_ASSERT_GE(device->fd, 0); free((void *)cdev_path); - /* - * Require device->iommufd to be >0 so that a simple non-0 check can be - * used to check if iommufd is enabled. In practice open() will never - * return 0 unless stdin is closed. - */ - device->iommu->iommufd = open("/dev/iommu", O_RDWR); - VFIO_ASSERT_GT(device->iommu->iommufd, 0); - vfio_device_bind_iommufd(device->fd, device->iommu->iommufd); - device->iommu->ioas_id = iommufd_ioas_alloc(device->iommu->iommufd); vfio_device_attach_iommufd_pt(device->fd, device->iommu->ioas_id); } -struct vfio_pci_device *vfio_pci_device_init(const char *bdf, const char *iommu_mode) +struct iommu *iommu_init(const char *iommu_mode) +{ + const char *container_path; + struct iommu *iommu; + int version; + + iommu = calloc(1, sizeof(*iommu)); + VFIO_ASSERT_NOT_NULL(iommu); + + INIT_LIST_HEAD(&iommu->dma_regions); + + iommu->mode = lookup_iommu_mode(iommu_mode); + + container_path = iommu->mode->container_path; + if (container_path) { + iommu->container_fd = open(container_path, O_RDWR); + VFIO_ASSERT_GE(iommu->container_fd, 0, "open(%s) failed\n", container_path); + + version = ioctl(iommu->container_fd, VFIO_GET_API_VERSION); + VFIO_ASSERT_EQ(version, VFIO_API_VERSION, "Unsupported version: %d\n", version); + } else { + /* + * Require device->iommufd to be >0 so that a simple non-0 check can be + * used to check if iommufd is enabled. In practice open() will never + * return 0 unless stdin is closed. + */ + iommu->iommufd = open("/dev/iommu", O_RDWR); + VFIO_ASSERT_GT(iommu->iommufd, 0); + + iommu->ioas_id = iommufd_ioas_alloc(iommu->iommufd); + } + + return iommu; +} + +struct vfio_pci_device *__vfio_pci_device_init(const char *bdf, struct iommu *iommu) { struct vfio_pci_device *device; device = calloc(1, sizeof(*device)); VFIO_ASSERT_NOT_NULL(device); - device->iommu = calloc(1, sizeof(*device->iommu)); - VFIO_ASSERT_NOT_NULL(device->iommu); - - INIT_LIST_HEAD(&device->iommu->dma_regions); - - device->iommu->mode = lookup_iommu_mode(iommu_mode); + device->iommu = iommu; if (device->iommu->mode->container_path) vfio_pci_container_setup(device, bdf); @@ -532,7 +551,14 @@ struct vfio_pci_device *vfio_pci_device_init(const char *bdf, const char *iommu_ return device; } -void vfio_pci_device_cleanup(struct vfio_pci_device *device) +struct vfio_pci_device *vfio_pci_device_init(const char *bdf, const char *iommu_mode) +{ + struct iommu *iommu = iommu_init(iommu_mode); + + return __vfio_pci_device_init(bdf, iommu); +} + +void __vfio_pci_device_cleanup(struct vfio_pci_device *device) { int i; @@ -550,17 +576,30 @@ void vfio_pci_device_cleanup(struct vfio_pci_device *device) VFIO_ASSERT_EQ(close(device->msi_eventfds[i]), 0); } - if (device->iommu->iommufd) { - VFIO_ASSERT_EQ(close(device->iommu->iommufd), 0); - } else { + if (device->group_fd) VFIO_ASSERT_EQ(close(device->group_fd), 0); - VFIO_ASSERT_EQ(close(device->iommu->container_fd), 0); - } - free(device->iommu); free(device); } +void iommu_cleanup(struct iommu *iommu) +{ + if (iommu->iommufd) + VFIO_ASSERT_EQ(close(iommu->iommufd), 0); + else + VFIO_ASSERT_EQ(close(iommu->container_fd), 0); + + free(iommu); +} + +void vfio_pci_device_cleanup(struct vfio_pci_device *device) +{ + struct iommu *iommu = device->iommu; + + __vfio_pci_device_cleanup(device); + iommu_cleanup(iommu); +} + static bool is_bdf(const char *str) { unsigned int s, b, d, f; -- 2.51.0.710.ga91ca5db03-goog