Add an opt-in iommufd_noiommu mode (MODE_IOMMUFD_NOIOMMU) for devices bound to vfio-pci in noiommu mode. When a device is in noiommu mode, its VFIO character device is named /dev/vfio/devices/noiommu-vfioX (opening it requires CAP_SYS_RAWIO), and binding and attaching to an iommufd IOAS use a software page table. Add __vfio_pci_get_cdev_path(bdf, noiommu) to look up either the regular or the noiommu cdev path and return NULL when none is present, and make vfio_pci_cdev_open() select the cdev based on mode->noiommu. Because a device in noiommu mode DMAs directly to physical addresses, tests that program arbitrary or unmapped IOVAs would corrupt memory. Keep MODE_IOMMUFD_NOIOMMU out of FIXTURE_VARIANT_ADD_ALL_IOMMU_MODES() so that only noiommu-aware tests opt into it. Signed-off-by: Vipin Sharma --- .../vfio/lib/include/libvfio/iommu.h | 17 ++++++++++ .../lib/include/libvfio/vfio_pci_device.h | 7 +++++ tools/testing/selftests/vfio/lib/iommu.c | 9 +++++- .../selftests/vfio/lib/vfio_pci_device.c | 31 +++++++++++++------ 4 files changed, 53 insertions(+), 11 deletions(-) diff --git a/tools/testing/selftests/vfio/lib/include/libvfio/iommu.h b/tools/testing/selftests/vfio/lib/include/libvfio/iommu.h index e9a3386a4719..e652b8caf5bb 100644 --- a/tools/testing/selftests/vfio/lib/include/libvfio/iommu.h +++ b/tools/testing/selftests/vfio/lib/include/libvfio/iommu.h @@ -13,6 +13,12 @@ struct iommu_mode { const char *name; const char *container_path; unsigned long iommu_type; + /* + * For devices bound to vfio-pci in noiommu mode, i.e. not behind an + * IOMMU. Such devices DMA to the addresses programmed into them as + * physical addresses. + */ + bool noiommu; }; extern const char *default_iommu_mode; @@ -67,6 +73,17 @@ struct iommu_iova_range *iommu_iova_ranges(struct iommu *iommu, u32 *nranges); #define MODE_IOMMUFD_COMPAT_TYPE1V2 "iommufd_compat_type1v2" #define MODE_IOMMUFD "iommufd" +/* + * iommufd with a device bound to vfio-pci in noiommu mode + * (vfio.enable_unsafe_noiommu_mode=1 and CONFIG_IOMMUFD_NOIOMMU), whose cdev + * is /dev/vfio/devices/noiommu-vfioX. Opening it requires CAP_SYS_RAWIO. + * + * Not part of FIXTURE_VARIANT_ADD_ALL_IOMMU_MODES(): tests must opt in, since + * the device DMAs to IOVAs as physical addresses, and tests that DMA to + * arbitrary or unmapped IOVAs would corrupt memory. + */ +#define MODE_IOMMUFD_NOIOMMU "iommufd_noiommu" + /* * Generator for VFIO selftests fixture variants that replicate across all * possible IOMMU modes. Tests must define FIXTURE_VARIANT_ADD_IOMMU_MODE() diff --git a/tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h b/tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h index 9a62bfb6b440..bfadeb362723 100644 --- a/tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h +++ b/tools/testing/selftests/vfio/lib/include/libvfio/vfio_pci_device.h @@ -154,6 +154,13 @@ static inline bool vfio_pci_device_match(struct vfio_pci_device *device, (device_id == vfio_pci_config_readw(device, PCI_DEVICE_ID)); } +/* + * Return the path of the VFIO cdev of @bdf, or NULL if there is none. The cdev + * of a device bound to vfio-pci in noiommu mode is named noiommu-vfioX rather + * than vfioX, and @noiommu selects which of the two to look for. Free with + * free(). + */ +const char *__vfio_pci_get_cdev_path(const char *bdf, bool noiommu); const char *vfio_pci_get_cdev_path(const char *bdf); void vfio_pci_group_setup(struct vfio_pci_device *device, const char *bdf); diff --git a/tools/testing/selftests/vfio/lib/iommu.c b/tools/testing/selftests/vfio/lib/iommu.c index b6f3c5c84e01..0df49646d016 100644 --- a/tools/testing/selftests/vfio/lib/iommu.c +++ b/tools/testing/selftests/vfio/lib/iommu.c @@ -22,7 +22,10 @@ const char *default_iommu_mode = MODE_IOMMUFD; -/* Reminder: Keep in sync with FIXTURE_VARIANT_ADD_ALL_IOMMU_MODES(). */ +/* + * Reminder: Keep in sync with FIXTURE_VARIANT_ADD_ALL_IOMMU_MODES(), which + * deliberately leaves out the noiommu modes. + */ static const struct iommu_mode iommu_modes[] = { { .name = MODE_VFIO_TYPE1_IOMMU, @@ -47,6 +50,10 @@ static const struct iommu_mode iommu_modes[] = { { .name = MODE_IOMMUFD, }, + { + .name = MODE_IOMMUFD_NOIOMMU, + .noiommu = true, + }, }; static const struct iommu_mode *lookup_iommu_mode(const char *iommu_mode) diff --git a/tools/testing/selftests/vfio/lib/vfio_pci_device.c b/tools/testing/selftests/vfio/lib/vfio_pci_device.c index af789c859586..b0deef2b0e3f 100644 --- a/tools/testing/selftests/vfio/lib/vfio_pci_device.c +++ b/tools/testing/selftests/vfio/lib/vfio_pci_device.c @@ -380,35 +380,42 @@ static void vfio_pci_device_setup(struct vfio_pci_device *device) device->msi_eventfds[i] = -1; } -const char *vfio_pci_get_cdev_path(const char *bdf) +const char *__vfio_pci_get_cdev_path(const char *bdf, bool noiommu) { + const char *prefix = noiommu ? "noiommu-vfio" : "vfio"; char dir_path[PATH_MAX]; struct dirent *entry; - char *cdev_path; + char *cdev_path = NULL; DIR *dir; - cdev_path = calloc_assert(PATH_MAX, 1); - snprintf_assert(dir_path, sizeof(dir_path), "/sys/bus/pci/devices/%s/vfio-dev/", bdf); dir = opendir(dir_path); - VFIO_ASSERT_NOT_NULL(dir, "Failed to open directory %s\n", dir_path); + if (!dir) + return NULL; while ((entry = readdir(dir)) != NULL) { - /* Find the file that starts with "vfio" */ - if (strncmp("vfio", entry->d_name, 4)) + if (strncmp(prefix, entry->d_name, strlen(prefix))) continue; + cdev_path = calloc_assert(PATH_MAX, 1); snprintf_assert(cdev_path, PATH_MAX, "/dev/vfio/devices/%s", entry->d_name); break; } - VFIO_ASSERT_NE(cdev_path[0], 0, "Failed to find vfio cdev file.\n"); VFIO_ASSERT_EQ(closedir(dir), 0); return cdev_path; } +const char *vfio_pci_get_cdev_path(const char *bdf) +{ + const char *cdev_path = __vfio_pci_get_cdev_path(bdf, false); + + VFIO_ASSERT_NOT_NULL(cdev_path, "Failed to find vfio cdev file for %s\n", bdf); + return cdev_path; +} + int __vfio_device_bind_iommufd(int device_fd, int iommufd, const char *vf_token) { struct vfio_device_bind_iommufd args = { @@ -449,10 +456,14 @@ static void vfio_device_attach_iommufd_pt(int device_fd, u32 pt_id) void vfio_pci_cdev_open(struct vfio_pci_device *device, const char *bdf) { - const char *cdev_path = vfio_pci_get_cdev_path(bdf); + bool noiommu = device->iommu->mode->noiommu; + const char *cdev_path = __vfio_pci_get_cdev_path(bdf, noiommu); + + VFIO_ASSERT_NOT_NULL(cdev_path, "Failed to find %svfio cdev file for %s\n", + noiommu ? "noiommu-" : "", bdf); device->fd = open(cdev_path, O_RDWR); - VFIO_ASSERT_GE(device->fd, 0); + VFIO_ASSERT_GE(device->fd, 0, "open(%s) failed\n", cdev_path); free((void *)cdev_path); } -- 2.56.0.317.g454f8302e1-goog