AI Trajectory

Trajectory: (Expand/Collapse All)
Seq	Timestamp	Type	Name	Duration
0/0	2026/07/02 00:20	flow	patch-triage	0m Args: null Results: { "EnableConfigs": [ "VFIO_PCI", "PCI_P2PDMA", "ZONE_DEVICE" ], "FocusSymbols": [ "vfio_pci_core_feature_dma_buf_revoke", "vfio_pci_core_feature_dma_buf_memattr", "vfio_pci_dma_buf_mmap", "vfio_pci_dma_buf_find_pfn", "vfio_pci_dmabuf_export", "vfio_pci_core_mmap_prep_dmabuf", "vfio_pci_dma_buf_set_status" ], "Reasoning": "The patch introduces new VFIO features (VFIO_DEVICE_FEATURE_DMA_BUF_REVOKE and VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR) and modifies the mmap handling to use DMABUF-backed mmaps. These are functional changes to reachable core kernel logic and should be fuzzed.", "WorthFuzzing": true }
1/1	2026/07/02 00:20	action	read-patch-diff	0m Args: null Results: { "PatchDiff": "commit be307977b6149355089ba083133af2137dd145b0\nAuthor: syz-cluster \u003ctriage@syzkaller.com\u003e\nDate: Thu Jul 2 00:20:04 2026 +0000\n\n syz-cluster: applied patch under review\n\ndiff --git a/MAINTAINERS b/MAINTAINERS\nindex c8d4b913f26c12..713861af44846a 100644\n--- a/MAINTAINERS\n+++ b/MAINTAINERS\n@@ -20626,7 +20626,7 @@ B:\thttps://bugzilla.kernel.org\n C:\tirc://irc.oftc.net/linux-pci\n T:\tgit git://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git\n F:\tDocumentation/driver-api/pci/p2pdma.rst\n-F:\tdrivers/pci/p2pdma.c\n+F:\tdrivers/pci/p2pdma\n F:\tinclude/linux/pci-p2pdma.h\n \n PCI POWER CONTROL\ndiff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig\nindex 33c88432b72875..59d70bc84cc9ac 100644\n--- a/drivers/pci/Kconfig\n+++ b/drivers/pci/Kconfig\n@@ -206,11 +206,7 @@ config PCIE_TPH\n config PCI_P2PDMA\n \tbool \"PCI peer-to-peer transfer support\"\n \tdepends on ZONE_DEVICE\n-\t#\n-\t# The need for the scatterlist DMA bus address flag means PCI P2PDMA\n-\t# requires 64bit\n-\t#\n-\tdepends on 64BIT\n+\tselect PCI_P2PDMA_CORE\n \tselect GENERIC_ALLOCATOR\n \tselect NEED_SG_DMA_FLAGS\n \thelp\n@@ -226,6 +222,10 @@ config PCI_P2PDMA\n \n \t If unsure, say N.\n \n+config PCI_P2PDMA_CORE\n+\tdefault n\n+\tbool\n+\n config PCI_LABEL\n \tdef_bool y if (DMI \|\| ACPI)\n \tselect NLS\ndiff --git a/drivers/pci/Makefile b/drivers/pci/Makefile\nindex 41ebc3b9a51824..0b32572d57a159 100644\n--- a/drivers/pci/Makefile\n+++ b/drivers/pci/Makefile\n@@ -30,6 +30,7 @@ obj-$(CONFIG_PCI_SYSCALL)\t+= syscall.o\n obj-$(CONFIG_PCI_STUB)\t\t+= pci-stub.o\n obj-$(CONFIG_PCI_PF_STUB)\t+= pci-pf-stub.o\n obj-$(CONFIG_PCI_ECAM)\t\t+= ecam.o\n+obj-$(CONFIG_PCI_P2PDMA_CORE)\t+= p2pdma_core.o\n obj-$(CONFIG_PCI_P2PDMA)\t+= p2pdma.o\n obj-$(CONFIG_XEN_PCIDEV_FRONTEND) += xen-pcifront.o\n obj-$(CONFIG_VGA_ARB)\t\t+= vgaarb.o\ndiff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c\nindex 7c898542af8d5e..50b1a7daf55cc4 100644\n--- a/drivers/pci/p2pdma.c\n+++ b/drivers/pci/p2pdma.c\n@@ -21,12 +21,7 @@\n #include \u003clinux/seq_buf.h\u003e\n #include \u003clinux/xarray.h\u003e\n \n-struct pci_p2pdma {\n-\tstruct gen_pool pool;\n-\tbool p2pmem_published;\n-\tstruct xarray map_types;\n-\tstruct p2pdma_provider mem[PCI_STD_NUM_BARS];\n-};\n+#include \"p2pdma.h\"\n \n struct pci_p2pdma_pagemap {\n \tstruct dev_pagemap pgmap;\n@@ -226,110 +221,16 @@ static const struct dev_pagemap_ops p2pdma_pgmap_ops = {\n \t.folio_free = p2pdma_folio_free,\n };\n \n-static void pci_p2pdma_release(void data)\n+void pci_p2pdma_release_pool(struct pci_dev pdev, struct pci_p2pdma p2pdma)\n {\n-\tstruct pci_dev pdev = data;\n-\tstruct pci_p2pdma p2pdma;\n-\n-\tp2pdma = rcu_dereference_protected(pdev-\u003ep2pdma, 1);\n-\tif (!p2pdma)\n-\t\treturn;\n-\n-\t/ Flush and disable pci_alloc_p2p_mem() /\n-\tpdev-\u003ep2pdma = NULL;\n-\tif (p2pdma-\u003epool)\n-\t\tsynchronize_rcu();\n-\txa_destroy(\u0026p2pdma-\u003emap_types);\n-\n \tif (!p2pdma-\u003epool)\n \t\treturn;\n \n+\tsynchronize_rcu();\n \tgen_pool_destroy(p2pdma-\u003epool);\n \tsysfs_remove_group(\u0026pdev-\u003edev.kobj, \u0026p2pmem_group);\n }\n \n-/\n- pcim_p2pdma_init - Initialise peer-to-peer DMA providers\n- * @pdev: The PCI device to enable P2PDMA for\n- \n- This function initializes the peer-to-peer DMA infrastructure\n- * for a PCI device. It allocates and sets up the necessary data\n- * structures to support P2PDMA operations, including mapping type\n- * tracking.\n- /\n-int pcim_p2pdma_init(struct pci_dev pdev)\n-{\n-\tstruct pci_p2pdma p2p;\n-\tint i, ret;\n-\n-\tp2p = rcu_dereference_protected(pdev-\u003ep2pdma, 1);\n-\tif (p2p)\n-\t\treturn 0;\n-\n-\tp2p = devm_kzalloc(\u0026pdev-\u003edev, sizeof(p2p), GFP_KERNEL);\n-\tif (!p2p)\n-\t\treturn -ENOMEM;\n-\n-\txa_init(\u0026p2p-\u003emap_types);\n-\t/\n-\t Iterate over all standard PCI BARs and record only those that\n-\t * correspond to MMIO regions. Skip non-memory resources (e.g. I/O\n-\t * port BARs) since they cannot be used for peer-to-peer (P2P)\n-\t * transactions.\n-\t /\n-\tfor (i = 0; i \u003c PCI_STD_NUM_BARS; i++) {\n-\t\tif (!(pci_resource_flags(pdev, i) \u0026 IORESOURCE_MEM))\n-\t\t\tcontinue;\n-\n-\t\tp2p-\u003emem[i].owner = \u0026pdev-\u003edev;\n-\t\tp2p-\u003emem[i].bus_offset =\n-\t\t\tpci_bus_address(pdev, i) - pci_resource_start(pdev, i);\n-\t}\n-\n-\tret = devm_add_action_or_reset(\u0026pdev-\u003edev, pci_p2pdma_release, pdev);\n-\tif (ret)\n-\t\tgoto out_p2p;\n-\n-\trcu_assign_pointer(pdev-\u003ep2pdma, p2p);\n-\treturn 0;\n-\n-out_p2p:\n-\tdevm_kfree(\u0026pdev-\u003edev, p2p);\n-\treturn ret;\n-}\n-EXPORT_SYMBOL_GPL(pcim_p2pdma_init);\n-\n-/\n- pcim_p2pdma_provider - Get peer-to-peer DMA provider\n- * @pdev: The PCI device to enable P2PDMA for\n- * @bar: BAR index to get provider\n- \n- This function gets peer-to-peer DMA provider for a PCI device. The lifetime\n- * of the provider (and of course the MMIO) is bound to the lifetime of the\n- * driver. A driver calling this function must ensure that all references to the\n- * provider, and any DMA mappings created for any MMIO, are all cleaned up\n- * before the driver remove() completes.\n- \n- Since P2P is almost always shared with a second driver this means some system\n- * to notify, invalidate and revoke the MMIO's DMA must be in place to use this\n- * function. For example a revoke can be built using DMABUF.\n- /\n-struct p2pdma_provider pcim_p2pdma_provider(struct pci_dev pdev, int bar)\n-{\n-\tstruct pci_p2pdma p2p;\n-\n-\tif (!(pci_resource_flags(pdev, bar) \u0026 IORESOURCE_MEM))\n-\t\treturn NULL;\n-\n-\tp2p = rcu_dereference_protected(pdev-\u003ep2pdma, 1);\n-\tif (WARN_ON(!p2p))\n-\t\t/* Someone forgot to call to pcim_p2pdma_init() before /\n-\t\treturn NULL;\n-\n-\treturn \u0026p2p-\u003emem[bar];\n-}\n-EXPORT_SYMBOL_GPL(pcim_p2pdma_provider);\n-\n static int pci_p2pdma_setup_pool(struct pci_dev pdev)\n {\n \tstruct pci_p2pdma p2pdma;\n@@ -932,8 +833,8 @@ void pci_alloc_p2pmem(struct pci_dev pdev, size_t size)\n \tstruct pci_p2pdma p2pdma;\n \n \t/\n-\t Pairs with synchronize_rcu() in pci_p2pdma_release() to\n-\t * ensure pdev-\u003ep2pdma is non-NULL for the duration of the\n+\t * Pairs with synchronize_rcu() in pci_p2pdma_release_pool()\n+\t * to ensure pdev-\u003ep2pdma is non-NULL for the duration of the\n \t * read-lock.\n \t /\n \trcu_read_lock();\ndiff --git a/drivers/pci/p2pdma.h b/drivers/pci/p2pdma.h\nnew file mode 100644\nindex 00000000000000..94638380998149\n--- /dev/null\n+++ b/drivers/pci/p2pdma.h\n@@ -0,0 +1,29 @@\n+/ SPDX-License-Identifier: GPL-2.0 /\n+/\n+ * PCI peer-to-peer DMA support.\n+ /\n+\n+#ifndef _PCI_P2PDMA_H\n+#define _PCI_P2PDMA_H\n+\n+#include \u003clinux/genalloc.h\u003e\n+#include \u003clinux/pci-p2pdma.h\u003e\n+#include \u003clinux/xarray.h\u003e\n+\n+struct pci_p2pdma {\n+\tstruct gen_pool pool;\n+\tbool p2pmem_published;\n+\tstruct xarray map_types;\n+\tstruct p2pdma_provider mem[PCI_STD_NUM_BARS];\n+};\n+\n+#ifdef CONFIG_PCI_P2PDMA\n+void pci_p2pdma_release_pool(struct pci_dev pdev, struct pci_p2pdma p2pdma);\n+#else\n+static inline void pci_p2pdma_release_pool(struct pci_dev pdev,\n+\t\t\t\t\t struct pci_p2pdma p2pdma)\n+{\n+}\n+#endif\n+\n+#endif\ndiff --git a/drivers/pci/p2pdma_core.c b/drivers/pci/p2pdma_core.c\nnew file mode 100644\nindex 00000000000000..bb2138bf2bc7b7\n--- /dev/null\n+++ b/drivers/pci/p2pdma_core.c\n@@ -0,0 +1,118 @@\n+// SPDX-License-Identifier: GPL-2.0\n+/\n+ PCI peer-to-peer DMA support core, providing a bare-bones\n+ * pcim_p2pdma_provider() interface to drivers even if full P2PDMA\n+ * isn't present. The full P2PDMA feature is in p2pdma.c (see\n+ * CONFIG_PCI_P2PDMA).\n+ \n+ Copyright (c) 2016-2018, Logan Gunthorpe\n+ * Copyright (c) 2016-2017, Microsemi Corporation\n+ * Copyright (c) 2017, Christoph Hellwig\n+ * Copyright (c) 2018, Eideticom Inc.\n+ /\n+\n+#define pr_fmt(fmt) \"pci-p2pdma: \" fmt\n+#include \u003clinux/ctype.h\u003e\n+#include \u003clinux/genalloc.h\u003e\n+#include \u003clinux/memremap.h\u003e\n+#include \u003clinux/pci-p2pdma.h\u003e\n+#include \u003clinux/xarray.h\u003e\n+\n+#include \"p2pdma.h\"\n+\n+static void pci_p2pdma_release(void data)\n+{\n+\tstruct pci_dev pdev = data;\n+\tstruct pci_p2pdma p2pdma;\n+\n+\tp2pdma = rcu_dereference_protected(pdev-\u003ep2pdma, 1);\n+\tif (!p2pdma)\n+\t\treturn;\n+\n+\t/* Flush and disable pci_alloc_p2p_mem() /\n+\tpdev-\u003ep2pdma = NULL;\n+\tpci_p2pdma_release_pool(pdev, p2pdma);\n+\txa_destroy(\u0026p2pdma-\u003emap_types);\n+}\n+\n+/\n+ pcim_p2pdma_init - Initialise peer-to-peer DMA providers\n+ * @pdev: The PCI device to enable P2PDMA for\n+ \n+ This function initializes the peer-to-peer DMA infrastructure\n+ * for a PCI device. It allocates and sets up the necessary data\n+ * structures to support P2PDMA operations, including mapping type\n+ * tracking.\n+ /\n+int pcim_p2pdma_init(struct pci_dev pdev)\n+{\n+\tstruct pci_p2pdma p2p;\n+\tint i, ret;\n+\n+\tp2p = rcu_dereference_protected(pdev-\u003ep2pdma, 1);\n+\tif (p2p)\n+\t\treturn 0;\n+\n+\tp2p = devm_kzalloc(\u0026pdev-\u003edev, sizeof(p2p), GFP_KERNEL);\n+\tif (!p2p)\n+\t\treturn -ENOMEM;\n+\n+\txa_init(\u0026p2p-\u003emap_types);\n+\t/\n+\t Iterate over all standard PCI BARs and record only those that\n+\t * correspond to MMIO regions. Skip non-memory resources (e.g. I/O\n+\t * port BARs) since they cannot be used for peer-to-peer (P2P)\n+\t * transactions.\n+\t /\n+\tfor (i = 0; i \u003c PCI_STD_NUM_BARS; i++) {\n+\t\tif (!(pci_resource_flags(pdev, i) \u0026 IORESOURCE_MEM))\n+\t\t\tcontinue;\n+\n+\t\tp2p-\u003emem[i].owner = \u0026pdev-\u003edev;\n+\t\tp2p-\u003emem[i].bus_offset =\n+\t\t\tpci_bus_address(pdev, i) - pci_resource_start(pdev, i);\n+\t}\n+\n+\tret = devm_add_action_or_reset(\u0026pdev-\u003edev, pci_p2pdma_release, pdev);\n+\tif (ret)\n+\t\tgoto out_p2p;\n+\n+\trcu_assign_pointer(pdev-\u003ep2pdma, p2p);\n+\treturn 0;\n+\n+out_p2p:\n+\tdevm_kfree(\u0026pdev-\u003edev, p2p);\n+\treturn ret;\n+}\n+EXPORT_SYMBOL_GPL(pcim_p2pdma_init);\n+\n+/\n+ pcim_p2pdma_provider - Get peer-to-peer DMA provider\n+ * @pdev: The PCI device to enable P2PDMA for\n+ * @bar: BAR index to get provider\n+ \n+ This function gets peer-to-peer DMA provider for a PCI device. The lifetime\n+ * of the provider (and of course the MMIO) is bound to the lifetime of the\n+ * driver. A driver calling this function must ensure that all references to the\n+ * provider, and any DMA mappings created for any MMIO, are all cleaned up\n+ * before the driver remove() completes.\n+ \n+ Since P2P is almost always shared with a second driver this means some system\n+ * to notify, invalidate and revoke the MMIO's DMA must be in place to use this\n+ * function. For example a revoke can be built using DMABUF.\n+ /\n+struct p2pdma_provider pcim_p2pdma_provider(struct pci_dev pdev, int bar)\n+{\n+\tstruct pci_p2pdma p2p;\n+\n+\tif (!(pci_resource_flags(pdev, bar) \u0026 IORESOURCE_MEM))\n+\t\treturn NULL;\n+\n+\tp2p = rcu_dereference_protected(pdev-\u003ep2pdma, 1);\n+\tif (WARN_ON(!p2p))\n+\t\t/* Someone forgot to call to pcim_p2pdma_init() before /\n+\t\treturn NULL;\n+\n+\treturn \u0026p2p-\u003emem[bar];\n+}\n+EXPORT_SYMBOL_GPL(pcim_p2pdma_provider);\ndiff --git a/drivers/vfio/pci/Kconfig b/drivers/vfio/pci/Kconfig\nindex 296bf01e185eca..67a2ae1fbc04ba 100644\n--- a/drivers/vfio/pci/Kconfig\n+++ b/drivers/vfio/pci/Kconfig\n@@ -6,6 +6,8 @@ config VFIO_PCI_CORE\n \ttristate\n \tselect VFIO_VIRQFD\n \tselect IRQ_BYPASS_MANAGER\n+\tselect PCI_P2PDMA_CORE\n+\tselect DMA_SHARED_BUFFER\n \n config VFIO_PCI_INTX\n \tdef_bool y if !S390\n@@ -56,7 +58,8 @@ config VFIO_PCI_ZDEV_KVM\n \t To enable s390x KVM vfio-pci extensions, say Y.\n \n config VFIO_PCI_DMABUF\n-\tdef_bool y if VFIO_PCI_CORE \u0026\u0026 PCI_P2PDMA \u0026\u0026 DMA_SHARED_BUFFER\n+\tdef_bool y if PCI_P2PDMA\n+\tdepends on VFIO_PCI_CORE\n \n source \"drivers/vfio/pci/mlx5/Kconfig\"\n \ndiff --git a/drivers/vfio/pci/Makefile b/drivers/vfio/pci/Makefile\nindex 6138f1bf241df0..881452ea89be0c 100644\n--- a/drivers/vfio/pci/Makefile\n+++ b/drivers/vfio/pci/Makefile\n@@ -1,8 +1,7 @@\n # SPDX-License-Identifier: GPL-2.0-only\n \n-vfio-pci-core-y := vfio_pci_core.o vfio_pci_intrs.o vfio_pci_rdwr.o vfio_pci_config.o\n+vfio-pci-core-y := vfio_pci_core.o vfio_pci_intrs.o vfio_pci_rdwr.o vfio_pci_config.o vfio_pci_dmabuf.o\n vfio-pci-core-$(CONFIG_VFIO_PCI_ZDEV_KVM) += vfio_pci_zdev.o\n-vfio-pci-core-$(CONFIG_VFIO_PCI_DMABUF) += vfio_pci_dmabuf.o\n obj-$(CONFIG_VFIO_PCI_CORE) += vfio-pci-core.o\n \n vfio-pci-y := vfio_pci.o\ndiff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c\nindex bb121f635b9f7f..d9f26b0ce991d9 100644\n--- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c\n+++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c\n@@ -1725,6 +1725,14 @@ static int hisi_acc_vfio_pci_probe(struct pci_dev pdev, const struct pci_device\n \tif (ret)\n \t\tgoto out_put_vdev;\n \n+\t/\n+\t hisi_acc_vfio_pci_mmap() calls down to\n+\t * vfio_pci_core_mmap(), so BAR mappings are still\n+\t * DMABUF-backed. They don't require a zap on revoke, so opt\n+\t * out:\n+\t /\n+\thisi_acc_vdev-\u003ecore_device.zap_bars_on_revoke = false;\n+\n \thisi_acc_vfio_debug_init(hisi_acc_vdev);\n \treturn 0;\n \ndiff --git a/drivers/vfio/pci/vfio_pci_config.c b/drivers/vfio/pci/vfio_pci_config.c\nindex a10ed733f0e3af..bc6ccb2e135c3c 100644\n--- a/drivers/vfio/pci/vfio_pci_config.c\n+++ b/drivers/vfio/pci/vfio_pci_config.c\n@@ -590,12 +590,10 @@ static int vfio_basic_config_write(struct vfio_pci_core_device vdev, int pos,\n \t\tvirt_mem = !!(le16_to_cpu(virt_cmd) \u0026 PCI_COMMAND_MEMORY);\n \t\tnew_mem = !!(new_cmd \u0026 PCI_COMMAND_MEMORY);\n \n-\t\tif (!new_mem) {\n-\t\t\tvfio_pci_zap_and_down_write_memory_lock(vdev);\n-\t\t\tvfio_pci_dma_buf_move(vdev, true);\n-\t\t} else {\n+\t\tif (!new_mem)\n+\t\t\tvfio_pci_lock_revoke_bars(vdev);\n+\t\telse\n \t\t\tdown_write(\u0026vdev-\u003ememory_lock);\n-\t\t}\n \n \t\t/\n \t\t * If the user is writing mem/io enable (new_mem/io) and we\n@@ -631,7 +629,7 @@ static int vfio_basic_config_write(struct vfio_pci_core_device vdev, int pos,\n \t\tvirt_cmd \|= cpu_to_le16(new_cmd \u0026 mask);\n \n \t\tif (__vfio_pci_memory_enabled(vdev))\n-\t\t\tvfio_pci_dma_buf_move(vdev, false);\n+\t\t\tvfio_pci_unrevoke_bars(vdev);\n \t\tup_write(\u0026vdev-\u003ememory_lock);\n \t}\n \n@@ -712,16 +710,14 @@ static int __init init_pci_cap_basic_perm(struct perm_bits perm)\n static void vfio_lock_and_set_power_state(struct vfio_pci_core_device vdev,\n \t\t\t\t\t pci_power_t state)\n {\n-\tif (state \u003e= PCI_D3hot) {\n-\t\tvfio_pci_zap_and_down_write_memory_lock(vdev);\n-\t\tvfio_pci_dma_buf_move(vdev, true);\n-\t} else {\n+\tif (state \u003e= PCI_D3hot)\n+\t\tvfio_pci_lock_revoke_bars(vdev);\n+\telse\n \t\tdown_write(\u0026vdev-\u003ememory_lock);\n-\t}\n \n \tvfio_pci_set_power_state(vdev, state);\n \tif (__vfio_pci_memory_enabled(vdev))\n-\t\tvfio_pci_dma_buf_move(vdev, false);\n+\t\tvfio_pci_unrevoke_bars(vdev);\n \tup_write(\u0026vdev-\u003ememory_lock);\n }\n \n@@ -908,11 +904,10 @@ static int vfio_exp_config_write(struct vfio_pci_core_device vdev, int pos,\n \t\t\t\t\t\t \u0026cap);\n \n \t\tif (!ret \u0026\u0026 (cap \u0026 PCI_EXP_DEVCAP_FLR)) {\n-\t\t\tvfio_pci_zap_and_down_write_memory_lock(vdev);\n-\t\t\tvfio_pci_dma_buf_move(vdev, true);\n+\t\t\tvfio_pci_lock_revoke_bars(vdev);\n \t\t\tpci_try_reset_function(vdev-\u003epdev);\n \t\t\tif (__vfio_pci_memory_enabled(vdev))\n-\t\t\t\tvfio_pci_dma_buf_move(vdev, false);\n+\t\t\t\tvfio_pci_unrevoke_bars(vdev);\n \t\t\tup_write(\u0026vdev-\u003ememory_lock);\n \t\t}\n \t}\n@@ -993,11 +988,10 @@ static int vfio_af_config_write(struct vfio_pci_core_device vdev, int pos,\n \t\t\t\t\t\t\u0026cap);\n \n \t\tif (!ret \u0026\u0026 (cap \u0026 PCI_AF_CAP_FLR) \u0026\u0026 (cap \u0026 PCI_AF_CAP_TP)) {\n-\t\t\tvfio_pci_zap_and_down_write_memory_lock(vdev);\n-\t\t\tvfio_pci_dma_buf_move(vdev, true);\n+\t\t\tvfio_pci_lock_revoke_bars(vdev);\n \t\t\tpci_try_reset_function(vdev-\u003epdev);\n \t\t\tif (__vfio_pci_memory_enabled(vdev))\n-\t\t\t\tvfio_pci_dma_buf_move(vdev, false);\n+\t\t\t\tvfio_pci_unrevoke_bars(vdev);\n \t\t\tup_write(\u0026vdev-\u003ememory_lock);\n \t\t}\n \t}\ndiff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c\nindex 050e7542952ed1..0059c176d62706 100644\n--- a/drivers/vfio/pci/vfio_pci_core.c\n+++ b/drivers/vfio/pci/vfio_pci_core.c\n@@ -12,6 +12,8 @@\n \n #include \u003clinux/aperture.h\u003e\n #include \u003clinux/device.h\u003e\n+#include \u003clinux/dma-buf.h\u003e\n+#include \u003clinux/dma-resv.h\u003e\n #include \u003clinux/eventfd.h\u003e\n #include \u003clinux/file.h\u003e\n #include \u003clinux/interrupt.h\u003e\n@@ -319,8 +321,7 @@ static int vfio_pci_runtime_pm_entry(struct vfio_pci_core_device vdev,\n \t The vdev power related flags are protected with 'memory_lock'\n \t * semaphore.\n \t /\n-\tvfio_pci_zap_and_down_write_memory_lock(vdev);\n-\tvfio_pci_dma_buf_move(vdev, true);\n+\tvfio_pci_lock_revoke_bars(vdev);\n \n \tif (vdev-\u003epm_runtime_engaged) {\n \t\tup_write(\u0026vdev-\u003ememory_lock);\n@@ -406,7 +407,7 @@ static void vfio_pci_runtime_pm_exit(struct vfio_pci_core_device vdev)\n \tdown_write(\u0026vdev-\u003ememory_lock);\n \t__vfio_pci_runtime_pm_exit(vdev);\n \tif (__vfio_pci_memory_enabled(vdev))\n-\t\tvfio_pci_dma_buf_move(vdev, false);\n+\t\tvfio_pci_unrevoke_bars(vdev);\n \tup_write(\u0026vdev-\u003ememory_lock);\n }\n \n@@ -1256,6 +1257,8 @@ static int vfio_pci_ioctl_set_irqs(struct vfio_pci_core_device vdev,\n \treturn ret;\n }\n \n+static void vfio_pci_revoke_bars(struct vfio_pci_core_device vdev);\n+\n static int vfio_pci_ioctl_reset(struct vfio_pci_core_device vdev,\n \t\t\t\tvoid __user arg)\n {\n@@ -1264,7 +1267,7 @@ static int vfio_pci_ioctl_reset(struct vfio_pci_core_device vdev,\n \tif (!vdev-\u003ereset_works)\n \t\treturn -EINVAL;\n \n-\tvfio_pci_zap_and_down_write_memory_lock(vdev);\n+\tdown_write(\u0026vdev-\u003ememory_lock);\n \n \t/\n \t * This function can be invoked while the power state is non-D0. If\n@@ -1274,13 +1277,18 @@ static int vfio_pci_ioctl_reset(struct vfio_pci_core_device vdev,\n \t have NoSoftRst-, the reset function can cause the PCI config space\n \t * reset without restoring the original state (saved locally in\n \t * 'vdev-\u003epm_save').\n+\t \n+\t The zap is done after making the device accessible in D0,\n+\t * because a DMABUF importer could access the device as part\n+\t * of its revocation cleanup.\n \t /\n \tvfio_pci_set_power_state(vdev, PCI_D0);\n \n-\tvfio_pci_dma_buf_move(vdev, true);\n+\tvfio_pci_revoke_bars(vdev);\n+\n \tret = pci_try_reset_function(vdev-\u003epdev);\n \tif (__vfio_pci_memory_enabled(vdev))\n-\t\tvfio_pci_dma_buf_move(vdev, false);\n+\t\tvfio_pci_unrevoke_bars(vdev);\n \tup_write(\u0026vdev-\u003ememory_lock);\n \n \treturn ret;\n@@ -1569,6 +1577,10 @@ int vfio_pci_core_ioctl_feature(struct vfio_device device, u32 flags,\n \t\treturn vfio_pci_core_feature_token(vdev, flags, arg, argsz);\n \tcase VFIO_DEVICE_FEATURE_DMA_BUF:\n \t\treturn vfio_pci_core_feature_dma_buf(vdev, flags, arg, argsz);\n+\tcase VFIO_DEVICE_FEATURE_DMA_BUF_REVOKE:\n+\t\treturn vfio_pci_core_feature_dma_buf_revoke(vdev, flags, arg, argsz);\n+\tcase VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR:\n+\t\treturn vfio_pci_core_feature_dma_buf_memattr(vdev, flags, arg, argsz);\n \tdefault:\n \t\treturn -ENOTTY;\n \t}\n@@ -1648,20 +1660,37 @@ ssize_t vfio_pci_core_write(struct vfio_device core_vdev, const char __user bu\n }\n EXPORT_SYMBOL_GPL(vfio_pci_core_write);\n \n-static void vfio_pci_zap_bars(struct vfio_pci_core_device vdev)\n+static void vfio_pci_revoke_bars(struct vfio_pci_core_device vdev)\n {\n-\tstruct vfio_device core_vdev = \u0026vdev-\u003evdev;\n-\tloff_t start = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR0_REGION_INDEX);\n-\tloff_t end = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_ROM_REGION_INDEX);\n-\tloff_t len = end - start;\n+\tlockdep_assert_held_write(\u0026vdev-\u003ememory_lock);\n+\tvfio_pci_dma_buf_move(vdev, true);\n \n-\tunmap_mapping_range(core_vdev-\u003einode-\u003ei_mapping, start, len, true);\n+\t/\n+\t * If a driver could possibly create BAR mappings in the\n+\t * vdev's address_space, do an additional zap on revoke. See\n+\t * vfio_pci_core_register_device().\n+\t /\n+\tif (vdev-\u003ezap_bars_on_revoke) {\n+\t\tstruct vfio_device core_vdev = \u0026vdev-\u003evdev;\n+\t\tloff_t start = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR0_REGION_INDEX);\n+\t\tloff_t end = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_ROM_REGION_INDEX);\n+\t\tloff_t len = end - start;\n+\n+\t\tunmap_mapping_range(core_vdev-\u003einode-\u003ei_mapping,\n+\t\t\t\t start, len, true);\n+\t}\n }\n \n-void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device vdev)\n+void vfio_pci_lock_revoke_bars(struct vfio_pci_core_device vdev)\n {\n \tdown_write(\u0026vdev-\u003ememory_lock);\n-\tvfio_pci_zap_bars(vdev);\n+\tvfio_pci_revoke_bars(vdev);\n+}\n+\n+void vfio_pci_unrevoke_bars(struct vfio_pci_core_device vdev)\n+{\n+\tlockdep_assert_held_write(\u0026vdev-\u003ememory_lock);\n+\tvfio_pci_dma_buf_move(vdev, false);\n }\n \n u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device vdev)\n@@ -1683,18 +1712,6 @@ void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device vdev, u16 c\n \tup_write(\u0026vdev-\u003ememory_lock);\n }\n \n-static unsigned long vma_to_pfn(struct vm_area_struct vma)\n-{\n-\tstruct vfio_pci_core_device vdev = vma-\u003evm_private_data;\n-\tint index = vma-\u003evm_pgoff \u003e\u003e (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);\n-\tu64 pgoff;\n-\n-\tpgoff = vma-\u003evm_pgoff \u0026\n-\t\t((1U \u003c\u003c (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);\n-\n-\treturn (pci_resource_start(vdev-\u003epdev, index) \u003e\u003e PAGE_SHIFT) + pgoff;\n-}\n-\n vm_fault_t vfio_pci_vmf_insert_pfn(struct vfio_pci_core_device vdev,\n \t\t\t\t struct vm_fault vmf,\n \t\t\t\t unsigned long pfn,\n@@ -1722,24 +1739,103 @@ static vm_fault_t vfio_pci_mmap_huge_fault(struct vm_fault vmf,\n \t\t\t\t\t unsigned int order)\n {\n \tstruct vm_area_struct vma = vmf-\u003evma;\n-\tstruct vfio_pci_core_device vdev = vma-\u003evm_private_data;\n-\tunsigned long addr = vmf-\u003eaddress \u0026 ~((PAGE_SIZE \u003c\u003c order) - 1);\n-\tunsigned long pgoff = (addr - vma-\u003evm_start) \u003e\u003e PAGE_SHIFT;\n-\tunsigned long pfn = vma_to_pfn(vma) + pgoff;\n-\tvm_fault_t ret = VM_FAULT_FALLBACK;\n-\n-\tif (is_aligned_for_order(vma, addr, pfn, order)) {\n-\t\tscoped_guard(rwsem_read, \u0026vdev-\u003ememory_lock)\n-\t\t\tret = vfio_pci_vmf_insert_pfn(vdev, vmf, pfn, order);\n+\tstruct vfio_pci_dma_buf priv = vma-\u003evm_private_data;\n+\tstruct vfio_pci_core_device vdev;\n+\tunsigned long pfn = 0;\n+\tvm_fault_t ret = VM_FAULT_SIGBUS;\n+\n+\t/\n+\t The only thing this can rely on is that the DMABUF relating\n+\t * to the VMA's vm_file exists (priv).\n+\t \n+\t A DMABUF for a VFIO device fd mmap() holds a reference to\n+\t * the original VFIO device fd, but an explicitly-exported\n+\t * DMABUF does not. The original fd might have closed,\n+\t * meaning this fault can race with\n+\t * vfio_pci_dma_buf_cleanup(), meaning the buffer could have\n+\t * been revoked (in which case priv-\u003evdev might be NULL), and\n+\t * the VFIO device registration might have been dropped.\n+\t \n+\t With the goal of taking vdev-\u003ememory_lock in a world where\n+\t * vdev might not still exist:\n+\t \n+\t 1. Take the resv lock on the DMABUF:\n+\t * - If racing cleanup got in first, the buffer is revoked;\n+\t * stop/exit if so.\n+\t * - If we got in first, the buffer is not revoked so vdev is\n+\t * non-NULL, accessible, and cleanup _has not yet put the\n+\t * VFIO device registration_. So, the device refcount must\n+\t * be \u003e0.\n+\t \n+\t 2. Take vfio_device registration (refcount guaranteed \u003e0\n+\t * hereafter).\n+\t \n+\t 3. Unlock the DMABUF's resv lock:\n+\t * - A racing cleanup can now complete.\n+\t * - But, the device refcount \u003e0, meaning the vfio_device\n+\t * (and vfio_pcie_core device vdev) have not yet been\n+\t * freed. vdev is accessible, even if the DMABUF has been\n+\t * revoked or cleanup has happened, because\n+\t * vfio_unregister_group_dev() can't complete.\n+\t \n+\t 4. Take the vdev-\u003ememory_lock\n+\t * - Either the DMABUF is usable, or has been cleaned up.\n+\t * Whichever, it can no longer change under us.\n+\t * - Test the DMABUF revocation status again: if it was\n+\t * revoked between 1 and 4 return a SIGBUS. Otherwise,\n+\t * return a PFN.\n+\t * - It's not necessary to also take the resv lock, because\n+\t * the status/vdev can't change while memory_lock is held.\n+\t \n+\t 5. Unlock, done.\n+\t /\n+\n+\tdma_resv_lock(priv-\u003edmabuf-\u003eresv, NULL);\n+\n+\tif (priv-\u003estatus != VFIO_PCI_DMABUF_OK) {\n+\t\tpr_debug_ratelimited(\"%s VA 0x%lx, pgoff 0x%lx: DMABUF revoked/cleaned up\\n\",\n+\t\t\t\t __func__, vmf-\u003eaddress, vma-\u003evm_pgoff);\n+\t\tdma_resv_unlock(priv-\u003edmabuf-\u003eresv);\n+\t\treturn VM_FAULT_SIGBUS;\n+\t}\n+\n+\t/ If the buffer isn't revoked, vdev is valid /\n+\tvdev = priv-\u003evdev;\n+\n+\tif (!vfio_device_try_get_registration(\u0026vdev-\u003evdev)) {\n+\t\t/\n+\t\t * If vdev != NULL (above), the registration should\n+\t\t * already be \u003e0 and so this try_get should never\n+\t\t * fail.\n+\t\t /\n+\t\tdev_warn(\u0026vdev-\u003epdev-\u003edev, \"%s: Unexpected registration failure\\n\",\n+\t\t\t __func__);\n+\t\tdma_resv_unlock(priv-\u003edmabuf-\u003eresv);\n+\t\treturn VM_FAULT_SIGBUS;\n+\t}\n+\tdma_resv_unlock(priv-\u003edmabuf-\u003eresv);\n+\n+\tscoped_guard(rwsem_read, \u0026vdev-\u003ememory_lock) {\n+\t\t/ Revocation status must be re-read, under memory_lock /\n+\t\tif (priv-\u003estatus == VFIO_PCI_DMABUF_OK) {\n+\t\t\tint pres = vfio_pci_dma_buf_find_pfn(priv, vma,\n+\t\t\t\t\t\t\t vmf-\u003eaddress,\n+\t\t\t\t\t\t\t order, \u0026pfn);\n+\n+\t\t\tif (pres == 0)\n+\t\t\t\tret = vfio_pci_vmf_insert_pfn(vdev, vmf,\n+\t\t\t\t\t\t\t pfn, order);\n+\t\t\telse if (pres == -EAGAIN)\n+\t\t\t\tret = VM_FAULT_FALLBACK;\n+\t\t}\n \t}\n \n \tdev_dbg_ratelimited(\u0026vdev-\u003epdev-\u003edev,\n-\t\t\t \"%s(,order = %d) BAR %ld page offset 0x%lx: 0x%x\\n\",\n-\t\t\t __func__, order,\n-\t\t\t vma-\u003evm_pgoff \u003e\u003e\n-\t\t\t\t(VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT),\n-\t\t\t pgoff, (unsigned int)ret);\n+\t\t\t \"%s(order = %d) PFN 0x%lx, VA 0x%lx, pgoff 0x%lx: 0x%x\\n\",\n+\t\t\t __func__, order, pfn, vmf-\u003eaddress,\n+\t\t\t vma-\u003evm_pgoff, (unsigned int)ret);\n \n+\tvfio_device_put_registration(\u0026vdev-\u003evdev);\n \treturn ret;\n }\n \n@@ -1755,6 +1851,11 @@ static const struct vm_operations_struct vfio_pci_mmap_ops = {\n #endif\n };\n \n+void vfio_pci_set_vma_ops(struct vm_area_struct vma)\n+{\n+\tvma-\u003evm_ops = \u0026vfio_pci_mmap_ops;\n+}\n+\n int vfio_pci_core_mmap(struct vfio_device core_vdev, struct vm_area_struct vma)\n {\n \tstruct vfio_pci_core_device vdev =\n@@ -1803,7 +1904,12 @@ int vfio_pci_core_mmap(struct vfio_device core_vdev, struct vm_area_struct vma\n \tif (ret)\n \t\treturn ret;\n \n-\tvma-\u003evm_private_data = vdev;\n+\tret = vfio_pci_core_mmap_prep_dmabuf(vdev, vma,\n+\t\t\t\t\t pci_resource_start(pdev, index),\n+\t\t\t\t\t req_len, index);\n+\tif (ret)\n+\t\treturn ret;\n+\n \tvma-\u003evm_page_prot = pgprot_noncached(vma-\u003evm_page_prot);\n \tvma-\u003evm_page_prot = pgprot_decrypted(vma-\u003evm_page_prot);\n \n@@ -2222,6 +2328,17 @@ int vfio_pci_core_register_device(struct vfio_pci_core_device vdev)\n \tif (ret)\n \t\tgoto out_vf;\n \n+\t/\n+\t If a driver overrides .mmap, it has to be assumed that it\n+\t * might not use the DMABUF-backed core mmap; this flag\n+\t * enables a zap at revoke time. Drivers can opt out by\n+\t * clearing this flag after registration, e.g. if their .mmap\n+\t * override calls down to vfio_pci_core_mmap() so that maps\n+\t * are DMABUF-backed.\n+\t /\n+\tif (vdev-\u003evdev.ops-\u003emmap != vfio_pci_core_mmap)\n+\t\tvdev-\u003ezap_bars_on_revoke = true;\n+\n \tvfio_pci_probe_power_state(vdev);\n \n \t/\n@@ -2497,9 +2614,10 @@ static int vfio_pci_dev_set_hot_reset(struct vfio_device_set dev_set,\n \t\t}\n \n \t\t/\n-\t\t * Take the memory write lock for each device and zap BAR\n-\t\t * mappings to prevent the user accessing the device while in\n-\t\t * reset. Locking multiple devices is prone to deadlock,\n+\t\t * Take the memory write lock for each device and\n+\t\t * zap/revoke BAR mappings to prevent the user (or\n+\t\t * peers) accessing the device while in reset.\n+\t\t * Locking multiple devices is prone to deadlock,\n \t\t * runaway and unwind if we hit contention.\n \t\t /\n \t\tif (!down_write_trylock(\u0026vdev-\u003ememory_lock)) {\n@@ -2507,8 +2625,7 @@ static int vfio_pci_dev_set_hot_reset(struct vfio_device_set dev_set,\n \t\t\tbreak;\n \t\t}\n \n-\t\tvfio_pci_dma_buf_move(vdev, true);\n-\t\tvfio_pci_zap_bars(vdev);\n+\t\tvfio_pci_revoke_bars(vdev);\n \t}\n \n \tif (!list_entry_is_head(vdev,\n@@ -2538,7 +2655,7 @@ static int vfio_pci_dev_set_hot_reset(struct vfio_device_set dev_set,\n \tlist_for_each_entry_from_reverse(vdev, \u0026dev_set-\u003edevice_list,\n \t\t\t\t\t vdev.dev_set_list) {\n \t\tif (vdev-\u003evdev.open_count \u0026\u0026 __vfio_pci_memory_enabled(vdev))\n-\t\t\tvfio_pci_dma_buf_move(vdev, false);\n+\t\t\tvfio_pci_unrevoke_bars(vdev);\n \t\tup_write(\u0026vdev-\u003ememory_lock);\n \t}\n \ndiff --git a/drivers/vfio/pci/vfio_pci_dmabuf.c b/drivers/vfio/pci/vfio_pci_dmabuf.c\nindex 1a177ce7de546a..31fbad93f89b1c 100644\n--- a/drivers/vfio/pci/vfio_pci_dmabuf.c\n+++ b/drivers/vfio/pci/vfio_pci_dmabuf.c\n@@ -4,24 +4,13 @@\n #include \u003clinux/dma-buf-mapping.h\u003e\n #include \u003clinux/pci-p2pdma.h\u003e\n #include \u003clinux/dma-resv.h\u003e\n+#include \u003cuapi/linux/dma-buf.h\u003e\n \n #include \"vfio_pci_priv.h\"\n \n MODULE_IMPORT_NS(\"DMA_BUF\");\n \n-struct vfio_pci_dma_buf {\n-\tstruct dma_buf dmabuf;\n-\tstruct vfio_pci_core_device vdev;\n-\tstruct list_head dmabufs_elm;\n-\tsize_t size;\n-\tstruct phys_vec phys_vec;\n-\tstruct p2pdma_provider provider;\n-\tu32 nr_ranges;\n-\tstruct kref kref;\n-\tstruct completion comp;\n-\tu8 revoked : 1;\n-};\n-\n+#ifdef CONFIG_VFIO_PCI_DMABUF\n static int vfio_pci_dma_buf_attach(struct dma_buf dmabuf,\n \t\t\t\t struct dma_buf_attachment attachment)\n {\n@@ -30,7 +19,7 @@ static int vfio_pci_dma_buf_attach(struct dma_buf dmabuf,\n \tif (!attachment-\u003epeer2peer)\n \t\treturn -EOPNOTSUPP;\n \n-\tif (priv-\u003erevoked)\n+\tif (priv-\u003estatus != VFIO_PCI_DMABUF_OK)\n \t\treturn -ENODEV;\n \n \tif (!dma_buf_attach_revocable(attachment))\n@@ -39,6 +28,43 @@ static int vfio_pci_dma_buf_attach(struct dma_buf dmabuf,\n \treturn 0;\n }\n \n+static int vfio_pci_dma_buf_mmap(struct dma_buf dmabuf, struct vm_area_struct vma)\n+{\n+\tstruct vfio_pci_dma_buf priv = dmabuf-\u003epriv;\n+\n+\t/\n+\t dma_buf_mmap_internal() has asserted that the VMA is\n+\t * contained within the DMABUF size before calling this.\n+\t \n+\t Also, if we observe that the buffer is revoked now then\n+\t * refuse the mmap(). This is a belt-and-braces early failure\n+\t * to ease debugging a revoked buffer being used. Userspace\n+\t * might also race an mmap() against an explicit revocation,\n+\t * or an action doing a temporary revoke; race scenarios are\n+\t * still safe because the fault handler ultimately prevents\n+\t * access to a revoked buffer if it isn't caught here.\n+\t /\n+\tif (READ_ONCE(priv-\u003estatus) != VFIO_PCI_DMABUF_OK)\n+\t\treturn -ENODEV;\n+\tif ((vma-\u003evm_flags \u0026 VM_SHARED) == 0)\n+\t\treturn -EINVAL;\n+\n+\tif (READ_ONCE(priv-\u003ememattr) == VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR_WC)\n+\t\tvma-\u003evm_page_prot = pgprot_writecombine(vma-\u003evm_page_prot);\n+\telse\n+\t\tvma-\u003evm_page_prot = pgprot_noncached(vma-\u003evm_page_prot);\n+\tvma-\u003evm_page_prot = pgprot_decrypted(vma-\u003evm_page_prot);\n+\n+\t/ See comments in vfio_pci_core_mmap() re VM_ALLOW_ANY_UNCACHED. /\n+\tvm_flags_set(vma, VM_ALLOW_ANY_UNCACHED \| VM_IO \| VM_PFNMAP \|\n+\t\t VM_DONTEXPAND \| VM_DONTDUMP);\n+\tvma-\u003evm_private_data = priv;\n+\tvfio_pci_set_vma_ops(vma);\n+\n+\treturn 0;\n+}\n+#endif / CONFIG_VFIO_PCI_DMABUF /\n+\n static void vfio_pci_dma_buf_done(struct kref kref)\n {\n \tstruct vfio_pci_dma_buf priv =\n@@ -56,7 +82,7 @@ vfio_pci_dma_buf_map(struct dma_buf_attachment attachment,\n \n \tdma_resv_assert_held(priv-\u003edmabuf-\u003eresv);\n \n-\tif (priv-\u003erevoked)\n+\tif (priv-\u003estatus != VFIO_PCI_DMABUF_OK)\n \t\treturn ERR_PTR(-ENODEV);\n \n \tret = dma_buf_phys_vec_to_sgt(attachment, priv-\u003eprovider,\n@@ -95,17 +121,203 @@ static void vfio_pci_dma_buf_release(struct dma_buf dmabuf)\n \t\tup_write(\u0026priv-\u003evdev-\u003ememory_lock);\n \t\tvfio_device_put_registration(\u0026priv-\u003evdev-\u003evdev);\n \t}\n+\tif (priv-\u003evfile)\n+\t\tfput(priv-\u003evfile);\n \tkfree(priv-\u003ephys_vec);\n \tkfree(priv);\n }\n \n static const struct dma_buf_ops vfio_pci_dmabuf_ops = {\n+#ifdef CONFIG_VFIO_PCI_DMABUF\n \t.attach = vfio_pci_dma_buf_attach,\n+\t.mmap = vfio_pci_dma_buf_mmap,\n+#endif\n \t.map_dma_buf = vfio_pci_dma_buf_map,\n \t.unmap_dma_buf = vfio_pci_dma_buf_unmap,\n \t.release = vfio_pci_dma_buf_release,\n };\n \n+int vfio_pci_dma_buf_find_pfn(struct vfio_pci_dma_buf priv,\n+\t\t\t struct vm_area_struct vma,\n+\t\t\t unsigned long fault_addr,\n+\t\t\t unsigned int order,\n+\t\t\t unsigned long out_pfn)\n+{\n+\t/\n+\t Given a VMA (start, end, pgoffs) and a fault address,\n+\t * search the corresponding DMABUF's phys_vec[] to find the\n+\t * range representing the address's offset into the VMA, and\n+\t * its PFN.\n+\t \n+\t The phys_vec[] ranges represent contiguous spans of VAs\n+\t * upwards from the buffer offset 0; the actual PFNs might be\n+\t * in any order, overlap/alias, etc. Calculate an offset of\n+\t * the desired page given VMA start/pgoff and address, then\n+\t * search upwards from 0 to find which span contains it.\n+\t \n+\t On success, a valid PFN for a page sized by 'order' is\n+\t * returned into out_pfn.\n+\t \n+\t Failure occurs if:\n+\t * - A hugepage would cross the edge of the VMA,\n+\t * - A hugepage isn't entirely contained within a range\n+\t * (including where it straddles the boundary between\n+\t * ranges),\n+\t * - We find a range, but the final PFN isn't aligned to the\n+\t * requested order.\n+\t \n+\t Upon failure, -EAGAIN is returned and the caller is\n+\t * expected to try again with a smaller order, which will\n+\t * eventually succeed (order=0 will always work).\n+\t \n+\t It's suboptimal if DMABUFs are created with neighbouring\n+\t * ranges that are physically contiguous, since hugepages\n+\t * can't straddle range boundaries. (The construction of the\n+\t * ranges should merge them in this case.)\n+\t \n+\t Finally, vma_pgoff_adjust is used with a DMABUF created for\n+\t * a VFIO BAR mmap: a BAR mapped with vm_pgoff \u003e 0 creates a\n+\t * DMABUF such that byte 0 of the VMA corresponds to byte 0 of\n+\t * the DMABUF and byte 'vm_pgoff \u003c\u003c PAGE_SHIFT' into the BAR.\n+\t * To avoid double-offsetting in this scenario, subtracting\n+\t * vma_pgoff_adjust from this (non-zero) vm_pgoff generates\n+\t * the effective offset.\n+\t /\n+\n+\tconst unsigned long pagesize = PAGE_SIZE \u003c\u003c order;\n+\tunsigned long vma_off = ((vma-\u003evm_pgoff - priv-\u003evma_pgoff_adjust) \u003c\u003c\n+\t\t\t\t PAGE_SHIFT) \u0026 VFIO_PCI_OFFSET_MASK;\n+\tunsigned long rounded_page_addr = ALIGN_DOWN(fault_addr, pagesize);\n+\tunsigned long rounded_page_end = rounded_page_addr + pagesize;\n+\tunsigned long fault_offset;\n+\tunsigned long fault_offset_end;\n+\tunsigned long range_start_offset = 0;\n+\tunsigned int i;\n+\tint ret;\n+\n+\tif (rounded_page_addr \u003c vma-\u003evm_start \|\| rounded_page_end \u003e vma-\u003evm_end) {\n+\t\tif (order \u003e 0)\n+\t\t\treturn -EAGAIN;\n+\n+\t\t/ A fault address outside of the VMA is absurd. /\n+\t\tdev_warn_ratelimited(\n+\t\t\t\u0026priv-\u003evdev-\u003epdev-\u003edev,\n+\t\t\t\"Fault addr 0x%lx outside VMA 0x%lx-0x%lx\\n\",\n+\t\t\tfault_addr, vma-\u003evm_start, vma-\u003evm_end);\n+\t\treturn -EFAULT;\n+\t}\n+\n+\t/\n+\t * fault_offset[_end] is the span within the DMABUF\n+\t * corresponding to the faulting page:\n+\t /\n+\tif (unlikely(check_add_overflow(rounded_page_addr - vma-\u003evm_start,\n+\t\t\t\t\tvma_off, \u0026fault_offset) \|\|\n+\t\t check_add_overflow(fault_offset, pagesize,\n+\t\t\t\t\t\u0026fault_offset_end)))\n+\t\treturn -EFAULT;\n+\n+\t/\n+\t * Iterate over ranges in the buffer, summing their lengths:\n+\t * range_start_offset represents the current range's starting\n+\t * offset in the buffer (from 0 upwards).\n+\t \n+\t A failure for order == 0 is unexpected, and triggers a\n+\t * fault/warn.\n+\t /\n+\tret = (order == 0) ? -EFAULT : -EAGAIN;\n+\n+\tfor (i = 0; i \u003c priv-\u003enr_ranges; i++) {\n+\t\tsize_t range_len = priv-\u003ephys_vec[i].len;\n+\n+\t\t/ Early exit if range starts after the page end /\n+\t\tif (fault_offset_end \u003c= range_start_offset)\n+\t\t\tbreak;\n+\n+\t\tif (fault_offset \u003e= range_start_offset \u0026\u0026\n+\t\t fault_offset_end \u003c= range_start_offset + range_len) {\n+\t\t\t/\n+\t\t\t * The faulting page is wholly contained\n+\t\t\t * within the span represented by this range,\n+\t\t\t * so validate PFN alignment for the order.\n+\t\t\t * The if() condition ensures the pfn\n+\t\t\t * arithmetic won't overflow.\n+\t\t\t /\n+\t\t\tunsigned long pfn =\n+\t\t\t\t((fault_offset - range_start_offset) +\n+\t\t\t\t priv-\u003ephys_vec[i].paddr) \u003e\u003e PAGE_SHIFT;\n+\n+\t\t\tif (IS_ALIGNED(pfn, 1 \u003c\u003c order)) {\n+\t\t\t\tout_pfn = pfn;\n+\t\t\t\tret = 0;\n+\t\t\t}\n+\t\t\t/\n+\t\t\t Else order \u003e 0; EAGAIN retries with smaller\n+\t\t\t * order\n+\t\t\t /\n+\t\t\tbreak;\n+\t\t}\n+\t\trange_start_offset += range_len;\n+\t}\n+\n+\tif (order == 0 \u0026\u0026 ret != 0)\n+\t\t/\n+\t\t * The address fell outside of the span represented by\n+\t\t * the (concatenated) ranges. As setup of a mapping\n+\t\t * ensures that the VMA is \u003c= the total size of the\n+\t\t * ranges this should never happen. If it does, warn\n+\t\t * and SIGBUS.\n+\t\t /\n+\t\tdev_warn_ratelimited(\n+\t\t\t\u0026priv-\u003evdev-\u003epdev-\u003edev,\n+\t\t\t\"No range for addr 0x%lx, order %d: VMA 0x%lx-0x%lx pgoff 0x%lx, %u ranges, size 0x%zx\\n\",\n+\t\t\tfault_addr, order, vma-\u003evm_start, vma-\u003evm_end,\n+\t\t\tvma-\u003evm_pgoff, priv-\u003enr_ranges, priv-\u003esize);\n+\n+\treturn ret;\n+}\n+\n+/\n+ * Create a DMABUF corresponding to priv, add it to vdev-\u003edmabufs list\n+ * for tracking (meaning cleanup or revocation will zap it), and take\n+ * a vfio_device registration.\n+ /\n+static int vfio_pci_dmabuf_export(struct vfio_pci_core_device vdev,\n+\t\t\t\t struct vfio_pci_dma_buf priv, u32 flags)\n+{\n+\tDEFINE_DMA_BUF_EXPORT_INFO(exp_info);\n+\n+\tif (!vfio_device_try_get_registration(\u0026vdev-\u003evdev))\n+\t\treturn -ENODEV;\n+\n+\texp_info.ops = \u0026vfio_pci_dmabuf_ops;\n+\texp_info.size = priv-\u003esize;\n+\texp_info.flags = flags;\n+\texp_info.priv = priv;\n+\n+\tpriv-\u003edmabuf = dma_buf_export(\u0026exp_info);\n+\tif (IS_ERR(priv-\u003edmabuf)) {\n+\t\tvfio_device_put_registration(\u0026vdev-\u003evdev);\n+\t\treturn PTR_ERR(priv-\u003edmabuf);\n+\t}\n+\n+\tkref_init(\u0026priv-\u003ekref);\n+\tinit_completion(\u0026priv-\u003ecomp);\n+\n+\t/ dma_buf_put() now frees priv /\n+\tINIT_LIST_HEAD(\u0026priv-\u003edmabufs_elm);\n+\tdown_write(\u0026vdev-\u003ememory_lock);\n+\tdma_resv_lock(priv-\u003edmabuf-\u003eresv, NULL);\n+\tpriv-\u003estatus = __vfio_pci_memory_enabled(vdev) ? VFIO_PCI_DMABUF_OK :\n+\t\tVFIO_PCI_DMABUF_TEMP_REVOKED;\n+\tlist_add_tail(\u0026priv-\u003edmabufs_elm, \u0026vdev-\u003edmabufs);\n+\tdma_resv_unlock(priv-\u003edmabuf-\u003eresv);\n+\tup_write(\u0026vdev-\u003ememory_lock);\n+\n+\treturn 0;\n+}\n+\n+#ifdef CONFIG_VFIO_PCI_DMABUF\n /\n * This is a temporary \"private interconnect\" between VFIO DMABUF and iommufd.\n * It allows the two co-operating drivers to exchange the physical address of\n@@ -128,7 +340,7 @@ int vfio_pci_dma_buf_iommufd_map(struct dma_buf_attachment attachment,\n \t\treturn -EOPNOTSUPP;\n \n \tpriv = attachment-\u003edmabuf-\u003epriv;\n-\tif (priv-\u003erevoked)\n+\tif (priv-\u003estatus != VFIO_PCI_DMABUF_OK)\n \t\treturn -ENODEV;\n \n \t/ More than one range to iommufd will require proper DMABUF support /\n@@ -224,7 +436,6 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device vdev, u32 flags,\n {\n \tstruct vfio_device_feature_dma_buf get_dma_buf = {};\n \tstruct vfio_region_dma_range dma_ranges;\n-\tDEFINE_DMA_BUF_EXPORT_INFO(exp_info);\n \tstruct vfio_pci_dma_buf priv;\n \tsize_t length;\n \tint ret;\n@@ -274,6 +485,7 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device vdev, u32 flags,\n \tpriv-\u003evdev = vdev;\n \tpriv-\u003enr_ranges = get_dma_buf.nr_ranges;\n \tpriv-\u003esize = length;\n+\tpriv-\u003ememattr = VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR_NC;\n \tret = vdev-\u003epci_ops-\u003eget_dmabuf_phys(vdev, \u0026priv-\u003eprovider,\n \t\t\t\t\t get_dma_buf.region_index,\n \t\t\t\t\t priv-\u003ephys_vec, dma_ranges,\n@@ -284,34 +496,9 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device vdev, u32 flags,\n \tkfree(dma_ranges);\n \tdma_ranges = NULL;\n \n-\tif (!vfio_device_try_get_registration(\u0026vdev-\u003evdev)) {\n-\t\tret = -ENODEV;\n+\tret = vfio_pci_dmabuf_export(vdev, priv, get_dma_buf.open_flags);\n+\tif (ret)\n \t\tgoto err_free_phys;\n-\t}\n-\n-\texp_info.ops = \u0026vfio_pci_dmabuf_ops;\n-\texp_info.size = priv-\u003esize;\n-\texp_info.flags = get_dma_buf.open_flags;\n-\texp_info.priv = priv;\n-\n-\tpriv-\u003edmabuf = dma_buf_export(\u0026exp_info);\n-\tif (IS_ERR(priv-\u003edmabuf)) {\n-\t\tret = PTR_ERR(priv-\u003edmabuf);\n-\t\tgoto err_dev_put;\n-\t}\n-\n-\tkref_init(\u0026priv-\u003ekref);\n-\tinit_completion(\u0026priv-\u003ecomp);\n-\n-\t/* dma_buf_put() now frees priv /\n-\tINIT_LIST_HEAD(\u0026priv-\u003edmabufs_elm);\n-\tdown_write(\u0026vdev-\u003ememory_lock);\n-\tdma_resv_lock(priv-\u003edmabuf-\u003eresv, NULL);\n-\tpriv-\u003erevoked = !__vfio_pci_memory_enabled(vdev);\n-\tlist_add_tail(\u0026priv-\u003edmabufs_elm, \u0026vdev-\u003edmabufs);\n-\tdma_resv_unlock(priv-\u003edmabuf-\u003eresv);\n-\tup_write(\u0026vdev-\u003ememory_lock);\n-\n \t/\n \t * dma_buf_fd() consumes the reference, when the file closes the dmabuf\n \t * will be released.\n@@ -322,8 +509,6 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device vdev, u32 flags,\n \n \treturn ret;\n \n-err_dev_put:\n-\tvfio_device_put_registration(\u0026vdev-\u003evdev);\n err_free_phys:\n \tkfree(priv-\u003ephys_vec);\n err_free_priv:\n@@ -332,47 +517,164 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device vdev, u32 flags,\n \tkfree(dma_ranges);\n \treturn ret;\n }\n+#endif /* CONFIG_VFIO_PCI_DMABUF /\n+\n+int vfio_pci_core_mmap_prep_dmabuf(struct vfio_pci_core_device vdev,\n+\t\t\t\t struct vm_area_struct vma,\n+\t\t\t\t u64 phys_start, u64 req_len,\n+\t\t\t\t unsigned int res_index)\n+{\n+\tstruct vfio_pci_dma_buf priv;\n+\tunsigned long vma_pgoff = vma-\u003evm_pgoff \u0026 (VFIO_PCI_OFFSET_MASK \u003e\u003e PAGE_SHIFT);\n+\tchar bufname;\n+\tint ret;\n+\n+\tpriv = kzalloc_obj(priv);\n+\tif (!priv)\n+\t\treturn -ENOMEM;\n+\n+\tpriv-\u003ephys_vec = kzalloc_obj(priv-\u003ephys_vec);\n+\tif (!priv-\u003ephys_vec) {\n+\t\tret = -ENOMEM;\n+\t\tgoto err_free_priv;\n+\t}\n+\n+\tbufname = kasprintf(GFP_KERNEL, \"%s:%s/%x\",\n+\t\t\t dev_name(\u0026vdev-\u003evdev.device), pci_name(vdev-\u003epdev),\n+\t\t\t res_index);\n+\n+\tif (!bufname) {\n+\t\tret = -ENOMEM;\n+\t\tgoto err_free_phys;\n+\t}\n+\n+\t/\n+\t * The DMABUF begins from the mmap()'s BAR offset, i.e. the\n+\t * start of the VMA corresponds to byte 0 of the DMABUF and\n+\t * byte (vma_pgoff \u003c\u003c PAGE_SHIFT) of the BAR.\n+\t \n+\t vfio_pci_dma_buf_find_pfn() reverses this offset using\n+\t * vma_pgoff_adjust, so that ultimately a fault's offset from\n+\t * the start of the _VMA_ has a consistent usage whether the\n+\t * VMA originates from an mmap() of the VFIO device here or a\n+\t * direct DMABUF mmap().\n+\t /\n+\tpriv-\u003evdev = vdev;\n+\tpriv-\u003esize = req_len;\n+\tpriv-\u003enr_ranges = 1;\n+\tpriv-\u003evma_pgoff_adjust = vma_pgoff;\n+\tpriv-\u003eprovider = pcim_p2pdma_provider(vdev-\u003epdev, res_index);\n+\tif (!priv-\u003eprovider) {\n+\t\tret = -EINVAL;\n+\t\tgoto err_free_name;\n+\t}\n+\n+\tpriv-\u003ephys_vec[0].paddr = phys_start + ((u64)vma_pgoff \u003c\u003c PAGE_SHIFT);\n+\tpriv-\u003ephys_vec[0].len = priv-\u003esize;\n+\n+\tret = vfio_pci_dmabuf_export(vdev, priv, O_CLOEXEC \| O_RDWR);\n+\tif (ret)\n+\t\tgoto err_free_name;\n+\n+\t/\n+\t * Ownership of the DMABUF file transfers to the VMA so that\n+\t * other users can locate the DMABUF via a VA. Ownership of\n+\t * the original VFIO device file being mmap()ed transfers to\n+\t * priv, and is put when the DMABUF is released. This\n+\t * intentionally does not use get_file()/vma_set_file()\n+\t * because the references are already held, and ownership\n+\t * moves.\n+\t /\n+\tpriv-\u003evfile = vma-\u003evm_file;\n+\tvma-\u003evm_file = priv-\u003edmabuf-\u003efile;\n+\tvma-\u003evm_private_data = priv;\n+\n+\tspin_lock(\u0026priv-\u003edmabuf-\u003ename_lock);\n+\tkfree(priv-\u003edmabuf-\u003ename);\n+\tpriv-\u003edmabuf-\u003ename = bufname;\n+\tspin_unlock(\u0026priv-\u003edmabuf-\u003ename_lock);\n+\n+\treturn 0;\n+\n+err_free_name:\n+\tkfree(bufname);\n+err_free_phys:\n+\tkfree(priv-\u003ephys_vec);\n+err_free_priv:\n+\tkfree(priv);\n+\treturn ret;\n+}\n+\n+/ Set the DMABUF's revocation status (OK or temporarily/permanently revoked) /\n+static void vfio_pci_dma_buf_set_status(struct vfio_pci_dma_buf priv,\n+\t\t\t\t\tenum vfio_pci_dma_buf_status new_status)\n+{\n+\tbool was_revoked;\n+\n+\tlockdep_assert_held_write(\u0026priv-\u003evdev-\u003ememory_lock);\n+\n+\tif (priv-\u003estatus == VFIO_PCI_DMABUF_PERM_REVOKED \|\|\n+\t priv-\u003estatus == new_status)\n+\t\treturn;\n+\n+\tdma_resv_lock(priv-\u003edmabuf-\u003eresv, NULL);\n+\twas_revoked = (priv-\u003estatus == VFIO_PCI_DMABUF_TEMP_REVOKED);\n+\n+\tif (new_status != VFIO_PCI_DMABUF_OK) {\n+\t\tpriv-\u003estatus = new_status; /* Temp or permanently revoked /\n+\n+\t\tif (was_revoked) {\n+\t\t\t/\n+\t\t\t * TEMP_REVOKED is being upgraded to\n+\t\t\t * PERM_REVOKED. The buffer is already gone,\n+\t\t\t * don't wait on it again.\n+\t\t\t /\n+\t\t\tdma_resv_unlock(priv-\u003edmabuf-\u003eresv);\n+\t\t\treturn;\n+\t\t}\n+\t\tdma_buf_invalidate_mappings(priv-\u003edmabuf);\n+\t\tdma_resv_wait_timeout(priv-\u003edmabuf-\u003eresv,\n+\t\t\t\t DMA_RESV_USAGE_BOOKKEEP, false,\n+\t\t\t\t MAX_SCHEDULE_TIMEOUT);\n+\t\tdma_resv_unlock(priv-\u003edmabuf-\u003eresv);\n+\t\tkref_put(\u0026priv-\u003ekref, vfio_pci_dma_buf_done);\n+\t\twait_for_completion(\u0026priv-\u003ecomp);\n+\t\tunmap_mapping_range(priv-\u003edmabuf-\u003efile-\u003ef_mapping,\n+\t\t\t\t 0, priv-\u003esize, true);\n+\t\t/\n+\t\t * Re-arm the registered kref reference and the\n+\t\t * completion so the post-revoke state matches the\n+\t\t * post-creation state. An un-revoke followed by a\n+\t\t * new mapping needs the kref to be non-zero before\n+\t\t * kref_get(), and vfio_pci_dma_buf_cleanup()\n+\t\t * delegates its drain back through this revoke\n+\t\t * path on a possibly-already-revoked dma-buf.\n+\t\t /\n+\t\tkref_init(\u0026priv-\u003ekref);\n+\t\treinit_completion(\u0026priv-\u003ecomp);\n+\t} else {\n+\t\tpriv-\u003estatus = VFIO_PCI_DMABUF_OK;\n+\t\tdma_resv_unlock(priv-\u003edmabuf-\u003eresv);\n+\t}\n+}\n \n void vfio_pci_dma_buf_move(struct vfio_pci_core_device vdev, bool revoked)\n {\n \tstruct vfio_pci_dma_buf priv;\n \tstruct vfio_pci_dma_buf tmp;\n \n+\t/\n+\t Holding memory_lock ensures a racing VMA fault observes\n+\t * priv-\u003estatus properly.\n+\t /\n \tlockdep_assert_held_write(\u0026vdev-\u003ememory_lock);\n \n \tlist_for_each_entry_safe(priv, tmp, \u0026vdev-\u003edmabufs, dmabufs_elm) {\n \t\tif (!get_file_active(\u0026priv-\u003edmabuf-\u003efile))\n \t\t\tcontinue;\n-\n-\t\tif (priv-\u003erevoked != revoked) {\n-\t\t\tdma_resv_lock(priv-\u003edmabuf-\u003eresv, NULL);\n-\t\t\tif (revoked)\n-\t\t\t\tpriv-\u003erevoked = true;\n-\t\t\tdma_buf_invalidate_mappings(priv-\u003edmabuf);\n-\t\t\tdma_resv_wait_timeout(priv-\u003edmabuf-\u003eresv,\n-\t\t\t\t\t DMA_RESV_USAGE_BOOKKEEP, false,\n-\t\t\t\t\t MAX_SCHEDULE_TIMEOUT);\n-\t\t\tdma_resv_unlock(priv-\u003edmabuf-\u003eresv);\n-\t\t\tif (revoked) {\n-\t\t\t\tkref_put(\u0026priv-\u003ekref, vfio_pci_dma_buf_done);\n-\t\t\t\twait_for_completion(\u0026priv-\u003ecomp);\n-\t\t\t\t/\n-\t\t\t\t * Re-arm the registered kref reference and the\n-\t\t\t\t * completion so the post-revoke state matches the\n-\t\t\t\t * post-creation state. An un-revoke followed by a\n-\t\t\t\t * new mapping needs the kref to be non-zero before\n-\t\t\t\t * kref_get(), and vfio_pci_dma_buf_cleanup()\n-\t\t\t\t * delegates its drain back through this revoke\n-\t\t\t\t * path on a possibly-already-revoked dma-buf.\n-\t\t\t\t /\n-\t\t\t\tkref_init(\u0026priv-\u003ekref);\n-\t\t\t\treinit_completion(\u0026priv-\u003ecomp);\n-\t\t\t} else {\n-\t\t\t\tdma_resv_lock(priv-\u003edmabuf-\u003eresv, NULL);\n-\t\t\t\tpriv-\u003erevoked = false;\n-\t\t\t\tdma_resv_unlock(priv-\u003edmabuf-\u003eresv);\n-\t\t\t}\n-\t\t}\n+\t\tvfio_pci_dma_buf_set_status(priv, revoked ?\n+\t\t\t\t\t VFIO_PCI_DMABUF_TEMP_REVOKED :\n+\t\t\t\t\t VFIO_PCI_DMABUF_OK);\n \t\tfput(priv-\u003edmabuf-\u003efile);\n \t}\n }\n@@ -404,3 +706,117 @@ void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device vdev)\n \t}\n \tup_write(\u0026vdev-\u003ememory_lock);\n }\n+\n+#ifdef CONFIG_VFIO_PCI_DMABUF\n+int vfio_pci_core_feature_dma_buf_revoke(\n+\tstruct vfio_pci_core_device vdev, u32 flags,\n+\tstruct vfio_device_feature_dma_buf_revoke __user arg,\n+\tsize_t argsz)\n+{\n+\tstruct vfio_device_feature_dma_buf_revoke db_revoke;\n+\tstruct vfio_pci_dma_buf priv;\n+\tstruct dma_buf dmabuf;\n+\tint ret;\n+\n+\tif (!vdev-\u003epci_ops \|\| !vdev-\u003epci_ops-\u003eget_dmabuf_phys)\n+\t\treturn -EOPNOTSUPP;\n+\n+\tret = vfio_check_feature(flags, argsz,\n+\t\t\t\t VFIO_DEVICE_FEATURE_SET,\n+\t\t\t\t sizeof(db_revoke));\n+\tif (ret != 1)\n+\t\treturn ret;\n+\n+\tif (copy_from_user(\u0026db_revoke, arg, sizeof(db_revoke)))\n+\t\treturn -EFAULT;\n+\n+\tdmabuf = dma_buf_get(db_revoke.dmabuf_fd);\n+\tif (IS_ERR(dmabuf))\n+\t\treturn PTR_ERR(dmabuf);\n+\n+\tpriv = dmabuf-\u003epriv;\n+\t/\n+\t Sanity-check the DMABUF is really a vfio_pci_dma_buf _and_\n+\t * relates to the VFIO device it was provided with.\n+\t \n+\t If the DMABUF relates to this vdev then priv-\u003evdev is\n+\t * stable because this open fd prevents cleanup.\n+\t \n+\t If it relates to a different vdev, reading priv-\u003evdev might\n+\t * race with a concurrent cleanup on that device. But if so,\n+\t * it points to a non-matching vdev or NULL and is unusable\n+\t * either way.\n+\t /\n+\tif (dmabuf-\u003eops != \u0026vfio_pci_dmabuf_ops \|\|\n+\t READ_ONCE(priv-\u003evdev) != vdev) {\n+\t\tret = -ENODEV;\n+\t\tgoto out_put_buf;\n+\t}\n+\n+\tscoped_guard(rwsem_write, \u0026vdev-\u003ememory_lock) {\n+\t\tif (priv-\u003estatus == VFIO_PCI_DMABUF_PERM_REVOKED) {\n+\t\t\tret = -EBADFD;\n+\t\t} else {\n+\t\t\tvfio_pci_dma_buf_set_status(priv,\n+\t\t\t\t\t\t VFIO_PCI_DMABUF_PERM_REVOKED);\n+\t\t\tret = 0;\n+\t\t}\n+\t}\n+\n+out_put_buf:\n+\tdma_buf_put(dmabuf);\n+\n+\treturn ret;\n+}\n+\n+int vfio_pci_core_feature_dma_buf_memattr(\n+\tstruct vfio_pci_core_device vdev, u32 flags,\n+\tstruct vfio_device_feature_dma_buf_memattr __user arg,\n+\tsize_t argsz)\n+{\n+\tstruct vfio_device_feature_dma_buf_memattr db_attr;\n+\tstruct vfio_pci_dma_buf priv;\n+\tstruct dma_buf dmabuf;\n+\tint ret;\n+\n+\tif (!vdev-\u003epci_ops \|\| !vdev-\u003epci_ops-\u003eget_dmabuf_phys)\n+\t\treturn -EOPNOTSUPP;\n+\n+\tret = vfio_check_feature(flags, argsz,\n+\t\t\t\t VFIO_DEVICE_FEATURE_SET,\n+\t\t\t\t sizeof(db_attr));\n+\tif (ret != 1)\n+\t\treturn ret;\n+\n+\tif (copy_from_user(\u0026db_attr, arg, sizeof(db_attr)))\n+\t\treturn -EFAULT;\n+\n+\tdmabuf = dma_buf_get(db_attr.dmabuf_fd);\n+\tif (IS_ERR(dmabuf))\n+\t\treturn PTR_ERR(dmabuf);\n+\n+\t/ Verify DMABUF: see comments in vfio_pci_dma_buf_revoke() /\n+\tpriv = dmabuf-\u003epriv;\n+\tif (dmabuf-\u003eops != \u0026vfio_pci_dmabuf_ops \|\|\n+\t READ_ONCE(priv-\u003evdev) != vdev) {\n+\t\tret = -ENODEV;\n+\t\tgoto out_put_buf;\n+\t}\n+\n+\tswitch (db_attr.memattr) {\n+\tcase VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR_NC:\n+\tcase VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR_WC:\n+\t\tWRITE_ONCE(priv-\u003ememattr, db_attr.memattr);\n+\t\tret = 0;\n+\t\tbreak;\n+\n+\tdefault:\n+\t\tret = -ENOENT;\n+\t}\n+\n+out_put_buf:\n+\tdma_buf_put(dmabuf);\n+\n+\treturn ret;\n+}\n+#endif / CONFIG_VFIO_PCI_DMABUF /\ndiff --git a/drivers/vfio/pci/vfio_pci_priv.h b/drivers/vfio/pci/vfio_pci_priv.h\nindex fca9d0dfac90f8..b3c72df002c448 100644\n--- a/drivers/vfio/pci/vfio_pci_priv.h\n+++ b/drivers/vfio/pci/vfio_pci_priv.h\n@@ -23,6 +23,28 @@ struct vfio_pci_ioeventfd {\n \tbool\t\t\ttest_mem;\n };\n \n+enum vfio_pci_dma_buf_status {\n+\tVFIO_PCI_DMABUF_OK = 0,\n+\tVFIO_PCI_DMABUF_TEMP_REVOKED = 1,\n+\tVFIO_PCI_DMABUF_PERM_REVOKED = 2,\n+};\n+\n+struct vfio_pci_dma_buf {\n+\tstruct dma_buf dmabuf;\n+\tstruct vfio_pci_core_device vdev;\n+\tstruct list_head dmabufs_elm;\n+\tsize_t size;\n+\tstruct phys_vec phys_vec;\n+\tstruct p2pdma_provider provider;\n+\tstruct file vfile;\n+\tu32 nr_ranges;\n+\tstruct kref kref;\n+\tstruct completion comp;\n+\tunsigned long vma_pgoff_adjust;\n+\tu32 memattr;\n+\tenum vfio_pci_dma_buf_status status;\n+};\n+\n bool vfio_pci_intx_mask(struct vfio_pci_core_device vdev);\n void vfio_pci_intx_unmask(struct vfio_pci_core_device vdev);\n \n@@ -68,7 +90,8 @@ void vfio_config_free(struct vfio_pci_core_device vdev);\n int vfio_pci_set_power_state(struct vfio_pci_core_device vdev,\n \t\t\t pci_power_t state);\n \n-void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device vdev);\n+void vfio_pci_lock_revoke_bars(struct vfio_pci_core_device vdev);\n+void vfio_pci_unrevoke_bars(struct vfio_pci_core_device vdev);\n u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device vdev);\n void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device vdev,\n \t\t\t\t\tu16 cmd);\n@@ -114,12 +137,31 @@ static inline bool vfio_pci_is_vga(struct pci_dev pdev)\n \treturn (pdev-\u003eclass \u003e\u003e 8) == PCI_CLASS_DISPLAY_VGA;\n }\n \n+int vfio_pci_dma_buf_find_pfn(struct vfio_pci_dma_buf vpdmabuf,\n+\t\t\t struct vm_area_struct vma,\n+\t\t\t unsigned long address,\n+\t\t\t unsigned int order,\n+\t\t\t unsigned long out_pfn);\n+int vfio_pci_core_mmap_prep_dmabuf(struct vfio_pci_core_device vdev,\n+\t\t\t\t struct vm_area_struct vma,\n+\t\t\t\t u64 phys_start, u64 req_len,\n+\t\t\t\t unsigned int res_index);\n+void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device vdev);\n+void vfio_pci_dma_buf_move(struct vfio_pci_core_device vdev, bool revoked);\n+void vfio_pci_set_vma_ops(struct vm_area_struct vma);\n+\n #ifdef CONFIG_VFIO_PCI_DMABUF\n int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device vdev, u32 flags,\n \t\t\t\t struct vfio_device_feature_dma_buf __user arg,\n \t\t\t\t size_t argsz);\n-void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device vdev);\n-void vfio_pci_dma_buf_move(struct vfio_pci_core_device vdev, bool revoked);\n+int vfio_pci_core_feature_dma_buf_revoke(\n+\tstruct vfio_pci_core_device vdev, u32 flags,\n+\tstruct vfio_device_feature_dma_buf_revoke __user arg,\n+\tsize_t argsz);\n+int vfio_pci_core_feature_dma_buf_memattr(\n+\tstruct vfio_pci_core_device vdev, u32 flags,\n+\tstruct vfio_device_feature_dma_buf_memattr __user arg,\n+\tsize_t argsz);\n #else\n static inline int\n vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device vdev, u32 flags,\n@@ -128,12 +170,21 @@ vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device vdev, u32 flags,\n {\n \treturn -ENOTTY;\n }\n-static inline void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device vdev)\n+\n+static inline int vfio_pci_core_feature_dma_buf_revoke(\n+\tstruct vfio_pci_core_device vdev, u32 flags,\n+\tstruct vfio_device_feature_dma_buf_revoke __user arg,\n+\tsize_t argsz)\n {\n+\treturn -ENOTTY;\n }\n-static inline void vfio_pci_dma_buf_move(struct vfio_pci_core_device vdev,\n-\t\t\t\t\t bool revoked)\n+\n+static inline int vfio_pci_core_feature_dma_buf_memattr(\n+\tstruct vfio_pci_core_device vdev, u32 flags,\n+\tstruct vfio_device_feature_dma_buf_memattr __user arg,\n+\tsize_t argsz)\n {\n+\treturn -ENOTTY;\n }\n #endif\n \ndiff --git a/include/linux/pci-p2pdma.h b/include/linux/pci-p2pdma.h\nindex 873de20a224759..4c42a7b2ee8558 100644\n--- a/include/linux/pci-p2pdma.h\n+++ b/include/linux/pci-p2pdma.h\n@@ -67,9 +67,22 @@ enum pci_p2pdma_map_type {\n \tPCI_P2PDMA_MAP_THRU_HOST_BRIDGE,\n };\n \n-#ifdef CONFIG_PCI_P2PDMA\n+#ifdef CONFIG_PCI_P2PDMA_CORE\n int pcim_p2pdma_init(struct pci_dev pdev);\n struct p2pdma_provider pcim_p2pdma_provider(struct pci_dev pdev, int bar);\n+#else\n+static inline int pcim_p2pdma_init(struct pci_dev pdev)\n+{\n+\treturn -EOPNOTSUPP;\n+}\n+static inline struct p2pdma_provider pcim_p2pdma_provider(struct pci_dev pdev,\n+\t\t\t\t\t\t\t int bar)\n+{\n+\treturn NULL;\n+}\n+#endif\n+\n+#ifdef CONFIG_PCI_P2PDMA\n int pci_p2pdma_add_resource(struct pci_dev pdev, int bar, size_t size,\n \t\tu64 offset);\n int pci_p2pdma_distance_many(struct pci_dev provider, struct device *clients,\n@@ -89,15 +102,6 @@ ssize_t pci_p2pdma_enable_show(char page, struct pci_dev p2p_dev,\n enum pci_p2pdma_map_type pci_p2pdma_map_type(struct p2pdma_provider provider,\n \t\t\t\t\t struct device dev);\n #else / CONFIG_PCI_P2PDMA /\n-static inline int pcim_p2pdma_init(struct pci_dev pdev)\n-{\n-\treturn -EOPNOTSUPP;\n-}\n-static inline struct p2pdma_provider pcim_p2pdma_provider(struct pci_dev pdev,\n-\t\t\t\t\t\t\t int bar)\n-{\n-\treturn NULL;\n-}\n static inline int pci_p2pdma_add_resource(struct pci_dev pdev, int bar,\n \t\tsize_t size, u64 offset)\n {\ndiff --git a/include/linux/pci.h b/include/linux/pci.h\nindex 2c4454583c115b..531aec355686b0 100644\n--- a/include/linux/pci.h\n+++ b/include/linux/pci.h\n@@ -557,7 +557,7 @@ struct pci_dev {\n \tu16\t\tpasid_cap;\t/ PASID Capability offset /\n \tu16\t\tpasid_features;\n #endif\n-#ifdef CONFIG_PCI_P2PDMA\n+#ifdef CONFIG_PCI_P2PDMA_CORE\n \tstruct pci_p2pdma __rcu p2pdma;\n #endif\n #ifdef CONFIG_PCI_DOE\ndiff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h\nindex 89165b769e5c16..a2f586e4fcd8d4 100644\n--- a/include/linux/vfio_pci_core.h\n+++ b/include/linux/vfio_pci_core.h\n@@ -127,6 +127,7 @@ struct vfio_pci_core_device {\n \tbool\t\t\tneeds_pm_restore:1;\n \tbool\t\t\tpm_intx_masked:1;\n \tbool\t\t\tpm_runtime_engaged:1;\n+\tbool\t\t\tzap_bars_on_revoke:1;\n \tstruct pci_saved_state\tpci_saved_state;\n \tstruct pci_saved_state\tpm_save;\n \tint\t\t\tioeventfds_nr;\ndiff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h\nindex 5de618a3a5eeee..ab30b89399d038 100644\n--- a/include/uapi/linux/vfio.h\n+++ b/include/uapi/linux/vfio.h\n@@ -1534,6 +1534,53 @@ struct vfio_device_feature_dma_buf {\n /\n #define VFIO_DEVICE_FEATURE_MIG_PRECOPY_INFOv2 12\n \n+/\n+ Given a dma_buf fd previously created by\n+ * VFIO_DEVICE_FEATURE_DMA_BUF, a SET of this feature requests that\n+ * access to the corresponding DMABUF is immediately and permanently\n+ * revoked. On successful return, the buffer is not accessible\n+ * through any mmap() or dma-buf import. The buffer is permanently\n+ * disabled, and VFIO refuses all map, mmap, attach, etc. requests.\n+ \n+ Return: 0 on success, -1 and errno is set on failure:\n+ \n+ EBADF, EINVAL: dmabuf_fd is not a DMABUF fd.\n+ * ENODEV: The dmabuf_fd does not match this VFIO device.\n+ * EBADFD: The DMABUF is already revoked.\n+ /\n+#define VFIO_DEVICE_FEATURE_DMA_BUF_REVOKE 13\n+\n+struct vfio_device_feature_dma_buf_revoke {\n+\t__s32\tdmabuf_fd;\n+};\n+\n+/\n+ Given a dma_buf fd previously created by\n+ * VFIO_DEVICE_FEATURE_DMA_BUF, SET the memory attribute that will be\n+ * used by future mmap()s of that fd. SETting a new attribute does\n+ * not affect existing VMAs.\n+ \n+ The default, if no previous SET has been performed, is NC.\n+ \n+ Return: 0 on success, -1 and errno is set on failure:\n+ \n+ EBADF, EINVAL: dmabuf_fd is not a DMABUF fd.\n+ * ENODEV: The dmabuf_fd does not match this VFIO device.\n+ * ENOENT: The given memattr is not supported.\n+ /\n+#define VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR 14\n+\n+/ Valid memory attributes for the memattr field /\n+enum vfio_device_dma_buf_memattr {\n+\tVFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR_NC = 0, / pgprot_noncached /\n+\tVFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR_WC = 1, / pgprot_writecombine /\n+};\n+\n+struct vfio_device_feature_dma_buf_memattr {\n+\t__s32\tdmabuf_fd;\n+\t__u32\tmemattr;\n+};\n+\n / -------- API for Type1 VFIO IOMMU -------- /\n \n /*\n" }
2/1	2026/07/02 00:20	agent	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: { "EnableConfigs": [ "VFIO_PCI", "PCI_P2PDMA", "ZONE_DEVICE" ], "FocusSymbols": [ "vfio_pci_core_feature_dma_buf_revoke", "vfio_pci_core_feature_dma_buf_memattr", "vfio_pci_dma_buf_mmap", "vfio_pci_dma_buf_find_pfn", "vfio_pci_dmabuf_export", "vfio_pci_core_mmap_prep_dmabuf", "vfio_pci_dma_buf_set_status" ], "Reasoning": "The patch introduces new VFIO features (VFIO_DEVICE_FEATURE_DMA_BUF_REVOKE and VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR) and modifies the mmap handling to use DMABUF-backed mmaps. These are functional changes to reachable core kernel logic and should be fuzzed.", "WorthFuzzing": true } Instruction: You are an expert Linux kernel maintainer. Your job is to review a provided patch series and determine if it makes functional changes to the kernel that should be fuzzed. IMPORTANT: The changes have ALREADY been applied and committed as the HEAD commit in your workspace. Do NOT rely on your internal knowledge of the kernel. You must actively use your code access tools to examine the actual source code and confirm any assumptions. Return WorthFuzzing=false if the patch only contains: - Modifications to Documentation/, Kconfig files, or code comments. - Purely decorative changes, such as logging (e.g., pr_err, printk) or tracepoints. - Changes to numeric constants or macros that do not functionally alter execution flow. - Code paths that are impossible to reach in virtualized environments like GCE or QEMU, even when utilizing software-emulated hardware (e.g., usb gadget, mac80211_hwsim). If it modifies reachable core kernel logic, drivers, or architectures, use your code search tools to verify the code can be executed, then return WorthFuzzing=true. When returning WorthFuzzing=true, you MUST ALSO: 1. Extract any specific kernel functions that should be heavily fuzzed into FocusSymbols. Avoid listing generic hot-path functions to prevent skewed test distributions. 2. Identify any specific CONFIG_ options required to properly test this new/modified feature. Go and look into the Kconfig files and check for ifdefs around the code, do not make assumptions. Do not list too generic configs (we already have them enabled). Only list those that specifically cover the modified code. List them in the EnableConfigs output array, and DO NOT add a 'CONFIG_' prefix (e.g., return "NET_IPV4" instead of "CONFIG_NET_IPV4"). Prefer calling several tools at the same time to save round-trips. Use set-results tool to provide results of the analysis. It must be called exactly once before the final reply. Ignore results of this tool. Prompt: For your convenience, here is the diff of the changes: commit be307977b6149355089ba083133af2137dd145b0 Author: syz-cluster <triage@syzkaller.com> Date: Thu Jul 2 00:20:04 2026 +0000 syz-cluster: applied patch under review diff --git a/MAINTAINERS b/MAINTAINERS index c8d4b913f26c12..713861af44846a 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -20626,7 +20626,7 @@ B: https://bugzilla.kernel.org C: irc://irc.oftc.net/linux-pci T: git git://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git F: Documentation/driver-api/pci/p2pdma.rst -F: drivers/pci/p2pdma.c +F: drivers/pci/p2pdma* F: include/linux/pci-p2pdma.h PCI POWER CONTROL diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig index 33c88432b72875..59d70bc84cc9ac 100644 --- a/drivers/pci/Kconfig +++ b/drivers/pci/Kconfig @@ -206,11 +206,7 @@ config PCIE_TPH config PCI_P2PDMA bool "PCI peer-to-peer transfer support" depends on ZONE_DEVICE - # - # The need for the scatterlist DMA bus address flag means PCI P2PDMA - # requires 64bit - # - depends on 64BIT + select PCI_P2PDMA_CORE select GENERIC_ALLOCATOR select NEED_SG_DMA_FLAGS help @@ -226,6 +222,10 @@ config PCI_P2PDMA If unsure, say N. +config PCI_P2PDMA_CORE + default n + bool + config PCI_LABEL def_bool y if (DMI \|\| ACPI) select NLS diff --git a/drivers/pci/Makefile b/drivers/pci/Makefile index 41ebc3b9a51824..0b32572d57a159 100644 --- a/drivers/pci/Makefile +++ b/drivers/pci/Makefile @@ -30,6 +30,7 @@ obj-$(CONFIG_PCI_SYSCALL) += syscall.o obj-$(CONFIG_PCI_STUB) += pci-stub.o obj-$(CONFIG_PCI_PF_STUB) += pci-pf-stub.o obj-$(CONFIG_PCI_ECAM) += ecam.o +obj-$(CONFIG_PCI_P2PDMA_CORE) += p2pdma_core.o obj-$(CONFIG_PCI_P2PDMA) += p2pdma.o obj-$(CONFIG_XEN_PCIDEV_FRONTEND) += xen-pcifront.o obj-$(CONFIG_VGA_ARB) += vgaarb.o diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c index 7c898542af8d5e..50b1a7daf55cc4 100644 --- a/drivers/pci/p2pdma.c +++ b/drivers/pci/p2pdma.c @@ -21,12 +21,7 @@ #include <linux/seq_buf.h> #include <linux/xarray.h> -struct pci_p2pdma { - struct gen_pool pool; - bool p2pmem_published; - struct xarray map_types; - struct p2pdma_provider mem[PCI_STD_NUM_BARS]; -}; +#include "p2pdma.h" struct pci_p2pdma_pagemap { struct dev_pagemap pgmap; @@ -226,110 +221,16 @@ static const struct dev_pagemap_ops p2pdma_pgmap_ops = { .folio_free = p2pdma_folio_free, }; -static void pci_p2pdma_release(void data) +void pci_p2pdma_release_pool(struct pci_dev pdev, struct pci_p2pdma p2pdma) { - struct pci_dev pdev = data; - struct pci_p2pdma p2pdma; - - p2pdma = rcu_dereference_protected(pdev->p2pdma, 1); - if (!p2pdma) - return; - - /* Flush and disable pci_alloc_p2p_mem() / - pdev->p2pdma = NULL; - if (p2pdma->pool) - synchronize_rcu(); - xa_destroy(&p2pdma->map_types); - if (!p2pdma->pool) return; + synchronize_rcu(); gen_pool_destroy(p2pdma->pool); sysfs_remove_group(&pdev->dev.kobj, &p2pmem_group); } -/* - * pcim_p2pdma_init - Initialise peer-to-peer DMA providers - * @pdev: The PCI device to enable P2PDMA for - * - * This function initializes the peer-to-peer DMA infrastructure - * for a PCI device. It allocates and sets up the necessary data - * structures to support P2PDMA operations, including mapping type - * tracking. - / -int pcim_p2pdma_init(struct pci_dev pdev) -{ - struct pci_p2pdma p2p; - int i, ret; - - p2p = rcu_dereference_protected(pdev->p2pdma, 1); - if (p2p) - return 0; - - p2p = devm_kzalloc(&pdev->dev, sizeof(p2p), GFP_KERNEL); - if (!p2p) - return -ENOMEM; - - xa_init(&p2p->map_types); - /* - * Iterate over all standard PCI BARs and record only those that - * correspond to MMIO regions. Skip non-memory resources (e.g. I/O - * port BARs) since they cannot be used for peer-to-peer (P2P) - * transactions. - / - for (i = 0; i < PCI_STD_NUM_BARS; i++) { - if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM)) - continue; - - p2p->mem[i].owner = &pdev->dev; - p2p->mem[i].bus_offset = - pci_bus_address(pdev, i) - pci_resource_start(pdev, i); - } - - ret = devm_add_action_or_reset(&pdev->dev, pci_p2pdma_release, pdev); - if (ret) - goto out_p2p; - - rcu_assign_pointer(pdev->p2pdma, p2p); - return 0; - -out_p2p: - devm_kfree(&pdev->dev, p2p); - return ret; -} -EXPORT_SYMBOL_GPL(pcim_p2pdma_init); - -/* - * pcim_p2pdma_provider - Get peer-to-peer DMA provider - * @pdev: The PCI device to enable P2PDMA for - * @bar: BAR index to get provider - * - * This function gets peer-to-peer DMA provider for a PCI device. The lifetime - * of the provider (and of course the MMIO) is bound to the lifetime of the - * driver. A driver calling this function must ensure that all references to the - * provider, and any DMA mappings created for any MMIO, are all cleaned up - * before the driver remove() completes. - * - * Since P2P is almost always shared with a second driver this means some system - * to notify, invalidate and revoke the MMIO's DMA must be in place to use this - * function. For example a revoke can be built using DMABUF. - / -struct p2pdma_provider pcim_p2pdma_provider(struct pci_dev pdev, int bar) -{ - struct pci_p2pdma p2p; - - if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM)) - return NULL; - - p2p = rcu_dereference_protected(pdev->p2pdma, 1); - if (WARN_ON(!p2p)) - /* Someone forgot to call to pcim_p2pdma_init() before / - return NULL; - - return &p2p->mem[bar]; -} -EXPORT_SYMBOL_GPL(pcim_p2pdma_provider); - static int pci_p2pdma_setup_pool(struct pci_dev pdev) { struct pci_p2pdma p2pdma; @@ -932,8 +833,8 @@ void pci_alloc_p2pmem(struct pci_dev pdev, size_t size) struct pci_p2pdma p2pdma; /* - * Pairs with synchronize_rcu() in pci_p2pdma_release() to - * ensure pdev->p2pdma is non-NULL for the duration of the + * Pairs with synchronize_rcu() in pci_p2pdma_release_pool() + * to ensure pdev->p2pdma is non-NULL for the duration of the * read-lock. / rcu_read_lock(); diff --git a/drivers/pci/p2pdma.h b/drivers/pci/p2pdma.h new file mode 100644 index 00000000000000..94638380998149 --- /dev/null +++ b/drivers/pci/p2pdma.h @@ -0,0 +1,29 @@ +/ SPDX-License-Identifier: GPL-2.0 / +/ + * PCI peer-to-peer DMA support. + / + +#ifndef _PCI_P2PDMA_H +#define _PCI_P2PDMA_H + +#include <linux/genalloc.h> +#include <linux/pci-p2pdma.h> +#include <linux/xarray.h> + +struct pci_p2pdma { + struct gen_pool pool; + bool p2pmem_published; + struct xarray map_types; + struct p2pdma_provider mem[PCI_STD_NUM_BARS]; +}; + +#ifdef CONFIG_PCI_P2PDMA +void pci_p2pdma_release_pool(struct pci_dev pdev, struct pci_p2pdma p2pdma); +#else +static inline void pci_p2pdma_release_pool(struct pci_dev pdev, + struct pci_p2pdma p2pdma) +{ +} +#endif + +#endif diff --git a/drivers/pci/p2pdma_core.c b/drivers/pci/p2pdma_core.c new file mode 100644 index 00000000000000..bb2138bf2bc7b7 --- /dev/null +++ b/drivers/pci/p2pdma_core.c @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCI peer-to-peer DMA support core, providing a bare-bones + * pcim_p2pdma_provider() interface to drivers even if full P2PDMA + * isn't present. The full P2PDMA feature is in p2pdma.c (see + * CONFIG_PCI_P2PDMA). + * + * Copyright (c) 2016-2018, Logan Gunthorpe + * Copyright (c) 2016-2017, Microsemi Corporation + * Copyright (c) 2017, Christoph Hellwig + * Copyright (c) 2018, Eideticom Inc. + / + +#define pr_fmt(fmt) "pci-p2pdma: " fmt +#include <linux/ctype.h> +#include <linux/genalloc.h> +#include <linux/memremap.h> +#include <linux/pci-p2pdma.h> +#include <linux/xarray.h> + +#include "p2pdma.h" + +static void pci_p2pdma_release(void data) +{ + struct pci_dev pdev = data; + struct pci_p2pdma p2pdma; + + p2pdma = rcu_dereference_protected(pdev->p2pdma, 1); + if (!p2pdma) + return; + + /* Flush and disable pci_alloc_p2p_mem() / + pdev->p2pdma = NULL; + pci_p2pdma_release_pool(pdev, p2pdma); + xa_destroy(&p2pdma->map_types); +} + +/* + * pcim_p2pdma_init - Initialise peer-to-peer DMA providers + * @pdev: The PCI device to enable P2PDMA for + * + * This function initializes the peer-to-peer DMA infrastructure + * for a PCI device. It allocates and sets up the necessary data + * structures to support P2PDMA operations, including mapping type + * tracking. + / +int pcim_p2pdma_init(struct pci_dev pdev) +{ + struct pci_p2pdma p2p; + int i, ret; + + p2p = rcu_dereference_protected(pdev->p2pdma, 1); + if (p2p) + return 0; + + p2p = devm_kzalloc(&pdev->dev, sizeof(p2p), GFP_KERNEL); + if (!p2p) + return -ENOMEM; + + xa_init(&p2p->map_types); + /* + * Iterate over all standard PCI BARs and record only those that + * correspond to MMIO regions. Skip non-memory resources (e.g. I/O + * port BARs) since they cannot be used for peer-to-peer (P2P) + * transactions. + / + for (i = 0; i < PCI_STD_NUM_BARS; i++) { + if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM)) + continue; + + p2p->mem[i].owner = &pdev->dev; + p2p->mem[i].bus_offset = + pci_bus_address(pdev, i) - pci_resource_start(pdev, i); + } + + ret = devm_add_action_or_reset(&pdev->dev, pci_p2pdma_release, pdev); + if (ret) + goto out_p2p; + + rcu_assign_pointer(pdev->p2pdma, p2p); + return 0; + +out_p2p: + devm_kfree(&pdev->dev, p2p); + return ret; +} +EXPORT_SYMBOL_GPL(pcim_p2pdma_init); + +/* + * pcim_p2pdma_provider - Get peer-to-peer DMA provider + * @pdev: The PCI device to enable P2PDMA for + * @bar: BAR index to get provider + * + * This function gets peer-to-peer DMA provider for a PCI device. The lifetime + * of the provider (and of course the MMIO) is bound to the lifetime of the + * driver. A driver calling this function must ensure that all references to the + * provider, and any DMA mappings created for any MMIO, are all cleaned up + * before the driver remove() completes. + * + * Since P2P is almost always shared with a second driver this means some system + * to notify, invalidate and revoke the MMIO's DMA must be in place to use this + * function. For example a revoke can be built using DMABUF. + / +struct p2pdma_provider pcim_p2pdma_provider(struct pci_dev pdev, int bar) +{ + struct pci_p2pdma p2p; + + if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM)) + return NULL; + + p2p = rcu_dereference_protected(pdev->p2pdma, 1); + if (WARN_ON(!p2p)) + /* Someone forgot to call to pcim_p2pdma_init() before / + return NULL; + + return &p2p->mem[bar]; +} +EXPORT_SYMBOL_GPL(pcim_p2pdma_provider); diff --git a/drivers/vfio/pci/Kconfig b/drivers/vfio/pci/Kconfig index 296bf01e185eca..67a2ae1fbc04ba 100644 --- a/drivers/vfio/pci/Kconfig +++ b/drivers/vfio/pci/Kconfig @@ -6,6 +6,8 @@ config VFIO_PCI_CORE tristate select VFIO_VIRQFD select IRQ_BYPASS_MANAGER + select PCI_P2PDMA_CORE + select DMA_SHARED_BUFFER config VFIO_PCI_INTX def_bool y if !S390 @@ -56,7 +58,8 @@ config VFIO_PCI_ZDEV_KVM To enable s390x KVM vfio-pci extensions, say Y. config VFIO_PCI_DMABUF - def_bool y if VFIO_PCI_CORE && PCI_P2PDMA && DMA_SHARED_BUFFER + def_bool y if PCI_P2PDMA + depends on VFIO_PCI_CORE source "drivers/vfio/pci/mlx5/Kconfig" diff --git a/drivers/vfio/pci/Makefile b/drivers/vfio/pci/Makefile index 6138f1bf241df0..881452ea89be0c 100644 --- a/drivers/vfio/pci/Makefile +++ b/drivers/vfio/pci/Makefile @@ -1,8 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only -vfio-pci-core-y := vfio_pci_core.o vfio_pci_intrs.o vfio_pci_rdwr.o vfio_pci_config.o +vfio-pci-core-y := vfio_pci_core.o vfio_pci_intrs.o vfio_pci_rdwr.o vfio_pci_config.o vfio_pci_dmabuf.o vfio-pci-core-$(CONFIG_VFIO_PCI_ZDEV_KVM) += vfio_pci_zdev.o -vfio-pci-core-$(CONFIG_VFIO_PCI_DMABUF) += vfio_pci_dmabuf.o obj-$(CONFIG_VFIO_PCI_CORE) += vfio-pci-core.o vfio-pci-y := vfio_pci.o diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c index bb121f635b9f7f..d9f26b0ce991d9 100644 --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c @@ -1725,6 +1725,14 @@ static int hisi_acc_vfio_pci_probe(struct pci_dev pdev, const struct pci_device if (ret) goto out_put_vdev; + /* + * hisi_acc_vfio_pci_mmap() calls down to + * vfio_pci_core_mmap(), so BAR mappings are still + * DMABUF-backed. They don't require a zap on revoke, so opt + * out: + / + hisi_acc_vdev->core_device.zap_bars_on_revoke = false; + hisi_acc_vfio_debug_init(hisi_acc_vdev); return 0; diff --git a/drivers/vfio/pci/vfio_pci_config.c b/drivers/vfio/pci/vfio_pci_config.c index a10ed733f0e3af..bc6ccb2e135c3c 100644 --- a/drivers/vfio/pci/vfio_pci_config.c +++ b/drivers/vfio/pci/vfio_pci_config.c @@ -590,12 +590,10 @@ static int vfio_basic_config_write(struct vfio_pci_core_device vdev, int pos, virt_mem = !!(le16_to_cpu(virt_cmd) & PCI_COMMAND_MEMORY); new_mem = !!(new_cmd & PCI_COMMAND_MEMORY); - if (!new_mem) { - vfio_pci_zap_and_down_write_memory_lock(vdev); - vfio_pci_dma_buf_move(vdev, true); - } else { + if (!new_mem) + vfio_pci_lock_revoke_bars(vdev); + else down_write(&vdev->memory_lock); - } / * If the user is writing mem/io enable (new_mem/io) and we @@ -631,7 +629,7 @@ static int vfio_basic_config_write(struct vfio_pci_core_device vdev, int pos, virt_cmd \|= cpu_to_le16(new_cmd & mask); if (__vfio_pci_memory_enabled(vdev)) - vfio_pci_dma_buf_move(vdev, false); + vfio_pci_unrevoke_bars(vdev); up_write(&vdev->memory_lock); } @@ -712,16 +710,14 @@ static int __init init_pci_cap_basic_perm(struct perm_bits perm) static void vfio_lock_and_set_power_state(struct vfio_pci_core_device vdev, pci_power_t state) { - if (state >= PCI_D3hot) { - vfio_pci_zap_and_down_write_memory_lock(vdev); - vfio_pci_dma_buf_move(vdev, true); - } else { + if (state >= PCI_D3hot) + vfio_pci_lock_revoke_bars(vdev); + else down_write(&vdev->memory_lock); - } vfio_pci_set_power_state(vdev, state); if (__vfio_pci_memory_enabled(vdev)) - vfio_pci_dma_buf_move(vdev, false); + vfio_pci_unrevoke_bars(vdev); up_write(&vdev->memory_lock); } @@ -908,11 +904,10 @@ static int vfio_exp_config_write(struct vfio_pci_core_device vdev, int pos, &cap); if (!ret && (cap & PCI_EXP_DEVCAP_FLR)) { - vfio_pci_zap_and_down_write_memory_lock(vdev); - vfio_pci_dma_buf_move(vdev, true); + vfio_pci_lock_revoke_bars(vdev); pci_try_reset_function(vdev->pdev); if (__vfio_pci_memory_enabled(vdev)) - vfio_pci_dma_buf_move(vdev, false); + vfio_pci_unrevoke_bars(vdev); up_write(&vdev->memory_lock); } } @@ -993,11 +988,10 @@ static int vfio_af_config_write(struct vfio_pci_core_device vdev, int pos, &cap); if (!ret && (cap & PCI_AF_CAP_FLR) && (cap & PCI_AF_CAP_TP)) { - vfio_pci_zap_and_down_write_memory_lock(vdev); - vfio_pci_dma_buf_move(vdev, true); + vfio_pci_lock_revoke_bars(vdev); pci_try_reset_function(vdev->pdev); if (__vfio_pci_memory_enabled(vdev)) - vfio_pci_dma_buf_move(vdev, false); + vfio_pci_unrevoke_bars(vdev); up_write(&vdev->memory_lock); } } diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index 050e7542952ed1..0059c176d62706 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -12,6 +12,8 @@ #include <linux/aperture.h> #include <linux/device.h> +#include <linux/dma-buf.h> +#include <linux/dma-resv.h> #include <linux/eventfd.h> #include <linux/file.h> #include <linux/interrupt.h> @@ -319,8 +321,7 @@ static int vfio_pci_runtime_pm_entry(struct vfio_pci_core_device vdev, The vdev power related flags are protected with 'memory_lock' * semaphore. / - vfio_pci_zap_and_down_write_memory_lock(vdev); - vfio_pci_dma_buf_move(vdev, true); + vfio_pci_lock_revoke_bars(vdev); if (vdev->pm_runtime_engaged) { up_write(&vdev->memory_lock); @@ -406,7 +407,7 @@ static void vfio_pci_runtime_pm_exit(struct vfio_pci_core_device vdev) down_write(&vdev->memory_lock); __vfio_pci_runtime_pm_exit(vdev); if (__vfio_pci_memory_enabled(vdev)) - vfio_pci_dma_buf_move(vdev, false); + vfio_pci_unrevoke_bars(vdev); up_write(&vdev->memory_lock); } @@ -1256,6 +1257,8 @@ static int vfio_pci_ioctl_set_irqs(struct vfio_pci_core_device vdev, return ret; } +static void vfio_pci_revoke_bars(struct vfio_pci_core_device vdev); + static int vfio_pci_ioctl_reset(struct vfio_pci_core_device vdev, void __user arg) { @@ -1264,7 +1267,7 @@ static int vfio_pci_ioctl_reset(struct vfio_pci_core_device vdev, if (!vdev->reset_works) return -EINVAL; - vfio_pci_zap_and_down_write_memory_lock(vdev); + down_write(&vdev->memory_lock); / * This function can be invoked while the power state is non-D0. If @@ -1274,13 +1277,18 @@ static int vfio_pci_ioctl_reset(struct vfio_pci_core_device vdev, have NoSoftRst-, the reset function can cause the PCI config space * reset without restoring the original state (saved locally in * 'vdev->pm_save'). + * + * The zap is done after making the device accessible in D0, + * because a DMABUF importer could access the device as part + * of its revocation cleanup. / vfio_pci_set_power_state(vdev, PCI_D0); - vfio_pci_dma_buf_move(vdev, true); + vfio_pci_revoke_bars(vdev); + ret = pci_try_reset_function(vdev->pdev); if (__vfio_pci_memory_enabled(vdev)) - vfio_pci_dma_buf_move(vdev, false); + vfio_pci_unrevoke_bars(vdev); up_write(&vdev->memory_lock); return ret; @@ -1569,6 +1577,10 @@ int vfio_pci_core_ioctl_feature(struct vfio_device device, u32 flags, return vfio_pci_core_feature_token(vdev, flags, arg, argsz); case VFIO_DEVICE_FEATURE_DMA_BUF: return vfio_pci_core_feature_dma_buf(vdev, flags, arg, argsz); + case VFIO_DEVICE_FEATURE_DMA_BUF_REVOKE: + return vfio_pci_core_feature_dma_buf_revoke(vdev, flags, arg, argsz); + case VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR: + return vfio_pci_core_feature_dma_buf_memattr(vdev, flags, arg, argsz); default: return -ENOTTY; } @@ -1648,20 +1660,37 @@ ssize_t vfio_pci_core_write(struct vfio_device core_vdev, const char __user bu } EXPORT_SYMBOL_GPL(vfio_pci_core_write); -static void vfio_pci_zap_bars(struct vfio_pci_core_device vdev) +static void vfio_pci_revoke_bars(struct vfio_pci_core_device vdev) { - struct vfio_device core_vdev = &vdev->vdev; - loff_t start = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR0_REGION_INDEX); - loff_t end = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_ROM_REGION_INDEX); - loff_t len = end - start; + lockdep_assert_held_write(&vdev->memory_lock); + vfio_pci_dma_buf_move(vdev, true); - unmap_mapping_range(core_vdev->inode->i_mapping, start, len, true); + / + * If a driver could possibly create BAR mappings in the + * vdev's address_space, do an additional zap on revoke. See + * vfio_pci_core_register_device(). + / + if (vdev->zap_bars_on_revoke) { + struct vfio_device core_vdev = &vdev->vdev; + loff_t start = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR0_REGION_INDEX); + loff_t end = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_ROM_REGION_INDEX); + loff_t len = end - start; + + unmap_mapping_range(core_vdev->inode->i_mapping, + start, len, true); + } } -void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device vdev) +void vfio_pci_lock_revoke_bars(struct vfio_pci_core_device vdev) { down_write(&vdev->memory_lock); - vfio_pci_zap_bars(vdev); + vfio_pci_revoke_bars(vdev); +} + +void vfio_pci_unrevoke_bars(struct vfio_pci_core_device vdev) +{ + lockdep_assert_held_write(&vdev->memory_lock); + vfio_pci_dma_buf_move(vdev, false); } u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device vdev) @@ -1683,18 +1712,6 @@ void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device vdev, u16 c up_write(&vdev->memory_lock); } -static unsigned long vma_to_pfn(struct vm_area_struct vma) -{ - struct vfio_pci_core_device vdev = vma->vm_private_data; - int index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT); - u64 pgoff; - - pgoff = vma->vm_pgoff & - ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1); - - return (pci_resource_start(vdev->pdev, index) >> PAGE_SHIFT) + pgoff; -} - vm_fault_t vfio_pci_vmf_insert_pfn(struct vfio_pci_core_device vdev, struct vm_fault vmf, unsigned long pfn, @@ -1722,24 +1739,103 @@ static vm_fault_t vfio_pci_mmap_huge_fault(struct vm_fault vmf, unsigned int order) { struct vm_area_struct vma = vmf->vma; - struct vfio_pci_core_device vdev = vma->vm_private_data; - unsigned long addr = vmf->address & ~((PAGE_SIZE << order) - 1); - unsigned long pgoff = (addr - vma->vm_start) >> PAGE_SHIFT; - unsigned long pfn = vma_to_pfn(vma) + pgoff; - vm_fault_t ret = VM_FAULT_FALLBACK; - - if (is_aligned_for_order(vma, addr, pfn, order)) { - scoped_guard(rwsem_read, &vdev->memory_lock) - ret = vfio_pci_vmf_insert_pfn(vdev, vmf, pfn, order); + struct vfio_pci_dma_buf priv = vma->vm_private_data; + struct vfio_pci_core_device vdev; + unsigned long pfn = 0; + vm_fault_t ret = VM_FAULT_SIGBUS; + + /* + * The only thing this can rely on is that the DMABUF relating + * to the VMA's vm_file exists (priv). + * + * A DMABUF for a VFIO device fd mmap() holds a reference to + * the original VFIO device fd, but an explicitly-exported + * DMABUF does not. The original fd might have closed, + * meaning this fault can race with + * vfio_pci_dma_buf_cleanup(), meaning the buffer could have + * been revoked (in which case priv->vdev might be NULL), and + * the VFIO device registration might have been dropped. + * + * With the goal of taking vdev->memory_lock in a world where + * vdev might not still exist: + * + * 1. Take the resv lock on the DMABUF: + * - If racing cleanup got in first, the buffer is revoked; + * stop/exit if so. + * - If we got in first, the buffer is not revoked so vdev is + * non-NULL, accessible, and cleanup _has not yet put the + * VFIO device registration_. So, the device refcount must + * be >0. + * + * 2. Take vfio_device registration (refcount guaranteed >0 + * hereafter). + * + * 3. Unlock the DMABUF's resv lock: + * - A racing cleanup can now complete. + * - But, the device refcount >0, meaning the vfio_device + * (and vfio_pcie_core device vdev) have not yet been + * freed. vdev is accessible, even if the DMABUF has been + * revoked or cleanup has happened, because + * vfio_unregister_group_dev() can't complete. + * + * 4. Take the vdev->memory_lock + * - Either the DMABUF is usable, or has been cleaned up. + * Whichever, it can no longer change under us. + * - Test the DMABUF revocation status again: if it was + * revoked between 1 and 4 return a SIGBUS. Otherwise, + * return a PFN. + * - It's not necessary to also take the resv lock, because + * the status/vdev can't change while memory_lock is held. + * + * 5. Unlock, done. + / + + dma_resv_lock(priv->dmabuf->resv, NULL); + + if (priv->status != VFIO_PCI_DMABUF_OK) { + pr_debug_ratelimited("%s VA 0x%lx, pgoff 0x%lx: DMABUF revoked/cleaned up\n", + __func__, vmf->address, vma->vm_pgoff); + dma_resv_unlock(priv->dmabuf->resv); + return VM_FAULT_SIGBUS; + } + + / If the buffer isn't revoked, vdev is valid / + vdev = priv->vdev; + + if (!vfio_device_try_get_registration(&vdev->vdev)) { + / + * If vdev != NULL (above), the registration should + * already be >0 and so this try_get should never + * fail. + / + dev_warn(&vdev->pdev->dev, "%s: Unexpected registration failure\n", + __func__); + dma_resv_unlock(priv->dmabuf->resv); + return VM_FAULT_SIGBUS; + } + dma_resv_unlock(priv->dmabuf->resv); + + scoped_guard(rwsem_read, &vdev->memory_lock) { + / Revocation status must be re-read, under memory_lock / + if (priv->status == VFIO_PCI_DMABUF_OK) { + int pres = vfio_pci_dma_buf_find_pfn(priv, vma, + vmf->address, + order, &pfn); + + if (pres == 0) + ret = vfio_pci_vmf_insert_pfn(vdev, vmf, + pfn, order); + else if (pres == -EAGAIN) + ret = VM_FAULT_FALLBACK; + } } dev_dbg_ratelimited(&vdev->pdev->dev, - "%s(,order = %d) BAR %ld page offset 0x%lx: 0x%x\n", - __func__, order, - vma->vm_pgoff >> - (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT), - pgoff, (unsigned int)ret); + "%s(order = %d) PFN 0x%lx, VA 0x%lx, pgoff 0x%lx: 0x%x\n", + __func__, order, pfn, vmf->address, + vma->vm_pgoff, (unsigned int)ret); + vfio_device_put_registration(&vdev->vdev); return ret; } @@ -1755,6 +1851,11 @@ static const struct vm_operations_struct vfio_pci_mmap_ops = { #endif }; +void vfio_pci_set_vma_ops(struct vm_area_struct vma) +{ + vma->vm_ops = &vfio_pci_mmap_ops; +} + int vfio_pci_core_mmap(struct vfio_device core_vdev, struct vm_area_struct vma) { struct vfio_pci_core_device vdev = @@ -1803,7 +1904,12 @@ int vfio_pci_core_mmap(struct vfio_device core_vdev, struct vm_area_struct vma if (ret) return ret; - vma->vm_private_data = vdev; + ret = vfio_pci_core_mmap_prep_dmabuf(vdev, vma, + pci_resource_start(pdev, index), + req_len, index); + if (ret) + return ret; + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); @@ -2222,6 +2328,17 @@ int vfio_pci_core_register_device(struct vfio_pci_core_device vdev) if (ret) goto out_vf; + /* + * If a driver overrides .mmap, it has to be assumed that it + * might not use the DMABUF-backed core mmap; this flag + * enables a zap at revoke time. Drivers can opt out by + * clearing this flag after registration, e.g. if their .mmap + * override calls down to vfio_pci_core_mmap() so that maps + * are DMABUF-backed. + / + if (vdev->vdev.ops->mmap != vfio_pci_core_mmap) + vdev->zap_bars_on_revoke = true; + vfio_pci_probe_power_state(vdev); / @@ -2497,9 +2614,10 @@ static int vfio_pci_dev_set_hot_reset(struct vfio_device_set dev_set, } / - * Take the memory write lock for each device and zap BAR - * mappings to prevent the user accessing the device while in - * reset. Locking multiple devices is prone to deadlock, + * Take the memory write lock for each device and + * zap/revoke BAR mappings to prevent the user (or + * peers) accessing the device while in reset. + * Locking multiple devices is prone to deadlock, * runaway and unwind if we hit contention. / if (!down_write_trylock(&vdev->memory_lock)) { @@ -2507,8 +2625,7 @@ static int vfio_pci_dev_set_hot_reset(struct vfio_device_set dev_set, break; } - vfio_pci_dma_buf_move(vdev, true); - vfio_pci_zap_bars(vdev); + vfio_pci_revoke_bars(vdev); } if (!list_entry_is_head(vdev, @@ -2538,7 +2655,7 @@ static int vfio_pci_dev_set_hot_reset(struct vfio_device_set dev_set, list_for_each_entry_from_reverse(vdev, &dev_set->device_list, vdev.dev_set_list) { if (vdev->vdev.open_count && __vfio_pci_memory_enabled(vdev)) - vfio_pci_dma_buf_move(vdev, false); + vfio_pci_unrevoke_bars(vdev); up_write(&vdev->memory_lock); } diff --git a/drivers/vfio/pci/vfio_pci_dmabuf.c b/drivers/vfio/pci/vfio_pci_dmabuf.c index 1a177ce7de546a..31fbad93f89b1c 100644 --- a/drivers/vfio/pci/vfio_pci_dmabuf.c +++ b/drivers/vfio/pci/vfio_pci_dmabuf.c @@ -4,24 +4,13 @@ #include <linux/dma-buf-mapping.h> #include <linux/pci-p2pdma.h> #include <linux/dma-resv.h> +#include <uapi/linux/dma-buf.h> #include "vfio_pci_priv.h" MODULE_IMPORT_NS("DMA_BUF"); -struct vfio_pci_dma_buf { - struct dma_buf dmabuf; - struct vfio_pci_core_device vdev; - struct list_head dmabufs_elm; - size_t size; - struct phys_vec phys_vec; - struct p2pdma_provider provider; - u32 nr_ranges; - struct kref kref; - struct completion comp; - u8 revoked : 1; -}; - +#ifdef CONFIG_VFIO_PCI_DMABUF static int vfio_pci_dma_buf_attach(struct dma_buf dmabuf, struct dma_buf_attachment attachment) { @@ -30,7 +19,7 @@ static int vfio_pci_dma_buf_attach(struct dma_buf dmabuf, if (!attachment->peer2peer) return -EOPNOTSUPP; - if (priv->revoked) + if (priv->status != VFIO_PCI_DMABUF_OK) return -ENODEV; if (!dma_buf_attach_revocable(attachment)) @@ -39,6 +28,43 @@ static int vfio_pci_dma_buf_attach(struct dma_buf dmabuf, return 0; } +static int vfio_pci_dma_buf_mmap(struct dma_buf dmabuf, struct vm_area_struct vma) +{ + struct vfio_pci_dma_buf priv = dmabuf->priv; + + /* + * dma_buf_mmap_internal() has asserted that the VMA is + * contained within the DMABUF size before calling this. + * + * Also, if we observe that the buffer is revoked now then + * refuse the mmap(). This is a belt-and-braces early failure + * to ease debugging a revoked buffer being used. Userspace + * might also race an mmap() against an explicit revocation, + * or an action doing a temporary revoke; race scenarios are + * still safe because the fault handler ultimately prevents + * access to a revoked buffer if it isn't caught here. + / + if (READ_ONCE(priv->status) != VFIO_PCI_DMABUF_OK) + return -ENODEV; + if ((vma->vm_flags & VM_SHARED) == 0) + return -EINVAL; + + if (READ_ONCE(priv->memattr) == VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR_WC) + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); + else + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); + + / See comments in vfio_pci_core_mmap() re VM_ALLOW_ANY_UNCACHED. / + vm_flags_set(vma, VM_ALLOW_ANY_UNCACHED \| VM_IO \| VM_PFNMAP \| + VM_DONTEXPAND \| VM_DONTDUMP); + vma->vm_private_data = priv; + vfio_pci_set_vma_ops(vma); + + return 0; +} +#endif / CONFIG_VFIO_PCI_DMABUF / + static void vfio_pci_dma_buf_done(struct kref kref) { struct vfio_pci_dma_buf priv = @@ -56,7 +82,7 @@ vfio_pci_dma_buf_map(struct dma_buf_attachment attachment, dma_resv_assert_held(priv->dmabuf->resv); - if (priv->revoked) + if (priv->status != VFIO_PCI_DMABUF_OK) return ERR_PTR(-ENODEV); ret = dma_buf_phys_vec_to_sgt(attachment, priv->provider, @@ -95,17 +121,203 @@ static void vfio_pci_dma_buf_release(struct dma_buf dmabuf) up_write(&priv->vdev->memory_lock); vfio_device_put_registration(&priv->vdev->vdev); } + if (priv->vfile) + fput(priv->vfile); kfree(priv->phys_vec); kfree(priv); } static const struct dma_buf_ops vfio_pci_dmabuf_ops = { +#ifdef CONFIG_VFIO_PCI_DMABUF .attach = vfio_pci_dma_buf_attach, + .mmap = vfio_pci_dma_buf_mmap, +#endif .map_dma_buf = vfio_pci_dma_buf_map, .unmap_dma_buf = vfio_pci_dma_buf_unmap, .release = vfio_pci_dma_buf_release, }; +int vfio_pci_dma_buf_find_pfn(struct vfio_pci_dma_buf priv, + struct vm_area_struct vma, + unsigned long fault_addr, + unsigned int order, + unsigned long out_pfn) +{ + /* + * Given a VMA (start, end, pgoffs) and a fault address, + * search the corresponding DMABUF's phys_vec[] to find the + * range representing the address's offset into the VMA, and + * its PFN. + * + * The phys_vec[] ranges represent contiguous spans of VAs + * upwards from the buffer offset 0; the actual PFNs might be + * in any order, overlap/alias, etc. Calculate an offset of + * the desired page given VMA start/pgoff and address, then + * search upwards from 0 to find which span contains it. + * + * On success, a valid PFN for a page sized by 'order' is + * returned into out_pfn. + * + * Failure occurs if: + * - A hugepage would cross the edge of the VMA, + * - A hugepage isn't entirely contained within a range + * (including where it straddles the boundary between + * ranges), + * - We find a range, but the final PFN isn't aligned to the + * requested order. + * + * Upon failure, -EAGAIN is returned and the caller is + * expected to try again with a smaller order, which will + * eventually succeed (order=0 will always work). + * + * It's suboptimal if DMABUFs are created with neighbouring + * ranges that are physically contiguous, since hugepages + * can't straddle range boundaries. (The construction of the + * ranges should merge them in this case.) + * + * Finally, vma_pgoff_adjust is used with a DMABUF created for + * a VFIO BAR mmap: a BAR mapped with vm_pgoff > 0 creates a + * DMABUF such that byte 0 of the VMA corresponds to byte 0 of + * the DMABUF and byte 'vm_pgoff << PAGE_SHIFT' into the BAR. + * To avoid double-offsetting in this scenario, subtracting + * vma_pgoff_adjust from this (non-zero) vm_pgoff generates + * the effective offset. + / + + const unsigned long pagesize = PAGE_SIZE << order; + unsigned long vma_off = ((vma->vm_pgoff - priv->vma_pgoff_adjust) << + PAGE_SHIFT) & VFIO_PCI_OFFSET_MASK; + unsigned long rounded_page_addr = ALIGN_DOWN(fault_addr, pagesize); + unsigned long rounded_page_end = rounded_page_addr + pagesize; + unsigned long fault_offset; + unsigned long fault_offset_end; + unsigned long range_start_offset = 0; + unsigned int i; + int ret; + + if (rounded_page_addr < vma->vm_start \|\| rounded_page_end > vma->vm_end) { + if (order > 0) + return -EAGAIN; + + / A fault address outside of the VMA is absurd. / + dev_warn_ratelimited( + &priv->vdev->pdev->dev, + "Fault addr 0x%lx outside VMA 0x%lx-0x%lx\n", + fault_addr, vma->vm_start, vma->vm_end); + return -EFAULT; + } + + / + * fault_offset[_end] is the span within the DMABUF + * corresponding to the faulting page: + / + if (unlikely(check_add_overflow(rounded_page_addr - vma->vm_start, + vma_off, &fault_offset) \|\| + check_add_overflow(fault_offset, pagesize, + &fault_offset_end))) + return -EFAULT; + + / + * Iterate over ranges in the buffer, summing their lengths: + * range_start_offset represents the current range's starting + * offset in the buffer (from 0 upwards). + * + * A failure for order == 0 is unexpected, and triggers a + * fault/warn. + / + ret = (order == 0) ? -EFAULT : -EAGAIN; + + for (i = 0; i < priv->nr_ranges; i++) { + size_t range_len = priv->phys_vec[i].len; + + / Early exit if range starts after the page end / + if (fault_offset_end <= range_start_offset) + break; + + if (fault_offset >= range_start_offset && + fault_offset_end <= range_start_offset + range_len) { + / + * The faulting page is wholly contained + * within the span represented by this range, + * so validate PFN alignment for the order. + * The if() condition ensures the pfn + * arithmetic won't overflow. + / + unsigned long pfn = + ((fault_offset - range_start_offset) + + priv->phys_vec[i].paddr) >> PAGE_SHIFT; + + if (IS_ALIGNED(pfn, 1 << order)) { + out_pfn = pfn; + ret = 0; + } + /* + * Else order > 0; EAGAIN retries with smaller + * order + / + break; + } + range_start_offset += range_len; + } + + if (order == 0 && ret != 0) + / + * The address fell outside of the span represented by + * the (concatenated) ranges. As setup of a mapping + * ensures that the VMA is <= the total size of the + * ranges this should never happen. If it does, warn + * and SIGBUS. + / + dev_warn_ratelimited( + &priv->vdev->pdev->dev, + "No range for addr 0x%lx, order %d: VMA 0x%lx-0x%lx pgoff 0x%lx, %u ranges, size 0x%zx\n", + fault_addr, order, vma->vm_start, vma->vm_end, + vma->vm_pgoff, priv->nr_ranges, priv->size); + + return ret; +} + +/ + * Create a DMABUF corresponding to priv, add it to vdev->dmabufs list + * for tracking (meaning cleanup or revocation will zap it), and take + * a vfio_device registration. + / +static int vfio_pci_dmabuf_export(struct vfio_pci_core_device vdev, + struct vfio_pci_dma_buf priv, u32 flags) +{ + DEFINE_DMA_BUF_EXPORT_INFO(exp_info); + + if (!vfio_device_try_get_registration(&vdev->vdev)) + return -ENODEV; + + exp_info.ops = &vfio_pci_dmabuf_ops; + exp_info.size = priv->size; + exp_info.flags = flags; + exp_info.priv = priv; + + priv->dmabuf = dma_buf_export(&exp_info); + if (IS_ERR(priv->dmabuf)) { + vfio_device_put_registration(&vdev->vdev); + return PTR_ERR(priv->dmabuf); + } + + kref_init(&priv->kref); + init_completion(&priv->comp); + + / dma_buf_put() now frees priv / + INIT_LIST_HEAD(&priv->dmabufs_elm); + down_write(&vdev->memory_lock); + dma_resv_lock(priv->dmabuf->resv, NULL); + priv->status = __vfio_pci_memory_enabled(vdev) ? VFIO_PCI_DMABUF_OK : + VFIO_PCI_DMABUF_TEMP_REVOKED; + list_add_tail(&priv->dmabufs_elm, &vdev->dmabufs); + dma_resv_unlock(priv->dmabuf->resv); + up_write(&vdev->memory_lock); + + return 0; +} + +#ifdef CONFIG_VFIO_PCI_DMABUF / * This is a temporary "private interconnect" between VFIO DMABUF and iommufd. * It allows the two co-operating drivers to exchange the physical address of @@ -128,7 +340,7 @@ int vfio_pci_dma_buf_iommufd_map(struct dma_buf_attachment attachment, return -EOPNOTSUPP; priv = attachment->dmabuf->priv; - if (priv->revoked) + if (priv->status != VFIO_PCI_DMABUF_OK) return -ENODEV; / More than one range to iommufd will require proper DMABUF support / @@ -224,7 +436,6 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device vdev, u32 flags, { struct vfio_device_feature_dma_buf get_dma_buf = {}; struct vfio_region_dma_range dma_ranges; - DEFINE_DMA_BUF_EXPORT_INFO(exp_info); struct vfio_pci_dma_buf priv; size_t length; int ret; @@ -274,6 +485,7 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device vdev, u32 flags, priv->vdev = vdev; priv->nr_ranges = get_dma_buf.nr_ranges; priv->size = length; + priv->memattr = VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR_NC; ret = vdev->pci_ops->get_dmabuf_phys(vdev, &priv->provider, get_dma_buf.region_index, priv->phys_vec, dma_ranges, @@ -284,34 +496,9 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device vdev, u32 flags, kfree(dma_ranges); dma_ranges = NULL; - if (!vfio_device_try_get_registration(&vdev->vdev)) { - ret = -ENODEV; + ret = vfio_pci_dmabuf_export(vdev, priv, get_dma_buf.open_flags); + if (ret) goto err_free_phys; - } - - exp_info.ops = &vfio_pci_dmabuf_ops; - exp_info.size = priv->size; - exp_info.flags = get_dma_buf.open_flags; - exp_info.priv = priv; - - priv->dmabuf = dma_buf_export(&exp_info); - if (IS_ERR(priv->dmabuf)) { - ret = PTR_ERR(priv->dmabuf); - goto err_dev_put; - } - - kref_init(&priv->kref); - init_completion(&priv->comp); - - /* dma_buf_put() now frees priv / - INIT_LIST_HEAD(&priv->dmabufs_elm); - down_write(&vdev->memory_lock); - dma_resv_lock(priv->dmabuf->resv, NULL); - priv->revoked = !__vfio_pci_memory_enabled(vdev); - list_add_tail(&priv->dmabufs_elm, &vdev->dmabufs); - dma_resv_unlock(priv->dmabuf->resv); - up_write(&vdev->memory_lock); - / * dma_buf_fd() consumes the reference, when the file closes the dmabuf * will be released. @@ -322,8 +509,6 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device vdev, u32 flags, return ret; -err_dev_put: - vfio_device_put_registration(&vdev->vdev); err_free_phys: kfree(priv->phys_vec); err_free_priv: @@ -332,47 +517,164 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device vdev, u32 flags, kfree(dma_ranges); return ret; } +#endif /* CONFIG_VFIO_PCI_DMABUF / + +int vfio_pci_core_mmap_prep_dmabuf(struct vfio_pci_core_device vdev, + struct vm_area_struct vma, + u64 phys_start, u64 req_len, + unsigned int res_index) +{ + struct vfio_pci_dma_buf priv; + unsigned long vma_pgoff = vma->vm_pgoff & (VFIO_PCI_OFFSET_MASK >> PAGE_SHIFT); + char bufname; + int ret; + + priv = kzalloc_obj(priv); + if (!priv) + return -ENOMEM; + + priv->phys_vec = kzalloc_obj(priv->phys_vec); + if (!priv->phys_vec) { + ret = -ENOMEM; + goto err_free_priv; + } + + bufname = kasprintf(GFP_KERNEL, "%s:%s/%x", + dev_name(&vdev->vdev.device), pci_name(vdev->pdev), + res_index); + + if (!bufname) { + ret = -ENOMEM; + goto err_free_phys; + } + + / + * The DMABUF begins from the mmap()'s BAR offset, i.e. the + * start of the VMA corresponds to byte 0 of the DMABUF and + * byte (vma_pgoff << PAGE_SHIFT) of the BAR. + * + * vfio_pci_dma_buf_find_pfn() reverses this offset using + * vma_pgoff_adjust, so that ultimately a fault's offset from + * the start of the _VMA_ has a consistent usage whether the + * VMA originates from an mmap() of the VFIO device here or a + * direct DMABUF mmap(). + / + priv->vdev = vdev; + priv->size = req_len; + priv->nr_ranges = 1; + priv->vma_pgoff_adjust = vma_pgoff; + priv->provider = pcim_p2pdma_provider(vdev->pdev, res_index); + if (!priv->provider) { + ret = -EINVAL; + goto err_free_name; + } + + priv->phys_vec[0].paddr = phys_start + ((u64)vma_pgoff << PAGE_SHIFT); + priv->phys_vec[0].len = priv->size; + + ret = vfio_pci_dmabuf_export(vdev, priv, O_CLOEXEC \| O_RDWR); + if (ret) + goto err_free_name; + + / + * Ownership of the DMABUF file transfers to the VMA so that + * other users can locate the DMABUF via a VA. Ownership of + * the original VFIO device file being mmap()ed transfers to + * priv, and is put when the DMABUF is released. This + * intentionally does not use get_file()/vma_set_file() + * because the references are already held, and ownership + * moves. + / + priv->vfile = vma->vm_file; + vma->vm_file = priv->dmabuf->file; + vma->vm_private_data = priv; + + spin_lock(&priv->dmabuf->name_lock); + kfree(priv->dmabuf->name); + priv->dmabuf->name = bufname; + spin_unlock(&priv->dmabuf->name_lock); + + return 0; + +err_free_name: + kfree(bufname); +err_free_phys: + kfree(priv->phys_vec); +err_free_priv: + kfree(priv); + return ret; +} + +/ Set the DMABUF's revocation status (OK or temporarily/permanently revoked) / +static void vfio_pci_dma_buf_set_status(struct vfio_pci_dma_buf priv, + enum vfio_pci_dma_buf_status new_status) +{ + bool was_revoked; + + lockdep_assert_held_write(&priv->vdev->memory_lock); + + if (priv->status == VFIO_PCI_DMABUF_PERM_REVOKED \|\| + priv->status == new_status) + return; + + dma_resv_lock(priv->dmabuf->resv, NULL); + was_revoked = (priv->status == VFIO_PCI_DMABUF_TEMP_REVOKED); + + if (new_status != VFIO_PCI_DMABUF_OK) { + priv->status = new_status; /* Temp or permanently revoked / + + if (was_revoked) { + / + * TEMP_REVOKED is being upgraded to + * PERM_REVOKED. The buffer is already gone, + * don't wait on it again. + / + dma_resv_unlock(priv->dmabuf->resv); + return; + } + dma_buf_invalidate_mappings(priv->dmabuf); + dma_resv_wait_timeout(priv->dmabuf->resv, + DMA_RESV_USAGE_BOOKKEEP, false, + MAX_SCHEDULE_TIMEOUT); + dma_resv_unlock(priv->dmabuf->resv); + kref_put(&priv->kref, vfio_pci_dma_buf_done); + wait_for_completion(&priv->comp); + unmap_mapping_range(priv->dmabuf->file->f_mapping, + 0, priv->size, true); + / + * Re-arm the registered kref reference and the + * completion so the post-revoke state matches the + * post-creation state. An un-revoke followed by a + * new mapping needs the kref to be non-zero before + * kref_get(), and vfio_pci_dma_buf_cleanup() + * delegates its drain back through this revoke + * path on a possibly-already-revoked dma-buf. + / + kref_init(&priv->kref); + reinit_completion(&priv->comp); + } else { + priv->status = VFIO_PCI_DMABUF_OK; + dma_resv_unlock(priv->dmabuf->resv); + } +} void vfio_pci_dma_buf_move(struct vfio_pci_core_device vdev, bool revoked) { struct vfio_pci_dma_buf priv; struct vfio_pci_dma_buf tmp; + /* + * Holding memory_lock ensures a racing VMA fault observes + * priv->status properly. + / lockdep_assert_held_write(&vdev->memory_lock); list_for_each_entry_safe(priv, tmp, &vdev->dmabufs, dmabufs_elm) { if (!get_file_active(&priv->dmabuf->file)) continue; - - if (priv->revoked != revoked) { - dma_resv_lock(priv->dmabuf->resv, NULL); - if (revoked) - priv->revoked = true; - dma_buf_invalidate_mappings(priv->dmabuf); - dma_resv_wait_timeout(priv->dmabuf->resv, - DMA_RESV_USAGE_BOOKKEEP, false, - MAX_SCHEDULE_TIMEOUT); - dma_resv_unlock(priv->dmabuf->resv); - if (revoked) { - kref_put(&priv->kref, vfio_pci_dma_buf_done); - wait_for_completion(&priv->comp); - / - * Re-arm the registered kref reference and the - * completion so the post-revoke state matches the - * post-creation state. An un-revoke followed by a - * new mapping needs the kref to be non-zero before - * kref_get(), and vfio_pci_dma_buf_cleanup() - * delegates its drain back through this revoke - * path on a possibly-already-revoked dma-buf. - / - kref_init(&priv->kref); - reinit_completion(&priv->comp); - } else { - dma_resv_lock(priv->dmabuf->resv, NULL); - priv->revoked = false; - dma_resv_unlock(priv->dmabuf->resv); - } - } + vfio_pci_dma_buf_set_status(priv, revoked ? + VFIO_PCI_DMABUF_TEMP_REVOKED : + VFIO_PCI_DMABUF_OK); fput(priv->dmabuf->file); } } @@ -404,3 +706,117 @@ void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device vdev) } up_write(&vdev->memory_lock); } + +#ifdef CONFIG_VFIO_PCI_DMABUF +int vfio_pci_core_feature_dma_buf_revoke( + struct vfio_pci_core_device vdev, u32 flags, + struct vfio_device_feature_dma_buf_revoke __user arg, + size_t argsz) +{ + struct vfio_device_feature_dma_buf_revoke db_revoke; + struct vfio_pci_dma_buf priv; + struct dma_buf dmabuf; + int ret; + + if (!vdev->pci_ops \|\| !vdev->pci_ops->get_dmabuf_phys) + return -EOPNOTSUPP; + + ret = vfio_check_feature(flags, argsz, + VFIO_DEVICE_FEATURE_SET, + sizeof(db_revoke)); + if (ret != 1) + return ret; + + if (copy_from_user(&db_revoke, arg, sizeof(db_revoke))) + return -EFAULT; + + dmabuf = dma_buf_get(db_revoke.dmabuf_fd); + if (IS_ERR(dmabuf)) + return PTR_ERR(dmabuf); + + priv = dmabuf->priv; + /* + * Sanity-check the DMABUF is really a vfio_pci_dma_buf _and_ + * relates to the VFIO device it was provided with. + * + * If the DMABUF relates to this vdev then priv->vdev is + * stable because this open fd prevents cleanup. + * + * If it relates to a different vdev, reading priv->vdev might + * race with a concurrent cleanup on that device. But if so, + * it points to a non-matching vdev or NULL and is unusable + * either way. + / + if (dmabuf->ops != &vfio_pci_dmabuf_ops \|\| + READ_ONCE(priv->vdev) != vdev) { + ret = -ENODEV; + goto out_put_buf; + } + + scoped_guard(rwsem_write, &vdev->memory_lock) { + if (priv->status == VFIO_PCI_DMABUF_PERM_REVOKED) { + ret = -EBADFD; + } else { + vfio_pci_dma_buf_set_status(priv, + VFIO_PCI_DMABUF_PERM_REVOKED); + ret = 0; + } + } + +out_put_buf: + dma_buf_put(dmabuf); + + return ret; +} + +int vfio_pci_core_feature_dma_buf_memattr( + struct vfio_pci_core_device vdev, u32 flags, + struct vfio_device_feature_dma_buf_memattr __user arg, + size_t argsz) +{ + struct vfio_device_feature_dma_buf_memattr db_attr; + struct vfio_pci_dma_buf priv; + struct dma_buf dmabuf; + int ret; + + if (!vdev->pci_ops \|\| !vdev->pci_ops->get_dmabuf_phys) + return -EOPNOTSUPP; + + ret = vfio_check_feature(flags, argsz, + VFIO_DEVICE_FEATURE_SET, + sizeof(db_attr)); + if (ret != 1) + return ret; + + if (copy_from_user(&db_attr, arg, sizeof(db_attr))) + return -EFAULT; + + dmabuf = dma_buf_get(db_attr.dmabuf_fd); + if (IS_ERR(dmabuf)) + return PTR_ERR(dmabuf); + + / Verify DMABUF: see comments in vfio_pci_dma_buf_revoke() / + priv = dmabuf->priv; + if (dmabuf->ops != &vfio_pci_dmabuf_ops \|\| + READ_ONCE(priv->vdev) != vdev) { + ret = -ENODEV; + goto out_put_buf; + } + + switch (db_attr.memattr) { + case VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR_NC: + case VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR_WC: + WRITE_ONCE(priv->memattr, db_attr.memattr); + ret = 0; + break; + + default: + ret = -ENOENT; + } + +out_put_buf: + dma_buf_put(dmabuf); + + return ret; +} +#endif / CONFIG_VFIO_PCI_DMABUF / diff --git a/drivers/vfio/pci/vfio_pci_priv.h b/drivers/vfio/pci/vfio_pci_priv.h index fca9d0dfac90f8..b3c72df002c448 100644 --- a/drivers/vfio/pci/vfio_pci_priv.h +++ b/drivers/vfio/pci/vfio_pci_priv.h @@ -23,6 +23,28 @@ struct vfio_pci_ioeventfd { bool test_mem; }; +enum vfio_pci_dma_buf_status { + VFIO_PCI_DMABUF_OK = 0, + VFIO_PCI_DMABUF_TEMP_REVOKED = 1, + VFIO_PCI_DMABUF_PERM_REVOKED = 2, +}; + +struct vfio_pci_dma_buf { + struct dma_buf dmabuf; + struct vfio_pci_core_device vdev; + struct list_head dmabufs_elm; + size_t size; + struct phys_vec phys_vec; + struct p2pdma_provider provider; + struct file vfile; + u32 nr_ranges; + struct kref kref; + struct completion comp; + unsigned long vma_pgoff_adjust; + u32 memattr; + enum vfio_pci_dma_buf_status status; +}; + bool vfio_pci_intx_mask(struct vfio_pci_core_device vdev); void vfio_pci_intx_unmask(struct vfio_pci_core_device vdev); @@ -68,7 +90,8 @@ void vfio_config_free(struct vfio_pci_core_device vdev); int vfio_pci_set_power_state(struct vfio_pci_core_device vdev, pci_power_t state); -void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device vdev); +void vfio_pci_lock_revoke_bars(struct vfio_pci_core_device vdev); +void vfio_pci_unrevoke_bars(struct vfio_pci_core_device vdev); u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device vdev); void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device vdev, u16 cmd); @@ -114,12 +137,31 @@ static inline bool vfio_pci_is_vga(struct pci_dev pdev) return (pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA; } +int vfio_pci_dma_buf_find_pfn(struct vfio_pci_dma_buf vpdmabuf, + struct vm_area_struct vma, + unsigned long address, + unsigned int order, + unsigned long out_pfn); +int vfio_pci_core_mmap_prep_dmabuf(struct vfio_pci_core_device vdev, + struct vm_area_struct vma, + u64 phys_start, u64 req_len, + unsigned int res_index); +void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device vdev); +void vfio_pci_dma_buf_move(struct vfio_pci_core_device vdev, bool revoked); +void vfio_pci_set_vma_ops(struct vm_area_struct vma); + #ifdef CONFIG_VFIO_PCI_DMABUF int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device vdev, u32 flags, struct vfio_device_feature_dma_buf __user arg, size_t argsz); -void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device vdev); -void vfio_pci_dma_buf_move(struct vfio_pci_core_device vdev, bool revoked); +int vfio_pci_core_feature_dma_buf_revoke( + struct vfio_pci_core_device vdev, u32 flags, + struct vfio_device_feature_dma_buf_revoke __user arg, + size_t argsz); +int vfio_pci_core_feature_dma_buf_memattr( + struct vfio_pci_core_device vdev, u32 flags, + struct vfio_device_feature_dma_buf_memattr __user arg, + size_t argsz); #else static inline int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device vdev, u32 flags, @@ -128,12 +170,21 @@ vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device vdev, u32 flags, { return -ENOTTY; } -static inline void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device vdev) + +static inline int vfio_pci_core_feature_dma_buf_revoke( + struct vfio_pci_core_device vdev, u32 flags, + struct vfio_device_feature_dma_buf_revoke __user arg, + size_t argsz) { + return -ENOTTY; } -static inline void vfio_pci_dma_buf_move(struct vfio_pci_core_device vdev, - bool revoked) + +static inline int vfio_pci_core_feature_dma_buf_memattr( + struct vfio_pci_core_device vdev, u32 flags, + struct vfio_device_feature_dma_buf_memattr __user arg, + size_t argsz) { + return -ENOTTY; } #endif diff --git a/include/linux/pci-p2pdma.h b/include/linux/pci-p2pdma.h index 873de20a224759..4c42a7b2ee8558 100644 --- a/include/linux/pci-p2pdma.h +++ b/include/linux/pci-p2pdma.h @@ -67,9 +67,22 @@ enum pci_p2pdma_map_type { PCI_P2PDMA_MAP_THRU_HOST_BRIDGE, }; -#ifdef CONFIG_PCI_P2PDMA +#ifdef CONFIG_PCI_P2PDMA_CORE int pcim_p2pdma_init(struct pci_dev pdev); struct p2pdma_provider pcim_p2pdma_provider(struct pci_dev pdev, int bar); +#else +static inline int pcim_p2pdma_init(struct pci_dev pdev) +{ + return -EOPNOTSUPP; +} +static inline struct p2pdma_provider pcim_p2pdma_provider(struct pci_dev pdev, + int bar) +{ + return NULL; +} +#endif + +#ifdef CONFIG_PCI_P2PDMA int pci_p2pdma_add_resource(struct pci_dev pdev, int bar, size_t size, u64 offset); int pci_p2pdma_distance_many(struct pci_dev provider, struct device *clients, @@ -89,15 +102,6 @@ ssize_t pci_p2pdma_enable_show(char page, struct pci_dev p2p_dev, enum pci_p2pdma_map_type pci_p2pdma_map_type(struct p2pdma_provider provider, struct device dev); #else / CONFIG_PCI_P2PDMA / -static inline int pcim_p2pdma_init(struct pci_dev pdev) -{ - return -EOPNOTSUPP; -} -static inline struct p2pdma_provider pcim_p2pdma_provider(struct pci_dev pdev, - int bar) -{ - return NULL; -} static inline int pci_p2pdma_add_resource(struct pci_dev pdev, int bar, size_t size, u64 offset) { diff --git a/include/linux/pci.h b/include/linux/pci.h index 2c4454583c115b..531aec355686b0 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -557,7 +557,7 @@ struct pci_dev { u16 pasid_cap; / PASID Capability offset / u16 pasid_features; #endif -#ifdef CONFIG_PCI_P2PDMA +#ifdef CONFIG_PCI_P2PDMA_CORE struct pci_p2pdma __rcu p2pdma; #endif #ifdef CONFIG_PCI_DOE diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h index 89165b769e5c16..a2f586e4fcd8d4 100644 --- a/include/linux/vfio_pci_core.h +++ b/include/linux/vfio_pci_core.h @@ -127,6 +127,7 @@ struct vfio_pci_core_device { bool needs_pm_restore:1; bool pm_intx_masked:1; bool pm_runtime_engaged:1; + bool zap_bars_on_revoke:1; struct pci_saved_state pci_saved_state; struct pci_saved_state pm_save; int ioeventfds_nr; diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h index 5de618a3a5eeee..ab30b89399d038 100644 --- a/include/uapi/linux/vfio.h +++ b/include/uapi/linux/vfio.h @@ -1534,6 +1534,53 @@ struct vfio_device_feature_dma_buf { / #define VFIO_DEVICE_FEATURE_MIG_PRECOPY_INFOv2 12 +/* + * Given a dma_buf fd previously created by + * VFIO_DEVICE_FEATURE_DMA_BUF, a SET of this feature requests that + * access to the corresponding DMABUF is immediately and permanently + * revoked. On successful return, the buffer is not accessible + * through any mmap() or dma-buf import. The buffer is permanently + * disabled, and VFIO refuses all map, mmap, attach, etc. requests. + * + * Return: 0 on success, -1 and errno is set on failure: + * + * EBADF, EINVAL: dmabuf_fd is not a DMABUF fd. + * ENODEV: The dmabuf_fd does not match this VFIO device. + * EBADFD: The DMABUF is already revoked. + / +#define VFIO_DEVICE_FEATURE_DMA_BUF_REVOKE 13 + +struct vfio_device_feature_dma_buf_revoke { + __s32 dmabuf_fd; +}; + +/* + * Given a dma_buf fd previously created by + * VFIO_DEVICE_FEATURE_DMA_BUF, SET the memory attribute that will be + * used by future mmap()s of that fd. SETting a new attribute does + * not affect existing VMAs. + * + * The default, if no previous SET has been performed, is NC. + * + * Return: 0 on success, -1 and errno is set on failure: + * + * EBADF, EINVAL: dmabuf_fd is not a DMABUF fd. + * ENODEV: The dmabuf_fd does not match this VFIO device. + * ENOENT: The given memattr is not supported. + / +#define VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR 14 + +/ Valid memory attributes for the memattr field / +enum vfio_device_dma_buf_memattr { + VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR_NC = 0, / pgprot_noncached / + VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR_WC = 1, / pgprot_writecombine / +}; + +struct vfio_device_feature_dma_buf_memattr { + __s32 dmabuf_fd; + __u32 memattr; +}; + / -------- API for Type1 VFIO IOMMU -------- / /*
3/2	2026/07/02 00:20	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 25790 output: 30 thoughts: 798 Thoughts: Analyzing the New VFIO PCI DMA-BUF Patch Series Okay, so I'm looking at this patch series, and it's making some significant changes to how VFIO interacts with PCI P2PDMA and DMA-BUF. The goal, it seems, is to improve DMA-BUF integration within the VFIO framework. I need to figure out where the code is touched and if it’s interesting enough for fuzzing. Firstly, they've introduced `CONFIG_PCI_P2PDMA_CORE`, and crucially, they've shifted `CONFIG_VFIO_PCI_DMABUF` to now depend on `PCI_P2PDMA` rather than requiring both `PCI_P2PDMA` and `DMA_SHARED_BUFFER`, which is something to immediately note. They've also added these new VFIO features: `VFIO_DEVICE_FEATURE_DMA_BUF_REVOKE` and `VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR`. This is interesting; these features suggest enhanced control and management over DMA-BUFs. The code modifications are happening in `drivers/vfio/pci/vfio_pci_core.c`, `drivers/vfio/pci/vfio_pci_dmabuf.c`, and `drivers/pci/p2pdma_core.c`. We know VFIO itself is reachable and is actively fuzzed by syzkaller, so that’s a good starting point. I can see the new features are handled in `vfio_pci_core_ioctl_feature`, which also handles other ioctls that may be good to fuzz. Now, to get a handle on the build configuration. We need: `CONFIG_VFIO_PCI_CORE`, `CONFIG_VFIO_PCI`, `CONFIG_PCI_P2PDMA`, and `CONFIG_VFIO_PCI_DMABUF`. `VFIO_PCI_DMABUF` is specifically conditional in the Kconfig with: `def_bool y if PCI_P2PDMA`, and dependent on `VFIO_PCI_CORE`, so to enable everything it seems we need `VFIO_PCI` as that brings in `VFIO_PCI_CORE` which then pulls in `VFIO_PCI_DMABUF`. `PCI_P2PDMA`, in the PCI Kconfig, requires `ZONE_DEVICE`. So, the minimal set is: `PCI_P2PDMA`, `ZONE_DEVICE`, and `VFIO_PCI`. The key symbols to focus on are the new feature handlers: `vfio_pci_core_feature_dma_buf_revoke`, `vfio_pci_core_feature_dma_buf_memattr`, and the changes to `vfio_pci_core_mmap`. Also, the new function `vfio_pci_dma_buf_mmap` and supporting functions like `vfio_pci_dma_buf_find_pfn`, `vfio_pci_dmabuf_export`, `vfio_pci_core_mmap_prep_dmabuf`, and `vfio_pci_dma_buf_set_status`. They’re all called from `vfio_pci_core_ioctl_feature` and seem like prime targets for further examination and fuzzing. Now, let’s get the testing tools ready.
4/2	2026/07/02 00:20	tool	grepper	0m Args: { "Expression": "vfio_pci_core_feature_dma_buf_revoke" } Results: { "Output": "drivers/vfio/pci/vfio_pci_core.c=1562=int vfio_pci_core_ioctl_feature(struct vfio_device device, u32 flags,\n--\ndrivers/vfio/pci/vfio_pci_core.c-1580-\tcase VFIO_DEVICE_FEATURE_DMA_BUF_REVOKE:\ndrivers/vfio/pci/vfio_pci_core.c:1581:\t\treturn vfio_pci_core_feature_dma_buf_revoke(vdev, flags, arg, argsz);\ndrivers/vfio/pci/vfio_pci_core.c-1582-\tcase VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR:\n--\ndrivers/vfio/pci/vfio_pci_dmabuf.c=682=void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device vdev)\n--\ndrivers/vfio/pci/vfio_pci_dmabuf.c-710-#ifdef CONFIG_VFIO_PCI_DMABUF\ndrivers/vfio/pci/vfio_pci_dmabuf.c:711:int vfio_pci_core_feature_dma_buf_revoke(\ndrivers/vfio/pci/vfio_pci_dmabuf.c-712-\tstruct vfio_pci_core_device vdev, u32 flags,\n--\ndrivers/vfio/pci/vfio_pci_priv.h=154=int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device vdev, u32 flags,\n--\ndrivers/vfio/pci/vfio_pci_priv.h-156-\t\t\t\t size_t argsz);\ndrivers/vfio/pci/vfio_pci_priv.h:157:int vfio_pci_core_feature_dma_buf_revoke(\ndrivers/vfio/pci/vfio_pci_priv.h-158-\tstruct vfio_pci_core_device vdev, u32 flags,\n--\ndrivers/vfio/pci/vfio_pci_priv.h=167=vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device vdev, u32 flags,\n--\ndrivers/vfio/pci/vfio_pci_priv.h-173-\ndrivers/vfio/pci/vfio_pci_priv.h:174:static inline int vfio_pci_core_feature_dma_buf_revoke(\ndrivers/vfio/pci/vfio_pci_priv.h-175-\tstruct vfio_pci_core_device *vdev, u32 flags,\n" }
5/2	2026/07/02 00:20	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 28087 output: 27 thoughts: 38 Thoughts: Okay, here's my interpretation of those thoughts, tailored for an expert audience, presented in the first person: Navigating the VFIO PCI Landscape: Feature Access and DMA Buffer Mapping Alright, so I'm looking at VFIO-PCI's inner workings. My immediate focus is on how features are accessed. It seems like the primary entry point for managing those features is the `vfio_pci_core_ioctl_feature` function. I need to understand that interaction flow precisely. Specifically, I need to understand how we set and get things using this method. I need to understand how we are able to do that, and in what situations. Beyond that, I'm thinking about DMA buffer mapping, and how that relates to the features of the device that we are accessing. I'm going to turn my attention to `vfio_pci_dma_buf_mmap`. I'm curious about the specific interaction of that functionality. I will also be looking into what data structures support the interaction. How does this fit into the larger picture of secure DMA access within VFIO? Are there specific protections involved, and if so, how do they work? I need to understand this to make sure things are working as they should be.
6/2	2026/07/02 00:20	tool	grepper	0m Args: { "Expression": "vfio_pci_dma_buf_mmap" } Results: { "Output": "drivers/vfio/pci/vfio_pci_dmabuf.c=14=static int vfio_pci_dma_buf_attach(struct dma_buf dmabuf,\n--\ndrivers/vfio/pci/vfio_pci_dmabuf.c-30-\ndrivers/vfio/pci/vfio_pci_dmabuf.c:31:static int vfio_pci_dma_buf_mmap(struct dma_buf dmabuf, struct vm_area_struct *vma)\ndrivers/vfio/pci/vfio_pci_dmabuf.c-32-{\n--\ndrivers/vfio/pci/vfio_pci_dmabuf.c=130=static const struct dma_buf_ops vfio_pci_dmabuf_ops = {\n--\ndrivers/vfio/pci/vfio_pci_dmabuf.c-132-\t.attach = vfio_pci_dma_buf_attach,\ndrivers/vfio/pci/vfio_pci_dmabuf.c:133:\t.mmap = vfio_pci_dma_buf_mmap,\ndrivers/vfio/pci/vfio_pci_dmabuf.c-134-#endif\n" }
7/2	2026/07/02 00:20	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 28749 output: 29 thoughts: 21 Thoughts: Here's a summary of my thought process, as requested: Diving into `vfio_pci_dma_buf_find_pfn` Okay, I'm thinking I need to refresh my memory on `vfio_pci_dma_buf_find_pfn`. It's been a while since I last dug into the DMA buffer management within the VFIO-PCI context. I need to understand how that function works, particularly its role in locating the physical frame number (PFN) associated with a DMA buffer. This is key for any further investigation of memory mapping or DMA operations within this driver. Knowing its behavior will help me understand how VFIO interacts with DMA and page allocation. I need to remember the arguments, the return values, and what the function is supposed to accomplish. Specifically, I should check the context in which it's used within the wider VFIO-PCI framework. What other VFIO functions does it depend on or coordinate with? Time to break out the code and the commit logs.
8/2	2026/07/02 00:20	tool	grepper	0m Args: { "Expression": "vfio_pci_dma_buf_find_pfn" } Results: { "Output": "drivers/vfio/pci/vfio_pci_core.c=1738=static vm_fault_t vfio_pci_mmap_huge_fault(struct vm_fault vmf,\n--\ndrivers/vfio/pci/vfio_pci_core.c-1820-\t\tif (priv-\u003estatus == VFIO_PCI_DMABUF_OK) {\ndrivers/vfio/pci/vfio_pci_core.c:1821:\t\t\tint pres = vfio_pci_dma_buf_find_pfn(priv, vma,\ndrivers/vfio/pci/vfio_pci_core.c-1822-\t\t\t\t\t\t\t vmf-\u003eaddress,\n--\ndrivers/vfio/pci/vfio_pci_dmabuf.c=130=static const struct dma_buf_ops vfio_pci_dmabuf_ops = {\n--\ndrivers/vfio/pci/vfio_pci_dmabuf.c-139-\ndrivers/vfio/pci/vfio_pci_dmabuf.c:140:int vfio_pci_dma_buf_find_pfn(struct vfio_pci_dma_buf priv,\ndrivers/vfio/pci/vfio_pci_dmabuf.c-141-\t\t\t struct vm_area_struct vma,\n--\ndrivers/vfio/pci/vfio_pci_dmabuf.c=522=int vfio_pci_core_mmap_prep_dmabuf(struct vfio_pci_core_device vdev,\n--\ndrivers/vfio/pci/vfio_pci_dmabuf.c-555-\t \ndrivers/vfio/pci/vfio_pci_dmabuf.c:556:\t vfio_pci_dma_buf_find_pfn() reverses this offset using\ndrivers/vfio/pci/vfio_pci_dmabuf.c-557-\t * vma_pgoff_adjust, so that ultimately a fault's offset from\n--\ndrivers/vfio/pci/vfio_pci_priv.h=135=static inline bool vfio_pci_is_vga(struct pci_dev pdev)\n--\ndrivers/vfio/pci/vfio_pci_priv.h-139-\ndrivers/vfio/pci/vfio_pci_priv.h:140:int vfio_pci_dma_buf_find_pfn(struct vfio_pci_dma_buf vpdmabuf,\ndrivers/vfio/pci/vfio_pci_priv.h-141-\t\t\t struct vm_area_struct *vma,\n" }
9/2	2026/07/02 00:20	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 29720 output: 25 thoughts: 17 Thoughts: Okay, let's dive into this. Here's my take on those thoughts, summarized from my expert perspective: Diving into `vfio_pci_dmabuf_export` Alright, time to get my hands dirty. My mind's automatically jumping to `vfio_pci_dmabuf_export`. This function, more than likely, is the linchpin we're going to be focusing on. It's the point where things get interesting, the place where we transition data, and it is going to set up where all the core functionality will be handled. I'm already anticipating the nuances of DMA-BUF export within the VFIO context. What are the key considerations? Memory mappings, access permissions, synchronization primitives – those are the types of things I'm mentally parsing as I go. Let's see how they've handled that, and how it is all plumbed. I want to understand the exact mechanics of this export. Let's dig in and see what they've done.
10/2	2026/07/02 00:20	tool	grepper	0m Args: { "Expression": "vfio_pci_dmabuf_export" } Results: { "Output": "drivers/vfio/pci/vfio_pci_dmabuf.c=140=int vfio_pci_dma_buf_find_pfn(struct vfio_pci_dma_buf priv,\n--\ndrivers/vfio/pci/vfio_pci_dmabuf.c-284- /\ndrivers/vfio/pci/vfio_pci_dmabuf.c:285:static int vfio_pci_dmabuf_export(struct vfio_pci_core_device vdev,\ndrivers/vfio/pci/vfio_pci_dmabuf.c-286-\t\t\t\t struct vfio_pci_dma_buf priv, u32 flags)\n--\ndrivers/vfio/pci/vfio_pci_dmabuf.c=433=int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device vdev, u32 flags,\n--\ndrivers/vfio/pci/vfio_pci_dmabuf.c-498-\ndrivers/vfio/pci/vfio_pci_dmabuf.c:499:\tret = vfio_pci_dmabuf_export(vdev, priv, get_dma_buf.open_flags);\ndrivers/vfio/pci/vfio_pci_dmabuf.c-500-\tif (ret)\n--\ndrivers/vfio/pci/vfio_pci_dmabuf.c=522=int vfio_pci_core_mmap_prep_dmabuf(struct vfio_pci_core_device vdev,\n--\ndrivers/vfio/pci/vfio_pci_dmabuf.c-574-\ndrivers/vfio/pci/vfio_pci_dmabuf.c:575:\tret = vfio_pci_dmabuf_export(vdev, priv, O_CLOEXEC \| O_RDWR);\ndrivers/vfio/pci/vfio_pci_dmabuf.c-576-\tif (ret)\n" }
11/2	2026/07/02 00:20	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 30523 output: 28 thoughts: 20 Thoughts: Here's a summary of my thought process, as requested: Diving into `vfio_pci_dma_buf_set_status` Okay, time to dig into `vfio_pci_dma_buf_set_status`. This function is critical for managing DMA buffer status within the VFIO-PCI framework. I need to understand exactly how it's interacting with the DMA buffer itself, and what implications it has for the virtualized environment. My primary concern is always performance and security; how is this handled here? I'm particularly interested in memory barriers, cache coherency, and the potential for race conditions – this area is notoriously tricky. I'll need to trace the calls, especially the ones related to setting flags or updating state. I need to make sure this is robust, especially with regard to the various possible guest OS behaviors and potential malicious attempts at accessing data they shouldn't. Let's see how they deal with the buffer status and how they do error handling, just to be sure.
12/2	2026/07/02 00:20	tool	grepper	0m Args: { "Expression": "vfio_pci_dma_buf_set_status" } Results: { "Output": "drivers/vfio/pci/vfio_pci_dmabuf.c=522=int vfio_pci_core_mmap_prep_dmabuf(struct vfio_pci_core_device vdev,\n--\ndrivers/vfio/pci/vfio_pci_dmabuf.c-608-/ Set the DMABUF's revocation status (OK or temporarily/permanently revoked) /\ndrivers/vfio/pci/vfio_pci_dmabuf.c:609:static void vfio_pci_dma_buf_set_status(struct vfio_pci_dma_buf priv,\ndrivers/vfio/pci/vfio_pci_dmabuf.c-610-\t\t\t\t\tenum vfio_pci_dma_buf_status new_status)\n--\ndrivers/vfio/pci/vfio_pci_dmabuf.c=661=void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev, bool revoked)\n--\ndrivers/vfio/pci/vfio_pci_dmabuf.c-674-\t\t\tcontinue;\ndrivers/vfio/pci/vfio_pci_dmabuf.c:675:\t\tvfio_pci_dma_buf_set_status(priv, revoked ?\ndrivers/vfio/pci/vfio_pci_dmabuf.c-676-\t\t\t\t\t VFIO_PCI_DMABUF_TEMP_REVOKED :\n--\ndrivers/vfio/pci/vfio_pci_dmabuf.c=711=int vfio_pci_core_feature_dma_buf_revoke(\n--\ndrivers/vfio/pci/vfio_pci_dmabuf.c-759-\t\t} else {\ndrivers/vfio/pci/vfio_pci_dmabuf.c:760:\t\t\tvfio_pci_dma_buf_set_status(priv,\ndrivers/vfio/pci/vfio_pci_dmabuf.c-761-\t\t\t\t\t\t VFIO_PCI_DMABUF_PERM_REVOKED);\n" }
13/2	2026/07/02 00:20	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 31341 output: 242 thoughts: 417 Thoughts: My Assessment of the VFIO PCI DMA-BUF Patch Okay, so this patch introduces some significant functional changes to VFIO PCI DMA-BUF support. We're talking new ioctls and mmap handling here, which means it's a prime target for fuzzing. I need to make a note to add that to the fuzzing queue. Now, for the important symbols to focus on: `vfio_pci_core_feature_dma_buf_revoke`, `vfio_pci_core_feature_dma_buf_memattr`, `vfio_pci_dma_buf_mmap`, `vfio_pci_dma_buf_find_pfn`, `vfio_pci_dmabuf_export`, `vfio_pci_core_mmap_prep_dmabuf`, and `vfio_pci_dma_buf_set_status`. These are the key functions to examine for potential vulnerabilities. Next, I need to verify the required configurations. We have `VFIO_PCI`, `PCI_P2PDMA`, and `ZONE_DEVICE`. Hmm, let's just confirm that `ZONE_DEVICE` is indeed a hard requirement. Ah, yes, `PCI_P2PDMA` depends on `ZONE_DEVICE`, so we absolutely need to have that enabled. Good, double-checked. Now, I need to determine if just `VFIO_PCI` is sufficient, or if `VFIO_PCI_CORE` is also necessary. Excellent, `VFIO_PCI` selects `VFIO_PCI_CORE`, so that simplifies things. We're good with just enabling `VFIO_PCI`. Let's see, what about `VFIO_PCI_DMABUF`? Is that a separate config option? Yes, it is! Wait... it's a `def_bool y if PCI_P2PDMA`. So, it's automatically enabled if `PCI_P2PDMA` is enabled. Okay, that's good to know. We don't have to worry about enabling that explicitly. Let me see, are there any other related configurations to keep an eye on? `PCI_P2PDMA_CORE` looks relevant, but it's selected by both `PCI_P2PDMA` and `VFIO_PCI_CORE`. Since we already know that these configs are already activated, it's enabled automatically. Good. Alright, with all that out of the way, I think I'm ready to use the `set-results` tool to document these findings.
14/2	2026/07/02 00:20	tool	set-results	0m Args: { "EnableConfigs": [ "VFIO_PCI", "PCI_P2PDMA", "ZONE_DEVICE" ], "FocusSymbols": [ "vfio_pci_core_feature_dma_buf_revoke", "vfio_pci_core_feature_dma_buf_memattr", "vfio_pci_dma_buf_mmap", "vfio_pci_dma_buf_find_pfn", "vfio_pci_dmabuf_export", "vfio_pci_core_mmap_prep_dmabuf", "vfio_pci_dma_buf_set_status" ], "Reasoning": "The patch introduces new VFIO features (VFIO_DEVICE_FEATURE_DMA_BUF_REVOKE and VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR) and modifies the mmap handling to use DMABUF-backed mmaps. These are functional changes to reachable core kernel logic and should be fuzzed.", "WorthFuzzing": true } Results: { "EnableConfigs": [ "VFIO_PCI", "PCI_P2PDMA", "ZONE_DEVICE" ], "FocusSymbols": [ "vfio_pci_core_feature_dma_buf_revoke", "vfio_pci_core_feature_dma_buf_memattr", "vfio_pci_dma_buf_mmap", "vfio_pci_dma_buf_find_pfn", "vfio_pci_dmabuf_export", "vfio_pci_core_mmap_prep_dmabuf", "vfio_pci_dma_buf_set_status" ], "Reasoning": "The patch introduces new VFIO features (VFIO_DEVICE_FEATURE_DMA_BUF_REVOKE and VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR) and modifies the mmap handling to use DMABUF-backed mmaps. These are functional changes to reachable core kernel logic and should be fuzzed.", "WorthFuzzing": true }

{
  "PatchDiff": "commit be307977b6149355089ba083133af2137dd145b0\nAuthor: syz-cluster \u003ctriage@syzkaller.com\u003e\nDate:   Thu Jul 2 00:20:04 2026 +0000\n\n    syz-cluster: applied patch under review\n\ndiff --git a/MAINTAINERS b/MAINTAINERS\nindex c8d4b913f26c12..713861af44846a 100644\n--- a/MAINTAINERS\n+++ b/MAINTAINERS\n@@ -20626,7 +20626,7 @@ B:\thttps://bugzilla.kernel.org\n C:\tirc://irc.oftc.net/linux-pci\n T:\tgit git://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git\n F:\tDocumentation/driver-api/pci/p2pdma.rst\n-F:\tdrivers/pci/p2pdma.c\n+F:\tdrivers/pci/p2pdma*\n F:\tinclude/linux/pci-p2pdma.h\n \n PCI POWER CONTROL\ndiff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig\nindex 33c88432b72875..59d70bc84cc9ac 100644\n--- a/drivers/pci/Kconfig\n+++ b/drivers/pci/Kconfig\n@@ -206,11 +206,7 @@ config PCIE_TPH\n config PCI_P2PDMA\n \tbool \"PCI peer-to-peer transfer support\"\n \tdepends on ZONE_DEVICE\n-\t#\n-\t# The need for the scatterlist DMA bus address flag means PCI P2PDMA\n-\t# requires 64bit\n-\t#\n-\tdepends on 64BIT\n+\tselect PCI_P2PDMA_CORE\n \tselect GENERIC_ALLOCATOR\n \tselect NEED_SG_DMA_FLAGS\n \thelp\n@@ -226,6 +222,10 @@ config PCI_P2PDMA\n \n \t  If unsure, say N.\n \n+config PCI_P2PDMA_CORE\n+\tdefault n\n+\tbool\n+\n config PCI_LABEL\n \tdef_bool y if (DMI || ACPI)\n \tselect NLS\ndiff --git a/drivers/pci/Makefile b/drivers/pci/Makefile\nindex 41ebc3b9a51824..0b32572d57a159 100644\n--- a/drivers/pci/Makefile\n+++ b/drivers/pci/Makefile\n@@ -30,6 +30,7 @@ obj-$(CONFIG_PCI_SYSCALL)\t+= syscall.o\n obj-$(CONFIG_PCI_STUB)\t\t+= pci-stub.o\n obj-$(CONFIG_PCI_PF_STUB)\t+= pci-pf-stub.o\n obj-$(CONFIG_PCI_ECAM)\t\t+= ecam.o\n+obj-$(CONFIG_PCI_P2PDMA_CORE)\t+= p2pdma_core.o\n obj-$(CONFIG_PCI_P2PDMA)\t+= p2pdma.o\n obj-$(CONFIG_XEN_PCIDEV_FRONTEND) += xen-pcifront.o\n obj-$(CONFIG_VGA_ARB)\t\t+= vgaarb.o\ndiff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c\nindex 7c898542af8d5e..50b1a7daf55cc4 100644\n--- a/drivers/pci/p2pdma.c\n+++ b/drivers/pci/p2pdma.c\n@@ -21,12 +21,7 @@\n #include \u003clinux/seq_buf.h\u003e\n #include \u003clinux/xarray.h\u003e\n \n-struct pci_p2pdma {\n-\tstruct gen_pool *pool;\n-\tbool p2pmem_published;\n-\tstruct xarray map_types;\n-\tstruct p2pdma_provider mem[PCI_STD_NUM_BARS];\n-};\n+#include \"p2pdma.h\"\n \n struct pci_p2pdma_pagemap {\n \tstruct dev_pagemap pgmap;\n@@ -226,110 +221,16 @@ static const struct dev_pagemap_ops p2pdma_pgmap_ops = {\n \t.folio_free = p2pdma_folio_free,\n };\n \n-static void pci_p2pdma_release(void *data)\n+void pci_p2pdma_release_pool(struct pci_dev *pdev, struct pci_p2pdma *p2pdma)\n {\n-\tstruct pci_dev *pdev = data;\n-\tstruct pci_p2pdma *p2pdma;\n-\n-\tp2pdma = rcu_dereference_protected(pdev-\u003ep2pdma, 1);\n-\tif (!p2pdma)\n-\t\treturn;\n-\n-\t/* Flush and disable pci_alloc_p2p_mem() */\n-\tpdev-\u003ep2pdma = NULL;\n-\tif (p2pdma-\u003epool)\n-\t\tsynchronize_rcu();\n-\txa_destroy(\u0026p2pdma-\u003emap_types);\n-\n \tif (!p2pdma-\u003epool)\n \t\treturn;\n \n+\tsynchronize_rcu();\n \tgen_pool_destroy(p2pdma-\u003epool);\n \tsysfs_remove_group(\u0026pdev-\u003edev.kobj, \u0026p2pmem_group);\n }\n \n-/**\n- * pcim_p2pdma_init - Initialise peer-to-peer DMA providers\n- * @pdev: The PCI device to enable P2PDMA for\n- *\n- * This function initializes the peer-to-peer DMA infrastructure\n- * for a PCI device. It allocates and sets up the necessary data\n- * structures to support P2PDMA operations, including mapping type\n- * tracking.\n- */\n-int pcim_p2pdma_init(struct pci_dev *pdev)\n-{\n-\tstruct pci_p2pdma *p2p;\n-\tint i, ret;\n-\n-\tp2p = rcu_dereference_protected(pdev-\u003ep2pdma, 1);\n-\tif (p2p)\n-\t\treturn 0;\n-\n-\tp2p = devm_kzalloc(\u0026pdev-\u003edev, sizeof(*p2p), GFP_KERNEL);\n-\tif (!p2p)\n-\t\treturn -ENOMEM;\n-\n-\txa_init(\u0026p2p-\u003emap_types);\n-\t/*\n-\t * Iterate over all standard PCI BARs and record only those that\n-\t * correspond to MMIO regions. Skip non-memory resources (e.g. I/O\n-\t * port BARs) since they cannot be used for peer-to-peer (P2P)\n-\t * transactions.\n-\t */\n-\tfor (i = 0; i \u003c PCI_STD_NUM_BARS; i++) {\n-\t\tif (!(pci_resource_flags(pdev, i) \u0026 IORESOURCE_MEM))\n-\t\t\tcontinue;\n-\n-\t\tp2p-\u003emem[i].owner = \u0026pdev-\u003edev;\n-\t\tp2p-\u003emem[i].bus_offset =\n-\t\t\tpci_bus_address(pdev, i) - pci_resource_start(pdev, i);\n-\t}\n-\n-\tret = devm_add_action_or_reset(\u0026pdev-\u003edev, pci_p2pdma_release, pdev);\n-\tif (ret)\n-\t\tgoto out_p2p;\n-\n-\trcu_assign_pointer(pdev-\u003ep2pdma, p2p);\n-\treturn 0;\n-\n-out_p2p:\n-\tdevm_kfree(\u0026pdev-\u003edev, p2p);\n-\treturn ret;\n-}\n-EXPORT_SYMBOL_GPL(pcim_p2pdma_init);\n-\n-/**\n- * pcim_p2pdma_provider - Get peer-to-peer DMA provider\n- * @pdev: The PCI device to enable P2PDMA for\n- * @bar: BAR index to get provider\n- *\n- * This function gets peer-to-peer DMA provider for a PCI device. The lifetime\n- * of the provider (and of course the MMIO) is bound to the lifetime of the\n- * driver. A driver calling this function must ensure that all references to the\n- * provider, and any DMA mappings created for any MMIO, are all cleaned up\n- * before the driver remove() completes.\n- *\n- * Since P2P is almost always shared with a second driver this means some system\n- * to notify, invalidate and revoke the MMIO's DMA must be in place to use this\n- * function. For example a revoke can be built using DMABUF.\n- */\n-struct p2pdma_provider *pcim_p2pdma_provider(struct pci_dev *pdev, int bar)\n-{\n-\tstruct pci_p2pdma *p2p;\n-\n-\tif (!(pci_resource_flags(pdev, bar) \u0026 IORESOURCE_MEM))\n-\t\treturn NULL;\n-\n-\tp2p = rcu_dereference_protected(pdev-\u003ep2pdma, 1);\n-\tif (WARN_ON(!p2p))\n-\t\t/* Someone forgot to call to pcim_p2pdma_init() before */\n-\t\treturn NULL;\n-\n-\treturn \u0026p2p-\u003emem[bar];\n-}\n-EXPORT_SYMBOL_GPL(pcim_p2pdma_provider);\n-\n static int pci_p2pdma_setup_pool(struct pci_dev *pdev)\n {\n \tstruct pci_p2pdma *p2pdma;\n@@ -932,8 +833,8 @@ void *pci_alloc_p2pmem(struct pci_dev *pdev, size_t size)\n \tstruct pci_p2pdma *p2pdma;\n \n \t/*\n-\t * Pairs with synchronize_rcu() in pci_p2pdma_release() to\n-\t * ensure pdev-\u003ep2pdma is non-NULL for the duration of the\n+\t * Pairs with synchronize_rcu() in pci_p2pdma_release_pool()\n+\t * to ensure pdev-\u003ep2pdma is non-NULL for the duration of the\n \t * read-lock.\n \t */\n \trcu_read_lock();\ndiff --git a/drivers/pci/p2pdma.h b/drivers/pci/p2pdma.h\nnew file mode 100644\nindex 00000000000000..94638380998149\n--- /dev/null\n+++ b/drivers/pci/p2pdma.h\n@@ -0,0 +1,29 @@\n+/* SPDX-License-Identifier: GPL-2.0 */\n+/*\n+ * PCI peer-to-peer DMA support.\n+ */\n+\n+#ifndef _PCI_P2PDMA_H\n+#define _PCI_P2PDMA_H\n+\n+#include \u003clinux/genalloc.h\u003e\n+#include \u003clinux/pci-p2pdma.h\u003e\n+#include \u003clinux/xarray.h\u003e\n+\n+struct pci_p2pdma {\n+\tstruct gen_pool *pool;\n+\tbool p2pmem_published;\n+\tstruct xarray map_types;\n+\tstruct p2pdma_provider mem[PCI_STD_NUM_BARS];\n+};\n+\n+#ifdef CONFIG_PCI_P2PDMA\n+void pci_p2pdma_release_pool(struct pci_dev *pdev, struct pci_p2pdma *p2pdma);\n+#else\n+static inline void pci_p2pdma_release_pool(struct pci_dev *pdev,\n+\t\t\t\t\t   struct pci_p2pdma *p2pdma)\n+{\n+}\n+#endif\n+\n+#endif\ndiff --git a/drivers/pci/p2pdma_core.c b/drivers/pci/p2pdma_core.c\nnew file mode 100644\nindex 00000000000000..bb2138bf2bc7b7\n--- /dev/null\n+++ b/drivers/pci/p2pdma_core.c\n@@ -0,0 +1,118 @@\n+// SPDX-License-Identifier: GPL-2.0\n+/*\n+ * PCI peer-to-peer DMA support core, providing a bare-bones\n+ * pcim_p2pdma_provider() interface to drivers even if full P2PDMA\n+ * isn't present.  The full P2PDMA feature is in p2pdma.c (see\n+ * CONFIG_PCI_P2PDMA).\n+ *\n+ * Copyright (c) 2016-2018, Logan Gunthorpe\n+ * Copyright (c) 2016-2017, Microsemi Corporation\n+ * Copyright (c) 2017, Christoph Hellwig\n+ * Copyright (c) 2018, Eideticom Inc.\n+ */\n+\n+#define pr_fmt(fmt) \"pci-p2pdma: \" fmt\n+#include \u003clinux/ctype.h\u003e\n+#include \u003clinux/genalloc.h\u003e\n+#include \u003clinux/memremap.h\u003e\n+#include \u003clinux/pci-p2pdma.h\u003e\n+#include \u003clinux/xarray.h\u003e\n+\n+#include \"p2pdma.h\"\n+\n+static void pci_p2pdma_release(void *data)\n+{\n+\tstruct pci_dev *pdev = data;\n+\tstruct pci_p2pdma *p2pdma;\n+\n+\tp2pdma = rcu_dereference_protected(pdev-\u003ep2pdma, 1);\n+\tif (!p2pdma)\n+\t\treturn;\n+\n+\t/* Flush and disable pci_alloc_p2p_mem() */\n+\tpdev-\u003ep2pdma = NULL;\n+\tpci_p2pdma_release_pool(pdev, p2pdma);\n+\txa_destroy(\u0026p2pdma-\u003emap_types);\n+}\n+\n+/**\n+ * pcim_p2pdma_init - Initialise peer-to-peer DMA providers\n+ * @pdev: The PCI device to enable P2PDMA for\n+ *\n+ * This function initializes the peer-to-peer DMA infrastructure\n+ * for a PCI device. It allocates and sets up the necessary data\n+ * structures to support P2PDMA operations, including mapping type\n+ * tracking.\n+ */\n+int pcim_p2pdma_init(struct pci_dev *pdev)\n+{\n+\tstruct pci_p2pdma *p2p;\n+\tint i, ret;\n+\n+\tp2p = rcu_dereference_protected(pdev-\u003ep2pdma, 1);\n+\tif (p2p)\n+\t\treturn 0;\n+\n+\tp2p = devm_kzalloc(\u0026pdev-\u003edev, sizeof(*p2p), GFP_KERNEL);\n+\tif (!p2p)\n+\t\treturn -ENOMEM;\n+\n+\txa_init(\u0026p2p-\u003emap_types);\n+\t/*\n+\t * Iterate over all standard PCI BARs and record only those that\n+\t * correspond to MMIO regions. Skip non-memory resources (e.g. I/O\n+\t * port BARs) since they cannot be used for peer-to-peer (P2P)\n+\t * transactions.\n+\t */\n+\tfor (i = 0; i \u003c PCI_STD_NUM_BARS; i++) {\n+\t\tif (!(pci_resource_flags(pdev, i) \u0026 IORESOURCE_MEM))\n+\t\t\tcontinue;\n+\n+\t\tp2p-\u003emem[i].owner = \u0026pdev-\u003edev;\n+\t\tp2p-\u003emem[i].bus_offset =\n+\t\t\tpci_bus_address(pdev, i) - pci_resource_start(pdev, i);\n+\t}\n+\n+\tret = devm_add_action_or_reset(\u0026pdev-\u003edev, pci_p2pdma_release, pdev);\n+\tif (ret)\n+\t\tgoto out_p2p;\n+\n+\trcu_assign_pointer(pdev-\u003ep2pdma, p2p);\n+\treturn 0;\n+\n+out_p2p:\n+\tdevm_kfree(\u0026pdev-\u003edev, p2p);\n+\treturn ret;\n+}\n+EXPORT_SYMBOL_GPL(pcim_p2pdma_init);\n+\n+/**\n+ * pcim_p2pdma_provider - Get peer-to-peer DMA provider\n+ * @pdev: The PCI device to enable P2PDMA for\n+ * @bar: BAR index to get provider\n+ *\n+ * This function gets peer-to-peer DMA provider for a PCI device. The lifetime\n+ * of the provider (and of course the MMIO) is bound to the lifetime of the\n+ * driver. A driver calling this function must ensure that all references to the\n+ * provider, and any DMA mappings created for any MMIO, are all cleaned up\n+ * before the driver remove() completes.\n+ *\n+ * Since P2P is almost always shared with a second driver this means some system\n+ * to notify, invalidate and revoke the MMIO's DMA must be in place to use this\n+ * function. For example a revoke can be built using DMABUF.\n+ */\n+struct p2pdma_provider *pcim_p2pdma_provider(struct pci_dev *pdev, int bar)\n+{\n+\tstruct pci_p2pdma *p2p;\n+\n+\tif (!(pci_resource_flags(pdev, bar) \u0026 IORESOURCE_MEM))\n+\t\treturn NULL;\n+\n+\tp2p = rcu_dereference_protected(pdev-\u003ep2pdma, 1);\n+\tif (WARN_ON(!p2p))\n+\t\t/* Someone forgot to call to pcim_p2pdma_init() before */\n+\t\treturn NULL;\n+\n+\treturn \u0026p2p-\u003emem[bar];\n+}\n+EXPORT_SYMBOL_GPL(pcim_p2pdma_provider);\ndiff --git a/drivers/vfio/pci/Kconfig b/drivers/vfio/pci/Kconfig\nindex 296bf01e185eca..67a2ae1fbc04ba 100644\n--- a/drivers/vfio/pci/Kconfig\n+++ b/drivers/vfio/pci/Kconfig\n@@ -6,6 +6,8 @@ config VFIO_PCI_CORE\n \ttristate\n \tselect VFIO_VIRQFD\n \tselect IRQ_BYPASS_MANAGER\n+\tselect PCI_P2PDMA_CORE\n+\tselect DMA_SHARED_BUFFER\n \n config VFIO_PCI_INTX\n \tdef_bool y if !S390\n@@ -56,7 +58,8 @@ config VFIO_PCI_ZDEV_KVM\n \t  To enable s390x KVM vfio-pci extensions, say Y.\n \n config VFIO_PCI_DMABUF\n-\tdef_bool y if VFIO_PCI_CORE \u0026\u0026 PCI_P2PDMA \u0026\u0026 DMA_SHARED_BUFFER\n+\tdef_bool y if PCI_P2PDMA\n+\tdepends on VFIO_PCI_CORE\n \n source \"drivers/vfio/pci/mlx5/Kconfig\"\n \ndiff --git a/drivers/vfio/pci/Makefile b/drivers/vfio/pci/Makefile\nindex 6138f1bf241df0..881452ea89be0c 100644\n--- a/drivers/vfio/pci/Makefile\n+++ b/drivers/vfio/pci/Makefile\n@@ -1,8 +1,7 @@\n # SPDX-License-Identifier: GPL-2.0-only\n \n-vfio-pci-core-y := vfio_pci_core.o vfio_pci_intrs.o vfio_pci_rdwr.o vfio_pci_config.o\n+vfio-pci-core-y := vfio_pci_core.o vfio_pci_intrs.o vfio_pci_rdwr.o vfio_pci_config.o vfio_pci_dmabuf.o\n vfio-pci-core-$(CONFIG_VFIO_PCI_ZDEV_KVM) += vfio_pci_zdev.o\n-vfio-pci-core-$(CONFIG_VFIO_PCI_DMABUF) += vfio_pci_dmabuf.o\n obj-$(CONFIG_VFIO_PCI_CORE) += vfio-pci-core.o\n \n vfio-pci-y := vfio_pci.o\ndiff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c\nindex bb121f635b9f7f..d9f26b0ce991d9 100644\n--- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c\n+++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c\n@@ -1725,6 +1725,14 @@ static int hisi_acc_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device\n \tif (ret)\n \t\tgoto out_put_vdev;\n \n+\t/*\n+\t * hisi_acc_vfio_pci_mmap() calls down to\n+\t * vfio_pci_core_mmap(), so BAR mappings are still\n+\t * DMABUF-backed.  They don't require a zap on revoke, so opt\n+\t * out:\n+\t */\n+\thisi_acc_vdev-\u003ecore_device.zap_bars_on_revoke = false;\n+\n \thisi_acc_vfio_debug_init(hisi_acc_vdev);\n \treturn 0;\n \ndiff --git a/drivers/vfio/pci/vfio_pci_config.c b/drivers/vfio/pci/vfio_pci_config.c\nindex a10ed733f0e3af..bc6ccb2e135c3c 100644\n--- a/drivers/vfio/pci/vfio_pci_config.c\n+++ b/drivers/vfio/pci/vfio_pci_config.c\n@@ -590,12 +590,10 @@ static int vfio_basic_config_write(struct vfio_pci_core_device *vdev, int pos,\n \t\tvirt_mem = !!(le16_to_cpu(*virt_cmd) \u0026 PCI_COMMAND_MEMORY);\n \t\tnew_mem = !!(new_cmd \u0026 PCI_COMMAND_MEMORY);\n \n-\t\tif (!new_mem) {\n-\t\t\tvfio_pci_zap_and_down_write_memory_lock(vdev);\n-\t\t\tvfio_pci_dma_buf_move(vdev, true);\n-\t\t} else {\n+\t\tif (!new_mem)\n+\t\t\tvfio_pci_lock_revoke_bars(vdev);\n+\t\telse\n \t\t\tdown_write(\u0026vdev-\u003ememory_lock);\n-\t\t}\n \n \t\t/*\n \t\t * If the user is writing mem/io enable (new_mem/io) and we\n@@ -631,7 +629,7 @@ static int vfio_basic_config_write(struct vfio_pci_core_device *vdev, int pos,\n \t\t*virt_cmd |= cpu_to_le16(new_cmd \u0026 mask);\n \n \t\tif (__vfio_pci_memory_enabled(vdev))\n-\t\t\tvfio_pci_dma_buf_move(vdev, false);\n+\t\t\tvfio_pci_unrevoke_bars(vdev);\n \t\tup_write(\u0026vdev-\u003ememory_lock);\n \t}\n \n@@ -712,16 +710,14 @@ static int __init init_pci_cap_basic_perm(struct perm_bits *perm)\n static void vfio_lock_and_set_power_state(struct vfio_pci_core_device *vdev,\n \t\t\t\t\t  pci_power_t state)\n {\n-\tif (state \u003e= PCI_D3hot) {\n-\t\tvfio_pci_zap_and_down_write_memory_lock(vdev);\n-\t\tvfio_pci_dma_buf_move(vdev, true);\n-\t} else {\n+\tif (state \u003e= PCI_D3hot)\n+\t\tvfio_pci_lock_revoke_bars(vdev);\n+\telse\n \t\tdown_write(\u0026vdev-\u003ememory_lock);\n-\t}\n \n \tvfio_pci_set_power_state(vdev, state);\n \tif (__vfio_pci_memory_enabled(vdev))\n-\t\tvfio_pci_dma_buf_move(vdev, false);\n+\t\tvfio_pci_unrevoke_bars(vdev);\n \tup_write(\u0026vdev-\u003ememory_lock);\n }\n \n@@ -908,11 +904,10 @@ static int vfio_exp_config_write(struct vfio_pci_core_device *vdev, int pos,\n \t\t\t\t\t\t \u0026cap);\n \n \t\tif (!ret \u0026\u0026 (cap \u0026 PCI_EXP_DEVCAP_FLR)) {\n-\t\t\tvfio_pci_zap_and_down_write_memory_lock(vdev);\n-\t\t\tvfio_pci_dma_buf_move(vdev, true);\n+\t\t\tvfio_pci_lock_revoke_bars(vdev);\n \t\t\tpci_try_reset_function(vdev-\u003epdev);\n \t\t\tif (__vfio_pci_memory_enabled(vdev))\n-\t\t\t\tvfio_pci_dma_buf_move(vdev, false);\n+\t\t\t\tvfio_pci_unrevoke_bars(vdev);\n \t\t\tup_write(\u0026vdev-\u003ememory_lock);\n \t\t}\n \t}\n@@ -993,11 +988,10 @@ static int vfio_af_config_write(struct vfio_pci_core_device *vdev, int pos,\n \t\t\t\t\t\t\u0026cap);\n \n \t\tif (!ret \u0026\u0026 (cap \u0026 PCI_AF_CAP_FLR) \u0026\u0026 (cap \u0026 PCI_AF_CAP_TP)) {\n-\t\t\tvfio_pci_zap_and_down_write_memory_lock(vdev);\n-\t\t\tvfio_pci_dma_buf_move(vdev, true);\n+\t\t\tvfio_pci_lock_revoke_bars(vdev);\n \t\t\tpci_try_reset_function(vdev-\u003epdev);\n \t\t\tif (__vfio_pci_memory_enabled(vdev))\n-\t\t\t\tvfio_pci_dma_buf_move(vdev, false);\n+\t\t\t\tvfio_pci_unrevoke_bars(vdev);\n \t\t\tup_write(\u0026vdev-\u003ememory_lock);\n \t\t}\n \t}\ndiff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c\nindex 050e7542952ed1..0059c176d62706 100644\n--- a/drivers/vfio/pci/vfio_pci_core.c\n+++ b/drivers/vfio/pci/vfio_pci_core.c\n@@ -12,6 +12,8 @@\n \n #include \u003clinux/aperture.h\u003e\n #include \u003clinux/device.h\u003e\n+#include \u003clinux/dma-buf.h\u003e\n+#include \u003clinux/dma-resv.h\u003e\n #include \u003clinux/eventfd.h\u003e\n #include \u003clinux/file.h\u003e\n #include \u003clinux/interrupt.h\u003e\n@@ -319,8 +321,7 @@ static int vfio_pci_runtime_pm_entry(struct vfio_pci_core_device *vdev,\n \t * The vdev power related flags are protected with 'memory_lock'\n \t * semaphore.\n \t */\n-\tvfio_pci_zap_and_down_write_memory_lock(vdev);\n-\tvfio_pci_dma_buf_move(vdev, true);\n+\tvfio_pci_lock_revoke_bars(vdev);\n \n \tif (vdev-\u003epm_runtime_engaged) {\n \t\tup_write(\u0026vdev-\u003ememory_lock);\n@@ -406,7 +407,7 @@ static void vfio_pci_runtime_pm_exit(struct vfio_pci_core_device *vdev)\n \tdown_write(\u0026vdev-\u003ememory_lock);\n \t__vfio_pci_runtime_pm_exit(vdev);\n \tif (__vfio_pci_memory_enabled(vdev))\n-\t\tvfio_pci_dma_buf_move(vdev, false);\n+\t\tvfio_pci_unrevoke_bars(vdev);\n \tup_write(\u0026vdev-\u003ememory_lock);\n }\n \n@@ -1256,6 +1257,8 @@ static int vfio_pci_ioctl_set_irqs(struct vfio_pci_core_device *vdev,\n \treturn ret;\n }\n \n+static void vfio_pci_revoke_bars(struct vfio_pci_core_device *vdev);\n+\n static int vfio_pci_ioctl_reset(struct vfio_pci_core_device *vdev,\n \t\t\t\tvoid __user *arg)\n {\n@@ -1264,7 +1267,7 @@ static int vfio_pci_ioctl_reset(struct vfio_pci_core_device *vdev,\n \tif (!vdev-\u003ereset_works)\n \t\treturn -EINVAL;\n \n-\tvfio_pci_zap_and_down_write_memory_lock(vdev);\n+\tdown_write(\u0026vdev-\u003ememory_lock);\n \n \t/*\n \t * This function can be invoked while the power state is non-D0. If\n@@ -1274,13 +1277,18 @@ static int vfio_pci_ioctl_reset(struct vfio_pci_core_device *vdev,\n \t * have NoSoftRst-, the reset function can cause the PCI config space\n \t * reset without restoring the original state (saved locally in\n \t * 'vdev-\u003epm_save').\n+\t *\n+\t * The zap is done after making the device accessible in D0,\n+\t * because a DMABUF importer could access the device as part\n+\t * of its revocation cleanup.\n \t */\n \tvfio_pci_set_power_state(vdev, PCI_D0);\n \n-\tvfio_pci_dma_buf_move(vdev, true);\n+\tvfio_pci_revoke_bars(vdev);\n+\n \tret = pci_try_reset_function(vdev-\u003epdev);\n \tif (__vfio_pci_memory_enabled(vdev))\n-\t\tvfio_pci_dma_buf_move(vdev, false);\n+\t\tvfio_pci_unrevoke_bars(vdev);\n \tup_write(\u0026vdev-\u003ememory_lock);\n \n \treturn ret;\n@@ -1569,6 +1577,10 @@ int vfio_pci_core_ioctl_feature(struct vfio_device *device, u32 flags,\n \t\treturn vfio_pci_core_feature_token(vdev, flags, arg, argsz);\n \tcase VFIO_DEVICE_FEATURE_DMA_BUF:\n \t\treturn vfio_pci_core_feature_dma_buf(vdev, flags, arg, argsz);\n+\tcase VFIO_DEVICE_FEATURE_DMA_BUF_REVOKE:\n+\t\treturn vfio_pci_core_feature_dma_buf_revoke(vdev, flags, arg, argsz);\n+\tcase VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR:\n+\t\treturn vfio_pci_core_feature_dma_buf_memattr(vdev, flags, arg, argsz);\n \tdefault:\n \t\treturn -ENOTTY;\n \t}\n@@ -1648,20 +1660,37 @@ ssize_t vfio_pci_core_write(struct vfio_device *core_vdev, const char __user *bu\n }\n EXPORT_SYMBOL_GPL(vfio_pci_core_write);\n \n-static void vfio_pci_zap_bars(struct vfio_pci_core_device *vdev)\n+static void vfio_pci_revoke_bars(struct vfio_pci_core_device *vdev)\n {\n-\tstruct vfio_device *core_vdev = \u0026vdev-\u003evdev;\n-\tloff_t start = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR0_REGION_INDEX);\n-\tloff_t end = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_ROM_REGION_INDEX);\n-\tloff_t len = end - start;\n+\tlockdep_assert_held_write(\u0026vdev-\u003ememory_lock);\n+\tvfio_pci_dma_buf_move(vdev, true);\n \n-\tunmap_mapping_range(core_vdev-\u003einode-\u003ei_mapping, start, len, true);\n+\t/*\n+\t * If a driver could possibly create BAR mappings in the\n+\t * vdev's address_space, do an additional zap on revoke.  See\n+\t * vfio_pci_core_register_device().\n+\t */\n+\tif (vdev-\u003ezap_bars_on_revoke) {\n+\t\tstruct vfio_device *core_vdev = \u0026vdev-\u003evdev;\n+\t\tloff_t start = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR0_REGION_INDEX);\n+\t\tloff_t end = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_ROM_REGION_INDEX);\n+\t\tloff_t len = end - start;\n+\n+\t\tunmap_mapping_range(core_vdev-\u003einode-\u003ei_mapping,\n+\t\t\t\t    start, len, true);\n+\t}\n }\n \n-void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device *vdev)\n+void vfio_pci_lock_revoke_bars(struct vfio_pci_core_device *vdev)\n {\n \tdown_write(\u0026vdev-\u003ememory_lock);\n-\tvfio_pci_zap_bars(vdev);\n+\tvfio_pci_revoke_bars(vdev);\n+}\n+\n+void vfio_pci_unrevoke_bars(struct vfio_pci_core_device *vdev)\n+{\n+\tlockdep_assert_held_write(\u0026vdev-\u003ememory_lock);\n+\tvfio_pci_dma_buf_move(vdev, false);\n }\n \n u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device *vdev)\n@@ -1683,18 +1712,6 @@ void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device *vdev, u16 c\n \tup_write(\u0026vdev-\u003ememory_lock);\n }\n \n-static unsigned long vma_to_pfn(struct vm_area_struct *vma)\n-{\n-\tstruct vfio_pci_core_device *vdev = vma-\u003evm_private_data;\n-\tint index = vma-\u003evm_pgoff \u003e\u003e (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);\n-\tu64 pgoff;\n-\n-\tpgoff = vma-\u003evm_pgoff \u0026\n-\t\t((1U \u003c\u003c (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);\n-\n-\treturn (pci_resource_start(vdev-\u003epdev, index) \u003e\u003e PAGE_SHIFT) + pgoff;\n-}\n-\n vm_fault_t vfio_pci_vmf_insert_pfn(struct vfio_pci_core_device *vdev,\n \t\t\t\t   struct vm_fault *vmf,\n \t\t\t\t   unsigned long pfn,\n@@ -1722,24 +1739,103 @@ static vm_fault_t vfio_pci_mmap_huge_fault(struct vm_fault *vmf,\n \t\t\t\t\t   unsigned int order)\n {\n \tstruct vm_area_struct *vma = vmf-\u003evma;\n-\tstruct vfio_pci_core_device *vdev = vma-\u003evm_private_data;\n-\tunsigned long addr = vmf-\u003eaddress \u0026 ~((PAGE_SIZE \u003c\u003c order) - 1);\n-\tunsigned long pgoff = (addr - vma-\u003evm_start) \u003e\u003e PAGE_SHIFT;\n-\tunsigned long pfn = vma_to_pfn(vma) + pgoff;\n-\tvm_fault_t ret = VM_FAULT_FALLBACK;\n-\n-\tif (is_aligned_for_order(vma, addr, pfn, order)) {\n-\t\tscoped_guard(rwsem_read, \u0026vdev-\u003ememory_lock)\n-\t\t\tret = vfio_pci_vmf_insert_pfn(vdev, vmf, pfn, order);\n+\tstruct vfio_pci_dma_buf *priv = vma-\u003evm_private_data;\n+\tstruct vfio_pci_core_device *vdev;\n+\tunsigned long pfn = 0;\n+\tvm_fault_t ret = VM_FAULT_SIGBUS;\n+\n+\t/*\n+\t * The only thing this can rely on is that the DMABUF relating\n+\t * to the VMA's vm_file exists (priv).\n+\t *\n+\t * A DMABUF for a VFIO device fd mmap() holds a reference to\n+\t * the original VFIO device fd, but an explicitly-exported\n+\t * DMABUF does not.  The original fd might have closed,\n+\t * meaning this fault can race with\n+\t * vfio_pci_dma_buf_cleanup(), meaning the buffer could have\n+\t * been revoked (in which case priv-\u003evdev might be NULL), and\n+\t * the VFIO device registration might have been dropped.\n+\t *\n+\t * With the goal of taking vdev-\u003ememory_lock in a world where\n+\t * vdev might not still exist:\n+\t *\n+\t * 1. Take the resv lock on the DMABUF:\n+\t *  - If racing cleanup got in first, the buffer is revoked;\n+\t *    stop/exit if so.\n+\t *  - If we got in first, the buffer is not revoked so vdev is\n+\t *    non-NULL, accessible, and cleanup _has not yet put the\n+\t *    VFIO device registration_.  So, the device refcount must\n+\t *    be \u003e0.\n+\t *\n+\t * 2. Take vfio_device registration (refcount guaranteed \u003e0\n+\t *    hereafter).\n+\t *\n+\t * 3. Unlock the DMABUF's resv lock:\n+\t *  - A racing cleanup can now complete.\n+\t *  - But, the device refcount \u003e0, meaning the vfio_device\n+\t *    (and vfio_pcie_core device vdev) have not yet been\n+\t *    freed.  vdev is accessible, even if the DMABUF has been\n+\t *    revoked or cleanup has happened, because\n+\t *    vfio_unregister_group_dev() can't complete.\n+\t *\n+\t * 4. Take the vdev-\u003ememory_lock\n+\t *  - Either the DMABUF is usable, or has been cleaned up.\n+\t *    Whichever, it can no longer change under us.\n+\t *  - Test the DMABUF revocation status again: if it was\n+\t *    revoked between 1 and 4 return a SIGBUS. Otherwise,\n+\t *    return a PFN.\n+\t *  - It's not necessary to also take the resv lock, because\n+\t *    the status/vdev can't change while memory_lock is held.\n+\t *\n+\t * 5. Unlock, done.\n+\t */\n+\n+\tdma_resv_lock(priv-\u003edmabuf-\u003eresv, NULL);\n+\n+\tif (priv-\u003estatus != VFIO_PCI_DMABUF_OK) {\n+\t\tpr_debug_ratelimited(\"%s VA 0x%lx, pgoff 0x%lx: DMABUF revoked/cleaned up\\n\",\n+\t\t\t\t     __func__, vmf-\u003eaddress, vma-\u003evm_pgoff);\n+\t\tdma_resv_unlock(priv-\u003edmabuf-\u003eresv);\n+\t\treturn VM_FAULT_SIGBUS;\n+\t}\n+\n+\t/* If the buffer isn't revoked, vdev is valid */\n+\tvdev = priv-\u003evdev;\n+\n+\tif (!vfio_device_try_get_registration(\u0026vdev-\u003evdev)) {\n+\t\t/*\n+\t\t * If vdev != NULL (above), the registration should\n+\t\t * already be \u003e0 and so this try_get should never\n+\t\t * fail.\n+\t\t */\n+\t\tdev_warn(\u0026vdev-\u003epdev-\u003edev, \"%s: Unexpected registration failure\\n\",\n+\t\t\t __func__);\n+\t\tdma_resv_unlock(priv-\u003edmabuf-\u003eresv);\n+\t\treturn VM_FAULT_SIGBUS;\n+\t}\n+\tdma_resv_unlock(priv-\u003edmabuf-\u003eresv);\n+\n+\tscoped_guard(rwsem_read, \u0026vdev-\u003ememory_lock) {\n+\t\t/* Revocation status must be re-read, under memory_lock */\n+\t\tif (priv-\u003estatus == VFIO_PCI_DMABUF_OK) {\n+\t\t\tint pres = vfio_pci_dma_buf_find_pfn(priv, vma,\n+\t\t\t\t\t\t\t     vmf-\u003eaddress,\n+\t\t\t\t\t\t\t     order, \u0026pfn);\n+\n+\t\t\tif (pres == 0)\n+\t\t\t\tret = vfio_pci_vmf_insert_pfn(vdev, vmf,\n+\t\t\t\t\t\t\t      pfn, order);\n+\t\t\telse if (pres == -EAGAIN)\n+\t\t\t\tret = VM_FAULT_FALLBACK;\n+\t\t}\n \t}\n \n \tdev_dbg_ratelimited(\u0026vdev-\u003epdev-\u003edev,\n-\t\t\t   \"%s(,order = %d) BAR %ld page offset 0x%lx: 0x%x\\n\",\n-\t\t\t    __func__, order,\n-\t\t\t    vma-\u003evm_pgoff \u003e\u003e\n-\t\t\t\t(VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT),\n-\t\t\t    pgoff, (unsigned int)ret);\n+\t\t\t    \"%s(order = %d) PFN 0x%lx, VA 0x%lx, pgoff 0x%lx: 0x%x\\n\",\n+\t\t\t    __func__, order, pfn, vmf-\u003eaddress,\n+\t\t\t    vma-\u003evm_pgoff, (unsigned int)ret);\n \n+\tvfio_device_put_registration(\u0026vdev-\u003evdev);\n \treturn ret;\n }\n \n@@ -1755,6 +1851,11 @@ static const struct vm_operations_struct vfio_pci_mmap_ops = {\n #endif\n };\n \n+void vfio_pci_set_vma_ops(struct vm_area_struct *vma)\n+{\n+\tvma-\u003evm_ops = \u0026vfio_pci_mmap_ops;\n+}\n+\n int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma)\n {\n \tstruct vfio_pci_core_device *vdev =\n@@ -1803,7 +1904,12 @@ int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma\n \tif (ret)\n \t\treturn ret;\n \n-\tvma-\u003evm_private_data = vdev;\n+\tret = vfio_pci_core_mmap_prep_dmabuf(vdev, vma,\n+\t\t\t\t\t     pci_resource_start(pdev, index),\n+\t\t\t\t\t     req_len, index);\n+\tif (ret)\n+\t\treturn ret;\n+\n \tvma-\u003evm_page_prot = pgprot_noncached(vma-\u003evm_page_prot);\n \tvma-\u003evm_page_prot = pgprot_decrypted(vma-\u003evm_page_prot);\n \n@@ -2222,6 +2328,17 @@ int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev)\n \tif (ret)\n \t\tgoto out_vf;\n \n+\t/*\n+\t * If a driver overrides .mmap, it has to be assumed that it\n+\t * might not use the DMABUF-backed core mmap; this flag\n+\t * enables a zap at revoke time.  Drivers can opt out by\n+\t * clearing this flag after registration, e.g. if their .mmap\n+\t * override calls down to vfio_pci_core_mmap() so that maps\n+\t * are DMABUF-backed.\n+\t */\n+\tif (vdev-\u003evdev.ops-\u003emmap != vfio_pci_core_mmap)\n+\t\tvdev-\u003ezap_bars_on_revoke = true;\n+\n \tvfio_pci_probe_power_state(vdev);\n \n \t/*\n@@ -2497,9 +2614,10 @@ static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set,\n \t\t}\n \n \t\t/*\n-\t\t * Take the memory write lock for each device and zap BAR\n-\t\t * mappings to prevent the user accessing the device while in\n-\t\t * reset.  Locking multiple devices is prone to deadlock,\n+\t\t * Take the memory write lock for each device and\n+\t\t * zap/revoke BAR mappings to prevent the user (or\n+\t\t * peers) accessing the device while in reset.\n+\t\t * Locking multiple devices is prone to deadlock,\n \t\t * runaway and unwind if we hit contention.\n \t\t */\n \t\tif (!down_write_trylock(\u0026vdev-\u003ememory_lock)) {\n@@ -2507,8 +2625,7 @@ static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set,\n \t\t\tbreak;\n \t\t}\n \n-\t\tvfio_pci_dma_buf_move(vdev, true);\n-\t\tvfio_pci_zap_bars(vdev);\n+\t\tvfio_pci_revoke_bars(vdev);\n \t}\n \n \tif (!list_entry_is_head(vdev,\n@@ -2538,7 +2655,7 @@ static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set,\n \tlist_for_each_entry_from_reverse(vdev, \u0026dev_set-\u003edevice_list,\n \t\t\t\t\t vdev.dev_set_list) {\n \t\tif (vdev-\u003evdev.open_count \u0026\u0026 __vfio_pci_memory_enabled(vdev))\n-\t\t\tvfio_pci_dma_buf_move(vdev, false);\n+\t\t\tvfio_pci_unrevoke_bars(vdev);\n \t\tup_write(\u0026vdev-\u003ememory_lock);\n \t}\n \ndiff --git a/drivers/vfio/pci/vfio_pci_dmabuf.c b/drivers/vfio/pci/vfio_pci_dmabuf.c\nindex 1a177ce7de546a..31fbad93f89b1c 100644\n--- a/drivers/vfio/pci/vfio_pci_dmabuf.c\n+++ b/drivers/vfio/pci/vfio_pci_dmabuf.c\n@@ -4,24 +4,13 @@\n #include \u003clinux/dma-buf-mapping.h\u003e\n #include \u003clinux/pci-p2pdma.h\u003e\n #include \u003clinux/dma-resv.h\u003e\n+#include \u003cuapi/linux/dma-buf.h\u003e\n \n #include \"vfio_pci_priv.h\"\n \n MODULE_IMPORT_NS(\"DMA_BUF\");\n \n-struct vfio_pci_dma_buf {\n-\tstruct dma_buf *dmabuf;\n-\tstruct vfio_pci_core_device *vdev;\n-\tstruct list_head dmabufs_elm;\n-\tsize_t size;\n-\tstruct phys_vec *phys_vec;\n-\tstruct p2pdma_provider *provider;\n-\tu32 nr_ranges;\n-\tstruct kref kref;\n-\tstruct completion comp;\n-\tu8 revoked : 1;\n-};\n-\n+#ifdef CONFIG_VFIO_PCI_DMABUF\n static int vfio_pci_dma_buf_attach(struct dma_buf *dmabuf,\n \t\t\t\t   struct dma_buf_attachment *attachment)\n {\n@@ -30,7 +19,7 @@ static int vfio_pci_dma_buf_attach(struct dma_buf *dmabuf,\n \tif (!attachment-\u003epeer2peer)\n \t\treturn -EOPNOTSUPP;\n \n-\tif (priv-\u003erevoked)\n+\tif (priv-\u003estatus != VFIO_PCI_DMABUF_OK)\n \t\treturn -ENODEV;\n \n \tif (!dma_buf_attach_revocable(attachment))\n@@ -39,6 +28,43 @@ static int vfio_pci_dma_buf_attach(struct dma_buf *dmabuf,\n \treturn 0;\n }\n \n+static int vfio_pci_dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)\n+{\n+\tstruct vfio_pci_dma_buf *priv = dmabuf-\u003epriv;\n+\n+\t/*\n+\t * dma_buf_mmap_internal() has asserted that the VMA is\n+\t * contained within the DMABUF size before calling this.\n+\t *\n+\t * Also, if we observe that the buffer is revoked now then\n+\t * refuse the mmap().  This is a belt-and-braces early failure\n+\t * to ease debugging a revoked buffer being used.  Userspace\n+\t * might also race an mmap() against an explicit revocation,\n+\t * or an action doing a temporary revoke; race scenarios are\n+\t * still safe because the fault handler ultimately prevents\n+\t * access to a revoked buffer if it isn't caught here.\n+\t */\n+\tif (READ_ONCE(priv-\u003estatus) != VFIO_PCI_DMABUF_OK)\n+\t\treturn -ENODEV;\n+\tif ((vma-\u003evm_flags \u0026 VM_SHARED) == 0)\n+\t\treturn -EINVAL;\n+\n+\tif (READ_ONCE(priv-\u003ememattr) == VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR_WC)\n+\t\tvma-\u003evm_page_prot = pgprot_writecombine(vma-\u003evm_page_prot);\n+\telse\n+\t\tvma-\u003evm_page_prot = pgprot_noncached(vma-\u003evm_page_prot);\n+\tvma-\u003evm_page_prot = pgprot_decrypted(vma-\u003evm_page_prot);\n+\n+\t/* See comments in vfio_pci_core_mmap() re VM_ALLOW_ANY_UNCACHED. */\n+\tvm_flags_set(vma, VM_ALLOW_ANY_UNCACHED | VM_IO | VM_PFNMAP |\n+\t\t     VM_DONTEXPAND | VM_DONTDUMP);\n+\tvma-\u003evm_private_data = priv;\n+\tvfio_pci_set_vma_ops(vma);\n+\n+\treturn 0;\n+}\n+#endif /* CONFIG_VFIO_PCI_DMABUF */\n+\n static void vfio_pci_dma_buf_done(struct kref *kref)\n {\n \tstruct vfio_pci_dma_buf *priv =\n@@ -56,7 +82,7 @@ vfio_pci_dma_buf_map(struct dma_buf_attachment *attachment,\n \n \tdma_resv_assert_held(priv-\u003edmabuf-\u003eresv);\n \n-\tif (priv-\u003erevoked)\n+\tif (priv-\u003estatus != VFIO_PCI_DMABUF_OK)\n \t\treturn ERR_PTR(-ENODEV);\n \n \tret = dma_buf_phys_vec_to_sgt(attachment, priv-\u003eprovider,\n@@ -95,17 +121,203 @@ static void vfio_pci_dma_buf_release(struct dma_buf *dmabuf)\n \t\tup_write(\u0026priv-\u003evdev-\u003ememory_lock);\n \t\tvfio_device_put_registration(\u0026priv-\u003evdev-\u003evdev);\n \t}\n+\tif (priv-\u003evfile)\n+\t\tfput(priv-\u003evfile);\n \tkfree(priv-\u003ephys_vec);\n \tkfree(priv);\n }\n \n static const struct dma_buf_ops vfio_pci_dmabuf_ops = {\n+#ifdef CONFIG_VFIO_PCI_DMABUF\n \t.attach = vfio_pci_dma_buf_attach,\n+\t.mmap = vfio_pci_dma_buf_mmap,\n+#endif\n \t.map_dma_buf = vfio_pci_dma_buf_map,\n \t.unmap_dma_buf = vfio_pci_dma_buf_unmap,\n \t.release = vfio_pci_dma_buf_release,\n };\n \n+int vfio_pci_dma_buf_find_pfn(struct vfio_pci_dma_buf *priv,\n+\t\t\t      struct vm_area_struct *vma,\n+\t\t\t      unsigned long fault_addr,\n+\t\t\t      unsigned int order,\n+\t\t\t      unsigned long *out_pfn)\n+{\n+\t/*\n+\t * Given a VMA (start, end, pgoffs) and a fault address,\n+\t * search the corresponding DMABUF's phys_vec[] to find the\n+\t * range representing the address's offset into the VMA, and\n+\t * its PFN.\n+\t *\n+\t * The phys_vec[] ranges represent contiguous spans of VAs\n+\t * upwards from the buffer offset 0; the actual PFNs might be\n+\t * in any order, overlap/alias, etc.  Calculate an offset of\n+\t * the desired page given VMA start/pgoff and address, then\n+\t * search upwards from 0 to find which span contains it.\n+\t *\n+\t * On success, a valid PFN for a page sized by 'order' is\n+\t * returned into out_pfn.\n+\t *\n+\t * Failure occurs if:\n+\t * - A hugepage would cross the edge of the VMA,\n+\t * - A hugepage isn't entirely contained within a range\n+\t *   (including where it straddles the boundary between\n+\t *   ranges),\n+\t * - We find a range, but the final PFN isn't aligned to the\n+\t *   requested order.\n+\t *\n+\t * Upon failure, -EAGAIN is returned and the caller is\n+\t * expected to try again with a smaller order, which will\n+\t * eventually succeed (order=0 will always work).\n+\t *\n+\t * It's suboptimal if DMABUFs are created with neighbouring\n+\t * ranges that are physically contiguous, since hugepages\n+\t * can't straddle range boundaries.  (The construction of the\n+\t * ranges should merge them in this case.)\n+\t *\n+\t * Finally, vma_pgoff_adjust is used with a DMABUF created for\n+\t * a VFIO BAR mmap: a BAR mapped with vm_pgoff \u003e 0 creates a\n+\t * DMABUF such that byte 0 of the VMA corresponds to byte 0 of\n+\t * the DMABUF and byte 'vm_pgoff \u003c\u003c PAGE_SHIFT' into the BAR.\n+\t * To avoid double-offsetting in this scenario, subtracting\n+\t * vma_pgoff_adjust from this (non-zero) vm_pgoff generates\n+\t * the effective offset.\n+\t */\n+\n+\tconst unsigned long pagesize = PAGE_SIZE \u003c\u003c order;\n+\tunsigned long vma_off = ((vma-\u003evm_pgoff - priv-\u003evma_pgoff_adjust) \u003c\u003c\n+\t\t\t\t PAGE_SHIFT) \u0026 VFIO_PCI_OFFSET_MASK;\n+\tunsigned long rounded_page_addr = ALIGN_DOWN(fault_addr, pagesize);\n+\tunsigned long rounded_page_end = rounded_page_addr + pagesize;\n+\tunsigned long fault_offset;\n+\tunsigned long fault_offset_end;\n+\tunsigned long range_start_offset = 0;\n+\tunsigned int i;\n+\tint ret;\n+\n+\tif (rounded_page_addr \u003c vma-\u003evm_start || rounded_page_end \u003e vma-\u003evm_end) {\n+\t\tif (order \u003e 0)\n+\t\t\treturn -EAGAIN;\n+\n+\t\t/* A fault address outside of the VMA is absurd. */\n+\t\tdev_warn_ratelimited(\n+\t\t\t\u0026priv-\u003evdev-\u003epdev-\u003edev,\n+\t\t\t\"Fault addr 0x%lx outside VMA 0x%lx-0x%lx\\n\",\n+\t\t\tfault_addr, vma-\u003evm_start, vma-\u003evm_end);\n+\t\treturn -EFAULT;\n+\t}\n+\n+\t/*\n+\t * fault_offset[_end] is the span within the DMABUF\n+\t * corresponding to the faulting page:\n+\t */\n+\tif (unlikely(check_add_overflow(rounded_page_addr - vma-\u003evm_start,\n+\t\t\t\t\tvma_off, \u0026fault_offset) ||\n+\t\t     check_add_overflow(fault_offset, pagesize,\n+\t\t\t\t\t\u0026fault_offset_end)))\n+\t\treturn -EFAULT;\n+\n+\t/*\n+\t * Iterate over ranges in the buffer, summing their lengths:\n+\t * range_start_offset represents the current range's starting\n+\t * offset in the buffer (from 0 upwards).\n+\t *\n+\t * A failure for order == 0 is unexpected, and triggers a\n+\t * fault/warn.\n+\t */\n+\tret = (order == 0) ? -EFAULT : -EAGAIN;\n+\n+\tfor (i = 0; i \u003c priv-\u003enr_ranges; i++) {\n+\t\tsize_t range_len = priv-\u003ephys_vec[i].len;\n+\n+\t\t/* Early exit if range starts after the page end */\n+\t\tif (fault_offset_end \u003c= range_start_offset)\n+\t\t\tbreak;\n+\n+\t\tif (fault_offset \u003e= range_start_offset \u0026\u0026\n+\t\t    fault_offset_end \u003c= range_start_offset + range_len) {\n+\t\t\t/*\n+\t\t\t * The faulting page is wholly contained\n+\t\t\t * within the span represented by this range,\n+\t\t\t * so validate PFN alignment for the order.\n+\t\t\t * The if() condition ensures the pfn\n+\t\t\t * arithmetic won't overflow.\n+\t\t\t */\n+\t\t\tunsigned long pfn =\n+\t\t\t\t((fault_offset - range_start_offset) +\n+\t\t\t\t priv-\u003ephys_vec[i].paddr) \u003e\u003e PAGE_SHIFT;\n+\n+\t\t\tif (IS_ALIGNED(pfn, 1 \u003c\u003c order)) {\n+\t\t\t\t*out_pfn = pfn;\n+\t\t\t\tret = 0;\n+\t\t\t}\n+\t\t\t/*\n+\t\t\t * Else order \u003e 0; EAGAIN retries with smaller\n+\t\t\t * order\n+\t\t\t */\n+\t\t\tbreak;\n+\t\t}\n+\t\trange_start_offset += range_len;\n+\t}\n+\n+\tif (order == 0 \u0026\u0026 ret != 0)\n+\t\t/*\n+\t\t * The address fell outside of the span represented by\n+\t\t * the (concatenated) ranges.  As setup of a mapping\n+\t\t * ensures that the VMA is \u003c= the total size of the\n+\t\t * ranges this should never happen.  If it does, warn\n+\t\t * and SIGBUS.\n+\t\t */\n+\t\tdev_warn_ratelimited(\n+\t\t\t\u0026priv-\u003evdev-\u003epdev-\u003edev,\n+\t\t\t\"No range for addr 0x%lx, order %d: VMA 0x%lx-0x%lx pgoff 0x%lx, %u ranges, size 0x%zx\\n\",\n+\t\t\tfault_addr, order, vma-\u003evm_start, vma-\u003evm_end,\n+\t\t\tvma-\u003evm_pgoff, priv-\u003enr_ranges, priv-\u003esize);\n+\n+\treturn ret;\n+}\n+\n+/*\n+ * Create a DMABUF corresponding to priv, add it to vdev-\u003edmabufs list\n+ * for tracking (meaning cleanup or revocation will zap it), and take\n+ * a vfio_device registration.\n+ */\n+static int vfio_pci_dmabuf_export(struct vfio_pci_core_device *vdev,\n+\t\t\t\t  struct vfio_pci_dma_buf *priv, u32 flags)\n+{\n+\tDEFINE_DMA_BUF_EXPORT_INFO(exp_info);\n+\n+\tif (!vfio_device_try_get_registration(\u0026vdev-\u003evdev))\n+\t\treturn -ENODEV;\n+\n+\texp_info.ops = \u0026vfio_pci_dmabuf_ops;\n+\texp_info.size = priv-\u003esize;\n+\texp_info.flags = flags;\n+\texp_info.priv = priv;\n+\n+\tpriv-\u003edmabuf = dma_buf_export(\u0026exp_info);\n+\tif (IS_ERR(priv-\u003edmabuf)) {\n+\t\tvfio_device_put_registration(\u0026vdev-\u003evdev);\n+\t\treturn PTR_ERR(priv-\u003edmabuf);\n+\t}\n+\n+\tkref_init(\u0026priv-\u003ekref);\n+\tinit_completion(\u0026priv-\u003ecomp);\n+\n+\t/* dma_buf_put() now frees priv */\n+\tINIT_LIST_HEAD(\u0026priv-\u003edmabufs_elm);\n+\tdown_write(\u0026vdev-\u003ememory_lock);\n+\tdma_resv_lock(priv-\u003edmabuf-\u003eresv, NULL);\n+\tpriv-\u003estatus = __vfio_pci_memory_enabled(vdev) ? VFIO_PCI_DMABUF_OK :\n+\t\tVFIO_PCI_DMABUF_TEMP_REVOKED;\n+\tlist_add_tail(\u0026priv-\u003edmabufs_elm, \u0026vdev-\u003edmabufs);\n+\tdma_resv_unlock(priv-\u003edmabuf-\u003eresv);\n+\tup_write(\u0026vdev-\u003ememory_lock);\n+\n+\treturn 0;\n+}\n+\n+#ifdef CONFIG_VFIO_PCI_DMABUF\n /*\n  * This is a temporary \"private interconnect\" between VFIO DMABUF and iommufd.\n  * It allows the two co-operating drivers to exchange the physical address of\n@@ -128,7 +340,7 @@ int vfio_pci_dma_buf_iommufd_map(struct dma_buf_attachment *attachment,\n \t\treturn -EOPNOTSUPP;\n \n \tpriv = attachment-\u003edmabuf-\u003epriv;\n-\tif (priv-\u003erevoked)\n+\tif (priv-\u003estatus != VFIO_PCI_DMABUF_OK)\n \t\treturn -ENODEV;\n \n \t/* More than one range to iommufd will require proper DMABUF support */\n@@ -224,7 +436,6 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,\n {\n \tstruct vfio_device_feature_dma_buf get_dma_buf = {};\n \tstruct vfio_region_dma_range *dma_ranges;\n-\tDEFINE_DMA_BUF_EXPORT_INFO(exp_info);\n \tstruct vfio_pci_dma_buf *priv;\n \tsize_t length;\n \tint ret;\n@@ -274,6 +485,7 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,\n \tpriv-\u003evdev = vdev;\n \tpriv-\u003enr_ranges = get_dma_buf.nr_ranges;\n \tpriv-\u003esize = length;\n+\tpriv-\u003ememattr = VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR_NC;\n \tret = vdev-\u003epci_ops-\u003eget_dmabuf_phys(vdev, \u0026priv-\u003eprovider,\n \t\t\t\t\t     get_dma_buf.region_index,\n \t\t\t\t\t     priv-\u003ephys_vec, dma_ranges,\n@@ -284,34 +496,9 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,\n \tkfree(dma_ranges);\n \tdma_ranges = NULL;\n \n-\tif (!vfio_device_try_get_registration(\u0026vdev-\u003evdev)) {\n-\t\tret = -ENODEV;\n+\tret = vfio_pci_dmabuf_export(vdev, priv, get_dma_buf.open_flags);\n+\tif (ret)\n \t\tgoto err_free_phys;\n-\t}\n-\n-\texp_info.ops = \u0026vfio_pci_dmabuf_ops;\n-\texp_info.size = priv-\u003esize;\n-\texp_info.flags = get_dma_buf.open_flags;\n-\texp_info.priv = priv;\n-\n-\tpriv-\u003edmabuf = dma_buf_export(\u0026exp_info);\n-\tif (IS_ERR(priv-\u003edmabuf)) {\n-\t\tret = PTR_ERR(priv-\u003edmabuf);\n-\t\tgoto err_dev_put;\n-\t}\n-\n-\tkref_init(\u0026priv-\u003ekref);\n-\tinit_completion(\u0026priv-\u003ecomp);\n-\n-\t/* dma_buf_put() now frees priv */\n-\tINIT_LIST_HEAD(\u0026priv-\u003edmabufs_elm);\n-\tdown_write(\u0026vdev-\u003ememory_lock);\n-\tdma_resv_lock(priv-\u003edmabuf-\u003eresv, NULL);\n-\tpriv-\u003erevoked = !__vfio_pci_memory_enabled(vdev);\n-\tlist_add_tail(\u0026priv-\u003edmabufs_elm, \u0026vdev-\u003edmabufs);\n-\tdma_resv_unlock(priv-\u003edmabuf-\u003eresv);\n-\tup_write(\u0026vdev-\u003ememory_lock);\n-\n \t/*\n \t * dma_buf_fd() consumes the reference, when the file closes the dmabuf\n \t * will be released.\n@@ -322,8 +509,6 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,\n \n \treturn ret;\n \n-err_dev_put:\n-\tvfio_device_put_registration(\u0026vdev-\u003evdev);\n err_free_phys:\n \tkfree(priv-\u003ephys_vec);\n err_free_priv:\n@@ -332,47 +517,164 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,\n \tkfree(dma_ranges);\n \treturn ret;\n }\n+#endif /* CONFIG_VFIO_PCI_DMABUF */\n+\n+int vfio_pci_core_mmap_prep_dmabuf(struct vfio_pci_core_device *vdev,\n+\t\t\t\t   struct vm_area_struct *vma,\n+\t\t\t\t   u64 phys_start, u64 req_len,\n+\t\t\t\t   unsigned int res_index)\n+{\n+\tstruct vfio_pci_dma_buf *priv;\n+\tunsigned long vma_pgoff = vma-\u003evm_pgoff \u0026 (VFIO_PCI_OFFSET_MASK \u003e\u003e PAGE_SHIFT);\n+\tchar *bufname;\n+\tint ret;\n+\n+\tpriv = kzalloc_obj(*priv);\n+\tif (!priv)\n+\t\treturn -ENOMEM;\n+\n+\tpriv-\u003ephys_vec = kzalloc_obj(*priv-\u003ephys_vec);\n+\tif (!priv-\u003ephys_vec) {\n+\t\tret = -ENOMEM;\n+\t\tgoto err_free_priv;\n+\t}\n+\n+\tbufname = kasprintf(GFP_KERNEL, \"%s:%s/%x\",\n+\t\t\t    dev_name(\u0026vdev-\u003evdev.device), pci_name(vdev-\u003epdev),\n+\t\t\t    res_index);\n+\n+\tif (!bufname) {\n+\t\tret = -ENOMEM;\n+\t\tgoto err_free_phys;\n+\t}\n+\n+\t/*\n+\t * The DMABUF begins from the mmap()'s BAR offset, i.e. the\n+\t * start of the VMA corresponds to byte 0 of the DMABUF and\n+\t * byte (vma_pgoff \u003c\u003c PAGE_SHIFT) of the BAR.\n+\t *\n+\t * vfio_pci_dma_buf_find_pfn() reverses this offset using\n+\t * vma_pgoff_adjust, so that ultimately a fault's offset from\n+\t * the start of the _VMA_ has a consistent usage whether the\n+\t * VMA originates from an mmap() of the VFIO device here or a\n+\t * direct DMABUF mmap().\n+\t */\n+\tpriv-\u003evdev = vdev;\n+\tpriv-\u003esize = req_len;\n+\tpriv-\u003enr_ranges = 1;\n+\tpriv-\u003evma_pgoff_adjust = vma_pgoff;\n+\tpriv-\u003eprovider = pcim_p2pdma_provider(vdev-\u003epdev, res_index);\n+\tif (!priv-\u003eprovider) {\n+\t\tret = -EINVAL;\n+\t\tgoto err_free_name;\n+\t}\n+\n+\tpriv-\u003ephys_vec[0].paddr = phys_start + ((u64)vma_pgoff \u003c\u003c PAGE_SHIFT);\n+\tpriv-\u003ephys_vec[0].len = priv-\u003esize;\n+\n+\tret = vfio_pci_dmabuf_export(vdev, priv, O_CLOEXEC | O_RDWR);\n+\tif (ret)\n+\t\tgoto err_free_name;\n+\n+\t/*\n+\t * Ownership of the DMABUF file transfers to the VMA so that\n+\t * other users can locate the DMABUF via a VA.  Ownership of\n+\t * the original VFIO device file being mmap()ed transfers to\n+\t * priv, and is put when the DMABUF is released.  This\n+\t * intentionally does not use get_file()/vma_set_file()\n+\t * because the references are already held, and ownership\n+\t * moves.\n+\t */\n+\tpriv-\u003evfile = vma-\u003evm_file;\n+\tvma-\u003evm_file = priv-\u003edmabuf-\u003efile;\n+\tvma-\u003evm_private_data = priv;\n+\n+\tspin_lock(\u0026priv-\u003edmabuf-\u003ename_lock);\n+\tkfree(priv-\u003edmabuf-\u003ename);\n+\tpriv-\u003edmabuf-\u003ename = bufname;\n+\tspin_unlock(\u0026priv-\u003edmabuf-\u003ename_lock);\n+\n+\treturn 0;\n+\n+err_free_name:\n+\tkfree(bufname);\n+err_free_phys:\n+\tkfree(priv-\u003ephys_vec);\n+err_free_priv:\n+\tkfree(priv);\n+\treturn ret;\n+}\n+\n+/* Set the DMABUF's revocation status (OK or temporarily/permanently revoked) */\n+static void vfio_pci_dma_buf_set_status(struct vfio_pci_dma_buf *priv,\n+\t\t\t\t\tenum vfio_pci_dma_buf_status new_status)\n+{\n+\tbool was_revoked;\n+\n+\tlockdep_assert_held_write(\u0026priv-\u003evdev-\u003ememory_lock);\n+\n+\tif (priv-\u003estatus == VFIO_PCI_DMABUF_PERM_REVOKED ||\n+\t    priv-\u003estatus == new_status)\n+\t\treturn;\n+\n+\tdma_resv_lock(priv-\u003edmabuf-\u003eresv, NULL);\n+\twas_revoked = (priv-\u003estatus == VFIO_PCI_DMABUF_TEMP_REVOKED);\n+\n+\tif (new_status != VFIO_PCI_DMABUF_OK) {\n+\t\tpriv-\u003estatus = new_status; /* Temp or permanently revoked */\n+\n+\t\tif (was_revoked) {\n+\t\t\t/*\n+\t\t\t * TEMP_REVOKED is being upgraded to\n+\t\t\t * PERM_REVOKED.  The buffer is already gone,\n+\t\t\t * don't wait on it again.\n+\t\t\t */\n+\t\t\tdma_resv_unlock(priv-\u003edmabuf-\u003eresv);\n+\t\t\treturn;\n+\t\t}\n+\t\tdma_buf_invalidate_mappings(priv-\u003edmabuf);\n+\t\tdma_resv_wait_timeout(priv-\u003edmabuf-\u003eresv,\n+\t\t\t\t      DMA_RESV_USAGE_BOOKKEEP, false,\n+\t\t\t\t      MAX_SCHEDULE_TIMEOUT);\n+\t\tdma_resv_unlock(priv-\u003edmabuf-\u003eresv);\n+\t\tkref_put(\u0026priv-\u003ekref, vfio_pci_dma_buf_done);\n+\t\twait_for_completion(\u0026priv-\u003ecomp);\n+\t\tunmap_mapping_range(priv-\u003edmabuf-\u003efile-\u003ef_mapping,\n+\t\t\t\t    0, priv-\u003esize, true);\n+\t\t/*\n+\t\t * Re-arm the registered kref reference and the\n+\t\t * completion so the post-revoke state matches the\n+\t\t * post-creation state.  An un-revoke followed by a\n+\t\t * new mapping needs the kref to be non-zero before\n+\t\t * kref_get(), and vfio_pci_dma_buf_cleanup()\n+\t\t * delegates its drain back through this revoke\n+\t\t * path on a possibly-already-revoked dma-buf.\n+\t\t */\n+\t\tkref_init(\u0026priv-\u003ekref);\n+\t\treinit_completion(\u0026priv-\u003ecomp);\n+\t} else {\n+\t\tpriv-\u003estatus = VFIO_PCI_DMABUF_OK;\n+\t\tdma_resv_unlock(priv-\u003edmabuf-\u003eresv);\n+\t}\n+}\n \n void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev, bool revoked)\n {\n \tstruct vfio_pci_dma_buf *priv;\n \tstruct vfio_pci_dma_buf *tmp;\n \n+\t/*\n+\t * Holding memory_lock ensures a racing VMA fault observes\n+\t * priv-\u003estatus properly.\n+\t */\n \tlockdep_assert_held_write(\u0026vdev-\u003ememory_lock);\n \n \tlist_for_each_entry_safe(priv, tmp, \u0026vdev-\u003edmabufs, dmabufs_elm) {\n \t\tif (!get_file_active(\u0026priv-\u003edmabuf-\u003efile))\n \t\t\tcontinue;\n-\n-\t\tif (priv-\u003erevoked != revoked) {\n-\t\t\tdma_resv_lock(priv-\u003edmabuf-\u003eresv, NULL);\n-\t\t\tif (revoked)\n-\t\t\t\tpriv-\u003erevoked = true;\n-\t\t\tdma_buf_invalidate_mappings(priv-\u003edmabuf);\n-\t\t\tdma_resv_wait_timeout(priv-\u003edmabuf-\u003eresv,\n-\t\t\t\t\t      DMA_RESV_USAGE_BOOKKEEP, false,\n-\t\t\t\t\t      MAX_SCHEDULE_TIMEOUT);\n-\t\t\tdma_resv_unlock(priv-\u003edmabuf-\u003eresv);\n-\t\t\tif (revoked) {\n-\t\t\t\tkref_put(\u0026priv-\u003ekref, vfio_pci_dma_buf_done);\n-\t\t\t\twait_for_completion(\u0026priv-\u003ecomp);\n-\t\t\t\t/*\n-\t\t\t\t * Re-arm the registered kref reference and the\n-\t\t\t\t * completion so the post-revoke state matches the\n-\t\t\t\t * post-creation state.  An un-revoke followed by a\n-\t\t\t\t * new mapping needs the kref to be non-zero before\n-\t\t\t\t * kref_get(), and vfio_pci_dma_buf_cleanup()\n-\t\t\t\t * delegates its drain back through this revoke\n-\t\t\t\t * path on a possibly-already-revoked dma-buf.\n-\t\t\t\t */\n-\t\t\t\tkref_init(\u0026priv-\u003ekref);\n-\t\t\t\treinit_completion(\u0026priv-\u003ecomp);\n-\t\t\t} else {\n-\t\t\t\tdma_resv_lock(priv-\u003edmabuf-\u003eresv, NULL);\n-\t\t\t\tpriv-\u003erevoked = false;\n-\t\t\t\tdma_resv_unlock(priv-\u003edmabuf-\u003eresv);\n-\t\t\t}\n-\t\t}\n+\t\tvfio_pci_dma_buf_set_status(priv, revoked ?\n+\t\t\t\t\t    VFIO_PCI_DMABUF_TEMP_REVOKED :\n+\t\t\t\t\t    VFIO_PCI_DMABUF_OK);\n \t\tfput(priv-\u003edmabuf-\u003efile);\n \t}\n }\n@@ -404,3 +706,117 @@ void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device *vdev)\n \t}\n \tup_write(\u0026vdev-\u003ememory_lock);\n }\n+\n+#ifdef CONFIG_VFIO_PCI_DMABUF\n+int vfio_pci_core_feature_dma_buf_revoke(\n+\tstruct vfio_pci_core_device *vdev, u32 flags,\n+\tstruct vfio_device_feature_dma_buf_revoke __user *arg,\n+\tsize_t argsz)\n+{\n+\tstruct vfio_device_feature_dma_buf_revoke db_revoke;\n+\tstruct vfio_pci_dma_buf *priv;\n+\tstruct dma_buf *dmabuf;\n+\tint ret;\n+\n+\tif (!vdev-\u003epci_ops || !vdev-\u003epci_ops-\u003eget_dmabuf_phys)\n+\t\treturn -EOPNOTSUPP;\n+\n+\tret = vfio_check_feature(flags, argsz,\n+\t\t\t\t VFIO_DEVICE_FEATURE_SET,\n+\t\t\t\t sizeof(db_revoke));\n+\tif (ret != 1)\n+\t\treturn ret;\n+\n+\tif (copy_from_user(\u0026db_revoke, arg, sizeof(db_revoke)))\n+\t\treturn -EFAULT;\n+\n+\tdmabuf = dma_buf_get(db_revoke.dmabuf_fd);\n+\tif (IS_ERR(dmabuf))\n+\t\treturn PTR_ERR(dmabuf);\n+\n+\tpriv = dmabuf-\u003epriv;\n+\t/*\n+\t * Sanity-check the DMABUF is really a vfio_pci_dma_buf _and_\n+\t * relates to the VFIO device it was provided with.\n+\t *\n+\t * If the DMABUF relates to this vdev then priv-\u003evdev is\n+\t * stable because this open fd prevents cleanup.\n+\t *\n+\t * If it relates to a different vdev, reading priv-\u003evdev might\n+\t * race with a concurrent cleanup on that device.  But if so,\n+\t * it points to a non-matching vdev or NULL and is unusable\n+\t * either way.\n+\t */\n+\tif (dmabuf-\u003eops != \u0026vfio_pci_dmabuf_ops ||\n+\t    READ_ONCE(priv-\u003evdev) != vdev) {\n+\t\tret = -ENODEV;\n+\t\tgoto out_put_buf;\n+\t}\n+\n+\tscoped_guard(rwsem_write, \u0026vdev-\u003ememory_lock) {\n+\t\tif (priv-\u003estatus == VFIO_PCI_DMABUF_PERM_REVOKED) {\n+\t\t\tret = -EBADFD;\n+\t\t} else {\n+\t\t\tvfio_pci_dma_buf_set_status(priv,\n+\t\t\t\t\t\t    VFIO_PCI_DMABUF_PERM_REVOKED);\n+\t\t\tret = 0;\n+\t\t}\n+\t}\n+\n+out_put_buf:\n+\tdma_buf_put(dmabuf);\n+\n+\treturn ret;\n+}\n+\n+int vfio_pci_core_feature_dma_buf_memattr(\n+\tstruct vfio_pci_core_device *vdev, u32 flags,\n+\tstruct vfio_device_feature_dma_buf_memattr __user *arg,\n+\tsize_t argsz)\n+{\n+\tstruct vfio_device_feature_dma_buf_memattr db_attr;\n+\tstruct vfio_pci_dma_buf *priv;\n+\tstruct dma_buf *dmabuf;\n+\tint ret;\n+\n+\tif (!vdev-\u003epci_ops || !vdev-\u003epci_ops-\u003eget_dmabuf_phys)\n+\t\treturn -EOPNOTSUPP;\n+\n+\tret = vfio_check_feature(flags, argsz,\n+\t\t\t\t VFIO_DEVICE_FEATURE_SET,\n+\t\t\t\t sizeof(db_attr));\n+\tif (ret != 1)\n+\t\treturn ret;\n+\n+\tif (copy_from_user(\u0026db_attr, arg, sizeof(db_attr)))\n+\t\treturn -EFAULT;\n+\n+\tdmabuf = dma_buf_get(db_attr.dmabuf_fd);\n+\tif (IS_ERR(dmabuf))\n+\t\treturn PTR_ERR(dmabuf);\n+\n+\t/* Verify DMABUF: see comments in vfio_pci_dma_buf_revoke() */\n+\tpriv = dmabuf-\u003epriv;\n+\tif (dmabuf-\u003eops != \u0026vfio_pci_dmabuf_ops ||\n+\t    READ_ONCE(priv-\u003evdev) != vdev) {\n+\t\tret = -ENODEV;\n+\t\tgoto out_put_buf;\n+\t}\n+\n+\tswitch (db_attr.memattr) {\n+\tcase VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR_NC:\n+\tcase VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR_WC:\n+\t\tWRITE_ONCE(priv-\u003ememattr, db_attr.memattr);\n+\t\tret = 0;\n+\t\tbreak;\n+\n+\tdefault:\n+\t\tret = -ENOENT;\n+\t}\n+\n+out_put_buf:\n+\tdma_buf_put(dmabuf);\n+\n+\treturn ret;\n+}\n+#endif /* CONFIG_VFIO_PCI_DMABUF */\ndiff --git a/drivers/vfio/pci/vfio_pci_priv.h b/drivers/vfio/pci/vfio_pci_priv.h\nindex fca9d0dfac90f8..b3c72df002c448 100644\n--- a/drivers/vfio/pci/vfio_pci_priv.h\n+++ b/drivers/vfio/pci/vfio_pci_priv.h\n@@ -23,6 +23,28 @@ struct vfio_pci_ioeventfd {\n \tbool\t\t\ttest_mem;\n };\n \n+enum vfio_pci_dma_buf_status {\n+\tVFIO_PCI_DMABUF_OK = 0,\n+\tVFIO_PCI_DMABUF_TEMP_REVOKED = 1,\n+\tVFIO_PCI_DMABUF_PERM_REVOKED = 2,\n+};\n+\n+struct vfio_pci_dma_buf {\n+\tstruct dma_buf *dmabuf;\n+\tstruct vfio_pci_core_device *vdev;\n+\tstruct list_head dmabufs_elm;\n+\tsize_t size;\n+\tstruct phys_vec *phys_vec;\n+\tstruct p2pdma_provider *provider;\n+\tstruct file *vfile;\n+\tu32 nr_ranges;\n+\tstruct kref kref;\n+\tstruct completion comp;\n+\tunsigned long vma_pgoff_adjust;\n+\tu32 memattr;\n+\tenum vfio_pci_dma_buf_status status;\n+};\n+\n bool vfio_pci_intx_mask(struct vfio_pci_core_device *vdev);\n void vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev);\n \n@@ -68,7 +90,8 @@ void vfio_config_free(struct vfio_pci_core_device *vdev);\n int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev,\n \t\t\t     pci_power_t state);\n \n-void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device *vdev);\n+void vfio_pci_lock_revoke_bars(struct vfio_pci_core_device *vdev);\n+void vfio_pci_unrevoke_bars(struct vfio_pci_core_device *vdev);\n u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device *vdev);\n void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device *vdev,\n \t\t\t\t\tu16 cmd);\n@@ -114,12 +137,31 @@ static inline bool vfio_pci_is_vga(struct pci_dev *pdev)\n \treturn (pdev-\u003eclass \u003e\u003e 8) == PCI_CLASS_DISPLAY_VGA;\n }\n \n+int vfio_pci_dma_buf_find_pfn(struct vfio_pci_dma_buf *vpdmabuf,\n+\t\t\t      struct vm_area_struct *vma,\n+\t\t\t      unsigned long address,\n+\t\t\t      unsigned int order,\n+\t\t\t      unsigned long *out_pfn);\n+int vfio_pci_core_mmap_prep_dmabuf(struct vfio_pci_core_device *vdev,\n+\t\t\t\t   struct vm_area_struct *vma,\n+\t\t\t\t   u64 phys_start, u64 req_len,\n+\t\t\t\t   unsigned int res_index);\n+void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device *vdev);\n+void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev, bool revoked);\n+void vfio_pci_set_vma_ops(struct vm_area_struct *vma);\n+\n #ifdef CONFIG_VFIO_PCI_DMABUF\n int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,\n \t\t\t\t  struct vfio_device_feature_dma_buf __user *arg,\n \t\t\t\t  size_t argsz);\n-void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device *vdev);\n-void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev, bool revoked);\n+int vfio_pci_core_feature_dma_buf_revoke(\n+\tstruct vfio_pci_core_device *vdev, u32 flags,\n+\tstruct vfio_device_feature_dma_buf_revoke __user *arg,\n+\tsize_t argsz);\n+int vfio_pci_core_feature_dma_buf_memattr(\n+\tstruct vfio_pci_core_device *vdev, u32 flags,\n+\tstruct vfio_device_feature_dma_buf_memattr __user *arg,\n+\tsize_t argsz);\n #else\n static inline int\n vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,\n@@ -128,12 +170,21 @@ vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,\n {\n \treturn -ENOTTY;\n }\n-static inline void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device *vdev)\n+\n+static inline int vfio_pci_core_feature_dma_buf_revoke(\n+\tstruct vfio_pci_core_device *vdev, u32 flags,\n+\tstruct vfio_device_feature_dma_buf_revoke __user *arg,\n+\tsize_t argsz)\n {\n+\treturn -ENOTTY;\n }\n-static inline void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev,\n-\t\t\t\t\t bool revoked)\n+\n+static inline int vfio_pci_core_feature_dma_buf_memattr(\n+\tstruct vfio_pci_core_device *vdev, u32 flags,\n+\tstruct vfio_device_feature_dma_buf_memattr __user *arg,\n+\tsize_t argsz)\n {\n+\treturn -ENOTTY;\n }\n #endif\n \ndiff --git a/include/linux/pci-p2pdma.h b/include/linux/pci-p2pdma.h\nindex 873de20a224759..4c42a7b2ee8558 100644\n--- a/include/linux/pci-p2pdma.h\n+++ b/include/linux/pci-p2pdma.h\n@@ -67,9 +67,22 @@ enum pci_p2pdma_map_type {\n \tPCI_P2PDMA_MAP_THRU_HOST_BRIDGE,\n };\n \n-#ifdef CONFIG_PCI_P2PDMA\n+#ifdef CONFIG_PCI_P2PDMA_CORE\n int pcim_p2pdma_init(struct pci_dev *pdev);\n struct p2pdma_provider *pcim_p2pdma_provider(struct pci_dev *pdev, int bar);\n+#else\n+static inline int pcim_p2pdma_init(struct pci_dev *pdev)\n+{\n+\treturn -EOPNOTSUPP;\n+}\n+static inline struct p2pdma_provider *pcim_p2pdma_provider(struct pci_dev *pdev,\n+\t\t\t\t\t\t\t   int bar)\n+{\n+\treturn NULL;\n+}\n+#endif\n+\n+#ifdef CONFIG_PCI_P2PDMA\n int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size,\n \t\tu64 offset);\n int pci_p2pdma_distance_many(struct pci_dev *provider, struct device **clients,\n@@ -89,15 +102,6 @@ ssize_t pci_p2pdma_enable_show(char *page, struct pci_dev *p2p_dev,\n enum pci_p2pdma_map_type pci_p2pdma_map_type(struct p2pdma_provider *provider,\n \t\t\t\t\t     struct device *dev);\n #else /* CONFIG_PCI_P2PDMA */\n-static inline int pcim_p2pdma_init(struct pci_dev *pdev)\n-{\n-\treturn -EOPNOTSUPP;\n-}\n-static inline struct p2pdma_provider *pcim_p2pdma_provider(struct pci_dev *pdev,\n-\t\t\t\t\t\t\t   int bar)\n-{\n-\treturn NULL;\n-}\n static inline int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar,\n \t\tsize_t size, u64 offset)\n {\ndiff --git a/include/linux/pci.h b/include/linux/pci.h\nindex 2c4454583c115b..531aec355686b0 100644\n--- a/include/linux/pci.h\n+++ b/include/linux/pci.h\n@@ -557,7 +557,7 @@ struct pci_dev {\n \tu16\t\tpasid_cap;\t/* PASID Capability offset */\n \tu16\t\tpasid_features;\n #endif\n-#ifdef CONFIG_PCI_P2PDMA\n+#ifdef CONFIG_PCI_P2PDMA_CORE\n \tstruct pci_p2pdma __rcu *p2pdma;\n #endif\n #ifdef CONFIG_PCI_DOE\ndiff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h\nindex 89165b769e5c16..a2f586e4fcd8d4 100644\n--- a/include/linux/vfio_pci_core.h\n+++ b/include/linux/vfio_pci_core.h\n@@ -127,6 +127,7 @@ struct vfio_pci_core_device {\n \tbool\t\t\tneeds_pm_restore:1;\n \tbool\t\t\tpm_intx_masked:1;\n \tbool\t\t\tpm_runtime_engaged:1;\n+\tbool\t\t\tzap_bars_on_revoke:1;\n \tstruct pci_saved_state\t*pci_saved_state;\n \tstruct pci_saved_state\t*pm_save;\n \tint\t\t\tioeventfds_nr;\ndiff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h\nindex 5de618a3a5eeee..ab30b89399d038 100644\n--- a/include/uapi/linux/vfio.h\n+++ b/include/uapi/linux/vfio.h\n@@ -1534,6 +1534,53 @@ struct vfio_device_feature_dma_buf {\n  */\n #define VFIO_DEVICE_FEATURE_MIG_PRECOPY_INFOv2  12\n \n+/**\n+ * Given a dma_buf fd previously created by\n+ * VFIO_DEVICE_FEATURE_DMA_BUF, a SET of this feature requests that\n+ * access to the corresponding DMABUF is immediately and permanently\n+ * revoked.  On successful return, the buffer is not accessible\n+ * through any mmap() or dma-buf import.  The buffer is permanently\n+ * disabled, and VFIO refuses all map, mmap, attach, etc. requests.\n+ *\n+ * Return: 0 on success, -1 and errno is set on failure:\n+ *\n+ *  EBADF, EINVAL: dmabuf_fd is not a DMABUF fd.\n+ *  ENODEV: The dmabuf_fd does not match this VFIO device.\n+ *  EBADFD: The DMABUF is already revoked.\n+ */\n+#define VFIO_DEVICE_FEATURE_DMA_BUF_REVOKE 13\n+\n+struct vfio_device_feature_dma_buf_revoke {\n+\t__s32\tdmabuf_fd;\n+};\n+\n+/**\n+ * Given a dma_buf fd previously created by\n+ * VFIO_DEVICE_FEATURE_DMA_BUF, SET the memory attribute that will be\n+ * used by future mmap()s of that fd.  SETting a new attribute does\n+ * not affect existing VMAs.\n+ *\n+ * The default, if no previous SET has been performed, is NC.\n+ *\n+ * Return: 0 on success, -1 and errno is set on failure:\n+ *\n+ *  EBADF, EINVAL: dmabuf_fd is not a DMABUF fd.\n+ *  ENODEV: The dmabuf_fd does not match this VFIO device.\n+ *  ENOENT: The given memattr is not supported.\n+ */\n+#define VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR 14\n+\n+/* Valid memory attributes for the memattr field */\n+enum vfio_device_dma_buf_memattr {\n+\tVFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR_NC = 0, /* pgprot_noncached */\n+\tVFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR_WC = 1, /* pgprot_writecombine */\n+};\n+\n+struct vfio_device_feature_dma_buf_memattr {\n+\t__s32\tdmabuf_fd;\n+\t__u32\tmemattr;\n+};\n+\n /* -------- API for Type1 VFIO IOMMU -------- */\n \n /**\n"
}

For your convenience, here is the diff of the changes:
commit be307977b6149355089ba083133af2137dd145b0
Author: syz-cluster <triage@syzkaller.com>
Date:   Thu Jul 2 00:20:04 2026 +0000

    syz-cluster: applied patch under review

diff --git a/MAINTAINERS b/MAINTAINERS
index c8d4b913f26c12..713861af44846a 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -20626,7 +20626,7 @@ B:	https://bugzilla.kernel.org
 C:	irc://irc.oftc.net/linux-pci
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git
 F:	Documentation/driver-api/pci/p2pdma.rst
-F:	drivers/pci/p2pdma.c
+F:	drivers/pci/p2pdma*
 F:	include/linux/pci-p2pdma.h
 
 PCI POWER CONTROL
diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig
index 33c88432b72875..59d70bc84cc9ac 100644
--- a/drivers/pci/Kconfig
+++ b/drivers/pci/Kconfig
@@ -206,11 +206,7 @@ config PCIE_TPH
 config PCI_P2PDMA
 	bool "PCI peer-to-peer transfer support"
 	depends on ZONE_DEVICE
-	#
-	# The need for the scatterlist DMA bus address flag means PCI P2PDMA
-	# requires 64bit
-	#
-	depends on 64BIT
+	select PCI_P2PDMA_CORE
 	select GENERIC_ALLOCATOR
 	select NEED_SG_DMA_FLAGS
 	help
@@ -226,6 +222,10 @@ config PCI_P2PDMA
 
 	  If unsure, say N.
 
+config PCI_P2PDMA_CORE
+	default n
+	bool
+
 config PCI_LABEL
 	def_bool y if (DMI || ACPI)
 	select NLS
diff --git a/drivers/pci/Makefile b/drivers/pci/Makefile
index 41ebc3b9a51824..0b32572d57a159 100644
--- a/drivers/pci/Makefile
+++ b/drivers/pci/Makefile
@@ -30,6 +30,7 @@ obj-$(CONFIG_PCI_SYSCALL)	+= syscall.o
 obj-$(CONFIG_PCI_STUB)		+= pci-stub.o
 obj-$(CONFIG_PCI_PF_STUB)	+= pci-pf-stub.o
 obj-$(CONFIG_PCI_ECAM)		+= ecam.o
+obj-$(CONFIG_PCI_P2PDMA_CORE)	+= p2pdma_core.o
 obj-$(CONFIG_PCI_P2PDMA)	+= p2pdma.o
 obj-$(CONFIG_XEN_PCIDEV_FRONTEND) += xen-pcifront.o
 obj-$(CONFIG_VGA_ARB)		+= vgaarb.o
diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c
index 7c898542af8d5e..50b1a7daf55cc4 100644
--- a/drivers/pci/p2pdma.c
+++ b/drivers/pci/p2pdma.c
@@ -21,12 +21,7 @@
 #include <linux/seq_buf.h>
 #include <linux/xarray.h>
 
-struct pci_p2pdma {
-	struct gen_pool *pool;
-	bool p2pmem_published;
-	struct xarray map_types;
-	struct p2pdma_provider mem[PCI_STD_NUM_BARS];
-};
+#include "p2pdma.h"
 
 struct pci_p2pdma_pagemap {
 	struct dev_pagemap pgmap;
@@ -226,110 +221,16 @@ static const struct dev_pagemap_ops p2pdma_pgmap_ops = {
 	.folio_free = p2pdma_folio_free,
 };
 
-static void pci_p2pdma_release(void *data)
+void pci_p2pdma_release_pool(struct pci_dev *pdev, struct pci_p2pdma *p2pdma)
 {
-	struct pci_dev *pdev = data;
-	struct pci_p2pdma *p2pdma;
-
-	p2pdma = rcu_dereference_protected(pdev->p2pdma, 1);
-	if (!p2pdma)
-		return;
-
-	/* Flush and disable pci_alloc_p2p_mem() */
-	pdev->p2pdma = NULL;
-	if (p2pdma->pool)
-		synchronize_rcu();
-	xa_destroy(&p2pdma->map_types);
-
 	if (!p2pdma->pool)
 		return;
 
+	synchronize_rcu();
 	gen_pool_destroy(p2pdma->pool);
 	sysfs_remove_group(&pdev->dev.kobj, &p2pmem_group);
 }
 
-/**
- * pcim_p2pdma_init - Initialise peer-to-peer DMA providers
- * @pdev: The PCI device to enable P2PDMA for
- *
- * This function initializes the peer-to-peer DMA infrastructure
- * for a PCI device. It allocates and sets up the necessary data
- * structures to support P2PDMA operations, including mapping type
- * tracking.
- */
-int pcim_p2pdma_init(struct pci_dev *pdev)
-{
-	struct pci_p2pdma *p2p;
-	int i, ret;
-
-	p2p = rcu_dereference_protected(pdev->p2pdma, 1);
-	if (p2p)
-		return 0;
-
-	p2p = devm_kzalloc(&pdev->dev, sizeof(*p2p), GFP_KERNEL);
-	if (!p2p)
-		return -ENOMEM;
-
-	xa_init(&p2p->map_types);
-	/*
-	 * Iterate over all standard PCI BARs and record only those that
-	 * correspond to MMIO regions. Skip non-memory resources (e.g. I/O
-	 * port BARs) since they cannot be used for peer-to-peer (P2P)
-	 * transactions.
-	 */
-	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
-		if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM))
-			continue;
-
-		p2p->mem[i].owner = &pdev->dev;
-		p2p->mem[i].bus_offset =
-			pci_bus_address(pdev, i) - pci_resource_start(pdev, i);
-	}
-
-	ret = devm_add_action_or_reset(&pdev->dev, pci_p2pdma_release, pdev);
-	if (ret)
-		goto out_p2p;
-
-	rcu_assign_pointer(pdev->p2pdma, p2p);
-	return 0;
-
-out_p2p:
-	devm_kfree(&pdev->dev, p2p);
-	return ret;
-}
-EXPORT_SYMBOL_GPL(pcim_p2pdma_init);
-
-/**
- * pcim_p2pdma_provider - Get peer-to-peer DMA provider
- * @pdev: The PCI device to enable P2PDMA for
- * @bar: BAR index to get provider
- *
- * This function gets peer-to-peer DMA provider for a PCI device. The lifetime
- * of the provider (and of course the MMIO) is bound to the lifetime of the
- * driver. A driver calling this function must ensure that all references to the
- * provider, and any DMA mappings created for any MMIO, are all cleaned up
- * before the driver remove() completes.
- *
- * Since P2P is almost always shared with a second driver this means some system
- * to notify, invalidate and revoke the MMIO's DMA must be in place to use this
- * function. For example a revoke can be built using DMABUF.
- */
-struct p2pdma_provider *pcim_p2pdma_provider(struct pci_dev *pdev, int bar)
-{
-	struct pci_p2pdma *p2p;
-
-	if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM))
-		return NULL;
-
-	p2p = rcu_dereference_protected(pdev->p2pdma, 1);
-	if (WARN_ON(!p2p))
-		/* Someone forgot to call to pcim_p2pdma_init() before */
-		return NULL;
-
-	return &p2p->mem[bar];
-}
-EXPORT_SYMBOL_GPL(pcim_p2pdma_provider);
-
 static int pci_p2pdma_setup_pool(struct pci_dev *pdev)
 {
 	struct pci_p2pdma *p2pdma;
@@ -932,8 +833,8 @@ void *pci_alloc_p2pmem(struct pci_dev *pdev, size_t size)
 	struct pci_p2pdma *p2pdma;
 
 	/*
-	 * Pairs with synchronize_rcu() in pci_p2pdma_release() to
-	 * ensure pdev->p2pdma is non-NULL for the duration of the
+	 * Pairs with synchronize_rcu() in pci_p2pdma_release_pool()
+	 * to ensure pdev->p2pdma is non-NULL for the duration of the
 	 * read-lock.
 	 */
 	rcu_read_lock();
diff --git a/drivers/pci/p2pdma.h b/drivers/pci/p2pdma.h
new file mode 100644
index 00000000000000..94638380998149
--- /dev/null
+++ b/drivers/pci/p2pdma.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * PCI peer-to-peer DMA support.
+ */
+
+#ifndef _PCI_P2PDMA_H
+#define _PCI_P2PDMA_H
+
+#include <linux/genalloc.h>
+#include <linux/pci-p2pdma.h>
+#include <linux/xarray.h>
+
+struct pci_p2pdma {
+	struct gen_pool *pool;
+	bool p2pmem_published;
+	struct xarray map_types;
+	struct p2pdma_provider mem[PCI_STD_NUM_BARS];
+};
+
+#ifdef CONFIG_PCI_P2PDMA
+void pci_p2pdma_release_pool(struct pci_dev *pdev, struct pci_p2pdma *p2pdma);
+#else
+static inline void pci_p2pdma_release_pool(struct pci_dev *pdev,
+					   struct pci_p2pdma *p2pdma)
+{
+}
+#endif
+
+#endif
diff --git a/drivers/pci/p2pdma_core.c b/drivers/pci/p2pdma_core.c
new file mode 100644
index 00000000000000..bb2138bf2bc7b7
--- /dev/null
+++ b/drivers/pci/p2pdma_core.c
@@ -0,0 +1,118 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * PCI peer-to-peer DMA support core, providing a bare-bones
+ * pcim_p2pdma_provider() interface to drivers even if full P2PDMA
+ * isn't present.  The full P2PDMA feature is in p2pdma.c (see
+ * CONFIG_PCI_P2PDMA).
+ *
+ * Copyright (c) 2016-2018, Logan Gunthorpe
+ * Copyright (c) 2016-2017, Microsemi Corporation
+ * Copyright (c) 2017, Christoph Hellwig
+ * Copyright (c) 2018, Eideticom Inc.
+ */
+
+#define pr_fmt(fmt) "pci-p2pdma: " fmt
+#include <linux/ctype.h>
+#include <linux/genalloc.h>
+#include <linux/memremap.h>
+#include <linux/pci-p2pdma.h>
+#include <linux/xarray.h>
+
+#include "p2pdma.h"
+
+static void pci_p2pdma_release(void *data)
+{
+	struct pci_dev *pdev = data;
+	struct pci_p2pdma *p2pdma;
+
+	p2pdma = rcu_dereference_protected(pdev->p2pdma, 1);
+	if (!p2pdma)
+		return;
+
+	/* Flush and disable pci_alloc_p2p_mem() */
+	pdev->p2pdma = NULL;
+	pci_p2pdma_release_pool(pdev, p2pdma);
+	xa_destroy(&p2pdma->map_types);
+}
+
+/**
+ * pcim_p2pdma_init - Initialise peer-to-peer DMA providers
+ * @pdev: The PCI device to enable P2PDMA for
+ *
+ * This function initializes the peer-to-peer DMA infrastructure
+ * for a PCI device. It allocates and sets up the necessary data
+ * structures to support P2PDMA operations, including mapping type
+ * tracking.
+ */
+int pcim_p2pdma_init(struct pci_dev *pdev)
+{
+	struct pci_p2pdma *p2p;
+	int i, ret;
+
+	p2p = rcu_dereference_protected(pdev->p2pdma, 1);
+	if (p2p)
+		return 0;
+
+	p2p = devm_kzalloc(&pdev->dev, sizeof(*p2p), GFP_KERNEL);
+	if (!p2p)
+		return -ENOMEM;
+
+	xa_init(&p2p->map_types);
+	/*
+	 * Iterate over all standard PCI BARs and record only those that
+	 * correspond to MMIO regions. Skip non-memory resources (e.g. I/O
+	 * port BARs) since they cannot be used for peer-to-peer (P2P)
+	 * transactions.
+	 */
+	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
+		if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM))
+			continue;
+
+		p2p->mem[i].owner = &pdev->dev;
+		p2p->mem[i].bus_offset =
+			pci_bus_address(pdev, i) - pci_resource_start(pdev, i);
+	}
+
+	ret = devm_add_action_or_reset(&pdev->dev, pci_p2pdma_release, pdev);
+	if (ret)
+		goto out_p2p;
+
+	rcu_assign_pointer(pdev->p2pdma, p2p);
+	return 0;
+
+out_p2p:
+	devm_kfree(&pdev->dev, p2p);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(pcim_p2pdma_init);
+
+/**
+ * pcim_p2pdma_provider - Get peer-to-peer DMA provider
+ * @pdev: The PCI device to enable P2PDMA for
+ * @bar: BAR index to get provider
+ *
+ * This function gets peer-to-peer DMA provider for a PCI device. The lifetime
+ * of the provider (and of course the MMIO) is bound to the lifetime of the
+ * driver. A driver calling this function must ensure that all references to the
+ * provider, and any DMA mappings created for any MMIO, are all cleaned up
+ * before the driver remove() completes.
+ *
+ * Since P2P is almost always shared with a second driver this means some system
+ * to notify, invalidate and revoke the MMIO's DMA must be in place to use this
+ * function. For example a revoke can be built using DMABUF.
+ */
+struct p2pdma_provider *pcim_p2pdma_provider(struct pci_dev *pdev, int bar)
+{
+	struct pci_p2pdma *p2p;
+
+	if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM))
+		return NULL;
+
+	p2p = rcu_dereference_protected(pdev->p2pdma, 1);
+	if (WARN_ON(!p2p))
+		/* Someone forgot to call to pcim_p2pdma_init() before */
+		return NULL;
+
+	return &p2p->mem[bar];
+}
+EXPORT_SYMBOL_GPL(pcim_p2pdma_provider);
diff --git a/drivers/vfio/pci/Kconfig b/drivers/vfio/pci/Kconfig
index 296bf01e185eca..67a2ae1fbc04ba 100644
--- a/drivers/vfio/pci/Kconfig
+++ b/drivers/vfio/pci/Kconfig
@@ -6,6 +6,8 @@ config VFIO_PCI_CORE
 	tristate
 	select VFIO_VIRQFD
 	select IRQ_BYPASS_MANAGER
+	select PCI_P2PDMA_CORE
+	select DMA_SHARED_BUFFER
 
 config VFIO_PCI_INTX
 	def_bool y if !S390
@@ -56,7 +58,8 @@ config VFIO_PCI_ZDEV_KVM
 	  To enable s390x KVM vfio-pci extensions, say Y.
 
 config VFIO_PCI_DMABUF
-	def_bool y if VFIO_PCI_CORE && PCI_P2PDMA && DMA_SHARED_BUFFER
+	def_bool y if PCI_P2PDMA
+	depends on VFIO_PCI_CORE
 
 source "drivers/vfio/pci/mlx5/Kconfig"
 
diff --git a/drivers/vfio/pci/Makefile b/drivers/vfio/pci/Makefile
index 6138f1bf241df0..881452ea89be0c 100644
--- a/drivers/vfio/pci/Makefile
+++ b/drivers/vfio/pci/Makefile
@@ -1,8 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 
-vfio-pci-core-y := vfio_pci_core.o vfio_pci_intrs.o vfio_pci_rdwr.o vfio_pci_config.o
+vfio-pci-core-y := vfio_pci_core.o vfio_pci_intrs.o vfio_pci_rdwr.o vfio_pci_config.o vfio_pci_dmabuf.o
 vfio-pci-core-$(CONFIG_VFIO_PCI_ZDEV_KVM) += vfio_pci_zdev.o
-vfio-pci-core-$(CONFIG_VFIO_PCI_DMABUF) += vfio_pci_dmabuf.o
 obj-$(CONFIG_VFIO_PCI_CORE) += vfio-pci-core.o
 
 vfio-pci-y := vfio_pci.o
diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
index bb121f635b9f7f..d9f26b0ce991d9 100644
--- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
+++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
@@ -1725,6 +1725,14 @@ static int hisi_acc_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device
 	if (ret)
 		goto out_put_vdev;
 
+	/*
+	 * hisi_acc_vfio_pci_mmap() calls down to
+	 * vfio_pci_core_mmap(), so BAR mappings are still
+	 * DMABUF-backed.  They don't require a zap on revoke, so opt
+	 * out:
+	 */
+	hisi_acc_vdev->core_device.zap_bars_on_revoke = false;
+
 	hisi_acc_vfio_debug_init(hisi_acc_vdev);
 	return 0;
 
diff --git a/drivers/vfio/pci/vfio_pci_config.c b/drivers/vfio/pci/vfio_pci_config.c
index a10ed733f0e3af..bc6ccb2e135c3c 100644
--- a/drivers/vfio/pci/vfio_pci_config.c
+++ b/drivers/vfio/pci/vfio_pci_config.c
@@ -590,12 +590,10 @@ static int vfio_basic_config_write(struct vfio_pci_core_device *vdev, int pos,
 		virt_mem = !!(le16_to_cpu(*virt_cmd) & PCI_COMMAND_MEMORY);
 		new_mem = !!(new_cmd & PCI_COMMAND_MEMORY);
 
-		if (!new_mem) {
-			vfio_pci_zap_and_down_write_memory_lock(vdev);
-			vfio_pci_dma_buf_move(vdev, true);
-		} else {
+		if (!new_mem)
+			vfio_pci_lock_revoke_bars(vdev);
+		else
 			down_write(&vdev->memory_lock);
-		}
 
 		/*
 		 * If the user is writing mem/io enable (new_mem/io) and we
@@ -631,7 +629,7 @@ static int vfio_basic_config_write(struct vfio_pci_core_device *vdev, int pos,
 		*virt_cmd |= cpu_to_le16(new_cmd & mask);
 
 		if (__vfio_pci_memory_enabled(vdev))
-			vfio_pci_dma_buf_move(vdev, false);
+			vfio_pci_unrevoke_bars(vdev);
 		up_write(&vdev->memory_lock);
 	}
 
@@ -712,16 +710,14 @@ static int __init init_pci_cap_basic_perm(struct perm_bits *perm)
 static void vfio_lock_and_set_power_state(struct vfio_pci_core_device *vdev,
 					  pci_power_t state)
 {
-	if (state >= PCI_D3hot) {
-		vfio_pci_zap_and_down_write_memory_lock(vdev);
-		vfio_pci_dma_buf_move(vdev, true);
-	} else {
+	if (state >= PCI_D3hot)
+		vfio_pci_lock_revoke_bars(vdev);
+	else
 		down_write(&vdev->memory_lock);
-	}
 
 	vfio_pci_set_power_state(vdev, state);
 	if (__vfio_pci_memory_enabled(vdev))
-		vfio_pci_dma_buf_move(vdev, false);
+		vfio_pci_unrevoke_bars(vdev);
 	up_write(&vdev->memory_lock);
 }
 
@@ -908,11 +904,10 @@ static int vfio_exp_config_write(struct vfio_pci_core_device *vdev, int pos,
 						 &cap);
 
 		if (!ret && (cap & PCI_EXP_DEVCAP_FLR)) {
-			vfio_pci_zap_and_down_write_memory_lock(vdev);
-			vfio_pci_dma_buf_move(vdev, true);
+			vfio_pci_lock_revoke_bars(vdev);
 			pci_try_reset_function(vdev->pdev);
 			if (__vfio_pci_memory_enabled(vdev))
-				vfio_pci_dma_buf_move(vdev, false);
+				vfio_pci_unrevoke_bars(vdev);
 			up_write(&vdev->memory_lock);
 		}
 	}
@@ -993,11 +988,10 @@ static int vfio_af_config_write(struct vfio_pci_core_device *vdev, int pos,
 						&cap);
 
 		if (!ret && (cap & PCI_AF_CAP_FLR) && (cap & PCI_AF_CAP_TP)) {
-			vfio_pci_zap_and_down_write_memory_lock(vdev);
-			vfio_pci_dma_buf_move(vdev, true);
+			vfio_pci_lock_revoke_bars(vdev);
 			pci_try_reset_function(vdev->pdev);
 			if (__vfio_pci_memory_enabled(vdev))
-				vfio_pci_dma_buf_move(vdev, false);
+				vfio_pci_unrevoke_bars(vdev);
 			up_write(&vdev->memory_lock);
 		}
 	}
diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c
index 050e7542952ed1..0059c176d62706 100644
--- a/drivers/vfio/pci/vfio_pci_core.c
+++ b/drivers/vfio/pci/vfio_pci_core.c
@@ -12,6 +12,8 @@
 
 #include <linux/aperture.h>
 #include <linux/device.h>
+#include <linux/dma-buf.h>
+#include <linux/dma-resv.h>
 #include <linux/eventfd.h>
 #include <linux/file.h>
 #include <linux/interrupt.h>
@@ -319,8 +321,7 @@ static int vfio_pci_runtime_pm_entry(struct vfio_pci_core_device *vdev,
 	 * The vdev power related flags are protected with 'memory_lock'
 	 * semaphore.
 	 */
-	vfio_pci_zap_and_down_write_memory_lock(vdev);
-	vfio_pci_dma_buf_move(vdev, true);
+	vfio_pci_lock_revoke_bars(vdev);
 
 	if (vdev->pm_runtime_engaged) {
 		up_write(&vdev->memory_lock);
@@ -406,7 +407,7 @@ static void vfio_pci_runtime_pm_exit(struct vfio_pci_core_device *vdev)
 	down_write(&vdev->memory_lock);
 	__vfio_pci_runtime_pm_exit(vdev);
 	if (__vfio_pci_memory_enabled(vdev))
-		vfio_pci_dma_buf_move(vdev, false);
+		vfio_pci_unrevoke_bars(vdev);
 	up_write(&vdev->memory_lock);
 }
 
@@ -1256,6 +1257,8 @@ static int vfio_pci_ioctl_set_irqs(struct vfio_pci_core_device *vdev,
 	return ret;
 }
 
+static void vfio_pci_revoke_bars(struct vfio_pci_core_device *vdev);
+
 static int vfio_pci_ioctl_reset(struct vfio_pci_core_device *vdev,
 				void __user *arg)
 {
@@ -1264,7 +1267,7 @@ static int vfio_pci_ioctl_reset(struct vfio_pci_core_device *vdev,
 	if (!vdev->reset_works)
 		return -EINVAL;
 
-	vfio_pci_zap_and_down_write_memory_lock(vdev);
+	down_write(&vdev->memory_lock);
 
 	/*
 	 * This function can be invoked while the power state is non-D0. If
@@ -1274,13 +1277,18 @@ static int vfio_pci_ioctl_reset(struct vfio_pci_core_device *vdev,
 	 * have NoSoftRst-, the reset function can cause the PCI config space
 	 * reset without restoring the original state (saved locally in
 	 * 'vdev->pm_save').
+	 *
+	 * The zap is done after making the device accessible in D0,
+	 * because a DMABUF importer could access the device as part
+	 * of its revocation cleanup.
 	 */
 	vfio_pci_set_power_state(vdev, PCI_D0);
 
-	vfio_pci_dma_buf_move(vdev, true);
+	vfio_pci_revoke_bars(vdev);
+
 	ret = pci_try_reset_function(vdev->pdev);
 	if (__vfio_pci_memory_enabled(vdev))
-		vfio_pci_dma_buf_move(vdev, false);
+		vfio_pci_unrevoke_bars(vdev);
 	up_write(&vdev->memory_lock);
 
 	return ret;
@@ -1569,6 +1577,10 @@ int vfio_pci_core_ioctl_feature(struct vfio_device *device, u32 flags,
 		return vfio_pci_core_feature_token(vdev, flags, arg, argsz);
 	case VFIO_DEVICE_FEATURE_DMA_BUF:
 		return vfio_pci_core_feature_dma_buf(vdev, flags, arg, argsz);
+	case VFIO_DEVICE_FEATURE_DMA_BUF_REVOKE:
+		return vfio_pci_core_feature_dma_buf_revoke(vdev, flags, arg, argsz);
+	case VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR:
+		return vfio_pci_core_feature_dma_buf_memattr(vdev, flags, arg, argsz);
 	default:
 		return -ENOTTY;
 	}
@@ -1648,20 +1660,37 @@ ssize_t vfio_pci_core_write(struct vfio_device *core_vdev, const char __user *bu
 }
 EXPORT_SYMBOL_GPL(vfio_pci_core_write);
 
-static void vfio_pci_zap_bars(struct vfio_pci_core_device *vdev)
+static void vfio_pci_revoke_bars(struct vfio_pci_core_device *vdev)
 {
-	struct vfio_device *core_vdev = &vdev->vdev;
-	loff_t start = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR0_REGION_INDEX);
-	loff_t end = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_ROM_REGION_INDEX);
-	loff_t len = end - start;
+	lockdep_assert_held_write(&vdev->memory_lock);
+	vfio_pci_dma_buf_move(vdev, true);
 
-	unmap_mapping_range(core_vdev->inode->i_mapping, start, len, true);
+	/*
+	 * If a driver could possibly create BAR mappings in the
+	 * vdev's address_space, do an additional zap on revoke.  See
+	 * vfio_pci_core_register_device().
+	 */
+	if (vdev->zap_bars_on_revoke) {
+		struct vfio_device *core_vdev = &vdev->vdev;
+		loff_t start = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR0_REGION_INDEX);
+		loff_t end = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_ROM_REGION_INDEX);
+		loff_t len = end - start;
+
+		unmap_mapping_range(core_vdev->inode->i_mapping,
+				    start, len, true);
+	}
 }
 
-void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device *vdev)
+void vfio_pci_lock_revoke_bars(struct vfio_pci_core_device *vdev)
 {
 	down_write(&vdev->memory_lock);
-	vfio_pci_zap_bars(vdev);
+	vfio_pci_revoke_bars(vdev);
+}
+
+void vfio_pci_unrevoke_bars(struct vfio_pci_core_device *vdev)
+{
+	lockdep_assert_held_write(&vdev->memory_lock);
+	vfio_pci_dma_buf_move(vdev, false);
 }
 
 u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device *vdev)
@@ -1683,18 +1712,6 @@ void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device *vdev, u16 c
 	up_write(&vdev->memory_lock);
 }
 
-static unsigned long vma_to_pfn(struct vm_area_struct *vma)
-{
-	struct vfio_pci_core_device *vdev = vma->vm_private_data;
-	int index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
-	u64 pgoff;
-
-	pgoff = vma->vm_pgoff &
-		((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
-
-	return (pci_resource_start(vdev->pdev, index) >> PAGE_SHIFT) + pgoff;
-}
-
 vm_fault_t vfio_pci_vmf_insert_pfn(struct vfio_pci_core_device *vdev,
 				   struct vm_fault *vmf,
 				   unsigned long pfn,
@@ -1722,24 +1739,103 @@ static vm_fault_t vfio_pci_mmap_huge_fault(struct vm_fault *vmf,
 					   unsigned int order)
 {
 	struct vm_area_struct *vma = vmf->vma;
-	struct vfio_pci_core_device *vdev = vma->vm_private_data;
-	unsigned long addr = vmf->address & ~((PAGE_SIZE << order) - 1);
-	unsigned long pgoff = (addr - vma->vm_start) >> PAGE_SHIFT;
-	unsigned long pfn = vma_to_pfn(vma) + pgoff;
-	vm_fault_t ret = VM_FAULT_FALLBACK;
-
-	if (is_aligned_for_order(vma, addr, pfn, order)) {
-		scoped_guard(rwsem_read, &vdev->memory_lock)
-			ret = vfio_pci_vmf_insert_pfn(vdev, vmf, pfn, order);
+	struct vfio_pci_dma_buf *priv = vma->vm_private_data;
+	struct vfio_pci_core_device *vdev;
+	unsigned long pfn = 0;
+	vm_fault_t ret = VM_FAULT_SIGBUS;
+
+	/*
+	 * The only thing this can rely on is that the DMABUF relating
+	 * to the VMA's vm_file exists (priv).
+	 *
+	 * A DMABUF for a VFIO device fd mmap() holds a reference to
+	 * the original VFIO device fd, but an explicitly-exported
+	 * DMABUF does not.  The original fd might have closed,
+	 * meaning this fault can race with
+	 * vfio_pci_dma_buf_cleanup(), meaning the buffer could have
+	 * been revoked (in which case priv->vdev might be NULL), and
+	 * the VFIO device registration might have been dropped.
+	 *
+	 * With the goal of taking vdev->memory_lock in a world where
+	 * vdev might not still exist:
+	 *
+	 * 1. Take the resv lock on the DMABUF:
+	 *  - If racing cleanup got in first, the buffer is revoked;
+	 *    stop/exit if so.
+	 *  - If we got in first, the buffer is not revoked so vdev is
+	 *    non-NULL, accessible, and cleanup _has not yet put the
+	 *    VFIO device registration_.  So, the device refcount must
+	 *    be >0.
+	 *
+	 * 2. Take vfio_device registration (refcount guaranteed >0
+	 *    hereafter).
+	 *
+	 * 3. Unlock the DMABUF's resv lock:
+	 *  - A racing cleanup can now complete.
+	 *  - But, the device refcount >0, meaning the vfio_device
+	 *    (and vfio_pcie_core device vdev) have not yet been
+	 *    freed.  vdev is accessible, even if the DMABUF has been
+	 *    revoked or cleanup has happened, because
+	 *    vfio_unregister_group_dev() can't complete.
+	 *
+	 * 4. Take the vdev->memory_lock
+	 *  - Either the DMABUF is usable, or has been cleaned up.
+	 *    Whichever, it can no longer change under us.
+	 *  - Test the DMABUF revocation status again: if it was
+	 *    revoked between 1 and 4 return a SIGBUS. Otherwise,
+	 *    return a PFN.
+	 *  - It's not necessary to also take the resv lock, because
+	 *    the status/vdev can't change while memory_lock is held.
+	 *
+	 * 5. Unlock, done.
+	 */
+
+	dma_resv_lock(priv->dmabuf->resv, NULL);
+
+	if (priv->status != VFIO_PCI_DMABUF_OK) {
+		pr_debug_ratelimited("%s VA 0x%lx, pgoff 0x%lx: DMABUF revoked/cleaned up\n",
+				     __func__, vmf->address, vma->vm_pgoff);
+		dma_resv_unlock(priv->dmabuf->resv);
+		return VM_FAULT_SIGBUS;
+	}
+
+	/* If the buffer isn't revoked, vdev is valid */
+	vdev = priv->vdev;
+
+	if (!vfio_device_try_get_registration(&vdev->vdev)) {
+		/*
+		 * If vdev != NULL (above), the registration should
+		 * already be >0 and so this try_get should never
+		 * fail.
+		 */
+		dev_warn(&vdev->pdev->dev, "%s: Unexpected registration failure\n",
+			 __func__);
+		dma_resv_unlock(priv->dmabuf->resv);
+		return VM_FAULT_SIGBUS;
+	}
+	dma_resv_unlock(priv->dmabuf->resv);
+
+	scoped_guard(rwsem_read, &vdev->memory_lock) {
+		/* Revocation status must be re-read, under memory_lock */
+		if (priv->status == VFIO_PCI_DMABUF_OK) {
+			int pres = vfio_pci_dma_buf_find_pfn(priv, vma,
+							     vmf->address,
+							     order, &pfn);
+
+			if (pres == 0)
+				ret = vfio_pci_vmf_insert_pfn(vdev, vmf,
+							      pfn, order);
+			else if (pres == -EAGAIN)
+				ret = VM_FAULT_FALLBACK;
+		}
 	}
 
 	dev_dbg_ratelimited(&vdev->pdev->dev,
-			   "%s(,order = %d) BAR %ld page offset 0x%lx: 0x%x\n",
-			    __func__, order,
-			    vma->vm_pgoff >>
-				(VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT),
-			    pgoff, (unsigned int)ret);
+			    "%s(order = %d) PFN 0x%lx, VA 0x%lx, pgoff 0x%lx: 0x%x\n",
+			    __func__, order, pfn, vmf->address,
+			    vma->vm_pgoff, (unsigned int)ret);
 
+	vfio_device_put_registration(&vdev->vdev);
 	return ret;
 }
 
@@ -1755,6 +1851,11 @@ static const struct vm_operations_struct vfio_pci_mmap_ops = {
 #endif
 };
 
+void vfio_pci_set_vma_ops(struct vm_area_struct *vma)
+{
+	vma->vm_ops = &vfio_pci_mmap_ops;
+}
+
 int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma)
 {
 	struct vfio_pci_core_device *vdev =
@@ -1803,7 +1904,12 @@ int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma
 	if (ret)
 		return ret;
 
-	vma->vm_private_data = vdev;
+	ret = vfio_pci_core_mmap_prep_dmabuf(vdev, vma,
+					     pci_resource_start(pdev, index),
+					     req_len, index);
+	if (ret)
+		return ret;
+
 	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 	vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
 
@@ -2222,6 +2328,17 @@ int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev)
 	if (ret)
 		goto out_vf;
 
+	/*
+	 * If a driver overrides .mmap, it has to be assumed that it
+	 * might not use the DMABUF-backed core mmap; this flag
+	 * enables a zap at revoke time.  Drivers can opt out by
+	 * clearing this flag after registration, e.g. if their .mmap
+	 * override calls down to vfio_pci_core_mmap() so that maps
+	 * are DMABUF-backed.
+	 */
+	if (vdev->vdev.ops->mmap != vfio_pci_core_mmap)
+		vdev->zap_bars_on_revoke = true;
+
 	vfio_pci_probe_power_state(vdev);
 
 	/*
@@ -2497,9 +2614,10 @@ static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set,
 		}
 
 		/*
-		 * Take the memory write lock for each device and zap BAR
-		 * mappings to prevent the user accessing the device while in
-		 * reset.  Locking multiple devices is prone to deadlock,
+		 * Take the memory write lock for each device and
+		 * zap/revoke BAR mappings to prevent the user (or
+		 * peers) accessing the device while in reset.
+		 * Locking multiple devices is prone to deadlock,
 		 * runaway and unwind if we hit contention.
 		 */
 		if (!down_write_trylock(&vdev->memory_lock)) {
@@ -2507,8 +2625,7 @@ static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set,
 			break;
 		}
 
-		vfio_pci_dma_buf_move(vdev, true);
-		vfio_pci_zap_bars(vdev);
+		vfio_pci_revoke_bars(vdev);
 	}
 
 	if (!list_entry_is_head(vdev,
@@ -2538,7 +2655,7 @@ static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set,
 	list_for_each_entry_from_reverse(vdev, &dev_set->device_list,
 					 vdev.dev_set_list) {
 		if (vdev->vdev.open_count && __vfio_pci_memory_enabled(vdev))
-			vfio_pci_dma_buf_move(vdev, false);
+			vfio_pci_unrevoke_bars(vdev);
 		up_write(&vdev->memory_lock);
 	}
 
diff --git a/drivers/vfio/pci/vfio_pci_dmabuf.c b/drivers/vfio/pci/vfio_pci_dmabuf.c
index 1a177ce7de546a..31fbad93f89b1c 100644
--- a/drivers/vfio/pci/vfio_pci_dmabuf.c
+++ b/drivers/vfio/pci/vfio_pci_dmabuf.c
@@ -4,24 +4,13 @@
 #include <linux/dma-buf-mapping.h>
 #include <linux/pci-p2pdma.h>
 #include <linux/dma-resv.h>
+#include <uapi/linux/dma-buf.h>
 
 #include "vfio_pci_priv.h"
 
 MODULE_IMPORT_NS("DMA_BUF");
 
-struct vfio_pci_dma_buf {
-	struct dma_buf *dmabuf;
-	struct vfio_pci_core_device *vdev;
-	struct list_head dmabufs_elm;
-	size_t size;
-	struct phys_vec *phys_vec;
-	struct p2pdma_provider *provider;
-	u32 nr_ranges;
-	struct kref kref;
-	struct completion comp;
-	u8 revoked : 1;
-};
-
+#ifdef CONFIG_VFIO_PCI_DMABUF
 static int vfio_pci_dma_buf_attach(struct dma_buf *dmabuf,
 				   struct dma_buf_attachment *attachment)
 {
@@ -30,7 +19,7 @@ static int vfio_pci_dma_buf_attach(struct dma_buf *dmabuf,
 	if (!attachment->peer2peer)
 		return -EOPNOTSUPP;
 
-	if (priv->revoked)
+	if (priv->status != VFIO_PCI_DMABUF_OK)
 		return -ENODEV;
 
 	if (!dma_buf_attach_revocable(attachment))
@@ -39,6 +28,43 @@ static int vfio_pci_dma_buf_attach(struct dma_buf *dmabuf,
 	return 0;
 }
 
+static int vfio_pci_dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
+{
+	struct vfio_pci_dma_buf *priv = dmabuf->priv;
+
+	/*
+	 * dma_buf_mmap_internal() has asserted that the VMA is
+	 * contained within the DMABUF size before calling this.
+	 *
+	 * Also, if we observe that the buffer is revoked now then
+	 * refuse the mmap().  This is a belt-and-braces early failure
+	 * to ease debugging a revoked buffer being used.  Userspace
+	 * might also race an mmap() against an explicit revocation,
+	 * or an action doing a temporary revoke; race scenarios are
+	 * still safe because the fault handler ultimately prevents
+	 * access to a revoked buffer if it isn't caught here.
+	 */
+	if (READ_ONCE(priv->status) != VFIO_PCI_DMABUF_OK)
+		return -ENODEV;
+	if ((vma->vm_flags & VM_SHARED) == 0)
+		return -EINVAL;
+
+	if (READ_ONCE(priv->memattr) == VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR_WC)
+		vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
+	else
+		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+	vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
+
+	/* See comments in vfio_pci_core_mmap() re VM_ALLOW_ANY_UNCACHED. */
+	vm_flags_set(vma, VM_ALLOW_ANY_UNCACHED | VM_IO | VM_PFNMAP |
+		     VM_DONTEXPAND | VM_DONTDUMP);
+	vma->vm_private_data = priv;
+	vfio_pci_set_vma_ops(vma);
+
+	return 0;
+}
+#endif /* CONFIG_VFIO_PCI_DMABUF */
+
 static void vfio_pci_dma_buf_done(struct kref *kref)
 {
 	struct vfio_pci_dma_buf *priv =
@@ -56,7 +82,7 @@ vfio_pci_dma_buf_map(struct dma_buf_attachment *attachment,
 
 	dma_resv_assert_held(priv->dmabuf->resv);
 
-	if (priv->revoked)
+	if (priv->status != VFIO_PCI_DMABUF_OK)
 		return ERR_PTR(-ENODEV);
 
 	ret = dma_buf_phys_vec_to_sgt(attachment, priv->provider,
@@ -95,17 +121,203 @@ static void vfio_pci_dma_buf_release(struct dma_buf *dmabuf)
 		up_write(&priv->vdev->memory_lock);
 		vfio_device_put_registration(&priv->vdev->vdev);
 	}
+	if (priv->vfile)
+		fput(priv->vfile);
 	kfree(priv->phys_vec);
 	kfree(priv);
 }
 
 static const struct dma_buf_ops vfio_pci_dmabuf_ops = {
+#ifdef CONFIG_VFIO_PCI_DMABUF
 	.attach = vfio_pci_dma_buf_attach,
+	.mmap = vfio_pci_dma_buf_mmap,
+#endif
 	.map_dma_buf = vfio_pci_dma_buf_map,
 	.unmap_dma_buf = vfio_pci_dma_buf_unmap,
 	.release = vfio_pci_dma_buf_release,
 };
 
+int vfio_pci_dma_buf_find_pfn(struct vfio_pci_dma_buf *priv,
+			      struct vm_area_struct *vma,
+			      unsigned long fault_addr,
+			      unsigned int order,
+			      unsigned long *out_pfn)
+{
+	/*
+	 * Given a VMA (start, end, pgoffs) and a fault address,
+	 * search the corresponding DMABUF's phys_vec[] to find the
+	 * range representing the address's offset into the VMA, and
+	 * its PFN.
+	 *
+	 * The phys_vec[] ranges represent contiguous spans of VAs
+	 * upwards from the buffer offset 0; the actual PFNs might be
+	 * in any order, overlap/alias, etc.  Calculate an offset of
+	 * the desired page given VMA start/pgoff and address, then
+	 * search upwards from 0 to find which span contains it.
+	 *
+	 * On success, a valid PFN for a page sized by 'order' is
+	 * returned into out_pfn.
+	 *
+	 * Failure occurs if:
+	 * - A hugepage would cross the edge of the VMA,
+	 * - A hugepage isn't entirely contained within a range
+	 *   (including where it straddles the boundary between
+	 *   ranges),
+	 * - We find a range, but the final PFN isn't aligned to the
+	 *   requested order.
+	 *
+	 * Upon failure, -EAGAIN is returned and the caller is
+	 * expected to try again with a smaller order, which will
+	 * eventually succeed (order=0 will always work).
+	 *
+	 * It's suboptimal if DMABUFs are created with neighbouring
+	 * ranges that are physically contiguous, since hugepages
+	 * can't straddle range boundaries.  (The construction of the
+	 * ranges should merge them in this case.)
+	 *
+	 * Finally, vma_pgoff_adjust is used with a DMABUF created for
+	 * a VFIO BAR mmap: a BAR mapped with vm_pgoff > 0 creates a
+	 * DMABUF such that byte 0 of the VMA corresponds to byte 0 of
+	 * the DMABUF and byte 'vm_pgoff << PAGE_SHIFT' into the BAR.
+	 * To avoid double-offsetting in this scenario, subtracting
+	 * vma_pgoff_adjust from this (non-zero) vm_pgoff generates
+	 * the effective offset.
+	 */
+
+	const unsigned long pagesize = PAGE_SIZE << order;
+	unsigned long vma_off = ((vma->vm_pgoff - priv->vma_pgoff_adjust) <<
+				 PAGE_SHIFT) & VFIO_PCI_OFFSET_MASK;
+	unsigned long rounded_page_addr = ALIGN_DOWN(fault_addr, pagesize);
+	unsigned long rounded_page_end = rounded_page_addr + pagesize;
+	unsigned long fault_offset;
+	unsigned long fault_offset_end;
+	unsigned long range_start_offset = 0;
+	unsigned int i;
+	int ret;
+
+	if (rounded_page_addr < vma->vm_start || rounded_page_end > vma->vm_end) {
+		if (order > 0)
+			return -EAGAIN;
+
+		/* A fault address outside of the VMA is absurd. */
+		dev_warn_ratelimited(
+			&priv->vdev->pdev->dev,
+			"Fault addr 0x%lx outside VMA 0x%lx-0x%lx\n",
+			fault_addr, vma->vm_start, vma->vm_end);
+		return -EFAULT;
+	}
+
+	/*
+	 * fault_offset[_end] is the span within the DMABUF
+	 * corresponding to the faulting page:
+	 */
+	if (unlikely(check_add_overflow(rounded_page_addr - vma->vm_start,
+					vma_off, &fault_offset) ||
+		     check_add_overflow(fault_offset, pagesize,
+					&fault_offset_end)))
+		return -EFAULT;
+
+	/*
+	 * Iterate over ranges in the buffer, summing their lengths:
+	 * range_start_offset represents the current range's starting
+	 * offset in the buffer (from 0 upwards).
+	 *
+	 * A failure for order == 0 is unexpected, and triggers a
+	 * fault/warn.
+	 */
+	ret = (order == 0) ? -EFAULT : -EAGAIN;
+
+	for (i = 0; i < priv->nr_ranges; i++) {
+		size_t range_len = priv->phys_vec[i].len;
+
+		/* Early exit if range starts after the page end */
+		if (fault_offset_end <= range_start_offset)
+			break;
+
+		if (fault_offset >= range_start_offset &&
+		    fault_offset_end <= range_start_offset + range_len) {
+			/*
+			 * The faulting page is wholly contained
+			 * within the span represented by this range,
+			 * so validate PFN alignment for the order.
+			 * The if() condition ensures the pfn
+			 * arithmetic won't overflow.
+			 */
+			unsigned long pfn =
+				((fault_offset - range_start_offset) +
+				 priv->phys_vec[i].paddr) >> PAGE_SHIFT;
+
+			if (IS_ALIGNED(pfn, 1 << order)) {
+				*out_pfn = pfn;
+				ret = 0;
+			}
+			/*
+			 * Else order > 0; EAGAIN retries with smaller
+			 * order
+			 */
+			break;
+		}
+		range_start_offset += range_len;
+	}
+
+	if (order == 0 && ret != 0)
+		/*
+		 * The address fell outside of the span represented by
+		 * the (concatenated) ranges.  As setup of a mapping
+		 * ensures that the VMA is <= the total size of the
+		 * ranges this should never happen.  If it does, warn
+		 * and SIGBUS.
+		 */
+		dev_warn_ratelimited(
+			&priv->vdev->pdev->dev,
+			"No range for addr 0x%lx, order %d: VMA 0x%lx-0x%lx pgoff 0x%lx, %u ranges, size 0x%zx\n",
+			fault_addr, order, vma->vm_start, vma->vm_end,
+			vma->vm_pgoff, priv->nr_ranges, priv->size);
+
+	return ret;
+}
+
+/*
+ * Create a DMABUF corresponding to priv, add it to vdev->dmabufs list
+ * for tracking (meaning cleanup or revocation will zap it), and take
+ * a vfio_device registration.
+ */
+static int vfio_pci_dmabuf_export(struct vfio_pci_core_device *vdev,
+				  struct vfio_pci_dma_buf *priv, u32 flags)
+{
+	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
+
+	if (!vfio_device_try_get_registration(&vdev->vdev))
+		return -ENODEV;
+
+	exp_info.ops = &vfio_pci_dmabuf_ops;
+	exp_info.size = priv->size;
+	exp_info.flags = flags;
+	exp_info.priv = priv;
+
+	priv->dmabuf = dma_buf_export(&exp_info);
+	if (IS_ERR(priv->dmabuf)) {
+		vfio_device_put_registration(&vdev->vdev);
+		return PTR_ERR(priv->dmabuf);
+	}
+
+	kref_init(&priv->kref);
+	init_completion(&priv->comp);
+
+	/* dma_buf_put() now frees priv */
+	INIT_LIST_HEAD(&priv->dmabufs_elm);
+	down_write(&vdev->memory_lock);
+	dma_resv_lock(priv->dmabuf->resv, NULL);
+	priv->status = __vfio_pci_memory_enabled(vdev) ? VFIO_PCI_DMABUF_OK :
+		VFIO_PCI_DMABUF_TEMP_REVOKED;
+	list_add_tail(&priv->dmabufs_elm, &vdev->dmabufs);
+	dma_resv_unlock(priv->dmabuf->resv);
+	up_write(&vdev->memory_lock);
+
+	return 0;
+}
+
+#ifdef CONFIG_VFIO_PCI_DMABUF
 /*
  * This is a temporary "private interconnect" between VFIO DMABUF and iommufd.
  * It allows the two co-operating drivers to exchange the physical address of
@@ -128,7 +340,7 @@ int vfio_pci_dma_buf_iommufd_map(struct dma_buf_attachment *attachment,
 		return -EOPNOTSUPP;
 
 	priv = attachment->dmabuf->priv;
-	if (priv->revoked)
+	if (priv->status != VFIO_PCI_DMABUF_OK)
 		return -ENODEV;
 
 	/* More than one range to iommufd will require proper DMABUF support */
@@ -224,7 +436,6 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,
 {
 	struct vfio_device_feature_dma_buf get_dma_buf = {};
 	struct vfio_region_dma_range *dma_ranges;
-	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
 	struct vfio_pci_dma_buf *priv;
 	size_t length;
 	int ret;
@@ -274,6 +485,7 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,
 	priv->vdev = vdev;
 	priv->nr_ranges = get_dma_buf.nr_ranges;
 	priv->size = length;
+	priv->memattr = VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR_NC;
 	ret = vdev->pci_ops->get_dmabuf_phys(vdev, &priv->provider,
 					     get_dma_buf.region_index,
 					     priv->phys_vec, dma_ranges,
@@ -284,34 +496,9 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,
 	kfree(dma_ranges);
 	dma_ranges = NULL;
 
-	if (!vfio_device_try_get_registration(&vdev->vdev)) {
-		ret = -ENODEV;
+	ret = vfio_pci_dmabuf_export(vdev, priv, get_dma_buf.open_flags);
+	if (ret)
 		goto err_free_phys;
-	}
-
-	exp_info.ops = &vfio_pci_dmabuf_ops;
-	exp_info.size = priv->size;
-	exp_info.flags = get_dma_buf.open_flags;
-	exp_info.priv = priv;
-
-	priv->dmabuf = dma_buf_export(&exp_info);
-	if (IS_ERR(priv->dmabuf)) {
-		ret = PTR_ERR(priv->dmabuf);
-		goto err_dev_put;
-	}
-
-	kref_init(&priv->kref);
-	init_completion(&priv->comp);
-
-	/* dma_buf_put() now frees priv */
-	INIT_LIST_HEAD(&priv->dmabufs_elm);
-	down_write(&vdev->memory_lock);
-	dma_resv_lock(priv->dmabuf->resv, NULL);
-	priv->revoked = !__vfio_pci_memory_enabled(vdev);
-	list_add_tail(&priv->dmabufs_elm, &vdev->dmabufs);
-	dma_resv_unlock(priv->dmabuf->resv);
-	up_write(&vdev->memory_lock);
-
 	/*
 	 * dma_buf_fd() consumes the reference, when the file closes the dmabuf
 	 * will be released.
@@ -322,8 +509,6 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,
 
 	return ret;
 
-err_dev_put:
-	vfio_device_put_registration(&vdev->vdev);
 err_free_phys:
 	kfree(priv->phys_vec);
 err_free_priv:
@@ -332,47 +517,164 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,
 	kfree(dma_ranges);
 	return ret;
 }
+#endif /* CONFIG_VFIO_PCI_DMABUF */
+
+int vfio_pci_core_mmap_prep_dmabuf(struct vfio_pci_core_device *vdev,
+				   struct vm_area_struct *vma,
+				   u64 phys_start, u64 req_len,
+				   unsigned int res_index)
+{
+	struct vfio_pci_dma_buf *priv;
+	unsigned long vma_pgoff = vma->vm_pgoff & (VFIO_PCI_OFFSET_MASK >> PAGE_SHIFT);
+	char *bufname;
+	int ret;
+
+	priv = kzalloc_obj(*priv);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->phys_vec = kzalloc_obj(*priv->phys_vec);
+	if (!priv->phys_vec) {
+		ret = -ENOMEM;
+		goto err_free_priv;
+	}
+
+	bufname = kasprintf(GFP_KERNEL, "%s:%s/%x",
+			    dev_name(&vdev->vdev.device), pci_name(vdev->pdev),
+			    res_index);
+
+	if (!bufname) {
+		ret = -ENOMEM;
+		goto err_free_phys;
+	}
+
+	/*
+	 * The DMABUF begins from the mmap()'s BAR offset, i.e. the
+	 * start of the VMA corresponds to byte 0 of the DMABUF and
+	 * byte (vma_pgoff << PAGE_SHIFT) of the BAR.
+	 *
+	 * vfio_pci_dma_buf_find_pfn() reverses this offset using
+	 * vma_pgoff_adjust, so that ultimately a fault's offset from
+	 * the start of the _VMA_ has a consistent usage whether the
+	 * VMA originates from an mmap() of the VFIO device here or a
+	 * direct DMABUF mmap().
+	 */
+	priv->vdev = vdev;
+	priv->size = req_len;
+	priv->nr_ranges = 1;
+	priv->vma_pgoff_adjust = vma_pgoff;
+	priv->provider = pcim_p2pdma_provider(vdev->pdev, res_index);
+	if (!priv->provider) {
+		ret = -EINVAL;
+		goto err_free_name;
+	}
+
+	priv->phys_vec[0].paddr = phys_start + ((u64)vma_pgoff << PAGE_SHIFT);
+	priv->phys_vec[0].len = priv->size;
+
+	ret = vfio_pci_dmabuf_export(vdev, priv, O_CLOEXEC | O_RDWR);
+	if (ret)
+		goto err_free_name;
+
+	/*
+	 * Ownership of the DMABUF file transfers to the VMA so that
+	 * other users can locate the DMABUF via a VA.  Ownership of
+	 * the original VFIO device file being mmap()ed transfers to
+	 * priv, and is put when the DMABUF is released.  This
+	 * intentionally does not use get_file()/vma_set_file()
+	 * because the references are already held, and ownership
+	 * moves.
+	 */
+	priv->vfile = vma->vm_file;
+	vma->vm_file = priv->dmabuf->file;
+	vma->vm_private_data = priv;
+
+	spin_lock(&priv->dmabuf->name_lock);
+	kfree(priv->dmabuf->name);
+	priv->dmabuf->name = bufname;
+	spin_unlock(&priv->dmabuf->name_lock);
+
+	return 0;
+
+err_free_name:
+	kfree(bufname);
+err_free_phys:
+	kfree(priv->phys_vec);
+err_free_priv:
+	kfree(priv);
+	return ret;
+}
+
+/* Set the DMABUF's revocation status (OK or temporarily/permanently revoked) */
+static void vfio_pci_dma_buf_set_status(struct vfio_pci_dma_buf *priv,
+					enum vfio_pci_dma_buf_status new_status)
+{
+	bool was_revoked;
+
+	lockdep_assert_held_write(&priv->vdev->memory_lock);
+
+	if (priv->status == VFIO_PCI_DMABUF_PERM_REVOKED ||
+	    priv->status == new_status)
+		return;
+
+	dma_resv_lock(priv->dmabuf->resv, NULL);
+	was_revoked = (priv->status == VFIO_PCI_DMABUF_TEMP_REVOKED);
+
+	if (new_status != VFIO_PCI_DMABUF_OK) {
+		priv->status = new_status; /* Temp or permanently revoked */
+
+		if (was_revoked) {
+			/*
+			 * TEMP_REVOKED is being upgraded to
+			 * PERM_REVOKED.  The buffer is already gone,
+			 * don't wait on it again.
+			 */
+			dma_resv_unlock(priv->dmabuf->resv);
+			return;
+		}
+		dma_buf_invalidate_mappings(priv->dmabuf);
+		dma_resv_wait_timeout(priv->dmabuf->resv,
+				      DMA_RESV_USAGE_BOOKKEEP, false,
+				      MAX_SCHEDULE_TIMEOUT);
+		dma_resv_unlock(priv->dmabuf->resv);
+		kref_put(&priv->kref, vfio_pci_dma_buf_done);
+		wait_for_completion(&priv->comp);
+		unmap_mapping_range(priv->dmabuf->file->f_mapping,
+				    0, priv->size, true);
+		/*
+		 * Re-arm the registered kref reference and the
+		 * completion so the post-revoke state matches the
+		 * post-creation state.  An un-revoke followed by a
+		 * new mapping needs the kref to be non-zero before
+		 * kref_get(), and vfio_pci_dma_buf_cleanup()
+		 * delegates its drain back through this revoke
+		 * path on a possibly-already-revoked dma-buf.
+		 */
+		kref_init(&priv->kref);
+		reinit_completion(&priv->comp);
+	} else {
+		priv->status = VFIO_PCI_DMABUF_OK;
+		dma_resv_unlock(priv->dmabuf->resv);
+	}
+}
 
 void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev, bool revoked)
 {
 	struct vfio_pci_dma_buf *priv;
 	struct vfio_pci_dma_buf *tmp;
 
+	/*
+	 * Holding memory_lock ensures a racing VMA fault observes
+	 * priv->status properly.
+	 */
 	lockdep_assert_held_write(&vdev->memory_lock);
 
 	list_for_each_entry_safe(priv, tmp, &vdev->dmabufs, dmabufs_elm) {
 		if (!get_file_active(&priv->dmabuf->file))
 			continue;
-
-		if (priv->revoked != revoked) {
-			dma_resv_lock(priv->dmabuf->resv, NULL);
-			if (revoked)
-				priv->revoked = true;
-			dma_buf_invalidate_mappings(priv->dmabuf);
-			dma_resv_wait_timeout(priv->dmabuf->resv,
-					      DMA_RESV_USAGE_BOOKKEEP, false,
-					      MAX_SCHEDULE_TIMEOUT);
-			dma_resv_unlock(priv->dmabuf->resv);
-			if (revoked) {
-				kref_put(&priv->kref, vfio_pci_dma_buf_done);
-				wait_for_completion(&priv->comp);
-				/*
-				 * Re-arm the registered kref reference and the
-				 * completion so the post-revoke state matches the
-				 * post-creation state.  An un-revoke followed by a
-				 * new mapping needs the kref to be non-zero before
-				 * kref_get(), and vfio_pci_dma_buf_cleanup()
-				 * delegates its drain back through this revoke
-				 * path on a possibly-already-revoked dma-buf.
-				 */
-				kref_init(&priv->kref);
-				reinit_completion(&priv->comp);
-			} else {
-				dma_resv_lock(priv->dmabuf->resv, NULL);
-				priv->revoked = false;
-				dma_resv_unlock(priv->dmabuf->resv);
-			}
-		}
+		vfio_pci_dma_buf_set_status(priv, revoked ?
+					    VFIO_PCI_DMABUF_TEMP_REVOKED :
+					    VFIO_PCI_DMABUF_OK);
 		fput(priv->dmabuf->file);
 	}
 }
@@ -404,3 +706,117 @@ void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device *vdev)
 	}
 	up_write(&vdev->memory_lock);
 }
+
+#ifdef CONFIG_VFIO_PCI_DMABUF
+int vfio_pci_core_feature_dma_buf_revoke(
+	struct vfio_pci_core_device *vdev, u32 flags,
+	struct vfio_device_feature_dma_buf_revoke __user *arg,
+	size_t argsz)
+{
+	struct vfio_device_feature_dma_buf_revoke db_revoke;
+	struct vfio_pci_dma_buf *priv;
+	struct dma_buf *dmabuf;
+	int ret;
+
+	if (!vdev->pci_ops || !vdev->pci_ops->get_dmabuf_phys)
+		return -EOPNOTSUPP;
+
+	ret = vfio_check_feature(flags, argsz,
+				 VFIO_DEVICE_FEATURE_SET,
+				 sizeof(db_revoke));
+	if (ret != 1)
+		return ret;
+
+	if (copy_from_user(&db_revoke, arg, sizeof(db_revoke)))
+		return -EFAULT;
+
+	dmabuf = dma_buf_get(db_revoke.dmabuf_fd);
+	if (IS_ERR(dmabuf))
+		return PTR_ERR(dmabuf);
+
+	priv = dmabuf->priv;
+	/*
+	 * Sanity-check the DMABUF is really a vfio_pci_dma_buf _and_
+	 * relates to the VFIO device it was provided with.
+	 *
+	 * If the DMABUF relates to this vdev then priv->vdev is
+	 * stable because this open fd prevents cleanup.
+	 *
+	 * If it relates to a different vdev, reading priv->vdev might
+	 * race with a concurrent cleanup on that device.  But if so,
+	 * it points to a non-matching vdev or NULL and is unusable
+	 * either way.
+	 */
+	if (dmabuf->ops != &vfio_pci_dmabuf_ops ||
+	    READ_ONCE(priv->vdev) != vdev) {
+		ret = -ENODEV;
+		goto out_put_buf;
+	}
+
+	scoped_guard(rwsem_write, &vdev->memory_lock) {
+		if (priv->status == VFIO_PCI_DMABUF_PERM_REVOKED) {
+			ret = -EBADFD;
+		} else {
+			vfio_pci_dma_buf_set_status(priv,
+						    VFIO_PCI_DMABUF_PERM_REVOKED);
+			ret = 0;
+		}
+	}
+
+out_put_buf:
+	dma_buf_put(dmabuf);
+
+	return ret;
+}
+
+int vfio_pci_core_feature_dma_buf_memattr(
+	struct vfio_pci_core_device *vdev, u32 flags,
+	struct vfio_device_feature_dma_buf_memattr __user *arg,
+	size_t argsz)
+{
+	struct vfio_device_feature_dma_buf_memattr db_attr;
+	struct vfio_pci_dma_buf *priv;
+	struct dma_buf *dmabuf;
+	int ret;
+
+	if (!vdev->pci_ops || !vdev->pci_ops->get_dmabuf_phys)
+		return -EOPNOTSUPP;
+
+	ret = vfio_check_feature(flags, argsz,
+				 VFIO_DEVICE_FEATURE_SET,
+				 sizeof(db_attr));
+	if (ret != 1)
+		return ret;
+
+	if (copy_from_user(&db_attr, arg, sizeof(db_attr)))
+		return -EFAULT;
+
+	dmabuf = dma_buf_get(db_attr.dmabuf_fd);
+	if (IS_ERR(dmabuf))
+		return PTR_ERR(dmabuf);
+
+	/* Verify DMABUF: see comments in vfio_pci_dma_buf_revoke() */
+	priv = dmabuf->priv;
+	if (dmabuf->ops != &vfio_pci_dmabuf_ops ||
+	    READ_ONCE(priv->vdev) != vdev) {
+		ret = -ENODEV;
+		goto out_put_buf;
+	}
+
+	switch (db_attr.memattr) {
+	case VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR_NC:
+	case VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR_WC:
+		WRITE_ONCE(priv->memattr, db_attr.memattr);
+		ret = 0;
+		break;
+
+	default:
+		ret = -ENOENT;
+	}
+
+out_put_buf:
+	dma_buf_put(dmabuf);
+
+	return ret;
+}
+#endif /* CONFIG_VFIO_PCI_DMABUF */
diff --git a/drivers/vfio/pci/vfio_pci_priv.h b/drivers/vfio/pci/vfio_pci_priv.h
index fca9d0dfac90f8..b3c72df002c448 100644
--- a/drivers/vfio/pci/vfio_pci_priv.h
+++ b/drivers/vfio/pci/vfio_pci_priv.h
@@ -23,6 +23,28 @@ struct vfio_pci_ioeventfd {
 	bool			test_mem;
 };
 
+enum vfio_pci_dma_buf_status {
+	VFIO_PCI_DMABUF_OK = 0,
+	VFIO_PCI_DMABUF_TEMP_REVOKED = 1,
+	VFIO_PCI_DMABUF_PERM_REVOKED = 2,
+};
+
+struct vfio_pci_dma_buf {
+	struct dma_buf *dmabuf;
+	struct vfio_pci_core_device *vdev;
+	struct list_head dmabufs_elm;
+	size_t size;
+	struct phys_vec *phys_vec;
+	struct p2pdma_provider *provider;
+	struct file *vfile;
+	u32 nr_ranges;
+	struct kref kref;
+	struct completion comp;
+	unsigned long vma_pgoff_adjust;
+	u32 memattr;
+	enum vfio_pci_dma_buf_status status;
+};
+
 bool vfio_pci_intx_mask(struct vfio_pci_core_device *vdev);
 void vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev);
 
@@ -68,7 +90,8 @@ void vfio_config_free(struct vfio_pci_core_device *vdev);
 int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev,
 			     pci_power_t state);
 
-void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device *vdev);
+void vfio_pci_lock_revoke_bars(struct vfio_pci_core_device *vdev);
+void vfio_pci_unrevoke_bars(struct vfio_pci_core_device *vdev);
 u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device *vdev);
 void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device *vdev,
 					u16 cmd);
@@ -114,12 +137,31 @@ static inline bool vfio_pci_is_vga(struct pci_dev *pdev)
 	return (pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA;
 }
 
+int vfio_pci_dma_buf_find_pfn(struct vfio_pci_dma_buf *vpdmabuf,
+			      struct vm_area_struct *vma,
+			      unsigned long address,
+			      unsigned int order,
+			      unsigned long *out_pfn);
+int vfio_pci_core_mmap_prep_dmabuf(struct vfio_pci_core_device *vdev,
+				   struct vm_area_struct *vma,
+				   u64 phys_start, u64 req_len,
+				   unsigned int res_index);
+void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device *vdev);
+void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev, bool revoked);
+void vfio_pci_set_vma_ops(struct vm_area_struct *vma);
+
 #ifdef CONFIG_VFIO_PCI_DMABUF
 int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,
 				  struct vfio_device_feature_dma_buf __user *arg,
 				  size_t argsz);
-void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device *vdev);
-void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev, bool revoked);
+int vfio_pci_core_feature_dma_buf_revoke(
+	struct vfio_pci_core_device *vdev, u32 flags,
+	struct vfio_device_feature_dma_buf_revoke __user *arg,
+	size_t argsz);
+int vfio_pci_core_feature_dma_buf_memattr(
+	struct vfio_pci_core_device *vdev, u32 flags,
+	struct vfio_device_feature_dma_buf_memattr __user *arg,
+	size_t argsz);
 #else
 static inline int
 vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,
@@ -128,12 +170,21 @@ vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,
 {
 	return -ENOTTY;
 }
-static inline void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device *vdev)
+
+static inline int vfio_pci_core_feature_dma_buf_revoke(
+	struct vfio_pci_core_device *vdev, u32 flags,
+	struct vfio_device_feature_dma_buf_revoke __user *arg,
+	size_t argsz)
 {
+	return -ENOTTY;
 }
-static inline void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev,
-					 bool revoked)
+
+static inline int vfio_pci_core_feature_dma_buf_memattr(
+	struct vfio_pci_core_device *vdev, u32 flags,
+	struct vfio_device_feature_dma_buf_memattr __user *arg,
+	size_t argsz)
 {
+	return -ENOTTY;
 }
 #endif
 
diff --git a/include/linux/pci-p2pdma.h b/include/linux/pci-p2pdma.h
index 873de20a224759..4c42a7b2ee8558 100644
--- a/include/linux/pci-p2pdma.h
+++ b/include/linux/pci-p2pdma.h
@@ -67,9 +67,22 @@ enum pci_p2pdma_map_type {
 	PCI_P2PDMA_MAP_THRU_HOST_BRIDGE,
 };
 
-#ifdef CONFIG_PCI_P2PDMA
+#ifdef CONFIG_PCI_P2PDMA_CORE
 int pcim_p2pdma_init(struct pci_dev *pdev);
 struct p2pdma_provider *pcim_p2pdma_provider(struct pci_dev *pdev, int bar);
+#else
+static inline int pcim_p2pdma_init(struct pci_dev *pdev)
+{
+	return -EOPNOTSUPP;
+}
+static inline struct p2pdma_provider *pcim_p2pdma_provider(struct pci_dev *pdev,
+							   int bar)
+{
+	return NULL;
+}
+#endif
+
+#ifdef CONFIG_PCI_P2PDMA
 int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size,
 		u64 offset);
 int pci_p2pdma_distance_many(struct pci_dev *provider, struct device **clients,
@@ -89,15 +102,6 @@ ssize_t pci_p2pdma_enable_show(char *page, struct pci_dev *p2p_dev,
 enum pci_p2pdma_map_type pci_p2pdma_map_type(struct p2pdma_provider *provider,
 					     struct device *dev);
 #else /* CONFIG_PCI_P2PDMA */
-static inline int pcim_p2pdma_init(struct pci_dev *pdev)
-{
-	return -EOPNOTSUPP;
-}
-static inline struct p2pdma_provider *pcim_p2pdma_provider(struct pci_dev *pdev,
-							   int bar)
-{
-	return NULL;
-}
 static inline int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar,
 		size_t size, u64 offset)
 {
diff --git a/include/linux/pci.h b/include/linux/pci.h
index 2c4454583c115b..531aec355686b0 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -557,7 +557,7 @@ struct pci_dev {
 	u16		pasid_cap;	/* PASID Capability offset */
 	u16		pasid_features;
 #endif
-#ifdef CONFIG_PCI_P2PDMA
+#ifdef CONFIG_PCI_P2PDMA_CORE
 	struct pci_p2pdma __rcu *p2pdma;
 #endif
 #ifdef CONFIG_PCI_DOE
diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h
index 89165b769e5c16..a2f586e4fcd8d4 100644
--- a/include/linux/vfio_pci_core.h
+++ b/include/linux/vfio_pci_core.h
@@ -127,6 +127,7 @@ struct vfio_pci_core_device {
 	bool			needs_pm_restore:1;
 	bool			pm_intx_masked:1;
 	bool			pm_runtime_engaged:1;
+	bool			zap_bars_on_revoke:1;
 	struct pci_saved_state	*pci_saved_state;
 	struct pci_saved_state	*pm_save;
 	int			ioeventfds_nr;
diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
index 5de618a3a5eeee..ab30b89399d038 100644
--- a/include/uapi/linux/vfio.h
+++ b/include/uapi/linux/vfio.h
@@ -1534,6 +1534,53 @@ struct vfio_device_feature_dma_buf {
  */
 #define VFIO_DEVICE_FEATURE_MIG_PRECOPY_INFOv2  12
 
+/**
+ * Given a dma_buf fd previously created by
+ * VFIO_DEVICE_FEATURE_DMA_BUF, a SET of this feature requests that
+ * access to the corresponding DMABUF is immediately and permanently
+ * revoked.  On successful return, the buffer is not accessible
+ * through any mmap() or dma-buf import.  The buffer is permanently
+ * disabled, and VFIO refuses all map, mmap, attach, etc. requests.
+ *
+ * Return: 0 on success, -1 and errno is set on failure:
+ *
+ *  EBADF, EINVAL: dmabuf_fd is not a DMABUF fd.
+ *  ENODEV: The dmabuf_fd does not match this VFIO device.
+ *  EBADFD: The DMABUF is already revoked.
+ */
+#define VFIO_DEVICE_FEATURE_DMA_BUF_REVOKE 13
+
+struct vfio_device_feature_dma_buf_revoke {
+	__s32	dmabuf_fd;
+};
+
+/**
+ * Given a dma_buf fd previously created by
+ * VFIO_DEVICE_FEATURE_DMA_BUF, SET the memory attribute that will be
+ * used by future mmap()s of that fd.  SETting a new attribute does
+ * not affect existing VMAs.
+ *
+ * The default, if no previous SET has been performed, is NC.
+ *
+ * Return: 0 on success, -1 and errno is set on failure:
+ *
+ *  EBADF, EINVAL: dmabuf_fd is not a DMABUF fd.
+ *  ENODEV: The dmabuf_fd does not match this VFIO device.
+ *  ENOENT: The given memattr is not supported.
+ */
+#define VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR 14
+
+/* Valid memory attributes for the memattr field */
+enum vfio_device_dma_buf_memattr {
+	VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR_NC = 0, /* pgprot_noncached */
+	VFIO_DEVICE_FEATURE_DMA_BUF_MEMATTR_WC = 1, /* pgprot_writecombine */
+};
+
+struct vfio_device_feature_dma_buf_memattr {
+	__s32	dmabuf_fd;
+	__u32	memattr;
+};
+
 /* -------- API for Type1 VFIO IOMMU -------- */
 
 /**