get_rp_completer_type() returns the Root Port's "TPH Completer Supported" field (bits 13:12 of Device Capabilities 2) verbatim. The 0b10 encoding is reserved, but pcie_enable_tph() feeds the raw value into the requester type: pdev->tph_req_type = min(pdev->tph_req_type, rp_req_type); and later writes tph_req_type to the TPH Requester Enable field, which only defines 0b00 (disable), 0b01 (TPH only) and 0b11 (extended TPH). No known hardware presents the reserved 0b10 in this field, so this is defensive hardening rather than a fix for observed silicon: fold the reserved encoding into "not supported" so only the three architected values can ever reach the Requester Enable field. Signed-off-by: Zhiping Zhang Acked-by: Chengwen Feng --- drivers/pci/tph.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/drivers/pci/tph.c b/drivers/pci/tph.c index 655ffd60e62f..5cbb381e2a02 100644 --- a/drivers/pci/tph.c +++ b/drivers/pci/tph.c @@ -200,6 +200,7 @@ EXPORT_SYMBOL(pcie_tph_get_st_table_size); static u8 get_rp_completer_type(struct pci_dev *pdev) { struct pci_dev *rp; + u8 tph_comp; u32 reg; int ret; @@ -211,7 +212,15 @@ static u8 get_rp_completer_type(struct pci_dev *pdev) if (ret) return 0; - return FIELD_GET(PCI_EXP_DEVCAP2_TPH_COMP_MASK, reg); + /* + * 0b10 is reserved; treat it as "not supported" so only the + * architected encodings reach the Requester Enable field. + */ + tph_comp = FIELD_GET(PCI_EXP_DEVCAP2_TPH_COMP_MASK, reg); + if (tph_comp == PCI_EXP_DEVCAP2_TPH_COMP_TPH_ONLY || + tph_comp == PCI_EXP_DEVCAP2_TPH_COMP_EXT_TPH) + return tph_comp; + return PCI_EXP_DEVCAP2_TPH_COMP_NONE; } /* Write tag to ST table - Return 0 if OK, otherwise -errno */ -- 2.53.0-Meta Add pcie_tph_enabled_req_type() so drivers can query the enabled TPH requester mode without reaching into pci_dev internals. Add pcie_tph_completer_type() so drivers that publish TPH metadata for a device acting as a completer can gate on the "TPH Completer Supported" field of Device Capabilities 2 (bits 13:12, PCI_EXP_DEVCAP2_TPH_COMP_MASK) rather than reusing requester-side state. It applies the same reserved- encoding fold as get_rp_completer_type(), inlined, so the reserved 0b10 value never reaches callers. This keeps pci_dev::tph_req_type and the completer-capability decode inside the PCI/TPH code and provides !CONFIG_PCIE_TPH stubs for callers. Signed-off-by: Zhiping Zhang --- drivers/pci/tph.c | 44 +++++++++++++++++++++++++++++++++++++++++ include/linux/pci-tph.h | 8 ++++++++ 2 files changed, 52 insertions(+) diff --git a/drivers/pci/tph.c b/drivers/pci/tph.c index 5cbb381e2a02..d0202ff7a1b2 100644 --- a/drivers/pci/tph.c +++ b/drivers/pci/tph.c @@ -223,6 +223,50 @@ static u8 get_rp_completer_type(struct pci_dev *pdev) return PCI_EXP_DEVCAP2_TPH_COMP_NONE; } +/** + * pcie_tph_enabled_req_type - Return the device's enabled TPH requester type + * @pdev: PCI device to query + * + * Return: PCI_TPH_REQ_DISABLE, PCI_TPH_REQ_TPH_ONLY or PCI_TPH_REQ_EXT_TPH. + */ +u8 pcie_tph_enabled_req_type(struct pci_dev *pdev) +{ + return pdev->tph_req_type; +} +EXPORT_SYMBOL(pcie_tph_enabled_req_type); + +/** + * pcie_tph_completer_type - Return the device's TPH Completer support + * @pdev: PCI device to query + * + * Reads the "TPH Completer Supported" field (bits 13:12) of Device + * Capabilities 2. The reserved 0b10 encoding is folded into + * "not supported" so callers only need to compare against the three + * defined values. + * + * Return: one of %PCI_EXP_DEVCAP2_TPH_COMP_NONE, + * %PCI_EXP_DEVCAP2_TPH_COMP_TPH_ONLY or + * %PCI_EXP_DEVCAP2_TPH_COMP_EXT_TPH. + */ +u8 pcie_tph_completer_type(struct pci_dev *pdev) +{ + u8 tph_comp; + u32 reg; + + if (pcie_capability_read_dword(pdev, PCI_EXP_DEVCAP2, ®)) + return PCI_EXP_DEVCAP2_TPH_COMP_NONE; + if (PCI_POSSIBLE_ERROR(reg)) + return PCI_EXP_DEVCAP2_TPH_COMP_NONE; + + /* 0b10 is reserved; fold it into "not supported" (see above). */ + tph_comp = FIELD_GET(PCI_EXP_DEVCAP2_TPH_COMP_MASK, reg); + if (tph_comp == PCI_EXP_DEVCAP2_TPH_COMP_TPH_ONLY || + tph_comp == PCI_EXP_DEVCAP2_TPH_COMP_EXT_TPH) + return tph_comp; + return PCI_EXP_DEVCAP2_TPH_COMP_NONE; +} +EXPORT_SYMBOL(pcie_tph_completer_type); + /* Write tag to ST table - Return 0 if OK, otherwise -errno */ static int write_tag_to_st_table(struct pci_dev *pdev, int index, u16 tag) { diff --git a/include/linux/pci-tph.h b/include/linux/pci-tph.h index be68cd17f2f8..7743af6fe432 100644 --- a/include/linux/pci-tph.h +++ b/include/linux/pci-tph.h @@ -9,6 +9,8 @@ #ifndef LINUX_PCI_TPH_H #define LINUX_PCI_TPH_H +#include + /* * According to the ECN for PCI Firmware Spec, Steering Tag can be different * depending on the memory type: Volatile Memory or Persistent Memory. When a @@ -30,6 +32,8 @@ void pcie_disable_tph(struct pci_dev *pdev); int pcie_enable_tph(struct pci_dev *pdev, int mode); u16 pcie_tph_get_st_table_size(struct pci_dev *pdev); u32 pcie_tph_get_st_table_loc(struct pci_dev *pdev); +u8 pcie_tph_enabled_req_type(struct pci_dev *pdev); +u8 pcie_tph_completer_type(struct pci_dev *pdev); #else static inline int pcie_tph_set_st_entry(struct pci_dev *pdev, unsigned int index, u16 tag) @@ -41,6 +45,10 @@ static inline int pcie_tph_get_cpu_st(struct pci_dev *dev, static inline void pcie_disable_tph(struct pci_dev *pdev) { } static inline int pcie_enable_tph(struct pci_dev *pdev, int mode) { return -EINVAL; } +static inline u8 pcie_tph_enabled_req_type(struct pci_dev *pdev) +{ return PCI_TPH_REQ_DISABLE; } +static inline u8 pcie_tph_completer_type(struct pci_dev *pdev) +{ return PCI_EXP_DEVCAP2_TPH_COMP_NONE; } #endif #endif /* LINUX_PCI_TPH_H */ -- 2.53.0-Meta Add an optional dma_buf_ops.get_pci_tph callback and a DMA-buf importer wrapper, dma_buf_get_pci_tph(). TPH is PCIe TLP Processing Hint. 8-bit ST and 16-bit Extended ST are distinct PCIe TPH namespaces, so the importer requests the namespace it can emit and the exporter returns the matching ST/PH tuple or -EOPNOTSUPP. dma_buf_get_pci_tph() is the importer entry point. It requires &dmabuf->resv to be held while the callback runs and returns -EOPNOTSUPP when the exporter does not provide PCI TPH metadata. The first user is VFIO_DEVICE_FEATURE_DMA_BUF_TPH in vfio-pci, with mlx5 as the first importer. Signed-off-by: Zhiping Zhang Acked-by: Chengwen Feng --- drivers/dma-buf/dma-buf.c | 32 ++++++++++++++++++++++++++++++++ include/linux/dma-buf.h | 25 +++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index d504c636dc29..71f335442ba7 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -939,6 +939,7 @@ dma_buf_pin_on_map(struct dma_buf_attachment *attach) * - dma_buf_unmap_attachment() * - dma_buf_vmap() * - dma_buf_vunmap() + * - dma_buf_get_pci_tph() * * 2. Importers must not hold the dma-buf reservation lock when calling these * functions: @@ -979,6 +980,7 @@ dma_buf_pin_on_map(struct dma_buf_attachment *attach) * - &dma_buf_ops.unmap_dma_buf() * - &dma_buf_ops.vmap() * - &dma_buf_ops.vunmap() + * - &dma_buf_ops.get_pci_tph() * * 3. Exporters must hold the dma-buf reservation lock when calling these * functions: @@ -1144,6 +1146,36 @@ void dma_buf_unpin(struct dma_buf_attachment *attach) } EXPORT_SYMBOL_NS_GPL(dma_buf_unpin, "DMA_BUF"); +/** + * dma_buf_get_pci_tph - Retrieve PCIe TLP Processing Hint (TPH) metadata + * @dmabuf: DMA buffer to query + * @extended: false for 8-bit ST, true for 16-bit Extended ST + * @steering_tag: returns the raw steering tag for the requested namespace + * @ph: returns the TPH processing hint + * + * Wrapper for the optional &dma_buf_ops.get_pci_tph callback. + * + * Must be called with &dma_buf.resv held. Returns -EOPNOTSUPP if the + * exporter does not implement the callback or has no metadata for the + * requested namespace. + * + * The returned steering tag and processing hint are only valid until the + * exporter invalidates the current mapping (signalled to the importer via + * &dma_buf_attach_ops.invalidate_mappings). Importers must re-query after a new + * mapping is established following invalidation. + */ +int dma_buf_get_pci_tph(struct dma_buf *dmabuf, bool extended, + u16 *steering_tag, u8 *ph) +{ + dma_resv_assert_held(dmabuf->resv); + + if (!dmabuf->ops->get_pci_tph) + return -EOPNOTSUPP; + + return dmabuf->ops->get_pci_tph(dmabuf, extended, steering_tag, ph); +} +EXPORT_SYMBOL_NS_GPL(dma_buf_get_pci_tph, "DMA_BUF"); + /** * dma_buf_map_attachment - Returns the scatterlist table of the attachment; * mapped into _device_ address space. Is a wrapper for map_dma_buf() of the diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h index d1203da56fc5..14e8eb61a62c 100644 --- a/include/linux/dma-buf.h +++ b/include/linux/dma-buf.h @@ -113,6 +113,29 @@ struct dma_buf_ops { */ void (*unpin)(struct dma_buf_attachment *attach); + /** + * @get_pci_tph: + * + * Retrieve PCIe TLP Processing Hint (TPH) steering metadata for + * this buffer so an importer can program a matching ST/PH hint on + * outbound TLPs targeting the exporter for peer-to-peer DMA. + * + * @dmabuf: DMA buffer for which to retrieve TPH metadata + * @extended: false for 8-bit ST, true for 16-bit Extended ST + * @steering_tag: Returns the raw TPH steering tag for the requested + * namespace + * @ph: Returns the TPH processing hint (2-bit value) + * + * Optional callback for dma_buf_get_pci_tph(). Called with + * &dma_buf.resv held. Returns 0 on success or -EOPNOTSUPP when + * the exporter has no metadata for the requested namespace. + * + * Returned metadata follows the lifetime rules documented for + * dma_buf_get_pci_tph(). + */ + int (*get_pci_tph)(struct dma_buf *dmabuf, bool extended, + u16 *steering_tag, u8 *ph); + /** * @map_dma_buf: * @@ -563,6 +586,8 @@ void dma_buf_detach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach); int dma_buf_pin(struct dma_buf_attachment *attach); void dma_buf_unpin(struct dma_buf_attachment *attach); +int dma_buf_get_pci_tph(struct dma_buf *dmabuf, bool extended, + u16 *steering_tag, u8 *ph); struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info); -- 2.53.0-Meta Implement dma-buf get_pci_tph for vfio-pci exported dma-bufs and add VFIO_DEVICE_FEATURE_DMA_BUF_TPH so userspace can publish TPH metadata for a VFIO-owned device. 8-bit ST and 16-bit Extended ST are distinct PCIe TPH namespaces; the uAPI carries both with explicit validity flags, and get_pci_tph() returns the value matching the importer's requested namespace or -EOPNOTSUPP. Publish and read the TPH descriptor under dmabuf->resv, matching the locking used for other importer-visible dma-buf state. The SET ioctl takes dma_resv_lock_interruptible(), while the callback runs under DMA-buf's asserted resv lock. Hold memory_lock in the SET path while checking that the dma-buf still belongs to this vfio-pci device and while publishing the descriptor. Cleanup clears the dma-buf/vdev association under memory_lock; taking the same lock prevents publishing into a dma-buf that has already been detached from @vdev, and preserves the memory_lock -> dmabuf->resv order used by the move path. The attach path reads @revoked without holding memory_lock. Use READ_ONCE() for that lockless attach-path test and WRITE_ONCE() for the revoke/unrevoke writers to document this intentional lockless access: the read is a benign early-out, and a racing revocation is re-checked under dmabuf->resv in vfio_pci_dma_buf_map() before any mapping is handed out. The annotation only needs to keep the access well-formed against the memory_lock-protected writers. Reject requests the device cannot consume as a completer: pcie_tph_completer_type() must report at least PCI_EXP_DEVCAP2_TPH_COMP_TPH_ONLY, and Extended ST requires PCI_EXP_DEVCAP2_TPH_COMP_EXT_TPH. Make PROBE follow the same hardware gate so the feature only probes as supported when the device can really consume it. Gate on vdev->pci_ops->get_dmabuf_phys as well, matching vfio_pci_core_feature_dma_buf(). Without it PROBE reports the feature as supported on a device that advertises TPH Completer support but cannot export a vfio dma-buf at all, so no dma-buf can ever exist to carry the metadata and every SET would fail later at the ownership check. Signed-off-by: Zhiping Zhang Acked-by: Chengwen Feng --- drivers/vfio/pci/vfio_pci_core.c | 3 + drivers/vfio/pci/vfio_pci_dmabuf.c | 120 ++++++++++++++++++++++++++++- drivers/vfio/pci/vfio_pci_priv.h | 13 ++++ include/uapi/linux/vfio.h | 43 +++++++++++ 4 files changed, 175 insertions(+), 4 deletions(-) diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index 3f11a9624b9c..4e5e34a77b76 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -1625,6 +1625,9 @@ int vfio_pci_core_ioctl_feature(struct vfio_device *device, u32 flags, return vfio_pci_core_feature_token(vdev, flags, arg, argsz); case VFIO_DEVICE_FEATURE_DMA_BUF: return vfio_pci_core_feature_dma_buf(vdev, flags, arg, argsz); + case VFIO_DEVICE_FEATURE_DMA_BUF_TPH: + return vfio_pci_core_feature_dma_buf_tph(vdev, flags, arg, + argsz); default: return -ENOTTY; } diff --git a/drivers/vfio/pci/vfio_pci_dmabuf.c b/drivers/vfio/pci/vfio_pci_dmabuf.c index c16f460c01d6..f630158204b4 100644 --- a/drivers/vfio/pci/vfio_pci_dmabuf.c +++ b/drivers/vfio/pci/vfio_pci_dmabuf.c @@ -3,6 +3,7 @@ */ #include #include +#include #include #include "vfio_pci_priv.h" @@ -19,7 +20,17 @@ struct vfio_pci_dma_buf { u32 nr_ranges; struct kref kref; struct completion comp; - u8 revoked : 1; + + /* + * Updates protected by dmabuf->resv, @revoked additionally + * protected by memory_lock. + */ + u16 tph_st_ext; + u8 tph_st; + bool revoked; + u8 tph_st_valid:1; + u8 tph_st_ext_valid:1; + u8 tph_ph:2; }; static int vfio_pci_dma_buf_attach(struct dma_buf *dmabuf, @@ -30,7 +41,7 @@ static int vfio_pci_dma_buf_attach(struct dma_buf *dmabuf, if (!attachment->peer2peer) return -EOPNOTSUPP; - if (priv->revoked) + if (READ_ONCE(priv->revoked)) return -ENODEV; if (!dma_buf_attach_revocable(attachment)) @@ -69,6 +80,26 @@ vfio_pci_dma_buf_map(struct dma_buf_attachment *attachment, return ret; } +static int vfio_pci_dma_buf_get_pci_tph(struct dma_buf *dmabuf, bool extended, + u16 *steering_tag, u8 *ph) +{ + struct vfio_pci_dma_buf *priv = dmabuf->priv; + + dma_resv_assert_held(dmabuf->resv); + + if (extended) { + if (!priv->tph_st_ext_valid) + return -EOPNOTSUPP; + *steering_tag = priv->tph_st_ext; + } else { + if (!priv->tph_st_valid) + return -EOPNOTSUPP; + *steering_tag = priv->tph_st; + } + *ph = priv->tph_ph; + return 0; +} + static void vfio_pci_dma_buf_unmap(struct dma_buf_attachment *attachment, struct sg_table *sgt, enum dma_data_direction dir) @@ -101,6 +132,7 @@ static void vfio_pci_dma_buf_release(struct dma_buf *dmabuf) static const struct dma_buf_ops vfio_pci_dmabuf_ops = { .attach = vfio_pci_dma_buf_attach, + .get_pci_tph = vfio_pci_dma_buf_get_pci_tph, .map_dma_buf = vfio_pci_dma_buf_map, .unmap_dma_buf = vfio_pci_dma_buf_unmap, .release = vfio_pci_dma_buf_release, @@ -333,6 +365,86 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags, return ret; } +int vfio_pci_core_feature_dma_buf_tph(struct vfio_pci_core_device *vdev, + u32 flags, + struct vfio_device_feature_dma_buf_tph __user *arg, + size_t argsz) +{ + struct vfio_device_feature_dma_buf_tph set_tph; + struct vfio_pci_dma_buf *priv; + struct dma_buf *dmabuf; + u8 comp; + int ret; + + if (!vdev->pci_ops || !vdev->pci_ops->get_dmabuf_phys) + return -EOPNOTSUPP; + + comp = pcie_tph_completer_type(vdev->pdev); + if (comp == PCI_EXP_DEVCAP2_TPH_COMP_NONE) + return -EOPNOTSUPP; + + ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET, + sizeof(set_tph)); + if (ret != 1) + return ret; + + if (copy_from_user(&set_tph, arg, sizeof(set_tph))) + return -EFAULT; + + if (set_tph.flags & ~(VFIO_DMA_BUF_TPH_ST | VFIO_DMA_BUF_TPH_ST_EXT)) + return -EINVAL; + + if (set_tph.ph & ~0x3) + return -EINVAL; + + if ((set_tph.flags & VFIO_DMA_BUF_TPH_ST_EXT) && + comp != PCI_EXP_DEVCAP2_TPH_COMP_EXT_TPH) + return -EOPNOTSUPP; + + dmabuf = dma_buf_get(set_tph.dmabuf_fd); + if (IS_ERR(dmabuf)) + return PTR_ERR(dmabuf); + + if (dmabuf->ops != &vfio_pci_dmabuf_ops) { + ret = -EINVAL; + goto out_put; + } + + priv = dmabuf->priv; + + /* + * @revoked and the dma-buf/vdev association are torn down under + * memory_lock by vfio_pci_dma_buf_cleanup(); hold it while validating + * ownership and publishing so the dma-buf cannot be disassociated from + * @vdev underneath us. Lock order memory_lock -> dmabuf->resv matches + * vfio_pci_dma_buf_move(). + */ + down_read(&vdev->memory_lock); + if (priv->vdev != vdev) { + ret = -EINVAL; + goto out_unlock_mem; + } + + ret = dma_resv_lock_interruptible(dmabuf->resv, NULL); + if (ret) + goto out_unlock_mem; + + priv->tph_st = set_tph.steering_tag; + priv->tph_st_ext = set_tph.steering_tag_ext; + priv->tph_ph = set_tph.ph; + priv->tph_st_valid = !!(set_tph.flags & VFIO_DMA_BUF_TPH_ST); + priv->tph_st_ext_valid = + !!(set_tph.flags & VFIO_DMA_BUF_TPH_ST_EXT); + dma_resv_unlock(dmabuf->resv); + ret = 0; + +out_unlock_mem: + up_read(&vdev->memory_lock); +out_put: + dma_buf_put(dmabuf); + return ret; +} + void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev, bool revoked) { struct vfio_pci_dma_buf *priv; @@ -347,7 +459,7 @@ void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev, bool revoked) if (priv->revoked != revoked) { dma_resv_lock(priv->dmabuf->resv, NULL); if (revoked) - priv->revoked = true; + WRITE_ONCE(priv->revoked, true); dma_buf_invalidate_mappings(priv->dmabuf); dma_resv_wait_timeout(priv->dmabuf->resv, DMA_RESV_USAGE_BOOKKEEP, false, @@ -369,7 +481,7 @@ void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev, bool revoked) reinit_completion(&priv->comp); } else { dma_resv_lock(priv->dmabuf->resv, NULL); - priv->revoked = false; + WRITE_ONCE(priv->revoked, false); dma_resv_unlock(priv->dmabuf->resv); } } diff --git a/drivers/vfio/pci/vfio_pci_priv.h b/drivers/vfio/pci/vfio_pci_priv.h index fca9d0dfac90..063f3748478a 100644 --- a/drivers/vfio/pci/vfio_pci_priv.h +++ b/drivers/vfio/pci/vfio_pci_priv.h @@ -118,6 +118,10 @@ static inline bool vfio_pci_is_vga(struct pci_dev *pdev) int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags, struct vfio_device_feature_dma_buf __user *arg, size_t argsz); +int vfio_pci_core_feature_dma_buf_tph(struct vfio_pci_core_device *vdev, + u32 flags, + struct vfio_device_feature_dma_buf_tph __user *arg, + size_t argsz); void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device *vdev); void vfio_pci_dma_buf_move(struct vfio_pci_core_device *vdev, bool revoked); #else @@ -128,6 +132,15 @@ vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags, { return -ENOTTY; } + +static inline int +vfio_pci_core_feature_dma_buf_tph(struct vfio_pci_core_device *vdev, u32 flags, + struct vfio_device_feature_dma_buf_tph __user *arg, + size_t argsz) +{ + return -ENOTTY; +} + static inline void vfio_pci_dma_buf_cleanup(struct vfio_pci_core_device *vdev) { } diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h index 5de618a3a5ee..4c1c70aac150 100644 --- a/include/uapi/linux/vfio.h +++ b/include/uapi/linux/vfio.h @@ -1534,6 +1534,49 @@ struct vfio_device_feature_dma_buf { */ #define VFIO_DEVICE_FEATURE_MIG_PRECOPY_INFOv2 12 +/** + * Upon VFIO_DEVICE_FEATURE_SET associate TPH (TLP Processing Hints) metadata + * with a vfio-exported dma-buf. The dma-buf must have been created by + * VFIO_DEVICE_FEATURE_DMA_BUF on this device, and the device must report + * TPH Completer support in Device Capabilities 2 (bits 13:12); requests + * carrying VFIO_DMA_BUF_TPH_ST_EXT additionally require the device to + * report the Extended TPH Completer encoding. Otherwise the ioctl + * returns -EOPNOTSUPP. + * + * @dmabuf_fd is the file descriptor returned by VFIO_DEVICE_FEATURE_DMA_BUF. + * + * 8-bit ST (@steering_tag) and 16-bit Extended ST (@steering_tag_ext) are + * distinct namespaces. @flags is the authoritative validity mask for the + * two namespaces: VFIO_DMA_BUF_TPH_ST marks @steering_tag valid and + * VFIO_DMA_BUF_TPH_ST_EXT marks @steering_tag_ext valid. An importer + * requests one namespace and receives the matching value. + * + * Each SET fully replaces the dma-buf's TPH state for future queries + * only. A namespace whose flag bit is clear is reported as unsupported + * to future importers; an importer that has already retrieved a value + * is unaffected. @flags == 0 leaves both namespaces unsupported for + * future importers. + * + * @ph is the 2-bit TLP Processing Hint and must be in the range [0, 3]. + * + * Undefined @flags and @ph bits must always be zero. + * + * Userspace must configure TPH before handing the dma-buf fd to an + * importer. + */ +#define VFIO_DEVICE_FEATURE_DMA_BUF_TPH 13 + +#define VFIO_DMA_BUF_TPH_ST (1 << 0) +#define VFIO_DMA_BUF_TPH_ST_EXT (1 << 1) + +struct vfio_device_feature_dma_buf_tph { + __s32 dmabuf_fd; + __u32 flags; + __u16 steering_tag_ext; + __u8 steering_tag; + __u8 ph; +}; + /* -------- API for Type1 VFIO IOMMU -------- */ /** -- 2.53.0-Meta Peer-to-peer DMA between a mlx5 NIC and a foreign PCIe endpoint (typically a GPU or a vfio-pci passthrough device) traverses the host PCIe fabric. The endpoint exporting the dma-buf knows which PCIe TLP Processing Hint (TPH) Steering Tag yields the best placement for the traffic it will sink: per-endpoint hint selection lets the root complex or switch direct DMA to a specific cache slice / NUMA node, cutting cross-socket snoop traffic and DRAM pressure under sustained p2p workloads. Until now the mlx5 importer had no way to learn the exporter's chosen ST tag, so dma-buf MRs were registered without TPH and ran with the default (no-hint) routing. With dma_buf_get_pci_tph() in place this patch wires up mlx5_ib to query that metadata at MR registration time for p2p access and use it to program requester-side TPH on the outbound mkey. If the exporter has no metadata, fall back to the existing no-TPH path so behavior for non-TPH-aware exporters is unchanged. Use mlx5_st_alloc_index_by_tag() to translate exporter-provided steering tags into local ST entries when table mode is active, and add mlx5_st_get_index() for DMAH-backed flows that already carry an ST index. Splitting mlx5_st_alloc_index() moves the !dev->st check into the new by-tag helper, so keep an equivalent early-out in mlx5_st_alloc_index() itself, ahead of the pcie_tph_get_cpu_st() call. A device without ST support therefore still returns -EOPNOTSUPP without an ACPI _DSM invocation, exactly as it did before the split. The check is repeated in mlx5_st_alloc_index_by_tag() because that is now an exported entry point in its own right. For TPH-backed FRMRs, keep the extra ST-table reference tied to MR lifetime rather than pooled mkey lifetime. Acquire the ref before MR creation and release it again when the MR is returned to the pool or the backing mkey is destroyed, while leaving the generic FRMR pool core unchanged. The exporter's steering tag is only valid for the mapping it was queried against (see dma_buf_get_pci_tph()). The mkey's TPH fields cannot be reprogrammed in place: they are create-time fields with no UMR update mask, and rebuilding the mkey would change the rkey/lkey, which must stay stable across invalidate/refault. Keep the registration-time TPH tuple in the MR and re-query the exporter after each dma-buf mapping is established under dmabuf->resv. If the tuple is unchanged, continue using the existing mkey. If the tuple changed or disappeared, unmap the pages and return -EFAULT rather than continue with a stale mkey hint; userspace must create a new MR to get a mkey programmed with the new tuple. For vfio-pci BAR dma-bufs this validation is expected to be a no-op because invalidation is revoke/quiesce of BAR access, not movement to a different backing placement, and the userspace-provided TPH tuple is not changed by the revoke/un-revoke path. Import the DMA_BUF namespace for the new dma_buf_get_pci_tph() call so modular mlx5_ib builds link cleanly. Signed-off-by: Zhiping Zhang --- drivers/infiniband/hw/mlx5/main.c | 1 + drivers/infiniband/hw/mlx5/mlx5_ib.h | 11 ++ drivers/infiniband/hw/mlx5/mr.c | 151 +++++++++++++++++- drivers/infiniband/hw/mlx5/odp.c | 7 + .../net/ethernet/mellanox/mlx5/core/lib/st.c | 52 +++++- include/linux/mlx5/driver.h | 15 ++ 6 files changed, 223 insertions(+), 14 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 02809114fc79..a2b497f6b16b 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -60,6 +60,7 @@ MODULE_AUTHOR("Eli Cohen "); MODULE_DESCRIPTION("Mellanox 5th generation network adapters (ConnectX series) IB driver"); MODULE_LICENSE("Dual BSD/GPL"); +MODULE_IMPORT_NS("DMA_BUF"); struct mlx5_ib_event_work { struct work_struct work; diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 522984d958bb..c6a956b2ee87 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -646,6 +646,15 @@ enum mlx5_mkey_type { /* Used for non-existent ph value */ #define MLX5_IB_NO_PH 0xff +struct dma_buf; + +struct mlx5_ib_dmabuf_tph { + u16 steering_tag; + u8 ph; + bool extended; + bool valid; +}; + struct mlx5_ib_mkey { u32 key; enum mlx5_mkey_type type; @@ -726,6 +735,7 @@ struct mlx5_ib_mr { u8 revoked :1; /* Indicates previous dmabuf page fault occurred */ u8 dmabuf_faulted:1; + struct mlx5_ib_dmabuf_tph dmabuf_tph; struct mlx5_ib_mkey null_mmkey; }; }; @@ -1341,6 +1351,7 @@ struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, u64 length, u64 virt_addr, int access_flags, struct ib_pd *pd, struct ib_udata *udata); int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata); +int mlx5_ib_validate_dmabuf_tph(struct mlx5_ib_mr *mr, struct dma_buf *dmabuf); struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, u32 max_num_sg); struct ib_mr *mlx5_ib_alloc_mr_integrity(struct ib_pd *pd, diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index e6b74955d95d..404c5d081b8a 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include "dm.h" @@ -167,6 +168,61 @@ static int get_unchangeable_access_flags(struct mlx5_ib_dev *dev, #define MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK GENMASK_ULL(23, 16) #define MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK GENMASK_ULL(15, 0) +static u8 mlx5_ib_tph_key_ph(u8 ph) +{ + if (ph == MLX5_IB_NO_PH || ph == 0) + ph ^= MLX5_IB_NO_PH; + + return ph; +} + +static int mlx5_ib_get_st_handle_ref(struct mlx5_ib_dev *dev, u16 st_index) +{ + if (st_index == MLX5_MKC_PCIE_TPH_NO_STEERING_TAG_INDEX) + return 0; + + return mlx5_st_get_index(dev->mdev, st_index); +} + +static void mlx5_ib_put_st_index_ref(struct mlx5_ib_dev *dev, u16 st_index) +{ + if (st_index == MLX5_MKC_PCIE_TPH_NO_STEERING_TAG_INDEX) + return; + + mlx5_st_dealloc_index(dev->mdev, st_index); +} + +static void mlx5_ib_put_st_handle_ref(struct mlx5_ib_dev *dev, + u64 kernel_vendor_key) +{ + u16 st_index = FIELD_GET(MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK, + kernel_vendor_key); + + mlx5_ib_put_st_index_ref(dev, st_index); +} + +int mlx5_ib_validate_dmabuf_tph(struct mlx5_ib_mr *mr, struct dma_buf *dmabuf) +{ + struct mlx5_ib_dmabuf_tph *tph = &mr->dmabuf_tph; + u16 steering_tag; + u8 ph; + int ret; + + if (!tph->valid) + return 0; + + dma_resv_assert_held(dmabuf->resv); + + ret = dma_buf_get_pci_tph(dmabuf, tph->extended, &steering_tag, &ph); + if (ret) + return -EFAULT; + + if (steering_tag != tph->steering_tag || ph != tph->ph) + return -EFAULT; + + return 0; +} + static struct mlx5_ib_mr * _mlx5_frmr_pool_alloc(struct mlx5_ib_dev *dev, struct ib_umem *umem, int access_flags, int access_mode, @@ -189,13 +245,10 @@ _mlx5_frmr_pool_alloc(struct mlx5_ib_dev *dev, struct ib_umem *umem, MLX5_FRMR_POOLS_KEY_ACCESS_MODE_KSM_MASK : 0; - /* Normalize ph: swap 0 and MLX5_IB_NO_PH */ - if (ph == MLX5_IB_NO_PH || ph == 0) - ph ^= MLX5_IB_NO_PH; - mr->ibmr.frmr.key.kernel_vendor_key = FIELD_PREP(MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK, st_index) | - FIELD_PREP(MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK, ph); + FIELD_PREP(MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK, + mlx5_ib_tph_key_ph(ph)); err = ib_frmr_pool_pop(&dev->ib_dev, &mr->ibmr); if (err) { kfree(mr); @@ -218,7 +271,9 @@ struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, 0 : MLX5_FRMR_POOLS_KEY_ACCESS_MODE_KSM_MASK, .num_dma_blocks = ndescs, - .kernel_vendor_key = 0, /* no PH and no ST index */ + .kernel_vendor_key = + FIELD_PREP(MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK, + MLX5_MKC_PCIE_TPH_NO_STEERING_TAG_INDEX), }; struct mlx5_ib_mr *mr; int ret; @@ -557,6 +612,10 @@ static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem, mr->ibmr.pd = pd; mr->access_flags = access_flags; mr->page_shift = order_base_2(page_size); + mr->ibmr.frmr.key.kernel_vendor_key = + FIELD_PREP(MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK, st_index) | + FIELD_PREP(MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK, + mlx5_ib_tph_key_ph(ph)); inlen = MLX5_ST_SZ_BYTES(create_mkey_in); if (populate) @@ -753,6 +812,12 @@ static struct ib_mr *create_real_mr(struct ib_pd *pd, struct ib_umem *umem, st_index = mdmah->st_index; } + err = mlx5_ib_get_st_handle_ref(dev, st_index); + if (err) { + ib_umem_release(umem); + return ERR_PTR(err); + } + xlt_with_umr = mlx5r_umr_can_load_pas(dev, umem->length); if (xlt_with_umr) { mr = alloc_cacheable_mr(pd, umem, iova, access_flags, @@ -769,6 +834,7 @@ static struct ib_mr *create_real_mr(struct ib_pd *pd, struct ib_umem *umem, mutex_unlock(&dev->slow_path_mutex); } if (IS_ERR(mr)) { + mlx5_ib_put_st_index_ref(dev, st_index); ib_umem_release(umem); return ERR_CAST(mr); } @@ -903,6 +969,63 @@ static struct dma_buf_attach_ops mlx5_ib_dmabuf_attach_ops = { .invalidate_mappings = mlx5_ib_dmabuf_invalidate_cb, }; +static void get_pci_tph_mr_dmabuf(struct mlx5_ib_dev *dev, struct dma_buf *dmabuf, + struct mlx5_ib_dmabuf_tph *tph, + u16 *st_index, u8 *ph) +{ + u16 local_st_index; + u16 steering_tag; + u8 local_ph; + bool extended; + int ret; + + switch (pcie_tph_enabled_req_type(dev->mdev->pdev)) { + case PCI_TPH_REQ_TPH_ONLY: + extended = false; + break; + case PCI_TPH_REQ_EXT_TPH: + extended = true; + break; + default: + return; + } + + /* + * dma_buf_get_pci_tph() must run under dmabuf->resv, but drop the lock + * before mlx5_st_alloc_index_by_tag(): that allocates with GFP_KERNEL, + * which must not run under dmabuf->resv since reclaim can re-enter the + * same reservation object through a dynamic exporter's move_notify path. + */ + dma_resv_lock(dmabuf->resv, NULL); + ret = dma_buf_get_pci_tph(dmabuf, extended, &steering_tag, &local_ph); + dma_resv_unlock(dmabuf->resv); + if (ret) { + if (ret != -EOPNOTSUPP) + mlx5_ib_dbg(dev, "get_pci_tph failed (%d)\n", ret); + return; + } + + ret = mlx5_st_alloc_index_by_tag(dev->mdev, steering_tag, + &local_st_index); + if (ret) { + mlx5_ib_dbg(dev, "st_alloc_index_by_tag failed (%d)\n", ret); + return; + } + + *st_index = local_st_index; + *ph = local_ph; + tph->steering_tag = steering_tag; + tph->ph = local_ph; + tph->extended = extended; + tph->valid = true; +} + +static void mlx5_ib_mr_put_st_handle_ref(struct mlx5_ib_mr *mr) +{ + mlx5_ib_put_st_handle_ref(mr_to_mdev(mr), + mr->ibmr.frmr.key.kernel_vendor_key); +} + static struct ib_mr * reg_user_mr_dmabuf(struct ib_pd *pd, struct device *dma_device, u64 offset, u64 length, u64 virt_addr, @@ -913,6 +1036,7 @@ reg_user_mr_dmabuf(struct ib_pd *pd, struct device *dma_device, struct mlx5_ib_dev *dev = to_mdev(pd->device); struct mlx5_ib_mr *mr = NULL; struct ib_umem_dmabuf *umem_dmabuf; + struct mlx5_ib_dmabuf_tph tph = {}; u16 st_index = MLX5_MKC_PCIE_TPH_NO_STEERING_TAG_INDEX; u8 ph = MLX5_IB_NO_PH; int err; @@ -945,18 +1069,29 @@ reg_user_mr_dmabuf(struct ib_pd *pd, struct device *dma_device, ph = dmah->ph; if (dmah->valid_fields & BIT(IB_DMAH_CPU_ID_EXISTS)) st_index = mdmah->st_index; + + err = mlx5_ib_get_st_handle_ref(dev, st_index); + if (err) { + ib_umem_release(&umem_dmabuf->umem); + return ERR_PTR(err); + } + } else { + get_pci_tph_mr_dmabuf(dev, umem_dmabuf->attach->dmabuf, + &tph, &st_index, &ph); } mr = alloc_cacheable_mr(pd, &umem_dmabuf->umem, virt_addr, access_flags, access_mode, st_index, ph); if (IS_ERR(mr)) { + mlx5_ib_put_st_index_ref(dev, st_index); ib_umem_release(&umem_dmabuf->umem); return ERR_CAST(mr); } mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmkey.key); + mr->dmabuf_tph = tph; atomic_add(ib_umem_num_pages(mr->umem), &dev->mdev->priv.reg_pages); umem_dmabuf->private = mr; if (!pinned_mode) { @@ -1405,6 +1540,7 @@ static int mlx5r_handle_mkey_cleanup(struct mlx5_ib_mr *mr) if (mr->ibmr.frmr.pool) { if (!mlx5_umr_revoke_mr_with_lock(mr)) { ib_frmr_pool_push(mr->ibmr.device, &mr->ibmr); + mlx5_ib_mr_put_st_handle_ref(mr); return 0; } } @@ -1432,6 +1568,9 @@ static int mlx5r_handle_mkey_cleanup(struct mlx5_ib_mr *mr) if (mr->ibmr.frmr.pool && !ret) ib_frmr_pool_drop(&mr->ibmr); + if (!ret) + mlx5_ib_mr_put_st_handle_ref(mr); + return ret; } diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index 1badec9bf527..7c1731aa3dd9 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -857,6 +857,13 @@ static int pagefault_dmabuf_mr(struct mlx5_ib_mr *mr, size_t bcnt, return err; } + err = mlx5_ib_validate_dmabuf_tph(mr, umem_dmabuf->attach->dmabuf); + if (err) { + ib_umem_dmabuf_unmap_pages(umem_dmabuf); + dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv); + return err; + } + page_size = mlx5_umem_dmabuf_find_best_pgsz(umem_dmabuf, access_mode); if (!page_size) { ib_umem_dmabuf_unmap_pages(umem_dmabuf); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/st.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/st.c index 7cedc348790d..c62bcae7c542 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/st.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/st.c @@ -92,23 +92,18 @@ void mlx5_st_destroy(struct mlx5_core_dev *dev) kfree(st); } -int mlx5_st_alloc_index(struct mlx5_core_dev *dev, enum tph_mem_type mem_type, - unsigned int cpu_uid, u16 *st_index) +int mlx5_st_alloc_index_by_tag(struct mlx5_core_dev *dev, u16 tag, + u16 *st_index) { struct mlx5_st_idx_data *idx_data; struct mlx5_st *st = dev->st; unsigned long index; u32 xa_id; - u16 tag; - int ret; + int ret = 0; if (!st) return -EOPNOTSUPP; - ret = pcie_tph_get_cpu_st(dev->pdev, mem_type, cpu_uid, &tag); - if (ret) - return ret; - if (st->direct_mode) { *st_index = tag; return 0; @@ -152,8 +147,49 @@ int mlx5_st_alloc_index(struct mlx5_core_dev *dev, enum tph_mem_type mem_type, mutex_unlock(&st->lock); return ret; } +EXPORT_SYMBOL_GPL(mlx5_st_alloc_index_by_tag); + +int mlx5_st_alloc_index(struct mlx5_core_dev *dev, enum tph_mem_type mem_type, + unsigned int cpu_uid, u16 *st_index) +{ + u16 tag; + int ret; + + if (!dev->st) + return -EOPNOTSUPP; + + ret = pcie_tph_get_cpu_st(dev->pdev, mem_type, cpu_uid, &tag); + if (ret) + return ret; + + return mlx5_st_alloc_index_by_tag(dev, tag, st_index); +} EXPORT_SYMBOL_GPL(mlx5_st_alloc_index); +int mlx5_st_get_index(struct mlx5_core_dev *dev, u16 st_index) +{ + struct mlx5_st_idx_data *idx_data; + struct mlx5_st *st = dev->st; + int ret = 0; + + if (!st) + return -EOPNOTSUPP; + + if (st->direct_mode) + return 0; + + mutex_lock(&st->lock); + idx_data = xa_load(&st->idx_xa, st_index); + if (WARN_ON_ONCE(!idx_data)) + ret = -EINVAL; + else + refcount_inc(&idx_data->usecount); + mutex_unlock(&st->lock); + + return ret; +} +EXPORT_SYMBOL_GPL(mlx5_st_get_index); + int mlx5_st_dealloc_index(struct mlx5_core_dev *dev, u16 st_index) { struct mlx5_st_idx_data *idx_data; diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index b1871c0821d0..d26e5744c843 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -1182,16 +1182,31 @@ int mlx5_dm_sw_icm_dealloc(struct mlx5_core_dev *dev, enum mlx5_sw_icm_type type u64 length, u16 uid, phys_addr_t addr, u32 obj_id); #ifdef CONFIG_PCIE_TPH +int mlx5_st_alloc_index_by_tag(struct mlx5_core_dev *dev, u16 tag, + u16 *st_index); +int mlx5_st_get_index(struct mlx5_core_dev *dev, u16 st_index); int mlx5_st_alloc_index(struct mlx5_core_dev *dev, enum tph_mem_type mem_type, unsigned int cpu_uid, u16 *st_index); int mlx5_st_dealloc_index(struct mlx5_core_dev *dev, u16 st_index); #else +static inline int mlx5_st_alloc_index_by_tag(struct mlx5_core_dev *dev, + u16 tag, u16 *st_index) +{ + return -EOPNOTSUPP; +} + +static inline int mlx5_st_get_index(struct mlx5_core_dev *dev, u16 st_index) +{ + return -EOPNOTSUPP; +} + static inline int mlx5_st_alloc_index(struct mlx5_core_dev *dev, enum tph_mem_type mem_type, unsigned int cpu_uid, u16 *st_index) { return -EOPNOTSUPP; } + static inline int mlx5_st_dealloc_index(struct mlx5_core_dev *dev, u16 st_index) { return -EOPNOTSUPP; -- 2.53.0-Meta