From: Dragos Tatulea There are three per device data direct objects: mlx5_data_direct_dev, mlx5_data_direct_resources and mlx5_data_direct_registration with separate allocations. This patch consolidates all of them into the registration object. For this, a few things needed to happen: - mlx5_ib_dev keeps a pointer to mlx5_data_direct. - mlx5_data_direct_dev is now accessible from mlx5_data_direct. mlx5_data_direct_get_dev() is added as a helper to access the device pointer. - Since mlx5_data_direct is part of mlx5_data_direct_registration, getting to the registration is a simple container_of. No need for searching the list. Since mlx5_data_direct_resources no longer has an accurate name, this patch renames it. This is a preparation for the upcoming move of data_direct to mlx5_core. This patch has no functional changes. Signed-off-by: Dragos Tatulea Signed-off-by: Tariq Toukan --- drivers/infiniband/hw/mlx5/data_direct.c | 73 ++++++++++-------------- drivers/infiniband/hw/mlx5/data_direct.h | 7 --- drivers/infiniband/hw/mlx5/mlx5_ib.h | 3 +- drivers/infiniband/hw/mlx5/mr.c | 6 +- drivers/infiniband/hw/mlx5/odp.c | 2 +- drivers/infiniband/hw/mlx5/std_types.c | 2 +- drivers/infiniband/hw/mlx5/umr.c | 11 ++-- include/linux/mlx5/data_direct.h | 17 ++++++ 8 files changed, 59 insertions(+), 62 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c index 16afacdd3601..f58a144d95e3 100644 --- a/drivers/infiniband/hw/mlx5/data_direct.c +++ b/drivers/infiniband/hw/mlx5/data_direct.c @@ -18,10 +18,10 @@ static LIST_HEAD(mlx5_data_direct_reg_list); static DEFINE_MUTEX(mlx5_data_direct_mutex); struct mlx5_data_direct_registration { - struct mlx5_ib_dev *ibdev; char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1]; struct list_head list; struct blocking_notifier_head users; + struct mlx5_data_direct dd; }; static int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, @@ -139,8 +139,8 @@ int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev) if (err) goto err_mkey; - dev->ddr.mkey = mkey; - dev->ddr.pdn = pdn; + dev->data_direct->mkey = mkey; + dev->data_direct->pdn = pdn; /* create another mkey with RO support */ if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write)) { @@ -157,8 +157,8 @@ int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev) err = mlx5_core_create_mkey(mdev, &mkey, in, inlen); /* RO is defined as best effort */ if (!err) { - dev->ddr.mkey_ro = mkey; - dev->ddr.mkey_ro_valid = true; + dev->data_direct->mkey_ro = mkey; + dev->data_direct->mkey_ro_valid = true; } } @@ -174,38 +174,25 @@ int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev) void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev) { - if (dev->ddr.mkey_ro_valid) - mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey_ro); + if (dev->data_direct->mkey_ro_valid) + mlx5_core_destroy_mkey(dev->mdev, dev->data_direct->mkey_ro); - mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey); - mlx5_core_dealloc_pd(dev->mdev, dev->ddr.pdn); + mlx5_core_destroy_mkey(dev->mdev, dev->data_direct->mkey); + mlx5_core_dealloc_pd(dev->mdev, dev->data_direct->pdn); - memset(&dev->ddr, 0, sizeof(dev->ddr)); + memset(dev->data_direct, 0, sizeof(*dev->data_direct)); } -static struct mlx5_data_direct_registration * -mlx5_data_direct_get_reg(struct mlx5_ib_dev *ibdev) -{ - struct mlx5_data_direct_registration *reg; - - list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) - if (reg->ibdev == ibdev) - return reg; - return NULL; -} - -static void mlx5_data_direct_bind(struct mlx5_ib_dev *ibdev, +static void mlx5_data_direct_bind(struct mlx5_data_direct_registration *reg, struct mlx5_data_direct_dev *dev) { - WRITE_ONCE(ibdev->data_direct_dev, dev); + WRITE_ONCE(reg->dd.dev, dev); } static void mlx5_data_direct_do_unbind(struct mlx5_data_direct_registration *reg) { - struct mlx5_ib_dev *ibdev = reg->ibdev; - - WRITE_ONCE(ibdev->data_direct_dev, NULL); + WRITE_ONCE(reg->dd.dev, NULL); blocking_notifier_call_chain(®->users, MLX5_DATA_DIRECT_UNBIND, NULL); } @@ -223,7 +210,6 @@ int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev) if (!reg) return -ENOMEM; - reg->ibdev = ibdev; BLOCKING_INIT_NOTIFIER_HEAD(®->users); err = mlx5_data_direct_query_vuid(ibdev->mdev, reg->vuid); @@ -234,10 +220,12 @@ int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev) return err; } + ibdev->data_direct = ®->dd; + mutex_lock(&mlx5_data_direct_mutex); list_for_each_entry(dev, &mlx5_data_direct_dev_list, list) { if (strcmp(dev->vuid, reg->vuid) == 0) { - mlx5_data_direct_bind(ibdev, dev); + mlx5_data_direct_bind(reg, dev); break; } } @@ -257,14 +245,14 @@ void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev) if (!mlx5_data_direct_supported(ibdev->mdev)) return; + reg = container_of(ibdev->data_direct, + struct mlx5_data_direct_registration, dd); mutex_lock(&mlx5_data_direct_mutex); - reg = mlx5_data_direct_get_reg(ibdev); - if (reg) { - list_del(®->list); - mlx5_data_direct_do_unbind(reg); - } + list_del(®->list); + mlx5_data_direct_do_unbind(reg); mutex_unlock(&mlx5_data_direct_mutex); + ibdev->data_direct = NULL; kfree(reg); } @@ -276,11 +264,10 @@ int mlx5_data_direct_register(struct mlx5_ib_dev *ibdev, if (!mlx5_data_direct_supported(ibdev->mdev)) return 0; - mutex_lock(&mlx5_data_direct_mutex); - reg = mlx5_data_direct_get_reg(ibdev); - if (reg) - blocking_notifier_chain_register(®->users, nb); - mutex_unlock(&mlx5_data_direct_mutex); + reg = container_of(ibdev->data_direct, + struct mlx5_data_direct_registration, dd); + blocking_notifier_chain_register(®->users, nb); + return 0; } EXPORT_SYMBOL_GPL(mlx5_data_direct_register); @@ -293,11 +280,9 @@ void mlx5_data_direct_unregister(struct mlx5_ib_dev *ibdev, if (!mlx5_data_direct_supported(ibdev->mdev)) return; - mutex_lock(&mlx5_data_direct_mutex); - reg = mlx5_data_direct_get_reg(ibdev); - if (reg) - blocking_notifier_chain_unregister(®->users, nb); - mutex_unlock(&mlx5_data_direct_mutex); + reg = container_of(ibdev->data_direct, + struct mlx5_data_direct_registration, dd); + blocking_notifier_chain_unregister(®->users, nb); } EXPORT_SYMBOL_GPL(mlx5_data_direct_unregister); @@ -308,7 +293,7 @@ static void mlx5_data_direct_dev_reg(struct mlx5_data_direct_dev *dev) mutex_lock(&mlx5_data_direct_mutex); list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) { if (strcmp(dev->vuid, reg->vuid) == 0) - mlx5_data_direct_bind(reg->ibdev, dev); + mlx5_data_direct_bind(reg, dev); } /* Add the data direct device to the global list, further IB devices may diff --git a/drivers/infiniband/hw/mlx5/data_direct.h b/drivers/infiniband/hw/mlx5/data_direct.h index f207239461bf..de2730f00852 100644 --- a/drivers/infiniband/hw/mlx5/data_direct.h +++ b/drivers/infiniband/hw/mlx5/data_direct.h @@ -23,13 +23,6 @@ struct mlx5_data_direct_dev { struct list_head list; }; -struct mlx5_data_direct_resources { - u32 pdn; - u32 mkey; - u32 mkey_ro; - u8 mkey_ro_valid :1; -}; - int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev); void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev); diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 8792e10340c5..6952170f472b 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -1091,7 +1091,7 @@ struct mlx5_macsec { struct mlx5_ib_dev { struct ib_device ib_dev; struct mlx5_core_dev *mdev; - struct mlx5_data_direct_dev *data_direct_dev; + struct mlx5_data_direct *data_direct; /* Protects data_direct_mr_list and serializes mr * registration/deregistration with data direct device unbind. */ @@ -1152,7 +1152,6 @@ struct mlx5_ib_dev { u16 pkey_table_len; u8 lag_ports; struct mlx5_special_mkeys mkeys; - struct mlx5_data_direct_resources ddr; #ifdef CONFIG_MLX5_MACSEC struct mlx5_macsec macsec; diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 1fc61a680f8a..5efe78cb34c1 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -587,7 +587,7 @@ static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem, populate ? pd : dev->umrc.pd); /* In case a data direct flow, overwrite the pdn field by its internal kernel PD */ if (umem->is_dmabuf && ksm_mode) - MLX5_SET(mkc, mkc, pd, dev->ddr.pdn); + MLX5_SET(mkc, mkc, pd, dev->data_direct->pdn); MLX5_SET(mkc, mkc, free, !populate); MLX5_SET(mkc, mkc, access_mode_1_0, access_mode); @@ -997,7 +997,7 @@ reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset, return ERR_PTR(-EOPNOTSUPP); mutex_lock(&dev->data_direct_lock); - data_direct_dev = READ_ONCE(dev->data_direct_dev); + data_direct_dev = mlx5_data_direct_get_dev(dev->data_direct); if (!data_direct_dev) { ret = -EINVAL; goto end; @@ -1006,7 +1006,7 @@ reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset, /* If no device's 'data direct mkey' with RO flags exists * mask it out accordingly. */ - if (!dev->ddr.mkey_ro_valid) + if (!dev->data_direct->mkey_ro_valid) access_flags &= ~IB_ACCESS_RELAXED_ORDERING; crossed_mr = reg_user_mr_dmabuf(pd, &data_direct_dev->pdev->dev, offset, length, virt_addr, fd, diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index 1badec9bf527..7b9f73b76d0d 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -952,7 +952,7 @@ int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd) int ret; if (mr->data_direct) - pdn = dev->ddr.pdn; + pdn = dev->data_direct->pdn; else pdn = to_mpd(pd)->pdn; diff --git a/drivers/infiniband/hw/mlx5/std_types.c b/drivers/infiniband/hw/mlx5/std_types.c index df949fa634c9..3828100fdc6c 100644 --- a/drivers/infiniband/hw/mlx5/std_types.c +++ b/drivers/infiniband/hw/mlx5/std_types.c @@ -203,7 +203,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_GET_DATA_DIRECT_SYSFS_PATH)( return PTR_ERR(c); dev = to_mdev(c->ibucontext.device); mutex_lock(&dev->data_direct_lock); - data_direct_dev = READ_ONCE(dev->data_direct_dev); + data_direct_dev = mlx5_data_direct_get_dev(dev->data_direct); if (!data_direct_dev) { ret = -ENODEV; goto end; diff --git a/drivers/infiniband/hw/mlx5/umr.c b/drivers/infiniband/hw/mlx5/umr.c index 48cae5cc1c1b..1ca83d2d8a0b 100644 --- a/drivers/infiniband/hw/mlx5/umr.c +++ b/drivers/infiniband/hw/mlx5/umr.c @@ -760,12 +760,14 @@ static int _mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, } if (dd) { + struct mlx5_data_direct *mdd = dev->data_direct; + cur_ksm->va = cpu_to_be64(rdma_block_iter_dma_address(&biter)); if (mr->access_flags & IB_ACCESS_RELAXED_ORDERING && - dev->ddr.mkey_ro_valid) - cur_ksm->key = cpu_to_be32(dev->ddr.mkey_ro); + mdd->mkey_ro_valid) + cur_ksm->key = cpu_to_be32(mdd->mkey_ro); else - cur_ksm->key = cpu_to_be32(dev->ddr.mkey); + cur_ksm->key = cpu_to_be32(mdd->mkey); if (mr->umem->is_dmabuf && (flags & MLX5_IB_UPD_XLT_ZAP)) { cur_ksm->va = 0; @@ -810,7 +812,8 @@ int mlx5r_umr_update_data_direct_ksm_pas_range(struct mlx5_ib_mr *mr, !(flags & MLX5_IB_UPD_XLT_KEEP_PGSZ))) return -EINVAL; - return _mlx5r_umr_update_mr_pas(mr, flags, mr_to_mdev(mr)->ddr.pdn, + return _mlx5r_umr_update_mr_pas(mr, flags, + mr_to_mdev(mr)->data_direct->pdn, true, start_block, nblocks); } diff --git a/include/linux/mlx5/data_direct.h b/include/linux/mlx5/data_direct.h index 71f47547aebb..eaa2dc6bc607 100644 --- a/include/linux/mlx5/data_direct.h +++ b/include/linux/mlx5/data_direct.h @@ -6,10 +6,27 @@ #ifndef _MLX5_DATA_DIRECT_H #define _MLX5_DATA_DIRECT_H +#include + struct mlx5_core_dev; +struct mlx5_data_direct_dev; struct mlx5_ib_dev; struct notifier_block; +struct mlx5_data_direct { + struct mlx5_data_direct_dev *dev; + u32 pdn; + u32 mkey; + u32 mkey_ro; + u8 mkey_ro_valid :1; +}; + +static inline struct mlx5_data_direct_dev * +mlx5_data_direct_get_dev(struct mlx5_data_direct *dd) +{ + return dd ? READ_ONCE(dd->dev) : NULL; +} + int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev); void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev); -- 2.44.0