Register auxiliary module to create ibdevice for ionic ethernet adapter. Co-developed-by: Andrew Boyer Signed-off-by: Andrew Boyer Co-developed-by: Allen Hubbe Signed-off-by: Allen Hubbe Signed-off-by: Abhijit Gangurde --- v1->v2 - Removed netdev references from ionic RDMA driver - Moved to ionic_lif* instead of void* to convey information between aux devices and drivers. drivers/infiniband/hw/ionic/ionic_ibdev.c | 131 ++++++++++++++++++++ drivers/infiniband/hw/ionic/ionic_ibdev.h | 18 +++ drivers/infiniband/hw/ionic/ionic_lif_cfg.c | 101 +++++++++++++++ drivers/infiniband/hw/ionic/ionic_lif_cfg.h | 64 ++++++++++ 4 files changed, 314 insertions(+) create mode 100644 drivers/infiniband/hw/ionic/ionic_ibdev.c create mode 100644 drivers/infiniband/hw/ionic/ionic_ibdev.h create mode 100644 drivers/infiniband/hw/ionic/ionic_lif_cfg.c create mode 100644 drivers/infiniband/hw/ionic/ionic_lif_cfg.h diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.c b/drivers/infiniband/hw/ionic/ionic_ibdev.c new file mode 100644 index 000000000000..d79470dae13a --- /dev/null +++ b/drivers/infiniband/hw/ionic/ionic_ibdev.c @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */ + +#include +#include +#include + +#include "ionic_ibdev.h" + +#define DRIVER_DESCRIPTION "AMD Pensando RoCE HCA driver" +#define DEVICE_DESCRIPTION "AMD Pensando RoCE HCA" + +MODULE_AUTHOR("Allen Hubbe "); +MODULE_DESCRIPTION(DRIVER_DESCRIPTION); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("NET_IONIC"); + +static void ionic_destroy_ibdev(struct ionic_ibdev *dev) +{ + ib_unregister_device(&dev->ibdev); + ib_dealloc_device(&dev->ibdev); +} + +static struct ionic_ibdev *ionic_create_ibdev(struct ionic_aux_dev *ionic_adev) +{ + struct ib_device *ibdev; + struct ionic_ibdev *dev; + struct net_device *ndev; + int rc; + + dev = ib_alloc_device(ionic_ibdev, ibdev); + if (!dev) + return ERR_PTR(-EINVAL); + + ionic_fill_lif_cfg(ionic_adev->lif, &dev->lif_cfg); + + ibdev = &dev->ibdev; + ibdev->dev.parent = dev->lif_cfg.hwdev; + + strscpy(ibdev->name, "ionic_%d", IB_DEVICE_NAME_MAX); + strscpy(ibdev->node_desc, DEVICE_DESCRIPTION, IB_DEVICE_NODE_DESC_MAX); + + ibdev->node_type = RDMA_NODE_IB_CA; + ibdev->phys_port_cnt = 1; + + /* the first two eq are reserved for async events */ + ibdev->num_comp_vectors = dev->lif_cfg.eq_count - 2; + + ndev = ionic_lif_netdev(ionic_adev->lif); + addrconf_ifid_eui48((u8 *)&ibdev->node_guid, ndev); + rc = ib_device_set_netdev(ibdev, ndev, 1); + /* ionic_lif_netdev() returns ndev with refcount held */ + dev_put(ndev); + if (rc) + goto err_admin; + + rc = ib_register_device(ibdev, "ionic_%d", ibdev->dev.parent); + if (rc) + goto err_register; + + return dev; + +err_register: +err_admin: + ib_dealloc_device(&dev->ibdev); + + return ERR_PTR(rc); +} + +static int ionic_aux_probe(struct auxiliary_device *adev, + const struct auxiliary_device_id *id) +{ + struct ionic_aux_dev *ionic_adev; + struct ionic_ibdev *dev; + + ionic_adev = container_of(adev, struct ionic_aux_dev, adev); + dev = ionic_create_ibdev(ionic_adev); + if (IS_ERR(dev)) + return dev_err_probe(&adev->dev, PTR_ERR(dev), + "Failed to register ibdev\n"); + + auxiliary_set_drvdata(adev, dev); + ibdev_dbg(&dev->ibdev, "registered\n"); + + return 0; +} + +static void ionic_aux_remove(struct auxiliary_device *adev) +{ + struct ionic_ibdev *dev = auxiliary_get_drvdata(adev); + + dev_dbg(&adev->dev, "unregister ibdev\n"); + ionic_destroy_ibdev(dev); + dev_dbg(&adev->dev, "unregistered\n"); +} + +static const struct auxiliary_device_id ionic_aux_id_table[] = { + { .name = "ionic.rdma", }, + {}, +}; + +MODULE_DEVICE_TABLE(auxiliary, ionic_aux_id_table); + +static struct auxiliary_driver ionic_aux_r_driver = { + .name = "rdma", + .probe = ionic_aux_probe, + .remove = ionic_aux_remove, + .id_table = ionic_aux_id_table, +}; + +static int __init ionic_mod_init(void) +{ + int rc; + + rc = auxiliary_driver_register(&ionic_aux_r_driver); + if (rc) + goto err_aux; + + return 0; + +err_aux: + return rc; +} + +static void __exit ionic_mod_exit(void) +{ + auxiliary_driver_unregister(&ionic_aux_r_driver); +} + +module_init(ionic_mod_init); +module_exit(ionic_mod_exit); diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.h b/drivers/infiniband/hw/ionic/ionic_ibdev.h new file mode 100644 index 000000000000..224e5e74056d --- /dev/null +++ b/drivers/infiniband/hw/ionic/ionic_ibdev.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */ + +#ifndef _IONIC_IBDEV_H_ +#define _IONIC_IBDEV_H_ + +#include +#include + +#include "ionic_lif_cfg.h" + +struct ionic_ibdev { + struct ib_device ibdev; + + struct ionic_lif_cfg lif_cfg; +}; + +#endif /* _IONIC_IBDEV_H_ */ diff --git a/drivers/infiniband/hw/ionic/ionic_lif_cfg.c b/drivers/infiniband/hw/ionic/ionic_lif_cfg.c new file mode 100644 index 000000000000..8d0d209227e9 --- /dev/null +++ b/drivers/infiniband/hw/ionic/ionic_lif_cfg.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */ + +#include + +#include +#include + +#include "ionic_lif_cfg.h" + +#define IONIC_MIN_RDMA_VERSION 0 +#define IONIC_MAX_RDMA_VERSION 2 + +static u8 ionic_get_expdb(struct ionic_lif *lif) +{ + u8 expdb_support = 0; + + if (lif->ionic->idev.phy_cmb_expdb64_pages) + expdb_support |= IONIC_EXPDB_64B_WQE; + if (lif->ionic->idev.phy_cmb_expdb128_pages) + expdb_support |= IONIC_EXPDB_128B_WQE; + if (lif->ionic->idev.phy_cmb_expdb256_pages) + expdb_support |= IONIC_EXPDB_256B_WQE; + if (lif->ionic->idev.phy_cmb_expdb512_pages) + expdb_support |= IONIC_EXPDB_512B_WQE; + + return expdb_support; +} + +void ionic_fill_lif_cfg(struct ionic_lif *lif, struct ionic_lif_cfg *cfg) +{ + union ionic_lif_identity *ident = &lif->ionic->ident.lif; + + cfg->lif = lif; + cfg->hwdev = &lif->ionic->pdev->dev; + cfg->lif_index = lif->index; + cfg->lif_hw_index = lif->hw_index; + + cfg->dbid = lif->kern_pid; + cfg->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif); + cfg->dbpage = lif->kern_dbpage; + cfg->intr_ctrl = lif->ionic->idev.intr_ctrl; + + cfg->db_phys = lif->ionic->bars[IONIC_PCI_BAR_DBELL].bus_addr; + + if (IONIC_VERSION(ident->rdma.version, ident->rdma.minor_version) >= + IONIC_VERSION(2, 1)) + cfg->page_size_supported = + le64_to_cpu(ident->rdma.page_size_cap); + else + cfg->page_size_supported = IONIC_PAGE_SIZE_SUPPORTED; + + cfg->rdma_version = ident->rdma.version; + cfg->qp_opcodes = ident->rdma.qp_opcodes; + cfg->admin_opcodes = ident->rdma.admin_opcodes; + + cfg->stats_type = le16_to_cpu(ident->rdma.stats_type); + cfg->npts_per_lif = le32_to_cpu(ident->rdma.npts_per_lif); + cfg->nmrs_per_lif = le32_to_cpu(ident->rdma.nmrs_per_lif); + cfg->nahs_per_lif = le32_to_cpu(ident->rdma.nahs_per_lif); + + cfg->aq_base = le32_to_cpu(ident->rdma.aq_qtype.qid_base); + cfg->cq_base = le32_to_cpu(ident->rdma.cq_qtype.qid_base); + cfg->eq_base = le32_to_cpu(ident->rdma.eq_qtype.qid_base); + + /* + * ionic_create_rdma_admin() may reduce aq_count or eq_count if + * it is unable to allocate all that were requested. + * aq_count is tunable; see ionic_aq_count + * eq_count is tunable; see ionic_eq_count + */ + cfg->aq_count = le32_to_cpu(ident->rdma.aq_qtype.qid_count); + cfg->eq_count = le32_to_cpu(ident->rdma.eq_qtype.qid_count); + cfg->cq_count = le32_to_cpu(ident->rdma.cq_qtype.qid_count); + cfg->qp_count = le32_to_cpu(ident->rdma.sq_qtype.qid_count); + cfg->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif); + + cfg->aq_qtype = ident->rdma.aq_qtype.qtype; + cfg->sq_qtype = ident->rdma.sq_qtype.qtype; + cfg->rq_qtype = ident->rdma.rq_qtype.qtype; + cfg->cq_qtype = ident->rdma.cq_qtype.qtype; + cfg->eq_qtype = ident->rdma.eq_qtype.qtype; + cfg->udma_qgrp_shift = ident->rdma.udma_shift; + cfg->udma_count = 2; + + cfg->max_stride = ident->rdma.max_stride; + cfg->expdb_mask = ionic_get_expdb(lif); + + cfg->sq_expdb = + !!(lif->qtype_info[IONIC_QTYPE_TXQ].features & IONIC_QIDENT_F_EXPDB); + cfg->rq_expdb = + !!(lif->qtype_info[IONIC_QTYPE_RXQ].features & IONIC_QIDENT_F_EXPDB); +} + +struct net_device *ionic_lif_netdev(struct ionic_lif *lif) +{ + struct net_device *netdev = lif->netdev; + + dev_hold(netdev); + return netdev; +} diff --git a/drivers/infiniband/hw/ionic/ionic_lif_cfg.h b/drivers/infiniband/hw/ionic/ionic_lif_cfg.h new file mode 100644 index 000000000000..5b04b8a9937e --- /dev/null +++ b/drivers/infiniband/hw/ionic/ionic_lif_cfg.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */ + +#ifndef _IONIC_LIF_CFG_H_ + +#define IONIC_VERSION(a, b) (((a) << 16) + ((b) << 8)) +#define IONIC_PAGE_SIZE_SUPPORTED 0x40201000 /* 4kb, 2Mb, 1Gb */ + +#define IONIC_EXPDB_64B_WQE BIT(0) +#define IONIC_EXPDB_128B_WQE BIT(1) +#define IONIC_EXPDB_256B_WQE BIT(2) +#define IONIC_EXPDB_512B_WQE BIT(3) + +struct ionic_lif_cfg { + struct device *hwdev; + struct ionic_lif *lif; + + int lif_index; + int lif_hw_index; + + u32 dbid; + int dbid_count; + u64 __iomem *dbpage; + struct ionic_intr __iomem *intr_ctrl; + phys_addr_t db_phys; + + u64 page_size_supported; + u32 npts_per_lif; + u32 nmrs_per_lif; + u32 nahs_per_lif; + + u32 aq_base; + u32 cq_base; + u32 eq_base; + + int aq_count; + int eq_count; + int cq_count; + int qp_count; + + u16 stats_type; + u8 aq_qtype; + u8 sq_qtype; + u8 rq_qtype; + u8 cq_qtype; + u8 eq_qtype; + + u8 udma_count; + u8 udma_qgrp_shift; + + u8 rdma_version; + u8 qp_opcodes; + u8 admin_opcodes; + + u8 max_stride; + bool sq_expdb; + bool rq_expdb; + u8 expdb_mask; +}; + +void ionic_fill_lif_cfg(struct ionic_lif *lif, struct ionic_lif_cfg *cfg); +struct net_device *ionic_lif_netdev(struct ionic_lif *lif); + +#endif /* _IONIC_LIF_CFG_H_ */ -- 2.43.0