Add the queue pair control path: create, destroy, modify and query, for both kernel and user queue pairs, including the address path setup for RoCE. Signed-off-by: Xin Tian --- drivers/infiniband/hw/xsc/main.c | 6 + drivers/infiniband/hw/xsc/qp.c | 1321 ++++++++++++++++++++++++++++++ 2 files changed, 1327 insertions(+) diff --git a/drivers/infiniband/hw/xsc/main.c b/drivers/infiniband/hw/xsc/main.c index e9a9ba114..c4f306318 100644 --- a/drivers/infiniband/hw/xsc/main.c +++ b/drivers/infiniband/hw/xsc/main.c @@ -570,6 +570,11 @@ static void xsc_ib_dev_setting(struct xsc_ib_dev *dev) dev->ib_dev.ops.get_link_layer = xsc_ib_port_link_layer; dev->ib_dev.ops.get_netdev = xsc_get_netdev; + dev->ib_dev.ops.create_qp = xsc_ib_create_qp; + dev->ib_dev.ops.modify_qp = xsc_ib_modify_qp; + dev->ib_dev.ops.query_qp = xsc_ib_query_qp; + dev->ib_dev.ops.destroy_qp = xsc_ib_destroy_qp; + dev->ib_dev.ops.post_recv = xsc_ib_post_recv; dev->ib_dev.ops.create_cq = xsc_ib_create_cq; dev->ib_dev.ops.destroy_cq = xsc_ib_destroy_cq; dev->ib_dev.ops.poll_cq = xsc_ib_poll_cq; @@ -592,6 +597,7 @@ static void xsc_ib_dev_setting(struct xsc_ib_dev *dev) dev->ib_dev.ops INIT_RDMA_OBJ_SIZE(ib_ah, xsc_ib_ah, ibah); dev->ib_dev.ops INIT_RDMA_OBJ_SIZE(ib_cq, xsc_ib_cq, ibcq); dev->ib_dev.ops INIT_RDMA_OBJ_SIZE(ib_pd, xsc_ib_pd, ibpd); + dev->ib_dev.ops INIT_RDMA_OBJ_SIZE(ib_qp, xsc_ib_qp, ibqp); dev->ib_dev.ops INIT_RDMA_OBJ_SIZE(ib_srq, xsc_ib_srq, ibsrq); } diff --git a/drivers/infiniband/hw/xsc/qp.c b/drivers/infiniband/hw/xsc/qp.c index 28c8a1246..6a83adc9c 100644 --- a/drivers/infiniband/hw/xsc/qp.c +++ b/drivers/infiniband/hw/xsc/qp.c @@ -28,11 +28,21 @@ enum { #define UDP_SPORT_MASK 0xffff #define UDP_SPORT_OFFSET 0 +static int is_qp0(enum ib_qp_type qp_type) +{ + return qp_type == IB_QPT_SMI; +} + static int is_qp1(enum ib_qp_type qp_type) { return qp_type == IB_QPT_GSI; } +static int is_sqp(enum ib_qp_type qp_type) +{ + return is_qp0(qp_type) || is_qp1(qp_type); +} + static void *get_wqe(struct xsc_ib_qp *qp, int offset) { return xsc_buf_offset(&qp->buf, offset); @@ -48,6 +58,258 @@ static void *get_seg_wqe(void *first, int n) return first + (n << XSC_BASE_WQE_SHIFT); } +static void xsc_ib_qp_event(struct xsc_core_qp *qp, int type) +{ + struct ib_qp *ibqp = &core_to_xqp(qp)->ibqp; + struct ib_event event; + + if (ibqp->event_handler) { + event.device = ibqp->device; + event.element.qp = ibqp; + switch (type) { + case XSC_EVENT_TYPE_WQ_CATAS_ERROR: + event.event = IB_EVENT_QP_FATAL; + break; + case XSC_EVENT_TYPE_WQ_INVAL_REQ_ERROR: + event.event = IB_EVENT_QP_REQ_ERR; + break; + case XSC_EVENT_TYPE_WQ_ACCESS_ERROR: + event.event = IB_EVENT_QP_ACCESS_ERR; + break; + default: + pr_warn("xsc_ib: Unexpected event type %d on QP %06x\n", type, qp->qpn); + return; + } + + ibqp->event_handler(&event, ibqp->qp_context); + pr_info("xsc_ib: xsc async event 0x%x, ib event %d\n", type, event.event); + } +} + +static int set_rq_size(struct xsc_ib_dev *dev, struct ib_qp_init_attr *init_attr, + int has_rq, struct xsc_ib_qp *qp, struct xsc_ib_create_qp *ucmd) +{ + bool check_res = false; + u32 wqe_cnt = init_attr->cap.max_recv_wr ? + roundup_pow_of_two(init_attr->cap.max_recv_wr) : 0; + + if (init_attr->srq) + wqe_cnt = to_xsrq(init_attr->srq)->xsrq.max; + + /* Sanity check RQ size before proceeding */ + check_res = xsc_check_max_qp_depth(dev->xdev, &wqe_cnt, dev->xdev->caps.max_wqes); + if (check_res) { + ibdev_err(&dev->ib_dev, "max_recv_wr:%d exceed max rq depth\n", + init_attr->cap.max_recv_wr); + return -EINVAL; + } + + if (!has_rq) { + qp->rq.max_gs = 0; + qp->rq.wqe_cnt = 0; + qp->rq.wqe_shift = 0; + } else { + if (ucmd) { + qp->rq.wqe_cnt = ucmd->rq_wqe_count; + qp->rq.wqe_shift = ucmd->rq_wqe_shift; + qp->rq.max_gs = 1; + qp->rq.max_post = qp->rq.wqe_cnt; + } else { + qp->rq.wqe_cnt = wqe_cnt; + if (qp->xqp.qp_type == IB_QPT_GSI) + qp->rq.wqe_shift = ilog2(xsc_get_recv_ds_num(dev->xdev)) + + XSC_RECV_WQE_SHIFT; + else + qp->rq.wqe_shift = dev->xdev->caps.recv_wqe_shift; + qp->rq.max_gs = dev->xdev->caps.recv_ds_num; + qp->rq.max_post = qp->rq.wqe_cnt; + } + } + + return 0; +} + +static int calc_sq_size(struct xsc_ib_dev *dev, struct ib_qp_init_attr *attr, + struct xsc_ib_qp *qp) +{ + int wqe_size; + int wq_size; + bool check_res = false; + + if (!attr->cap.max_send_wr) { + ibdev_err(&dev->ib_dev, "invalid max_send_wr:%d\n", attr->cap.max_send_wr); + return -1; + } + + wqe_size = 1 << dev->xdev->caps.send_wqe_shift; + qp->max_inline_data = (dev->xdev->caps.send_ds_num - 2) * sizeof(struct xsc_wqe_data_seg); + attr->cap.max_inline_data = qp->max_inline_data; + + attr->cap.max_send_wr = min_t(u32, attr->cap.max_send_wr, dev->xdev->caps.max_wqes); + qp->sq.wqe_cnt = roundup_pow_of_two(attr->cap.max_send_wr); + + check_res = xsc_check_max_qp_depth(dev->xdev, &qp->sq.wqe_cnt, dev->xdev->caps.max_wqes); + if (check_res) { + ibdev_err(&dev->ib_dev, "max_send_wr:%d exceed max sq depth\n", + attr->cap.max_send_wr); + return -EINVAL; + } + qp->sq.ds_cnt = qp->sq.wqe_cnt << (dev->xdev->caps.send_wqe_shift - XSC_BASE_WQE_SHIFT); + wq_size = qp->sq.wqe_cnt * wqe_size; + qp->sq.wqe_shift = ilog2(wqe_size); + qp->sq.max_gs = dev->xdev->caps.send_ds_num - XSC_CTRL_SEG_NUM - XSC_RADDR_SEG_NUM; + qp->sq.max_post = qp->sq.wqe_cnt; + + return wq_size; +} + +static int qp_has_rq(struct ib_qp_init_attr *attr) +{ + if (!attr->srq && !attr->cap.max_recv_wr) + return 0; + + return 1; +} + +static enum xsc_qp_state to_xsc_state(enum ib_qp_state state) +{ + switch (state) { + case IB_QPS_RESET: return XSC_QP_STATE_RST; + case IB_QPS_INIT: return XSC_QP_STATE_INIT; + case IB_QPS_RTR: return XSC_QP_STATE_RTR; + case IB_QPS_RTS: return XSC_QP_STATE_RTS; + case IB_QPS_SQD: return XSC_QP_STATE_SQD; + case IB_QPS_SQE: return XSC_QP_STATE_SQER; + case IB_QPS_ERR: return XSC_QP_STATE_ERR; + default: return -1; + } +} + +static char *qp_state_to_str(enum ib_qp_state state) +{ + switch (state) { + case IB_QPS_RESET: return "RST"; + case IB_QPS_INIT: return "INIT"; + case IB_QPS_RTR: return "RTR"; + case IB_QPS_RTS: return "RTS"; + case IB_QPS_SQD: return "SQD"; + case IB_QPS_SQE: return "SQE"; + case IB_QPS_ERR: return "ERR"; + default: return "UNKNOWN"; + } +} + +static struct ib_umem *qp_umem_get(struct xsc_ib_dev *dev, struct xsc_ib_qp *qp, + struct ib_udata *udata, struct ib_pd *pd, + struct xsc_ib_create_qp *ucmd) +{ + struct ib_umem *umem; + + umem = ib_umem_get_va(&dev->ib_dev, ucmd->buf_addr, qp->buf_size, + IB_ACCESS_LOCAL_WRITE); + return umem; +} + +static int create_user_qp(struct xsc_ib_dev *dev, struct ib_pd *pd, + struct xsc_ib_qp *qp, struct ib_udata *udata, + struct xsc_create_qp_mbox_in **in, + struct xsc_ib_create_qp_resp *resp, int *inlen) +{ + struct xsc_ib_create_qp ucmd; + int page_shift; + int npages; + u32 offset; + int ncont; + int err; + int hw_npages; + struct ib_umem_dmabuf *umem_dmabuf; + + err = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)); + if (err) { + ibdev_err(&dev->ib_dev, "failed to copy from udata, err=%d\n", err); + return err; + } + ibdev_dbg(&dev->ib_dev, "buf_addr:0x%lx db_addr:0x%lx sq cnt:%u, rq cnt:%u, rq shift:%u\n", + (uintptr_t)ucmd.buf_addr, (uintptr_t)ucmd.db_addr, + ucmd.sq_wqe_count, ucmd.rq_wqe_count, ucmd.rq_wqe_shift); + + qp->sq.ds_cnt = ucmd.sq_wqe_count; + qp->sq.wqe_cnt = ucmd.sq_wqe_count; + qp->sq.wqe_shift = XSC_BASE_WQE_SHIFT; + qp->rq.ds_cnt = ucmd.rq_wqe_count; + qp->rq.wqe_cnt = ucmd.rq_wqe_count; + qp->rq.wqe_shift = XSC_BASE_WQE_SHIFT; + + qp->buf_size = (qp->sq.wqe_cnt << qp->sq.wqe_shift) + (qp->rq.wqe_cnt << qp->rq.wqe_shift); + + if (ucmd.dmabuf_fd != -1) { + umem_dmabuf = ib_umem_dmabuf_get_pinned(&dev->ib_dev, 0, ucmd.dmabuf_sz, + ucmd.dmabuf_fd, IB_ACCESS_LOCAL_WRITE); + + if (IS_ERR(umem_dmabuf)) { + ibdev_warn(&dev->ib_dev, "qp umem_dmabuf get failed (%ld)\n", + PTR_ERR(umem_dmabuf)); + return -ENOMEM; + } + + qp->umem = &umem_dmabuf->umem; + } else { + qp->umem = qp_umem_get(dev, qp, udata, pd, &ucmd); + } + + if (IS_ERR(qp->umem)) { + ibdev_err(&dev->ib_dev, "umem_get failed\n"); + err = PTR_ERR(qp->umem); + goto err_uuar; + } + + xsc_ib_cont_pages(qp->umem, ucmd.buf_addr, &npages, &page_shift, + &ncont); + if (ncont != npages) { + /* TODO: peer memory support failed */ + page_shift = PAGE_SHIFT; + ncont = npages; + } + + hw_npages = DIV_ROUND_UP(qp->buf_size, PAGE_SIZE_4K); + err = xsc_ib_get_buf_offset(ucmd.buf_addr, page_shift, &offset); + if (err) { + ibdev_err(&dev->ib_dev, "bad offset:%d\n", offset); + goto err_umem; + } + ibdev_dbg(&dev->ib_dev, "npage:%d, page_shift:%d, ncont:%d, offset:%d, hw_npages %d\n", + npages, page_shift, ncont, offset, hw_npages); + + *inlen = sizeof(**in) + sizeof(*((*in)->req.pas)) * hw_npages; + *in = xsc_vzalloc(*inlen); + if (!*in) { + err = -ENOMEM; + goto err_umem; + } + xsc_ib_populate_pas(dev, qp->umem, page_shift, (*in)->req.pas, hw_npages, true); + (*in)->req.pa_num = cpu_to_be16(hw_npages); + + err = ib_copy_to_udata(udata, resp, sizeof(*resp)); + if (err) { + ibdev_err(&dev->ib_dev, "failed to copy to udata, err=%d\n", err); + goto err_umem; + } + qp->create_type = XSC_QP_USER; + + return 0; + +err_umem: + ib_umem_release(qp->umem); + +err_uuar: + return err; +} + +static void destroy_qp_user(struct ib_pd *__maybe_unused pd, struct xsc_ib_qp *qp) +{ + ib_umem_release(qp->umem); +} + #define MAX_QP1_SQ_HDR_SIZE_V2 512 #define MAX_QP1_SQ_HDR_SIZE 86 /* Ethernet header = 14 */ @@ -57,6 +319,690 @@ static void *get_seg_wqe(void *first, int n) /* iCRC = 4 */ #define MAX_QP1_RQ_HDR_SIZE_V2 512 +static int create_kernel_qp(struct xsc_ib_dev *dev, + struct ib_qp_init_attr *init_attr, + struct xsc_ib_qp *qp, + struct xsc_create_qp_mbox_in **in, int *inlen) +{ + int err; + int sq_size; + int hw_npages; + + sq_size = calc_sq_size(dev, init_attr, qp); + if (sq_size < 0) { + err = -EINVAL; + ibdev_err(&dev->ib_dev, "err %d\n", err); + return err; + } + + qp->rq.ds_cnt = qp->rq.wqe_cnt << (qp->rq.wqe_shift - XSC_BASE_WQE_SHIFT); + qp->rq.offset = 0; + qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift; + qp->buf_size = qp->sq.offset + sq_size; + qp->send_psn = 0; + + err = xsc_buf_alloc(dev->xdev, qp->buf_size, PAGE_SIZE, &qp->buf); + if (err) { + ibdev_err(&dev->ib_dev, "failed to alloc qp buffer,err=%d\n", err); + return err; + } + + qp->sq.qend = qp->buf.direct.buf + qp->sq.offset + sq_size; + hw_npages = DIV_ROUND_UP(qp->buf_size, PAGE_SIZE_4K); + *inlen = sizeof(**in) + sizeof(*(*in)->req.pas) * hw_npages; + *in = xsc_vzalloc(*inlen); + if (!*in) { + err = -ENOMEM; + goto err_buf; + } + + xsc_fill_page_array(&qp->buf, (*in)->req.pas, hw_npages); + (*in)->req.pa_num = cpu_to_be16(hw_npages); + + qp->sq.wrid = kcalloc(qp->sq.wqe_cnt, sizeof(*qp->sq.wrid), GFP_KERNEL); + qp->sq.wr_data = kcalloc(qp->sq.wqe_cnt, sizeof(*qp->sq.wr_data), GFP_KERNEL); + qp->rq.wrid = kcalloc(qp->rq.wqe_cnt, sizeof(*qp->rq.wrid), GFP_KERNEL); + qp->sq.w_list = kcalloc(qp->sq.wqe_cnt, sizeof(*qp->sq.w_list), GFP_KERNEL); + qp->sq.wqe_head = kcalloc(qp->sq.wqe_cnt, sizeof(*qp->sq.wqe_head), GFP_KERNEL); + qp->sq.wr_opcode = kcalloc(qp->sq.wqe_cnt, sizeof(*qp->sq.wr_opcode), GFP_KERNEL); + qp->sq.need_flush = kcalloc(qp->sq.wqe_cnt, sizeof(*qp->sq.need_flush), GFP_KERNEL); + qp->sq.prev_ce_wqe = kcalloc(qp->sq.wqe_cnt, sizeof(*qp->sq.prev_ce_wqe), GFP_KERNEL); + qp->sq.last_comp_ce_wqe = -1; + qp->sq.last_post_ce_wqe = -1; + + if (!qp->sq.wrid || !qp->sq.wr_data || !qp->rq.wrid || !qp->sq.prev_ce_wqe || + !qp->sq.w_list || !qp->sq.wqe_head || !qp->sq.wr_opcode || !qp->sq.need_flush) { + err = -ENOMEM; + goto err_wrid; + } + qp->create_type = XSC_QP_KERNEL; + + if (init_attr->qp_type == IB_QPT_GSI) { + qp->sq.mad_index = 0; + qp->sq.mad_queue_depth = MAD_QUEUE_DEPTH; + qp->sq.hdr_size = MAX_QP1_SQ_HDR_SIZE_V2 * MAD_QUEUE_DEPTH; + qp->sq.hdr_buf = dma_alloc_coherent(dev->ib_dev.dma_device, + qp->sq.hdr_size, + &qp->sq.hdr_dma, + GFP_KERNEL); + if (!qp->sq.hdr_buf) { + err = -ENOMEM; + ibdev_err(&dev->ib_dev, "Failed to create sq_hdr_buf"); + goto err_wrid; + } + } + + return 0; + +err_wrid: + kfree(qp->sq.prev_ce_wqe); + kfree(qp->sq.need_flush); + kfree(qp->sq.wr_opcode); + kfree(qp->sq.wqe_head); + kfree(qp->sq.w_list); + kfree(qp->sq.wrid); + kfree(qp->sq.wr_data); + kfree(qp->rq.wrid); + +err_buf: + xsc_buf_free(dev->xdev, &qp->buf); + return err; +} + +static void destroy_qp_kernel(struct xsc_ib_dev *dev, struct xsc_ib_qp *qp) +{ + if (qp->sq.hdr_buf) + dma_free_coherent(dev->ib_dev.dma_device, qp->sq.hdr_size, + qp->sq.hdr_buf, qp->sq.hdr_dma); + kfree(qp->sq.prev_ce_wqe); + kfree(qp->sq.need_flush); + kfree(qp->sq.wr_opcode); + kfree(qp->sq.wqe_head); + kfree(qp->sq.w_list); + kfree(qp->sq.wrid); + kfree(qp->sq.wr_data); + kfree(qp->rq.wrid); + xsc_buf_free(dev->xdev, &qp->buf); +} + +static u8 ib_to_xsc_qp_type(enum ib_qp_type qp_type, __u32 flags) +{ + if (qp_type == IB_QPT_RC) { + if (flags & XSC_QP_FLAG_WORC) + return XSC_QUEUE_TYPE_WORC; + else + return XSC_QUEUE_TYPE_RDMA_RC; + } else if ((qp_type == IB_QPT_GSI) || (qp_type == IB_QPT_SMI)) { + return XSC_QUEUE_TYPE_RDMA_MAD; + } else if (qp_type == IB_QPT_RAW_PACKET) { + if (flags & XSC_QP_FLAG_RAWPACKET_TSO) + return XSC_QUEUE_TYPE_RAW_TSO; + else if (flags & XSC_QP_FLAG_RAWPACKET_TX) + return XSC_QUEUE_TYPE_RAW_TX; + else if (flags & XSC_QP_FLAG_RAWPACKET_SNIFFER) + return XSC_QUEUE_TYPE_SNIFFER; + else + return XSC_QUEUE_TYPE_RAW; + } else { + return XSC_QUEUE_TYPE_INVALID; + } +} + +static void get_cqs(enum ib_qp_type qp_type, + struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq, + struct xsc_ib_cq **send_cq, struct xsc_ib_cq **recv_cq) +{ + switch (qp_type) { + case IB_QPT_SMI: + case IB_QPT_GSI: + case IB_QPT_RC: + case IB_QPT_UC: + case IB_QPT_UD: + case IB_QPT_RAW_PACKET: + *send_cq = ib_send_cq ? to_xcq(ib_send_cq) : NULL; + *recv_cq = ib_recv_cq ? to_xcq(ib_recv_cq) : NULL; + break; + default: + *send_cq = NULL; + *recv_cq = NULL; + break; + } +} + +static void xsc_ib_lock_cqs(struct xsc_ib_cq *send_cq, struct xsc_ib_cq *recv_cq) + __acquires(&send_cq->lock) __acquires(&recv_cq->lock) +{ + if (send_cq) { + if (recv_cq) { + if (send_cq->xcq.cqn < recv_cq->xcq.cqn) { + spin_lock_irq(&send_cq->lock); + spin_lock_nested(&recv_cq->lock, + SINGLE_DEPTH_NESTING); + } else if (send_cq->xcq.cqn == recv_cq->xcq.cqn) { + spin_lock_irq(&send_cq->lock); + __acquire(&recv_cq->lock); + } else { + spin_lock_irq(&recv_cq->lock); + spin_lock_nested(&send_cq->lock, + SINGLE_DEPTH_NESTING); + } + } else { + spin_lock_irq(&send_cq->lock); + } + } else if (recv_cq) { + spin_lock_irq(&recv_cq->lock); + } +} + +static void xsc_ib_unlock_cqs(struct xsc_ib_cq *send_cq, struct xsc_ib_cq *recv_cq) + __releases(&send_cq->lock) __releases(&recv_cq->lock) +{ + if (send_cq) { + if (recv_cq) { + if (send_cq->xcq.cqn < recv_cq->xcq.cqn) { + spin_unlock(&recv_cq->lock); + spin_unlock_irq(&send_cq->lock); + } else if (send_cq->xcq.cqn == recv_cq->xcq.cqn) { + __release(&recv_cq->lock); + spin_unlock_irq(&send_cq->lock); + } else { + spin_unlock(&send_cq->lock); + spin_unlock_irq(&recv_cq->lock); + } + } else { + spin_unlock_irq(&send_cq->lock); + } + } else if (recv_cq) { + spin_unlock_irq(&recv_cq->lock); + } +} + +static int create_qp_common(struct xsc_ib_dev *dev, struct ib_pd *pd, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata, struct xsc_ib_qp *qp) +{ + struct xsc_ib_create_qp_resp resp; + struct xsc_create_qp_mbox_in *in = NULL; + struct xsc_ib_create_qp ucmd; + struct xsc_ib_cq *send_cq; + struct xsc_ib_cq *recv_cq; + struct xsc_srq_qp_node *qp_node; + int inlen = sizeof(*in); + int err; + char buf[256]; + char *ptr = buf; + int ret = 0; + + mutex_init(&qp->mutex); + spin_lock_init(&qp->sq.lock); + spin_lock_init(&qp->rq.lock); + spin_lock_init(&qp->lock); + + if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) + qp->sq_signal_bits = XSC_WQE_CTRL_CQ_UPDATE; + + if (pd->uobject) { + if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) { + ibdev_err(&dev->ib_dev, "failed to copy from udata\n"); + return -EFAULT; + } + } + + qp->has_rq = qp_has_rq(init_attr); + + err = set_rq_size(dev, init_attr, qp->has_rq, + qp, pd->uobject ? &ucmd : NULL); + if (err) { + ibdev_err(&dev->ib_dev, "failed to set rq size %d\n", err); + return err; + } + + if (pd->uobject) { + err = create_user_qp(dev, pd, qp, udata, &in, &resp, &inlen); + if (err) + ibdev_err(&dev->ib_dev, "failed to create user qp, err = %d\n", err); + } else { + err = create_kernel_qp(dev, init_attr, qp, &in, &inlen); + if (err) + ibdev_err(&dev->ib_dev, "failed to create kernel qp, err = %d\n", err); + else + qp->pa_lkey = to_xpd(pd)->pa_lkey; + } + + if (err) + return err; + + if (is_sqp(init_attr->qp_type)) + qp->port = init_attr->port_num; + + if (is_qp1(init_attr->qp_type)) + in->req.input_qpn = cpu_to_be16(1); + + in->req.pdn = cpu_to_be32(to_xpd(pd)->pdn); + + if (qp->rq.ds_cnt) + in->req.log_rq_sz = ilog2(qp->rq.ds_cnt); + + if (qp->sq.ds_cnt) + in->req.log_sq_sz = ilog2(qp->sq.ds_cnt); + else + in->req.log_sq_sz = ilog2(0x80); + + if (init_attr->send_cq) { + qp->send_cq = init_attr->send_cq; + in->req.cqn_send = to_xcq(init_attr->send_cq)->xcq.cqn; + in->req.cqn_send = cpu_to_be16(in->req.cqn_send); + } + + if (init_attr->recv_cq) { + qp->recv_cq = init_attr->recv_cq; + in->req.cqn_recv = to_xcq(init_attr->recv_cq)->xcq.cqn; + in->req.cqn_recv = cpu_to_be16(in->req.cqn_recv); + } + + in->req.qp_type = ib_to_xsc_qp_type(init_attr->qp_type, pd->uobject ? + ucmd.flags : 0); + + if (in->req.qp_type == XSC_QUEUE_TYPE_INVALID) { + ibdev_err(&dev->ib_dev, "invalid qp type:%d\n", init_attr->qp_type); + goto err_create; + } + in->req.glb_funcid = cpu_to_be16(dev->xdev->glb_func_id); + in->req.page_shift = PAGE_SHIFT; + + qp->xqp.qp_type_internal = in->req.qp_type; + + err = xsc_core_create_qp(dev->xdev, &qp->xqp, in, inlen); + if (err) { + ibdev_err(&dev->ib_dev, "create qp failed, err=%d\n", err); + goto err_create; + } + + qp->doorbell_qpn = qp->xqp.qpn; + + qp->xqp.event = xsc_ib_qp_event; + qp->xqp.qp_type = init_attr->qp_type; + ret += snprintf(ptr + ret, 256 - ret, "pdn=%d,", to_xpd(pd)->pdn); + ret += snprintf(ptr + ret, 256 - ret, "log_rq_sz=%d,", in->req.log_rq_sz); + ret += snprintf(ptr + ret, 256 - ret, "log_sq_sz=%d,", in->req.log_sq_sz); + ret += snprintf(ptr + ret, 256 - ret, "scqn=%d,", to_xcq(init_attr->send_cq)->xcq.cqn); + ret += snprintf(ptr + ret, 256 - ret, "rcqn=%d", to_xcq(init_attr->recv_cq)->xcq.cqn); + + get_cqs(init_attr->qp_type, init_attr->send_cq, init_attr->recv_cq, &send_cq, &recv_cq); + xsc_ib_lock_cqs(send_cq, recv_cq); + if (send_cq) + list_add_tail(&qp->cq_send_list, &send_cq->send_qp_list); + if (recv_cq) + list_add_tail(&qp->cq_recv_list, &recv_cq->recv_qp_list); + xsc_ib_unlock_cqs(send_cq, recv_cq); + + if (!pd->uobject && init_attr->srq) { + qp_node = kvzalloc_obj(*qp_node, GFP_KERNEL); + if (!qp_node) + goto err_create; + qp_node->ibqp = &qp->ibqp; + qp_node->qp_id = qp->xqp.qpn; + ibdev_dbg(&dev->ib_dev, "qpn:%u add into srq_qp_list\n", qp_node->qp_id); + list_add_tail(&qp_node->entry, &to_xsrq(init_attr->srq)->xsrq.srq_qp_list); + } + + qp->xqp.qp_protocol_mode = dev->xdev->rdma_proto_mode; + + ibdev_dbg(&dev->ib_dev, "succeeded to create qp:%d, %s\n", qp->xqp.qpn, buf); + + xsc_vfree(in); + + return 0; + +err_create: + if (qp->create_type == XSC_QP_USER) + destroy_qp_user(pd, qp); + else if (qp->create_type == XSC_QP_KERNEL) + destroy_qp_kernel(dev, qp); + + xsc_vfree(in); + return err; +} + +static struct xsc_ib_pd *get_pd(struct xsc_ib_qp *qp) +{ + return to_xpd(qp->ibqp.pd); +} + +static void destroy_qp_common(struct xsc_ib_dev *dev, struct xsc_ib_qp *qp) +{ + struct xsc_ib_cq *send_cq = NULL; + struct xsc_ib_cq *recv_cq = NULL; + struct xsc_modify_qp_mbox_in *in; + int err; + + in = kzalloc_obj(*in, GFP_KERNEL); + if (!in) + return; + + if (qp->xqp.qp_type_internal == XSC_QUEUE_TYPE_RAW || + qp->xqp.qp_type_internal == XSC_QUEUE_TYPE_RAW_TSO || + qp->xqp.qp_type_internal == XSC_QUEUE_TYPE_RAW_TX || + qp->state != IB_QPS_RESET) + if (xsc_core_qp_modify(dev->xdev, to_xsc_state(qp->state), + XSC_QP_STATE_RST, in, sizeof(*in), &qp->xqp)) + ibdev_warn(&dev->ib_dev, "modify QP %06x to RESET failed\n", qp->xqp.qpn); + + get_cqs(qp->xqp.qp_type, qp->ibqp.send_cq, qp->ibqp.recv_cq, &send_cq, &recv_cq); + xsc_ib_lock_cqs(send_cq, recv_cq); + if (send_cq) + list_del(&qp->cq_send_list); + + if (recv_cq) + list_del(&qp->cq_recv_list); + + if (qp->create_type == XSC_QP_KERNEL) { + __xsc_ib_cq_clean(recv_cq, qp->xqp.qpn); + if (send_cq != recv_cq) + __xsc_ib_cq_clean(send_cq, qp->xqp.qpn); + } + xsc_ib_unlock_cqs(send_cq, recv_cq); + + err = xsc_core_destroy_qp(dev->xdev, &qp->xqp); + if (err) + ibdev_warn(&dev->ib_dev, "failed to destroy QP 0x%x\n", qp->xqp.qpn); + kfree(in); + + if (qp->create_type == XSC_QP_KERNEL) + destroy_qp_kernel(dev, qp); + else if (qp->create_type == XSC_QP_USER) + destroy_qp_user(&get_pd(qp)->ibpd, qp); +} + +static const char *ib_qp_type_str(enum ib_qp_type type) +{ + switch (type) { + case IB_QPT_SMI: + return "IB_QPT_SMI"; + case IB_QPT_GSI: + return "IB_QPT_GSI"; + case IB_QPT_RC: + return "IB_QPT_RC"; + case IB_QPT_UC: + return "IB_QPT_UC"; + case IB_QPT_UD: + return "IB_QPT_UD"; + case IB_QPT_RAW_IPV6: + return "IB_QPT_RAW_IPV6"; + case IB_QPT_RAW_ETHERTYPE: + return "IB_QPT_RAW_ETHERTYPE"; + case IB_QPT_RAW_PACKET: + return "IB_QPT_RAW_PACKET"; + case IB_QPT_MAX: + default: + return "Invalid QP type"; + } +} + +int xsc_ib_create_qp(struct ib_qp *ibqp, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) +{ + struct xsc_ib_dev *dev; + struct xsc_ib_qp *qp; + struct ib_pd *pd = ibqp->pd; + int err; + + if (pd) { + dev = to_xdev(pd->device); + } else { + pr_warn("%s: no PD for transport %s\n", __func__, + ib_qp_type_str(init_attr->qp_type)); + return -EINVAL; + } + + if (init_attr->qp_type != IB_QPT_RAW_PACKET) { + if (!is_support_rdma(dev->xdev) || + (is_qp1(init_attr->qp_type) && !is_support_rdma_cm(dev->xdev))) { + return -EPROTONOSUPPORT; + } + } + + qp = to_xqp(ibqp); + + qp->xqp.qp_type = init_attr->qp_type; + + switch (init_attr->qp_type) { + case IB_QPT_RC: + case IB_QPT_SMI: + case IB_QPT_GSI: + case IB_QPT_RAW_PACKET: + err = create_qp_common(dev, pd, init_attr, udata, qp); + if (err) { + ibdev_err(&dev->ib_dev, "create_qp_common failed\n"); + return err; + } + + if (is_qp0(init_attr->qp_type)) { + qp->ibqp.qp_num = 0; + } else if (is_qp1(init_attr->qp_type)) { + qp->ibqp.qp_num = 1; + dev->xdev->gsi_qpn = qp->xqp.qpn; + } else { + qp->ibqp.qp_num = qp->xqp.qpn; + } + + break; + + case IB_QPT_RAW_IPV6: + case IB_QPT_RAW_ETHERTYPE: + case IB_QPT_MAX: + default: + ibdev_err(&dev->ib_dev, "unsupported qp type %d\n", + init_attr->qp_type); + /* Don't support raw QPs */ + return -EINVAL; + } + + return 0; +} + +int xsc_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata) +{ + struct xsc_ib_dev *dev = to_xdev(qp->device); + struct xsc_ib_qp *xqp = to_xqp(qp); + struct xsc_err_state_qp_node *tmp = NULL, *err_rq_node = NULL, *err_sq_node = NULL; + struct xsc_srq_qp_node *qp_node = NULL, *tmp_qp_node = NULL; + unsigned long flags; + struct xsc_ib_srq *xsrq; + + if (qp->qp_type == IB_QPT_RC) { + if (xqp->recv_cq) { + spin_lock_irq(&to_xcq(xqp->recv_cq)->lock); + list_for_each_entry_safe(err_rq_node, tmp, + &to_xcq(xqp->recv_cq)->err_state_qp_list, entry) { + if (err_rq_node->qp_id == xqp->xqp.qpn && !err_rq_node->is_sq) { + list_del(&err_rq_node->entry); + kfree(err_rq_node); + } + } + spin_unlock_irq(&to_xcq(xqp->recv_cq)->lock); + } + + if (xqp->send_cq) { + spin_lock_irq(&to_xcq(xqp->send_cq)->lock); + list_for_each_entry_safe(err_sq_node, tmp, + &to_xcq(xqp->send_cq)->err_state_qp_list, entry) { + if (err_sq_node->qp_id == xqp->xqp.qpn && err_sq_node->is_sq) { + list_del(&err_sq_node->entry); + kfree(err_sq_node); + } + } + spin_unlock_irq(&to_xcq(xqp->send_cq)->lock); + } + } + + if (xqp->create_type == XSC_QP_KERNEL && qp->srq) { + if (!to_xsrq(qp->srq) || list_empty(&to_xsrq(qp->srq)->xsrq.srq_qp_list)) + ibdev_warn(&dev->ib_dev, "srq_qp_list is empty!\n"); + + xsrq = to_xsrq(qp->srq); + spin_lock_irqsave(&xsrq->lock, flags); + xsc_restore_cache_srq_wrs(xsrq, xqp); + list_for_each_entry_safe(qp_node, tmp_qp_node, + &to_xsrq(qp->srq)->xsrq.srq_qp_list, entry) { + if (qp_node->qp_id == xqp->xqp.qpn) { + to_xsrq(qp->srq)->qp_num--; + ibdev_dbg(&dev->ib_dev, "srq_qp_list del node qpn:%u qp_num:%d\n", + qp_node->qp_id, to_xsrq(qp->srq)->qp_num); + list_del_init(&qp_node->entry); + kvfree(qp_node); + break; + } + } + spin_unlock_irqrestore(&xsrq->lock, flags); + } + + destroy_qp_common(dev, xqp); + + return 0; +} + +static inline u16 xsc_calc_udp_sport(u32 lqpn, u32 rqpn) +{ + unsigned char *p; + u8 ports[2]; + u16 sport; + u64 tqpn; + + tqpn = ((u64)(lqpn & 0xffffff)) * ((u64)(rqpn & 0xffffff)); + p = (unsigned char *)&tqpn; + ports[0] = p[0] ^ p[2] ^ p[4]; + ports[1] = p[1] ^ p[3] ^ p[5]; + sport = *((u16 *)ports) | 0xC000; + + return sport; +} + +static inline void xsc_path_set_udp_sport(struct xsc_qp_path *path, + const struct rdma_ah_attr *ah, + u32 lqpn, u32 rqpn) +{ + if ((ah->grh.flow_label & UDP_SPORT_MASK) != 0) { + if ((ah->grh.flow_label & UDP_SPORT_MASK_EN) == 0) + path->sport = cpu_to_be16(rdma_flow_label_to_udp_sport(ah->grh.flow_label)); + else + path->sport = cpu_to_be16((ah->grh.flow_label & UDP_SPORT_MASK) >> + UDP_SPORT_OFFSET); + } else { + path->sport = cpu_to_be16(xsc_calc_udp_sport(lqpn, rqpn)); + } +} + +static int xsc_set_path(struct xsc_ib_dev *dev, const struct rdma_ah_attr *ah, + struct xsc_qp_path *path, u8 port, int attr_mask, + u32 path_flags, const struct ib_qp_attr *attr, struct xsc_ib_qp *qp) +{ + struct ib_global_route *grh = rdma_ah_retrieve_grh((struct rdma_ah_attr *)ah); + union ib_gid *dgid = &grh->dgid; + const struct ib_gid_attr *sgid_attr = grh->sgid_attr; + union ib_gid *sgid = &((struct ib_gid_attr *)sgid_attr)->gid; + union { + struct sockaddr _sockaddr; + struct sockaddr_in _sockaddr_in; + struct sockaddr_in6 _sockaddr_in6; + } sgid_addr, dgid_addr; + int force_pcp, force_dscp; + char buf[256] = {0}; + char *ptr = buf; + int ret = 0; + + if (ah->type == RDMA_AH_ATTR_TYPE_ROCE) { + if (!(rdma_ah_get_ah_flags(ah) & IB_AH_GRH)) + return -EINVAL; + + if (qp->ibqp.qp_type == IB_QPT_RC || + qp->ibqp.qp_type == IB_QPT_UC) { + if (((ah->grh.flow_label & UDP_SPORT_MASK_EN) != 0) && + qp->xqp.qp_protocol_mode == RDMA_PROTO_VEROCE) { + ibdev_err(&dev->ib_dev, + "qp type is RDMA_PROTO_VEROCE, using flow_label to set the UDP sport %d is not allowed.", + (ah->grh.flow_label & UDP_SPORT_MASK) >> + UDP_SPORT_OFFSET); + return -EINVAL; + } + xsc_path_set_udp_sport(path, ah, qp->ibqp.qp_num, attr->dest_qp_num); + } + + if (sgid_attr->gid_type != IB_GID_TYPE_ROCE_UDP_ENCAP) { + ibdev_err(&dev->ib_dev, "gid type not ROCEv2\n"); + return -EINVAL; + } + + force_dscp = dev->xdev->priv.force_dscp; + if (force_dscp == DSCP_PCP_UNSET) + path->ecn_dscp = (grh->traffic_class >> 2) & 0x3f; + else + path->ecn_dscp = force_dscp; + path->hop_limit = grh->hop_limit; + + rdma_gid2ip((struct sockaddr *)&sgid_addr, sgid); + rdma_gid2ip((struct sockaddr *)&dgid_addr, dgid); + + if (sgid_addr._sockaddr.sa_family == AF_INET && + dgid_addr._sockaddr.sa_family == AF_INET) { + memcpy(path->sip, &sgid_addr._sockaddr_in.sin_addr.s_addr, + sizeof(struct in_addr)); + memcpy(path->dip, &dgid_addr._sockaddr_in.sin_addr.s_addr, + sizeof(struct in_addr)); + path->af_type = AF_INET; + ret += snprintf(ptr + ret, 256 - ret, "sip=%#x,", + be32_to_cpu(path->sip[0])); + ret += snprintf(ptr + ret, 256 - ret, "dip=%#x,", + be32_to_cpu(path->dip[0])); + } else if (sgid_addr._sockaddr.sa_family == AF_INET6 && + dgid_addr._sockaddr.sa_family == AF_INET6) { + memcpy(path->sip, &sgid_addr._sockaddr_in6.sin6_addr.s6_addr, + sizeof(path->sip)); + memcpy(path->dip, &dgid_addr._sockaddr_in6.sin6_addr.s6_addr, + sizeof(path->dip)); + path->af_type = AF_INET6; + ret += snprintf(ptr + ret, 256 - ret, "sip=%08x%08x%08x%08x,", + be32_to_cpu(path->sip[0]), be32_to_cpu(path->sip[1]), + be32_to_cpu(path->sip[2]), be32_to_cpu(path->sip[3])); + ret += snprintf(ptr + ret, 256 - ret, "dip=%08x%08x%08x%08x,", + be32_to_cpu(path->dip[0]), be32_to_cpu(path->dip[1]), + be32_to_cpu(path->dip[2]), be32_to_cpu(path->dip[3])); + } else { + return -EINVAL; + } + + ether_addr_copy(path->smac, dev->netdev->dev_addr); + + memcpy(path->dmac, ah->roce.dmac, sizeof(ah->roce.dmac)); + ret += snprintf(ptr + ret, 256 - ret, "smac=%02x%02x%02x%02x%02x%02x,", + path->smac[0], path->smac[1], path->smac[2], + path->smac[3], path->smac[4], path->smac[5]); + ret += snprintf(ptr + ret, 256 - ret, "dmac=%02x%02x%02x%02x%02x%02x", + path->dmac[0], path->dmac[1], path->dmac[2], + path->dmac[3], path->dmac[4], path->dmac[5]); + if (is_zero_ether_addr(path->dmac)) + ibdev_err(&dev->ib_dev, "path dmac is zero!\n"); + else + ibdev_dbg(&dev->ib_dev, "ib path info:%s\n", buf); + + if (is_vlan_dev(sgid_attr->ndev)) { + path->vlan_valid = 1; + path->vlan_id = cpu_to_be16(vlan_dev_vlan_id(sgid_attr->ndev)); + + force_pcp = dev->xdev->priv.force_pcp; + if (force_pcp == DSCP_PCP_UNSET) + path->dci_cfi_prio_sl = (ah->sl & 0x7); + else + path->dci_cfi_prio_sl = force_pcp; + } else { + path->vlan_valid = 0; + } + } + ibdev_dbg(&dev->ib_dev, "path dscp %d pcp %d\n", path->ecn_dscp, path->dci_cfi_prio_sl); + return 0; +} + static inline void xsc_ib_trigger_cq_comp_handler(struct ib_cq *send_cq, struct ib_cq *recv_cq) { if (send_cq) @@ -127,10 +1073,285 @@ int xsc_ib_err_state_qp(struct xsc_ib_dev *dev, struct xsc_ib_qp *qp, return 0; } +static inline void xsc_set_qp_access_flag(struct xsc_modify_qp_mbox_in *in, + struct xsc_qp_context *context, int access_flags) +{ + in->hdr.ver = cpu_to_be16(XSC_QP_CONTEXT_V1); + context->qp_access_flags = 0; + + if (access_flags & IB_ACCESS_REMOTE_READ) + context->qp_access_flags |= QP_ACCESS_REMOTE_READ; + if (access_flags & IB_ACCESS_REMOTE_WRITE) + context->qp_access_flags |= QP_ACCESS_REMOTE_WRITE; + if (access_flags & IB_ACCESS_REMOTE_ATOMIC) + context->qp_access_flags |= QP_ACCESS_REMOTE_ATOMIC; + + context->qp_access_flags = cpu_to_be32(context->qp_access_flags); +} + +static int __xsc_ib_modify_qp(struct ib_qp *ibqp, + const struct ib_qp_attr *attr, int attr_mask, + enum ib_qp_state cur_state, enum ib_qp_state new_state, + const struct xsc_ib_modify_qp *ucmd, + struct ib_udata *udata) +{ + struct xsc_ib_dev *dev = to_xdev(ibqp->device); + struct xsc_ib_qp *qp = to_xqp(ibqp); + struct xsc_ib_cq *send_cq, *recv_cq; + struct xsc_qp_context *context; + struct xsc_modify_qp_mbox_in *in; + struct xsc_qp_path path; + int sqd_event; + int err; + char buf[256] = {0}; + char *ptr = buf; + int ret = 0; + struct xsc_core_device *xdev = dev->xdev; + + in = kzalloc_obj(*in, GFP_KERNEL); + if (!in) + return -ENOMEM; + + context = &qp->ctx; + + if (attr_mask & IB_QP_PATH_MTU) { + if (attr->path_mtu != IB_MTU_1024 && + attr->path_mtu != IB_MTU_4096 && + attr->path_mtu != IB_MTU_8192) { + ibdev_warn(&dev->ib_dev, "invalid mtu %d\n", attr->path_mtu); + } + + if (attr->path_mtu < IB_MTU_4096) + context->mtu_mode = 0; + else if (attr->path_mtu < IB_MTU_8192) + context->mtu_mode = 1; + else + context->mtu_mode = 2; + ret = snprintf(ptr, 256, "path_mtu=%d,", attr->path_mtu); + } + + if (attr_mask & IB_QP_DEST_QPN) { + context->remote_qpn = cpu_to_be32(attr->dest_qp_num); + ret += snprintf(ptr + ret, 256 - ret, "dest_qp_num=%d,", attr->dest_qp_num); + } + + if (attr_mask & IB_QP_AV) { + err = xsc_set_path(dev, &attr->ah_attr, &path, + attr_mask & IB_QP_PORT ? attr->port_num : qp->port, + attr_mask, 0, attr, qp); + if (err) + goto out; + + context->src_udp_port = path.sport; + context->dscp = path.ecn_dscp; + context->hop_limit = path.hop_limit; + context->ip_type = (path.af_type == AF_INET ? 0 : 1); + context->ip_type = cpu_to_be16(context->ip_type); + memcpy(context->dip, path.dip, sizeof(context->dip)); + memcpy(context->sip, path.sip, sizeof(context->sip)); + memcpy(context->dmac, path.dmac, sizeof(path.dmac)); + memcpy(context->smac, path.smac, sizeof(path.smac)); + + context->vlan_valid = path.vlan_valid; + context->dci_cfi_prio_sl = path.dci_cfi_prio_sl; + context->vlan_id = path.vlan_id; + } + + if (attr_mask & IB_QP_RNR_RETRY) { + context->rnr_retry = attr->rnr_retry; + ret += snprintf(ptr + ret, 256 - ret, "rnr_retry=%d,", attr->rnr_retry); + } + + if (attr_mask & IB_QP_RETRY_CNT) { + context->retry_cnt = attr->retry_cnt; + ret += snprintf(ptr + ret, 256 - ret, "retry_cnt=%d,", attr->retry_cnt); + } + + if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && attr->max_rd_atomic) + context->max_sra = min_t(u32, attr->max_rd_atomic, dev->xdev->caps.max_ra_req_qp); + + if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && attr->max_dest_rd_atomic) + context->max_rra = attr->max_dest_rd_atomic; + + if (attr_mask & IB_QP_SQ_PSN) { + context->next_send_psn = cpu_to_be32(attr->sq_psn); + ret += snprintf(ptr + ret, 256 - ret, "sq_psn=%#x,", attr->sq_psn); + } + + if (attr_mask & IB_QP_RQ_PSN) { + context->next_recv_psn = cpu_to_be32(attr->rq_psn); + ret += snprintf(ptr + ret, 256 - ret, "rq_psn=%#x,", attr->rq_psn); + } + + if (attr_mask & IB_QP_ACCESS_FLAGS) { + xsc_set_qp_access_flag(in, context, attr->qp_access_flags); + ret += snprintf(ptr + ret, 256 - ret, "qp_access_flags=%#x,\n", + attr->qp_access_flags); + } + + if (udata && udata->inlen && attr_mask & IB_QP_STATE && new_state == IB_QPS_RTR) { + if (!ucmd->profile && + xdev->rdma_proto_mode == RDMA_PROTO_VEROCE) { + context->swith_qp_type = 1; + context->qp_access_flags |= QP_ACCESS_MODIFY_QP_TYPE; + context->qp_access_flags = cpu_to_be32(context->qp_access_flags); + qp->xqp.qp_protocol_mode = + ucmd->profile ? RDMA_PROTO_VEROCE : RDMA_PROTO_ROCEV2; + in->hdr.ver = cpu_to_be16(XSC_QP_CONTEXT_V2); + ibdev_dbg(&dev->ib_dev, + "local is VEROCE,remote is RoCEv2, switch qp_type from VEROCE to RoCEv2!\n"); + } + } + + if (attr_mask & IB_QP_STATE && new_state == IB_QPS_RESET) + qp->xqp.qp_protocol_mode = xdev->rdma_proto_mode; + + if (cur_state == IB_QPS_RTS && new_state == IB_QPS_SQD && + attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY && attr->en_sqd_async_notify) + sqd_event = 1; + else + sqd_event = 0; + + memcpy(&in->ctx, context, sizeof(*context)); + err = xsc_core_qp_modify(xdev, to_xsc_state(cur_state), + to_xsc_state(new_state), in, sqd_event, + &qp->xqp); + if (err) { + ibdev_err(&dev->ib_dev, "failed to modify qp[%d] from %s to %s\n", + qp->xqp.qpn, qp_state_to_str(cur_state), qp_state_to_str(new_state)); + goto out; + } + + qp->state = new_state; + ibdev_dbg(&dev->ib_dev, "succeeded to modify qp[%d] from %s to %s with attr_mask=%#x, %s\n", + qp->xqp.qpn, qp_state_to_str(cur_state), qp_state_to_str(new_state), + attr_mask, buf); + + if (qp->state == IB_QPS_INIT) { + qp->post_recv_invalid = false; + qp->post_send_invalid = false; + } + + if (attr_mask & IB_QP_ACCESS_FLAGS) + qp->atomic_rd_en = attr->qp_access_flags; + if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) + qp->resp_depth = attr->max_dest_rd_atomic; + if (attr_mask & IB_QP_PORT) + qp->port = attr->port_num; + if (attr_mask & IB_QP_ALT_PATH) + qp->alt_port = attr->alt_port_num; + + /* + * If we moved a kernel QP to RESET, clean up all old CQ + * entries and reinitialize the QP. + */ + if (new_state == IB_QPS_RESET && !ibqp->uobject) { + get_cqs(qp->xqp.qp_type, qp->ibqp.send_cq, qp->ibqp.recv_cq, &send_cq, &recv_cq); + xsc_ib_cq_clean(recv_cq, qp->xqp.qpn); + if (send_cq != recv_cq) + xsc_ib_cq_clean(send_cq, qp->xqp.qpn); + + qp->rq.head = 0; + qp->rq.tail = 0; + qp->sq.head = 0; + qp->sq.tail = 0; + qp->sq.cur_post = 0; + qp->sq.last_poll = 0; + qp->sq.last_comp_ce_wqe = -1; + qp->sq.last_post_ce_wqe = -1; + qp->has_trig_cq_comp_handle = false; + } + +out: + kfree(in); + return err; +} + enum xsc_ib_qp_ex_attr_mask { XSC_IB_QP_FLUSH = 1 << 21, }; +int xsc_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata) +{ + struct xsc_ib_dev *dev = to_xdev(ibqp->device); + struct xsc_ib_qp *qp = to_xqp(ibqp); + enum ib_qp_state cur_state, new_state; + int err = -EINVAL; + int wr_num; + struct xsc_ib_srq *srq; + unsigned long flags; + struct xsc_ib_cq *send_cq; + struct xsc_ib_cq *recv_cq; + struct xsc_ib_modify_qp ucmd = {}; + + if (udata && udata->inlen) { + if (udata->inlen < offsetofend(typeof(ucmd), profile)) + return -EINVAL; + + if (ib_copy_from_udata(&ucmd, udata, + min(udata->inlen, sizeof(ucmd)))) + return -EFAULT; + } + + mutex_lock(&qp->mutex); + + cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state; + new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; + + if ((attr_mask & IB_QP_PORT) && + (attr->port_num == 0 || attr->port_num > dev->xdev->caps.num_ports)) { + ibdev_err(&dev->ib_dev, "error port num\n"); + goto out; + } + + if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && + attr->max_dest_rd_atomic > dev->xdev->caps.max_ra_res_qp) { + ibdev_err(&dev->ib_dev, "dest rd atomic:%u exeeded", attr->max_dest_rd_atomic); + goto out; + } + + if (cur_state == new_state && cur_state == IB_QPS_RESET) { + err = 0; + goto out; + } + + if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, attr_mask)) { + ibdev_err(&dev->ib_dev, "ib_modify_qp not ok, type=%d mask=0x%x state from%d to%d\n", + ibqp->qp_type, attr_mask, cur_state, new_state); + goto out; + } + + err = __xsc_ib_modify_qp(ibqp, attr, attr_mask, cur_state, new_state, &ucmd, udata); + + if (!err && (attr_mask & IB_QP_STATE) && !udata) { + get_cqs(qp->xqp.qp_type, qp->send_cq, qp->recv_cq, &send_cq, &recv_cq); + xsc_ib_lock_cqs(send_cq, recv_cq); + err = xsc_ib_err_state_qp(dev, qp, to_xsc_state(cur_state), + to_xsc_state(new_state)); + xsc_ib_unlock_cqs(send_cq, recv_cq); + + if (!ibqp->srq) + goto out; + + if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { + srq = to_xsrq(ibqp->srq); + spin_lock_irqsave(&srq->lock, flags); + srq->qp_num++; + wr_num = XSC_SRQ_MIN_WR_NUM; + ibdev_dbg(&dev->ib_dev, + "SRQ %d: qp %d to INIT, post %d srq wrs, qp_num %d\n", + srq->xsrq.srqn, qp->xqp.qpn, wr_num, srq->qp_num); + xsc_post_cache_wrs(srq, qp, wr_num, true); + spin_unlock_irqrestore(&srq->lock, flags); + } + } + +out: + mutex_unlock(&qp->mutex); + return err; +} + static int xsc_wq_overflow(struct xsc_ib_wq *wq, int nreq, struct xsc_ib_cq *cq) { unsigned int cur; @@ -231,6 +1452,106 @@ int xsc_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, return err; } +static inline enum ib_qp_state to_ib_qp_state(enum xsc_qp_state xsc_state) +{ + switch (xsc_state) { + case XSC_QP_STATE_RST: return IB_QPS_RESET; + case XSC_QP_STATE_INIT: return IB_QPS_INIT; + case XSC_QP_STATE_RTR: return IB_QPS_RTR; + case XSC_QP_STATE_RTS: return IB_QPS_RTS; + case XSC_QP_STATE_SQ_DRAINING: + case XSC_QP_STATE_SQD: return IB_QPS_SQD; + case XSC_QP_STATE_SQER: return IB_QPS_SQE; + case XSC_QP_STATE_ERR: return IB_QPS_ERR; + default: return -1; + } +} + +static inline enum ib_mig_state to_ib_mig_state(int xsc_mig_state) +{ + switch (xsc_mig_state) { + case XSC_QP_PM_ARMED: return IB_MIG_ARMED; + case XSC_QP_PM_REARM: return IB_MIG_REARM; + case XSC_QP_PM_MIGRATED: return IB_MIG_MIGRATED; + default: return -1; + } +} + +int xsc_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, + struct ib_qp_init_attr *qp_init_attr) +{ + struct xsc_ib_dev *dev = to_xdev(ibqp->device); + struct xsc_ib_qp *qp = to_xqp(ibqp); + struct xsc_query_qp_mbox_out *outb; + struct xsc_qp_context *context; + int err = 0; + + mutex_lock(&qp->mutex); + + if (qp_attr_mask & XSC_IB_QP_FLUSH) { + xsc_ib_trigger_cq_comp_handler(qp->send_cq, qp->recv_cq); + goto out; + } + + outb = kzalloc_obj(*outb, GFP_KERNEL); + if (!outb) { + err = -ENOMEM; + goto out; + } + context = &outb->ctx; + err = xsc_core_qp_query(dev->xdev, &qp->xqp, outb, sizeof(*outb)); + if (err) + goto out_free; + + if (qp->xqp.err_occurred) { + qp->state = IB_QPS_ERR; + qp->xqp.err_occurred = 0; + } + qp_attr->qp_state = qp->state; + qp_attr->path_mtu = context->mtu_mode ? IB_MTU_4096 : IB_MTU_1024; + qp_attr->rq_psn = be32_to_cpu(context->next_recv_psn) & 0xffffff; + qp_attr->sq_psn = be32_to_cpu(context->next_send_psn) & 0xffffff; + qp_attr->dest_qp_num = be32_to_cpu(context->remote_qpn) & 0xffffff; + qp_attr->sq_draining = 0; + qp_attr->retry_cnt = context->retry_cnt; + qp_attr->rnr_retry = context->rnr_retry; + qp_attr->cur_qp_state = qp_attr->qp_state; + qp_attr->cap.max_recv_wr = qp->rq.wqe_cnt; + qp_attr->cap.max_recv_sge = qp->rq.max_gs; + qp_attr->max_rd_atomic = context->max_sra; + qp_attr->max_dest_rd_atomic = context->max_rra; + qp_attr->port_num = 1; + + if (!ibqp->uobject) { + qp_attr->cap.max_send_wr = qp->sq.wqe_cnt; + qp_attr->cap.max_send_sge = qp->sq.max_gs; + } else { + qp_attr->cap.max_send_wr = qp->sq.wqe_cnt; + qp_attr->cap.max_send_sge = qp->sq.max_gs; + } + + /* We don't support inline sends for kernel QPs (yet), and we + * don't know what userspace's value should be. + */ + qp_attr->cap.max_inline_data = 0; + + qp_init_attr->cap = qp_attr->cap; + + qp_init_attr->create_flags = 0; + if (qp->flags & XSC_IB_QP_BLOCK_MULTICAST_LOOPBACK) + qp_init_attr->create_flags |= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK; + + qp_init_attr->sq_sig_type = qp->sq_signal_bits & XSC_WQE_CTRL_CQ_UPDATE ? + IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR; + +out_free: + kfree(outb); + +out: + mutex_unlock(&qp->mutex); + return err; +} + struct xsc_ib_drain_cqe { struct ib_cqe cqe; struct completion done;