From: Dragos Tatulea Previously the HW-GRO code was using a separate page_pool for the header buffer. The pages of the header buffer were replenished via UMR. This mechanism has some drawbacks: - Reference counting on the page_pool page frags is not cheap. - UMRs have HW overhead for updating and also for access. Especially for the KLM type which was previously used. - UMR code for headers is complex. This patch switches to using a static memory area (static MTT MKEY) for the header buffer and does a header memcpy. This happens only once per GRO session. The SKB is allocated from the per-cpu NAPI SKB cache. Performance numbers for x86: +---------------------------------------------------------+ | Test | Baseline | Header Copy | Change | |---------------------+------------+-------------+--------| | iperf3 oncpu | 59.5 Gbps | 64.00 Gbps | 7 % | | iperf3 offcpu | 102.5 Gbps | 104.20 Gbps | 2 % | | kperf oncpu | 115.0 Gbps | 130.00 Gbps | 12 % | | XDP_DROP (skb mode) | 3.9 Mpps | 3.9 Mpps | 0 % | +---------------------------------------------------------+ Notes on test: - System: Intel(R) Xeon(R) Platinum 8380 CPU @ 2.30GHz - oncpu: NAPI and application running on same CPU - offcpu: NAPI and application running on different CPUs - MTU: 1500 - iperf3 tests are single stream, 60s with IPv6 (for slightly larger headers) - kperf version [1] [1] git://git.kernel.dk/kperf.git Suggested-by: Eric Dumazet Signed-off-by: Dragos Tatulea Signed-off-by: Tariq Toukan --- drivers/net/ethernet/mellanox/mlx5/core/en.h | 20 +- .../net/ethernet/mellanox/mlx5/core/en/txrx.h | 1 - .../net/ethernet/mellanox/mlx5/core/en_main.c | 287 +++++++-------- .../net/ethernet/mellanox/mlx5/core/en_rx.c | 336 +++--------------- 4 files changed, 185 insertions(+), 459 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 262dc032e276..fcce50e46165 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -82,9 +82,10 @@ struct page_pool; #define MLX5E_RX_MAX_HEAD (256) #define MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE (8) -#define MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE (9) -#define MLX5E_SHAMPO_WQ_HEADER_PER_PAGE (PAGE_SIZE >> MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE) -#define MLX5E_SHAMPO_LOG_WQ_HEADER_PER_PAGE (PAGE_SHIFT - MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE) +#define MLX5E_SHAMPO_WQ_HEADER_PER_PAGE \ + (PAGE_SIZE >> MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE) +#define MLX5E_SHAMPO_LOG_WQ_HEADER_PER_PAGE \ + (PAGE_SHIFT - MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE) #define MLX5E_SHAMPO_WQ_BASE_HEAD_ENTRY_SIZE_SHIFT (6) #define MLX5E_SHAMPO_WQ_RESRV_SIZE_BASE_SHIFT (12) #define MLX5E_SHAMPO_WQ_LOG_RESRV_SIZE (16) @@ -632,16 +633,11 @@ struct mlx5e_dma_info { }; struct mlx5e_shampo_hd { - struct mlx5e_frag_page *pages; u32 hd_per_wq; - u32 hd_per_page; - u16 hd_per_wqe; - u8 log_hd_per_page; - u8 log_hd_entry_size; - unsigned long *bitmap; - u16 pi; - u16 ci; - __be32 mkey_be; + u32 hd_buf_size; + u32 mkey; + u32 nentries; + DECLARE_FLEX_ARRAY(struct mlx5e_dma_info, hd_buf_pages); }; struct mlx5e_hw_gro_data { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h index 7e191e1569e8..f2a8453d8dce 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h @@ -65,7 +65,6 @@ ktime_t mlx5e_cqe_ts_to_ns(cqe_ts_to_ns func, struct mlx5_clock *clock, u64 cqe_ enum mlx5e_icosq_wqe_type { MLX5E_ICOSQ_WQE_NOP, MLX5E_ICOSQ_WQE_UMR_RX, - MLX5E_ICOSQ_WQE_SHAMPO_HD_UMR, #ifdef CONFIG_MLX5_EN_TLS MLX5E_ICOSQ_WQE_UMR_TLS, MLX5E_ICOSQ_WQE_SET_PSV_TLS, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 136fa8f05607..4ee92eea1324 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -492,40 +492,6 @@ static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev, return err; } -static int mlx5e_create_umr_ksm_mkey(struct mlx5_core_dev *mdev, - u64 nentries, u8 log_entry_size, - u32 *umr_mkey) -{ - int inlen; - void *mkc; - u32 *in; - int err; - - inlen = MLX5_ST_SZ_BYTES(create_mkey_in); - - in = kvzalloc(inlen, GFP_KERNEL); - if (!in) - return -ENOMEM; - - mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); - - MLX5_SET(mkc, mkc, free, 1); - MLX5_SET(mkc, mkc, umr_en, 1); - MLX5_SET(mkc, mkc, lw, 1); - MLX5_SET(mkc, mkc, lr, 1); - MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KSM); - mlx5e_mkey_set_relaxed_ordering(mdev, mkc); - MLX5_SET(mkc, mkc, qpn, 0xffffff); - MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn); - MLX5_SET(mkc, mkc, translations_octword_size, nentries); - MLX5_SET(mkc, mkc, log_page_size, log_entry_size); - MLX5_SET64(mkc, mkc, len, nentries << log_entry_size); - err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen); - - kvfree(in); - return err; -} - static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq) { u32 xsk_chunk_size = rq->xsk_pool ? rq->xsk_pool->chunk_size : 0; @@ -551,29 +517,6 @@ static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq return err; } -static int mlx5e_create_rq_hd_umr_mkey(struct mlx5_core_dev *mdev, - u16 hd_per_wq, __be32 *umr_mkey) -{ - u32 max_ksm_size = BIT(MLX5_CAP_GEN(mdev, log_max_klm_list_size)); - u32 mkey; - int err; - - if (max_ksm_size < hd_per_wq) { - mlx5_core_err(mdev, "max ksm list size 0x%x is smaller than shampo header buffer list size 0x%x\n", - max_ksm_size, hd_per_wq); - return -EINVAL; - } - - err = mlx5e_create_umr_ksm_mkey(mdev, hd_per_wq, - MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE, - &mkey); - if (err) - return err; - - *umr_mkey = cpu_to_be32(mkey); - return 0; -} - static void mlx5e_init_frags_partition(struct mlx5e_rq *rq) { struct mlx5e_wqe_frag_info next_frag = {}; @@ -754,145 +697,169 @@ static int mlx5e_init_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *param xdp_frag_size); } -static int mlx5e_rq_shampo_hd_info_alloc(struct mlx5e_rq *rq, u16 hd_per_wq, - int node) +static void mlx5e_release_rq_hd_pages(struct mlx5e_rq *rq, + struct mlx5e_shampo_hd *shampo) + { - struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; + for (int i = 0; i < shampo->nentries; i++) { + struct mlx5e_dma_info *info = &shampo->hd_buf_pages[i]; - shampo->hd_per_wq = hd_per_wq; + if (!info->page) + continue; + + dma_unmap_page(rq->pdev, info->addr, PAGE_SIZE, + rq->buff.map_dir); + __free_page(info->page); + } +} + +static int mlx5e_alloc_rq_hd_pages(struct mlx5e_rq *rq, int node, + struct mlx5e_shampo_hd *shampo) +{ + int err, i; + + for (i = 0; i < shampo->nentries; i++) { + struct page *page = alloc_pages_node(node, GFP_KERNEL, 0); + dma_addr_t addr; + + if (!page) { + err = -ENOMEM; + goto err_free_pages; + } - shampo->bitmap = bitmap_zalloc_node(hd_per_wq, GFP_KERNEL, node); - shampo->pages = kvzalloc_node(array_size(hd_per_wq, - sizeof(*shampo->pages)), - GFP_KERNEL, node); - if (!shampo->bitmap || !shampo->pages) - goto err_nomem; + addr = dma_map_page(rq->pdev, page, 0, PAGE_SIZE, + rq->buff.map_dir); + err = dma_mapping_error(rq->pdev, addr); + if (err) { + __free_page(page); + goto err_free_pages; + } + + shampo->hd_buf_pages[i].page = page; + shampo->hd_buf_pages[i].addr = addr; + } return 0; -err_nomem: - kvfree(shampo->pages); - bitmap_free(shampo->bitmap); +err_free_pages: + mlx5e_release_rq_hd_pages(rq, shampo); - return -ENOMEM; + return err; } -static void mlx5e_rq_shampo_hd_info_free(struct mlx5e_rq *rq) +static int mlx5e_create_rq_hd_mkey(struct mlx5_core_dev *mdev, + struct mlx5e_shampo_hd *shampo) { - kvfree(rq->mpwqe.shampo->pages); - bitmap_free(rq->mpwqe.shampo->bitmap); + enum mlx5e_mpwrq_umr_mode umr_mode = MLX5E_MPWRQ_UMR_MODE_ALIGNED; + struct mlx5_mtt *mtt; + void *mkc, *in; + int inlen, err; + u32 octwords; + + octwords = mlx5e_mpwrq_umr_octowords(shampo->nentries, umr_mode); + inlen = MLX5_FLEXIBLE_INLEN(mdev, MLX5_ST_SZ_BYTES(create_mkey_in), + MLX5_OCTWORD, octwords); + if (inlen < 0) + return inlen; + + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) + return -ENOMEM; + + mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); + + MLX5_SET(mkc, mkc, lw, 1); + MLX5_SET(mkc, mkc, lr, 1); + MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); + mlx5e_mkey_set_relaxed_ordering(mdev, mkc); + MLX5_SET(mkc, mkc, qpn, 0xffffff); + MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn); + MLX5_SET64(mkc, mkc, len, shampo->hd_buf_size); + MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT); + MLX5_SET(mkc, mkc, translations_octword_size, octwords); + MLX5_SET(create_mkey_in, in, translations_octword_actual_size, + octwords); + + mtt = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); + for (int i = 0; i < shampo->nentries; i++) + mtt[i].ptag = cpu_to_be64(shampo->hd_buf_pages[i].addr); + + err = mlx5_core_create_mkey(mdev, &shampo->mkey, in, inlen); + + kvfree(in); + return err; } static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev, struct mlx5e_params *params, struct mlx5e_rq_param *rqp, struct mlx5e_rq *rq, - u32 *pool_size, int node) { - void *wqc = MLX5_ADDR_OF(rqc, rqp->rqc, wq); - u8 log_hd_per_page, log_hd_entry_size; - u16 hd_per_wq, hd_per_wqe; - u32 hd_pool_size; - int wq_size; - int err; + struct mlx5e_shampo_hd *shampo; + int nentries, err, shampo_sz; + u32 hd_per_wq, hd_buf_size; if (!test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) return 0; - rq->mpwqe.shampo = kvzalloc_node(sizeof(*rq->mpwqe.shampo), - GFP_KERNEL, node); - if (!rq->mpwqe.shampo) - return -ENOMEM; - - /* split headers data structures */ hd_per_wq = mlx5e_shampo_hd_per_wq(mdev, params, rqp); - err = mlx5e_rq_shampo_hd_info_alloc(rq, hd_per_wq, node); - if (err) - goto err_shampo_hd_info_alloc; - - err = mlx5e_create_rq_hd_umr_mkey(mdev, hd_per_wq, - &rq->mpwqe.shampo->mkey_be); - if (err) - goto err_umr_mkey; - - hd_per_wqe = mlx5e_shampo_hd_per_wqe(mdev, params, rqp); - wq_size = BIT(MLX5_GET(wq, wqc, log_wq_sz)); - - BUILD_BUG_ON(MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE > PAGE_SHIFT); - if (hd_per_wqe >= MLX5E_SHAMPO_WQ_HEADER_PER_PAGE) { - log_hd_per_page = MLX5E_SHAMPO_LOG_WQ_HEADER_PER_PAGE; - log_hd_entry_size = MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE; - } else { - log_hd_per_page = order_base_2(hd_per_wqe); - log_hd_entry_size = order_base_2(PAGE_SIZE / hd_per_wqe); + hd_buf_size = hd_per_wq * BIT(MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE); + nentries = hd_buf_size / PAGE_SIZE; + if (!nentries) { + mlx5_core_err(mdev, "SHAMPO header buffer size %u < %lu\n", + hd_buf_size, PAGE_SIZE); + return -EINVAL; } - rq->mpwqe.shampo->hd_per_wqe = hd_per_wqe; - rq->mpwqe.shampo->hd_per_page = BIT(log_hd_per_page); - rq->mpwqe.shampo->log_hd_per_page = log_hd_per_page; - rq->mpwqe.shampo->log_hd_entry_size = log_hd_entry_size; - - hd_pool_size = (hd_per_wqe * wq_size) >> log_hd_per_page; - - if (netif_rxq_has_unreadable_mp(rq->netdev, rq->ix)) { - /* Separate page pool for shampo headers */ - struct page_pool_params pp_params = { }; + shampo_sz = struct_size(shampo, hd_buf_pages, nentries); + shampo = kvzalloc_node(shampo_sz, GFP_KERNEL, node); + if (!shampo) + return -ENOMEM; - pp_params.order = 0; - pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; - pp_params.pool_size = hd_pool_size; - pp_params.nid = node; - pp_params.dev = rq->pdev; - pp_params.napi = rq->cq.napi; - pp_params.netdev = rq->netdev; - pp_params.dma_dir = rq->buff.map_dir; - pp_params.max_len = PAGE_SIZE; + shampo->hd_per_wq = hd_per_wq; + shampo->hd_buf_size = hd_buf_size; + shampo->nentries = nentries; + err = mlx5e_alloc_rq_hd_pages(rq, node, shampo); + if (err) + goto err_free; - rq->hd_page_pool = page_pool_create(&pp_params); - if (IS_ERR(rq->hd_page_pool)) { - err = PTR_ERR(rq->hd_page_pool); - rq->hd_page_pool = NULL; - goto err_hds_page_pool; - } - } else { - /* Common page pool, reserve space for headers. */ - *pool_size += hd_pool_size; - rq->hd_page_pool = NULL; - } + err = mlx5e_create_rq_hd_mkey(mdev, shampo); + if (err) + goto err_release_pages; /* gro only data structures */ rq->hw_gro_data = kvzalloc_node(sizeof(*rq->hw_gro_data), GFP_KERNEL, node); if (!rq->hw_gro_data) { err = -ENOMEM; - goto err_hw_gro_data; + goto err_destroy_mkey; } + rq->mpwqe.shampo = shampo; + return 0; -err_hw_gro_data: - page_pool_destroy(rq->hd_page_pool); -err_hds_page_pool: - mlx5_core_destroy_mkey(mdev, be32_to_cpu(rq->mpwqe.shampo->mkey_be)); -err_umr_mkey: - mlx5e_rq_shampo_hd_info_free(rq); -err_shampo_hd_info_alloc: - kvfree(rq->mpwqe.shampo); +err_destroy_mkey: + mlx5_core_destroy_mkey(mdev, shampo->mkey); +err_release_pages: + mlx5e_release_rq_hd_pages(rq, shampo); +err_free: + kvfree(shampo); + return err; } static void mlx5e_rq_free_shampo(struct mlx5e_rq *rq) { - if (!test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) + struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; + + if (!shampo) return; kvfree(rq->hw_gro_data); - if (rq->hd_page_pool != rq->page_pool) - page_pool_destroy(rq->hd_page_pool); - mlx5e_rq_shampo_hd_info_free(rq); - mlx5_core_destroy_mkey(rq->mdev, - be32_to_cpu(rq->mpwqe.shampo->mkey_be)); - kvfree(rq->mpwqe.shampo); + mlx5_core_destroy_mkey(rq->mdev, shampo->mkey); + mlx5e_release_rq_hd_pages(rq, shampo); + kvfree(shampo); } static int mlx5e_alloc_rq(struct mlx5e_params *params, @@ -970,7 +937,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, if (err) goto err_rq_mkey; - err = mlx5_rq_shampo_alloc(mdev, params, rqp, rq, &pool_size, node); + err = mlx5_rq_shampo_alloc(mdev, params, rqp, rq, node); if (err) goto err_free_mpwqe_info; @@ -1165,8 +1132,7 @@ int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_cou if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) { MLX5_SET(wq, wq, log_headers_buffer_entry_num, order_base_2(rq->mpwqe.shampo->hd_per_wq)); - MLX5_SET(wq, wq, headers_mkey, - be32_to_cpu(rq->mpwqe.shampo->mkey_be)); + MLX5_SET(wq, wq, headers_mkey, rq->mpwqe.shampo->mkey); } mlx5_fill_page_frag_array(&rq->wq_ctrl.buf, @@ -1326,14 +1292,6 @@ void mlx5e_free_rx_missing_descs(struct mlx5e_rq *rq) rq->mpwqe.actual_wq_head = wq->head; rq->mpwqe.umr_in_progress = 0; rq->mpwqe.umr_completed = 0; - - if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) { - struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; - u16 len; - - len = (shampo->pi - shampo->ci) & shampo->hd_per_wq; - mlx5e_shampo_fill_umr(rq, len); - } } void mlx5e_free_rx_descs(struct mlx5e_rq *rq) @@ -1356,9 +1314,6 @@ void mlx5e_free_rx_descs(struct mlx5e_rq *rq) mlx5_wq_ll_pop(wq, wqe_ix_be, &wqe->next.next_wqe_index); } - - if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) - mlx5e_shampo_dealloc_hd(rq); } else { struct mlx5_wq_cyc *wq = &rq->wqe.wq; u16 missing = mlx5_wq_cyc_missing(wq); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index aae4db392992..5ab70e057a5c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -611,165 +611,6 @@ static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq, u8 n) mlx5_wq_ll_update_db_record(wq); } -/* This function returns the size of the continuous free space inside a bitmap - * that starts from first and no longer than len including circular ones. - */ -static int bitmap_find_window(unsigned long *bitmap, int len, - int bitmap_size, int first) -{ - int next_one, count; - - next_one = find_next_bit(bitmap, bitmap_size, first); - if (next_one == bitmap_size) { - if (bitmap_size - first >= len) - return len; - next_one = find_next_bit(bitmap, bitmap_size, 0); - count = next_one + bitmap_size - first; - } else { - count = next_one - first; - } - - return min(len, count); -} - -static void build_ksm_umr(struct mlx5e_icosq *sq, struct mlx5e_umr_wqe *umr_wqe, - __be32 key, u16 offset, u16 ksm_len) -{ - memset(umr_wqe, 0, offsetof(struct mlx5e_umr_wqe, inline_ksms)); - umr_wqe->hdr.ctrl.opmod_idx_opcode = - cpu_to_be32((sq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) | - MLX5_OPCODE_UMR); - umr_wqe->hdr.ctrl.umr_mkey = key; - umr_wqe->hdr.ctrl.qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) - | MLX5E_KSM_UMR_DS_CNT(ksm_len)); - umr_wqe->hdr.uctrl.flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE; - umr_wqe->hdr.uctrl.xlt_offset = cpu_to_be16(offset); - umr_wqe->hdr.uctrl.xlt_octowords = cpu_to_be16(ksm_len); - umr_wqe->hdr.uctrl.mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE); -} - -static struct mlx5e_frag_page *mlx5e_shampo_hd_to_frag_page(struct mlx5e_rq *rq, - int header_index) -{ - struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; - - return &shampo->pages[header_index >> shampo->log_hd_per_page]; -} - -static u64 mlx5e_shampo_hd_offset(struct mlx5e_rq *rq, int header_index) -{ - struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; - u32 hd_per_page = shampo->hd_per_page; - - return (header_index & (hd_per_page - 1)) << shampo->log_hd_entry_size; -} - -static void mlx5e_free_rx_shampo_hd_entry(struct mlx5e_rq *rq, u16 header_index); - -static int mlx5e_build_shampo_hd_umr(struct mlx5e_rq *rq, - struct mlx5e_icosq *sq, - u16 ksm_entries, u16 index) -{ - struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; - u16 pi, header_offset, err, wqe_bbs; - u32 lkey = rq->mdev->mlx5e_res.hw_objs.mkey; - struct mlx5e_umr_wqe *umr_wqe; - int headroom, i; - - headroom = rq->buff.headroom; - wqe_bbs = MLX5E_KSM_UMR_WQEBBS(ksm_entries); - pi = mlx5e_icosq_get_next_pi(sq, wqe_bbs); - umr_wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi); - build_ksm_umr(sq, umr_wqe, shampo->mkey_be, index, ksm_entries); - - for (i = 0; i < ksm_entries; i++, index++) { - struct mlx5e_frag_page *frag_page; - u64 addr; - - frag_page = mlx5e_shampo_hd_to_frag_page(rq, index); - header_offset = mlx5e_shampo_hd_offset(rq, index); - if (!header_offset) { - err = mlx5e_page_alloc_fragmented(rq->hd_page_pool, - frag_page); - if (err) - goto err_unmap; - } - - addr = page_pool_get_dma_addr_netmem(frag_page->netmem); - umr_wqe->inline_ksms[i] = (struct mlx5_ksm) { - .key = cpu_to_be32(lkey), - .va = cpu_to_be64(addr + header_offset + headroom), - }; - } - - sq->db.wqe_info[pi] = (struct mlx5e_icosq_wqe_info) { - .wqe_type = MLX5E_ICOSQ_WQE_SHAMPO_HD_UMR, - .num_wqebbs = wqe_bbs, - .shampo.len = ksm_entries, - }; - - shampo->pi = (shampo->pi + ksm_entries) & (shampo->hd_per_wq - 1); - sq->pc += wqe_bbs; - sq->doorbell_cseg = &umr_wqe->hdr.ctrl; - - return 0; - -err_unmap: - while (--i >= 0) { - --index; - header_offset = mlx5e_shampo_hd_offset(rq, index); - if (!header_offset) { - struct mlx5e_frag_page *frag_page = mlx5e_shampo_hd_to_frag_page(rq, index); - - mlx5e_page_release_fragmented(rq->hd_page_pool, - frag_page); - } - } - - rq->stats->buff_alloc_err++; - return err; -} - -static int mlx5e_alloc_rx_hd_mpwqe(struct mlx5e_rq *rq) -{ - struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; - u16 ksm_entries, num_wqe, index, entries_before; - struct mlx5e_icosq *sq = rq->icosq; - int i, err, max_ksm_entries, len; - - max_ksm_entries = MLX5E_MAX_KSM_PER_WQE(rq->mdev); - ksm_entries = bitmap_find_window(shampo->bitmap, - shampo->hd_per_wqe, - shampo->hd_per_wq, shampo->pi); - ksm_entries = ALIGN_DOWN(ksm_entries, shampo->hd_per_page); - if (!ksm_entries) - return 0; - - /* pi is aligned to MLX5E_SHAMPO_WQ_HEADER_PER_PAGE */ - index = shampo->pi; - entries_before = shampo->hd_per_wq - index; - - if (unlikely(entries_before < ksm_entries)) - num_wqe = DIV_ROUND_UP(entries_before, max_ksm_entries) + - DIV_ROUND_UP(ksm_entries - entries_before, max_ksm_entries); - else - num_wqe = DIV_ROUND_UP(ksm_entries, max_ksm_entries); - - for (i = 0; i < num_wqe; i++) { - len = (ksm_entries > max_ksm_entries) ? max_ksm_entries : - ksm_entries; - if (unlikely(index + len > shampo->hd_per_wq)) - len = shampo->hd_per_wq - index; - err = mlx5e_build_shampo_hd_umr(rq, sq, len, index); - if (unlikely(err)) - return err; - index = (index + len) & (rq->mpwqe.shampo->hd_per_wq - 1); - ksm_entries -= len; - } - - return 0; -} - static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) { struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, ix); @@ -782,12 +623,6 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) int err; int i; - if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) { - err = mlx5e_alloc_rx_hd_mpwqe(rq); - if (unlikely(err)) - goto err; - } - pi = mlx5e_icosq_get_next_pi(sq, rq->mpwqe.umr_wqebbs); umr_wqe = mlx5_wq_cyc_get_wqe(wq, pi); memcpy(umr_wqe, &rq->mpwqe.umr_wqe, sizeof(struct mlx5e_umr_wqe)); @@ -848,34 +683,11 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) bitmap_fill(wi->skip_release_bitmap, rq->mpwqe.pages_per_wqe); -err: rq->stats->buff_alloc_err++; return err; } -static void -mlx5e_free_rx_shampo_hd_entry(struct mlx5e_rq *rq, u16 header_index) -{ - struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; - - if (((header_index + 1) & (shampo->hd_per_page - 1)) == 0) { - struct mlx5e_frag_page *frag_page = mlx5e_shampo_hd_to_frag_page(rq, header_index); - - mlx5e_page_release_fragmented(rq->hd_page_pool, frag_page); - } - clear_bit(header_index, shampo->bitmap); -} - -void mlx5e_shampo_dealloc_hd(struct mlx5e_rq *rq) -{ - struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; - int i; - - for_each_set_bit(i, shampo->bitmap, rq->mpwqe.shampo->hd_per_wq) - mlx5e_free_rx_shampo_hd_entry(rq, i); -} - static void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) { struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, ix); @@ -968,33 +780,6 @@ void mlx5e_free_icosq_descs(struct mlx5e_icosq *sq) sq->cc = sqcc; } -void mlx5e_shampo_fill_umr(struct mlx5e_rq *rq, int len) -{ - struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; - int end, from, full_len = len; - - end = shampo->hd_per_wq; - from = shampo->ci; - if (from + len > end) { - len -= end - from; - bitmap_set(shampo->bitmap, from, end - from); - from = 0; - } - - bitmap_set(shampo->bitmap, from, len); - shampo->ci = (shampo->ci + full_len) & (shampo->hd_per_wq - 1); -} - -static void mlx5e_handle_shampo_hd_umr(struct mlx5e_shampo_umr umr, - struct mlx5e_icosq *sq) -{ - struct mlx5e_channel *c = container_of(sq, struct mlx5e_channel, icosq); - /* assume 1:1 relationship between RQ and icosq */ - struct mlx5e_rq *rq = &c->rq; - - mlx5e_shampo_fill_umr(rq, umr.len); -} - int mlx5e_poll_ico_cq(struct mlx5e_cq *cq) { struct mlx5e_icosq *sq = container_of(cq, struct mlx5e_icosq, cq); @@ -1055,9 +840,6 @@ int mlx5e_poll_ico_cq(struct mlx5e_cq *cq) break; case MLX5E_ICOSQ_WQE_NOP: break; - case MLX5E_ICOSQ_WQE_SHAMPO_HD_UMR: - mlx5e_handle_shampo_hd_umr(wi->shampo, sq); - break; #ifdef CONFIG_MLX5_EN_TLS case MLX5E_ICOSQ_WQE_UMR_TLS: break; @@ -1245,15 +1027,6 @@ static unsigned int mlx5e_lro_update_hdr(struct sk_buff *skb, return (unsigned int)((unsigned char *)tcp + tcp->doff * 4 - skb->data); } -static void *mlx5e_shampo_get_packet_hd(struct mlx5e_rq *rq, u16 header_index) -{ - struct mlx5e_frag_page *frag_page = mlx5e_shampo_hd_to_frag_page(rq, header_index); - u16 head_offset = mlx5e_shampo_hd_offset(rq, header_index); - void *addr = netmem_address(frag_page->netmem); - - return addr + head_offset + rq->buff.headroom; -} - static void mlx5e_shampo_update_ipv4_udp_hdr(struct mlx5e_rq *rq, struct iphdr *ipv4) { int udp_off = rq->hw_gro_data->fk.control.thoff; @@ -1292,15 +1065,41 @@ static void mlx5e_shampo_update_ipv6_udp_hdr(struct mlx5e_rq *rq, struct ipv6hdr skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_L4; } +static inline u32 mlx5e_shampo_get_header_offset(int header_index) +{ + return (header_index & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1)) * + BIT(MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE); +} + +static void *mlx5e_shampo_get_hdr(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, + int len) +{ + struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; + u32 head_offset, header_index, di_index; + struct mlx5e_dma_info *di; + + header_index = mlx5e_shampo_get_cqe_header_index(rq, cqe); + head_offset = mlx5e_shampo_get_header_offset(header_index); + di_index = header_index >> MLX5E_SHAMPO_LOG_WQ_HEADER_PER_PAGE; + di = &shampo->hd_buf_pages[di_index]; + + dma_sync_single_range_for_cpu(rq->pdev, di->addr, head_offset, + len, rq->buff.map_dir); + + return page_address(di->page) + head_offset; +} + static void mlx5e_shampo_update_fin_psh_flags(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, struct tcphdr *skb_tcp_hd) { - u16 header_index = mlx5e_shampo_get_cqe_header_index(rq, cqe); + int nhoff = ETH_HLEN + rq->hw_gro_data->fk.control.thoff; + int len = nhoff + sizeof(struct tcphdr); struct tcphdr *last_tcp_hd; void *last_hd_addr; - last_hd_addr = mlx5e_shampo_get_packet_hd(rq, header_index); - last_tcp_hd = last_hd_addr + ETH_HLEN + rq->hw_gro_data->fk.control.thoff; + last_hd_addr = mlx5e_shampo_get_hdr(rq, cqe, len); + last_tcp_hd = (struct tcphdr *)(last_hd_addr + nhoff); + tcp_flag_word(skb_tcp_hd) |= tcp_flag_word(last_tcp_hd) & (TCP_FLAG_FIN | TCP_FLAG_PSH); } @@ -2299,52 +2098,29 @@ static struct sk_buff * mlx5e_skb_from_cqe_shampo(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, struct mlx5_cqe64 *cqe, u16 header_index) { - struct mlx5e_frag_page *frag_page = mlx5e_shampo_hd_to_frag_page(rq, header_index); - u16 head_offset = mlx5e_shampo_hd_offset(rq, header_index); struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; u16 head_size = cqe->shampo.header_size; - u16 rx_headroom = rq->buff.headroom; - struct sk_buff *skb = NULL; - dma_addr_t page_dma_addr; - dma_addr_t dma_addr; - void *hdr, *data; - u32 frag_size; - - page_dma_addr = page_pool_get_dma_addr_netmem(frag_page->netmem); - dma_addr = page_dma_addr + head_offset; + struct mlx5e_dma_info *di; + u32 head_offset, di_index; + struct sk_buff *skb; + int len; - hdr = netmem_address(frag_page->netmem) + head_offset; - data = hdr + rx_headroom; - frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + head_size); + len = ALIGN(head_size, sizeof(long)); + skb = napi_alloc_skb(rq->cq.napi, len); + if (unlikely(!skb)) { + rq->stats->buff_alloc_err++; + return NULL; + } - if (likely(frag_size <= BIT(shampo->log_hd_entry_size))) { - /* build SKB around header */ - dma_sync_single_range_for_cpu(rq->pdev, dma_addr, 0, frag_size, rq->buff.map_dir); - net_prefetchw(hdr); - net_prefetch(data); - skb = mlx5e_build_linear_skb(rq, hdr, frag_size, rx_headroom, head_size, 0); - if (unlikely(!skb)) - return NULL; + net_prefetchw(skb->data); - frag_page->frags++; - } else { - /* allocate SKB and copy header for large header */ - rq->stats->gro_large_hds++; - skb = napi_alloc_skb(rq->cq.napi, - ALIGN(head_size, sizeof(long))); - if (unlikely(!skb)) { - rq->stats->buff_alloc_err++; - return NULL; - } + head_offset = mlx5e_shampo_get_header_offset(header_index); + di_index = header_index >> MLX5E_SHAMPO_LOG_WQ_HEADER_PER_PAGE; + di = &shampo->hd_buf_pages[di_index]; - net_prefetchw(skb->data); - mlx5e_copy_skb_header(rq, skb, frag_page->netmem, dma_addr, - head_offset + rx_headroom, - rx_headroom, head_size); - /* skb linear part was allocated with headlen and aligned to long */ - skb->tail += head_size; - skb->len += head_size; - } + mlx5e_copy_skb_header(rq, skb, page_to_netmem(di->page), di->addr, + head_offset, head_offset, len); + __skb_put(skb, head_size); /* queue up for recycling/reuse */ skb_mark_for_recycle(skb); @@ -2445,7 +2221,7 @@ static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cq * prevent the kernel from touching it. */ if (unlikely(netmem_is_net_iov(frag_page->netmem))) - goto free_hd_entry; + goto mpwrq_cqe_out; *skb = mlx5e_skb_from_cqe_mpwrq_nonlinear(rq, wi, cqe, cqe_bcnt, data_offset, @@ -2453,19 +2229,22 @@ static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cq } if (unlikely(!*skb)) - goto free_hd_entry; + goto mpwrq_cqe_out; NAPI_GRO_CB(*skb)->count = 1; skb_shinfo(*skb)->gso_size = cqe_bcnt - head_size; } else { NAPI_GRO_CB(*skb)->count++; + if (NAPI_GRO_CB(*skb)->count == 2 && rq->hw_gro_data->fk.basic.n_proto == htons(ETH_P_IP)) { - void *hd_addr = mlx5e_shampo_get_packet_hd(rq, header_index); - int nhoff = ETH_HLEN + rq->hw_gro_data->fk.control.thoff - - sizeof(struct iphdr); - struct iphdr *iph = (struct iphdr *)(hd_addr + nhoff); + int len = ETH_HLEN + rq->hw_gro_data->fk.control.thoff; + int nhoff = len - sizeof(struct iphdr); + void *last_hd_addr; + struct iphdr *iph; + last_hd_addr = mlx5e_shampo_get_hdr(rq, cqe, len); + iph = (struct iphdr *)(last_hd_addr + nhoff); rq->hw_gro_data->second_ip_id = ntohs(iph->id); } } @@ -2487,13 +2266,10 @@ static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cq if (mlx5e_shampo_complete_rx_cqe(rq, cqe, cqe_bcnt, *skb)) { *skb = NULL; - goto free_hd_entry; + goto mpwrq_cqe_out; } if (flush && rq->hw_gro_data->skb) mlx5e_shampo_flush_skb(rq, cqe, match); -free_hd_entry: - if (likely(head_size)) - mlx5e_free_rx_shampo_hd_entry(rq, header_index); mpwrq_cqe_out: if (likely(wi->consumed_strides < rq->mpwqe.num_strides)) return; -- 2.31.1