From: Christoph Paasch Doing the call to dma_sync_single_for_cpu() earlier will allow us to adjust headlen based on the actual size of the protocol headers. Doing this earlier means that we don't need to call mlx5e_copy_skb_header() anymore and rather can call skb_copy_to_linear_data() directly. Reviewed-by: Eric Dumazet Reviewed-by: Saeed Mahameed Signed-off-by: Christoph Paasch Signed-off-by: Dragos Tatulea Signed-off-by: Tariq Toukan --- .../net/ethernet/mellanox/mlx5/core/en_rx.c | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index 5b60aa47c75b..75ccf40a7f17 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -1923,11 +1923,11 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w unsigned int truesize = 0; u32 pg_consumed_bytes; struct bpf_prog *prog; + void *va, *head_addr; struct sk_buff *skb; u32 linear_frame_sz; u16 linear_data_len; u16 linear_hr; - void *va; if (unlikely(cqe_bcnt > rq->hw_mtu)) { u8 lro_num_seg = get_cqe_lro_num_seg(cqe); @@ -1940,9 +1940,11 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w prog = rcu_dereference(rq->xdp_prog); + head_addr = netmem_address(head_page->netmem) + head_offset; + if (prog) { /* area for bpf_xdp_[store|load]_bytes */ - net_prefetchw(netmem_address(frag_page->netmem) + frag_offset); + net_prefetchw(head_addr); va = mlx5e_mpwqe_get_linear_page_frag(rq); if (!va) { @@ -1956,6 +1958,8 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w linear_frame_sz = MLX5_SKB_FRAG_SZ(linear_hr + MLX5E_RX_MAX_HEAD); linear_page = &rq->mpwqe.linear_info->frag_page; } else { + dma_addr_t addr; + skb = napi_alloc_skb(rq->cq.napi, ALIGN(MLX5E_RX_MAX_HEAD, sizeof(long))); if (unlikely(!skb)) { @@ -1967,6 +1971,11 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w net_prefetchw(va); /* xdp_frame data area */ net_prefetchw(skb->data); + addr = page_pool_get_dma_addr_netmem(head_page->netmem); + dma_sync_single_for_cpu(rq->pdev, addr + head_offset, + ALIGN(headlen, sizeof(long)), + rq->buff.map_dir); + frag_offset += headlen; byte_cnt -= headlen; linear_hr = skb_headroom(skb); @@ -2056,8 +2065,6 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w __pskb_pull_tail(skb, headlen); } } else { - dma_addr_t addr; - if (xdp_buff_has_frags(&mxbuf->xdp)) { struct mlx5e_frag_page *pagep; @@ -2071,10 +2078,11 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w pagep->frags++; while (++pagep < frag_page); } + /* copy header */ - addr = page_pool_get_dma_addr_netmem(head_page->netmem); - mlx5e_copy_skb_header(rq, skb, head_page->netmem, addr, - head_offset, head_offset, headlen); + skb_copy_to_linear_data(skb, head_addr, + ALIGN(headlen, sizeof(long))); + /* skb linear part was allocated with headlen and aligned to long */ skb->tail += headlen; skb->len += headlen; -- 2.44.0 From: Christoph Paasch mlx5e_skb_from_cqe_mpwrq_nonlinear() copies MLX5E_RX_MAX_HEAD (256) bytes from the page-pool to the skb's linear part. Those 256 bytes include part of the payload. When attempting to do GRO in skb_gro_receive, if headlen > data_offset (and skb->head_frag is not set), we end up aggregating packets in the frag_list. This is of course not good when we are CPU-limited. Also causes a worse skb->len/truesize ratio,... So, let's avoid copying parts of the payload to the linear part. We use eth_get_headlen() to parse the headers and compute the length of the protocol headers, which will be used to copy the relevant bits of the skb's linear part. We still allocate MLX5E_RX_MAX_HEAD for the skb so that if the networking stack needs to call pskb_may_pull() later on, we don't need to reallocate memory. This gives a nice throughput increase (ARM Neoverse-V2 with CX-7 NIC and LRO enabled): BEFORE: ======= (netserver pinned to core receiving interrupts) $ netperf -H 10.221.81.118 -T 80,9 -P 0 -l 60 -- -m 256K -M 256K 87380 16384 262144 60.01 32547.82 (netserver pinned to adjacent core receiving interrupts) $ netperf -H 10.221.81.118 -T 80,10 -P 0 -l 60 -- -m 256K -M 256K 87380 16384 262144 60.00 52531.67 AFTER: ====== (netserver pinned to core receiving interrupts) $ netperf -H 10.221.81.118 -T 80,9 -P 0 -l 60 -- -m 256K -M 256K 87380 16384 262144 60.00 52896.06 (netserver pinned to adjacent core receiving interrupts) $ netperf -H 10.221.81.118 -T 80,10 -P 0 -l 60 -- -m 256K -M 256K 87380 16384 262144 60.00 85094.90 Additional tests across a larger range of parameters w/ and w/o LRO, w/ and w/o IPv6-encapsulation, different MTUs (1500, 4096, 9000), different TCP read/write-sizes as well as UDP benchmarks, all have shown equal or better performance with this patch. For XDP pull at most ETH_HLEN bytes in the linear area so that XDP_PASS can also benefit from this improvement and keep things simple when dealing with skb geometry changes from the XDP program. Reviewed-by: Eric Dumazet Reviewed-by: Saeed Mahameed Signed-off-by: Christoph Paasch Signed-off-by: Dragos Tatulea Signed-off-by: Tariq Toukan --- drivers/net/ethernet/mellanox/mlx5/core/en_rx.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index 75ccf40a7f17..6fbc0441c4b8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -1912,7 +1912,6 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w u32 page_idx) { struct mlx5e_frag_page *frag_page = &wi->alloc_units.frag_pages[page_idx]; - u16 headlen = min_t(u16, MLX5E_RX_MAX_HEAD, cqe_bcnt); struct mlx5e_frag_page *head_page = frag_page; struct mlx5e_frag_page *linear_page = NULL; struct mlx5e_xdp_buff *mxbuf = &rq->mxbuf; @@ -1928,6 +1927,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w u32 linear_frame_sz; u16 linear_data_len; u16 linear_hr; + u16 headlen; if (unlikely(cqe_bcnt > rq->hw_mtu)) { u8 lro_num_seg = get_cqe_lro_num_seg(cqe); @@ -1971,11 +1971,14 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w net_prefetchw(va); /* xdp_frame data area */ net_prefetchw(skb->data); + headlen = min(MLX5E_RX_MAX_HEAD, cqe_bcnt); addr = page_pool_get_dma_addr_netmem(head_page->netmem); dma_sync_single_for_cpu(rq->pdev, addr + head_offset, ALIGN(headlen, sizeof(long)), rq->buff.map_dir); + headlen = eth_get_headlen(rq->netdev, head_addr, headlen); + frag_offset += headlen; byte_cnt -= headlen; linear_hr = skb_headroom(skb); @@ -2060,9 +2063,9 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w pagep->frags++; while (++pagep < frag_page); - headlen = min_t(u16, MLX5E_RX_MAX_HEAD - len, - skb->data_len); - __pskb_pull_tail(skb, headlen); + if (len < ETH_HLEN) + __pskb_pull_tail(skb, min(ETH_HLEN - len, + skb->data_len)); } } else { if (xdp_buff_has_frags(&mxbuf->xdp)) { -- 2.44.0