From: Jason Xing Three targeted optimizations for the batch copy-mode TX hot path: Replace skb_store_bits() with memcpy() for single-buffer first-desc path. After skb_reserve() + skb_put(), the SKB is freshly allocated with all data in the linear area and no frags, so skb_store_bits() degenerates to memcpy(skb->data, buffer, len) but carries unnecessary function call overhead, offset validation, and frag iteration logic. Inline UMEM address computation in Phase 3 and pass the pre-computed buffer pointer to xsk_build_skb(), avoiding the per-packet non-inlined xp_raw_get_data() (EXPORT_SYMBOL) call chain: xsk_buff_raw_get_data -> xp_raw_get_data -> __xp_raw_get_addr + __xp_raw_get_data. In the batch loop the pool->addrs and pool->unaligned are invariant, so we cache them once and compute each buffer address inline. Prefetch the *next* descriptor's UMEM data buffer at the top of the Phase 3 loop, hiding the memory latency of the upcoming memcpy. It improves 3-4% performance stably. Signed-off-by: Jason Xing --- include/net/xdp_sock.h | 3 ++- net/core/skbuff.c | 18 ++++++++++++++++-- net/xdp/xsk.c | 15 ++++++--------- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h index 0609e3b04279..5e05236c7fba 100644 --- a/include/net/xdp_sock.h +++ b/include/net/xdp_sock.h @@ -139,7 +139,8 @@ void __xsk_map_flush(struct list_head *flush_list); INDIRECT_CALLABLE_DECLARE(void xsk_destruct_skb(struct sk_buff *)); struct sk_buff *xsk_build_skb(struct xdp_sock *xs, struct sk_buff *allocated_skb, - struct xdp_desc *desc); + struct xdp_desc *desc, + void *buffer); int xsk_alloc_batch_skb(struct xdp_sock *xs, u32 nb_pkts, u32 nb_descs, int *err); int xsk_direct_xmit_batch(struct xdp_sock *xs, struct net_device *dev); diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 5726b1566b2b..bef5270e6332 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -752,14 +752,28 @@ int xsk_alloc_batch_skb(struct xdp_sock *xs, u32 nb_pkts, u32 nb_descs, int *err if (total_truesize) refcount_add(total_truesize, &xs->sk.sk_wmem_alloc); - /* Phase 3: Build SKBs with packet data */ + /* Phase 3: Build SKBs with packet data. */ + struct xsk_buff_pool *pool = xs->pool; + void *pool_addrs = pool->addrs; + bool unaligned = pool->unaligned; + for (j = 0; j < alloc_descs; j++) { + u64 addr = descs[j].addr; + void *buffer; + + if (unaligned) + addr = xp_unaligned_add_offset_to_addr(addr); + buffer = pool_addrs + addr; + + if (j + 1 < alloc_descs) + prefetch(pool_addrs + descs[j + 1].addr); + if (!xs->skb) { skb = skbs[skb_count - 1 - k]; k++; } - skb = xsk_build_skb(xs, skb, &descs[j]); + skb = xsk_build_skb(xs, skb, &descs[j], buffer); if (IS_ERR(skb)) { *err = PTR_ERR(skb); break; diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index be341290e42c..3bf81b838075 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -811,7 +811,8 @@ static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs, struct sk_buff *xsk_build_skb(struct xdp_sock *xs, struct sk_buff *allocated_skb, - struct xdp_desc *desc) + struct xdp_desc *desc, + void *buffer) { struct net_device *dev = xs->dev; struct sk_buff *skb = xs->skb; @@ -825,11 +826,10 @@ struct sk_buff *xsk_build_skb(struct xdp_sock *xs, goto free_err; } } else { - u32 hr, tr, len; - void *buffer; + u32 hr, tr, len = desc->len; - buffer = xsk_buff_raw_get_data(xs->pool, desc->addr); - len = desc->len; + if (!buffer) + buffer = xsk_buff_raw_get_data(xs->pool, desc->addr); if (!skb) { hr = max(NET_SKB_PAD, L1_CACHE_ALIGN(dev->needed_headroom)); @@ -844,10 +844,7 @@ struct sk_buff *xsk_build_skb(struct xdp_sock *xs, skb_reserve(skb, hr); skb_put(skb, len); - - err = skb_store_bits(skb, 0, buffer, len); - if (unlikely(err)) - goto free_err; + memcpy(skb->data, buffer, len); xsk_skb_init_misc(skb, xs, desc->addr); if (desc->options & XDP_TX_METADATA) { -- 2.41.3