We are unnecessarily setting a bunch of skb fields per each processed descriptor, which is redundant for fragmented frames. Let us set these respective members for first fragment only. Signed-off-by: Maciej Fijalkowski --- net/xdp/xsk.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index 72e34bd2d925..72194f0a3fc0 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -758,6 +758,10 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, goto free_err; xsk_set_destructor_arg(skb, desc->addr); + skb->dev = dev; + skb->priority = READ_ONCE(xs->sk.sk_priority); + skb->mark = READ_ONCE(xs->sk.sk_mark); + skb->destructor = xsk_destruct_skb; } else { int nr_frags = skb_shinfo(skb)->nr_frags; struct xsk_addr_node *xsk_addr; @@ -826,14 +830,10 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, if (meta->flags & XDP_TXMD_FLAGS_LAUNCH_TIME) skb->skb_mstamp_ns = meta->request.launch_time; + xsk_tx_metadata_to_compl(meta, &skb_shinfo(skb)->xsk_meta); } } - skb->dev = dev; - skb->priority = READ_ONCE(xs->sk.sk_priority); - skb->mark = READ_ONCE(xs->sk.sk_mark); - skb->destructor = xsk_destruct_skb; - xsk_tx_metadata_to_compl(meta, &skb_shinfo(skb)->xsk_meta); xsk_inc_num_desc(skb); return skb; -- 2.43.0 Devices that set IFF_TX_SKB_NO_LINEAR will not execute branch that handles metadata, as we set @first_frag only for !IFF_TX_SKB_NO_LINEAR code in xsk_build_skb(). Same functionality can be achieved with checking if xsk_get_num_desc() returns 0. To replace current usage of @first_frag with XSKCB(skb)->num_descs check, pull out the code from xsk_set_destructor_arg() that initializes sk_buff::cb and call it before skb_store_bits() in branch that creates skb against first processed frag. This so error path has the XSKCB(skb)->num_descs initialized and can free skb in case skb_store_bits() failed. Signed-off-by: Maciej Fijalkowski --- net/xdp/xsk.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index 72194f0a3fc0..064238400036 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -605,6 +605,13 @@ static u32 xsk_get_num_desc(struct sk_buff *skb) return XSKCB(skb)->num_descs; } +static void xsk_init_cb(struct sk_buff *skb) +{ + BUILD_BUG_ON(sizeof(struct xsk_addr_head) > sizeof(skb->cb)); + INIT_LIST_HEAD(&XSKCB(skb)->addrs_list); + XSKCB(skb)->num_descs = 0; +} + static void xsk_destruct_skb(struct sk_buff *skb) { struct xsk_tx_metadata_compl *compl = &skb_shinfo(skb)->xsk_meta; @@ -620,9 +627,6 @@ static void xsk_destruct_skb(struct sk_buff *skb) static void xsk_set_destructor_arg(struct sk_buff *skb, u64 addr) { - BUILD_BUG_ON(sizeof(struct xsk_addr_head) > sizeof(skb->cb)); - INIT_LIST_HEAD(&XSKCB(skb)->addrs_list); - XSKCB(skb)->num_descs = 0; skb_shinfo(skb)->destructor_arg = (void *)(uintptr_t)addr; } @@ -672,7 +676,7 @@ static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs, return ERR_PTR(err); skb_reserve(skb, hr); - + xsk_init_cb(skb); xsk_set_destructor_arg(skb, desc->addr); } else { xsk_addr = kmem_cache_zalloc(xsk_tx_generic_cache, GFP_KERNEL); @@ -725,7 +729,6 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, struct xsk_tx_metadata *meta = NULL; struct net_device *dev = xs->dev; struct sk_buff *skb = xs->skb; - bool first_frag = false; int err; if (dev->priv_flags & IFF_TX_SKB_NO_LINEAR) { @@ -742,8 +745,6 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, len = desc->len; if (!skb) { - first_frag = true; - hr = max(NET_SKB_PAD, L1_CACHE_ALIGN(dev->needed_headroom)); tr = dev->needed_tailroom; skb = sock_alloc_send_skb(&xs->sk, hr + len + tr, 1, &err); @@ -752,6 +753,7 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, skb_reserve(skb, hr); skb_put(skb, len); + xsk_init_cb(skb); err = skb_store_bits(skb, 0, buffer, len); if (unlikely(err)) @@ -797,7 +799,7 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, list_add_tail(&xsk_addr->addr_node, &XSKCB(skb)->addrs_list); } - if (first_frag && desc->options & XDP_TX_METADATA) { + if (!xsk_get_num_desc(skb) && desc->options & XDP_TX_METADATA) { if (unlikely(xs->pool->tx_metadata_len == 0)) { err = -EINVAL; goto free_err; @@ -839,7 +841,7 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, return skb; free_err: - if (first_frag && skb) + if (skb && !xsk_get_num_desc(skb)) kfree_skb(skb); if (err == -EOVERFLOW) { -- 2.43.0 xsk_build_skb() has gone wild with its size and one of the things we can do about it is to pull out a branch that takes care of metadata handling and make it a separate function. Consider this as a good start of cleanup. No functional changes here. Signed-off-by: Maciej Fijalkowski --- net/xdp/xsk.c | 83 ++++++++++++++++++++++++++++----------------------- 1 file changed, 46 insertions(+), 37 deletions(-) diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index 064238400036..7121d4f99915 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -723,10 +723,48 @@ static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs, return skb; } +static int xsk_skb_metadata(struct sk_buff *skb, void *buffer, + struct xdp_desc *desc, struct xsk_buff_pool *pool, + u32 hr) +{ + struct xsk_tx_metadata *meta = NULL; + + if (unlikely(pool->tx_metadata_len == 0)) + return -EINVAL; + + meta = buffer - pool->tx_metadata_len; + if (unlikely(!xsk_buff_valid_tx_metadata(meta))) + return -EINVAL; + + if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM) { + if (unlikely(meta->request.csum_start + + meta->request.csum_offset + + sizeof(__sum16) > desc->len)) + return -EINVAL; + + skb->csum_start = hr + meta->request.csum_start; + skb->csum_offset = meta->request.csum_offset; + skb->ip_summed = CHECKSUM_PARTIAL; + + if (unlikely(pool->tx_sw_csum)) { + int err; + + err = skb_checksum_help(skb); + if (err) + return err; + } + } + + if (meta->flags & XDP_TXMD_FLAGS_LAUNCH_TIME) + skb->skb_mstamp_ns = meta->request.launch_time; + xsk_tx_metadata_to_compl(meta, &skb_shinfo(skb)->xsk_meta); + + return 0; +} + static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, struct xdp_desc *desc) { - struct xsk_tx_metadata *meta = NULL; struct net_device *dev = xs->dev; struct sk_buff *skb = xs->skb; int err; @@ -764,6 +802,13 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, skb->priority = READ_ONCE(xs->sk.sk_priority); skb->mark = READ_ONCE(xs->sk.sk_mark); skb->destructor = xsk_destruct_skb; + + if (desc->options & XDP_TX_METADATA) { + err = xsk_skb_metadata(skb, buffer, desc, + xs->pool, hr); + if (unlikely(err)) + goto free_err; + } } else { int nr_frags = skb_shinfo(skb)->nr_frags; struct xsk_addr_node *xsk_addr; @@ -798,42 +843,6 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, xsk_addr->addr = desc->addr; list_add_tail(&xsk_addr->addr_node, &XSKCB(skb)->addrs_list); } - - if (!xsk_get_num_desc(skb) && desc->options & XDP_TX_METADATA) { - if (unlikely(xs->pool->tx_metadata_len == 0)) { - err = -EINVAL; - goto free_err; - } - - meta = buffer - xs->pool->tx_metadata_len; - if (unlikely(!xsk_buff_valid_tx_metadata(meta))) { - err = -EINVAL; - goto free_err; - } - - if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM) { - if (unlikely(meta->request.csum_start + - meta->request.csum_offset + - sizeof(__sum16) > len)) { - err = -EINVAL; - goto free_err; - } - - skb->csum_start = hr + meta->request.csum_start; - skb->csum_offset = meta->request.csum_offset; - skb->ip_summed = CHECKSUM_PARTIAL; - - if (unlikely(xs->pool->tx_sw_csum)) { - err = skb_checksum_help(skb); - if (err) - goto free_err; - } - } - - if (meta->flags & XDP_TXMD_FLAGS_LAUNCH_TIME) - skb->skb_mstamp_ns = meta->request.launch_time; - xsk_tx_metadata_to_compl(meta, &skb_shinfo(skb)->xsk_meta); - } } xsk_inc_num_desc(skb); -- 2.43.0