Move the XDP tail-shrink logic out of bpf_xdp_adjust_tail() into net/core/xdp.c so it can be shared with drivers. The frag helpers (bpf_xdp_shrink_data_zc(), bpf_xdp_shrink_data() and bpf_xdp_frags_shrink_tail()) move along with it and are renamed to xdp_buff_shrink_data_zc(), xdp_buff_shrink_data() and xdp_frags_shrink_tail(). Add xdp_buff_shrink_tail(), which handles the linear case too and is exported with EXPORT_SYMBOL_GPL() for driver use. Unlike bpf_xdp_adjust_tail(), it takes the number of bytes to remove as a positive value; bpf_xdp_adjust_tail() keeps its signed offset ABI and passes -offset for the shrink case. Negative offsets are rejected. bpf_xdp_pull_data() keeps using the frag helper under its new name. Behavior for valid offsets is unchanged. This lets stmmac use the common helper to strip the Ethernet FCS from the xdp_buff it accumulates for a multi-descriptor frame. Signed-off-by: Lorenzo Bianconi --- include/net/xdp.h | 3 ++ net/core/filter.c | 80 +++---------------------------------------------- net/core/xdp.c | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 76 deletions(-) diff --git a/include/net/xdp.h b/include/net/xdp.h index 054748b37d7d..3400775d179b 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -415,6 +415,9 @@ struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf, struct sk_buff *xdp_build_skb_from_frame(struct xdp_frame *xdpf, struct net_device *dev); struct xdp_frame *xdpf_clone(struct xdp_frame *xdpf); +int xdp_buff_shrink_tail(struct xdp_buff *xdp, int offset); +bool xdp_buff_shrink_data(struct xdp_buff *xdp, skb_frag_t *frag, + int shrink, bool tail); static inline void xdp_convert_frame_to_buff(const struct xdp_frame *frame, diff --git a/net/core/filter.c b/net/core/filter.c index f42ecfb5841f..cdbb54a10b0b 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -4438,88 +4438,16 @@ static int bpf_xdp_frags_increase_tail(struct xdp_buff *xdp, int offset) return 0; } -static struct xdp_buff *bpf_xdp_shrink_data_zc(struct xdp_buff *xdp, int shrink, - bool tail, bool release) -{ - struct xdp_buff *zc_frag = tail ? xsk_buff_get_tail(xdp) : - xsk_buff_get_head(xdp); - - if (release) { - xsk_buff_del_frag(zc_frag); - } else { - if (tail) - zc_frag->data_end -= shrink; - else - zc_frag->data += shrink; - } - - return zc_frag; -} - -static bool bpf_xdp_shrink_data(struct xdp_buff *xdp, skb_frag_t *frag, - int shrink, bool tail) -{ - enum xdp_mem_type mem_type = xdp->rxq->mem.type; - bool release = skb_frag_size(frag) == shrink; - netmem_ref netmem = skb_frag_netmem(frag); - struct xdp_buff *zc_frag = NULL; - - if (mem_type == MEM_TYPE_XSK_BUFF_POOL) { - netmem = 0; - zc_frag = bpf_xdp_shrink_data_zc(xdp, shrink, tail, release); - } - - if (release) { - __xdp_return(netmem, mem_type, false, zc_frag); - } else { - if (!tail) - skb_frag_off_add(frag, shrink); - skb_frag_size_sub(frag, shrink); - } - - return release; -} - -static int bpf_xdp_frags_shrink_tail(struct xdp_buff *xdp, int offset) -{ - struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); - int i, n_frags_free = 0, len_free = 0; - - if (unlikely(offset > (int)xdp_get_buff_len(xdp) - ETH_HLEN)) - return -EINVAL; - - for (i = sinfo->nr_frags - 1; i >= 0 && offset > 0; i--) { - skb_frag_t *frag = &sinfo->frags[i]; - int shrink = min_t(int, offset, skb_frag_size(frag)); - - len_free += shrink; - offset -= shrink; - if (bpf_xdp_shrink_data(xdp, frag, shrink, true)) - n_frags_free++; - } - sinfo->nr_frags -= n_frags_free; - sinfo->xdp_frags_size -= len_free; - - if (unlikely(!sinfo->nr_frags)) { - xdp_buff_clear_frags_flag(xdp); - xdp_buff_clear_frag_pfmemalloc(xdp); - xdp->data_end -= offset; - } - - return 0; -} - BPF_CALL_2(bpf_xdp_adjust_tail, struct xdp_buff *, xdp, int, offset) { void *data_hard_end = xdp_data_hard_end(xdp); /* use xdp->frame_sz */ void *data_end = xdp->data_end + offset; - if (unlikely(xdp_buff_has_frags(xdp))) { /* non-linear xdp buff */ - if (offset < 0) - return bpf_xdp_frags_shrink_tail(xdp, -offset); + if (offset < 0) + return xdp_buff_shrink_tail(xdp, -offset); + if (unlikely(xdp_buff_has_frags(xdp))) /* non-linear xdp buff */ return bpf_xdp_frags_increase_tail(xdp, offset); - } /* Notice that xdp_data_hard_end have reserved some tailroom */ if (unlikely(data_end > data_hard_end)) @@ -12840,7 +12768,7 @@ __bpf_kfunc int bpf_xdp_pull_data(struct xdp_md *x, u32 len) xdp->data_end += shrink; sinfo->xdp_frags_size -= shrink; delta -= shrink; - if (bpf_xdp_shrink_data(xdp, frag, shrink, false)) + if (xdp_buff_shrink_data(xdp, frag, shrink, false)) n_frags_free++; } diff --git a/net/core/xdp.c b/net/core/xdp.c index 0db2f68afc5c..acad82982d77 100644 --- a/net/core/xdp.c +++ b/net/core/xdp.c @@ -676,6 +676,95 @@ struct sk_buff *__xdp_build_skb_from_buff(const struct xdp_buff *xdp) } EXPORT_SYMBOL_GPL(__xdp_build_skb_from_buff); +static struct xdp_buff *xdp_buff_shrink_data_zc(struct xdp_buff *xdp, + int shrink, bool tail, + bool release) +{ + struct xdp_buff *zc_frag = tail ? xsk_buff_get_tail(xdp) : + xsk_buff_get_head(xdp); + + if (release) { + xsk_buff_del_frag(zc_frag); + } else { + if (tail) + zc_frag->data_end -= shrink; + else + zc_frag->data += shrink; + } + + return zc_frag; +} + +bool xdp_buff_shrink_data(struct xdp_buff *xdp, skb_frag_t *frag, + int shrink, bool tail) +{ + enum xdp_mem_type mem_type = xdp->rxq->mem.type; + bool release = skb_frag_size(frag) == shrink; + netmem_ref netmem = skb_frag_netmem(frag); + struct xdp_buff *zc_frag = NULL; + + if (mem_type == MEM_TYPE_XSK_BUFF_POOL) { + netmem = 0; + zc_frag = xdp_buff_shrink_data_zc(xdp, shrink, tail, release); + } + + if (release) { + __xdp_return(netmem, mem_type, false, zc_frag); + } else { + if (!tail) + skb_frag_off_add(frag, shrink); + skb_frag_size_sub(frag, shrink); + } + + return release; +} + +static int xdp_frags_shrink_tail(struct xdp_buff *xdp, int offset) +{ + struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); + int i, n_frags_free = 0, len_free = 0; + + if (unlikely(offset > (int)xdp_get_buff_len(xdp) - ETH_HLEN)) + return -EINVAL; + + for (i = sinfo->nr_frags - 1; i >= 0 && offset > 0; i--) { + skb_frag_t *frag = &sinfo->frags[i]; + int shrink = min_t(int, offset, skb_frag_size(frag)); + + len_free += shrink; + offset -= shrink; + if (xdp_buff_shrink_data(xdp, frag, shrink, true)) + n_frags_free++; + } + sinfo->nr_frags -= n_frags_free; + sinfo->xdp_frags_size -= len_free; + + if (unlikely(!sinfo->nr_frags)) { + xdp_buff_clear_frags_flag(xdp); + xdp_buff_clear_frag_pfmemalloc(xdp); + xdp->data_end -= offset; + } + + return 0; +} + +int xdp_buff_shrink_tail(struct xdp_buff *xdp, int offset) +{ + if (unlikely(offset < 0)) + return -EINVAL; + + if (unlikely(xdp_buff_has_frags(xdp))) /* non-linear xdp buff */ + return xdp_frags_shrink_tail(xdp, offset); + + if (unlikely(offset > xdp->data_end - xdp->data - ETH_HLEN)) + return -EINVAL; + + xdp->data_end = xdp->data_end - offset; + + return 0; +} +EXPORT_SYMBOL_GPL(xdp_buff_shrink_tail); + /** * xdp_copy_frags_from_zc - copy frags from XSk buff to skb * @skb: skb to copy frags to -- 2.56.1