Currently, generic XDP hook uses xdp_rxq_info from netstack Rx queues which do not have its XDP memory model registered. There is a case when XDP program calls bpf_xdp_adjust_tail() BPF helper, which in turn releases underlying memory. This happens when it consumes enough amount of bytes and when XDP buffer has fragments. For this action the memory model knowledge passed to XDP program is crucial so that core can call suitable function for freeing/recycling the page. For netstack queues it defaults to MEM_TYPE_PAGE_SHARED (0) due to lack of mem model registration. The problem we're fixing here is when kernel copied the skb to new buffer backed by system's page_pool and XDP buffer is built around it. Then when bpf_xdp_adjust_tail() calls __xdp_return(), it acts incorrectly due to mem type not being set to MEM_TYPE_PAGE_POOL and causes a page leak. Pull out the existing code from bpf_prog_run_generic_xdp() that init/prepares xdp_buff onto new helper xdp_convert_skb_to_buff() and embed there rxq's mem_type initialization that is assigned to xdp_buff. Make it agnostic to current skb->data position. This problem was triggered by syzbot as well as AF_XDP test suite which is about to be integrated to BPF CI. Reported-by: syzbot+ff145014d6b0ce64a173@syzkaller.appspotmail.com Closes: https://lore.kernel.org/netdev/6756c37b.050a0220.a30f1.019a.GAE@google.com/ Fixes: e6d5dbdd20aa ("xdp: add multi-buff support for xdp running in generic mode") Tested-by: Ihor Solodrai Reviewed-by: Toke Høiland-Jørgensen Co-developed-by: Octavian Purdila Signed-off-by: Octavian Purdila # whole analysis, testing, initiating a fix Signed-off-by: Maciej Fijalkowski # commit msg and proposed more robust fix --- include/net/xdp.h | 27 +++++++++++++++++++++++++++ net/core/dev.c | 25 ++++--------------------- 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/include/net/xdp.h b/include/net/xdp.h index aa742f413c35..cec43f56ae9a 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -384,6 +384,33 @@ struct sk_buff *xdp_build_skb_from_frame(struct xdp_frame *xdpf, struct net_device *dev); struct xdp_frame *xdpf_clone(struct xdp_frame *xdpf); +static inline +void xdp_convert_skb_to_buff(struct sk_buff *skb, struct xdp_buff *xdp, + struct xdp_rxq_info *xdp_rxq) +{ + u32 frame_sz, pkt_len; + + /* SKB "head" area always have tailroom for skb_shared_info */ + frame_sz = skb_end_pointer(skb) - skb->head; + frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); + + DEBUG_NET_WARN_ON_ONCE(!skb_mac_header_was_set(skb)); + pkt_len = skb->tail - skb->mac_header; + + xdp_init_buff(xdp, frame_sz, xdp_rxq); + xdp_prepare_buff(xdp, skb->head, skb->mac_header, pkt_len, true); + + if (skb_is_nonlinear(skb)) { + skb_shinfo(skb)->xdp_frags_size = skb->data_len; + xdp_buff_set_frags_flag(xdp); + } else { + xdp_buff_clear_frags_flag(xdp); + } + + xdp->rxq->mem.type = page_pool_page_is_pp(virt_to_head_page(xdp->data)) ? + MEM_TYPE_PAGE_POOL : MEM_TYPE_PAGE_SHARED; +} + static inline void xdp_convert_frame_to_buff(const struct xdp_frame *frame, struct xdp_buff *xdp) diff --git a/net/core/dev.c b/net/core/dev.c index a64cef2c537e..3d607dce292b 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -5320,35 +5320,18 @@ static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb) u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp, const struct bpf_prog *xdp_prog) { - void *orig_data, *orig_data_end, *hard_start; + void *orig_data, *orig_data_end; struct netdev_rx_queue *rxqueue; bool orig_bcast, orig_host; - u32 mac_len, frame_sz; + u32 metalen, act, mac_len; __be16 orig_eth_type; struct ethhdr *eth; - u32 metalen, act; int off; - /* The XDP program wants to see the packet starting at the MAC - * header. - */ + rxqueue = netif_get_rxqueue(skb); mac_len = skb->data - skb_mac_header(skb); - hard_start = skb->data - skb_headroom(skb); - - /* SKB "head" area always have tailroom for skb_shared_info */ - frame_sz = (void *)skb_end_pointer(skb) - hard_start; - frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); - rxqueue = netif_get_rxqueue(skb); - xdp_init_buff(xdp, frame_sz, &rxqueue->xdp_rxq); - xdp_prepare_buff(xdp, hard_start, skb_headroom(skb) - mac_len, - skb_headlen(skb) + mac_len, true); - if (skb_is_nonlinear(skb)) { - skb_shinfo(skb)->xdp_frags_size = skb->data_len; - xdp_buff_set_frags_flag(xdp); - } else { - xdp_buff_clear_frags_flag(xdp); - } + xdp_convert_skb_to_buff(skb, xdp, &rxqueue->xdp_rxq); orig_data_end = xdp->data_end; orig_data = xdp->data; -- 2.43.0