skb_frag_unref() and skb_release_data() skip releasing references on managed zerocopy fragments (SKBFL_MANAGED_FRAG_REFS), whereas skb_frag_ref() acquires a reference unconditionally. If skb_frag_ref() is called on a managed zerocopy SKB, the acquired reference is never released by skb_frag_unref() or skb_release_data(). Skip acquiring a reference in skb_frag_ref() when skb_zcopy_managed(skb) is true so skb_frag_ref() matches skb_frag_unref() and skb_release_data(), and disable share_ok in iptfs_output_queued() when skb_zcopy_managed(skb) is true so iptfs_consume_frags() does not append unreferenced fragments into a managed zerocopy outer SKB. Signed-off-by: Mina Almasry --- include/linux/skbuff_ref.h | 10 +++++++--- net/core/net_test.c | 10 ++++++++++ net/xfrm/xfrm_iptfs.c | 3 ++- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/include/linux/skbuff_ref.h b/include/linux/skbuff_ref.h index b6f6da556ee90..e94c599ce83d2 100644 --- a/include/linux/skbuff_ref.h +++ b/include/linux/skbuff_ref.h @@ -36,12 +36,16 @@ static __always_inline void skb_netmem_ref(netmem_ref netmem, bool recycle) * * Acquires a reference on the @f'th fragment of @skb via skb_netmem_ref() * (pp_ref_count if skb->pp_recycle and page_pool-backed, else non-pp - * backing refcount). Counterpart to skb_frag_unref(). + * backing refcount), skipping managed zerocopy fragments. Counterpart to + * skb_frag_unref(). */ static __always_inline void skb_frag_ref(struct sk_buff *skb, int f) { - skb_netmem_ref(skb_frag_netmem(&skb_shinfo(skb)->frags[f]), - skb->pp_recycle); + struct skb_shared_info *shinfo = skb_shinfo(skb); + + if (!skb_zcopy_managed(skb)) + skb_netmem_ref(skb_frag_netmem(&shinfo->frags[f]), + skb->pp_recycle); } bool napi_pp_put_page(netmem_ref netmem); diff --git a/net/core/net_test.c b/net/core/net_test.c index 2b5080c226636..e20059a00003c 100644 --- a/net/core/net_test.c +++ b/net/core/net_test.c @@ -571,6 +571,16 @@ static void netmem_ref_test_page(struct kunit *test) skb_frag_unref(skb, 0); KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref); + /* SKBFL_MANAGED_FRAG_REFS skips both skb_frag_ref and skb_frag_unref */ + skb_shinfo(skb)->flags |= SKBFL_ZEROCOPY_ENABLE | + SKBFL_MANAGED_FRAG_REFS; + skb_frag_ref(skb, 0); + KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref); + skb_frag_unref(skb, 0); + KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref); + skb_shinfo(skb)->flags &= ~(SKBFL_ZEROCOPY_ENABLE | + SKBFL_MANAGED_FRAG_REFS); + skb_shinfo(skb)->nr_frags = 0; consume_skb(skb); __free_page(page); diff --git a/net/xfrm/xfrm_iptfs.c b/net/xfrm/xfrm_iptfs.c index 103cf7c5dbfb9..04ef833074fc8 100644 --- a/net/xfrm/xfrm_iptfs.c +++ b/net/xfrm/xfrm_iptfs.c @@ -2258,7 +2258,8 @@ static void iptfs_output_queued(struct xfrm_state *x, struct sk_buff_head *list) nextp = &(*nextp)->next; } - if (shi->frag_list || skb_cloned(skb) || skb_shared(skb)) + if (shi->frag_list || skb_cloned(skb) || skb_shared(skb) || + skb_zcopy_managed(skb)) share_ok = false; /* See if we have enough space to simply append. -- 2.56.0.455.g6c5a503b38-goog Update skb_splice_bits() (spd_fill_page(), sock_spd_release(), and skb_pipe_buf_ops) and skb_append_pagefrags() to acquire and release references via netmem_inc_ref(), netmem_dec_ref(), and skb_frag_ref() so page_pool RX fragments spliced into a pipe remain pinned in the pool while in the pipe and pair with skb_append_pagefrags() and skb_release_data() when spliced into a TX socket once netmem_inc_ref() and netmem_dec_ref() handle page_pool netmems. In tls_push_data(), fall back to copying when MSG_SPLICE_PAGES encounters a page_pool page because kTLS device offload record->frags[] tracks pages via raw struct page references rather than pp_ref_count before handing them to tls_push_sg(). Caveats: - Because pipe buffers created by skb_splice_bits() hold pp_ref_count until released (including across tee()), user pipes holding unconsumed spliced data keep those page_pool pages in-flight and DMA-mapped in the pool (bounded by pipe capacity and RLIMIT_NOFILE rather than sk_rcvbuf) until the pipe buffers are consumed or closed, with page_pool_destroy() deferring pool release via its periodic workqueue and page_pool_scrub() unmapping DMA. Signed-off-by: Mina Almasry --- net/core/skbuff.c | 31 +++++++++++++++++++++++++++---- net/tls/tls_device.c | 6 ++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 48f6764117243..19d4ce347c41b 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -3114,9 +3114,32 @@ EXPORT_SYMBOL(skb_copy_bits); */ static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i) { - put_page(spd->pages[i]); + netmem_dec_ref(page_to_netmem(spd->pages[i])); } +static void skb_pipe_buf_release(struct pipe_inode_info *pipe, + struct pipe_buffer *buf) +{ + netmem_dec_ref(page_to_netmem(buf->page)); +} + +static bool skb_pipe_buf_get(struct pipe_inode_info *pipe, + struct pipe_buffer *buf) +{ + struct page *page = compound_head(buf->page); + + if (WARN_ON_ONCE(page_ref_count(page) <= 0)) + return false; + + netmem_inc_ref(page_to_netmem(page)); + return true; +} + +static const struct pipe_buf_operations skb_pipe_buf_ops = { + .release = skb_pipe_buf_release, + .get = skb_pipe_buf_get, +}; + static struct page *linear_to_page(struct page *page, unsigned int *len, unsigned int *offset, struct sock *sk) @@ -3165,7 +3188,7 @@ static bool spd_fill_page(struct splice_pipe_desc *spd, struct page *page, spd->partial[spd->nr_pages - 1].len += *len; return false; } - get_page(page); + netmem_inc_ref(page_to_netmem(page)); spd->pages[spd->nr_pages] = page; spd->partial[spd->nr_pages].len = *len; spd->partial[spd->nr_pages].offset = offset; @@ -3281,7 +3304,7 @@ int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset, .pages = pages, .partial = partial, .nr_pages_max = MAX_SKB_FRAGS, - .ops = &nosteal_pipe_buf_ops, + .ops = &skb_pipe_buf_ops, .spd_release = sock_spd_release, }; int ret = 0; @@ -4692,8 +4715,8 @@ int skb_append_pagefrags(struct sk_buff *skb, struct page *page, skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size); } else if (i < max_frags) { skb_zcopy_downgrade_managed(skb); - get_page(page); skb_fill_page_desc_noacc(skb, i, page, offset, size); + skb_frag_ref(skb, i); } else { return -EMSGSIZE; } diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c index 4cdfbb57ecff0..05abf8e23f78a 100644 --- a/net/tls/tls_device.c +++ b/net/tls/tls_device.c @@ -513,10 +513,16 @@ static int tls_push_data(struct sock *sk, goto handle_error; } + if (page_pool_page_is_pp(compound_head(*pages))) { + iov_iter_revert(iter, copy); + goto copy_data; + } + zc_pfrag.offset = off; zc_pfrag.size = copy; tls_append_frag(record, &zc_pfrag, copy); } else if (copy) { +copy_data: copy = min_t(size_t, copy, pfrag->size - pfrag->offset); rc = tls_device_copy_data(page_address(pfrag->page) + -- 2.56.0.455.g6c5a503b38-goog Once tcp_zerocopy_receive() maps an order-0 RX fragment page into a userspace VMA via vm_insert_pages(), vm_insert_pages() increments page->_refcount, which prevents the page from being recycled back into its page_pool when the RX SKB is freed. However, while the RX SKB still holds its fragment reference, the page remains marked as a page_pool page (page_pool_page_is_pp(page) == true) inside the userspace VMA, so subsequent GUP callers such as sendmsg(MSG_ZEROCOPY) or vmsplice() can encounter a page_pool page while holding only a raw struct page reference. Establish the invariant that pp_ref_count is only incremented by callers that already hold an active pp_ref_count (>= 1) on that netmem, and a raw page_ref_count is never converted into a pp_ref_count: - In can_map_frag(), require !skb_cloned(skb), page_pool_ref_count == 1, and page_ref_count == 1 for page_pool pages so the fragment being mapped holds the sole reference to the page; cloned SKBs or page_pool pages with additional references (including drivers holding an undrained fragment bias) fall back to copying. - In tcp_zerocopy_receive(), convert the fragment's sole pp_ref_count into a regular page reference and detach the page from its page_pool via get_page(page) + napi_pp_put_page(page_to_netmem(page)) before queueing the page for vm_insert_pages(). If vm_insert_pages() subsequently fails, the detached page remains referenced by the RX SKB via its regular page reference and is freed back to the page allocator when the SKB is consumed. Signed-off-by: Mina Almasry --- include/net/page_pool/helpers.h | 5 ++++ net/core/datagram.c | 2 ++ net/ipv4/tcp.c | 44 ++++++++++++++++++++++++++++----- 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/include/net/page_pool/helpers.h b/include/net/page_pool/helpers.h index 28635ce9454e8..c814ce95282eb 100644 --- a/include/net/page_pool/helpers.h +++ b/include/net/page_pool/helpers.h @@ -242,6 +242,11 @@ page_pool_get_dma_dir(const struct page_pool *pool) return pool->p.dma_dir; } +static inline long page_pool_ref_count(netmem_ref netmem) +{ + return atomic_long_read(netmem_get_pp_ref_count_ref(netmem)); +} + static inline void page_pool_fragment_netmem(netmem_ref netmem, long nr) { atomic_long_set(netmem_get_pp_ref_count_ref(netmem), nr); diff --git a/net/core/datagram.c b/net/core/datagram.c index ecc30a6f83023..808eda4f7efb5 100644 --- a/net/core/datagram.c +++ b/net/core/datagram.c @@ -34,6 +34,7 @@ * */ +#include #include #include #include @@ -702,6 +703,7 @@ int zerocopy_fill_skb_from_iter(struct sk_buff *skb, } return 0; } +EXPORT_SYMBOL_IF_KUNIT(zerocopy_fill_skb_from_iter); static int zerocopy_fill_skb_from_devmem(struct sk_buff *skb, struct iov_iter *from, diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index b9de7c302ba83..a984e7e93e4c9 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -286,6 +286,7 @@ #include #include #include +#include #include #include @@ -1921,7 +1922,7 @@ static skb_frag_t *skb_advance_to_frag(struct sk_buff *skb, u32 offset_skb, return frag; } -static bool can_map_frag(const skb_frag_t *frag) +static bool can_map_frag(const struct sk_buff *skb, const skb_frag_t *frag) { struct page *page; @@ -1935,18 +1936,32 @@ static bool can_map_frag(const skb_frag_t *frag) if (PageCompound(page) || page->mapping) return false; + /* Before mapping a page_pool page into userspace, + * tcp_zerocopy_receive() converts this fragment's pp_ref_count into a + * regular page reference and detaches the page from its page_pool via + * napi_pp_put_page(). Require this skb_frag_t to hold the sole + * reference to the page so no other fragment or SKB clone expects + * pp_ref_count on it after detachment. + */ + if (page_pool_page_is_pp(page) && + (skb_cloned(skb) || + page_pool_ref_count(page_to_netmem(page)) != 1 || + page_ref_count(page) != 1)) + return false; + return true; } -static int find_next_mappable_frag(const skb_frag_t *frag, +static int find_next_mappable_frag(const struct sk_buff *skb, + const skb_frag_t *frag, int remaining_in_skb) { int offset = 0; - if (likely(can_map_frag(frag))) + if (likely(can_map_frag(skb, frag))) return 0; - while (offset < remaining_in_skb && !can_map_frag(frag)) { + while (offset < remaining_in_skb && !can_map_frag(skb, frag)) { offset += skb_frag_size(frag); ++frag; } @@ -1986,7 +2001,8 @@ static void tcp_zerocopy_set_hint_for_skb(struct sock *sk, * mappable_offset: Bytes till next mappable frag, *not* counting bytes * in partial_frag_remainder. */ - mappable_offset = find_next_mappable_frag(frag, zc->recv_skip_hint); + mappable_offset = find_next_mappable_frag(skb, frag, + zc->recv_skip_hint); zc->recv_skip_hint = mappable_offset + partial_frag_remainder; } @@ -2290,7 +2306,7 @@ static int tcp_zerocopy_receive(struct sock *sk, break; } - mappable_offset = find_next_mappable_frag(frags, + mappable_offset = find_next_mappable_frag(skb, frags, zc->recv_skip_hint); if (mappable_offset) { zc->recv_skip_hint = mappable_offset; @@ -2300,6 +2316,22 @@ static int tcp_zerocopy_receive(struct sock *sk, if (WARN_ON_ONCE(!page)) break; + /* Convert this fragment's sole pp_ref_count into a regular + * page reference and detach the page from its page_pool before + * mapping it into userspace. Once vm_insert_pages() elevates + * page_ref_count the page cannot be recycled into the + * page_pool anyway (and if vm_insert_pages() fails, the + * detached page is freed to the page allocator when the skb is + * consumed); detaching it up front ensures that a page_pool + * page is never mapped into a userspace VMA (where GUP callers + * such as MSG_ZEROCOPY or vmsplice() would otherwise observe a + * page_pool page while holding only a raw page_ref_count). + */ + if (page_pool_page_is_pp(page)) { + get_page(page); + napi_pp_put_page(page_to_netmem(page)); + } + prefetchw(page); pages[pages_to_map++] = page; length += PAGE_SIZE; -- 2.56.0.455.g6c5a503b38-goog skb->pp_recycle was introduced in commit 6a5bcd84e886 ("page_pool: Allow drivers to hint on SKB recycling") when drivers had to explicitly opt in to page_pool SKB recycling and to avoid touching struct page on the free path. Today every page_pool-backed netmem carries its own identity via netmem_is_pp(), making the per-SKB skb->pp_recycle bitfield redundant and error-prone when SKBs are cloned, split, segmented, shifted, or coalesced across page_pool and non-page_pool buffers. For non-page_pool pages, get_page(), put_page(), and skb_free_frag() immediately load the same first struct page cache line (compound_head, which shares the cache line with pp_magic), so checking netmem_is_pp() does not add an extra cache-line load. Remove skb->pp_recycle and check netmem_is_pp() directly via napi_pp_get_page() and napi_pp_put_page() inside netmem_inc_ref(), netmem_dec_ref(), and skb_pp_recycle(), dropping the now-redundant skb_netmem_ref() and skb_netmem_unref() wrappers. Keep skb_mark_for_recycle() as a no-op stub to remove its callers in a separate patch. Use READ_ONCE() and WRITE_ONCE() on pp_magic because every SKB fragment ref/unref now checks netmem_is_pp(), and page->pp_magic shares a struct page union word with folio->lru.next which MM code may concurrently modify on page-cache folios in TX SKBs. Because pp_ref_count is only incremented by callers that already hold an active pp_ref_count (>= 1) on that netmem and tcp_zerocopy_receive() detaches page_pool pages before mapping them into userspace, a raw page_ref_count is never converted into a pp_ref_count and page_pool_unref_netmem() keeps its non-RMW fast path unchanged. bench_page_pool_simple results (5-iteration interleaved UML runs): - no-softirq-page_pool01 (fast-path cache recycle): 40 cycles (20.3 ns) baseline vs 30-31 cycles (15.3 ns) patched (~9-10 cycles / -24.7% improvement: using __netmem_to_page() in page_pool_netmem_is_exclusively_owned() and __page_pool_page_can_be_recycled() after checking netmem_is_net_iov() eliminates two redundant WARN_ON_ONCE() expansions and out-of-line netmem_to_page() calls per recycle). - no-softirq-page_pool02 (ptr_ring recycle): 81 cycles (40.7 ns) baseline vs 74 cycles (37.2 ns) patched (~7 cycles / -8.6% improvement for the same reason). - no-softirq-page_pool03 (slow page-allocator path with elevated page_ref_count): 458-461 cycles (230.0 ns) baseline vs 454-455 cycles (227.6 ns) patched (~5 cycles / -1.0% improvement). Notes: - __frame_add_frag() in net/wireless/util.c is switched from get_page() to skb_frag_ref() (with a MAX_SKB_FRAGS check) so mac80211 A-MSDU subframe SKBs on page_pool wireless drivers (such as mt76) pair with skb_release_data(). - destroy_record() in net/tls/tls_device.c and fill_sg_in() in net/tls/tls_device_fallback.c are switched from netmem_dec_ref() and netmem_inc_ref() to put_page() and get_page() so kTLS record->frags[] and fallback scatterlists stay on struct page references matching tls_create_new_record(), tls_append_frag(), tls_sw_fallback(), and mlx5 tx_sync_info_get(). - __xdp_return(MEM_TYPE_PAGE_SHARED) in net/core/xdp.c, veth_xdp_get() in drivers/net/veth.c, and virtnet_skb_append_frag() in drivers/net/virtio_net.c are switched to netmem_inc_ref() and netmem_dec_ref() so page_pool-backed XDP and virtio_net buffers pair with netmem_dec_ref(). Caveats: - For struct page-backed netmems, page_pool membership is identified via page->pp_magic, which has a pre-existing union aliasing risk with non-networking struct page fields if a non-page_pool page's union field matches PP_SIGNATURE (in particular on 32-bit architectures where POISON_POINTER_DELTA is 0); removing skb->pp_recycle extends this check from pp_recycle=1 SKBs to all netmem_inc_ref(), netmem_dec_ref(), and skb_pp_recycle() operations. - netpoll_zap_completion_queue() has a pre-existing issue where skb_irq_freeable() does not exclude SKBs holding page_pool memory before calling __kfree_skb() with hardirqs disabled; TX SKBs carrying spliced page_pool fragments are subject to the same path. Signed-off-by: Mina Almasry --- drivers/net/veth.c | 3 +- drivers/net/virtio_net.c | 8 +- include/linux/mm.h | 2 +- include/linux/skbuff.h | 9 +- include/linux/skbuff_ref.h | 60 +------- include/net/netmem.h | 37 ++++- include/net/page_pool/helpers.h | 15 ++ net/core/gro.c | 9 -- net/core/net_test.c | 264 ++++++++++++++------------------ net/core/netmem_priv.h | 21 ++- net/core/page_pool.c | 6 +- net/core/skbuff.c | 42 ++--- net/core/xdp.c | 2 +- net/ipv4/esp4.c | 11 +- net/ipv6/esp6.c | 11 +- net/tls/tls_device.c | 2 +- net/tls/tls_device_fallback.c | 2 +- net/wireless/util.c | 31 +++- net/xfrm/trace_iptfs.h | 6 +- net/xfrm/xfrm_iptfs.c | 13 +- 20 files changed, 243 insertions(+), 311 deletions(-) diff --git a/drivers/net/veth.c b/drivers/net/veth.c index 3eca92d3dc60d..285958432ea0a 100644 --- a/drivers/net/veth.c +++ b/drivers/net/veth.c @@ -27,7 +27,6 @@ #include #include #include -#include #include #define DRV_NAME "veth" @@ -746,7 +745,7 @@ static void veth_xdp_get(struct xdp_buff *xdp) struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); int i; - get_page(virt_to_page(xdp->data)); + netmem_inc_ref(page_to_netmem(virt_to_page(xdp->data))); if (likely(!xdp_buff_has_frags(xdp))) return; diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index dd2c41ef6fbe7..3453689bfd341 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -2351,9 +2351,6 @@ static struct sk_buff *virtnet_skb_append_frag(struct receive_queue *rq, if (unlikely(!nskb)) return NULL; - if (head_skb->pp_recycle) - skb_mark_for_recycle(nskb); - if (curr_skb == head_skb) skb_shinfo(curr_skb)->frag_list = nskb; else @@ -2371,10 +2368,7 @@ static struct sk_buff *virtnet_skb_append_frag(struct receive_queue *rq, offset = buf - page_address(page); if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) { - if (head_skb->pp_recycle) - page_pool_put_page(rq->page_pool, page, -1, true); - else - put_page(page); + netmem_dec_ref(page_to_netmem(page)); skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1, len, truesize); } else { diff --git a/include/linux/mm.h b/include/linux/mm.h index 9f167950926a1..f13d9f6b3ed50 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -5698,7 +5698,7 @@ int arch_lock_shadow_stack_status(struct task_struct *t, unsigned long status); #ifdef CONFIG_PAGE_POOL static inline bool page_pool_page_is_pp(const struct page *page) { - return (page->pp_magic & PP_MAGIC_MASK) == PP_SIGNATURE; + return (READ_ONCE(page->pp_magic) & PP_MAGIC_MASK) == PP_SIGNATURE; } #else static inline bool page_pool_page_is_pp(const struct page *page) diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 6787fcf6ac3e4..4eaf4da1f7dad 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -834,8 +834,6 @@ enum skb_tstamp_type { * @head_frag: skb was allocated from page fragments, * not allocated by kmalloc() or vmalloc(). * @pfmemalloc: skbuff was allocated from PFMEMALLOC reserves - * @pp_recycle: mark the packet for recycling instead of freeing (implies - * page_pool support on driver) * @active_extensions: active extensions (skb_ext_id types) * @ndisc_nodetype: router type (from link layer) * @ooo_okay: allow the mapping of a socket to a queue to be changed @@ -964,8 +962,7 @@ struct sk_buff { fclone:2, peeked:1, head_frag:1, - pfmemalloc:1, - pp_recycle:1; /* page_pool recycle indicator */ + pfmemalloc:1; #ifdef CONFIG_SKB_EXTENSIONS __u8 active_extensions; #endif @@ -5476,11 +5473,9 @@ static inline struct kcov_common_handle_id skb_get_kcov_handle(struct sk_buff *s return skb->kcov_handle; } +/* Deprecated: page_pool recycling is now determined per netmem. */ static inline void skb_mark_for_recycle(struct sk_buff *skb) { -#ifdef CONFIG_PAGE_POOL - skb->pp_recycle = 1; -#endif } ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter, diff --git a/include/linux/skbuff_ref.h b/include/linux/skbuff_ref.h index e94c599ce83d2..b368eb4661ed7 100644 --- a/include/linux/skbuff_ref.h +++ b/include/linux/skbuff_ref.h @@ -9,63 +9,21 @@ #include -bool napi_pp_get_page(netmem_ref netmem); - -/** - * skb_netmem_ref - acquire a fragment reference on a netmem - * @netmem: netmem to reference - * @recycle: whether page_pool recycling is enabled (e.g. skb->pp_recycle) - * - * Acquires pp_ref_count via napi_pp_get_page() if @recycle is true and - * @netmem belongs to a page_pool; otherwise acquires a non-pp backing - * reference via netmem_inc_ref(). Counterpart to skb_netmem_unref(). - */ -static __always_inline void skb_netmem_ref(netmem_ref netmem, bool recycle) -{ -#ifdef CONFIG_PAGE_POOL - if (recycle && napi_pp_get_page(netmem)) - return; -#endif - netmem_inc_ref(netmem); -} - /** * skb_frag_ref - acquire a reference on a paged fragment of an skb * @skb: the buffer * @f: the fragment offset * - * Acquires a reference on the @f'th fragment of @skb via skb_netmem_ref() - * (pp_ref_count if skb->pp_recycle and page_pool-backed, else non-pp - * backing refcount), skipping managed zerocopy fragments. Counterpart to - * skb_frag_unref(). + * Acquires a reference on the @f'th fragment of @skb via netmem_inc_ref() + * (pp_ref_count if page_pool-backed, else non-pp backing refcount), skipping + * managed zerocopy fragments. Counterpart to skb_frag_unref(). */ static __always_inline void skb_frag_ref(struct sk_buff *skb, int f) { struct skb_shared_info *shinfo = skb_shinfo(skb); if (!skb_zcopy_managed(skb)) - skb_netmem_ref(skb_frag_netmem(&shinfo->frags[f]), - skb->pp_recycle); -} - -bool napi_pp_put_page(netmem_ref netmem); - -/** - * skb_netmem_unref - release a fragment reference on a netmem - * @netmem: netmem to unreference - * @recycle: whether page_pool recycling is enabled (e.g. skb->pp_recycle) - * - * Drops pp_ref_count via napi_pp_put_page() if @recycle is true and @netmem - * belongs to a page_pool; otherwise drops a non-pp backing reference via - * netmem_dec_ref(). Counterpart to skb_netmem_ref(). - */ -static __always_inline void skb_netmem_unref(netmem_ref netmem, bool recycle) -{ -#ifdef CONFIG_PAGE_POOL - if (recycle && napi_pp_put_page(netmem)) - return; -#endif - netmem_dec_ref(netmem); + netmem_inc_ref(skb_frag_netmem(&shinfo->frags[f])); } /** @@ -73,18 +31,16 @@ static __always_inline void skb_netmem_unref(netmem_ref netmem, bool recycle) * @skb: the buffer * @f: the fragment offset * - * Drops a reference on the @f'th fragment of @skb via skb_netmem_unref() - * (pp_ref_count if skb->pp_recycle and page_pool-backed, else non-pp - * backing refcount), skipping managed zerocopy fragments. Counterpart to - * skb_frag_ref(). + * Drops a reference on the @f'th fragment of @skb via netmem_dec_ref() + * (pp_ref_count if page_pool-backed, else non-pp backing refcount), skipping + * managed zerocopy fragments. Counterpart to skb_frag_ref(). */ static __always_inline void skb_frag_unref(struct sk_buff *skb, int f) { struct skb_shared_info *shinfo = skb_shinfo(skb); if (!skb_zcopy_managed(skb)) - skb_netmem_unref(skb_frag_netmem(&shinfo->frags[f]), - skb->pp_recycle); + netmem_dec_ref(skb_frag_netmem(&shinfo->frags[f])); } #endif /* _LINUX_SKBUFF_REF_H */ diff --git a/include/net/netmem.h b/include/net/netmem.h index fb693bc1702a4..75c26c8997431 100644 --- a/include/net/netmem.h +++ b/include/net/netmem.h @@ -412,16 +412,34 @@ static inline bool net_is_devmem_iov(const struct net_iov *niov) void net_iov_inc_ref(struct net_iov *niov); void net_iov_dec_ref(struct net_iov *niov); +#if IS_ENABLED(CONFIG_PAGE_POOL) +bool napi_pp_get_page(netmem_ref netmem); +bool napi_pp_put_page(netmem_ref netmem); +#else +static inline bool napi_pp_get_page(netmem_ref netmem) +{ + return false; +} + +static inline bool napi_pp_put_page(netmem_ref netmem) +{ + return false; +} +#endif + /** - * netmem_inc_ref - acquire a non-page_pool reference on a netmem + * netmem_inc_ref - acquire a reference on a netmem * @netmem: netmem to reference * - * Acquires a non-pp backing reference (page._refcount via get_page() or - * binding->ref via net_iov_inc_ref()) regardless of page_pool membership. - * Counterpart to netmem_dec_ref(). + * Acquires pp_ref_count via napi_pp_get_page() if @netmem belongs to a + * page_pool; otherwise acquires a non-pp backing reference (binding->ref via + * net_iov_inc_ref() or page._refcount via get_page()). Counterpart to + * netmem_dec_ref(). */ static __always_inline void netmem_inc_ref(netmem_ref netmem) { + if (napi_pp_get_page(netmem)) + return; if (netmem_is_net_iov(netmem)) net_iov_inc_ref(netmem_to_net_iov(netmem)); else @@ -429,15 +447,18 @@ static __always_inline void netmem_inc_ref(netmem_ref netmem) } /** - * netmem_dec_ref - release a non-page_pool reference on a netmem + * netmem_dec_ref - release a reference on a netmem * @netmem: netmem to unreference * - * Drops a non-pp backing reference (page._refcount via put_page() or - * binding->ref via net_iov_dec_ref()) regardless of page_pool membership. - * Counterpart to netmem_inc_ref(). + * Drops pp_ref_count via napi_pp_put_page() if @netmem belongs to a + * page_pool; otherwise drops a non-pp backing reference (binding->ref via + * net_iov_dec_ref() or page._refcount via put_page()). Counterpart to + * netmem_inc_ref(). */ static __always_inline void netmem_dec_ref(netmem_ref netmem) { + if (napi_pp_put_page(netmem)) + return; if (netmem_is_net_iov(netmem)) net_iov_dec_ref(netmem_to_net_iov(netmem)); else diff --git a/include/net/page_pool/helpers.h b/include/net/page_pool/helpers.h index c814ce95282eb..fdbb1a4fe11f5 100644 --- a/include/net/page_pool/helpers.h +++ b/include/net/page_pool/helpers.h @@ -275,6 +275,14 @@ static inline void page_pool_fragment_page(struct page *page, long nr) page_pool_fragment_netmem(page_to_netmem(page), nr); } +static inline bool page_pool_netmem_is_exclusively_owned(netmem_ref netmem) +{ + if (netmem_is_net_iov(netmem)) + return true; + + return page_ref_count(__netmem_to_page(netmem)) == 1; +} + static inline long page_pool_unref_netmem(netmem_ref netmem, long nr) { atomic_long_t *pp_ref_count = netmem_get_pp_ref_count_ref(netmem); @@ -293,6 +301,13 @@ static inline long page_pool_unref_netmem(netmem_ref netmem, long nr) * handling by ensuring all pages have partitioned into only 1 piece * initially, and only overwrite it when the page is partitioned into * more than one piece. + * + * This non-RMW fast path is safe because pp_ref_count is only ever + * incremented by callers that already hold an active pp_ref_count + * (>= 1) on @netmem; a raw page_ref_count is never converted into a + * pp_ref_count. Thus when pp_ref_count == nr, the caller holds all + * remaining pp_ref_count references and no concurrent + * page_pool_ref_netmem() / napi_pp_get_page() can occur. */ if (atomic_long_read(pp_ref_count) == nr) { /* As we have ensured nr is always one for constant case using diff --git a/net/core/gro.c b/net/core/gro.c index 1b033855a20db..124079f3eb9b0 100644 --- a/net/core/gro.c +++ b/net/core/gro.c @@ -100,15 +100,6 @@ int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb) struct sk_buff *lp; int segs; - /* Do not splice page pool based packets w/ non-page pool - * packets. This can result in reference count issues as page - * pool pages will not decrement the reference count and will - * instead be immediately returned to the pool or have frag - * count decremented. - */ - if (p->pp_recycle != skb->pp_recycle) - return -ETOOMANYREFS; - if (skb_zcopy(p) || skb_zcopy(skb)) return -ETOOMANYREFS; diff --git a/net/core/net_test.c b/net/core/net_test.c index e20059a00003c..788aa57c63418 100644 --- a/net/core/net_test.c +++ b/net/core/net_test.c @@ -516,6 +516,7 @@ static void ip_tunnel_flags_test_run(struct kunit *test) /* Netmem & SKB fragment refcounting */ #include +#include #include #include #include @@ -549,23 +550,11 @@ static void netmem_ref_test_page(struct kunit *test) KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref); #endif - /* Layer 3: skb_netmem_ref / unref fall back to non-pp on non-PP page */ - skb_netmem_ref(netmem, false); - KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref + 1); - skb_netmem_unref(netmem, false); - KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref); - - skb_netmem_ref(netmem, true); - KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref + 1); - skb_netmem_unref(netmem, true); - KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref); - - /* Layer 4: skb_frag_ref / skb_frag_unref on non-PP page */ + /* Layer 3: skb_frag_ref / skb_frag_unref on non-PP page */ skb = alloc_skb(64, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, skb); skb_fill_netmem_desc(skb, 0, netmem, 0, 64); - skb->pp_recycle = 1; skb_frag_ref(skb, 0); KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref + 1); skb_frag_unref(skb, 0); @@ -610,47 +599,31 @@ static void netmem_ref_test_pp_page(struct kunit *test) base_ref = page_ref_count(page); base_pp_ref = atomic_long_read(&page->pp_ref_count); - /* Layer 1: netmem_inc/dec_ref always use non-pp refcount */ + /* Layer 1: netmem_inc/dec_ref use pp_ref_count on PP page */ netmem_inc_ref(netmem); - KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref + 1); - KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), - base_pp_ref); - netmem_dec_ref(netmem); - KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref); - - /* Layer 2: napi_pp_get/put_page use pp_ref_count */ - KUNIT_EXPECT_TRUE(test, napi_pp_get_page(netmem)); KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), base_pp_ref + 1); KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref); - KUNIT_EXPECT_TRUE(test, napi_pp_put_page(netmem)); - KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), - base_pp_ref); - - /* Layer 3: skb_netmem_ref / unref obey recycle flag */ - skb_netmem_ref(netmem, false); - KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref + 1); + netmem_dec_ref(netmem); KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), base_pp_ref); - skb_netmem_unref(netmem, false); - KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref); - skb_netmem_ref(netmem, true); + /* Layer 2: napi_pp_get/put_page use pp_ref_count */ + KUNIT_EXPECT_TRUE(test, napi_pp_get_page(netmem)); KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), base_pp_ref + 1); KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref); - skb_netmem_unref(netmem, true); + KUNIT_EXPECT_TRUE(test, napi_pp_put_page(netmem)); KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), base_pp_ref); - /* Layer 4: skb_frag_ref / skb_frag_unref match on pp_recycle=1 & 0 */ + /* Layer 3: skb_frag_ref / skb_frag_unref use pp_ref_count on PP page */ skb = alloc_skb(64, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, skb); skb_fill_netmem_desc(skb, 0, netmem, 0, 64); skb->len = 64; skb->data_len = 64; - skb->pp_recycle = 1; skb_frag_ref(skb, 0); KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), base_pp_ref + 1); @@ -659,10 +632,9 @@ static void netmem_ref_test_pp_page(struct kunit *test) KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), base_pp_ref); - /* pskb_copy on pp_recycle=1 skb propagates pp_recycle & pp_ref_count */ + /* pskb_copy on PP page skb propagates pp_ref_count */ copy = pskb_copy(skb, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, copy); - KUNIT_EXPECT_TRUE(test, copy->pp_recycle); KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), base_pp_ref + 1); KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref); @@ -671,14 +643,13 @@ static void netmem_ref_test_pp_page(struct kunit *test) KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), base_pp_ref); - /* skb_zerocopy on pp_recycle=1 skb propagates pp_ref_count */ + /* skb_zerocopy on PP page skb propagates pp_ref_count */ copy = alloc_skb(0, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, copy); skb_put(copy, skb_tailroom(copy)); skb->head_frag = 1; KUNIT_ASSERT_EQ(test, skb_zerocopy(copy, skb, skb->len, 0), 0); skb->head_frag = 0; - KUNIT_EXPECT_TRUE(test, copy->pp_recycle); KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), base_pp_ref + 1); KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref); @@ -687,11 +658,10 @@ static void netmem_ref_test_pp_page(struct kunit *test) KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), base_pp_ref); - /* skb_clone + pskb_expand_head preserves pp_recycle and pp_ref_count */ + /* skb_clone + pskb_expand_head preserves pp_ref_count */ clone = skb_clone(skb, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, clone); KUNIT_ASSERT_EQ(test, pskb_expand_head(clone, 32, 0, GFP_KERNEL), 0); - KUNIT_EXPECT_TRUE(test, clone->pp_recycle); KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), base_pp_ref + 1); KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref); @@ -712,7 +682,6 @@ static void netmem_ref_test_pp_page(struct kunit *test) split = alloc_skb(32, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, split); skb_split(skb, split, 16); - KUNIT_EXPECT_TRUE(test, split->pp_recycle); KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), base_pp_ref); KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref); @@ -728,7 +697,6 @@ static void netmem_ref_test_pp_page(struct kunit *test) split = alloc_skb(0, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, split); skb_split(skb, split, 32); - KUNIT_EXPECT_TRUE(test, split->pp_recycle); KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), base_pp_ref + 1); KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref); @@ -737,14 +705,13 @@ static void netmem_ref_test_pp_page(struct kunit *test) KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), base_pp_ref); - /* skb_segment on pp_recycle=1 skb propagates pp_recycle to segments */ + /* skb_segment on PP page skb propagates pp_ref_count to segments */ skb->protocol = htons(ETH_P_IP); skb_reset_mac_header(skb); skb_reset_network_header(skb); skb_shinfo(skb)->gso_size = 16; split = skb_segment(skb, NETIF_F_SG | NETIF_F_HW_CSUM); KUNIT_ASSERT_FALSE(test, IS_ERR_OR_NULL(split)); - KUNIT_EXPECT_TRUE(test, split->pp_recycle); KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), base_pp_ref + 2); KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref); @@ -763,7 +730,6 @@ static void netmem_ref_test_pp_page(struct kunit *test) skb->data_len = 64; split = alloc_skb(0, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, split); - split->pp_recycle = 1; KUNIT_EXPECT_EQ(test, skb_shift(split, skb, 16), 16); KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), base_pp_ref + 2); @@ -780,17 +746,6 @@ static void netmem_ref_test_pp_page(struct kunit *test) skb_shinfo(split)->nr_frags = 0; consume_skb(split); page_pool_put_full_page(pool, page, false); - skb_fill_netmem_desc(skb, 0, netmem, 0, 64); - skb->len = 64; - skb->data_len = 64; - - skb->pp_recycle = 0; - skb_frag_ref(skb, 0); - KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref + 1); - KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), - base_pp_ref); - skb_frag_unref(skb, 0); - KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref); skb_shinfo(skb)->nr_frags = 0; consume_skb(skb); @@ -798,7 +753,7 @@ static void netmem_ref_test_pp_page(struct kunit *test) page_pool_destroy(pool); /* Order-1 compound page_pool page: tail netmem resolves to head & - * build_skb(head_frag=1, pp_recycle=1) skb_zerocopy uses pp_ref_count. + * build_skb(head_frag=1) skb_zerocopy uses pp_ref_count. */ pp_params.order = 1; pool = page_pool_create(&pp_params); @@ -819,7 +774,6 @@ static void netmem_ref_test_pp_page(struct kunit *test) skb = build_skb(page_address(page), PAGE_SIZE); KUNIT_ASSERT_NOT_NULL(test, skb); - skb->pp_recycle = 1; skb_put(skb, 64); page_pool_ref_page(page); @@ -827,7 +781,6 @@ static void netmem_ref_test_pp_page(struct kunit *test) KUNIT_ASSERT_NOT_NULL(test, copy); skb_put(copy, skb_tailroom(copy)); KUNIT_ASSERT_EQ(test, skb_zerocopy(copy, skb, skb->len, 0), 0); - KUNIT_EXPECT_TRUE(test, copy->pp_recycle); KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), base_pp_ref + 2); KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref); @@ -841,6 +794,78 @@ static void netmem_ref_test_pp_page(struct kunit *test) page_pool_put_full_page(pool, page, false); page_pool_destroy(pool); } + +static void netmem_ref_test_tcp_zerocopy_receive_pp(struct kunit *test) +{ + struct page_pool_params pp_params = { + .order = 0, + .pool_size = 4, + .nid = NUMA_NO_NODE, + }; + struct sk_buff *rx_skb, *tx_skb; + struct page_pool *pool; + struct bio_vec bvec; + struct iov_iter iter; + struct page *pp_page; + + pool = page_pool_create(&pp_params); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pool); + pp_page = page_pool_dev_alloc_pages(pool); + if (!pp_page) { + page_pool_destroy(pool); + KUNIT_ASSERT_NOT_NULL(test, pp_page); + } + + rx_skb = alloc_skb(0, GFP_KERNEL); + if (!rx_skb) { + page_pool_put_full_page(pool, pp_page, false); + page_pool_destroy(pool); + KUNIT_ASSERT_NOT_NULL(test, rx_skb); + } + skb_add_rx_frag(rx_skb, 0, pp_page, 0, PAGE_SIZE, PAGE_SIZE); + KUNIT_EXPECT_TRUE(test, page_pool_page_is_pp(pp_page)); + KUNIT_EXPECT_EQ(test, page_pool_ref_count(page_to_netmem(pp_page)), 1L); + KUNIT_EXPECT_EQ(test, page_ref_count(pp_page), 1); + + /* Simulate tcp_zerocopy_receive() detaching an order-0 page_pool RX + * fragment before vm_insert_pages() maps it into userspace: + * get_page() + napi_pp_put_page() converts the fragment's sole + * pp_ref_count into a regular page reference and releases the page + * from the page_pool. + */ + get_page(pp_page); + KUNIT_EXPECT_TRUE(test, napi_pp_put_page(page_to_netmem(pp_page))); + KUNIT_EXPECT_FALSE(test, page_pool_page_is_pp(pp_page)); + KUNIT_EXPECT_EQ(test, page_ref_count(pp_page), 1); + + /* Forwarding the detached page into MSG_ZEROCOPY TX via + * zerocopy_fill_skb_from_iter() (where iov_iter_get_pages2() pins + * the page via get_page()) now operates on a normal non-page_pool + * page and balances page_ref_count across TX and RX SKB release. + */ + tx_skb = alloc_skb(0, GFP_KERNEL); + if (!tx_skb) { + consume_skb(rx_skb); + page_pool_destroy(pool); + KUNIT_ASSERT_NOT_NULL(test, tx_skb); + } + bvec_set_page(&bvec, pp_page, 128, 0); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, 128); + KUNIT_EXPECT_EQ(test, zerocopy_fill_skb_from_iter(tx_skb, &iter, 128), + 0); + KUNIT_EXPECT_EQ(test, skb_shinfo(tx_skb)->nr_frags, 1); + KUNIT_EXPECT_EQ(test, page_ref_count(pp_page), 2); + consume_skb(tx_skb); + KUNIT_EXPECT_EQ(test, page_ref_count(pp_page), 1); + + /* Extra ref so we can inspect page_ref_count after freeing rx_skb */ + get_page(pp_page); + consume_skb(rx_skb); + KUNIT_EXPECT_EQ(test, page_ref_count(pp_page), 1); + put_page(pp_page); + + page_pool_destroy(pool); +} #endif #if defined(CONFIG_PAGE_POOL) && defined(CONFIG_NET_DEVMEM) @@ -882,10 +907,10 @@ static void netmem_ref_test_net_iov(struct kunit *test) KUNIT_EXPECT_FALSE(test, napi_pp_get_page(netmem)); KUNIT_EXPECT_FALSE(test, napi_pp_put_page(netmem)); - skb_netmem_ref(netmem, true); + netmem_inc_ref(netmem); KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count), base_ref + 1); - skb_netmem_unref(netmem, true); + netmem_dec_ref(netmem); KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count), base_ref); @@ -896,7 +921,7 @@ static void netmem_ref_test_net_iov(struct kunit *test) netmem_or_pp_magic(netmem, PP_SIGNATURE); atomic_long_set(&niov.desc.pp_ref_count, 2); - /* Layer 0 & 1: net_iov/netmem_inc_ref grab non-pp ref on PP */ + /* Layer 0 & 1: net_iov_inc_ref uses binding.ref, netmem_inc_ref pp */ net_iov_inc_ref(&niov); KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count), base_ref + 1); @@ -905,6 +930,13 @@ static void netmem_ref_test_net_iov(struct kunit *test) KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count), base_ref); + netmem_inc_ref(netmem); + KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 3L); + KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count), + base_ref); + netmem_dec_ref(netmem); + KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 2L); + /* Layer 2: napi_pp_get/put_page use pp_ref_count */ KUNIT_EXPECT_TRUE(test, napi_pp_get_page(netmem)); KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 3L); @@ -913,14 +945,13 @@ static void netmem_ref_test_net_iov(struct kunit *test) KUNIT_EXPECT_TRUE(test, napi_pp_put_page(netmem)); KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 2L); - /* Layer 4: skb_frag_ref / skb_frag_unref on PP net_iov */ + /* Layer 3: skb_frag_ref / skb_frag_unref on PP net_iov */ skb = alloc_skb(64, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, skb); skb_fill_netmem_desc(skb, 0, netmem, 0, 64); skb->len = 64; skb->data_len = 64; - skb->pp_recycle = 1; skb_frag_ref(skb, 0); KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 3L); KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count), @@ -928,17 +959,16 @@ static void netmem_ref_test_net_iov(struct kunit *test) skb_frag_unref(skb, 0); KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 2L); - /* pskb_copy on pp_recycle=1 net_iov skb keeps pp_ref_count */ + /* pskb_copy on PP net_iov skb keeps pp_ref_count */ copy = pskb_copy(skb, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, copy); - KUNIT_EXPECT_TRUE(test, copy->pp_recycle); KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 3L); KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count), base_ref); consume_skb(copy); KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 2L); - /* skb_zerocopy on pp_recycle=1 net_iov skb keeps pp_ref_count */ + /* skb_zerocopy on PP net_iov skb keeps pp_ref_count */ copy = alloc_skb(0, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, copy); skb_put(copy, skb_tailroom(copy)); @@ -946,7 +976,6 @@ static void netmem_ref_test_net_iov(struct kunit *test) skb->unreadable = 1; KUNIT_ASSERT_EQ(test, skb_zerocopy(copy, skb, skb->len, 0), 0); skb->head_frag = 0; - KUNIT_EXPECT_TRUE(test, copy->pp_recycle); KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 3L); KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count), base_ref); @@ -956,7 +985,6 @@ static void netmem_ref_test_net_iov(struct kunit *test) /* skb_shift balances pp_ref_count on split and merge */ copy = alloc_skb(0, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, copy); - copy->pp_recycle = 1; KUNIT_EXPECT_EQ(test, skb_shift(copy, skb, 16), 16); KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 3L); KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count), @@ -969,26 +997,16 @@ static void netmem_ref_test_net_iov(struct kunit *test) skb->len = 64; skb->data_len = 64; - /* Uncloning a cloned pp_recycle=1 net_iov skb keeps pp_ref_count */ + /* Uncloning a cloned PP net_iov skb keeps pp_ref_count */ clone = skb_clone(skb, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, clone); KUNIT_ASSERT_EQ(test, pskb_expand_head(skb, 32, 0, GFP_KERNEL), 0); - KUNIT_EXPECT_TRUE(test, skb->pp_recycle); KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 3L); KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count), base_ref); consume_skb(clone); KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 2L); - skb->pp_recycle = 0; - skb_frag_ref(skb, 0); - KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count), - base_ref + 1); - KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 2L); - skb_frag_unref(skb, 0); - KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count), - base_ref); - skb_shinfo(skb)->nr_frags = 0; consume_skb(skb); netmem_clear_pp_magic(netmem); @@ -1025,7 +1043,6 @@ static void netmem_ref_test_release_data(struct kunit *test) skb = alloc_skb(64, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, skb); skb_put(skb, 32); - skb->pp_recycle = 1; skb_fill_netmem_desc(skb, 0, netmem, 0, 64); skb->len += 64; skb->data_len = 64; @@ -1038,14 +1055,11 @@ static void netmem_ref_test_release_data(struct kunit *test) clone2 = skb_clone(clone1, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, clone2); - /* Freeing head skb first leaves clone1/clone2 pp_recycle=1 intact */ + /* Freeing head skb first leaves clone1/clone2 intact */ consume_skb(skb); - KUNIT_EXPECT_TRUE(test, clone1->pp_recycle); - KUNIT_EXPECT_TRUE(test, clone2->pp_recycle); KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), base_pp_ref + 1); consume_skb(clone2); - KUNIT_EXPECT_TRUE(test, clone1->pp_recycle); KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), base_pp_ref + 1); consume_skb(clone1); @@ -1057,7 +1071,6 @@ static void netmem_ref_test_release_data(struct kunit *test) skb = alloc_skb(64, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, skb); skb_put(skb, 32); - skb->pp_recycle = 1; skb_fill_netmem_desc(skb, 0, netmem, 0, 64); skb->len += 64; skb->data_len = 64; @@ -1068,8 +1081,6 @@ static void netmem_ref_test_release_data(struct kunit *test) KUNIT_ASSERT_EQ(test, pskb_expand_head((i & 1) ? clone1 : skb, 32, 0, GFP_KERNEL), 0); - KUNIT_EXPECT_TRUE(test, skb->pp_recycle); - KUNIT_EXPECT_TRUE(test, clone1->pp_recycle); KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), base_pp_ref + 2); KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref); @@ -1096,7 +1107,6 @@ static void netmem_ref_test_release_data(struct kunit *test) skb = alloc_skb(64, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, skb); skb_put(skb, 32); - skb->pp_recycle = 1; skb_fill_netmem_desc(skb, 0, netmem, 0, 64); skb->len += 64; skb->data_len = 64; @@ -1105,7 +1115,6 @@ static void netmem_ref_test_release_data(struct kunit *test) /* Carve inside header keeping frag -> pp_ref_count +1 until free */ ext = pskb_extract(skb, 16, skb->len - 16, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, ext); - KUNIT_EXPECT_TRUE(test, ext->pp_recycle); KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), base_pp_ref + 2); KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref); @@ -1116,7 +1125,6 @@ static void netmem_ref_test_release_data(struct kunit *test) /* Carve + trim + skb_condense ref/unrefs pp_ref_count cleanly */ ext = pskb_extract(skb, 16, 48, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, ext); - KUNIT_EXPECT_TRUE(test, ext->pp_recycle); KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), base_pp_ref + 1); KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref); @@ -1125,7 +1133,6 @@ static void netmem_ref_test_release_data(struct kunit *test) /* Carve inside nonlinear keeping frag -> pp_ref_count +1 until free */ ext = pskb_extract(skb, 48, skb->len - 48, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, ext); - KUNIT_EXPECT_TRUE(test, ext->pp_recycle); KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), base_pp_ref + 2); KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref); @@ -1137,12 +1144,10 @@ static void netmem_ref_test_release_data(struct kunit *test) page_pool_ref_netmem(netmem); clone1 = alloc_skb(64, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, clone1); - clone1->pp_recycle = 1; skb_fill_netmem_desc(clone1, 0, netmem, 0, 64); clone1->len = 64; clone1->data_len = 64; skb_morph(clone1, skb); - KUNIT_EXPECT_TRUE(test, clone1->pp_recycle); KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count), base_pp_ref + 1); consume_skb(clone1); @@ -1164,14 +1169,12 @@ static void netmem_ref_test_release_data(struct kunit *test) skb = alloc_skb(64, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, skb); skb_put(skb, 32); - skb->pp_recycle = 1; skb_fill_netmem_desc(skb, 0, netmem, 0, 64); skb->len += 64; skb->data_len = 64; ext = pskb_extract(skb, 48, skb->len - 48, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, ext); - KUNIT_EXPECT_TRUE(test, ext->pp_recycle); KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 3L); consume_skb(ext); KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 2L); @@ -1230,7 +1233,6 @@ static void netmem_ref_test_skb_segment(struct kunit *test) /* 1. Single GSO skb with page_pool net_iov frags */ head = alloc_skb(64, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, head); - head->pp_recycle = 1; head->protocol = htons(ETH_P_IP); skb_reset_mac_header(head); skb_reset_network_header(head); @@ -1241,8 +1243,6 @@ static void netmem_ref_test_skb_segment(struct kunit *test) segs = skb_segment(head, NETIF_F_SG | NETIF_F_HW_CSUM); KUNIT_ASSERT_FALSE(test, IS_ERR_OR_NULL(segs)); - for (seg = segs; seg; seg = seg->next) - KUNIT_EXPECT_TRUE(test, seg->pp_recycle); KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 6L); KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count), base_iov_ref); @@ -1251,13 +1251,11 @@ static void netmem_ref_test_skb_segment(struct kunit *test) skb_shinfo(head)->nr_frags = 0; consume_skb(head); - /* 2. GRO frag_list fast path (skb_clone of pp_recycle=1 list_skb) */ + /* 2. GRO frag_list fast path (skb_clone of list_skb) */ head = alloc_skb(64, GFP_KERNEL); list = alloc_skb(64, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, head); KUNIT_ASSERT_NOT_NULL(test, list); - head->pp_recycle = 1; - list->pp_recycle = 1; head->protocol = htons(ETH_P_IP); skb_reset_mac_header(head); skb_reset_network_header(head); @@ -1280,8 +1278,6 @@ static void netmem_ref_test_skb_segment(struct kunit *test) segs = skb_segment(head, NETIF_F_SG | NETIF_F_HW_CSUM); KUNIT_ASSERT_FALSE(test, IS_ERR_OR_NULL(segs)); - for (seg = segs; seg; seg = seg->next) - KUNIT_EXPECT_TRUE(test, seg->pp_recycle); KUNIT_EXPECT_EQ(test, atomic_long_read(&pp_page->pp_ref_count), base_pp_ref + 4); KUNIT_EXPECT_EQ(test, page_ref_count(pp_page), base_pp_page_ref); @@ -1292,17 +1288,14 @@ static void netmem_ref_test_skb_segment(struct kunit *test) KUNIT_EXPECT_EQ(test, atomic_long_read(&pp_page->pp_ref_count), base_pp_ref); - /* 3. Mixed GRO frag_list (head pp_recycle=1 + list pp_recycle=0): - * straddling segment linearizes its pp_recycle=1 prefix before - * attaching the pp_recycle=0 frag so no nskb mixes incompatible - * fragment refcount types; non-straddling segments stay zero-copy SG. + /* 3. Mixed GRO frag_list (head PP page + list non-PP page): + * straddling segment stays zero-copy SG and refs each frag by netmem + * type (pp_ref_count for pp_page, page_ref_count for reg_page). */ head = alloc_skb(64, GFP_KERNEL); list = alloc_skb(0, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, head); KUNIT_ASSERT_NOT_NULL(test, list); - head->pp_recycle = 1; - list->pp_recycle = 0; head->protocol = htons(ETH_P_IP); skb_reset_mac_header(head); skb_reset_network_header(head); @@ -1323,11 +1316,8 @@ static void netmem_ref_test_skb_segment(struct kunit *test) segs = skb_segment(head, NETIF_F_SG | NETIF_F_HW_CSUM); KUNIT_ASSERT_FALSE(test, IS_ERR_OR_NULL(segs)); - KUNIT_EXPECT_TRUE(test, segs->pp_recycle); - KUNIT_EXPECT_FALSE(test, segs->next->pp_recycle); - KUNIT_EXPECT_FALSE(test, segs->next->next->pp_recycle); KUNIT_EXPECT_EQ(test, atomic_long_read(&pp_page->pp_ref_count), - base_pp_ref + 2); + base_pp_ref + 3); KUNIT_EXPECT_EQ(test, page_ref_count(pp_page), base_pp_page_ref); KUNIT_EXPECT_EQ(test, page_ref_count(reg_page), base_reg_ref + 3); kfree_skb_list(segs); @@ -1337,16 +1327,14 @@ static void netmem_ref_test_skb_segment(struct kunit *test) KUNIT_EXPECT_EQ(test, page_ref_count(pp_page), base_pp_page_ref); KUNIT_EXPECT_EQ(test, page_ref_count(reg_page), base_reg_ref); - /* 4. Reverse mixed frag_list (head pp_recycle=0 + list pp_recycle=1): - * Crossing from pp_recycle=0 head into pp_recycle=1 list_skb must not - * elevate page_ref_count(pp_page) instead of pp_ref_count. + /* 4. Reverse mixed frag_list (head non-PP page + list PP page): + * Crossing from non-PP head into PP list_skb increments pp_ref_count + * on pp_page without elevating page_ref_count(pp_page). */ head = alloc_skb(64, GFP_KERNEL); list = alloc_skb(0, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, head); KUNIT_ASSERT_NOT_NULL(test, list); - head->pp_recycle = 0; - list->pp_recycle = 1; head->protocol = htons(ETH_P_IP); skb_reset_mac_header(head); skb_reset_network_header(head); @@ -1367,13 +1355,10 @@ static void netmem_ref_test_skb_segment(struct kunit *test) segs = skb_segment(head, NETIF_F_SG | NETIF_F_HW_CSUM); KUNIT_ASSERT_FALSE(test, IS_ERR_OR_NULL(segs)); - KUNIT_EXPECT_FALSE(test, segs->pp_recycle); - KUNIT_EXPECT_TRUE(test, segs->next->pp_recycle); - KUNIT_EXPECT_TRUE(test, segs->next->next->pp_recycle); KUNIT_EXPECT_EQ(test, atomic_long_read(&pp_page->pp_ref_count), base_pp_ref + 3); KUNIT_EXPECT_EQ(test, page_ref_count(pp_page), base_pp_page_ref); - KUNIT_EXPECT_EQ(test, page_ref_count(reg_page), base_reg_ref + 2); + KUNIT_EXPECT_EQ(test, page_ref_count(reg_page), base_reg_ref + 3); kfree_skb_list(segs); consume_skb(head); KUNIT_EXPECT_EQ(test, atomic_long_read(&pp_page->pp_ref_count), @@ -1381,10 +1366,10 @@ static void netmem_ref_test_skb_segment(struct kunit *test) KUNIT_EXPECT_EQ(test, page_ref_count(pp_page), base_pp_page_ref); KUNIT_EXPECT_EQ(test, page_ref_count(reg_page), base_reg_ref); - /* 5. Linear head_skb (nr_frags=0) + mixed pp_recycle frag_list where - * member A has linear + frag (pp_recycle=1) and tail member B has - * linear len == skb_headlen(A) (pp_recycle=0): keeps SG clone path - * synchronized without hitting BUG_ON(i >= nfrags). + /* 5. Linear head_skb (nr_frags=0) + mixed frag_list where member A has + * linear + frag (PP page) and tail member B has linear len == + * skb_headlen(A): keeps SG clone path synchronized without hitting + * BUG_ON(i >= nfrags). */ head = alloc_skb(64, GFP_KERNEL); list = alloc_skb(64, GFP_KERNEL); @@ -1392,9 +1377,6 @@ static void netmem_ref_test_skb_segment(struct kunit *test) KUNIT_ASSERT_NOT_NULL(test, head); KUNIT_ASSERT_NOT_NULL(test, list); KUNIT_ASSERT_NOT_NULL(test, seg); - head->pp_recycle = 0; - list->pp_recycle = 1; - seg->pp_recycle = 0; head->protocol = htons(ETH_P_IP); skb_reset_mac_header(head); skb_reset_network_header(head); @@ -1413,9 +1395,6 @@ static void netmem_ref_test_skb_segment(struct kunit *test) segs = skb_segment(head, NETIF_F_SG | NETIF_F_HW_CSUM); KUNIT_ASSERT_FALSE(test, IS_ERR_OR_NULL(segs)); - KUNIT_EXPECT_FALSE(test, segs->pp_recycle); - KUNIT_EXPECT_TRUE(test, segs->next->pp_recycle); - KUNIT_EXPECT_FALSE(test, segs->next->next->pp_recycle); KUNIT_EXPECT_EQ(test, atomic_long_read(&pp_page->pp_ref_count), base_pp_ref + 2); KUNIT_EXPECT_EQ(test, page_ref_count(pp_page), base_pp_page_ref); @@ -1430,7 +1409,6 @@ static void netmem_ref_test_skb_segment(struct kunit *test) */ head = build_skb(page_address(pp_page), PAGE_SIZE); KUNIT_ASSERT_NOT_NULL(test, head); - head->pp_recycle = 1; skb_put(head, 32); page_pool_ref_netmem(pp_nm); skb_fill_netmem_desc(head, 0, niov_nm, 0, 32); @@ -1456,7 +1434,6 @@ static void netmem_ref_test_skb_segment(struct kunit *test) */ head = build_skb(page_address(pp_page), PAGE_SIZE); KUNIT_ASSERT_NOT_NULL(test, head); - head->pp_recycle = 1; skb_put(head, 32); page_pool_ref_netmem(pp_nm); skb_zcopy_init(head, &uarg); @@ -1529,14 +1506,12 @@ static void netmem_ref_test_try_coalesce(struct kunit *test) base_reg_ref = page_ref_count(reg_page); base_iov_ref = atomic_long_read(&binding.ref.data->count); - /* 1. Uncloned pp_recycle=1 from: frags stolen without extra ref */ + /* 1. Uncloned from with PP page: frags stolen without extra ref */ to = alloc_skb(0, GFP_KERNEL); from = alloc_skb(0, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, to); KUNIT_ASSERT_NOT_NULL(test, from); skb_put(to, skb_tailroom(to)); - to->pp_recycle = 1; - from->pp_recycle = 1; skb_add_rx_frag(from, 0, pp_page, 0, 64, PAGE_SIZE); page_pool_ref_netmem(pp_nm); KUNIT_EXPECT_TRUE(test, skb_try_coalesce(to, from, &fragstolen, @@ -1549,7 +1524,7 @@ static void netmem_ref_test_try_coalesce(struct kunit *test) KUNIT_EXPECT_EQ(test, atomic_long_read(&pp_page->pp_ref_count), base_pp_ref); - /* 2. Cloned pp_recycle=1 from (pp_page + non-PP reg_page fallback): + /* 2. Cloned from with mixed pp_page + non-PP reg_page: * skb_frag_ref(from, i) increments pp_ref_count for pp_page and * page_ref_count for reg_page. */ @@ -1558,8 +1533,6 @@ static void netmem_ref_test_try_coalesce(struct kunit *test) KUNIT_ASSERT_NOT_NULL(test, to); KUNIT_ASSERT_NOT_NULL(test, from); skb_put(to, skb_tailroom(to)); - to->pp_recycle = 1; - from->pp_recycle = 1; skb_add_rx_frag(from, 0, pp_page, 0, 32, PAGE_SIZE); page_pool_ref_netmem(pp_nm); skb_add_rx_frag(from, 1, reg_page, 0, 32, PAGE_SIZE); @@ -1581,7 +1554,7 @@ static void netmem_ref_test_try_coalesce(struct kunit *test) KUNIT_EXPECT_EQ(test, page_ref_count(pp_page), base_pp_page_ref); KUNIT_EXPECT_EQ(test, page_ref_count(reg_page), base_reg_ref); - /* 3. Cloned unreadable pp_recycle=1 net_iov skb: skb_try_coalesce + /* 3. Cloned unreadable PP net_iov skb: skb_try_coalesce * increments niov.desc.pp_ref_count via skb_frag_ref(from, i). */ to = alloc_skb(0, GFP_KERNEL); @@ -1589,9 +1562,7 @@ static void netmem_ref_test_try_coalesce(struct kunit *test) KUNIT_ASSERT_NOT_NULL(test, to); KUNIT_ASSERT_NOT_NULL(test, from); skb_put(to, skb_tailroom(to)); - to->pp_recycle = 1; to->unreadable = 1; - from->pp_recycle = 1; skb_add_rx_frag_netmem(from, 0, niov_nm, 0, 64, PAGE_SIZE); napi_pp_get_page(niov_nm); @@ -1674,7 +1645,6 @@ static void netmem_ref_test_tcp_devmem_recv(struct kunit *test) /* 1. Receive via tcp_recvmsg_dmabuf -> SO_DEVMEM_DONTNEED release */ skb = alloc_skb(0, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, skb); - skb->pp_recycle = 1; skb_add_rx_frag_netmem(skb, 0, niov_nm, 0, 64, PAGE_SIZE); napi_pp_get_page(niov_nm); TCP_SKB_CB(skb)->seq = 0; @@ -1705,7 +1675,6 @@ static void netmem_ref_test_tcp_devmem_recv(struct kunit *test) */ skb = alloc_skb(0, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, skb); - skb->pp_recycle = 1; skb_add_rx_frag_netmem(skb, 0, niov_nm, 0, 64, PAGE_SIZE); napi_pp_get_page(niov_nm); TCP_SKB_CB(skb)->seq = 64; @@ -1738,6 +1707,7 @@ static struct kunit_case net_test_cases[] = { KUNIT_CASE(netmem_ref_test_page), #ifdef CONFIG_PAGE_POOL KUNIT_CASE(netmem_ref_test_pp_page), + KUNIT_CASE(netmem_ref_test_tcp_zerocopy_receive_pp), #endif #if defined(CONFIG_PAGE_POOL) && defined(CONFIG_NET_DEVMEM) KUNIT_CASE(netmem_ref_test_net_iov), diff --git a/net/core/netmem_priv.h b/net/core/netmem_priv.h index 23175cb2bd866..e229a1ac87a30 100644 --- a/net/core/netmem_priv.h +++ b/net/core/netmem_priv.h @@ -3,21 +3,30 @@ #ifndef __NETMEM_PRIV_H #define __NETMEM_PRIV_H +/* Use READ_ONCE() because every netmem_inc/dec_ref() checks netmem_is_pp(), + * and pp_magic shares a struct page union word with folio->lru.next which MM + * code may concurrently modify on page-cache folios in TX SKBs. + */ static inline unsigned long netmem_get_pp_magic(netmem_ref netmem) { - return netmem_to_nmdesc(netmem)->pp_magic & ~PP_DMA_INDEX_MASK; + return READ_ONCE(netmem_to_nmdesc(netmem)->pp_magic) & + ~PP_DMA_INDEX_MASK; } static inline void netmem_or_pp_magic(netmem_ref netmem, unsigned long pp_magic) { - netmem_to_nmdesc(netmem)->pp_magic |= pp_magic; + struct netmem_desc *nmdesc = netmem_to_nmdesc(netmem); + + WRITE_ONCE(nmdesc->pp_magic, READ_ONCE(nmdesc->pp_magic) | pp_magic); } static inline void netmem_clear_pp_magic(netmem_ref netmem) { - WARN_ON_ONCE(netmem_to_nmdesc(netmem)->pp_magic & PP_DMA_INDEX_MASK); + struct netmem_desc *nmdesc = netmem_to_nmdesc(netmem); + + WARN_ON_ONCE(READ_ONCE(nmdesc->pp_magic) & PP_DMA_INDEX_MASK); - netmem_to_nmdesc(netmem)->pp_magic = 0; + WRITE_ONCE(nmdesc->pp_magic, 0); } static inline bool netmem_is_pp(netmem_ref netmem) @@ -43,7 +52,7 @@ static inline unsigned long netmem_get_dma_index(netmem_ref netmem) if (WARN_ON_ONCE(netmem_is_net_iov(netmem))) return 0; - magic = netmem_to_nmdesc(netmem)->pp_magic; + magic = READ_ONCE(netmem_to_nmdesc(netmem)->pp_magic); return (magic & PP_DMA_INDEX_MASK) >> PP_DMA_INDEX_SHIFT; } @@ -57,6 +66,6 @@ static inline void netmem_set_dma_index(netmem_ref netmem, return; magic = netmem_get_pp_magic(netmem) | (id << PP_DMA_INDEX_SHIFT); - netmem_to_nmdesc(netmem)->pp_magic = magic; + WRITE_ONCE(netmem_to_nmdesc(netmem)->pp_magic, magic); } #endif diff --git a/net/core/page_pool.c b/net/core/page_pool.c index d7c88c0b67e53..33c625b850e33 100644 --- a/net/core/page_pool.c +++ b/net/core/page_pool.c @@ -825,9 +825,9 @@ static bool page_pool_recycle_in_cache(netmem_ref netmem, static bool __page_pool_page_can_be_recycled(netmem_ref netmem) { - return netmem_is_net_iov(netmem) || - (page_ref_count(netmem_to_page(netmem)) == 1 && - !page_is_pfmemalloc(netmem_to_page(netmem))); + return page_pool_netmem_is_exclusively_owned(netmem) && + (netmem_is_net_iov(netmem) || + !page_is_pfmemalloc(__netmem_to_page(netmem))); } /* If the page refcnt == 1, this will try to recycle the page. diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 19d4ce347c41b..a695c3f64dfd9 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1056,6 +1056,13 @@ EXPORT_SYMBOL(napi_pp_put_page); * Acquires a page_pool reference (pp_ref_count) on @netmem if it belongs to a * page_pool. Counterpart to napi_pp_put_page(). * + * If @netmem belongs to a page_pool, the caller MUST already hold an active + * pp_ref_count (>= 1) on @netmem (e.g. via an existing skb_frag_t in + * skb_frag_ref() or a spliced pipe buffer in skb_pipe_buf_ops). A raw + * page_ref_count is never converted into a pp_ref_count (in particular, + * tcp_zerocopy_receive() detaches page_pool pages before mapping them into + * userspace so GUP callers never observe page_pool pages). + * * Return: true if @netmem belongs to a page_pool, false otherwise. */ bool napi_pp_get_page(netmem_ref netmem) @@ -1072,9 +1079,9 @@ bool napi_pp_get_page(netmem_ref netmem) EXPORT_SYMBOL(napi_pp_get_page); #endif -static bool skb_pp_recycle(struct sk_buff *skb, void *data) +static bool skb_pp_recycle(void *data) { - if (!IS_ENABLED(CONFIG_PAGE_POOL) || !skb->pp_recycle) + if (!IS_ENABLED(CONFIG_PAGE_POOL)) return false; return napi_pp_put_page(page_to_netmem(virt_to_page(data))); } @@ -1089,7 +1096,7 @@ static void skb_free_head(struct sk_buff *skb) unsigned char *head = skb->head; if (skb->head_frag) { - if (skb_pp_recycle(skb, head)) + if (skb_pp_recycle(head)) return; skb_free_frag(head); } else { @@ -1114,8 +1121,7 @@ static void skb_release_data(struct sk_buff *skb, enum skb_drop_reason reason) } for (i = 0; i < shinfo->nr_frags; i++) - skb_netmem_unref(skb_frag_netmem(&shinfo->frags[i]), - skb->pp_recycle); + netmem_dec_ref(skb_frag_netmem(&shinfo->frags[i])); free_head: if (shinfo->frag_list) @@ -1613,7 +1619,6 @@ static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb) n->nohdr = 0; n->peeked = 0; C(pfmemalloc); - C(pp_recycle); n->destructor = NULL; C(alloc_cpu); C(tail); @@ -2251,7 +2256,6 @@ struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom, n = NULL; goto out; } - n->pp_recycle = skb->pp_recycle; for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i]; skb_frag_ref(n, i); @@ -3941,7 +3945,6 @@ skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen) if (len <= skb_tailroom(to)) return skb_copy_bits(from, 0, skb_put(to, len), len); - to->pp_recycle = from->pp_recycle; if (hlen) { ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen); if (unlikely(ret)) @@ -4314,7 +4317,6 @@ void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len) skb_shinfo(skb1)->flags |= skb_shinfo(skb)->flags & zc_flags; skb_zerocopy_clone(skb1, skb, 0); - skb1->pp_recycle = skb->pp_recycle; if (len < pos) /* Split line is inside header. */ skb_split_inside_header(skb, skb1, len, pos); else /* Second chunk has no header, nothing to copy. */ @@ -4364,7 +4366,6 @@ int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen) if (skb_zcopy(tgt) || skb_zcopy(skb)) return 0; - DEBUG_NET_WARN_ON_ONCE(tgt->pp_recycle != skb->pp_recycle); DEBUG_NET_WARN_ON_ONCE(skb_cmp_decrypted(tgt, skb)); todo = shiftlen; @@ -5108,16 +5109,6 @@ struct sk_buff *skb_segment(struct sk_buff *head_skb, while (pos < offset + len) { if (i >= nfrags) { - if (skb_shinfo(nskb)->nr_frags && - nskb->pp_recycle != list_skb->pp_recycle) { - nskb->data_len = pos - offset - hsize; - nskb->len += nskb->data_len; - nskb->truesize += nskb->data_len; - if (__skb_linearize(nskb)) - goto err; - hsize = pos - offset; - nskb_frag = skb_shinfo(nskb)->frags; - } if (skb_orphan_frags(list_skb, GFP_ATOMIC) || skb_zerocopy_clone(nskb, list_skb, GFP_ATOMIC)) @@ -5160,8 +5151,6 @@ struct sk_buff *skb_segment(struct sk_buff *head_skb, goto err; } - if (!skb_shinfo(nskb)->nr_frags) - nskb->pp_recycle = frag_skb->pp_recycle; *nskb_frag = (i < 0) ? skb_head_frag_to_page_desc(frag_skb) : *frag; skb_frag_ref(nskb, skb_shinfo(nskb)->nr_frags); size = skb_frag_size(nskb_frag); @@ -6311,15 +6300,6 @@ bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, if (skb_cloned(to)) return false; - /* In general, avoid mixing page_pool and non-page_pool allocated - * pages within the same SKB. In theory we could take full - * references if @from is cloned and !@to->pp_recycle but its - * tricky (due to potential race with the clone disappearing) and - * rare, so not worth dealing with. - */ - if (to->pp_recycle != from->pp_recycle) - return false; - /* All frags in an skb must either all be struct page-backed or all * belong to the same memory provider instance; do not coalesce skbs * that would mix them. diff --git a/net/core/xdp.c b/net/core/xdp.c index 1d679e8fd6493..6f2b9191af9ec 100644 --- a/net/core/xdp.c +++ b/net/core/xdp.c @@ -445,7 +445,7 @@ void __xdp_return(netmem_ref netmem, enum xdp_mem_type mem_type, napi_direct); break; case MEM_TYPE_PAGE_SHARED: - page_frag_free(__netmem_address(netmem)); + netmem_dec_ref(netmem); break; case MEM_TYPE_PAGE_ORDER0: put_page(__netmem_to_page(netmem)); diff --git a/net/ipv4/esp4.c b/net/ipv4/esp4.c index ae701561d9f12..1c37028146c98 100644 --- a/net/ipv4/esp4.c +++ b/net/ipv4/esp4.c @@ -96,7 +96,7 @@ static inline struct scatterlist *esp_req_sg(struct crypto_aead *aead, __alignof__(struct scatterlist)); } -static void esp_ssg_unref(struct xfrm_state *x, void *tmp, struct sk_buff *skb, bool already_unref) +static void esp_ssg_unref(struct xfrm_state *x, void *tmp, bool already_unref) { struct crypto_aead *aead = x->data; int extralen = 0; @@ -117,8 +117,7 @@ static void esp_ssg_unref(struct xfrm_state *x, void *tmp, struct sk_buff *skb, struct scatterlist *src = already_unref ? esp_req_sg(aead, req) : req->src; for (sg = sg_next(src); sg; sg = sg_next(sg)) - skb_netmem_unref(page_to_netmem(sg_page(sg)), - skb->pp_recycle); + netmem_dec_ref(page_to_netmem(sg_page(sg))); } } @@ -223,7 +222,7 @@ static void esp_output_done(void *data, int err) } tmp = ESP_SKB_CB(skb)->tmp; - esp_ssg_unref(x, tmp, skb, false); + esp_ssg_unref(x, tmp, false); kfree(tmp); if (xo && (xo->flags & XFRM_DEV_RESUME)) { @@ -579,7 +578,7 @@ int esp_output_tail(struct xfrm_state *x, struct sk_buff *skb, struct esp_info * (unsigned char *)esph - skb->data, assoclen + ivlen + esp->clen + alen); if (unlikely(err < 0)) { - esp_ssg_unref(x, tmp, skb, true); + esp_ssg_unref(x, tmp, true); goto error_free; } } @@ -613,7 +612,7 @@ int esp_output_tail(struct xfrm_state *x, struct sk_buff *skb, struct esp_info * } if (sg != dsg) - esp_ssg_unref(x, tmp, skb, false); + esp_ssg_unref(x, tmp, false); if (!err && x->encap && x->encap->encap_type == TCP_ENCAP_ESPINTCP) err = esp_output_tail_tcp(x, skb); diff --git a/net/ipv6/esp6.c b/net/ipv6/esp6.c index d654f0f442388..25dd0c01f908d 100644 --- a/net/ipv6/esp6.c +++ b/net/ipv6/esp6.c @@ -113,7 +113,7 @@ static inline struct scatterlist *esp_req_sg(struct crypto_aead *aead, __alignof__(struct scatterlist)); } -static void esp_ssg_unref(struct xfrm_state *x, void *tmp, struct sk_buff *skb, bool already_unref) +static void esp_ssg_unref(struct xfrm_state *x, void *tmp, bool already_unref) { struct crypto_aead *aead = x->data; int extralen = 0; @@ -134,8 +134,7 @@ static void esp_ssg_unref(struct xfrm_state *x, void *tmp, struct sk_buff *skb, struct scatterlist *src = already_unref ? esp_req_sg(aead, req) : req->src; for (sg = sg_next(src); sg; sg = sg_next(sg)) - skb_netmem_unref(page_to_netmem(sg_page(sg)), - skb->pp_recycle); + netmem_dec_ref(page_to_netmem(sg_page(sg))); } } @@ -258,7 +257,7 @@ static void esp_output_done(void *data, int err) } tmp = ESP_SKB_CB(skb)->tmp; - esp_ssg_unref(x, tmp, skb, false); + esp_ssg_unref(x, tmp, false); kfree(tmp); esp_output_encap_csum(skb); @@ -611,7 +610,7 @@ int esp6_output_tail(struct xfrm_state *x, struct sk_buff *skb, struct esp_info (unsigned char *)esph - skb->data, assoclen + ivlen + esp->clen + alen); if (unlikely(err < 0)) { - esp_ssg_unref(x, tmp, skb, true); + esp_ssg_unref(x, tmp, true); goto error_free; } } @@ -646,7 +645,7 @@ int esp6_output_tail(struct xfrm_state *x, struct sk_buff *skb, struct esp_info } if (sg != dsg) - esp_ssg_unref(x, tmp, skb, false); + esp_ssg_unref(x, tmp, false); if (!err && x->encap && x->encap->encap_type == TCP_ENCAP_ESPINTCP) err = esp_output_tail_tcp(x, skb); diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c index 05abf8e23f78a..3aa1557f107dd 100644 --- a/net/tls/tls_device.c +++ b/net/tls/tls_device.c @@ -143,7 +143,7 @@ static void destroy_record(struct tls_record_info *record) int i; for (i = 0; i < record->num_frags; i++) - netmem_dec_ref(skb_frag_netmem(&record->frags[i])); + put_page(skb_frag_page(&record->frags[i])); kfree(record); } diff --git a/net/tls/tls_device_fallback.c b/net/tls/tls_device_fallback.c index b547027cc879f..7d26546f11eb5 100644 --- a/net/tls/tls_device_fallback.c +++ b/net/tls/tls_device_fallback.c @@ -256,7 +256,7 @@ static int fill_sg_in(struct scatterlist *sg_in, for (i = 0; remaining > 0; i++) { skb_frag_t *frag = &record->frags[i]; - netmem_inc_ref(skb_frag_netmem(frag)); + get_page(skb_frag_page(frag)); sg_set_page(sg_in + i, skb_frag_page(frag), skb_frag_size(frag), skb_frag_off(frag)); diff --git a/net/wireless/util.c b/net/wireless/util.c index 59912f4a93c88..493d7e4a58617 100644 --- a/net/wireless/util.c +++ b/net/wireless/util.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -768,19 +769,25 @@ int ieee80211_data_to_8023_exthdr(struct sk_buff *skb, struct ethhdr *ehdr, } EXPORT_SYMBOL(ieee80211_data_to_8023_exthdr); -static void +static bool __frame_add_frag(struct sk_buff *skb, struct page *page, void *ptr, int len, int size) { struct skb_shared_info *sh = skb_shinfo(skb); + int i = sh->nr_frags; int page_offset; - get_page(page); + if (i >= MAX_SKB_FRAGS) + return false; + page_offset = ptr - page_address(page); - skb_add_rx_frag(skb, sh->nr_frags, page, page_offset, len, size); + skb_add_rx_frag(skb, i, page, page_offset, len, size); + skb_frag_ref(skb, i); + + return true; } -static void +static bool __ieee80211_amsdu_copy_frag(struct sk_buff *skb, struct sk_buff *frame, int offset, int len) { @@ -809,17 +816,22 @@ __ieee80211_amsdu_copy_frag(struct sk_buff *skb, struct sk_buff *frame, cur_len = min(len, frag_len); - __frame_add_frag(frame, frag_page, frag_ptr, cur_len, frag_size); + if (!__frame_add_frag(frame, frag_page, frag_ptr, cur_len, frag_size)) + return false; len -= cur_len; while (len > 0) { frag_len = skb_frag_size(frag); cur_len = min(len, frag_len); - __frame_add_frag(frame, skb_frag_page(frag), - skb_frag_address(frag), cur_len, frag_len); + if (!__frame_add_frag(frame, skb_frag_page(frag), + skb_frag_address(frag), cur_len, + frag_len)) + return false; len -= cur_len; frag++; } + + return true; } static struct sk_buff * @@ -858,7 +870,10 @@ __ieee80211_amsdu_copy(struct sk_buff *skb, unsigned int hlen, return frame; offset += cur_len; - __ieee80211_amsdu_copy_frag(skb, frame, offset, len); + if (!__ieee80211_amsdu_copy_frag(skb, frame, offset, len)) { + kfree_skb(frame); + return NULL; + } return frame; } diff --git a/net/xfrm/trace_iptfs.h b/net/xfrm/trace_iptfs.h index 74391ba244453..ecfd3f1f36d1f 100644 --- a/net/xfrm/trace_iptfs.h +++ b/net/xfrm/trace_iptfs.h @@ -167,7 +167,6 @@ TRACE_EVENT(iptfs_ingress_nth_add, TP_PROTO(struct sk_buff *skb, u8 share_ok), __field(u32, data_len) __field(u8, share_ok) __field(u8, head_frag) - __field(u8, pp_recycle) __field(u8, cloned) __field(u8, shared) __field(u8, nr_frags) @@ -178,15 +177,14 @@ TRACE_EVENT(iptfs_ingress_nth_add, TP_PROTO(struct sk_buff *skb, u8 share_ok), __entry->data_len = skb->data_len; __entry->share_ok = share_ok; __entry->head_frag = skb->head_frag; - __entry->pp_recycle = skb->pp_recycle; __entry->cloned = skb_cloned(skb); __entry->shared = skb_shared(skb); __entry->nr_frags = skb_shinfo(skb)->nr_frags; __entry->frag_list = (bool)skb_shinfo(skb)->frag_list; ), - TP_printk("INGRPSTQ: NTHADD: skb=%p len=%u data_len=%u share_ok=%u head_frag=%u pp_recycle=%u cloned=%u shared=%u nr_frags=%u frag_list=%u", + TP_printk("INGRPSTQ: NTHADD: skb=%p len=%u data_len=%u share_ok=%u head_frag=%u cloned=%u shared=%u nr_frags=%u frag_list=%u", __entry->skb, __entry->skb_len, __entry->data_len, __entry->share_ok, - __entry->head_frag, __entry->pp_recycle, __entry->cloned, __entry->shared, + __entry->head_frag, __entry->cloned, __entry->shared, __entry->nr_frags, __entry->frag_list)); DECLARE_EVENT_CLASS(iptfs_timer_event, diff --git a/net/xfrm/xfrm_iptfs.c b/net/xfrm/xfrm_iptfs.c index 04ef833074fc8..f441aa54dcf1f 100644 --- a/net/xfrm/xfrm_iptfs.c +++ b/net/xfrm/xfrm_iptfs.c @@ -310,7 +310,6 @@ static void iptfs_skb_head_to_frag(const struct sk_buff *skb, skb_frag_t *frag) * @nr_frags: number of fragments present in array * @initial_offset: the value passed in to skb_prepare_frag_walk() * @frags: the page fragments inc. room for head page - * @pp_recycle: copy of skb->pp_recycle */ struct iptfs_skb_frag_walk { u32 fragi; @@ -319,7 +318,6 @@ struct iptfs_skb_frag_walk { u32 nr_frags; u32 initial_offset; skb_frag_t frags[MAX_SKB_FRAGS + 1]; - bool pp_recycle; }; /** @@ -343,7 +341,6 @@ static void iptfs_skb_prepare_frag_walk(struct sk_buff *skb, u32 initial_offset, walk->past = 0; walk->total = 0; walk->nr_frags = 0; - walk->pp_recycle = skb->pp_recycle; if (skb->head_frag) { if (initial_offset >= skb_headlen(skb)) { @@ -413,7 +410,7 @@ static bool iptfs_skb_can_add_frags(const struct sk_buff *skb, struct skb_shared_info *shinfo = skb_shinfo(skb); u32 fragi, nr_frags, fraglen; - if (skb_has_frag_list(skb) || skb->pp_recycle != walk->pp_recycle) + if (skb_has_frag_list(skb)) return false; /* Reject an @offset that is at or beyond the end of the walk's data @@ -898,8 +895,7 @@ static u32 iptfs_reassem_cont(struct xfrm_iptfs_data *xtfs, u64 seq, /* If we may have the opportunity to share prepare a fragwalk. */ if (!skb_has_frag_list(skb) && !skb_has_frag_list(newskb) && - (skb->head_frag || skb->len == skb->data_len) && - skb->pp_recycle == newskb->pp_recycle) { + (skb->head_frag || skb->len == skb->data_len)) { fragwalk = &_fragwalk; iptfs_skb_prepare_frag_walk(skb, data, fragwalk); } @@ -2289,15 +2285,10 @@ static void iptfs_output_queued(struct xfrm_state *x, struct sk_buff_head *list) } } - /* skb->pp_recycle is passed to __skb_flag_unref for all - * frag pages so we can only share pages with skb's who - * match ourselves. - */ shi2 = skb_shinfo(skb2); if (share_ok && (shi2->frag_list || (!skb2->head_frag && skb_headlen(skb)) || - skb->pp_recycle != skb2->pp_recycle || skb_zcopy(skb2) || (shi->nr_frags + shi2->nr_frags + 1 > MAX_SKB_FRAGS))) share_ok = false; -- 2.56.0.455.g6c5a503b38-goog Now that page_pool reference counting is dispatched directly from each netmem's type and skb->pp_recycle has been removed, skb_mark_for_recycle() is a no-op stub with no remaining purpose. Remove its definition, all in-tree callers, and its reference in Documentation/networking/page_pool.rst. Signed-off-by: Mina Almasry --- Documentation/networking/page_pool.rst | 1 - drivers/net/ethernet/airoha/airoha_eth.c | 1 - drivers/net/ethernet/alibaba/eea/eea_rx.c | 3 --- drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c | 1 - drivers/net/ethernet/broadcom/bnge/bnge_txrx.c | 5 ----- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 8 -------- drivers/net/ethernet/broadcom/genet/bcmgenet.c | 2 -- drivers/net/ethernet/cisco/enic/enic_rq.c | 1 - drivers/net/ethernet/engleder/tsnep_main.c | 2 -- drivers/net/ethernet/freescale/fec_main.c | 1 - drivers/net/ethernet/google/gve/gve_rx_dqo.c | 9 --------- drivers/net/ethernet/hisilicon/hibmcge/hbg_txrx.c | 1 - drivers/net/ethernet/hisilicon/hns3/hns3_enet.c | 6 ------ drivers/net/ethernet/huawei/hinic3/hinic3_rx.c | 1 - drivers/net/ethernet/intel/iavf/iavf_txrx.c | 2 -- drivers/net/ethernet/marvell/mvneta.c | 2 -- drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c | 2 -- .../net/ethernet/marvell/octeontx2/nic/otx2_txrx.c | 1 - drivers/net/ethernet/mediatek/mtk_eth_soc.c | 1 - drivers/net/ethernet/mellanox/mlx4/en_rx.c | 1 - drivers/net/ethernet/mellanox/mlx5/core/en_rx.c | 8 -------- drivers/net/ethernet/mellanox/mlxsw/pci.c | 2 -- drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c | 2 -- .../ethernet/microchip/sparx5/lan969x/lan969x_fdma.c | 1 - drivers/net/ethernet/microsoft/mana/mana_en.c | 3 --- drivers/net/ethernet/mucse/rnpgbe/rnpgbe_lib.c | 1 - drivers/net/ethernet/pensando/ionic/ionic_txrx.c | 2 -- drivers/net/ethernet/realtek/rtase/rtase_main.c | 1 - drivers/net/ethernet/renesas/ravb_main.c | 2 -- drivers/net/ethernet/socionext/netsec.c | 1 - drivers/net/ethernet/stmicro/stmmac/stmmac_main.c | 1 - drivers/net/ethernet/tehuti/tn40.c | 1 - drivers/net/ethernet/ti/am65-cpsw-nuss.c | 1 - drivers/net/ethernet/ti/cpsw.c | 2 -- drivers/net/ethernet/ti/cpsw_new.c | 2 -- drivers/net/ethernet/ti/icssg/icssg_common.c | 1 - drivers/net/ethernet/wangxun/libwx/wx_lib.c | 2 -- drivers/net/virtio_net.c | 8 +------- drivers/net/vmxnet3/vmxnet3_xdp.c | 1 - drivers/net/wireless/mediatek/mt76/dma.c | 1 - drivers/net/wireless/mediatek/mt76/mt7996/mac.c | 1 - drivers/net/wireless/mediatek/mt76/npu.c | 1 - drivers/net/wireless/mediatek/mt76/usb.c | 1 - drivers/net/xen-netfront.c | 1 - include/linux/skbuff.h | 5 ----- include/net/page_pool/helpers.h | 5 ++--- net/core/skbuff.c | 1 - net/core/xdp.c | 11 ++--------- 48 files changed, 5 insertions(+), 115 deletions(-) diff --git a/Documentation/networking/page_pool.rst b/Documentation/networking/page_pool.rst index d0be2b800a060..5b2d6d011fc1b 100644 --- a/Documentation/networking/page_pool.rst +++ b/Documentation/networking/page_pool.rst @@ -166,7 +166,6 @@ NAPI poller if XDP_DROP: page_pool_recycle_direct(page_pool, page); } else (packet_is_skb) { - skb_mark_for_recycle(skb); new_page = page_pool_dev_alloc_pages(page_pool); } } diff --git a/drivers/net/ethernet/airoha/airoha_eth.c b/drivers/net/ethernet/airoha/airoha_eth.c index b164a78ddac9d..980b8c1d0dc49 100644 --- a/drivers/net/ethernet/airoha/airoha_eth.c +++ b/drivers/net/ethernet/airoha/airoha_eth.c @@ -925,7 +925,6 @@ static int airoha_qdma_rx_process(struct airoha_queue *q, int budget) skb_reserve(q->skb, AIROHA_RX_HEADROOM); __skb_put(q->skb, len); - skb_mark_for_recycle(q->skb); q->skb->dev = netdev; q->skb->ip_summed = CHECKSUM_UNNECESSARY; skb_record_rx_queue(q->skb, qid); diff --git a/drivers/net/ethernet/alibaba/eea/eea_rx.c b/drivers/net/ethernet/alibaba/eea/eea_rx.c index 5dfb7cb2ba6d2..27feb9cdedd7e 100644 --- a/drivers/net/ethernet/alibaba/eea/eea_rx.c +++ b/drivers/net/ethernet/alibaba/eea/eea_rx.c @@ -319,8 +319,6 @@ static struct sk_buff *eea_rx_build_split_hdr_skb(struct eea_net_rx *rx, skb_add_rx_frag(skb, 0, page, offset, ctx->len, truesize); } - skb_mark_for_recycle(skb); - return skb; } @@ -345,7 +343,6 @@ static struct sk_buff *eea_rx_build_skb(struct eea_net_rx *rx, return NULL; truesize = eea_consume_rx_buffer(rx, meta, bufsize); - skb_mark_for_recycle(skb); skb->truesize += truesize - bufsize; diff --git a/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c b/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c index f09054a02feb6..005476e150710 100644 --- a/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c +++ b/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c @@ -576,7 +576,6 @@ static int bcmasp_rx_poll(struct napi_struct *napi, int budget) skb_reserve(skb, NET_SKB_PAD); skb_put(skb, len); memcpy(skb->data, data, len); - skb_mark_for_recycle(skb); /* Skip the 2-byte hardware alignment pad. */ skb_pull(skb, 2); diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_txrx.c b/drivers/net/ethernet/broadcom/bnge/bnge_txrx.c index 472b02cc43b3c..de352d53628c9 100644 --- a/drivers/net/ethernet/broadcom/bnge/bnge_txrx.c +++ b/drivers/net/ethernet/broadcom/bnge/bnge_txrx.c @@ -240,7 +240,6 @@ static struct sk_buff *bnge_rx_agg_netmems_skb(struct bnge_net *bn, total_frag_len = __bnge_rx_agg_netmems(bn, cpr, idx, agg_bufs, tpa, skb); if (!total_frag_len) { - skb_mark_for_recycle(skb); dev_kfree_skb(skb); return NULL; } @@ -468,7 +467,6 @@ void bnge_reuse_rx_data(struct bnge_rx_ring_info *rxr, u16 cons, void *data) static void bnge_deliver_skb(struct bnge_net *bn, struct bnge_napi *bnapi, struct sk_buff *skb) { - skb_mark_for_recycle(skb); skb_record_rx_queue(skb, bnapi->index); napi_gro_receive(&bnapi->napi, skb); } @@ -659,7 +657,6 @@ static struct sk_buff *bnge_tpa_end(struct bnge_net *bn, bnge_abort_tpa(cpr, idx, agg_bufs); return NULL; } - skb_mark_for_recycle(skb); skb_reserve(skb, bn->rx_offset); skb_put(skb, len); } @@ -761,7 +758,6 @@ static struct sk_buff *bnge_rx_vlan(struct sk_buff *skb, u8 cmp_type, return skb; vlan_err: - skb_mark_for_recycle(skb); dev_kfree_skb(skb); return NULL; } @@ -790,7 +786,6 @@ static struct sk_buff *bnge_rx_skb(struct bnge_net *bn, return NULL; } - skb_mark_for_recycle(skb); skb_reserve(skb, bn->rx_offset); skb_put(skb, len); return skb; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 757c7266ce15d..af385c0c43b6e 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -1226,7 +1226,6 @@ static struct sk_buff *bnxt_rx_multi_page_skb(struct bnxt *bp, bnxt_reuse_rx_data(rxr, cons, data); return NULL; } - skb_mark_for_recycle(skb); skb_reserve(skb, data_ptr - (u8 *)frag_start); __skb_put(skb, len); @@ -1265,7 +1264,6 @@ static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp, return NULL; } - skb_mark_for_recycle(skb); off = (void *)data_ptr - page_address(page); skb_add_rx_frag(skb, 0, page, off, len, rxr->rx_page_size); memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN, @@ -1304,7 +1302,6 @@ static struct sk_buff *bnxt_rx_skb(struct bnxt *bp, return NULL; } - skb_mark_for_recycle(skb); skb_reserve(skb, bp->rx_offset); skb_put(skb, offset_and_len & 0xffff); return skb; @@ -1411,7 +1408,6 @@ static struct sk_buff *bnxt_rx_agg_netmems_skb(struct bnxt *bp, total_frag_len = __bnxt_rx_agg_netmems(bp, cpr, idx, agg_bufs, tpa, skb, NULL); if (!total_frag_len) { - skb_mark_for_recycle(skb); dev_kfree_skb(skb); return NULL; } @@ -2023,7 +2019,6 @@ static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp, cpr->sw_stats->rx.rx_oom_discards += 1; return NULL; } - skb_mark_for_recycle(skb); skb_reserve(skb, bp->rx_offset); skb_put(skb, len); } @@ -2088,8 +2083,6 @@ static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi, struct sk_buff *skb) { - skb_mark_for_recycle(skb); - if (skb->dev != bp->dev) { /* this packet belongs to a vf-rep */ bnxt_vf_rep_rx(bp, skb); @@ -2151,7 +2144,6 @@ static struct sk_buff *bnxt_rx_vlan(struct sk_buff *skb, u8 cmp_type, } return skb; vlan_err: - skb_mark_for_recycle(skb); dev_kfree_skb(skb); return NULL; } diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.c b/drivers/net/ethernet/broadcom/genet/bcmgenet.c index 4d2be2a21ed1b..240d2d5e4f5fe 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c @@ -2437,8 +2437,6 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring, goto next; } - skb_mark_for_recycle(skb); - /* Reserve the RSB + pad, then set the data length */ skb_reserve(skb, GENET_RSB_PAD); __skb_put(skb, len - GENET_RSB_PAD); diff --git a/drivers/net/ethernet/cisco/enic/enic_rq.c b/drivers/net/ethernet/cisco/enic/enic_rq.c index ccbf5c9a21d0f..1221a4a289294 100644 --- a/drivers/net/ethernet/cisco/enic/enic_rq.c +++ b/drivers/net/ethernet/cisco/enic/enic_rq.c @@ -369,7 +369,6 @@ static void enic_rq_indicate_buf(struct enic *enic, struct vnic_rq *rq, fcoe_fc_crc_ok, vlan_stripped, csum_not_calc, tcp_udp_csum_ok, ipv6, ipv4_csum_ok, vlan_tci, skb); - skb_mark_for_recycle(skb); napi_gro_frags(napi); if (enic->rx_coalesce_setting.use_adaptive_rx_coalesce) enic_intr_update_pkt_size(&cq->pkt_size_counter, diff --git a/drivers/net/ethernet/engleder/tsnep_main.c b/drivers/net/ethernet/engleder/tsnep_main.c index 5db04e209dc67..e0881dd53912d 100644 --- a/drivers/net/ethernet/engleder/tsnep_main.c +++ b/drivers/net/ethernet/engleder/tsnep_main.c @@ -1416,8 +1416,6 @@ static void tsnep_rx_page(struct tsnep_rx *rx, struct napi_struct *napi, skb = tsnep_build_skb(rx, page, length); if (skb) { - skb_mark_for_recycle(skb); - rx->packets++; rx->bytes += length; if (skb->pkt_type == PACKET_MULTICAST) diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c index f7636f23d1ebb..42499a269ad6c 100644 --- a/drivers/net/ethernet/freescale/fec_main.c +++ b/drivers/net/ethernet/freescale/fec_main.c @@ -1855,7 +1855,6 @@ static struct sk_buff *fec_build_skb(struct fec_enet_private *fep, skb_reserve(skb, FEC_ENET_XDP_HEADROOM + fep->rx_shift); skb_put(skb, len); - skb_mark_for_recycle(skb); /* Get offloads from the enhanced buffer descriptor */ if (fep->bufdesc_ex) { diff --git a/drivers/net/ethernet/google/gve/gve_rx_dqo.c b/drivers/net/ethernet/google/gve/gve_rx_dqo.c index 82ed5e48b00ab..3942e5d117c14 100644 --- a/drivers/net/ethernet/google/gve/gve_rx_dqo.c +++ b/drivers/net/ethernet/google/gve/gve_rx_dqo.c @@ -622,9 +622,6 @@ static int gve_rx_append_frags(struct napi_struct *napi, if (!skb) return -1; - if (rx->dqo.page_pool) - skb_mark_for_recycle(skb); - if (rx->ctx.skb_tail == rx->ctx.skb_head) skb_shinfo(rx->ctx.skb_head)->frag_list = skb; else @@ -883,9 +880,6 @@ static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx, if (unlikely(!rx->ctx.skb_head)) goto error; rx->ctx.skb_tail = rx->ctx.skb_head; - - if (rx->dqo.page_pool) - skb_mark_for_recycle(rx->ctx.skb_head); } else { unsplit = 1; } @@ -988,9 +982,6 @@ static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx, return 0; } - if (rx->dqo.page_pool) - skb_mark_for_recycle(rx->ctx.skb_head); - gve_skb_add_rx_frag(rx, buf_state, 0, buf_len); gve_reuse_buffer(rx, buf_state); return 0; diff --git a/drivers/net/ethernet/hisilicon/hibmcge/hbg_txrx.c b/drivers/net/ethernet/hisilicon/hibmcge/hbg_txrx.c index 2ea025fd964fd..3204203230795 100644 --- a/drivers/net/ethernet/hisilicon/hibmcge/hbg_txrx.c +++ b/drivers/net/ethernet/hisilicon/hibmcge/hbg_txrx.c @@ -471,7 +471,6 @@ static int hbg_build_skb(struct hbg_priv *priv, buffer->skb = napi_build_skb(buffer->page_addr, buffer->page_size); if (unlikely(!buffer->skb)) return -ENOMEM; - skb_mark_for_recycle(buffer->skb); /* page will be freed together with the skb */ buffer->page = NULL; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c index 47788be64be68..ed6e75b45c25e 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c @@ -4125,9 +4125,6 @@ static int hns3_alloc_skb(struct hns3_enet_ring *ring, unsigned int length, return 0; } - if (ring->page_pool) - skb_mark_for_recycle(skb); - hns3_ring_stats_update(ring, seg_pkt_cnt); ring->pull_len = eth_get_headlen(netdev, va, HNS3_RX_HEAD_SIZE); @@ -4165,9 +4162,6 @@ static int hns3_add_frag(struct hns3_enet_ring *ring) return -ENXIO; } - if (ring->page_pool) - skb_mark_for_recycle(new_skb); - ring->frag_num = 0; if (ring->tail_skb) { diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_rx.c b/drivers/net/ethernet/huawei/hinic3/hinic3_rx.c index 309ab59013797..4f94c13eedf3f 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_rx.c +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_rx.c @@ -234,7 +234,6 @@ static void hinic3_add_rx_frag(struct hinic3_rxq *rxq, skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, rx_info->page_offset, size, rxq->buf_len); - skb_mark_for_recycle(skb); } static void packaging_skb(struct hinic3_rxq *rxq, struct sk_buff *skb, diff --git a/drivers/net/ethernet/intel/iavf/iavf_txrx.c b/drivers/net/ethernet/intel/iavf/iavf_txrx.c index 8982cf75466fa..6b863035d3ae9 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_txrx.c +++ b/drivers/net/ethernet/intel/iavf/iavf_txrx.c @@ -1229,8 +1229,6 @@ static struct sk_buff *iavf_build_skb(const struct libeth_fqe *rx_buffer, if (unlikely(!skb)) return NULL; - skb_mark_for_recycle(skb); - /* update pointers within the skb to store the data */ skb_reserve(skb, hr); __skb_put(skb, size); diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c index ea755cf05b9ef..0554595fbd143 100644 --- a/drivers/net/ethernet/marvell/mvneta.c +++ b/drivers/net/ethernet/marvell/mvneta.c @@ -2407,8 +2407,6 @@ mvneta_swbm_build_skb(struct mvneta_port *pp, struct page_pool *pool, if (!skb) return ERR_PTR(-ENOMEM); - skb_mark_for_recycle(skb); - skb_reserve(skb, xdp->data - xdp->data_hard_start); skb_put(skb, xdp->data_end - xdp->data); if (metasize) diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c index 317af54369cfe..7d4c6ce505697 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c @@ -4030,8 +4030,6 @@ static int mvpp2_rx(struct mvpp2_port *port, struct napi_struct *napi, } goto err_drop_frame_retired; } - if (pp) - skb_mark_for_recycle(skb); /* If we have RX hardware timestamping enabled, grab the * timestamp from the queue and convert. diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c index 90af9b9044aa0..8869e9f1d6b5a 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c @@ -383,7 +383,6 @@ static void otx2_rcv_pkt_handler(struct otx2_nic *pfvf, if (pfvf->flags & OTX2_FLAG_TC_MARK_ENABLED) skb->mark = parse->match_id; - skb_mark_for_recycle(skb); if (metasize) skb_metadata_set(skb, metasize); diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c index ad0fdc4133fb6..51105efb0ed6f 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c @@ -2305,7 +2305,6 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget, metasize = xdp.data - xdp.data_meta; if (metasize) skb_metadata_set(skb, metasize); - skb_mark_for_recycle(skb); } else { if (ring->frag_size <= PAGE_SIZE) new_data = napi_alloc_frag(ring->frag_size); diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c index fe18606c2ab55..6c59761b903b0 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c @@ -856,7 +856,6 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud skb = napi_get_frags(&cq->napi); if (unlikely(!skb)) goto next; - skb_mark_for_recycle(skb); if (unlikely(ring->hwtstamp_rx_filter == HWTSTAMP_FILTER_ALL)) { u64 timestamp = mlx4_en_get_cqe_ts(cqe); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index 22fea4739a2c9..f2b5ab014d549 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -1614,7 +1614,6 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi, return NULL; /* queue up for recycling/reuse */ - skb_mark_for_recycle(skb); frag_page->frags++; return skb; @@ -1701,7 +1700,6 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi if (unlikely(!skb)) return NULL; - skb_mark_for_recycle(skb); head_wi->frag_page->frags++; if (xdp_buff_has_frags(&mxbuf->xdp)) { @@ -1966,7 +1964,6 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w rq->stats->buff_alloc_err++; return NULL; } - skb_mark_for_recycle(skb); va = skb->head; net_prefetchw(va); /* xdp_frame data area */ net_prefetchw(skb->data); @@ -2046,7 +2043,6 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w if (unlikely(!skb)) return NULL; - skb_mark_for_recycle(skb); linear_page->frags++; if (xdp_buff_has_frags(&mxbuf->xdp)) { @@ -2146,7 +2142,6 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, return NULL; /* queue up for recycling/reuse */ - skb_mark_for_recycle(skb); frag_page->frags++; return skb; @@ -2176,9 +2171,6 @@ mlx5e_skb_from_cqe_shampo(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, head_offset, head_offset, len); __skb_put(skb, head_size); - /* queue up for recycling/reuse */ - skb_mark_for_recycle(skb); - return skb; } diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c index dceb69945d26e..a81cff6a6b7b8 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.c +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c @@ -776,8 +776,6 @@ static void mlxsw_pci_cqe_rdq_handle(struct mlxsw_pci *mlxsw_pci, goto out; } - skb_mark_for_recycle(skb); - if (mlxsw_pci_cqe_lag_get(cqe_v, cqe)) { rx_info.is_lag = true; rx_info.u.lag_id = mlxsw_pci_cqe_lag_id_get(cqe_v, cqe); diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c index 21668bbcc9063..e4f091fd7956c 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c @@ -439,8 +439,6 @@ static struct sk_buff *lan966x_fdma_rx_get_frame(struct lan966x_rx *rx, if (unlikely(!skb)) goto free_page; - skb_mark_for_recycle(skb); - skb_reserve(skb, XDP_PACKET_HEADROOM); skb_put(skb, fdma_db_len_get(db)); diff --git a/drivers/net/ethernet/microchip/sparx5/lan969x/lan969x_fdma.c b/drivers/net/ethernet/microchip/sparx5/lan969x/lan969x_fdma.c index 053d125eda658..f9c610c18b2c3 100644 --- a/drivers/net/ethernet/microchip/sparx5/lan969x/lan969x_fdma.c +++ b/drivers/net/ethernet/microchip/sparx5/lan969x/lan969x_fdma.c @@ -120,7 +120,6 @@ static struct sk_buff *lan969x_fdma_rx_get_frame(struct sparx5 *sparx5, if (unlikely(!skb)) goto free_page; - skb_mark_for_recycle(skb); skb_put(skb, fdma_db_len_get(db)); skb_pull(skb, IFH_LEN * sizeof(u32)); diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c index 6b7c027e497ed..2f670e3c61d80 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_en.c +++ b/drivers/net/ethernet/microsoft/mana/mana_en.c @@ -2170,9 +2170,6 @@ static void mana_rx_skb(void *buf_va, bool from_pool, if (!skb) goto drop; - if (from_pool) - skb_mark_for_recycle(skb); - skb->dev = napi->dev; skb->protocol = eth_type_trans(skb, ndev); diff --git a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_lib.c b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_lib.c index b7b0c45a0f664..03ba794d6d727 100644 --- a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_lib.c +++ b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_lib.c @@ -332,7 +332,6 @@ static struct sk_buff *rnpgbe_build_skb(struct mucse_rx_buffer *rx_buffer, /* update pointers within the skb to store the data */ skb_reserve(skb, RNPGBE_SKB_PAD); __skb_put(skb, size); - skb_mark_for_recycle(skb); return skb; } diff --git a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c index 2543a8ff85476..8b85bb7effdd5 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c @@ -198,7 +198,6 @@ static struct sk_buff *ionic_rx_build_skb(struct ionic_queue *q, q_to_rx_stats(q)->alloc_err++; return NULL; } - skb_mark_for_recycle(skb); if (headroom) frag_len = min_t(u16, len, @@ -249,7 +248,6 @@ static struct sk_buff *ionic_rx_copybreak(struct net_device *netdev, q_to_rx_stats(q)->alloc_err++; return NULL; } - skb_mark_for_recycle(skb); if (!synced) page_pool_dma_sync_for_cpu(q->page_pool, diff --git a/drivers/net/ethernet/realtek/rtase/rtase_main.c b/drivers/net/ethernet/realtek/rtase/rtase_main.c index e3cd4f7c13804..4d1cd4f8c2d07 100644 --- a/drivers/net/ethernet/realtek/rtase/rtase_main.c +++ b/drivers/net/ethernet/realtek/rtase/rtase_main.c @@ -562,7 +562,6 @@ static int rx_handler(struct rtase_ring *ring, int budget) rtase_rx_csum(tp, skb, desc); skb_put(skb, pkt_size); - skb_mark_for_recycle(skb); skb->protocol = eth_type_trans(skb, dev); if (skb->pkt_type == PACKET_MULTICAST) diff --git a/drivers/net/ethernet/renesas/ravb_main.c b/drivers/net/ethernet/renesas/ravb_main.c index 5a07cc0180d8b..135081fce0bbd 100644 --- a/drivers/net/ethernet/renesas/ravb_main.c +++ b/drivers/net/ethernet/renesas/ravb_main.c @@ -877,7 +877,6 @@ static int ravb_rx_gbeth(struct net_device *ndev, int budget, int q) true); goto refill; } - skb_mark_for_recycle(skb); skb_put(skb, desc_len); /* Save this skb if the packet spans multiple @@ -1041,7 +1040,6 @@ static int ravb_rx_rcar(struct net_device *ndev, int budget, int q) rx_buff->page, 0, true); break; } - skb_mark_for_recycle(skb); ravb_rx_rcar_hwstamp(priv, q, desc, skb); diff --git a/drivers/net/ethernet/socionext/netsec.c b/drivers/net/ethernet/socionext/netsec.c index 79a0a324c921d..95da134dc847a 100644 --- a/drivers/net/ethernet/socionext/netsec.c +++ b/drivers/net/ethernet/socionext/netsec.c @@ -1044,7 +1044,6 @@ static int netsec_process_rx(struct netsec_priv *priv, int budget) "rx failed to build skb\n"); break; } - skb_mark_for_recycle(skb); skb_reserve(skb, xdp.data - xdp.data_hard_start); skb_put(skb, xdp.data_end - xdp.data); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 9ded31150fbe6..5ce03aec41024 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -6010,7 +6010,6 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) head_pad_len = ctx.xdp.data - ctx.xdp.data_hard_start; skb_reserve(skb, head_pad_len); skb_put(skb, buf1_len); - skb_mark_for_recycle(skb); buf->page = NULL; } else if (buf1_len) { dma_sync_single_for_cpu(priv->device, buf->addr, diff --git a/drivers/net/ethernet/tehuti/tn40.c b/drivers/net/ethernet/tehuti/tn40.c index 558b791a97edd..c1fcc28e44322 100644 --- a/drivers/net/ethernet/tehuti/tn40.c +++ b/drivers/net/ethernet/tehuti/tn40.c @@ -384,7 +384,6 @@ static int tn40_rx_receive(struct tn40_priv *priv, int budget) tn40_recycle_rx_buffer(priv, rxdd); break; } - skb_mark_for_recycle(skb); skb_put(skb, len); skb->protocol = eth_type_trans(skb, priv->ndev); skb->ip_summed = diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.c b/drivers/net/ethernet/ti/am65-cpsw-nuss.c index 2551554add17a..48be2ba48a7f5 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c +++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c @@ -1353,7 +1353,6 @@ static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_rx_flow *flow, skb_put(skb, pkt_len); if (port->rx_ts_filter) am65_cpts_rx_timestamp(common->cpts, port_id, skb); - skb_mark_for_recycle(skb); skb->protocol = eth_type_trans(skb, ndev); am65_cpsw_nuss_rx_csum(skb, csum_info); napi_gro_receive(&flow->napi_rx, skb); diff --git a/drivers/net/ethernet/ti/cpsw.c b/drivers/net/ethernet/ti/cpsw.c index 8d36124520ad3..2ffdd8086b45b 100644 --- a/drivers/net/ethernet/ti/cpsw.c +++ b/drivers/net/ethernet/ti/cpsw.c @@ -453,8 +453,6 @@ static void cpsw_rx_handler(void *token, int len, int status) cpts_rx_timestamp(cpsw->cpts, skb); skb->protocol = eth_type_trans(skb, ndev); - /* mark skb for recycling */ - skb_mark_for_recycle(skb); netif_receive_skb(skb); ndev->stats.rx_bytes += len; diff --git a/drivers/net/ethernet/ti/cpsw_new.c b/drivers/net/ethernet/ti/cpsw_new.c index 07832cad5e3c2..996b513f349f1 100644 --- a/drivers/net/ethernet/ti/cpsw_new.c +++ b/drivers/net/ethernet/ti/cpsw_new.c @@ -397,8 +397,6 @@ static void cpsw_rx_handler(void *token, int len, int status) cpts_rx_timestamp(cpsw->cpts, skb); skb->protocol = eth_type_trans(skb, ndev); - /* mark skb for recycling */ - skb_mark_for_recycle(skb); netif_receive_skb(skb); ndev->stats.rx_bytes += len; diff --git a/drivers/net/ethernet/ti/icssg/icssg_common.c b/drivers/net/ethernet/ti/icssg/icssg_common.c index 4a7d1a6f470b1..20c5f44df2578 100644 --- a/drivers/net/ethernet/ti/icssg/icssg_common.c +++ b/drivers/net/ethernet/ti/icssg/icssg_common.c @@ -1131,7 +1131,6 @@ static int emac_rx_packet(struct prueth_emac *emac, u32 flow_id, u32 *xdp_state) skb->offload_fwd_mark = emac->offload_fwd_mark; skb->protocol = eth_type_trans(skb, ndev); - skb_mark_for_recycle(skb); napi_gro_receive(&emac->napi_rx, skb); ndev->stats.rx_bytes += pkt_len; ndev->stats.rx_packets++; diff --git a/drivers/net/ethernet/wangxun/libwx/wx_lib.c b/drivers/net/ethernet/wangxun/libwx/wx_lib.c index dcbf5811046e9..8de18c10de99b 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_lib.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_lib.c @@ -259,8 +259,6 @@ static struct sk_buff *wx_build_skb(struct wx_ring *rx_ring, return skb; } - skb_mark_for_recycle(skb); - if (!wx_test_staterr(rx_desc, WX_RXD_STAT_EOP)) WX_CB(skb)->dma = rx_buffer->dma; diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 3453689bfd341..9fa7b1a40624f 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -1930,8 +1930,6 @@ static struct sk_buff *receive_small_xdp(struct net_device *dev, if (metasize) skb_metadata_set(skb, metasize); - skb_mark_for_recycle(skb); - return skb; err_xdp: @@ -1987,10 +1985,8 @@ static struct sk_buff *receive_small(struct net_device *dev, } skb = receive_small_build_skb(vi, xdp_headroom, buf, len, buflen); - if (likely(skb)) { - skb_mark_for_recycle(skb); + if (likely(skb)) return skb; - } u64_stats_inc(&stats->alloc_fail); err: @@ -2313,7 +2309,6 @@ static struct sk_buff *receive_mergeable_xdp(struct net_device *dev, break; } - skb_mark_for_recycle(head_skb); return head_skb; case XDP_TX: @@ -2425,7 +2420,6 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, goto err_skb; } - skb_mark_for_recycle(head_skb); while (--num_buf) { buf = virtnet_rq_get_buf(rq, &len, &ctx); if (unlikely(!buf)) { diff --git a/drivers/net/vmxnet3/vmxnet3_xdp.c b/drivers/net/vmxnet3/vmxnet3_xdp.c index 5f470499e6002..9d1da8809501a 100644 --- a/drivers/net/vmxnet3/vmxnet3_xdp.c +++ b/drivers/net/vmxnet3/vmxnet3_xdp.c @@ -325,7 +325,6 @@ vmxnet3_build_skb(struct vmxnet3_rx_queue *rq, struct page *page, /* bpf prog might change len and data position. */ skb_reserve(skb, xdp->data - xdp->data_hard_start); skb_put(skb, xdp->data_end - xdp->data); - skb_mark_for_recycle(skb); return skb; } diff --git a/drivers/net/wireless/mediatek/mt76/dma.c b/drivers/net/wireless/mediatek/mt76/dma.c index a67880a9281c5..2ba62421c5ae0 100644 --- a/drivers/net/wireless/mediatek/mt76/dma.c +++ b/drivers/net/wireless/mediatek/mt76/dma.c @@ -1075,7 +1075,6 @@ mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget) goto free_frag; skb_reserve(skb, q->buf_offset); - skb_mark_for_recycle(skb); *(u32 *)skb->cb = info; diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c index f36da545045b8..7a3fccc74dcce 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c @@ -2041,7 +2041,6 @@ void mt7996_rro_rx_process(struct mt76_dev *mdev, void *data) goto next_page; skb_reserve(skb, q->buf_offset); - skb_mark_for_recycle(skb); __skb_put(skb, len); if (ind_reason == 1 || ind_reason == 2) { diff --git a/drivers/net/wireless/mediatek/mt76/npu.c b/drivers/net/wireless/mediatek/mt76/npu.c index c4c7c0af63217..2e018d7116f33 100644 --- a/drivers/net/wireless/mediatek/mt76/npu.c +++ b/drivers/net/wireless/mediatek/mt76/npu.c @@ -91,7 +91,6 @@ static struct sk_buff *mt76_npu_dequeue(struct mt76_dev *dev, __skb_put(skb, len); skb_reset_mac_header(skb); - skb_mark_for_recycle(skb); } else { struct skb_shared_info *shinfo = skb_shinfo(skb); struct page *page = virt_to_head_page(e->buf); diff --git a/drivers/net/wireless/mediatek/mt76/usb.c b/drivers/net/wireless/mediatek/mt76/usb.c index be48a88a3ad02..f91f773ddcd68 100644 --- a/drivers/net/wireless/mediatek/mt76/usb.c +++ b/drivers/net/wireless/mediatek/mt76/usb.c @@ -563,7 +563,6 @@ mt76u_process_rx_entry(struct mt76_dev *dev, struct urb *urb, nsgs++; } - skb_mark_for_recycle(skb); dev->drv->rx_skb(dev, MT_RXQ_MAIN, skb, NULL); return nsgs; diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c index fdcb91042f290..82bef245d6d25 100644 --- a/drivers/net/xen-netfront.c +++ b/drivers/net/xen-netfront.c @@ -286,7 +286,6 @@ static struct sk_buff *xennet_alloc_one_rx_buffer(struct netfront_queue *queue) return NULL; } skb_add_rx_frag(skb, 0, page, 0, 0, PAGE_SIZE); - skb_mark_for_recycle(skb); /* Align ip header to a 16 bytes boundary */ skb_reserve(skb, NET_IP_ALIGN); diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 4eaf4da1f7dad..3dc77734c80a3 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -5473,11 +5473,6 @@ static inline struct kcov_common_handle_id skb_get_kcov_handle(struct sk_buff *s return skb->kcov_handle; } -/* Deprecated: page_pool recycling is now determined per netmem. */ -static inline void skb_mark_for_recycle(struct sk_buff *skb) -{ -} - ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter, ssize_t maxsize); diff --git a/include/net/page_pool/helpers.h b/include/net/page_pool/helpers.h index fdbb1a4fe11f5..ff005fbace84d 100644 --- a/include/net/page_pool/helpers.h +++ b/include/net/page_pool/helpers.h @@ -42,8 +42,7 @@ * The API keeps track of in-flight pages, in order to let API users know when * it is safe to free a page_pool object, the API users must call * page_pool_put_page() or page_pool_free_va() to free the page_pool object, or - * attach the page_pool object to a page_pool-aware object like skbs marked with - * skb_mark_for_recycle(). + * attach the page_pool object to a page_pool-aware object like an skb. * * page_pool_put_page() may be called multiple times on the same page if a page * is split into multiple fragments. For the last fragment, it will either @@ -260,7 +259,7 @@ static inline void page_pool_fragment_netmem(netmem_ref netmem, long nr) * pp_ref_count represents the number of outstanding references to the page, * which will be freed using page_pool APIs (rather than page allocator APIs * like put_page()). Such references are usually held by page_pool-aware - * objects like skbs marked for page pool recycling. + * objects like skbs. * * This helper allows the caller to take (set) multiple references to a * freshly allocated page. The page must be freshly allocated (have a diff --git a/net/core/skbuff.c b/net/core/skbuff.c index a695c3f64dfd9..6519380c713df 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -967,7 +967,6 @@ int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb, skb_reserve(nskb, headroom); skb_copy_header(nskb, skb); - skb_mark_for_recycle(nskb); err = skb_copy_bits(skb, 0, nskb->data, size); if (err) { diff --git a/net/core/xdp.c b/net/core/xdp.c index 6f2b9191af9ec..513a51484ecd1 100644 --- a/net/core/xdp.c +++ b/net/core/xdp.c @@ -625,8 +625,8 @@ EXPORT_SYMBOL_GPL(xdp_warn); * * Perform common operations to create a new skb to pass up the stack from * &xdp_buff: allocate an skb head from the NAPI percpu cache, initialize - * skb data pointers and offsets, set the recycle bit if the buff is - * PP-backed, Rx queue index, protocol and update frags info. + * skb data pointers and offsets, Rx queue index, protocol and update frags + * info. * * Return: new &sk_buff on success, %NULL on error. */ @@ -654,9 +654,6 @@ struct sk_buff *xdp_build_skb_from_buff(const struct xdp_buff *xdp) if (metalen > 0) skb_metadata_set(skb, metalen); - if (rxq->mem.type == MEM_TYPE_PAGE_POOL) - skb_mark_for_recycle(skb); - skb_record_rx_queue(skb, rxq->queue_index); if (unlikely(nr_frags)) { @@ -761,7 +758,6 @@ struct sk_buff *xdp_build_skb_from_zc(struct xdp_buff *xdp) goto out; } - skb_mark_for_recycle(skb); skb_reserve(skb, xdp->data_meta - xdp->data_hard_start); memcpy(__skb_put(skb, len), xdp->data_meta, LARGEST_ALIGN(len)); @@ -836,9 +832,6 @@ struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf, * - RX ring dev queue index (skb_record_rx_queue) */ - if (xdpf->mem_type == MEM_TYPE_PAGE_POOL) - skb_mark_for_recycle(skb); - /* Allow SKB to reuse area used by xdp_frame */ xdp_scrub_frame(xdpf); -- 2.56.0.455.g6c5a503b38-goog