Various networking features take additional references to frag pages that are still referenced elsewhere when rearranging skb frags, for example software GSO segmentation, TCP receive coalescing of clones, SACK shifting and MTU probing. This is an effective way for multiplying folio references held by an skb, allowing for exceeding folio refcount limits. This is not specific to large pagecache folios, but with them it is a lot easier to reach the refcount limit, as page table mappings can cheaply create many references. Let's use skb_frag_try_ref() and skb_frags_try_ref() in skb_segment(), skb_try_coalesce(), skb_shift() and tcp_clone_payload(). All of these can already fail, and their callers handle that. skb_split() can still add one reference when splitting a frag. But each split also allocates a new skb that is charged to the socket, reaching the refcount limit that way would require a lot of socket memory, and the references go away once the data was acked, so it's not that relevant in practice. Making skb_split() fail is more involved, so let's leave it alone for now. Fixes: 800d8c63b2e9 ("shmem: add huge pages support") Fixes: 99cb0dbd47a1 ("mm,thp: add read-only THP support for (non-shmem) FS") Fixes: 793917d997df ("mm/readahead: Add large folio readahead") Cc: stable@vger.kernel.org Signed-off-by: David Hildenbrand (Arm) --- net/core/skbuff.c | 32 +++++++++++++++++++++----------- net/ipv4/tcp_output.c | 8 +++++--- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 1d44e9a4152e5..f473c74956b8b 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -4326,7 +4326,8 @@ static int skb_prepare_for_shift(struct sk_buff *skb) * the length of the skb, from skb to tgt. Returns number bytes shifted. * It's up to caller to free skb if everything was shifted. * - * If @tgt runs out of frags, the whole operation is aborted. + * If @tgt runs out of frags, or we cannot take an additional frag reference, + * the whole operation is aborted. * * Skb cannot include anything else but paged data while tgt is allowed * to have non-paged data as well. @@ -4409,7 +4410,9 @@ int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen) } else { skb_frag_page_copy(fragto, fragfrom); - skb_frag_ref(tgt, to); + /* Nothing has been committed yet. */ + if (!skb_frag_try_ref(tgt, to)) + return 0; skb_frag_off_copy(fragto, fragfrom); skb_frag_size_set(fragto, todo); @@ -5148,7 +5151,10 @@ struct sk_buff *skb_segment(struct sk_buff *head_skb, if (!skb_shinfo(nskb)->nr_frags) nskb->pp_recycle = frag_skb->pp_recycle; *nskb_frag = (i < 0) ? skb_head_frag_to_page_desc(frag_skb) : *frag; - skb_frag_ref(nskb, skb_shinfo(nskb)->nr_frags); + if (!skb_frag_try_ref(nskb, skb_shinfo(nskb)->nr_frags)) { + err = -ENOMEM; + goto err; + } size = skb_frag_size(nskb_frag); if (pos < offset) { @@ -6289,7 +6295,8 @@ bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, bool *fragstolen, int *delta_truesize) { struct skb_shared_info *to_shinfo, *from_shinfo; - int i, delta, len = from->len; + int delta, len = from->len; + bool cloned; *fragstolen = false; @@ -6362,6 +6369,15 @@ bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, WARN_ON_ONCE(delta < len); + /* + * A clone keeps referencing its frags, so we need our own references. + * Nothing was added to @to yet: we only steal the head of @from above + * if it is not locked, and the head of a clone is always locked. + */ + cloned = skb_cloned(from); + if (cloned && !skb_frags_try_ref(from)) + return false; + memcpy(to_shinfo->frags + to_shinfo->nr_frags, from_shinfo->frags, from_shinfo->nr_frags * sizeof(skb_frag_t)); @@ -6369,15 +6385,9 @@ bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, if (from_shinfo->nr_frags) to_shinfo->flags |= from_shinfo->flags & SKBFL_SHARED_FRAG; - if (!skb_cloned(from)) + if (!cloned) from_shinfo->nr_frags = 0; - /* if the skb is not cloned this does nothing - * since we set nr_frags to 0. - */ - for (i = 0; i < from_shinfo->nr_frags; i++) - skb_frag_ref(from, i); - to->truesize += delta; to->len += len; to->data_len += len; diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index cf3ddc055d48d..043838d996404 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -2710,10 +2710,12 @@ static int tcp_clone_payload(struct sock *sk, struct sk_buff *to, } commit: WARN_ON_ONCE(len != probe_size); - for (i = 0; i < nr_frags; i++) - skb_frag_ref(to, i); - skb_shinfo(to)->nr_frags = nr_frags; + if (!skb_frags_try_ref(to)) { + skb_shinfo(to)->nr_frags = 0; + return -ENOMEM; + } + to->truesize += probe_size; to->len += probe_size; to->data_len += probe_size; -- 2.56.0