Various networking features support splicing data from sockets into pipes (TCP, AF_UNIX, KCM, kTLS), whereby spd_fill_page() takes an additional page reference for each fragment, while the skb keeps its own reference until it was fully consumed. This allows for exceeding folio refcount limits. This is not specific to large pagecache folios, but with them it is a lot easier to reach the refcount limit, as page table mappings can cheaply create many references. Let's use try_get_page() in spd_fill_page() and stop splicing with -EFAULT if that fails. To propagate the error, let spd_fill_page(), __splice_segment() and __skb_splice_bits() return an int instead of a bool. Keep returning true/false instead of 1/0 to reduce the churn in this fix. If we could not splice anything, return the error from skb_splice_bits(): returning 0 would make the callers (TCP, AF_UNIX, KCM, kTLS) report EOF, although data is still queued. The callers already handle errors. Use -EFAULT, consistent with splice_folio_into_pipe(). Fixes: 800d8c63b2e9 ("shmem: add huge pages support") Fixes: 99cb0dbd47a1 ("mm,thp: add read-only THP support for (non-shmem) FS") Fixes: 793917d997df ("mm/readahead: Add large folio readahead") Cc: stable@vger.kernel.org Acked-by: Pedro Falcato Signed-off-by: David Hildenbrand (Arm) --- net/core/skbuff.c | 90 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 51 insertions(+), 39 deletions(-) diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 5e3f910eeea0b..a5965f8fa1b00 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -3150,11 +3150,13 @@ static bool spd_can_coalesce(const struct splice_pipe_desc *spd, } /* - * Fill page/offset/length into spd, if it can hold more pages. + * Fill page/offset/length into spd, if it can hold more pages and we can + * take a page reference. Returns non-zero to stop, or a negative error + * if we cannot take a page reference. */ -static bool spd_fill_page(struct splice_pipe_desc *spd, struct page *page, - unsigned int *len, unsigned int offset, bool linear, - struct sock *sk) +static int spd_fill_page(struct splice_pipe_desc *spd, struct page *page, + unsigned int *len, unsigned int offset, bool linear, + struct sock *sk) { if (unlikely(spd->nr_pages == MAX_SKB_FRAGS)) return true; @@ -3168,7 +3170,8 @@ static bool spd_fill_page(struct splice_pipe_desc *spd, struct page *page, spd->partial[spd->nr_pages - 1].len += *len; return false; } - get_page(page); + if (!try_get_page(page)) + return -EFAULT; spd->pages[spd->nr_pages] = page; spd->partial[spd->nr_pages].len = *len; spd->partial[spd->nr_pages].offset = offset; @@ -3177,12 +3180,14 @@ static bool spd_fill_page(struct splice_pipe_desc *spd, struct page *page, return false; } -static bool __splice_segment(struct page *page, unsigned int poff, - unsigned int plen, unsigned int *off, - unsigned int *len, - struct splice_pipe_desc *spd, bool linear, - struct sock *sk) +static int __splice_segment(struct page *page, unsigned int poff, + unsigned int plen, unsigned int *off, + unsigned int *len, + struct splice_pipe_desc *spd, bool linear, + struct sock *sk) { + int ret; + if (!*len) return true; @@ -3200,8 +3205,9 @@ static bool __splice_segment(struct page *page, unsigned int poff, do { unsigned int flen = min(*len, plen); - if (spd_fill_page(spd, page, &flen, poff, linear, sk)) - return true; + ret = spd_fill_page(spd, page, &flen, poff, linear, sk); + if (ret) + return ret; poff += flen; plen -= flen; *len -= flen; @@ -3213,28 +3219,30 @@ static bool __splice_segment(struct page *page, unsigned int poff, } /* - * Map linear and fragment data from the skb to spd. It reports true if the - * pipe is full or if we already spliced the requested length. + * Map linear and fragment data from the skb to spd. It returns non-zero to + * stop: true if the pipe is full or if we already spliced the requested + * length, or a negative error if we cannot take a page reference. */ -static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe, - unsigned int *offset, unsigned int *len, - struct splice_pipe_desc *spd, struct sock *sk) +static int __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe, + unsigned int *offset, unsigned int *len, + struct splice_pipe_desc *spd, struct sock *sk) { struct sk_buff *iter; - int seg; + int seg, ret; /* map the linear part : * If skb->head_frag is set, this 'linear' part is backed by a * fragment, and if the head is not shared with any clones then * we can avoid a copy since we own the head portion of this page. */ - if (__splice_segment(virt_to_page(skb->data), - (unsigned long) skb->data & (PAGE_SIZE - 1), - skb_headlen(skb), - offset, len, spd, - skb_head_is_locked(skb), - sk)) - return true; + ret = __splice_segment(virt_to_page(skb->data), + (unsigned long)skb->data & (PAGE_SIZE - 1), + skb_headlen(skb), + offset, len, spd, + skb_head_is_locked(skb), + sk); + if (ret) + return ret; /* * then map the fragments @@ -3248,10 +3256,11 @@ static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe, if (WARN_ON_ONCE(!skb_frag_page(f))) return false; - if (__splice_segment(skb_frag_page(f), - skb_frag_off(f), skb_frag_size(f), - offset, len, spd, false, sk)) - return true; + ret = __splice_segment(skb_frag_page(f), + skb_frag_off(f), skb_frag_size(f), + offset, len, spd, false, sk); + if (ret) + return ret; } skb_walk_frags(skb, iter) { @@ -3260,11 +3269,12 @@ static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe, continue; } /* __skb_splice_bits() only fails if the output has no room - * left, so no point in going over the frag_list for the error - * case. + * left or on error, so no point in going over the frag_list + * for the error case. */ - if (__skb_splice_bits(iter, pipe, offset, len, spd, sk)) - return true; + ret = __skb_splice_bits(iter, pipe, offset, len, spd, sk); + if (ret) + return ret; } return false; @@ -3272,7 +3282,9 @@ static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe, /* * Map data from the skb to a pipe. Should handle both the linear part, - * the fragments, and the frag list. + * the fragments, and the frag list. Returns the number of bytes spliced, + * or an error if we could not splice anything because we could not take + * a page reference. */ int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset, struct pipe_inode_info *pipe, unsigned int tlen, @@ -3287,14 +3299,14 @@ int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset, .ops = &nosteal_pipe_buf_ops, .spd_release = sock_spd_release, }; - int ret = 0; + int ret; - __skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk); + ret = __skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk); if (spd.nr_pages) - ret = splice_to_pipe(pipe, &spd); - - return ret; + return splice_to_pipe(pipe, &spd); + /* Returning 0 would make callers report EOF. */ + return ret < 0 ? ret : 0; } EXPORT_SYMBOL_GPL(skb_splice_bits); -- 2.56.0