At the final stage, tcp_collapse() iterates over collapsible candidate skbs and repacks them to new skbs. In each iteration, a new skb is allocated and inserted to the receive queue or a temporary list for RB tree. If alloc_skb() fails while splitting a large bloated skb across multiple new skbs, the partially-copied skb is not removed and new skbs remain in the queue as duplicates, consuming more memory. It is only when all candidates are properly collapsed that we can save memory. To avoid complicated rollback, let's preallocate skbs before collapsing. Now all skb in [start, sub_end) must be allocated in a batch, so the section size is normally capped at TCP_COLLAPSE_MAX_TOTAL_SIZE (16 skbs), except when a single bloated skb or overlapping OOO skbs must be collapsed together to free the original skb or keep the rbtree monotonic. Once the size exceeds the limit, the remaining skbs are re-evaluated from restart:. Note that the next skb can be skipped if its density already meets tp->scaling_ratio, and each cut adds at most one remainder skb (~3.4% worst-case truesize overhead on 4 KiB pages), which is a small tradeoff to bound the temporary allocation. Signed-off-by: Kuniyuki Iwashima --- net/ipv4/tcp_input.c | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index fe71305f6e67..9f32577b0b53 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -5812,6 +5812,7 @@ static struct sk_buff *tcp_skb_next(struct sk_buff *skb, struct sk_buff_head *li #define TCP_COLLAPSE_SCALING_RATIO \ ((TCP_COLLAPSE_MAX_SIZE << TCP_RMEM_TO_WIN_SCALE) / \ (SKB_TRUESIZE(TCP_COLLAPSE_MAX_SIZE))) +#define TCP_COLLAPSE_MAX_TOTAL_SIZE (TCP_COLLAPSE_MAX_SIZE * 16) static bool tcp_collapse_skb_bloated(const struct sk_buff *skb, u32 start, u8 scaling_ratio) @@ -5852,8 +5853,9 @@ tcp_collapse(struct sock *sk, struct sk_buff_head *list, struct rb_root *root, { int old_truesize, new_truesize; struct sk_buff *skb = head, *n; - struct sk_buff_head tmp; + struct sk_buff_head tmp, pool; int remainder, nskbs; + struct sk_buff *nskb; u8 scaling_ratio; u32 sub_end; @@ -5861,6 +5863,7 @@ tcp_collapse(struct sock *sk, struct sk_buff_head *list, struct rb_root *root, TCP_COLLAPSE_SCALING_RATIO); __skb_queue_head_init(&tmp); + __skb_queue_head_init(&pool); /* First, check that queue is collapsible and find * the point where collapsing can be useful. @@ -5908,8 +5911,9 @@ tcp_collapse(struct sock *sk, struct sk_buff_head *list, struct rb_root *root, /* seq and end_seq must increase monotonically for rbtree. */ if ((list || !before(TCP_SKB_CB(n)->seq, sub_end)) && - !tcp_collapse_skb_bloated(n, sub_end, - TCP_COLLAPSE_SCALING_RATIO)) + (TCP_SKB_CB(n)->end_seq - start >= TCP_COLLAPSE_MAX_TOTAL_SIZE || + !tcp_collapse_skb_bloated(n, sub_end, + TCP_COLLAPSE_SCALING_RATIO))) break; sub_end = TCP_SKB_CB(n)->end_seq; @@ -5924,6 +5928,8 @@ tcp_collapse(struct sock *sk, struct sk_buff_head *list, struct rb_root *root, if (remainder) { nskbs++; new_truesize += SKB_ALLOCSIZE(remainder); + } else { + remainder = TCP_COLLAPSE_MAX_SIZE; } if (old_truesize <= new_truesize) { @@ -5932,14 +5938,19 @@ tcp_collapse(struct sock *sk, struct sk_buff_head *list, struct rb_root *root, goto restart; } - while (before(start, sub_end)) { - int copy = min_t(int, TCP_COLLAPSE_MAX_SIZE, sub_end - start); - struct sk_buff *nskb; - - nskb = alloc_skb(copy, GFP_ATOMIC); + while (nskbs--) { + nskb = alloc_skb(nskbs ? TCP_COLLAPSE_MAX_SIZE : remainder, + GFP_ATOMIC); if (!nskb) goto end; + __skb_queue_tail(&pool, nskb); + } + + while (before(start, sub_end)) { + int copy = min_t(int, TCP_COLLAPSE_MAX_SIZE, sub_end - start); + + nskb = __skb_dequeue(&pool); memcpy(nskb->cb, skb->cb, sizeof(skb->cb)); TCP_SKB_CB(nskb)->has_rxtstamp = false; skb_copy_decrypted(nskb, skb); @@ -5985,6 +5996,8 @@ tcp_collapse(struct sock *sk, struct sk_buff_head *list, struct rb_root *root, end: skb_queue_walk_safe(&tmp, skb, n) tcp_rbtree_insert(root, skb); + + __skb_queue_purge_reason(&pool, SKB_CONSUMED); } /* Collapse ofo queue. Algorithm: select contiguous sequence of skbs -- 2.56.0.611.g1030f0b940-goog