From: Chuck Lever Each record release via tls_strp_msg_done() triggers tls_strp_check_rcv(), which calls tls_rx_msg_ready() and fires saved_data_ready(). During a multi-record receive, the first N-1 wakeups are pure overhead: the caller is already running and will pick up subsequent records on the next loop iteration. The same waste occurs on the recvmsg and splice_read paths. Replace tls_strp_msg_done() with tls_strp_msg_release() in all three receive paths (read_sock, recvmsg, splice_read), deferring the tls_strp_check_rcv() call to each path's exit point. Factor tls_rx_msg_ready() out of tls_strp_read_sock() so that parsing a record no longer fires the callback directly, and introduce tls_strp_check_rcv_quiet() for use in tls_rx_rec_wait(), which parses queued data without notifying. With no remaining callers, tls_strp_msg_done() and its wrapper tls_rx_rec_done() are removed. Acked-by: Alistair Francis Reviewed-by: Hannes Reinecke Signed-off-by: Chuck Lever --- net/tls/tls.h | 2 +- net/tls/tls_strp.c | 27 +++++++++++++++++++-------- net/tls/tls_sw.c | 21 ++++++++++++++------- 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/net/tls/tls.h b/net/tls/tls.h index a97f1acef31d3a4ad1442c20c6914a9f5198a1e0..0ab3b83c37243ab393367ac80836b9fd8fec7f82 100644 --- a/net/tls/tls.h +++ b/net/tls/tls.h @@ -193,8 +193,8 @@ int tls_strp_init(struct tls_strparser *strp, struct sock *sk); void tls_strp_data_ready(struct tls_strparser *strp); void tls_strp_check_rcv(struct tls_strparser *strp); +void tls_strp_check_rcv_quiet(struct tls_strparser *strp); void tls_strp_msg_release(struct tls_strparser *strp); -void tls_strp_msg_done(struct tls_strparser *strp); int tls_rx_msg_size(struct tls_strparser *strp, struct sk_buff *skb); void tls_rx_msg_ready(struct tls_strparser *strp); diff --git a/net/tls/tls_strp.c b/net/tls/tls_strp.c index a7648ebde162b1bd944f53d44c89cd7da4cef082..6cf274380da2785b58206cf0e70cf4fee033a223 100644 --- a/net/tls/tls_strp.c +++ b/net/tls/tls_strp.c @@ -368,7 +368,6 @@ static int tls_strp_copyin(read_descriptor_t *desc, struct sk_buff *in_skb, desc->count = 0; WRITE_ONCE(strp->msg_ready, 1); - tls_rx_msg_ready(strp); } return ret; @@ -539,11 +538,27 @@ static int tls_strp_read_sock(struct tls_strparser *strp) return tls_strp_read_copy(strp, false); WRITE_ONCE(strp->msg_ready, 1); - tls_rx_msg_ready(strp); return 0; } +/** + * tls_strp_check_rcv_quiet - parse without consumer notification + * @strp: TLS stream parser instance + * + * Parse queued data without firing the consumer notification. A subsequent + * tls_strp_check_rcv() is required before the socket lock is released; + * otherwise queued data stalls until the next tls_strp_data_ready() event. + */ +void tls_strp_check_rcv_quiet(struct tls_strparser *strp) +{ + if (unlikely(strp->stopped) || strp->msg_ready) + return; + + if (tls_strp_read_sock(strp) == -ENOMEM) + queue_work(tls_strp_wq, &strp->work); +} + void tls_strp_check_rcv(struct tls_strparser *strp) { if (unlikely(strp->stopped) || strp->msg_ready) @@ -551,6 +566,8 @@ void tls_strp_check_rcv(struct tls_strparser *strp) if (tls_strp_read_sock(strp) == -ENOMEM) queue_work(tls_strp_wq, &strp->work); + else if (strp->msg_ready) + tls_rx_msg_ready(strp); } /* Lower sock lock held */ @@ -603,12 +620,6 @@ void tls_strp_msg_release(struct tls_strparser *strp) memset(&strp->stm, 0, sizeof(strp->stm)); } -void tls_strp_msg_done(struct tls_strparser *strp) -{ - tls_strp_msg_release(strp); - tls_strp_check_rcv(strp); -} - void tls_strp_stop(struct tls_strparser *strp) { strp->stopped = 1; diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c index 07f4a3d1a6f854acc7762608cc7741b3de95c195..381a723b6cacc669e333752af34f051f296d6f52 100644 --- a/net/tls/tls_sw.c +++ b/net/tls/tls_sw.c @@ -1384,7 +1384,10 @@ tls_rx_rec_wait(struct sock *sk, struct sk_psock *psock, bool nonblock, return ret; if (!skb_queue_empty(&sk->sk_receive_queue)) { - tls_strp_check_rcv(&ctx->strp); + /* tls_strp_check_rcv() is called at each receive + * path's exit before the socket lock is released. + */ + tls_strp_check_rcv_quiet(&ctx->strp); if (tls_strp_msg_ready(ctx)) break; } @@ -1867,9 +1870,9 @@ static int tls_record_content_type(struct msghdr *msg, struct tls_msg *tlm, return 1; } -static void tls_rx_rec_done(struct tls_sw_context_rx *ctx) +static void tls_rx_rec_release(struct tls_sw_context_rx *ctx) { - tls_strp_msg_done(&ctx->strp); + tls_strp_msg_release(&ctx->strp); } /* This function traverses the rx_list in tls receive context to copies the @@ -2150,7 +2153,7 @@ int tls_sw_recvmsg(struct sock *sk, err = tls_record_content_type(msg, tls_msg(darg.skb), &control); if (err <= 0) { DEBUG_NET_WARN_ON_ONCE(darg.zc); - tls_rx_rec_done(ctx); + tls_rx_rec_release(ctx); put_on_rx_list_err: __skb_queue_tail(&ctx->rx_list, darg.skb); goto recv_end; @@ -2164,7 +2167,8 @@ int tls_sw_recvmsg(struct sock *sk, /* TLS 1.3 may have updated the length by more than overhead */ rxm = strp_msg(darg.skb); chunk = rxm->full_len; - tls_rx_rec_done(ctx); + tls_rx_rec_release(ctx); + tls_strp_check_rcv_quiet(&ctx->strp); if (!darg.zc) { bool partially_consumed = chunk > len; @@ -2258,6 +2262,7 @@ int tls_sw_recvmsg(struct sock *sk, copied += decrypted; end: + tls_strp_check_rcv(&ctx->strp); tls_rx_reader_unlock(sk, ctx); if (psock) sk_psock_put(sk, psock); @@ -2298,7 +2303,7 @@ ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, if (err < 0) goto splice_read_end; - tls_rx_rec_done(ctx); + tls_rx_rec_release(ctx); skb = darg.skb; } @@ -2325,6 +2330,7 @@ ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, consume_skb(skb); splice_read_end: + tls_strp_check_rcv(&ctx->strp); tls_rx_reader_unlock(sk, ctx); return copied ? : err; @@ -2390,7 +2396,7 @@ int tls_sw_read_sock(struct sock *sk, read_descriptor_t *desc, tlm = tls_msg(skb); decrypted += rxm->full_len; - tls_rx_rec_done(ctx); + tls_rx_rec_release(ctx); } /* read_sock does not support reading control messages */ @@ -2420,6 +2426,7 @@ int tls_sw_read_sock(struct sock *sk, read_descriptor_t *desc, } read_sock_end: + tls_strp_check_rcv(&ctx->strp); tls_rx_reader_release(sk, ctx); return copied ? : err; -- 2.53.0