Add four trace events covering the rekey state machine in
tls_device.c:

  tls_device_rekey_start         - rekey accepted; inflight=1 means old-key
                                   data is still queued, dev_add deferred
  tls_device_rekey_reencrypt     - old-key undo pass for a boundary record
  tls_device_rekey_done          - boundary crossed, old_aead_recv freed,
                                   deferred dev_add issued if pending
  tls_device_complete_rekey_fail - TX rekey completion failed in sendmsg;
                                   READY is left set and the next sendmsg
                                   retries

Signed-off-by: Rishikesh Jethwani <rjethwani@purestorage.com>
---
 net/tls/tls_device.c | 17 +++++++-
 net/tls/trace.h      | 98 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 113 insertions(+), 2 deletions(-)

diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c
index c34268cd9531..9a4c121dbebb 100644
--- a/net/tls/tls_device.c
+++ b/net/tls/tls_device.c
@@ -760,8 +760,14 @@ int tls_device_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
 	lock_sock(sk);
 
 	/* Old-key records all ACKed; switch back to HW. */
-	if (test_bit(TLS_TX_REKEY_READY, &tls_ctx->flags))
-		tls_device_complete_rekey(sk, tls_ctx, true);
+	if (test_bit(TLS_TX_REKEY_READY, &tls_ctx->flags)) {
+		rc = tls_device_complete_rekey(sk, tls_ctx, true);
+		/* On failure the READY bit is left set; the next sendmsg
+		 * retries.
+		 */
+		if (rc)
+			trace_tls_device_complete_rekey_fail(sk, rc);
+	}
 
 	/* Use SW path if rekey is in progress (PENDING) or if HW rekey
 	 * failed (FAILED).
@@ -1267,6 +1273,9 @@ int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx)
 				return 0;
 			}
 
+			trace_tls_device_rekey_reencrypt(sk, rec_start_seq,
+							 ctx->rekey.old_nic_boundary);
+
 			/* rekey_fixup sets decrypted flags in case the NIC clears
 			 * decrypted flags on auth failure
 			 */
@@ -1277,6 +1286,8 @@ int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx)
 							    sw_ctx, tls_ctx);
 		}
 
+		trace_tls_device_rekey_done(sk, rec_start_seq,
+					    ctx->rekey.old_nic_boundary);
 		crypto_free_aead(ctx->rekey.old_aead_recv);
 		ctx->rekey.old_aead_recv = NULL;
 
@@ -1923,6 +1934,8 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx,
 					netdev->tlsdev_ops->tls_dev_rx_rekey_fixup;
 				context->dev_add_pending = 1;
 			}
+			trace_tls_device_rekey_start(sk, copied_seq, rcv_nxt,
+						     before(copied_seq, rcv_nxt));
 		}
 	}
 
diff --git a/net/tls/trace.h b/net/tls/trace.h
index 2d8ce4ff3265..2a90b77d75e8 100644
--- a/net/tls/trace.h
+++ b/net/tls/trace.h
@@ -192,6 +192,104 @@ TRACE_EVENT(tls_device_tx_resync_send,
 	)
 );
 
+TRACE_EVENT(tls_device_rekey_start,
+
+	TP_PROTO(struct sock *sk, u32 copied_seq, u32 nic_boundary,
+		 bool inflight),
+
+	TP_ARGS(sk, copied_seq, nic_boundary, inflight),
+
+	TP_STRUCT__entry(
+		__field(	struct sock *,	sk		)
+		__field(	u32,		copied_seq	)
+		__field(	u32,		nic_boundary	)
+		__field(	bool,		inflight	)
+	),
+
+	TP_fast_assign(
+		__entry->sk = sk;
+		__entry->copied_seq = copied_seq;
+		__entry->nic_boundary = nic_boundary;
+		__entry->inflight = inflight;
+	),
+
+	TP_printk(
+		"sk=%p copied_seq=%u nic_boundary=%u inflight=%d",
+		__entry->sk, __entry->copied_seq, __entry->nic_boundary,
+		__entry->inflight
+	)
+);
+
+TRACE_EVENT(tls_device_rekey_reencrypt,
+
+	TP_PROTO(struct sock *sk, u32 tcp_seq, u32 nic_boundary),
+
+	TP_ARGS(sk, tcp_seq, nic_boundary),
+
+	TP_STRUCT__entry(
+		__field(	struct sock *,	sk		)
+		__field(	u32,		tcp_seq		)
+		__field(	u32,		nic_boundary	)
+	),
+
+	TP_fast_assign(
+		__entry->sk = sk;
+		__entry->tcp_seq = tcp_seq;
+		__entry->nic_boundary = nic_boundary;
+	),
+
+	TP_printk(
+		"sk=%p tcp_seq=%u nic_boundary=%u",
+		__entry->sk, __entry->tcp_seq, __entry->nic_boundary
+	)
+);
+
+TRACE_EVENT(tls_device_rekey_done,
+
+	TP_PROTO(struct sock *sk, u32 tcp_seq, u32 nic_boundary),
+
+	TP_ARGS(sk, tcp_seq, nic_boundary),
+
+	TP_STRUCT__entry(
+		__field(	struct sock *,	sk		)
+		__field(	u32,		tcp_seq		)
+		__field(	u32,		nic_boundary	)
+	),
+
+	TP_fast_assign(
+		__entry->sk = sk;
+		__entry->tcp_seq = tcp_seq;
+		__entry->nic_boundary = nic_boundary;
+	),
+
+	TP_printk(
+		"sk=%p tcp_seq=%u nic_boundary=%u",
+		__entry->sk, __entry->tcp_seq, __entry->nic_boundary
+	)
+);
+
+TRACE_EVENT(tls_device_complete_rekey_fail,
+
+	TP_PROTO(struct sock *sk, int rc),
+
+	TP_ARGS(sk, rc),
+
+	TP_STRUCT__entry(
+		__field(	struct sock *,	sk	)
+		__field(	int,		rc	)
+	),
+
+	TP_fast_assign(
+		__entry->sk = sk;
+		__entry->rc = rc;
+	),
+
+	TP_printk(
+		"sk=%p rc=%d",
+		__entry->sk, __entry->rc
+	)
+);
+
 #endif /* _TLS_TRACE_H_ */
 
 #undef TRACE_INCLUDE_PATH
-- 
2.25.1