Add TLS 1.3 support to the kernel TLS hardware offload infrastructure, enabling hardware acceleration for TLS 1.3 connections on capable NICs. This patch implements the critical differences between TLS 1.2 and TLS 1.3 record formats for hardware offload: TLS 1.2 record structure: [Header (5)] + [Explicit IV (8)] + [Ciphertext] + [Tag (16)] TLS 1.3 record structure: [Header (5)] + [Ciphertext + ContentType (1)] + [Tag (16)] Key changes: 1. Content type handling: In TLS 1.3, the content type byte is appended to the plaintext before encryption and tag computation. This byte must be encrypted along with the ciphertext to compute the correct authentication tag. Modified tls_device_record_close() to append the content type before the tag for TLS 1.3 records. 2. Version validation: Both tls_set_device_offload() and tls_set_device_offload_rx() now accept TLS_1_3_VERSION in addition to TLS_1_2_VERSION. 3. Pre-populate dummy_page with valid record types for memory allocation failure fallback path. Note: TLS 1.3 protocol parameters (aad_size, tail_size, prepend_size) are already handled by init_prot_info() in tls_sw.c. Testing: Verified on Broadcom BCM957608 (Thor 2) and Mellanox ConnectX-6 Dx (Crypto Enabled) using ktls_test. Both TX and RX hardware offload working successfully with TLS 1.3 AES-GCM-128 and AES-GCM-256 cipher suites. The upstream Broadcom bnxt_en driver does not yet support kTLS offload. Testing was performed using the out-of-tree driver version bnxt_en-1.10.3-235.1.154.0, which works without modifications. Signed-off-by: Rishikesh Jethwani --- net/tls/tls_device.c | 49 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c index 82ea407e520a..f57e96862b1c 100644 --- a/net/tls/tls_device.c +++ b/net/tls/tls_device.c @@ -319,6 +319,36 @@ static void tls_device_record_close(struct sock *sk, struct tls_prot_info *prot = &ctx->prot_info; struct page_frag dummy_tag_frag; + /* TLS 1.3: append content type byte before tag. + * Record structure: [Header (5)] + [Ciphertext + ContentType (1)] + [Tag (16)] + * The content type is encrypted with the ciphertext for authentication. + */ + if (prot->version == TLS_1_3_VERSION) { + struct page_frag dummy_content_type_frag; + struct page_frag *content_type_pfrag = pfrag; + + /* Validate record type range */ + if (unlikely(record_type < TLS_RECORD_TYPE_CHANGE_CIPHER_SPEC || + record_type > TLS_RECORD_TYPE_ACK)) { + pr_err_once("tls_device: invalid record type %u\n", + record_type); + return; + } + + if (unlikely(pfrag->size - pfrag->offset < prot->tail_size) && + !skb_page_frag_refill(prot->tail_size, pfrag, sk->sk_allocation)) { + /* Out of memory: use pre-populated dummy_page */ + dummy_content_type_frag.page = dummy_page; + dummy_content_type_frag.offset = record_type; + content_type_pfrag = &dummy_content_type_frag; + } else { + /* Current pfrag has space or allocation succeeded - write content type */ + *(unsigned char *)(page_address(pfrag->page) + pfrag->offset) = + record_type; + } + tls_append_frag(record, content_type_pfrag, prot->tail_size); + } + /* append tag * device will fill in the tag, we just need to append a placeholder * use socket memory to improve coalescing (re-using a single buffer @@ -335,7 +365,7 @@ static void tls_device_record_close(struct sock *sk, /* fill prepend */ tls_fill_prepend(ctx, skb_frag_address(&record->frags[0]), - record->len - prot->overhead_size, + (record->len - prot->overhead_size) + prot->tail_size, record_type); } @@ -1089,7 +1119,8 @@ int tls_set_device_offload(struct sock *sk) } crypto_info = &ctx->crypto_send.info; - if (crypto_info->version != TLS_1_2_VERSION) { + if (crypto_info->version != TLS_1_2_VERSION && + crypto_info->version != TLS_1_3_VERSION) { rc = -EOPNOTSUPP; goto release_netdev; } @@ -1196,7 +1227,8 @@ int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx) struct net_device *netdev; int rc = 0; - if (ctx->crypto_recv.info.version != TLS_1_2_VERSION) + if (ctx->crypto_recv.info.version != TLS_1_2_VERSION && + ctx->crypto_recv.info.version != TLS_1_3_VERSION) return -EOPNOTSUPP; netdev = get_netdev_for_sock(sk); @@ -1409,12 +1441,21 @@ static struct notifier_block tls_dev_notifier = { int __init tls_device_init(void) { - int err; + unsigned char *page_addr; + int err, i; dummy_page = alloc_page(GFP_KERNEL); if (!dummy_page) return -ENOMEM; + /* Pre-populate dummy_page with all valid TLS record types + * at their corresponding offsets for TLS 1.3 content type + * fallback path + */ + page_addr = page_address(dummy_page); + for (i = TLS_RECORD_TYPE_CHANGE_CIPHER_SPEC; i <= TLS_RECORD_TYPE_ACK; i++) + page_addr[i] = (unsigned char)i; + destruct_wq = alloc_workqueue("ktls_device_destruct", WQ_PERCPU, 0); if (!destruct_wq) { err = -ENOMEM; -- 2.25.1 Add TLS 1.3 hardware offload support to mlx5 driver, enabling both TX and RX hardware acceleration for TLS 1.3 connections on Mellanox ConnectX-6 Dx and newer adapters. This patch enables: - TLS 1.3 version detection and validation with proper capability checking - TLS 1.3 crypto context configuration using MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_3 (0x3) - Correct IV handling for TLS 1.3 (12-byte IV vs TLS 1.2's 4-byte salt) - Hardware offload for both TLS 1.3 AES-GCM-128 and AES-GCM-256 cipher suites Key differences from TLS 1.2: - TLS 1.2: Only 4-byte salt copied to gcm_iv, explicit IV in each record - TLS 1.3: Full 12-byte IV (salt + iv) copied to gcm_iv + implicit_iv * salt (4 bytes) → gcm_iv[0:3] * iv (8 bytes) → gcm_iv[4:7] + implicit_iv[0:3] * Note: gcm_iv and implicit_iv are contiguous in memory The EXTRACT_INFO_FIELDS macro is updated to also extract the 'iv' field which is needed for TLS 1.3. Testing: Verified on Mellanox ConnectX-6 Dx (Crypto Enabled) (MT2892) using ktls_test suite. Both TX and RX hardware offload working successfully with TLS 1.3 AES-GCM-128 and AES-GCM-256 cipher suites. Signed-off-by: Rishikesh Jethwani --- .../ethernet/mellanox/mlx5/core/en_accel/ktls.h | 8 +++++++- .../mellanox/mlx5/core/en_accel/ktls_txrx.c | 14 +++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h index 07a04a142a2e..0469ca6a0762 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h @@ -30,7 +30,9 @@ static inline bool mlx5e_is_ktls_device(struct mlx5_core_dev *mdev) return false; return (MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_128) || - MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_256)); + MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_256) || + MLX5_CAP_TLS(mdev, tls_1_3_aes_gcm_128) || + MLX5_CAP_TLS(mdev, tls_1_3_aes_gcm_256)); } static inline bool mlx5e_ktls_type_check(struct mlx5_core_dev *mdev, @@ -40,10 +42,14 @@ static inline bool mlx5e_ktls_type_check(struct mlx5_core_dev *mdev, case TLS_CIPHER_AES_GCM_128: if (crypto_info->version == TLS_1_2_VERSION) return MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_128); + else if (crypto_info->version == TLS_1_3_VERSION) + return MLX5_CAP_TLS(mdev, tls_1_3_aes_gcm_128); break; case TLS_CIPHER_AES_GCM_256: if (crypto_info->version == TLS_1_2_VERSION) return MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_256); + else if (crypto_info->version == TLS_1_3_VERSION) + return MLX5_CAP_TLS(mdev, tls_1_3_aes_gcm_256); break; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c index 570a912dd6fa..f3f90ad6c6cf 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c @@ -6,6 +6,7 @@ enum { MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2 = 0x2, + MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_3 = 0x3, }; enum { @@ -15,8 +16,10 @@ enum { #define EXTRACT_INFO_FIELDS do { \ salt = info->salt; \ rec_seq = info->rec_seq; \ + iv = info->iv; \ salt_sz = sizeof(info->salt); \ rec_seq_sz = sizeof(info->rec_seq); \ + iv_sz = sizeof(info->iv); \ } while (0) static void @@ -25,8 +28,8 @@ fill_static_params(struct mlx5_wqe_tls_static_params_seg *params, u32 key_id, u32 resync_tcp_sn) { char *initial_rn, *gcm_iv; - u16 salt_sz, rec_seq_sz; - char *salt, *rec_seq; + u16 salt_sz, rec_seq_sz, iv_sz; + char *salt, *rec_seq, *iv; u8 tls_version; u8 *ctx; @@ -59,7 +62,12 @@ fill_static_params(struct mlx5_wqe_tls_static_params_seg *params, memcpy(gcm_iv, salt, salt_sz); memcpy(initial_rn, rec_seq, rec_seq_sz); - tls_version = MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2; + if (crypto_info->crypto_info.version == TLS_1_3_VERSION) { + memcpy(gcm_iv + salt_sz, iv, iv_sz); + tls_version = MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_3; + } else { + tls_version = MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2; + } MLX5_SET(tls_static_params, ctx, tls_version, tls_version); MLX5_SET(tls_static_params, ctx, const_1, 1); -- 2.25.1