From: Raed Salem Add PSP offload related structs, layouts, and enumerations. Implement .set_config and .rx_spi_alloc PSP device operations. Driver does not need to make use of the .set_config operation. Stub .assoc_add and .assoc_del PSP operations. Introduce the MLX5_EN_PSP configuration option for enabling PSP offload support on mlx5 devices. Signed-off-by: Raed Salem Signed-off-by: Rahul Rameshbabu Signed-off-by: Cosmin Ratiu Signed-off-by: Daniel Zahka --- Notes: v11: - check psp_crypto_offload cap in mlx5_is_psp_device() v7: - use flexible array declaration instead of 0-length array declaration in struct mlx5_ifc_psp_gen_spi_out_bits v4: - remove unneeded psp.c/psp.h files - remove unneeded struct psp_key_spi usage v1: - https://lore.kernel.org/netdev/20240510030435.120935-10-kuba@kernel.org/ .../net/ethernet/mellanox/mlx5/core/Kconfig | 11 ++ .../net/ethernet/mellanox/mlx5/core/Makefile | 2 + drivers/net/ethernet/mellanox/mlx5/core/en.h | 3 + .../ethernet/mellanox/mlx5/core/en/params.c | 4 +- .../mellanox/mlx5/core/en_accel/psp.c | 168 ++++++++++++++++++ .../mellanox/mlx5/core/en_accel/psp.h | 43 +++++ .../net/ethernet/mellanox/mlx5/core/en_main.c | 9 + 7 files changed, 239 insertions(+), 1 deletion(-) create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.c create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.h diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig index 8ef2ac2060ba..3c3e84100d5a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig +++ b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig @@ -207,3 +207,14 @@ config MLX5_DPLL help DPLL support in Mellanox Technologies ConnectX NICs. +config MLX5_EN_PSP + bool "Mellanox Technologies support for PSP cryptography-offload acceleration" + depends on INET_PSP + depends on MLX5_CORE_EN + default y + help + mlx5 device offload support for Google PSP Security Protocol offload. + Adds support for PSP encryption offload and for SPI and key generation + interfaces to PSP Stack which supports PSP crypto offload. + + If unsure, say Y. diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index d77696f46eb5..04395806255d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -112,6 +112,8 @@ mlx5_core-$(CONFIG_MLX5_EN_TLS) += en_accel/ktls_stats.o \ en_accel/fs_tcp.o en_accel/ktls.o en_accel/ktls_txrx.o \ en_accel/ktls_tx.o en_accel/ktls_rx.o +mlx5_core-$(CONFIG_MLX5_EN_PSP) += en_accel/psp.o + # # SW Steering # diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 0dd3bc0f4caa..32e7b791dd6a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -936,6 +936,9 @@ struct mlx5e_priv { #ifdef CONFIG_MLX5_EN_IPSEC struct mlx5e_ipsec *ipsec; #endif +#ifdef CONFIG_MLX5_EN_PSP + struct mlx5e_psp *psp; +#endif #ifdef CONFIG_MLX5_EN_TLS struct mlx5e_tls *tls; #endif diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c index 3cca06a74cf9..9975a9d8945b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c @@ -6,6 +6,7 @@ #include "en/port.h" #include "en_accel/en_accel.h" #include "en_accel/ipsec.h" +#include "en_accel/psp.h" #include #include #include @@ -1003,7 +1004,8 @@ void mlx5e_build_sq_param(struct mlx5_core_dev *mdev, bool allow_swp; allow_swp = mlx5_geneve_tx_allowed(mdev) || - (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_CRYPTO); + (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_CRYPTO) || + mlx5_is_psp_device(mdev); mlx5e_build_sq_param_common(mdev, param); MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size); MLX5_SET(sqc, sqc, allow_swp, allow_swp); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.c new file mode 100644 index 000000000000..87eba63451a3 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.c @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ +#include +#include +#include +#include "mlx5_core.h" +#include "psp.h" +#include "lib/crypto.h" +#include "en_accel/psp.h" + +static int +mlx5e_psp_set_config(struct psp_dev *psd, struct psp_dev_config *conf, + struct netlink_ext_ack *extack) +{ + return 0; /* TODO: this should actually do things to the device */ +} + +static int +mlx5e_psp_generate_key_spi(struct mlx5_core_dev *mdev, + enum mlx5_psp_gen_spi_in_key_size keysz, + unsigned int keysz_bytes, + struct psp_key_parsed *key) +{ + u32 out[MLX5_ST_SZ_DW(psp_gen_spi_out) + MLX5_ST_SZ_DW(key_spi)] = {}; + u32 in[MLX5_ST_SZ_DW(psp_gen_spi_in)] = {}; + void *outkey; + int err; + + WARN_ON_ONCE(keysz_bytes > PSP_MAX_KEY); + + MLX5_SET(psp_gen_spi_in, in, opcode, MLX5_CMD_OP_PSP_GEN_SPI); + MLX5_SET(psp_gen_spi_in, in, key_size, keysz); + MLX5_SET(psp_gen_spi_in, in, num_of_spi, 1); + err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); + if (err) + return err; + + outkey = MLX5_ADDR_OF(psp_gen_spi_out, out, key_spi); + key->spi = cpu_to_be32(MLX5_GET(key_spi, outkey, spi)); + memcpy(key->key, MLX5_ADDR_OF(key_spi, outkey, key) + 32 - keysz_bytes, + keysz_bytes); + + return 0; +} + +static int +mlx5e_psp_rx_spi_alloc(struct psp_dev *psd, u32 version, + struct psp_key_parsed *assoc, + struct netlink_ext_ack *extack) +{ + struct mlx5e_priv *priv = netdev_priv(psd->main_netdev); + enum mlx5_psp_gen_spi_in_key_size keysz; + u8 keysz_bytes; + + switch (version) { + case PSP_VERSION_HDR0_AES_GCM_128: + keysz = MLX5_PSP_GEN_SPI_IN_KEY_SIZE_128; + keysz_bytes = 16; + break; + case PSP_VERSION_HDR0_AES_GCM_256: + keysz = MLX5_PSP_GEN_SPI_IN_KEY_SIZE_256; + keysz_bytes = 32; + break; + default: + return -EINVAL; + } + + return mlx5e_psp_generate_key_spi(priv->mdev, keysz, keysz_bytes, assoc); +} + +static int mlx5e_psp_assoc_add(struct psp_dev *psd, struct psp_assoc *pas, + struct netlink_ext_ack *extack) +{ + struct mlx5e_priv *priv = netdev_priv(psd->main_netdev); + + mlx5_core_dbg(priv->mdev, "PSP assoc add: rx: %u, tx: %u\n", + be32_to_cpu(pas->rx.spi), be32_to_cpu(pas->tx.spi)); + + return -EINVAL; +} + +static void mlx5e_psp_assoc_del(struct psp_dev *psd, struct psp_assoc *pas) +{ +} + +static struct psp_dev_ops mlx5_psp_ops = { + .set_config = mlx5e_psp_set_config, + .rx_spi_alloc = mlx5e_psp_rx_spi_alloc, + .tx_key_add = mlx5e_psp_assoc_add, + .tx_key_del = mlx5e_psp_assoc_del, +}; + +void mlx5e_psp_unregister(struct mlx5e_priv *priv) +{ + if (!priv->psp || !priv->psp->psp) + return; + + psp_dev_unregister(priv->psp->psp); +} + +void mlx5e_psp_register(struct mlx5e_priv *priv) +{ + /* FW Caps missing */ + if (!priv->psp) + return; + + priv->psp->caps.assoc_drv_spc = sizeof(u32); + priv->psp->caps.versions = 1 << PSP_VERSION_HDR0_AES_GCM_128; + if (MLX5_CAP_PSP(priv->mdev, psp_crypto_esp_aes_gcm_256_encrypt) && + MLX5_CAP_PSP(priv->mdev, psp_crypto_esp_aes_gcm_256_decrypt)) + priv->psp->caps.versions |= 1 << PSP_VERSION_HDR0_AES_GCM_256; + + priv->psp->psp = psp_dev_create(priv->netdev, &mlx5_psp_ops, + &priv->psp->caps, NULL); + if (IS_ERR(priv->psp->psp)) + mlx5_core_err(priv->mdev, "PSP failed to register due to %pe\n", + priv->psp->psp); +} + +int mlx5e_psp_init(struct mlx5e_priv *priv) +{ + struct mlx5_core_dev *mdev = priv->mdev; + struct mlx5e_psp *psp; + + if (!mlx5_is_psp_device(mdev)) { + mlx5_core_dbg(mdev, "PSP offload not supported\n"); + return -EOPNOTSUPP; + } + + if (!MLX5_CAP_ETH(mdev, swp)) { + mlx5_core_dbg(mdev, "SWP not supported\n"); + return -EOPNOTSUPP; + } + + if (!MLX5_CAP_ETH(mdev, swp_csum)) { + mlx5_core_dbg(mdev, "SWP checksum not supported\n"); + return -EOPNOTSUPP; + } + + if (!MLX5_CAP_ETH(mdev, swp_csum_l4_partial)) { + mlx5_core_dbg(mdev, "SWP L4 partial checksum not supported\n"); + return -EOPNOTSUPP; + } + + if (!MLX5_CAP_ETH(mdev, swp_lso)) { + mlx5_core_dbg(mdev, "PSP LSO not supported\n"); + return -EOPNOTSUPP; + } + + psp = kzalloc(sizeof(*psp), GFP_KERNEL); + if (!psp) + return -ENOMEM; + + priv->psp = psp; + mlx5_core_dbg(priv->mdev, "PSP attached to netdevice\n"); + return 0; +} + +void mlx5e_psp_cleanup(struct mlx5e_priv *priv) +{ + struct mlx5e_psp *psp = priv->psp; + + if (!psp) + return; + + priv->psp = NULL; + kfree(psp); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.h new file mode 100644 index 000000000000..40dbdb3e5d73 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#ifndef __MLX5E_ACCEL_PSP_H__ +#define __MLX5E_ACCEL_PSP_H__ +#if IS_ENABLED(CONFIG_MLX5_EN_PSP) +#include +#include "en.h" + +struct mlx5e_psp { + struct psp_dev *psp; + struct psp_dev_caps caps; +}; + +static inline bool mlx5_is_psp_device(struct mlx5_core_dev *mdev) +{ + if (!MLX5_CAP_GEN(mdev, psp)) + return false; + + if (!MLX5_CAP_PSP(mdev, psp_crypto_offload) || + !MLX5_CAP_PSP(mdev, psp_crypto_esp_aes_gcm_128_encrypt) || + !MLX5_CAP_PSP(mdev, psp_crypto_esp_aes_gcm_128_decrypt)) + return false; + + return true; +} + +void mlx5e_psp_register(struct mlx5e_priv *priv); +void mlx5e_psp_unregister(struct mlx5e_priv *priv); +int mlx5e_psp_init(struct mlx5e_priv *priv); +void mlx5e_psp_cleanup(struct mlx5e_priv *priv); +#else +static inline bool mlx5_is_psp_device(struct mlx5_core_dev *mdev) +{ + return false; +} + +static inline void mlx5e_psp_register(struct mlx5e_priv *priv) { } +static inline void mlx5e_psp_unregister(struct mlx5e_priv *priv) { } +static inline int mlx5e_psp_init(struct mlx5e_priv *priv) { return 0; } +static inline void mlx5e_psp_cleanup(struct mlx5e_priv *priv) { } +#endif /* CONFIG_MLX5_EN_PSP */ +#endif /* __MLX5E_ACCEL_PSP_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 714cce595692..e44cdea5e2b3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -53,6 +53,7 @@ #include "en_tc.h" #include "en_rep.h" #include "en_accel/ipsec.h" +#include "en_accel/psp.h" #include "en_accel/macsec.h" #include "en_accel/en_accel.h" #include "en_accel/ktls.h" @@ -5904,6 +5905,7 @@ static int mlx5e_nic_init(struct mlx5_core_dev *mdev, if (take_rtnl) rtnl_lock(); + mlx5e_psp_register(priv); /* update XDP supported features */ mlx5e_set_xdp_feature(netdev); @@ -5916,6 +5918,7 @@ static int mlx5e_nic_init(struct mlx5_core_dev *mdev, static void mlx5e_nic_cleanup(struct mlx5e_priv *priv) { mlx5e_health_destroy_reporters(priv); + mlx5e_psp_unregister(priv); mlx5e_ktls_cleanup(priv); mlx5e_fs_cleanup(priv->fs); debugfs_remove_recursive(priv->dfs_root); @@ -6043,6 +6046,10 @@ static void mlx5e_nic_enable(struct mlx5e_priv *priv) if (err) mlx5_core_err(mdev, "MACsec initialization failed, %d\n", err); + err = mlx5e_psp_init(priv); + if (err) + mlx5_core_err(mdev, "PSP initialization failed, %d\n", err); + /* Marking the link as currently not needed by the Driver */ if (!netif_running(netdev)) mlx5e_modify_admin_state(mdev, MLX5_PORT_DOWN); @@ -6106,6 +6113,7 @@ static void mlx5e_nic_disable(struct mlx5e_priv *priv) mlx5e_disable_async_events(priv); mlx5_lag_remove_netdev(mdev, priv->netdev); mlx5_vxlan_reset_to_default(mdev->vxlan); + mlx5e_psp_cleanup(priv); mlx5e_macsec_cleanup(priv); mlx5e_ipsec_cleanup(priv); } @@ -6764,6 +6772,7 @@ static void _mlx5e_remove(struct auxiliary_device *adev) * is already unregistered before changing to NIC profile. */ if (priv->netdev->reg_state == NETREG_REGISTERED) { + mlx5e_psp_unregister(priv); unregister_netdev(priv->netdev); _mlx5e_suspend(adev, false); } else { -- 2.47.3