Each kTLS connection requires a crypto key context ID (KID). These KIDs are allocated from the firmware in batches. Add data structure to store these IDs. The bnxt_kid_info structure stores a batch of IDs and it can be linked as we allocate more batches. There is a bitmap in the structure to keep track of which ones are in use. Add APIs to allocate and free these KIDs. Reviewed-by: Andy Gospodarek Reviewed-by: Pavan Chebbi Signed-off-by: Michael Chan --- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 3 + .../net/ethernet/broadcom/bnxt/bnxt_crypto.c | 257 ++++++++++++++++++ .../net/ethernet/broadcom/bnxt/bnxt_crypto.h | 66 +++++ 3 files changed, 326 insertions(+) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 926665c8bb79..4f5de41d4e86 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -17219,6 +17219,9 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) rc = bnxt_dl_register(bp); if (rc) goto init_err_dl; + rc = bnxt_crypto_init(bp); + if (rc) + bnxt_free_crypto_info(bp); INIT_LIST_HEAD(&bp->usr_fltr_list); diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_crypto.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_crypto.c index ee154f1e4e19..4ea3e67be9f5 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_crypto.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_crypto.c @@ -8,6 +8,7 @@ #include #include "bnxt.h" +#include "bnxt_hwrm.h" #include "bnxt_crypto.h" static u32 bnxt_get_max_crypto_key_ctx(struct bnxt *bp, int key_type) @@ -57,12 +58,51 @@ void bnxt_alloc_crypto_info(struct bnxt *bp, kctx = &crypto->kctx[i]; kctx->type = i; kctx->max_ctx = bnxt_get_max_crypto_key_ctx(bp, i); + INIT_LIST_HEAD(&kctx->list); + spin_lock_init(&kctx->lock); + atomic_set(&kctx->alloc_pending, 0); + init_waitqueue_head(&kctx->alloc_pending_wq); } bp->crypto_info = crypto; } crypto->max_key_ctxs_alloc = max_keys; } +/** + * bnxt_clear_crypto - Clear all crypto key contexts + * @bp: pointer to bnxt device + * + * Clears all key context allocations during shutdown or firmware reset. + * Frees all key info structures and bitmaps, and increments the epoch + * counter to invalidate any outstanding key references. + * + * This function assumes serialization (called during shutdown) and does + * not use locking. + * + * Context: Process context during shutdown/reset + */ +void bnxt_clear_crypto(struct bnxt *bp) +{ + struct bnxt_crypto_info *crypto = bp->crypto_info; + struct bnxt_kid_info *kid, *tmp; + struct bnxt_kctx *kctx; + int i; + + if (!crypto) + return; + + /* Shutting down or FW reset, no need to protect the lists. */ + for (i = 0; i < BNXT_MAX_CRYPTO_KEY_TYPE; i++) { + kctx = &crypto->kctx[i]; + list_for_each_entry_safe(kid, tmp, &kctx->list, list) { + list_del(&kid->list); + kfree(kid); + } + kctx->total_alloc = 0; + kctx->epoch++; + } +} + /** * bnxt_free_crypto_info - Free crypto offload resources * @bp: pointer to bnxt device @@ -73,6 +113,7 @@ void bnxt_alloc_crypto_info(struct bnxt *bp, */ void bnxt_free_crypto_info(struct bnxt *bp) { + bnxt_clear_crypto(bp); kfree(bp->crypto_info); bp->crypto_info = NULL; } @@ -107,3 +148,219 @@ void bnxt_hwrm_reserve_pf_key_ctxs(struct bnxt *bp, req->enables |= cpu_to_le32(FUNC_CFG_REQ_ENABLES_KTLS_TX_KEY_CTXS | FUNC_CFG_REQ_ENABLES_KTLS_RX_KEY_CTXS); } + +static int bnxt_key_ctx_store(struct bnxt_kctx *kctx, __le32 *key_buf, u32 num, + bool contig, u8 kind, u32 *id) +{ + struct bnxt_kid_info *kid; + u32 i; + + for (i = 0; i < num; ) { + kid = kzalloc_obj(*kid); + if (!kid) + return -ENOMEM; + kid->start_id = le32_to_cpu(key_buf[i]); + kid->type = kctx->type; + kid->kind = kind; + if (contig) + kid->count = num; + else + kid->count = 1; + bitmap_set(kid->ids, 0, kid->count); + if (id && !i) { + clear_bit(0, kid->ids); + *id = BNXT_SET_KID(kctx, kid->start_id); + } + spin_lock(&kctx->lock); + list_add_tail_rcu(&kid->list, &kctx->list); + kctx->total_alloc += kid->count; + spin_unlock(&kctx->lock); + i += kid->count; + } + return 0; +} + +/* Note that the driver does not free the key contexts. They are freed + * by the FW during FLR and HWRM_FUNC_RESET. + */ +static int bnxt_hwrm_key_ctx_alloc(struct bnxt *bp, struct bnxt_kctx *kctx, + u8 kind, u32 num, u32 *id) +{ + struct bnxt_crypto_info *crypto = bp->crypto_info; + struct hwrm_func_key_ctx_alloc_output *resp; + struct hwrm_func_key_ctx_alloc_input *req; + dma_addr_t mapping; + int pending_count; + __le32 *key_buf; + u32 num_alloc; + bool contig; + int rc; + + num = min3(num, crypto->max_key_ctxs_alloc, (u32)BNXT_KID_BATCH_SIZE); + rc = hwrm_req_init(bp, req, HWRM_FUNC_KEY_CTX_ALLOC); + if (rc) + return rc; + + key_buf = hwrm_req_dma_slice(bp, req, num * 4, &mapping); + if (!key_buf) { + rc = -ENOMEM; + goto key_alloc_exit; + } + req->dma_bufr_size_bytes = cpu_to_le32(num * 4); + req->host_dma_addr = cpu_to_le64(mapping); + resp = hwrm_req_hold(bp, req); + + req->key_ctx_type = kctx->type; + req->num_key_ctxs = cpu_to_le16(num); + + pending_count = atomic_inc_return(&kctx->alloc_pending); + rc = hwrm_req_send(bp, req); + atomic_dec(&kctx->alloc_pending); + if (rc) + goto key_alloc_exit_wake; + + num_alloc = le16_to_cpu(resp->num_key_ctxs_allocated); + if (num_alloc > num) + netdev_warn(bp->dev, + "FW allocated more keys (%d) than requested (%d)\n", + num_alloc, num); + else + num = num_alloc; + contig = resp->flags & + FUNC_KEY_CTX_ALLOC_RESP_FLAGS_KEY_CTXS_CONTIGUOUS; + rc = bnxt_key_ctx_store(kctx, key_buf, num, contig, kind, id); + +key_alloc_exit_wake: + if (pending_count >= BNXT_KCTX_ALLOC_PENDING_MAX) + wake_up_all(&kctx->alloc_pending_wq); +key_alloc_exit: + hwrm_req_drop(bp, req); + return rc; +} + +bool bnxt_kid_valid(struct bnxt_kctx *kctx, u32 id) +{ + struct bnxt_kid_info *kid; + bool valid = false; + u32 epoch; + + epoch = BNXT_KID_EPOCH(id); + if (epoch != kctx->epoch) + return false; + + id = BNXT_KID_HW(id); + rcu_read_lock(); + list_for_each_entry_rcu(kid, &kctx->list, list) { + if (id >= kid->start_id && id < kid->start_id + kid->count) { + if (!test_bit(id - kid->start_id, kid->ids)) { + valid = true; + break; + } + } + } + rcu_read_unlock(); + return valid; +} + +static int bnxt_alloc_one_kctx(struct bnxt_kctx *kctx, u8 kind, u32 *id) +{ + struct bnxt_kid_info *kid; + int rc = -ENOMEM; + + rcu_read_lock(); + list_for_each_entry_rcu(kid, &kctx->list, list) { + u32 idx = 0; + + if (kid->kind != kind) + continue; + do { + idx = find_next_bit(kid->ids, kid->count, idx); + if (idx >= kid->count) + break; + if (test_and_clear_bit(idx, kid->ids)) { + *id = BNXT_SET_KID(kctx, kid->start_id + idx); + rc = 0; + goto alloc_done; + } + } while (1); + } + +alloc_done: + rcu_read_unlock(); + return rc; +} + +void bnxt_free_one_kctx(struct bnxt_kctx *kctx, u32 id) +{ + struct bnxt_kid_info *kid; + + id = BNXT_KID_HW(id); + rcu_read_lock(); + list_for_each_entry_rcu(kid, &kctx->list, list) { + if (id >= kid->start_id && id < kid->start_id + kid->count) { + set_bit(id - kid->start_id, kid->ids); + break; + } + } + rcu_read_unlock(); +} + +#define BNXT_KCTX_ALLOC_RETRY_MAX 3 + +int bnxt_key_ctx_alloc_one(struct bnxt *bp, struct bnxt_kctx *kctx, u8 kind, + u32 *id) +{ + int rc, retry = 0; + + while (retry++ < BNXT_KCTX_ALLOC_RETRY_MAX) { + rc = bnxt_alloc_one_kctx(kctx, kind, id); + if (!rc) + return 0; + + if ((kctx->total_alloc + BNXT_KID_BATCH_SIZE) > kctx->max_ctx) + return -ENOSPC; + + if (!BNXT_KCTX_ALLOC_OK(kctx)) { + wait_event(kctx->alloc_pending_wq, + BNXT_KCTX_ALLOC_OK(kctx)); + continue; + } + rc = bnxt_hwrm_key_ctx_alloc(bp, kctx, kind, + BNXT_KID_BATCH_SIZE, id); + if (!rc) + return 0; + } + return -EAGAIN; +} + +int bnxt_crypto_init(struct bnxt *bp) +{ + struct bnxt_crypto_info *crypto = bp->crypto_info; + struct bnxt_hw_resc *hw_resc = &bp->hw_resc; + struct bnxt_hw_crypto_resc *crypto_resc; + int rc; + + if (!crypto) + return 0; + + crypto_resc = &hw_resc->crypto_resc; + BNXT_TCK(crypto).max_ctx = crypto_resc->resv_tx_key_ctxs; + BNXT_RCK(crypto).max_ctx = crypto_resc->resv_rx_key_ctxs; + + if (!BNXT_TCK(crypto).max_ctx || !BNXT_RCK(crypto).max_ctx) { + bnxt_free_crypto_info(bp); + return 0; + } + + rc = bnxt_hwrm_key_ctx_alloc(bp, &BNXT_TCK(crypto), BNXT_CTX_KIND_CK_TX, + BNXT_KID_BATCH_SIZE, NULL); + if (rc) + return rc; + + rc = bnxt_hwrm_key_ctx_alloc(bp, &BNXT_RCK(crypto), BNXT_CTX_KIND_CK_RX, + BNXT_KID_BATCH_SIZE, NULL); + if (rc) + return rc; + + return 0; +} diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_crypto.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_crypto.h index e090491006db..ecdf18ba6d83 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_crypto.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_crypto.h @@ -16,11 +16,43 @@ enum bnxt_crypto_type { BNXT_MAX_CRYPTO_KEY_TYPE, }; +#define BNXT_KID_BATCH_SIZE 128 + +struct bnxt_kid_info { + struct list_head list; + u8 type; + u8 kind; + u32 start_id; + u32 count; + DECLARE_BITMAP(ids, BNXT_KID_BATCH_SIZE); +}; + struct bnxt_kctx { + struct list_head list; + /* to serialize update to the linked list and total_alloc */ + spinlock_t lock; u8 type; + u8 epoch; + u32 total_alloc; u32 max_ctx; + atomic_t alloc_pending; +#define BNXT_KCTX_ALLOC_PENDING_MAX 8 + wait_queue_head_t alloc_pending_wq; }; +#define BNXT_KID_HW_MASK 0xffffff +#define BNXT_KID_HW(kid) ((kid) & BNXT_KID_HW_MASK) +#define BNXT_KID_EPOCH_MASK 0xff000000 +#define BNXT_KID_EPOCH_SHIFT 24 +#define BNXT_KID_EPOCH(kid) (((kid) & BNXT_KID_EPOCH_MASK) >> \ + BNXT_KID_EPOCH_SHIFT) + +#define BNXT_SET_KID(kctx, kid) \ + ((kid) | ((kctx)->epoch << BNXT_KID_EPOCH_SHIFT)) + +#define BNXT_KCTX_ALLOC_OK(kctx) \ + (atomic_read(&((kctx)->alloc_pending)) < BNXT_KCTX_ALLOC_PENDING_MAX) + struct bnxt_crypto_info { u16 max_key_ctxs_alloc; @@ -30,18 +62,31 @@ struct bnxt_crypto_info { #define BNXT_TCK(crypto) ((crypto)->kctx[BNXT_TX_CRYPTO_KEY_TYPE]) #define BNXT_RCK(crypto) ((crypto)->kctx[BNXT_RX_CRYPTO_KEY_TYPE]) +#define BNXT_CTX_KIND_CK_TX 0x11 +#define BNXT_CTX_KIND_CK_RX 0x12 + #ifdef CONFIG_BNXT_TLS void bnxt_alloc_crypto_info(struct bnxt *bp, struct hwrm_func_qcaps_output *resp); +void bnxt_clear_crypto(struct bnxt *bp); void bnxt_free_crypto_info(struct bnxt *bp); void bnxt_hwrm_reserve_pf_key_ctxs(struct bnxt *bp, struct hwrm_func_cfg_input *req); +bool bnxt_kid_valid(struct bnxt_kctx *kctx, u32 id); +void bnxt_free_one_kctx(struct bnxt_kctx *kctx, u32 id); +int bnxt_key_ctx_alloc_one(struct bnxt *bp, struct bnxt_kctx *kctx, u8 kind, + u32 *id); +int bnxt_crypto_init(struct bnxt *bp); #else static inline void bnxt_alloc_crypto_info(struct bnxt *bp, struct hwrm_func_qcaps_output *resp) { } +static inline void bnxt_clear_crypto(struct bnxt *bp) +{ +} + static inline void bnxt_free_crypto_info(struct bnxt *bp) { } @@ -50,5 +95,26 @@ static inline void bnxt_hwrm_reserve_pf_key_ctxs(struct bnxt *bp, struct hwrm_func_cfg_input *req) { } + +static inline bool bnxt_kid_valid(struct bnxt_kctx *kctx, u32 id) +{ + return false; +} + +static inline void bnxt_free_one_kctx(struct bnxt_kctx *kctx, u32 id) +{ +} + +static inline int bnxt_key_ctx_alloc_one(struct bnxt *bp, + struct bnxt_kctx *kctx, u8 kind, + u32 *id) +{ + return -EOPNOTSUPP; +} + +static inline int bnxt_crypto_init(struct bnxt *bp) +{ + return 0; +} #endif /* CONFIG_BNXT_TLS */ #endif /* BNXT_CRYPTO_H */ -- 2.51.0