Producers of sp_events may store data for bnxt_sp_task() right before raising the event, e.g. the IRQ affinity notifier publishes the new TPH Steering Tag, the async event handler records the FW echo request data or the FW reset wait limits. set_bit() is not ordered, so on weakly ordered architectures the event bit may become visible first. If bnxt_sp_task() is already running it can consume the event and read the stale data. The queue_work() which follows only re-runs the task with the event bit already cleared, so the update is lost. Add a helper which issues smp_mb__before_atomic() before setting the bit, it pairs with the fully ordered test_and_clear_bit() in bnxt_sp_task(). Use it for all events, including those which carry no data, so that new producers don't have to think about it. Reported by an LLM reviewer while working on subsequent patches. Very unlikely to matter in practice. Signed-off-by: Jakub Kicinski --- v2: - convert the producers which set the bit directly (Björn) v1: https://lore.kernel.org/20261118023905.3050661-2-kuba@kernel.org --- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 33 ++++++++++++++--------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 52606db3f7b4..9ddd75a0fd2f 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -392,9 +392,18 @@ static void __bnxt_queue_sp_work(struct bnxt *bp) schedule_work(&bp->sp_task); } +static void bnxt_set_sp_event(struct bnxt *bp, unsigned int event) +{ + /* Make the data the event carries visible before the event bit, + * pairs with test_and_clear_bit() in bnxt_sp_task(). + */ + smp_mb__before_atomic(); + set_bit(event, &bp->sp_event); +} + static void bnxt_queue_sp_work(struct bnxt *bp, unsigned int event) { - set_bit(event, &bp->sp_event); + bnxt_set_sp_event(bp, event); __bnxt_queue_sp_work(bp); } @@ -403,9 +412,9 @@ static void bnxt_sched_reset_rxr(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) if (!rxr->bnapi->in_reset) { rxr->bnapi->in_reset = true; if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) - set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); + bnxt_set_sp_event(bp, BNXT_RESET_TASK_SP_EVENT); else - set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event); + bnxt_set_sp_event(bp, BNXT_RST_RING_SP_EVENT); __bnxt_queue_sp_work(bp); } rxr->rx_next_cons = 0xffff; @@ -2751,7 +2760,7 @@ static bool bnxt_event_error_report(struct bnxt *bp, u32 data1, u32 data2) BNXT_EVENT_THERMAL_THRESHOLD_TEMP(data2)); if (notify) { bp->thermal_threshold_type = type; - set_bit(BNXT_THERMAL_THRESHOLD_SP_EVENT, &bp->sp_event); + bnxt_set_sp_event(bp, BNXT_THERMAL_THRESHOLD_SP_EVENT); return true; } return false; @@ -2817,18 +2826,18 @@ static int bnxt_async_event_process(struct bnxt *bp, netdev_warn(bp->dev, "Link speed %d no longer supported\n", speed); } - set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event); + bnxt_set_sp_event(bp, BNXT_LINK_SPEED_CHNG_SP_EVENT); } fallthrough; case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE: case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE: - set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event); + bnxt_set_sp_event(bp, BNXT_LINK_CFG_CHANGE_SP_EVENT); fallthrough; case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE: - set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event); + bnxt_set_sp_event(bp, BNXT_LINK_CHNG_SP_EVENT); break; case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD: - set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event); + bnxt_set_sp_event(bp, BNXT_HWRM_PF_UNLOAD_SP_EVENT); break; case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: { u16 port_id = BNXT_GET_EVENT_PORT(data1); @@ -2839,13 +2848,13 @@ static int bnxt_async_event_process(struct bnxt *bp, if (bp->pf.port_id != port_id) break; - set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event); + bnxt_set_sp_event(bp, BNXT_HWRM_PORT_MODULE_SP_EVENT); break; } case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE: if (BNXT_PF(bp)) goto async_event_process_exit; - set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event); + bnxt_set_sp_event(bp, BNXT_RESET_TASK_SILENT_SP_EVENT); break; case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: { char *type_str = "Solicited"; @@ -2877,7 +2886,7 @@ static int bnxt_async_event_process(struct bnxt *bp, type_str, data1, data2, bp->fw_reset_min_dsecs * 100, bp->fw_reset_max_dsecs * 100); - set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event); + bnxt_set_sp_event(bp, BNXT_FW_RESET_NOTIFY_SP_EVENT); break; } case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: { @@ -2955,7 +2964,7 @@ static int bnxt_async_event_process(struct bnxt *bp, if (fw_health) { fw_health->echo_req_data1 = data1; fw_health->echo_req_data2 = data2; - set_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event); + bnxt_set_sp_event(bp, BNXT_FW_ECHO_REQUEST_SP_EVENT); break; } goto async_event_process_exit; -- 2.56.1 The driver picks the NUMA node for each Rx ring's page pool with cpumask_local_spread() at open. IRQ affinity configured by the user is already recorded by the driver and persists across close/open cycles. Use it instead of cpumask_local_spread(). The affinity notifier runs without any of the locks held by the allocation paths, and copying a multi-word cpumask is not atomic. Like for the TPH Steering Tag, have the notifier compute the node (of the first online CPU in the mask) and publish it with WRITE_ONCE(). Queue restart (bnxt_queue_mem_alloc()) needs to resolve the NUMA node instead of copying the nid of the old page pool. Reviewed-by: Björn Töpel Reviewed-by: Joe Damato Signed-off-by: Jakub Kicinski --- v2: - correct the comment on ring_nid, it's not only written by the notifier - allocate ring_nid with kmalloc_objs() v1: https://lore.kernel.org/20261118023905.3050661-3-kuba@kernel.org --- drivers/net/ethernet/broadcom/bnxt/bnxt.h | 5 +++ drivers/net/ethernet/broadcom/bnxt/bnxt.c | 46 +++++++++++++++++++---- 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h index 41d05efd6307..61b86fa9cadb 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h @@ -2488,6 +2488,11 @@ struct bnxt { * rather than being our own default placement. */ unsigned long *ring_affinity_set; + /* NUMA node of the first online CPU in the affinity mask if it was + * configured explicitly by the user, otherwise NUMA_NO_NODE. + * Written by the IRQ affinity notifier, readers must use READ_ONCE(). + */ + int *ring_nid; int max_irqs; int total_irqs; int ulp_num_msix_want; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 9ddd75a0fd2f..6e503822d7bb 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -3823,6 +3823,20 @@ static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem) return 0; } +/* Node of the CPU expected to service cp ring @n. If user did not configure + * the affinity explicitly, use the default spread (historic behavior). + */ +static int bnxt_cp_num_to_node(struct bnxt *bp, int n) +{ + int nid = READ_ONCE(bp->ring_nid[n]); + unsigned int cpu; + + if (nid != NUMA_NO_NODE) + return nid; + cpu = cpumask_local_spread(n, dev_to_node(&bp->pdev->dev)); + return cpu_to_node(cpu); +} + static void bnxt_free_one_tpa_info(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) { @@ -4025,8 +4039,7 @@ static int bnxt_alloc_rx_agg_bmap(struct bnxt *bp, struct bnxt_rx_ring_info *rxr static int bnxt_alloc_rx_rings(struct bnxt *bp) { - int numa_node = dev_to_node(&bp->pdev->dev); - int i, rc = 0, agg_rings = 0, cpu; + int i, rc = 0, agg_rings = 0; if (!bp->rx_ring) return -ENOMEM; @@ -4041,8 +4054,7 @@ static int bnxt_alloc_rx_rings(struct bnxt *bp) ring = &rxr->rx_ring_struct; - cpu = cpumask_local_spread(i, numa_node); - cpu_node = cpu_to_node(cpu); + cpu_node = bnxt_cp_num_to_node(bp, i); netdev_dbg(bp->dev, "Allocating page pool for rx_ring[%d] on numa_node: %d\n", i, cpu_node); rc = bnxt_alloc_rx_page_pool(bp, rxr, cpu_node); @@ -11917,6 +11929,8 @@ static void bnxt_irq_affinity_notify(struct irq_affinity_notify *notify, const cpumask_t *mask) { struct bnxt_irq *irq; + unsigned int cpu; + int nid; u16 tag; irq = container_of(notify, struct bnxt_irq, affinity_notify); @@ -11924,6 +11938,14 @@ static void bnxt_irq_affinity_notify(struct irq_affinity_notify *notify, cpumask_copy(irq->bp->ring_cpu_mask[irq->ring_nr], mask); set_bit(irq->ring_nr, irq->bp->ring_affinity_set); + /* Readers can't access ring_cpu_mask atomically, so pre-compute + * the info they care about (NUMA node, TPH tag) and store it + * with WRITE_ONCE(). + */ + cpu = cpumask_first_and(mask, cpu_online_mask); + nid = cpu < nr_cpu_ids ? cpu_to_node(cpu) : NUMA_NO_NODE; + WRITE_ONCE(irq->bp->ring_nid[irq->ring_nr], nid); + #ifdef CONFIG_RFS_ACCEL if (irq->bp->dev->rx_cpu_rmap && irq->ring_nr < irq->bp->rx_nr_rings) { int err; @@ -12001,9 +12023,15 @@ static int bnxt_alloc_ring_cpu_masks(struct bnxt *bp) if (!bp->ring_affinity_set) return -ENOMEM; - for (i = 0; i < bp->max_irqs; i++) + bp->ring_nid = kmalloc_objs(*bp->ring_nid, bp->max_irqs); + if (!bp->ring_nid) + return -ENOMEM; + + for (i = 0; i < bp->max_irqs; i++) { if (!zalloc_cpumask_var(&bp->ring_cpu_mask[i], GFP_KERNEL)) return -ENOMEM; + bp->ring_nid[i] = NUMA_NO_NODE; + } return 0; } @@ -12018,6 +12046,8 @@ static void bnxt_free_ring_cpu_masks(struct bnxt *bp) for (i = 0; i < bp->max_irqs; i++) free_cpumask_var(bp->ring_cpu_mask[i]); + kfree(bp->ring_nid); + bp->ring_nid = NULL; bitmap_free(bp->ring_affinity_set); bp->ring_affinity_set = NULL; kfree(bp->ring_cpu_mask); @@ -12113,6 +12143,7 @@ static int bnxt_request_irq(struct bnxt *bp) if (!test_bit(i, bp->ring_affinity_set) || !cpumask_intersects(cpu_mask, cpu_online_mask)) { clear_bit(i, bp->ring_affinity_set); + WRITE_ONCE(bp->ring_nid[i], NUMA_NO_NODE); cpumask_clear(cpu_mask); cpumask_set_cpu(cpumask_local_spread(i, numa_node), cpu_mask); @@ -16478,7 +16509,7 @@ static int bnxt_queue_mem_alloc(struct net_device *dev, struct bnxt_rx_ring_info *rxr, *clone; struct bnxt *bp = netdev_priv(dev); struct bnxt_ring_struct *ring; - int rc; + int nid, rc; if (!bp->rx_ring) return -ENETDOWN; @@ -16499,7 +16530,8 @@ static int bnxt_queue_mem_alloc(struct net_device *dev, clone->rx_tpa = NULL; clone->rx_tpa_idx_map = NULL; - rc = bnxt_alloc_rx_page_pool(bp, clone, rxr->page_pool->p.nid); + nid = bnxt_cp_num_to_node(bp, idx); + rc = bnxt_alloc_rx_page_pool(bp, clone, nid); if (rc) return rc; -- 2.56.1 Allocate the SW rings, agg bitmap, TPA state, completion ring page arrays, completion sub-ring structs, SW stats and the SW USO header buffer on the NUMA node of the CPU which will service the queue. This lowers the cost of cache misses for NAPI fast path (and xmit, for Tx). Previous change already allocated the page pools on the right node. Reviewed-by: Björn Töpel Reviewed-by: Joe Damato Signed-off-by: Jakub Kicinski --- v2: no change v1: https://lore.kernel.org/20261118023905.3050661-4-kuba@kernel.org --- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 87 ++++++++++++++--------- 1 file changed, 52 insertions(+), 35 deletions(-) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 6e503822d7bb..7acf74822e64 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -3770,7 +3770,8 @@ static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem) } } -static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem) +static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem, + int nid) { struct pci_dev *pdev = bp->pdev; u64 valid_bit = 0; @@ -3815,8 +3816,9 @@ static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem) } } + /* Only the SW ring follows @nid, DMA memory is on the device's node */ if (rmem->vmem_size) { - *rmem->vmem = vzalloc(rmem->vmem_size); + *rmem->vmem = vzalloc_node(rmem->vmem_size, nid); if (!(*rmem->vmem)) return -ENOMEM; } @@ -3866,25 +3868,27 @@ static void bnxt_free_tpa_info(struct bnxt *bp) } static int bnxt_alloc_one_tpa_info(struct bnxt *bp, - struct bnxt_rx_ring_info *rxr) + struct bnxt_rx_ring_info *rxr, int nid) { struct rx_agg_cmp *agg; int i; - rxr->rx_tpa = kzalloc_objs(struct bnxt_tpa_info, - bp->max_tpa_roundup_size); + rxr->rx_tpa = kcalloc_node(bp->max_tpa_roundup_size, + sizeof(*rxr->rx_tpa), GFP_KERNEL, nid); if (!rxr->rx_tpa) return -ENOMEM; if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) return 0; for (i = 0; i < bp->max_tpa_roundup_size; i++) { - agg = kzalloc_objs(*agg, MAX_SKB_FRAGS); + agg = kcalloc_node(MAX_SKB_FRAGS, sizeof(*agg), + GFP_KERNEL, nid); if (!agg) return -ENOMEM; rxr->rx_tpa[i].agg_arr = agg; } - rxr->rx_tpa_idx_map = kzalloc_obj(*rxr->rx_tpa_idx_map); + rxr->rx_tpa_idx_map = kzalloc_node(sizeof(*rxr->rx_tpa_idx_map), + GFP_KERNEL, nid); if (!rxr->rx_tpa_idx_map) return -ENOMEM; @@ -3911,8 +3915,10 @@ static int bnxt_alloc_tpa_info(struct bnxt *bp) for (i = 0; i < bp->rx_nr_rings; i++) { struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; + int nid; - rc = bnxt_alloc_one_tpa_info(bp, rxr); + nid = bnxt_cp_num_to_node(bp, i); + rc = bnxt_alloc_one_tpa_info(bp, rxr, nid); if (rc) return rc; } @@ -4024,13 +4030,14 @@ static void bnxt_enable_rx_page_pool(struct bnxt_rx_ring_info *rxr) page_pool_enable_direct_recycling(rxr->page_pool, &rxr->bnapi->napi); } -static int bnxt_alloc_rx_agg_bmap(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) +static int bnxt_alloc_rx_agg_bmap(struct bnxt *bp, + struct bnxt_rx_ring_info *rxr, int nid) { u16 mem_size; rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1; mem_size = rxr->rx_agg_bmap_size / 8; - rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL); + rxr->rx_agg_bmap = kzalloc_node(mem_size, GFP_KERNEL, nid); if (!rxr->rx_agg_bmap) return -ENOMEM; @@ -4074,19 +4081,19 @@ static int bnxt_alloc_rx_rings(struct bnxt *bp) return rc; } - rc = bnxt_alloc_ring(bp, &ring->ring_mem); + rc = bnxt_alloc_ring(bp, &ring->ring_mem, cpu_node); if (rc) return rc; ring->grp_idx = i; if (agg_rings) { ring = &rxr->rx_agg_ring_struct; - rc = bnxt_alloc_ring(bp, &ring->ring_mem); + rc = bnxt_alloc_ring(bp, &ring->ring_mem, cpu_node); if (rc) return rc; ring->grp_idx = i; - rc = bnxt_alloc_rx_agg_bmap(bp, rxr); + rc = bnxt_alloc_rx_agg_bmap(bp, rxr, cpu_node); if (rc) return rc; } @@ -4111,9 +4118,9 @@ static void bnxt_free_tx_inline_buf(struct bnxt_tx_ring_info *txr, static int bnxt_alloc_tx_inline_buf(struct bnxt_tx_ring_info *txr, struct pci_dev *pdev, - unsigned int size) + unsigned int size, int nid) { - txr->tx_inline_buf = kmalloc(size, GFP_KERNEL); + txr->tx_inline_buf = kmalloc_node(size, GFP_KERNEL, nid); if (!txr->tx_inline_buf) return -ENOMEM; @@ -4187,11 +4194,13 @@ static int bnxt_alloc_tx_rings(struct bnxt *bp) for (i = 0, j = 0; i < bp->tx_nr_rings; i++) { struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; struct bnxt_ring_struct *ring; + int nid; u8 qidx; ring = &txr->tx_ring_struct; - rc = bnxt_alloc_ring(bp, &ring->ring_mem); + nid = bnxt_cp_num_to_node(bp, txr->bnapi->index); + rc = bnxt_alloc_ring(bp, &ring->ring_mem, nid); if (rc) return rc; @@ -4217,7 +4226,7 @@ static int bnxt_alloc_tx_rings(struct bnxt *bp) if (!(bp->flags & BNXT_FLAG_UDP_GSO_CAP)) { rc = bnxt_alloc_tx_inline_buf(txr, pdev, BNXT_SW_USO_MAX_SEGS * - TSO_HEADER_SIZE); + TSO_HEADER_SIZE, nid); if (rc) return rc; } @@ -4244,12 +4253,15 @@ static void bnxt_free_cp_arrays(struct bnxt_cp_ring_info *cpr) ring->ring_mem.dma_arr = NULL; } -static int bnxt_alloc_cp_arrays(struct bnxt_cp_ring_info *cpr, int n) +static int bnxt_alloc_cp_arrays(struct bnxt_cp_ring_info *cpr, int n, + int nid) { - cpr->cp_desc_ring = kzalloc_objs(*cpr->cp_desc_ring, n); + cpr->cp_desc_ring = kcalloc_node(n, sizeof(*cpr->cp_desc_ring), + GFP_KERNEL, nid); if (!cpr->cp_desc_ring) return -ENOMEM; - cpr->cp_desc_mapping = kzalloc_objs(*cpr->cp_desc_mapping, n); + cpr->cp_desc_mapping = kcalloc_node(n, sizeof(*cpr->cp_desc_mapping), + GFP_KERNEL, nid); if (!cpr->cp_desc_mapping) return -ENOMEM; return 0; @@ -4276,11 +4288,12 @@ static int bnxt_alloc_all_cp_arrays(struct bnxt *bp) for (i = 0; i < bp->cp_nr_rings; i++) { struct bnxt_napi *bnapi = bp->bnapi[i]; - int rc; + int nid, rc; if (!bnapi) continue; - rc = bnxt_alloc_cp_arrays(&bnapi->cp_ring, n); + nid = bnxt_cp_num_to_node(bp, i); + rc = bnxt_alloc_cp_arrays(&bnapi->cp_ring, n, nid); if (rc) return rc; } @@ -4325,13 +4338,13 @@ static void bnxt_free_cp_rings(struct bnxt *bp) } static int bnxt_alloc_cp_sub_ring(struct bnxt *bp, - struct bnxt_cp_ring_info *cpr) + struct bnxt_cp_ring_info *cpr, int nid) { struct bnxt_ring_mem_info *rmem; struct bnxt_ring_struct *ring; int rc; - rc = bnxt_alloc_cp_arrays(cpr, bp->cp_nr_pages); + rc = bnxt_alloc_cp_arrays(cpr, bp->cp_nr_pages, nid); if (rc) { bnxt_free_cp_arrays(cpr); return -ENOMEM; @@ -4343,7 +4356,7 @@ static int bnxt_alloc_cp_sub_ring(struct bnxt *bp, rmem->pg_arr = (void **)cpr->cp_desc_ring; rmem->dma_arr = cpr->cp_desc_mapping; rmem->flags = BNXT_RMEM_RING_PTE_FLAG; - rc = bnxt_alloc_ring(bp, rmem); + rc = bnxt_alloc_ring(bp, rmem, NUMA_NO_NODE); if (rc) { bnxt_free_ring(bp, rmem); bnxt_free_cp_arrays(cpr); @@ -4364,7 +4377,7 @@ static int bnxt_alloc_cp_rings(struct bnxt *bp) struct bnxt_napi *bnapi = bp->bnapi[i]; struct bnxt_cp_ring_info *cpr, *cpr2; struct bnxt_ring_struct *ring; - int cp_count = 0, k; + int cp_count = 0, k, nid; int rx = 0, tx = 0; if (!bnapi) @@ -4374,7 +4387,7 @@ static int bnxt_alloc_cp_rings(struct bnxt *bp) cpr->bnapi = bnapi; ring = &cpr->cp_ring_struct; - rc = bnxt_alloc_ring(bp, &ring->ring_mem); + rc = bnxt_alloc_ring(bp, &ring->ring_mem, NUMA_NO_NODE); if (rc) return rc; @@ -4396,14 +4409,16 @@ static int bnxt_alloc_cp_rings(struct bnxt *bp) tx = 1; } - cpr->cp_ring_arr = kzalloc_objs(*cpr, cp_count); + nid = bnxt_cp_num_to_node(bp, i); + cpr->cp_ring_arr = kcalloc_node(cp_count, sizeof(*cpr), + GFP_KERNEL, nid); if (!cpr->cp_ring_arr) return -ENOMEM; cpr->cp_ring_count = cp_count; for (k = 0; k < cp_count; k++) { cpr2 = &cpr->cp_ring_arr[k]; - rc = bnxt_alloc_cp_sub_ring(bp, cpr2); + rc = bnxt_alloc_cp_sub_ring(bp, cpr2, nid); if (rc) return rc; cpr2->bnapi = bnapi; @@ -5415,8 +5430,10 @@ static int bnxt_alloc_stats(struct bnxt *bp) for (i = 0; i < bp->cp_nr_rings; i++) { struct bnxt_napi *bnapi = bp->bnapi[i]; struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; + int nid = bnxt_cp_num_to_node(bp, i); - cpr->sw_stats = kzalloc_obj(*cpr->sw_stats); + cpr->sw_stats = kzalloc_node(sizeof(*cpr->sw_stats), GFP_KERNEL, + nid); if (!cpr->sw_stats) return -ENOMEM; @@ -9242,7 +9259,7 @@ static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp, rmem->flags = BNXT_RMEM_VALID_PTE_FLAG; if (rmem->depth >= 1) rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG; - return bnxt_alloc_ring(bp, rmem); + return bnxt_alloc_ring(bp, rmem, NUMA_NO_NODE); } static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp, @@ -16546,23 +16563,23 @@ static int bnxt_queue_mem_alloc(struct net_device *dev, goto err_rxq_info_unreg; ring = &clone->rx_ring_struct; - rc = bnxt_alloc_ring(bp, &ring->ring_mem); + rc = bnxt_alloc_ring(bp, &ring->ring_mem, nid); if (rc) goto err_free_rx_ring; if (bp->flags & BNXT_FLAG_AGG_RINGS) { ring = &clone->rx_agg_ring_struct; - rc = bnxt_alloc_ring(bp, &ring->ring_mem); + rc = bnxt_alloc_ring(bp, &ring->ring_mem, nid); if (rc) goto err_free_rx_agg_ring; - rc = bnxt_alloc_rx_agg_bmap(bp, clone); + rc = bnxt_alloc_rx_agg_bmap(bp, clone, nid); if (rc) goto err_free_rx_agg_ring; } if (bp->flags & BNXT_FLAG_TPA) { - rc = bnxt_alloc_one_tpa_info(bp, clone); + rc = bnxt_alloc_one_tpa_info(bp, clone, nid); if (rc) goto err_free_tpa_info; } -- 2.56.1 Context memory (and the FW crash dump buffer which reuses the same helpers) is used by the device, once set up the host only touches it on the slow path (diagnostics, FW trace). Pass the device's node to bnxt_alloc_ring() rather than NUMA_NO_NODE to make it clear that this memory is not meant to follow any queue. Ring page tables are only read by the device, annotate their allocation so that they stay put once the ring pages start following the queue. Reviewed-by: Björn Töpel Reviewed-by: Joe Damato Signed-off-by: Jakub Kicinski --- v2: no change v1: https://lore.kernel.org/20261118023905.3050661-5-kuba@kernel.org --- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 7acf74822e64..265ee15b6b35 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -3784,6 +3784,7 @@ static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem, if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG) pg_tbl_size = rmem->page_size; + /* Only read by the device, keep it on the device's node */ rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size, &rmem->pg_tbl_map, GFP_KERNEL); @@ -9259,7 +9260,8 @@ static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp, rmem->flags = BNXT_RMEM_VALID_PTE_FLAG; if (rmem->depth >= 1) rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG; - return bnxt_alloc_ring(bp, rmem, NUMA_NO_NODE); + /* Host only uses it on the slow path, keep it on the device's node */ + return bnxt_alloc_ring(bp, rmem, dev_to_node(&bp->pdev->dev)); } static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp, -- 2.56.1 Descriptor rings, completion rings, NQs and the TX push buffer are accessed by the host in the fast path, and so is the SW state around them, which we already allocate on the node of the CPU expected to service the queue. The DMA memory itself comes from the device's node, though, as that's what dma_alloc_coherent() always uses. Use dma_alloc_coherent_node() for the ring pages and the TX push buffer. Pass the queue's node for completion rings and NQs as well, they used NUMA_NO_NODE so far since @nid only affected the SW rings. Ring page tables are only read by the device, keep them on the device's node. Note that queue restart only reallocates the Rx descriptor rings. Completion rings, NQs and Tx rings will follow IRQ affinity changes only on the next full reconfiguration. Reviewed-by: Björn Töpel Reviewed-by: Joe Damato Signed-off-by: Jakub Kicinski --- v2: no change v1: https://lore.kernel.org/20261118023905.3050661-6-kuba@kernel.org --- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 24 +++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 265ee15b6b35..a74599c1447a 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -3795,10 +3795,10 @@ static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem, for (i = 0; i < rmem->nr_pages; i++) { u64 extra_bits = valid_bit; - rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev, - rmem->page_size, - &rmem->dma_arr[i], - GFP_KERNEL); + rmem->pg_arr[i] = dma_alloc_coherent_node(&pdev->dev, + rmem->page_size, + &rmem->dma_arr[i], + GFP_KERNEL, nid); if (!rmem->pg_arr[i]) return -ENOMEM; @@ -3817,7 +3817,6 @@ static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem, } } - /* Only the SW ring follows @nid, DMA memory is on the device's node */ if (rmem->vmem_size) { *rmem->vmem = vzalloc_node(rmem->vmem_size, nid); if (!(*rmem->vmem)) @@ -4212,10 +4211,11 @@ static int bnxt_alloc_tx_rings(struct bnxt *bp) /* One pre-allocated DMA buffer to backup * TX push operation */ - txr->tx_push = dma_alloc_coherent(&pdev->dev, - bp->tx_push_size, - &txr->tx_push_mapping, - GFP_KERNEL); + txr->tx_push = + dma_alloc_coherent_node(&pdev->dev, + bp->tx_push_size, + &txr->tx_push_mapping, + GFP_KERNEL, nid); if (!txr->tx_push) return -ENOMEM; @@ -4357,7 +4357,7 @@ static int bnxt_alloc_cp_sub_ring(struct bnxt *bp, rmem->pg_arr = (void **)cpr->cp_desc_ring; rmem->dma_arr = cpr->cp_desc_mapping; rmem->flags = BNXT_RMEM_RING_PTE_FLAG; - rc = bnxt_alloc_ring(bp, rmem, NUMA_NO_NODE); + rc = bnxt_alloc_ring(bp, rmem, nid); if (rc) { bnxt_free_ring(bp, rmem); bnxt_free_cp_arrays(cpr); @@ -4388,7 +4388,8 @@ static int bnxt_alloc_cp_rings(struct bnxt *bp) cpr->bnapi = bnapi; ring = &cpr->cp_ring_struct; - rc = bnxt_alloc_ring(bp, &ring->ring_mem, NUMA_NO_NODE); + nid = bnxt_cp_num_to_node(bp, i); + rc = bnxt_alloc_ring(bp, &ring->ring_mem, nid); if (rc) return rc; @@ -4410,7 +4411,6 @@ static int bnxt_alloc_cp_rings(struct bnxt *bp) tx = 1; } - nid = bnxt_cp_num_to_node(bp, i); cpr->cp_ring_arr = kcalloc_node(cp_count, sizeof(*cpr), GFP_KERNEL, nid); if (!cpr->cp_ring_arr) -- 2.56.1 bnxt_irq_affinity_notify() refers to irq->bp in almost every statement. Add a local variable to shorten the lines, the next patch will make the conditions at the end of the function more complex. No functional change. Signed-off-by: Jakub Kicinski --- v2: new patch --- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index a74599c1447a..2c80a39eb8f0 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -11949,13 +11949,15 @@ static void bnxt_irq_affinity_notify(struct irq_affinity_notify *notify, { struct bnxt_irq *irq; unsigned int cpu; + struct bnxt *bp; int nid; u16 tag; irq = container_of(notify, struct bnxt_irq, affinity_notify); + bp = irq->bp; - cpumask_copy(irq->bp->ring_cpu_mask[irq->ring_nr], mask); - set_bit(irq->ring_nr, irq->bp->ring_affinity_set); + cpumask_copy(bp->ring_cpu_mask[irq->ring_nr], mask); + set_bit(irq->ring_nr, bp->ring_affinity_set); /* Readers can't access ring_cpu_mask atomically, so pre-compute * the info they care about (NUMA node, TPH tag) and store it @@ -11963,32 +11965,31 @@ static void bnxt_irq_affinity_notify(struct irq_affinity_notify *notify, */ cpu = cpumask_first_and(mask, cpu_online_mask); nid = cpu < nr_cpu_ids ? cpu_to_node(cpu) : NUMA_NO_NODE; - WRITE_ONCE(irq->bp->ring_nid[irq->ring_nr], nid); + WRITE_ONCE(bp->ring_nid[irq->ring_nr], nid); #ifdef CONFIG_RFS_ACCEL - if (irq->bp->dev->rx_cpu_rmap && irq->ring_nr < irq->bp->rx_nr_rings) { + if (bp->dev->rx_cpu_rmap && irq->ring_nr < bp->rx_nr_rings) { int err; - err = cpu_rmap_update(irq->bp->dev->rx_cpu_rmap, irq->ring_nr, - mask); + err = cpu_rmap_update(bp->dev->rx_cpu_rmap, irq->ring_nr, mask); if (err) - netdev_warn(irq->bp->dev, + netdev_warn(bp->dev, "aRFS rmap update failed: %d\n", err); } #endif - if (!irq->bp->tph_mode) + if (!bp->tph_mode) return; - if (irq->ring_nr >= irq->bp->rx_nr_rings) + if (irq->ring_nr >= bp->rx_nr_rings) return; - if (pcie_tph_get_cpu_st(irq->bp->pdev, TPH_MEM_TYPE_VM, + if (pcie_tph_get_cpu_st(bp->pdev, TPH_MEM_TYPE_VM, cpumask_first(mask), &tag)) return; WRITE_ONCE(irq->new_tag, tag); - bnxt_queue_sp_work(irq->bp, BNXT_TPH_UPDATE_SP_EVENT); + bnxt_queue_sp_work(bp, BNXT_TPH_UPDATE_SP_EVENT); } static void bnxt_irq_affinity_release(struct kref *ref) -- 2.56.1 Rx queue memory is allocated on the node of the CPU servicing the queue when the device is opened. On a running device it only follows IRQ affinity changes if TPH is enabled, as a side effect of the queue restart which applies the new TPH Steering Tag. Schedule the affinity update work whenever the device supports the queue API, not just when TPH Steering Tag changed. Queue restart places all the memory it reallocates, other than the ring page tables, on the page pool's node, so compare the page pool's nid to decide if the queue has to move. Only consider queues whose IRQ affinity was set from the outside, so that moving one IRQ doesn't restart other queues just because CPU hotplug shifted our default spread. Since we need queue state of the currently allocated queue to compare nid - check that the device is still up after errors. Reviewed-by: Björn Töpel Reviewed-by: Joe Damato Signed-off-by: Jakub Kicinski --- v2: - split the TPH condition in the notifier (Joe) - only compare the node of queues with IRQ affinity set from the outside v1: https://lore.kernel.org/20261118023905.3050661-7-kuba@kernel.org --- drivers/net/ethernet/broadcom/bnxt/bnxt.h | 2 +- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 53 +++++++++++++++-------- 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h index 61b86fa9cadb..0334f24b7522 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h @@ -2656,7 +2656,7 @@ struct bnxt { #define BNXT_RING_COAL_NOW_SP_EVENT 17 #define BNXT_FW_RESET_NOTIFY_SP_EVENT 18 #define BNXT_FW_EXCEPTION_SP_EVENT 19 -#define BNXT_TPH_UPDATE_SP_EVENT 20 +#define BNXT_IRQ_AFFINITY_SP_EVENT 20 #define BNXT_LINK_CFG_CHANGE_SP_EVENT 21 #define BNXT_THERMAL_THRESHOLD_SP_EVENT 22 #define BNXT_FW_ECHO_REQUEST_SP_EVENT 23 diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 2c80a39eb8f0..cbd249788198 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -11978,18 +11978,17 @@ static void bnxt_irq_affinity_notify(struct irq_affinity_notify *notify, } #endif - if (!bp->tph_mode) + /* Only Rx queues can be restarted to follow the IRQ */ + if (!BNXT_SUPPORTS_QUEUE_API(bp) || irq->ring_nr >= bp->rx_nr_rings) return; - if (irq->ring_nr >= bp->rx_nr_rings) - return; + if (bp->tph_mode) { + cpu = cpumask_first(mask); + if (!pcie_tph_get_cpu_st(bp->pdev, TPH_MEM_TYPE_VM, cpu, &tag)) + WRITE_ONCE(irq->new_tag, tag); + } - if (pcie_tph_get_cpu_st(bp->pdev, TPH_MEM_TYPE_VM, - cpumask_first(mask), &tag)) - return; - - WRITE_ONCE(irq->new_tag, tag); - bnxt_queue_sp_work(bp, BNXT_TPH_UPDATE_SP_EVENT); + bnxt_queue_sp_work(bp, BNXT_IRQ_AFFINITY_SP_EVENT); } static void bnxt_irq_affinity_release(struct kref *ref) @@ -14709,7 +14708,10 @@ static void bnxt_rtnl_unlock_sp(struct bnxt *bp) rtnl_unlock(); } -static void bnxt_tph_update(struct bnxt *bp) +/* Restart the Rx queues whose IRQ moved, if the move changed the TPH + * Steering Tag or the node the queue memory should be allocated on. + */ +static void bnxt_irq_affinity_update(struct bnxt *bp) { struct net_device *dev = bp->dev; int i; @@ -14719,27 +14721,40 @@ static void bnxt_tph_update(struct bnxt *bp) goto unlock; for (i = 0; i < bp->rx_nr_rings; i++) { + struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; + bool new_tag = false, new_node = false; + int map_idx, err, nid; struct bnxt_irq *irq; - int map_idx, err; u16 tag; map_idx = bnxt_cp_num_to_irq_num(bp, i); irq = &bp->irq_tbl[map_idx]; + tag = READ_ONCE(irq->new_tag); - if (irq->tag == tag) + if (irq->tag != tag) { + if (!pcie_tph_set_st_entry(bp->pdev, irq->msix_nr, tag)) + new_tag = true; + } + + nid = READ_ONCE(bp->ring_nid[i]); + if (nid != NUMA_NO_NODE && rxr->page_pool->p.nid != nid) + new_node = true; + + if (!new_tag && !new_node) continue; - if (pcie_tph_set_st_entry(bp->pdev, irq->msix_nr, tag)) - continue; - - err = netdev_rx_queue_restart(dev, irq->ring_nr); + err = netdev_rx_queue_restart(dev, i); if (err) { netdev_err(dev, "RX queue restart failed: err=%d\n", err); + /* Failing to start a new queue may close the device */ + if (!test_bit(BNXT_STATE_OPEN, &bp->state)) + break; continue; } - irq->tag = tag; + if (new_tag) + irq->tag = tag; } unlock: @@ -15183,8 +15198,8 @@ static void bnxt_sp_task(struct work_struct *work) bnxt_devlink_health_fw_report(bp); } - if (test_and_clear_bit(BNXT_TPH_UPDATE_SP_EVENT, &bp->sp_event)) - bnxt_tph_update(bp); + if (test_and_clear_bit(BNXT_IRQ_AFFINITY_SP_EVENT, &bp->sp_event)) + bnxt_irq_affinity_update(bp); smp_mb__before_atomic(); clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); -- 2.56.1