From: Chris Mi When host is powered off, firmware does not send vhca_state event for every probed host SF on the DPU because it may have deployed thousands of SFs to the host. Instead it sends a function changed event. Currently, only VFs handle this event. This commit extends support to SFs. When DPU user deactivates[1] SFs, mlx5 expects vhca_state event and leaves the SF in dangling state[2]. When DPU user deletes[3] SFs, mlx5 also expects vhca_state event and destroys the SF resources[4]. Fix it by changing SF to the right state and freeing SF resources when the function changed event is received. When this event is received, driver checks all SF states. - If state is in_use, change it to active. - If state is teardown_request, change it to allocated. And SF hardware table entry is freed if it is pending for delete. [1] # devlink port function set en3f0c1pf0sf0 state inactive [2] # devlink port function set en3f0c1pf0sf0 state active Error: mlx5_core: SF is inactivated but it is still attached. kernel answers: Device or resource busy [3] # devlink port show pci/0000:03:00.0/229376: type eth netdev en3f0c1pf0sf0 \ flavour pcisf controller 1 pfnum 0 sfnum 0 splittable false function: hw_addr 00:00:00:00:00:00 state active opstate attached \ roce enable trust off max_uc_macs 4096 max_io_eqs 8 # devlink port del en3f0c1pf0sf0 [4] # devlink port add pci/0000:03:00.0 flavour pcisf pfnum 0 sfnum 0 \ controller 1 Error: mlx5_core: SF already exist. Choose different sfnum. kernel answers: File exists Fixes: 6a3273217469 ("net/mlx5: SF, Port function state change support") Signed-off-by: Chris Mi Reviewed-by: Shay Drori Signed-off-by: Tariq Toukan --- .../mellanox/mlx5/core/eswitch_offloads.c | 10 +++++-- .../ethernet/mellanox/mlx5/core/sf/devlink.c | 29 +++++++++++++++++++ .../mlx5/core/sf/diag/sf_tracepoint.h | 8 +++++ .../ethernet/mellanox/mlx5/core/sf/hw_table.c | 22 ++++++++++++++ .../net/ethernet/mellanox/mlx5/core/sf/sf.h | 12 ++++++++ 5 files changed, 78 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index 907ee83a722d..247e5d85ec36 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -3986,7 +3986,7 @@ static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw) mutex_destroy(&esw->fdb_table.offloads.vports.lock); } -static void esw_vfs_changed_event_handler(struct mlx5_eswitch *esw) +static void esw_changed_event_handler(struct mlx5_eswitch *esw) { struct mlx5_esw_pf_info host_pf_info; u16 new_num_vfs; @@ -3999,6 +3999,11 @@ static void esw_vfs_changed_event_handler(struct mlx5_eswitch *esw) host_pf_info = mlx5_esw_get_host_pf_info(esw->dev, out); new_num_vfs = host_pf_info.num_of_vfs; + if (host_pf_info.pf_disabled) { + mlx5_sf_table_esw_changed_event_handler(esw->dev); + mlx5_sf_hw_table_esw_changed_event_handler(esw->dev); + } + if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_info.pf_disabled) goto free; @@ -4091,8 +4096,7 @@ int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb); esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs); - ret = mlx5_esw_add_work(esw, esw_vfs_changed_event_handler, - GFP_ATOMIC); + ret = mlx5_esw_add_work(esw, esw_changed_event_handler, GFP_ATOMIC); if (ret) return NOTIFY_DONE; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c index b6cecbcc392d..4564d460416b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c @@ -561,3 +561,32 @@ bool mlx5_sf_table_empty(const struct mlx5_core_dev *dev) return xa_empty(&table->function_ids); } + +void mlx5_sf_table_esw_changed_event_handler(struct mlx5_core_dev *dev) +{ + struct mlx5_sf_table *table = dev->priv.sf_table; + unsigned long index; + struct mlx5_sf *sf; + + trace_mlx5_sf_host_pf_disabled(dev); + + if (!table) + return; + + mutex_lock(&table->sf_state_lock); + xa_for_each(&table->function_ids, index, sf) { + if (!sf->controller) + continue; + + if (sf->hw_state == MLX5_VHCA_STATE_IN_USE) + sf->hw_state = MLX5_VHCA_STATE_ACTIVE; + else if (sf->hw_state == MLX5_VHCA_STATE_TEARDOWN_REQUEST) + sf->hw_state = MLX5_VHCA_STATE_ALLOCATED; + else + continue; + trace_mlx5_sf_update_state(table->dev, sf->port_index, + sf->controller, sf->hw_fn_id, + sf->hw_state); + } + mutex_unlock(&table->sf_state_lock); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/diag/sf_tracepoint.h b/drivers/net/ethernet/mellanox/mlx5/core/sf/diag/sf_tracepoint.h index 302ce00da5a9..9dc88a7e30ca 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/diag/sf_tracepoint.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/diag/sf_tracepoint.h @@ -11,6 +11,14 @@ #include #include "sf/vhca_event.h" +TRACE_EVENT(mlx5_sf_host_pf_disabled, + TP_PROTO(const struct mlx5_core_dev *dev), + TP_ARGS(dev), + TP_STRUCT__entry(__string(devname, dev_name(dev->device))), + TP_fast_assign(__assign_str(devname);), + TP_printk("(%s)\n", __get_str(devname)) +); + TRACE_EVENT(mlx5_sf_add, TP_PROTO(const struct mlx5_core_dev *dev, unsigned int port_index, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/hw_table.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/hw_table.c index 0bc9146a3598..95a8b1e64ba4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/hw_table.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/hw_table.c @@ -459,3 +459,25 @@ bool mlx5_sf_hw_table_supported(const struct mlx5_core_dev *dev) { return !!dev->priv.sf_hw_table; } + +void mlx5_sf_hw_table_esw_changed_event_handler(struct mlx5_core_dev *dev) +{ + struct mlx5_sf_hw_table *table; + struct mlx5_sf_hwc_table *hwc; + int i; + + table = dev->priv.sf_hw_table; + if (!table) + return; + + mutex_lock(&table->table_lock); + hwc = &table->hwc[MLX5_SF_HWC_EXT_HOST]; + for (i = 0; i < hwc->max_fn; i++) { + struct mlx5_sf_hw *sf_hw; + + sf_hw = &hwc->sfs[i]; + if (sf_hw->allocated && sf_hw->pending_delete) + mlx5_sf_hw_table_hwc_sf_free(dev, hwc, i); + } + mutex_unlock(&table->table_lock); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/sf.h b/drivers/net/ethernet/mellanox/mlx5/core/sf/sf.h index d8a934a0e968..ed784682b9b1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/sf.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/sf.h @@ -15,12 +15,14 @@ void mlx5_sf_hw_table_cleanup(struct mlx5_core_dev *dev); int mlx5_sf_hw_notifier_init(struct mlx5_core_dev *dev); void mlx5_sf_hw_notifier_cleanup(struct mlx5_core_dev *dev); void mlx5_sf_hw_table_destroy(struct mlx5_core_dev *dev); +void mlx5_sf_hw_table_esw_changed_event_handler(struct mlx5_core_dev *dev); int mlx5_sf_notifiers_init(struct mlx5_core_dev *dev); int mlx5_sf_table_init(struct mlx5_core_dev *dev); void mlx5_sf_notifiers_cleanup(struct mlx5_core_dev *dev); void mlx5_sf_table_cleanup(struct mlx5_core_dev *dev); bool mlx5_sf_table_empty(const struct mlx5_core_dev *dev); +void mlx5_sf_table_esw_changed_event_handler(struct mlx5_core_dev *dev); int mlx5_devlink_sf_port_new(struct devlink *devlink, const struct devlink_port_new_attrs *add_attr, @@ -60,6 +62,11 @@ static inline void mlx5_sf_hw_table_destroy(struct mlx5_core_dev *dev) { } +static inline void +mlx5_sf_hw_table_esw_changed_event_handler(struct mlx5_core_dev *dev) +{ +} + static inline int mlx5_sf_notifiers_init(struct mlx5_core_dev *dev) { return 0; @@ -83,6 +90,11 @@ static inline bool mlx5_sf_table_empty(const struct mlx5_core_dev *dev) return true; } +static inline void +mlx5_sf_table_esw_changed_event_handler(struct mlx5_core_dev *dev) +{ +} + #endif #endif base-commit: 3bd438a58e910db5dc369aa25dfed1fc95f1b596 -- 2.44.0