mlx5_trigger_health_work() queues the health recovery, which enters MLX5_DEVICE_STATE_INTERNAL_ERROR only if mlx5_health_check_fatal_sensors() reports a fault. A consumer can detect a fault the sensors cannot see, such as a device that keeps answering while misbehaving. It then needs the recovery to run with the error state forced, whatever the sensors say. Add mlx5_trigger_health_work_force(), which queues a dedicated work that runs the recovery with the error state forced. It only queues the work, so it can be called from atomic context. Before entering the error state the forced work stops the health poller and asks the firmware to stop with mlx5_cmd_fast_teardown_hca(), as the device might still be running. The error state is forced whether or not the teardown succeeds. The teardown runs under intf_state_mutex and only while the interface is up, so it cannot race an unload. Export it for mlx5_ib, which will use it in a later patch of this series when it detects a NIC forging completion queue entries, a fault the health sensors cannot see. Reviewed-by: Moshe Shemesh Reviewed-by: Tariq Toukan Signed-off-by: Yishai Hadas --- .../net/ethernet/mellanox/mlx5/core/health.c | 82 ++++++++++++++++--- include/linux/mlx5/driver.h | 2 + 2 files changed, 73 insertions(+), 11 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c index aeeb136f5ebc..d000ac335fa7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/health.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c @@ -627,18 +627,11 @@ mlx5_fw_fatal_reporter_dump(struct devlink_health_reporter *reporter, return err; } -static void mlx5_fw_fatal_reporter_err_work(struct work_struct *work) +static void mlx5_fw_fatal_reporter_err(struct mlx5_core_dev *dev, bool forced) { struct mlx5_fw_reporter_ctx fw_reporter_ctx; - struct mlx5_core_health *health; - struct mlx5_core_dev *dev; - struct devlink *devlink; - struct mlx5_priv *priv; - - health = container_of(work, struct mlx5_core_health, fatal_report_work); - priv = container_of(health, struct mlx5_priv, health); - dev = container_of(priv, struct mlx5_core_dev, priv); - devlink = priv_to_devlink(dev); + struct mlx5_core_health *health = &dev->priv.health; + struct devlink *devlink = priv_to_devlink(dev); mutex_lock(&dev->intf_state_mutex); if (test_bit(MLX5_DROP_HEALTH_WORK, &health->flags)) { @@ -646,8 +639,26 @@ static void mlx5_fw_fatal_reporter_err_work(struct work_struct *work) mutex_unlock(&dev->intf_state_mutex); return; } + /* + * The firmware might still be running, so stop it first. Do it under + * intf_state_mutex and only while UP, so the cmd interface is valid. + */ + if (forced && test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) { + int err; + + mlx5_stop_health_poll(dev, false); + err = mlx5_cmd_fast_teardown_hca(dev); + if (err) + mlx5_core_err(dev, "Failed to stop the device before forcing the error state, err %d\n", + err); + } + + /* + * Enter the error state before unlocking, or an unload could send + * TEARDOWN_HCA to the stopped NIC, fail and skip function disable. + */ + enter_error_state(dev, forced); mutex_unlock(&dev->intf_state_mutex); - enter_error_state(dev, false); if (IS_ERR_OR_NULL(health->fw_fatal_reporter)) { devl_lock(devlink); if (mlx5_health_try_recover(dev)) @@ -669,6 +680,33 @@ static void mlx5_fw_fatal_reporter_err_work(struct work_struct *work) } } +static void mlx5_fw_fatal_reporter_err_work(struct work_struct *work) +{ + struct mlx5_core_health *health; + struct mlx5_core_dev *dev; + struct mlx5_priv *priv; + + health = container_of(work, struct mlx5_core_health, fatal_report_work); + priv = container_of(health, struct mlx5_priv, health); + dev = container_of(priv, struct mlx5_core_dev, priv); + + mlx5_fw_fatal_reporter_err(dev, false); +} + +static void mlx5_fw_fatal_reporter_force_err_work(struct work_struct *work) +{ + struct mlx5_core_health *health; + struct mlx5_core_dev *dev; + struct mlx5_priv *priv; + + health = container_of(work, struct mlx5_core_health, + force_fatal_report_work); + priv = container_of(health, struct mlx5_priv, health); + dev = container_of(priv, struct mlx5_core_dev, priv); + + mlx5_fw_fatal_reporter_err(dev, true); +} + #define MLX5_FW_REPORTER_ECPF_GRACEFUL_PERIOD 180000 #define MLX5_FW_REPORTER_PF_GRACEFUL_PERIOD 60000 #define MLX5_FW_REPORTER_VF_GRACEFUL_PERIOD 30000 @@ -759,6 +797,25 @@ void mlx5_trigger_health_work(struct mlx5_core_dev *dev) queue_work(health->wq, &health->fatal_report_work); } +/** + * mlx5_trigger_health_work_force - enter the error state and recover, even + * if the health sensors report the device as healthy + * @dev: mlx5 core device + * + * For faults only a consumer can detect, where the firmware keeps answering + * and mlx5_health_check_fatal_sensors() sees nothing wrong. Without forcing, + * the device would stay in MLX5_DEVICE_STATE_UP and consumers waiting on + * outstanding work would never be released. + */ +void mlx5_trigger_health_work_force(struct mlx5_core_dev *dev) +{ + struct mlx5_core_health *health = &dev->priv.health; + + if (!mlx5_dev_is_lightweight(dev)) + queue_work(health->wq, &health->force_fatal_report_work); +} +EXPORT_SYMBOL_GPL(mlx5_trigger_health_work_force); + #define MLX5_MSEC_PER_HOUR (MSEC_PER_SEC * 60 * 60) static void mlx5_health_log_ts_update(struct work_struct *work) { @@ -874,6 +931,7 @@ void mlx5_drain_health_wq(struct mlx5_core_dev *dev) cancel_delayed_work_sync(&health->update_fw_log_ts_work); cancel_work_sync(&health->report_work); cancel_work_sync(&health->fatal_report_work); + cancel_work_sync(&health->force_fatal_report_work); } void mlx5_health_cleanup(struct mlx5_core_dev *dev) @@ -911,6 +969,8 @@ int mlx5_health_init(struct mlx5_core_dev *dev) if (!health->wq) goto out_err; INIT_WORK(&health->fatal_report_work, mlx5_fw_fatal_reporter_err_work); + INIT_WORK(&health->force_fatal_report_work, + mlx5_fw_fatal_reporter_force_err_work); INIT_WORK(&health->report_work, mlx5_fw_reporter_err_work); INIT_DELAYED_WORK(&health->update_fw_log_ts_work, mlx5_health_log_ts_update); diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index abb99371ff13..a1275e6aa6c1 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -450,6 +450,7 @@ struct mlx5_core_health { struct workqueue_struct *wq; unsigned long flags; struct work_struct fatal_report_work; + struct work_struct force_fatal_report_work; struct work_struct report_work; struct devlink_health_reporter *fw_reporter; struct devlink_health_reporter *fw_fatal_reporter; @@ -1034,6 +1035,7 @@ void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health); void mlx5_start_health_fw_log_up(struct mlx5_core_dev *dev); void mlx5_drain_health_wq(struct mlx5_core_dev *dev); void mlx5_trigger_health_work(struct mlx5_core_dev *dev); +void mlx5_trigger_health_work_force(struct mlx5_core_dev *dev); int mlx5_frag_buf_alloc_node(struct mlx5_core_dev *dev, int size, struct mlx5_frag_buf *buf, int node); void mlx5_frag_buf_free(struct mlx5_core_dev *dev, struct mlx5_frag_buf *buf); -- 2.18.1