Each recovery trigger queues recovery_check_work on workqueue_aux. If a recovery is already pending, the work blocks for up to 5 seconds waiting for it to finish, and marks the device WEDGED after four failed waits. On SDIO, one firmware crash can raise several triggers. workqueue_aux also runs SDIO RX and WMI SERVICE_READY handling, so the blocked check stalls the traffic the recovery itself is waiting for. Recovery then fails with "ctl_resp never came in (-110)", and a recoverable crash ends up WEDGED. Stop waiting on workqueue_aux. Let only the first trigger start a recovery, using atomic_cmpxchg() on pending_recovery, and ignore the rest. Replace the wait-and-count logic with a single 20 second delayed work, the same bound as four 5-second waits. If recovery has not completed when it fires, mark the device WEDGED. Since the deadline already covers elapsed time, fail_cont_count is no longer needed and is removed. Because pending_recovery is now set only once per recovery, any path that abandons a recovery must clear it. Otherwise the next crash is ignored: the device is neither restarted nor marked WEDGED. Clear it in ath10k_stop(), and in ath10k_core_restart() when the device is OFF and there is nothing to restart. The OFF case was found by code inspection and has not been reproduced. Tested-on: QCA6174 hw3.2 PCI WLAN.RM.4.4.1-00288-QCARMSWPZ-1 Tested-on: QCA6174 hw3.2 SDIO WLAN.RMH.4.4.1-00189 Fixes: f35a07a4842a ("wifi: ath10k: move recovery check logic into a new work") Signed-off-by: Kang Yang --- drivers/net/wireless/ath/ath10k/core.c | 75 +++++++++++++++++++--------------- drivers/net/wireless/ath/ath10k/core.h | 7 +--- drivers/net/wireless/ath/ath10k/mac.c | 15 ++++++- 3 files changed, 56 insertions(+), 41 deletions(-) diff --git a/drivers/net/wireless/ath/ath10k/core.c b/drivers/net/wireless/ath/ath10k/core.c index 2aa18adb31cc..b536c21630b1 100644 --- a/drivers/net/wireless/ath/ath10k/core.c +++ b/drivers/net/wireless/ath/ath10k/core.c @@ -2535,48 +2535,45 @@ static int ath10k_init_hw_params(struct ath10k *ar) static void ath10k_core_recovery_check_work(struct work_struct *work) { - struct ath10k *ar = container_of(work, struct ath10k, recovery_check_work); - long time_left; + struct ath10k *ar = container_of(work, struct ath10k, + recovery_check_work.work); - /* Sometimes the recovery will fail and then the next all recovery fail, - * so avoid infinite recovery. - */ - if (atomic_read(&ar->fail_cont_count) >= ATH10K_RECOVERY_MAX_FAIL_COUNT) { - ath10k_err(ar, "consecutive fail %d times, will shutdown driver!", - atomic_read(&ar->fail_cont_count)); - ar->state = ATH10K_STATE_WEDGED; + if (ar->state == ATH10K_STATE_WEDGED) return; - } - - ath10k_dbg(ar, ATH10K_DBG_BOOT, "total recovery count: %d", ++ar->recovery_count); - - if (atomic_read(&ar->pending_recovery)) { - /* Sometimes it happened another recovery work before the previous one - * completed, then the second recovery work will destroy the previous - * one, thus below is to avoid that. - */ - time_left = wait_for_completion_timeout(&ar->driver_recovery, - ATH10K_RECOVERY_TIMEOUT_HZ); - if (time_left) { - ath10k_warn(ar, "previous recovery succeeded, skip this!\n"); - return; - } - /* Record the continuous recovery fail count when recovery failed. */ - atomic_inc(&ar->fail_cont_count); - - /* Avoid having multiple recoveries at the same time. */ + /* + * Pure deadline monitor: it is armed by + * ath10k_core_start_recovery() and cancelled once recovery + * completes. Bail if there is nothing in flight. + */ + if (!atomic_read(&ar->pending_recovery)) return; - } - atomic_inc(&ar->pending_recovery); - queue_work(ar->workqueue, &ar->restart_work); + ath10k_err(ar, "recovery did not complete in %u ms, will shutdown driver!\n", + jiffies_to_msecs(ATH10K_RECOVERY_TIMEOUT_HZ)); + ar->state = ATH10K_STATE_WEDGED; } void ath10k_core_start_recovery(struct ath10k *ar) { - /* Use workqueue_aux to avoid blocking recovery tracking */ - queue_work(ar->workqueue_aux, &ar->recovery_check_work); + if (ar->state == ATH10K_STATE_WEDGED) + return; + + /* + * SDIO raises several triggers within microseconds of one crash; + * only the first winner starts a fresh cycle. Losers do nothing + * while the owner runs recovery. + */ + if (atomic_cmpxchg(&ar->pending_recovery, 0, 1) != 0) + return; + ath10k_dbg(ar, ATH10K_DBG_BOOT, "total recovery count: %d", + ++ar->recovery_count); + /* Arm the monitor before queueing restart_work so that restart_work + * can cancel it if there turns out to be nothing to restart. + */ + queue_delayed_work(ar->workqueue_aux, &ar->recovery_check_work, + ATH10K_RECOVERY_TIMEOUT_HZ); + queue_work(ar->workqueue, &ar->restart_work); } EXPORT_SYMBOL(ath10k_core_start_recovery); @@ -2655,6 +2652,13 @@ static void ath10k_core_restart(struct work_struct *work) * or if the crash happens during FW probing */ ath10k_warn(ar, "cannot restart a device that hasn't been started\n"); + /* Nothing will be restarted, so no recovery is pending any + * more. Leaving pending_recovery set would block the next real + * recovery and let the monitor wedge a device that is started + * later. + */ + cancel_delayed_work(&ar->recovery_check_work); + atomic_set(&ar->pending_recovery, 0); break; case ATH10K_STATE_RESTARTING: /* hw restart might be requested from multiple places */ @@ -3769,7 +3773,7 @@ struct ath10k *ath10k_core_create(size_t priv_size, struct device *dev, INIT_WORK(&ar->register_work, ath10k_core_register_work); INIT_WORK(&ar->restart_work, ath10k_core_restart); - INIT_WORK(&ar->recovery_check_work, ath10k_core_recovery_check_work); + INIT_DELAYED_WORK(&ar->recovery_check_work, ath10k_core_recovery_check_work); INIT_WORK(&ar->set_coverage_class_work, ath10k_core_set_coverage_class_work); @@ -3806,6 +3810,9 @@ EXPORT_SYMBOL(ath10k_core_create); void ath10k_core_destroy(struct ath10k *ar) { + cancel_work_sync(&ar->restart_work); + cancel_delayed_work_sync(&ar->recovery_check_work); + destroy_workqueue(ar->workqueue); destroy_workqueue(ar->workqueue_aux); diff --git a/drivers/net/wireless/ath/ath10k/core.h b/drivers/net/wireless/ath/ath10k/core.h index dfee432615eb..2e100dd246e2 100644 --- a/drivers/net/wireless/ath/ath10k/core.h +++ b/drivers/net/wireless/ath/ath10k/core.h @@ -87,8 +87,7 @@ IEEE80211_IFACE_SKIP_SDATA_NOT_IN_DRIVER) #define ATH10K_ITER_RESUME_FLAGS (IEEE80211_IFACE_ITER_RESUME_ALL |\ IEEE80211_IFACE_SKIP_SDATA_NOT_IN_DRIVER) -#define ATH10K_RECOVERY_TIMEOUT_HZ (5 * HZ) -#define ATH10K_RECOVERY_MAX_FAIL_COUNT 4 +#define ATH10K_RECOVERY_TIMEOUT_HZ (20 * HZ) struct ath10k; @@ -1207,14 +1206,12 @@ struct ath10k { struct work_struct register_work; struct work_struct restart_work; - struct work_struct recovery_check_work; + struct delayed_work recovery_check_work; struct work_struct bundle_tx_work; struct work_struct tx_complete_work; atomic_t pending_recovery; unsigned int recovery_count; - /* continuous recovery fail count */ - atomic_t fail_cont_count; /* cycle count is reported twice for each visited channel during scan. * access protected by data_lock diff --git a/drivers/net/wireless/ath/ath10k/mac.c b/drivers/net/wireless/ath/ath10k/mac.c index 8e604697d6c2..74aca0af11c8 100644 --- a/drivers/net/wireless/ath/ath10k/mac.c +++ b/drivers/net/wireless/ath/ath10k/mac.c @@ -5427,7 +5427,14 @@ static void ath10k_stop(struct ieee80211_hw *hw, bool suspend) cancel_work_sync(&ar->set_coverage_class_work); cancel_delayed_work_sync(&ar->scan.timeout); cancel_work_sync(&ar->restart_work); - cancel_work_sync(&ar->recovery_check_work); + cancel_delayed_work_sync(&ar->recovery_check_work); + + /* ath10k_stop() can race with an in-flight recovery. Clear the + * recovery state after the works are cancelled so a later + * ath10k_start() can start a fresh cycle instead of being blocked + * by a stale pending_recovery that nothing will clear. + */ + atomic_set(&ar->pending_recovery, 0); } static int ath10k_config_ps(struct ath10k *ar) @@ -8170,9 +8177,13 @@ static void ath10k_reconfig_complete(struct ieee80211_hw *hw, ar->state = ATH10K_STATE_ON; ieee80211_wake_queues(ar->hw); + /* Cancel the deadline monitor first so it cannot declare the + * device wedged after recovery has completed. + */ + cancel_delayed_work_sync(&ar->recovery_check_work); + /* Clear recovery state. */ complete(&ar->driver_recovery); - atomic_set(&ar->fail_cont_count, 0); atomic_set(&ar->pending_recovery, 0); if (ar->hw_params.hw_restart_disconnect) { -- 2.34.1