From: Mani Tofigh watchdog_check_skew() waits for the remote CPU with interrupts disabled and gives up after a timeout measured with ktime_get_raw_fast_ns(). While the clocksource is still jiffies during boot (or with clocksource=jiffies) that timeout can't expire, because only the boot CPU's tick advances jiffies and that's the CPU spinning with interrupts disabled. The boot then hangs silently when the remote CPU can't answer, e.g., in the stop_machine() of the switch away from jiffies: CPU0 CPU1 multi_cpu_stop() multi_cpu_stop() local_irq_disable() /* waits for CPU0 */ clocksource_watchdog() /* sends CSD to CPU1 */ watchdog_check_skew() /* waits for CPU1 forever */ Measure the timeout with the clocksource under test instead, which keeps counting with interrupts disabled. Also, when the remote CPU still hasn't answered by the next watchdog round, report it as stalled and mark the clocksource unstable instead of retrying forever. With the fix, the watchdog gives up after the timeout and the boot continues. Fixes: 763aacf86f1b ("clocksource: Rewrite watchdog code completely") Reported-by: Srikanth Aithal Closes: https://lore.kernel.org/all/e4a41d43-4d60-4132-aa2d-70e038a3e9dd@amd.com/ Suggested-by: Thomas Gleixner Signed-off-by: Mani Tofigh Tested-by: Srikanth Aithal Cc: stable@vger.kernel.org --- The reproducer hits it within about 80s on a kernel booted with clocksource=jiffies, with both vCPUs of the guest on one host core: https://github.com/manitofigh/files/blob/main/linux-repros/wdboot.c --- Changes in v2: - Fold in the stall check (Thomas) - Drop the hard lockup backtrace (Thomas) - Add Srikanth's Reported-by, Closes and Tested-by - Link to v1: https://patch.msgid.link/20261102-clocksource-skew-timeout-v1-1-4ff412918591@columbia.edu --- kernel/time/clocksource.c | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index 58c0ca11cdc5..5dfa72a2db68 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -250,6 +250,7 @@ enum wd_result { WD_FREQ_RESET, WD_FREQ_SKEWED, WD_CPU_TIMEOUT, + WD_CPU_STALLED, WD_CPU_SKEWED, }; @@ -305,11 +306,16 @@ static bool watchdog_wait_seq(struct watchdog_cpu_data *wd, u64 start, int seq) if (READ_ONCE(wd->result) != WD_SUCCESS) return false; - /* Prevent endless loops if the other CPU does not react. */ + /* + * Prevent endless loops if the other CPU does not react. The + * timeout uses the clocksource under test because the timekeeping + * clocksource can be jiffies, which doesn't advance while this CPU + * waits with interrupts disabled. + */ if (cnt == 5000) { - u64 nsecs = ktime_get_raw_fast_ns(); + struct clocksource *cs = wd->cs; - if (nsecs - start >=wd->timeout_ns) { + if (cycles_to_nsec_safe(cs, start, cs->read(cs)) >= wd->timeout_ns) { watchdog_set_result(wd, WD_CPU_TIMEOUT); return false; } @@ -322,12 +328,13 @@ static bool watchdog_wait_seq(struct watchdog_cpu_data *wd, u64 start, int seq) static void watchdog_check_skew(struct watchdog_cpu_data *wd, int index) { - u64 prev, now, delta, start = ktime_get_raw_fast_ns(); + u64 prev, now, delta, start; int local = index, remote = (index + 1) & 0x1; struct clocksource *cs = wd->cs; /* Set the local timestamp so that the first iteration works correctly */ - wd->cpu_ts[local] = cs->read(cs); + start = cs->read(cs); + wd->cpu_ts[local] = start; /* Signal arrival */ atomic_inc(&wd->seq); @@ -392,7 +399,7 @@ static void __watchdog_check_cpu_skew(struct clocksource *cs, unsigned int cpu) wd = per_cpu_ptr(&watchdog_cpu_data, cpu); if (atomic_read(&wd->remote_inprogress) || wd_csd_locked(wd)) { - watchdog_data.result = WD_CPU_TIMEOUT; + watchdog_data.result = WD_CPU_STALLED; return; } @@ -584,11 +591,16 @@ static void watchdog_print_freq_skew(struct clocksource *cs) pr_warn("Clocksource %20s interval: %16lluns\n", cs->name, watchdog_data.cs_delta); } -static void watchdog_handle_remote_timeout(struct clocksource *cs) +static void watchdog_print_remote_timeout(struct clocksource *cs) { pr_info_once("Watchdog remote CPU %u read timed out\n", watchdog_data.curr_cpu); } +static void watchdog_print_remote_stall(struct clocksource *cs) +{ + pr_warn("Watchdog remote CPU %u stalled\n", watchdog_data.curr_cpu); +} + static void watchdog_print_remote_skew(struct clocksource *cs) { pr_warn("Marking clocksource %s unstable due to inter CPU skew\n", cs->name); @@ -629,9 +641,13 @@ static void watchdog_check_result(struct clocksource *cs) case WD_CPU_TIMEOUT: /* Remote check timed out. Try again next cycle. */ - watchdog_handle_remote_timeout(cs); + watchdog_print_remote_timeout(cs); return; + case WD_CPU_STALLED: + watchdog_print_remote_stall(cs); + break; + case WD_CPU_SKEWED: watchdog_print_remote_skew(cs); break; --- base-commit: c1b96ba87e3caaa701d6dba254e58db7cc3c99be change-id: 20261102-clocksource-skew-timeout-f2a9e5fb3f75 Best regards, -- Mani Tofigh