Since commit 95ecba62e2fd ("net: fix races in netdev_tx_sent_queue()/dev_watchdog()"), netif_tx_stop_queue() writes txq->trans_start on every call, also when the queue is already stopped. Several drivers stop the queue from their periodic Tx hang check and leave the recovery to dev_watchdog(). e1000, e1000e, igb and igc call netif_stop_queue() or netif_stop_subqueue() each time the hang check, requested every 2 seconds by the driver watchdog, finds the hang again, and do not wake the queue. Each call now moves trans_start forward, so dev_watchdog() never sees the queue stopped for longer than watchdog_timeo (5 seconds for these drivers). ndo_tx_timeout() is never called and the adapter is never reset: the driver logs "Detected Tx Unit Hang" every 2 seconds while the NIC stays hung. Write trans_start only when the driver stops a queue that it had not stopped yet, i.e. when __QUEUE_STATE_DRV_XOFF is clear. The write still happens before set_bit(), which keeps the ordering against dev_watchdog() added by 95ecba62e2fd. A queue stopped only by BQL (__QUEUE_STATE_STACK_XOFF) still gets a new trans_start when the driver stops it. The test_bit() and set_bit() pair is not atomic. If netif_tx_wake_queue() runs between the two, trans_start keeps the time of the previous stop and dev_watchdog() measures from that earlier stop. A timeout is then reported only if both stops together last longer than watchdog_timeo. The issue was observed on a system in real use: a hung NIC kept reporting a Tx unit hang and was never reset. Code inspection then pointed to 95ecba62e2fd. I reproduced the issue and tested the fix on an AlmaLinux 10 kernel (6.12.0-211.68.1.el10_2, which contains 95ecba62e2fd) with the e1000 driver on a VMware-emulated Intel 82545EM. Reproducing required modifying the driver: a test-only module parameter makes e1000_clean_tx_irq() report a Tx unit hang on every hang check of one interface. netif_tx_stop_queue() is inlined into the driver, so the patched variant was e1000 built out of tree against the 6.12 headers with this change applied. With the fake hang held for 30 seconds: unpatched: 15 "Detected Tx Unit Hang", no NETDEV WATCHDOG, tx_timeout 0 -> 0 patched: first "NETDEV WATCHDOG ... timed out 5054 ms" 5 seconds after the first stop, followed by an adapter reset; tx_timeout 0 -> 5 This patch was not booted on the net tree. The KUnit tests added in the next patch pass with this change and fail without it. I wrote the fix. Claude Code (Anthropic) reviewed it, built the e1000 reproducer and wrote this changelog. Fixes: 95ecba62e2fd ("net: fix races in netdev_tx_sent_queue()/dev_watchdog()") Cc: stable@vger.kernel.org Signed-off-by: Curtis McBurney --- include/linux/netdevice.h | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 435be7880d7e..6a488245db1e 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -3805,8 +3805,15 @@ static inline void netif_tx_wake_all_queues(struct net_device *dev) static __always_inline void netif_tx_stop_queue(struct netdev_queue *dev_queue) { - /* Paired with READ_ONCE() from dev_watchdog() */ - WRITE_ONCE(dev_queue->trans_start, jiffies); + /* Only write trans_start on the initial state transition + * (when the DRV_XOFF bit changes from 0 to 1). + * This prevents recursive driver watchdog polls from continuously + * resetting the timeout timeline and blinding dev_watchdog(). + * + * Paired with READ_ONCE() from dev_watchdog() + */ + if (!test_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) + WRITE_ONCE(dev_queue->trans_start, jiffies); /* This barrier is paired with smp_mb() from dev_watchdog() */ smp_mb__before_atomic(); base-commit: aa40afa831ebfe45e40a23721711059f2ac584d4 -- 2.52.0 Add KUnit tests for the txq->trans_start handling of netif_tx_stop_queue(), which dev_watchdog() uses to detect a stuck queue: - a stop on a running queue writes trans_start - a stop on a queue already stopped by the driver keeps trans_start - a stop after netif_tx_start_queue() writes trans_start again - a stop on a queue stopped only by BQL (__QUEUE_STATE_STACK_XOFF) writes trans_start txq_stop_keeps_stamp_stopped_queue fails without the previous patch. Tested on UML with CONFIG_KUNIT, CONFIG_NET and CONFIG_NET_TEST: tools/testing/kunit/kunit.py run --kunitconfig= 'net_core.txq_*' Claude Code (Anthropic) wrote these tests and this changelog. Signed-off-by: Curtis McBurney --- net/core/net_test.c | 112 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/net/core/net_test.c b/net/core/net_test.c index 2b5080c22663..c69d34d0e230 100644 --- a/net/core/net_test.c +++ b/net/core/net_test.c @@ -1721,6 +1721,114 @@ static void netmem_ref_test_tcp_devmem_recv(struct kunit *test) #endif #endif +/* netif_tx_stop_queue() and txq->trans_start */ + +#include +#include +#include + +/* dev_watchdog() reports a timeout when a queue stays stopped for longer + * than watchdog_timeo after txq->trans_start. A trans_start value this far + * in the past is distinguishable from a fresh one. + */ +#define TXQ_TEST_OLD_DELTA (10 * HZ) + +KUNIT_DEFINE_ACTION_WRAPPER(txq_test_free_netdev, free_netdev, + struct net_device *); + +static struct netdev_queue *txq_test_alloc(struct kunit *test) +{ + struct net_device *dev; + + dev = alloc_etherdev(0); + KUNIT_ASSERT_NOT_NULL(test, dev); + KUNIT_ASSERT_EQ(test, 0, + kunit_add_action_or_reset(test, txq_test_free_netdev, + dev)); + + return netdev_get_tx_queue(dev, 0); +} + +static void txq_test_expect_fresh(struct kunit *test, + struct netdev_queue *txq, + unsigned long before) +{ + unsigned long stamp = READ_ONCE(txq->trans_start); + + KUNIT_EXPECT_TRUE_MSG(test, time_in_range(stamp, before, jiffies), + "trans_start=%lu not in [%lu, %lu]", + stamp, before, jiffies); +} + +/* A stop on a running queue records the stop time in trans_start */ +static void txq_stop_stamps_running_queue(struct kunit *test) +{ + struct netdev_queue *txq = txq_test_alloc(test); + unsigned long before; + + WRITE_ONCE(txq->trans_start, jiffies - TXQ_TEST_OLD_DELTA); + before = jiffies; + netif_tx_stop_queue(txq); + + KUNIT_EXPECT_TRUE(test, netif_tx_queue_stopped(txq)); + txq_test_expect_fresh(test, txq, before); +} + +/* A stop on a queue that is already stopped keeps the first stop time, so + * repeated stops from a driver hang check do not postpone dev_watchdog() + */ +static void txq_stop_keeps_stamp_stopped_queue(struct kunit *test) +{ + struct netdev_queue *txq = txq_test_alloc(test); + unsigned long old, stamp; + + netif_tx_stop_queue(txq); + old = jiffies - TXQ_TEST_OLD_DELTA; + WRITE_ONCE(txq->trans_start, old); + netif_tx_stop_queue(txq); + stamp = READ_ONCE(txq->trans_start); + + KUNIT_EXPECT_TRUE(test, netif_tx_queue_stopped(txq)); + KUNIT_EXPECT_EQ_MSG(test, stamp, old, + "second stop moved trans_start forward by %lu jiffies", + stamp - old); +} + +/* A stop after a restart records a new stop time */ +static void txq_stop_stamps_restarted_queue(struct kunit *test) +{ + struct netdev_queue *txq = txq_test_alloc(test); + unsigned long before; + + netif_tx_stop_queue(txq); + netif_tx_start_queue(txq); + KUNIT_EXPECT_FALSE(test, netif_tx_queue_stopped(txq)); + + WRITE_ONCE(txq->trans_start, jiffies - TXQ_TEST_OLD_DELTA); + before = jiffies; + netif_tx_stop_queue(txq); + + KUNIT_EXPECT_TRUE(test, netif_tx_queue_stopped(txq)); + txq_test_expect_fresh(test, txq, before); +} + +/* Only the driver XOFF bit decides whether the stop is a transition. A queue + * stopped by BQL (stack XOFF) gets a new stop time when the driver stops it. + */ +static void txq_stop_stamps_stack_stopped_queue(struct kunit *test) +{ + struct netdev_queue *txq = txq_test_alloc(test); + unsigned long before; + + set_bit(__QUEUE_STATE_STACK_XOFF, &txq->state); + WRITE_ONCE(txq->trans_start, jiffies - TXQ_TEST_OLD_DELTA); + before = jiffies; + netif_tx_stop_queue(txq); + + KUNIT_EXPECT_TRUE(test, netif_tx_queue_stopped(txq)); + txq_test_expect_fresh(test, txq, before); +} + static struct kunit_case net_test_cases[] = { KUNIT_CASE_PARAM(gso_test_func, gso_test_gen_params), KUNIT_CASE_PARAM(ip_tunnel_flags_test_run, @@ -1738,6 +1846,10 @@ static struct kunit_case net_test_cases[] = { KUNIT_CASE(netmem_ref_test_tcp_devmem_recv), #endif #endif + KUNIT_CASE(txq_stop_stamps_running_queue), + KUNIT_CASE(txq_stop_keeps_stamp_stopped_queue), + KUNIT_CASE(txq_stop_stamps_restarted_queue), + KUNIT_CASE(txq_stop_stamps_stack_stopped_queue), { }, }; -- 2.52.0