From: Li RongQing During device hot-unplug, the mlx5 driver expects quickly unregister notification chains. The standard atomic_notifier_chain_unregister() calls synchronize_rcu(), which introduces significant latency and can become a bottleneck during mass resource cleanup. Introduce atomic_notifier_chain_unregister_expedited() to leverage synchronize_rcu_expedited(), and use it significantly reducing wait times in the following paths: - Event Queue (EQ) notifier chain - Firmware event notifier chain - IRQ notifier chain On x86-64 with HZ=1000, 64 networking channels: - Average teardown time: 3.59s -> 1.9s (47% reduction) On x86-64 with HZ=250, 64 networking channels: - Average teardown time: 5.5s -> 1.9s (65% reduction) Co-developed-by: liyongkang Signed-off-by: liyongkang Signed-off-by: Li RongQing Signed-off-by: Tariq Toukan --- V3: rebased. V2: https://lore.kernel.org/all/20260317003544.2583-1-lirongqing@baidu.com/ drivers/net/ethernet/mellanox/mlx5/core/eq.c | 2 +- .../net/ethernet/mellanox/mlx5/core/events.c | 2 +- .../net/ethernet/mellanox/mlx5/core/pci_irq.c | 2 +- include/linux/notifier.h | 2 ++ kernel/notifier.c | 24 +++++++++++++++++++ 5 files changed, 29 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c index d11ec263d53c..c86090cb58c6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c @@ -1243,6 +1243,6 @@ int mlx5_eq_notifier_unregister(struct mlx5_core_dev *dev, struct mlx5_nb *nb) { struct mlx5_eq_table *eqt = dev->priv.eq_table; - return atomic_notifier_chain_unregister(&eqt->nh[nb->event_type], &nb->nb); + return atomic_notifier_chain_unregister_expedited(&eqt->nh[nb->event_type], &nb->nb); } EXPORT_SYMBOL(mlx5_eq_notifier_unregister); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/events.c b/drivers/net/ethernet/mellanox/mlx5/core/events.c index 4d7f35b96876..753cb15cdeee 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/events.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/events.c @@ -436,7 +436,7 @@ int mlx5_notifier_unregister(struct mlx5_core_dev *dev, struct notifier_block *n { struct mlx5_events *events = dev->priv.events; - return atomic_notifier_chain_unregister(&events->fw_nh, nb); + return atomic_notifier_chain_unregister_expedited(&events->fw_nh, nb); } EXPORT_SYMBOL(mlx5_notifier_unregister); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c index 0f5b8bc7861e..cb31e8c7a244 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c @@ -343,7 +343,7 @@ int mlx5_irq_detach_nb(struct mlx5_irq *irq, struct notifier_block *nb) { int err = 0; - err = atomic_notifier_chain_unregister(&irq->nh, nb); + err = atomic_notifier_chain_unregister_expedited(&irq->nh, nb); mlx5_irq_put(irq); return err; } diff --git a/include/linux/notifier.h b/include/linux/notifier.h index 01b6c9d9956f..156d958f98f0 100644 --- a/include/linux/notifier.h +++ b/include/linux/notifier.h @@ -159,6 +159,8 @@ extern int blocking_notifier_chain_register_unique_prio( extern int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh, struct notifier_block *nb); +extern int atomic_notifier_chain_unregister_expedited(struct atomic_notifier_head *nh, + struct notifier_block *nb); extern int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh, struct notifier_block *nb); extern int raw_notifier_chain_unregister(struct raw_notifier_head *nh, diff --git a/kernel/notifier.c b/kernel/notifier.c index 2f9fe7c30287..9b35822fc2c0 100644 --- a/kernel/notifier.c +++ b/kernel/notifier.c @@ -197,6 +197,30 @@ int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh, } EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister); +/** + * atomic_notifier_chain_unregister_expedited - Remove notifier from an atomic notifier chain + * @nh: Pointer to head of the atomic notifier chain + * @n: Entry to remove from notifier chain + * + * Removes a notifier from an atomic notifier chain and forcefully + * accelerates the RCU grace period. + * + * Return: 0 on success, or -ENOENT on failure. + */ +int atomic_notifier_chain_unregister_expedited(struct atomic_notifier_head *nh, + struct notifier_block *n) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&nh->lock, flags); + ret = notifier_chain_unregister(&nh->head, n); + spin_unlock_irqrestore(&nh->lock, flags); + synchronize_rcu_expedited(); + return ret; +} +EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister_expedited); + /** * atomic_notifier_call_chain - Call functions in an atomic notifier chain * @nh: Pointer to head of the atomic notifier chain base-commit: d67e5dbda22604d0fcde32fce58c65f88676e676 -- 2.44.0