From: Li RongQing During device hot-unplug, the mlx5 driver expects quickly unregister notification chains. The standard atomic_notifier_chain_unregister() calls synchronize_rcu(), which introduces significant latency and can become a bottleneck during mass resource cleanup. Introduce atomic_notifier_chain_unregister_expedited() to leverage synchronize_rcu_expedited(), and use it significantly reducing wait times in the following paths: - Event Queue (EQ) notifier chain - Firmware event notifier chain - IRQ notifier chain On x86-64 with HZ=1000, 64 networking channels: - Average teardown time: 3.59s -> 1.9s (47% reduction) On x86-64 with HZ=250, 64 networking channels: - Average teardown time: 5.5s -> 1.9s (65% reduction) Co-developed-by: liyongkang Signed-off-by: liyongkang Signed-off-by: Li RongQing --- Diff with v1: fix doc warning and add detailed example drivers/net/ethernet/mellanox/mlx5/core/eq.c | 2 +- drivers/net/ethernet/mellanox/mlx5/core/events.c | 2 +- drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c | 2 +- include/linux/notifier.h | 2 ++ kernel/notifier.c | 24 +++++++++++++++++++++++ 5 files changed, 29 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c index 22a6371..03ae6ed 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c @@ -1244,6 +1244,6 @@ int mlx5_eq_notifier_unregister(struct mlx5_core_dev *dev, struct mlx5_nb *nb) { struct mlx5_eq_table *eqt = dev->priv.eq_table; - return atomic_notifier_chain_unregister(&eqt->nh[nb->event_type], &nb->nb); + return atomic_notifier_chain_unregister_expedited(&eqt->nh[nb->event_type], &nb->nb); } EXPORT_SYMBOL(mlx5_eq_notifier_unregister); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/events.c b/drivers/net/ethernet/mellanox/mlx5/core/events.c index 4d7f35b..753cb15 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/events.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/events.c @@ -436,7 +436,7 @@ int mlx5_notifier_unregister(struct mlx5_core_dev *dev, struct notifier_block *n { struct mlx5_events *events = dev->priv.events; - return atomic_notifier_chain_unregister(&events->fw_nh, nb); + return atomic_notifier_chain_unregister_expedited(&events->fw_nh, nb); } EXPORT_SYMBOL(mlx5_notifier_unregister); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c index e051b9a..826685d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c @@ -356,7 +356,7 @@ int mlx5_irq_detach_nb(struct mlx5_irq *irq, struct notifier_block *nb) { int err = 0; - err = atomic_notifier_chain_unregister(&irq->nh, nb); + err = atomic_notifier_chain_unregister_expedited(&irq->nh, nb); mlx5_irq_put(irq); return err; } diff --git a/include/linux/notifier.h b/include/linux/notifier.h index 01b6c9d..156d958 100644 --- a/include/linux/notifier.h +++ b/include/linux/notifier.h @@ -159,6 +159,8 @@ extern int blocking_notifier_chain_register_unique_prio( extern int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh, struct notifier_block *nb); +extern int atomic_notifier_chain_unregister_expedited(struct atomic_notifier_head *nh, + struct notifier_block *nb); extern int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh, struct notifier_block *nb); extern int raw_notifier_chain_unregister(struct raw_notifier_head *nh, diff --git a/kernel/notifier.c b/kernel/notifier.c index 2f9fe7c..9b35822 100644 --- a/kernel/notifier.c +++ b/kernel/notifier.c @@ -198,6 +198,30 @@ int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh, EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister); /** + * atomic_notifier_chain_unregister_expedited - Remove notifier from an atomic notifier chain + * @nh: Pointer to head of the atomic notifier chain + * @n: Entry to remove from notifier chain + * + * Removes a notifier from an atomic notifier chain and forcefully + * accelerates the RCU grace period. + * + * Return: 0 on success, or -ENOENT on failure. + */ +int atomic_notifier_chain_unregister_expedited(struct atomic_notifier_head *nh, + struct notifier_block *n) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&nh->lock, flags); + ret = notifier_chain_unregister(&nh->head, n); + spin_unlock_irqrestore(&nh->lock, flags); + synchronize_rcu_expedited(); + return ret; +} +EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister_expedited); + +/** * atomic_notifier_call_chain - Call functions in an atomic notifier chain * @nh: Pointer to head of the atomic notifier chain * @val: Value passed unmodified to notifier function -- 2.9.4