Add and export an expedited version of call_srcu() so that users of
synchronize_srcu_expedited() can avoid "transferring" non-expedited grace
periods.  In response to userspace changes to guest devices and memory,
KVM uses call_srcu() when freeing an object to avoid having to wait for
readers to go away, but then often emits a synchronize_srcu_expedited() in
a largely unrelated path shortly thereafter (on the same SRCU object).

Due to differences in how VMMs manage guest devices, and in the
architecture being emulated by userspace, some updates trigger call_srcu()
with concurrent readers (i.e. while the VM is active), while others occur
without readers, e.g. when configuring devices during a pre-boot setup.
For the later case (no concurrent readers), using the vanilla call_srcu()
is problematic, as it can kick off a normal grace period (totally fine for
freeing the object) and effectively transfer the non-expedited grace period
to the upcoming synchronize_srcu_expedited().

For micro-VM use cases with CONFIG_HZ=100 kernels, the resulting ~20ms
delay on the would-be-expedited sync can increase the boot time of the VM
by 15% or more.

Link: https://lore.kernel.org/all/a84ddba8-12da-489a-9dd1-ccdf7451a1ba@amazon.com
Signed-off-by: Sean Christopherson <seanjc@google.com>
---
 include/linux/srcutiny.h | 6 ++++++
 include/linux/srcutree.h | 2 ++
 kernel/rcu/srcutree.c    | 7 +++++++
 3 files changed, 15 insertions(+)

diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h
index 4976536e8b28..e2fc8c138e6a 100644
--- a/include/linux/srcutiny.h
+++ b/include/linux/srcutiny.h
@@ -130,6 +130,12 @@ static inline void srcu_barrier(struct srcu_struct *ssp)
 	synchronize_srcu(ssp);
 }
 
+static inline void call_srcu_expedited(struct srcu_struct *ssp, struct rcu_head *rhp,
+				       rcu_callback_t func)
+{
+	call_srcu(ssp, rhp, func);
+}
+
 static inline void srcu_expedite_current(struct srcu_struct *ssp) { }
 #define srcu_check_read_flavor(ssp, read_flavor) do { } while (0)
 
diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h
index 958cb7ef41cb..ed3cbbe7f5ce 100644
--- a/include/linux/srcutree.h
+++ b/include/linux/srcutree.h
@@ -234,6 +234,8 @@ struct srcu_struct {
 					__DEFINE_SRCU(name, SRCU_READ_FLAVOR_FAST_UPDOWN, static)
 
 int __srcu_read_lock(struct srcu_struct *ssp) __acquires_shared(ssp);
+void call_srcu_expedited(struct srcu_struct *ssp, struct rcu_head *head,
+			 rcu_callback_t func);
 void synchronize_srcu_expedited(struct srcu_struct *ssp);
 void srcu_barrier(struct srcu_struct *ssp);
 void srcu_expedite_current(struct srcu_struct *ssp);
diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c
index aef8e91ad33e..77076d2a1c57 100644
--- a/kernel/rcu/srcutree.c
+++ b/kernel/rcu/srcutree.c
@@ -1495,6 +1495,13 @@ void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,
 }
 EXPORT_SYMBOL_GPL(call_srcu);
 
+void call_srcu_expedited(struct srcu_struct *ssp, struct rcu_head *rhp,
+			 rcu_callback_t func)
+{
+	__call_srcu(ssp, rhp, func, rcu_gp_is_normal());
+}
+EXPORT_SYMBOL_GPL(call_srcu_expedited);
+
 /*
  * Helper function for synchronize_srcu() and synchronize_srcu_expedited().
  */
-- 
2.53.0.473.g4a7958ca14-goog