Rust Binder currently uses PollCondVar, but it calls synchronize_rcu() in the destructor, which we would like to avoid. Add a variation of PollCondVar that kfree_rcu() instead. One could avoid the `rcu` field and allocate the rcu_head on drop using a fallback to synchronize_rcu() on ENOMEM. However, I'd prefer to avoid the potential for synchronize_rcu(), and Binder will only use this for a small fraction of processes, so even if it changes which kmalloc bucket it falls into, the extra memory is not a problem. Signed-off-by: Alice Ryhl --- rust/kernel/sync/poll.rs | 73 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/rust/kernel/sync/poll.rs b/rust/kernel/sync/poll.rs index 0ec985d560c8..684dfa242b1a 100644 --- a/rust/kernel/sync/poll.rs +++ b/rust/kernel/sync/poll.rs @@ -5,12 +5,18 @@ //! Utilities for working with `struct poll_table`. use crate::{ + alloc::AllocError, bindings, fs::File, prelude::*, sync::{CondVar, LockClassKey}, + types::Opaque, // +}; +use core::{ + marker::PhantomData, + mem::ManuallyDrop, + ops::Deref, // }; -use core::{marker::PhantomData, ops::Deref}; /// Creates a [`PollCondVar`] initialiser with the given name and a newly-created lock class. #[macro_export] @@ -66,6 +72,7 @@ pub fn register_wait(&self, file: &File, cv: &PollCondVar) { /// /// [`CondVar`]: crate::sync::CondVar #[pin_data(PinnedDrop)] +#[repr(transparent)] pub struct PollCondVar { #[pin] inner: CondVar, @@ -104,3 +111,67 @@ fn drop(self: Pin<&mut Self>) { unsafe { bindings::synchronize_rcu() }; } } + +/// A [`KBox`] that uses `kfree_rcu`. +/// +/// [`KBox`]: PollCondVar +pub struct PollCondVarBox { + inner: ManuallyDrop>>, +} + +#[pin_data] +#[repr(C)] +struct PollCondVarBoxInner { + #[pin] + inner: PollCondVar, + rcu: Opaque, +} + +// SAFETY: PollCondVar is Send +unsafe impl Send for PollCondVarBoxInner {} +// SAFETY: PollCondVar is Sync +unsafe impl Sync for PollCondVarBoxInner {} + +impl PollCondVarBox { + /// Constructs a new boxed [`PollCondVar`]. + pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> Result { + let b = KBox::pin_init( + pin_init!(PollCondVarBoxInner { + inner <- PollCondVar::new(name, key), + rcu: Opaque::uninit(), + }), + GFP_KERNEL, + ) + .map_err(|_| AllocError)?; + + Ok(PollCondVarBox { + inner: ManuallyDrop::new(b), + }) + } +} + +impl Deref for PollCondVarBox { + type Target = PollCondVar; + fn deref(&self) -> &PollCondVar { + &self.inner.inner + } +} + +impl Drop for PollCondVarBox { + #[inline] + fn drop(&mut self) { + // SAFETY: ManuallyDrop::take ok because not already taken. + let boxed = unsafe { ManuallyDrop::take(&mut self.inner) }; + + // SAFETY: The code below frees the box without calling the actual destructor of the type, + // but it's okay because it re-implements the destructor using `kfree_rcu()` in place of + // `synchronize_rcu()`. + let ptr = KBox::into_raw(unsafe { Pin::into_inner_unchecked(boxed) }); + + // SAFETY: The pointer points at a valid `wait_queue_head`. + unsafe { bindings::__wake_up_pollfree((*ptr).inner.inner.wait_queue_head.get()) }; + + // SAFETY: This was allocated using `KBox::pin_init`, so it can be freed with `kvfree`. + unsafe { bindings::kvfree_call_rcu((*ptr).rcu.get(), ptr.cast::()) }; + } +} -- 2.55.0.rc2.803.g1fd1e6609c-goog