From: Andreas Hindborg C block device drivers can attach private data to a `struct request`. This data is stored next to the request structure and is part of the request allocation set up during driver initialization. Expose this private request data area to Rust block device drivers. Signed-off-by: Andreas Hindborg --- drivers/block/rnull/rnull.rs | 5 +++++ rust/kernel/block/mq.rs | 6 ++++++ rust/kernel/block/mq/operations.rs | 26 +++++++++++++++++++++++++- rust/kernel/block/mq/request.rs | 24 +++++++++++++++++++----- rust/kernel/block/mq/tag_set.rs | 24 +++++++++++++++++++----- 5 files changed, 74 insertions(+), 11 deletions(-) diff --git a/drivers/block/rnull/rnull.rs b/drivers/block/rnull/rnull.rs index 69cf62475446..dd7a30519870 100644 --- a/drivers/block/rnull/rnull.rs +++ b/drivers/block/rnull/rnull.rs @@ -132,6 +132,11 @@ struct QueueData { #[vtable] impl Operations for NullBlkDevice { type QueueData = KBox; + type RequestData = (); + + fn new_request_data() -> impl PinInit { + Ok(()) + } #[inline(always)] fn queue_rq(queue_data: &QueueData, rq: Owned>, _is_last: bool) -> Result { diff --git a/rust/kernel/block/mq.rs b/rust/kernel/block/mq.rs index b8ecd69abe98..7718b106eb49 100644 --- a/rust/kernel/block/mq.rs +++ b/rust/kernel/block/mq.rs @@ -69,8 +69,14 @@ //! //! #[vtable] //! impl Operations for MyBlkDevice { +//! type RequestData = (); //! type QueueData = (); //! +//! fn new_request_data( +//! ) -> impl PinInit<()> { +//! Ok(()) +//! } +//! //! fn queue_rq(_queue_data: (), rq: Owned>, _is_last: bool) -> Result { //! rq.end_ok(); //! Ok(()) diff --git a/rust/kernel/block/mq/operations.rs b/rust/kernel/block/mq/operations.rs index bb23a32f3983..c49ca2e8bbb2 100644 --- a/rust/kernel/block/mq/operations.rs +++ b/rust/kernel/block/mq/operations.rs @@ -29,6 +29,7 @@ marker::PhantomData, ptr::NonNull, // }; +use pin_init::PinInit; type ForeignBorrowed<'a, T> = ::Borrowed<'a>; @@ -44,10 +45,27 @@ /// [module level documentation]: kernel::block::mq #[macros::vtable] pub trait Operations: Sized { + /// Data associated with a request. This data is located next to the request + /// structure. + /// + /// To be able to handle accessing this data from interrupt context, this + /// data must be `Sync`. + /// + /// Requests may be cleaned up by a thread different from the allocating thread, so + /// `RequestData` must be `Send`. + /// + /// The `RequestData` object is initialized when the requests are allocated + /// during queue initialization, and it is are dropped when the requests are + /// dropped during queue teardown. + type RequestData: Sized + Sync + Send; + /// Data associated with the `struct request_queue` that is allocated for /// the `GenDisk` associated with this `Operations` implementation. type QueueData: ForeignOwnable + Sync; + /// Called by the kernel to get an initializer for a `Pin<&mut RequestData>`. + fn new_request_data() -> impl PinInit; + /// Called by the kernel to queue a request with the driver. If `is_last` is /// `false`, the driver is allowed to defer committing the request. fn queue_rq( @@ -252,6 +270,12 @@ impl OperationsVTable { // it is valid for writes. unsafe { RequestDataWrapper::refcount_ptr(pdu.as_ptr()).write(Refcount::new(0)) }; + let initializer = T::new_request_data(); + + // SAFETY: `pdu` is a valid pointer as established above. We do not touch `pdu` if + // `__pinned_init` returns an error. We promise not to move the pointee of `pdu`. + unsafe { initializer.__pinned_init(RequestDataWrapper::data_ptr(pdu.as_ptr()))? }; + Ok(0) }) } @@ -271,7 +295,7 @@ impl OperationsVTable { ) { // SAFETY: The tagset invariants guarantee that all requests are allocated with extra memory // for the request data. - let pdu = unsafe { bindings::blk_mq_rq_to_pdu(rq) }.cast::(); + let pdu = unsafe { bindings::blk_mq_rq_to_pdu(rq) }.cast::>(); // SAFETY: `pdu` is valid for read and write and is properly initialised. unsafe { core::ptr::drop_in_place(pdu) }; diff --git a/rust/kernel/block/mq/request.rs b/rust/kernel/block/mq/request.rs index 7444de3c8522..1882d697dcf3 100644 --- a/rust/kernel/block/mq/request.rs +++ b/rust/kernel/block/mq/request.rs @@ -107,12 +107,12 @@ pub fn complete(this: ARef) { /// /// - `this` must point to a valid allocation of size at least size of /// [`Self`] plus size of [`RequestDataWrapper`]. - pub(crate) unsafe fn wrapper_ptr(this: *mut Self) -> NonNull { + pub(crate) unsafe fn wrapper_ptr(this: *mut Self) -> NonNull> { let request_ptr = this.cast::(); // SAFETY: By safety requirements for this function, `this` is a // valid allocation. let wrapper_ptr = - unsafe { bindings::blk_mq_rq_to_pdu(request_ptr).cast::() }; + unsafe { bindings::blk_mq_rq_to_pdu(request_ptr).cast::>() }; // SAFETY: By C API contract, `wrapper_ptr` points to a valid allocation // and is not null. unsafe { NonNull::new_unchecked(wrapper_ptr) } @@ -120,7 +120,7 @@ pub(crate) unsafe fn wrapper_ptr(this: *mut Self) -> NonNull /// Return a reference to the [`RequestDataWrapper`] stored in the private /// area of the request structure. - pub(crate) fn wrapper_ref(&self) -> &RequestDataWrapper { + pub(crate) fn wrapper_ref(&self) -> &RequestDataWrapper { // SAFETY: By type invariant, `self.0` is a valid allocation. Further, // the private data associated with this request is initialized and // valid. The existence of `&self` guarantees that the private data is @@ -132,16 +132,19 @@ pub(crate) fn wrapper_ref(&self) -> &RequestDataWrapper { /// A wrapper around data stored in the private area of the C [`struct request`]. /// /// [`struct request`]: srctree/include/linux/blk-mq.h -pub(crate) struct RequestDataWrapper { +pub(crate) struct RequestDataWrapper { /// The Rust request refcount has the following states: /// /// - 0: The request is owned by C block layer. /// - 1: The request is owned by Rust abstractions but there are no [`ARef`] references to it. /// - 2+: There are [`ARef`] references to the request. refcount: Refcount, + + /// Driver managed request data + data: T::RequestData, } -impl RequestDataWrapper { +impl RequestDataWrapper { /// Return a reference to the refcount of the request that is embedding /// `self`. pub(crate) fn refcount(&self) -> &Refcount { @@ -159,6 +162,17 @@ pub(crate) unsafe fn refcount_ptr(this: *mut Self) -> *mut Refcount { // field projection is safe. unsafe { &raw mut (*this).refcount } } + + /// Return a pointer to the `data` field of the `Self` pointed to by `this`. + /// + /// # Safety + /// + /// - `this` must point to a live allocation of at least the size of `Self`. + pub(crate) unsafe fn data_ptr(this: *mut Self) -> *mut T::RequestData { + // SAFETY: Because of the safety requirements of this function, the + // field projection is safe. + unsafe { &raw mut (*this).data } + } } // SAFETY: Exclusive access is thread-safe for `Request`. `Request` has no `&mut diff --git a/rust/kernel/block/mq/tag_set.rs b/rust/kernel/block/mq/tag_set.rs index dae9df408a86..ec5cac48b83f 100644 --- a/rust/kernel/block/mq/tag_set.rs +++ b/rust/kernel/block/mq/tag_set.rs @@ -8,13 +8,27 @@ use crate::{ bindings, - block::mq::{operations::OperationsVTable, request::RequestDataWrapper, Operations}, - error::{self, Result}, + block::mq::{ + operations::OperationsVTable, + request::RequestDataWrapper, + Operations, // + }, + error::{ + self, + Result, // + }, prelude::try_pin_init, types::Opaque, }; -use core::{convert::TryInto, marker::PhantomData}; -use pin_init::{pin_data, pinned_drop, PinInit}; +use core::{ + convert::TryInto, + marker::PhantomData, // +}; +use pin_init::{ + pin_data, + pinned_drop, + PinInit, // +}; /// A wrapper for the C `struct blk_mq_tag_set`. /// @@ -39,7 +53,7 @@ pub fn new( num_maps: u32, ) -> impl PinInit { let tag_set: bindings::blk_mq_tag_set = pin_init::zeroed(); - let tag_set: Result<_> = core::mem::size_of::() + let tag_set: Result<_> = size_of::>() .try_into() .map(|cmd_size| { bindings::blk_mq_tag_set { -- 2.51.2