C block device drivers can attach private data to a `struct request`. This data is stored next to the request structure and is part of the request allocation set up during driver initialization. Expose this private request data area to Rust block device drivers. Signed-off-by: Andreas Hindborg --- drivers/block/rnull/rnull.rs | 5 +++++ rust/kernel/block/mq.rs | 6 ++++++ rust/kernel/block/mq/operations.rs | 24 +++++++++++++++++++++++- rust/kernel/block/mq/request.rs | 24 +++++++++++++++++++----- rust/kernel/block/mq/tag_set.rs | 2 +- 5 files changed, 54 insertions(+), 7 deletions(-) diff --git a/drivers/block/rnull/rnull.rs b/drivers/block/rnull/rnull.rs index 6a7f660d31998..065639fc4f941 100644 --- a/drivers/block/rnull/rnull.rs +++ b/drivers/block/rnull/rnull.rs @@ -134,6 +134,11 @@ struct QueueData { #[vtable] impl Operations for NullBlkDevice { type QueueData = KBox; + type RequestData = (); + + fn new_request_data() -> impl PinInit { + pin_init::zeroed::() + } #[inline(always)] fn queue_rq(queue_data: &QueueData, rq: Owned>, _is_last: bool) -> Result { diff --git a/rust/kernel/block/mq.rs b/rust/kernel/block/mq.rs index b8ecd69abe980..a285b753ada88 100644 --- a/rust/kernel/block/mq.rs +++ b/rust/kernel/block/mq.rs @@ -69,8 +69,14 @@ //! //! #[vtable] //! impl Operations for MyBlkDevice { +//! type RequestData = (); //! type QueueData = (); //! +//! fn new_request_data( +//! ) -> impl PinInit<()> { +//! pin_init::zeroed::<()>() +//! } +//! //! fn queue_rq(_queue_data: (), rq: Owned>, _is_last: bool) -> Result { //! rq.end_ok(); //! Ok(()) diff --git a/rust/kernel/block/mq/operations.rs b/rust/kernel/block/mq/operations.rs index 3dea79d647ff7..cd37b939bbf30 100644 --- a/rust/kernel/block/mq/operations.rs +++ b/rust/kernel/block/mq/operations.rs @@ -13,6 +13,7 @@ types::{ForeignOwnable, Owned}, }; use core::{marker::PhantomData, ptr::NonNull}; +use pin_init::PinInit; type ForeignBorrowed<'a, T> = ::Borrowed<'a>; @@ -28,10 +29,24 @@ /// [module level documentation]: kernel::block::mq #[macros::vtable] pub trait Operations: Sized { + /// Data associated with a request. This data is located next to the request + /// structure. + /// + /// To be able to handle accessing this data from interrupt context, this + /// data must be `Sync`. + /// + /// The `RequestData` object is initialized when the requests are allocated + /// during queue initialization, and it is are dropped when the requests are + /// dropped during queue teardown. + type RequestData: Sized + Sync; + /// Data associated with the `struct request_queue` that is allocated for /// the `GenDisk` associated with this `Operations` implementation. type QueueData: ForeignOwnable; + /// Called by the kernel to get an initializer for a `Pin<&mut RequestData>`. + fn new_request_data() -> impl PinInit; + /// Called by the kernel to queue a request with the driver. If `is_last` is /// `false`, the driver is allowed to defer committing the request. fn queue_rq( @@ -236,6 +251,13 @@ impl OperationsVTable { // it is valid for writes. unsafe { RequestDataWrapper::refcount_ptr(pdu.as_ptr()).write(Refcount::new(0)) }; + let initializer = T::new_request_data(); + + // SAFETY: `pdu` is a valid pointer as established above. We do not + // touch `pdu` if `__pinned_init` returns an error. We promise ot to + // move the pointee of `pdu`. + unsafe { initializer.__pinned_init(RequestDataWrapper::data_ptr(pdu.as_ptr()))? }; + Ok(0) }) } @@ -255,7 +277,7 @@ impl OperationsVTable { ) { // SAFETY: The tagset invariants guarantee that all requests are allocated with extra memory // for the request data. - let pdu = unsafe { bindings::blk_mq_rq_to_pdu(rq) }.cast::(); + let pdu = unsafe { bindings::blk_mq_rq_to_pdu(rq) }.cast::>(); // SAFETY: `pdu` is valid for read and write and is properly initialised. unsafe { core::ptr::drop_in_place(pdu) }; diff --git a/rust/kernel/block/mq/request.rs b/rust/kernel/block/mq/request.rs index 148348b4ef245..8a6c29ac627ee 100644 --- a/rust/kernel/block/mq/request.rs +++ b/rust/kernel/block/mq/request.rs @@ -96,12 +96,12 @@ pub fn complete(this: ARef) { /// /// - `this` must point to a valid allocation of size at least size of /// [`Self`] plus size of [`RequestDataWrapper`]. - pub(crate) unsafe fn wrapper_ptr(this: *mut Self) -> NonNull { + pub(crate) unsafe fn wrapper_ptr(this: *mut Self) -> NonNull> { let request_ptr = this.cast::(); // SAFETY: By safety requirements for this function, `this` is a // valid allocation. let wrapper_ptr = - unsafe { bindings::blk_mq_rq_to_pdu(request_ptr).cast::() }; + unsafe { bindings::blk_mq_rq_to_pdu(request_ptr).cast::>() }; // SAFETY: By C API contract, `wrapper_ptr` points to a valid allocation // and is not null. unsafe { NonNull::new_unchecked(wrapper_ptr) } @@ -109,7 +109,7 @@ pub(crate) unsafe fn wrapper_ptr(this: *mut Self) -> NonNull /// Return a reference to the [`RequestDataWrapper`] stored in the private /// area of the request structure. - pub(crate) fn wrapper_ref(&self) -> &RequestDataWrapper { + pub(crate) fn wrapper_ref(&self) -> &RequestDataWrapper { // SAFETY: By type invariant, `self.0` is a valid allocation. Further, // the private data associated with this request is initialized and // valid. The existence of `&self` guarantees that the private data is @@ -121,16 +121,19 @@ pub(crate) fn wrapper_ref(&self) -> &RequestDataWrapper { /// A wrapper around data stored in the private area of the C [`struct request`]. /// /// [`struct request`]: srctree/include/linux/blk-mq.h -pub(crate) struct RequestDataWrapper { +pub(crate) struct RequestDataWrapper { /// The Rust request refcount has the following states: /// /// - 0: The request is owned by C block layer. /// - 1: The request is owned by Rust abstractions but there are no [`ARef`] references to it. /// - 2+: There are [`ARef`] references to the request. refcount: Refcount, + + /// Driver managed request data + data: T::RequestData, } -impl RequestDataWrapper { +impl RequestDataWrapper { /// Return a reference to the refcount of the request that is embedding /// `self`. pub(crate) fn refcount(&self) -> &Refcount { @@ -148,6 +151,17 @@ pub(crate) unsafe fn refcount_ptr(this: *mut Self) -> *mut Refcount { // field projection is safe. unsafe { &raw mut (*this).refcount } } + + /// Return a pointer to the `data` field of the `Self` pointed to by `this`. + /// + /// # Safety + /// + /// - `this` must point to a live allocation of at least the size of `Self`. + pub(crate) unsafe fn data_ptr(this: *mut Self) -> *mut T::RequestData { + // SAFETY: Because of the safety requirements of this function, the + // field projection is safe. + unsafe { &raw mut (*this).data } + } } // SAFETY: Exclusive access is thread-safe for `Request`. `Request` has no `&mut diff --git a/rust/kernel/block/mq/tag_set.rs b/rust/kernel/block/mq/tag_set.rs index c3cf56d52beec..46481754b1335 100644 --- a/rust/kernel/block/mq/tag_set.rs +++ b/rust/kernel/block/mq/tag_set.rs @@ -41,7 +41,7 @@ pub fn new( // SAFETY: `blk_mq_tag_set` only contains integers and pointers, which // all are allowed to be 0. let tag_set: bindings::blk_mq_tag_set = unsafe { core::mem::zeroed() }; - let tag_set: Result<_> = core::mem::size_of::() + let tag_set: Result<_> = core::mem::size_of::>() .try_into() .map(|cmd_size| { bindings::blk_mq_tag_set { -- 2.51.2