Add support for polled I/O completion to the Rust block layer bindings. This includes the `poll` callback in `Operations` and the `IoCompletionBatch` type for batched completions. The `poll` callback is invoked by the block layer when polling for completed requests on poll queues. Drivers can use `IoCompletionBatch` to batch multiple completions efficiently. Signed-off-by: Andreas Hindborg --- drivers/block/rnull/rnull.rs | 1 + rust/helpers/blk.c | 7 +++ rust/kernel/block/mq.rs | 4 +- rust/kernel/block/mq/operations.rs | 104 +++++++++++++++++++++++++++++++++++-- rust/kernel/block/mq/request.rs | 5 ++ 5 files changed, 117 insertions(+), 4 deletions(-) diff --git a/drivers/block/rnull/rnull.rs b/drivers/block/rnull/rnull.rs index 48b2bd598304c..765bbc8101d10 100644 --- a/drivers/block/rnull/rnull.rs +++ b/drivers/block/rnull/rnull.rs @@ -733,6 +733,7 @@ fn queue_rq( this: ArcBorrow<'_, Self>, rq: Owned>, _is_last: bool, + _is_poll: bool, ) -> BlkResult { if this.bandwidth_limit != 0 { if !this.bandwidth_timer.active() { diff --git a/rust/helpers/blk.c b/rust/helpers/blk.c index 53beba8c7782d..91b72131a04cd 100644 --- a/rust/helpers/blk.c +++ b/rust/helpers/blk.c @@ -20,3 +20,10 @@ void rust_helper_bio_advance_iter_single(const struct bio *bio, { bio_advance_iter_single(bio, iter, bytes); } + +bool rust_helper_blk_mq_add_to_batch(struct request *req, + struct io_comp_batch *iob, bool is_error, + void (*complete)(struct io_comp_batch *)) +{ + return blk_mq_add_to_batch(req, iob, is_error, complete); +} diff --git a/rust/kernel/block/mq.rs b/rust/kernel/block/mq.rs index cd0bfbcbf317a..1d58eea971bd7 100644 --- a/rust/kernel/block/mq.rs +++ b/rust/kernel/block/mq.rs @@ -89,7 +89,8 @@ //! _hw_data: (), //! _queue_data: (), //! rq: Owned>, -//! _is_last: bool +//! _is_last: bool, +//! is_poll: bool //! ) -> BlkResult { //! rq.start().end_ok(); //! Ok(()) @@ -130,6 +131,7 @@ mod request_queue; pub mod tag_set; +pub use operations::IoCompletionBatch; pub use operations::Operations; pub use request::Command; pub use request::IdleRequest; diff --git a/rust/kernel/block/mq/operations.rs b/rust/kernel/block/mq/operations.rs index 28dd4b28d203f..bccc1903a0d10 100644 --- a/rust/kernel/block/mq/operations.rs +++ b/rust/kernel/block/mq/operations.rs @@ -65,6 +65,7 @@ fn queue_rq( queue_data: ForeignBorrowed<'_, Self::QueueData>, rq: Owned>, is_last: bool, + is_poll: bool, ) -> BlkResult; /// Called by the kernel to indicate that queued requests should be submitted. @@ -84,7 +85,15 @@ fn init_hctx( /// Called by the kernel to poll the device for completed requests. Only /// used for poll queues. - fn poll(_hw_data: ForeignBorrowed<'_, Self::HwData>) -> bool { + /// + /// Should return `Ok(true)` if any requests were completed during the call, + /// `Ok(false)` if no requests were completed, and `Err(e)` to signal an + /// error condition. + fn poll( + _hw_data: ForeignBorrowed<'_, Self::HwData>, + _queue_data: ForeignBorrowed<'_, Self::QueueData>, + _batch: &mut IoCompletionBatch, + ) -> Result { build_error!(crate::error::VTABLE_DEFAULT_ERROR) } @@ -168,6 +177,11 @@ impl OperationsVTable { // `into_foreign` in `Self::init_hctx_callback`. let hw_data = unsafe { T::HwData::borrow((*hctx).driver_data) }; + let is_poll = u32::from( + // SAFETY: `hctx` is valid as required by this function. + unsafe { (*hctx).type_ }, + ) == bindings::hctx_type_HCTX_TYPE_POLL; + // SAFETY: `hctx` is valid as required by this function. let queue_data = unsafe { (*(*hctx).queue).queuedata }; @@ -184,6 +198,7 @@ impl OperationsVTable { // SAFETY: `bd` is valid as required by the safety requirement for // this function. unsafe { (*bd).last }, + is_poll, ); if let Err(e) = ret { @@ -242,13 +257,32 @@ impl OperationsVTable { /// previously initialized by a call to `init_hctx_callback`. unsafe extern "C" fn poll_callback( hctx: *mut bindings::blk_mq_hw_ctx, - _iob: *mut bindings::io_comp_batch, + iob: *mut bindings::io_comp_batch, ) -> crate::ffi::c_int { // SAFETY: By function safety requirement, `hctx` was initialized by // `init_hctx_callback` and thus `driver_data` came from a call to // `into_foreign`. let hw_data = unsafe { T::HwData::borrow((*hctx).driver_data) }; - T::poll(hw_data).into() + + // SAFETY: `hctx` is valid as required by this function. + let queue_data = unsafe { (*(*hctx).queue).queuedata }; + + // SAFETY: `queue.queuedata` was created by `GenDiskBuilder::build` with + // a call to `ForeignOwnable::into_foreign` to create `queuedata`. + // `ForeignOwnable::from_foreign` is only called when the tagset is + // dropped, which happens after we are dropped. + let queue_data = unsafe { T::QueueData::borrow(queue_data) }; + + let mut batch = IoCompletionBatch { + inner: iob, + _p: PhantomData, + }; + + let ret = T::poll(hw_data, queue_data, &mut batch); + match ret { + Ok(val) => val.into(), + Err(e) => e.to_errno(), + } } /// This function is called by the C kernel. A pointer to this function is @@ -448,3 +482,67 @@ pub(crate) const fn build() -> &'static bindings::blk_mq_ops { &Self::VTABLE } } + +/// A batch of I/O completions for polled I/O. +/// +/// This struct wraps the C `struct io_comp_batch` and is used to batch +/// multiple request completions together for improved efficiency during polled +/// I/O operations. +/// +/// When the kernel polls for completed requests via [`Operations::poll`], it +/// passes an `IoCompletionBatch` to collect completed requests. The driver can +/// add completed requests to the batch using [`add_request`], allowing the +/// kernel to process multiple completions together rather than one at a time. +/// +/// # Invariants +/// +/// - `inner` must point to a valid `io_comp_batch`. +/// +/// [`add_request`]: IoCompletionBatch::add_request +#[repr(transparent)] +pub struct IoCompletionBatch { + inner: *mut bindings::io_comp_batch, + _p: PhantomData, +} + +impl IoCompletionBatch { + /// Attempt to add a completed request to this batch. + /// + /// This method tries to add `rq` to the batch for deferred completion. If + /// the request is successfully added, ownership is transferred to the batch + /// and the request will be completed later when the batch is processed. + /// + /// # Arguments + /// + /// - `rq`: The completed request to add to the batch. + /// - `error`: Set to `true` if the request completed with an error. + /// + /// # Return + /// + /// When this method returns `Err`, the caller is responsible for completing + /// the request through other means, such as calling + /// [`Request::complete`](super::Request::complete). + pub fn add_request( + &mut self, + rq: Owned>, + error: bool, + ) -> Result<(), Owned>> { + // SAFETY: By type invariant, `self.inner` is a valid `io_comp_batch`. + let ret = unsafe { + bindings::blk_mq_add_to_batch( + rq.as_raw(), + self.inner, + error, + Some(bindings::blk_mq_end_request_batch), + ) + }; + + match ret { + true => { + core::mem::forget(rq); + Ok(()) + } + false => Err(rq), + } + } +} diff --git a/rust/kernel/block/mq/request.rs b/rust/kernel/block/mq/request.rs index 4e7adb10929f0..b4f5a98a97b16 100644 --- a/rust/kernel/block/mq/request.rs +++ b/rust/kernel/block/mq/request.rs @@ -135,6 +135,11 @@ pub fn queue(&self) -> &RequestQueue { // SAFETY: By type invariant, self.0 is guaranteed to be valid. unsafe { RequestQueue::from_raw((*self.0.get()).q) } } + + /// Return a raw pointer to the underlying C structure. + pub fn as_raw(&self) -> *mut bindings::request { + self.0.get() + } } /// A wrapper around a blk-mq [`struct request`]. This represents an IO request. -- 2.51.2