Per-CPU variables are an important tool for reducing lock contention, especially in systems with many processors. They also provide a convenient way to handle data that are logically associated with a particular CPU (e.g., the currently running task). Therefore, add a Rust API to make use of per-CPU variables. Add a `CpuGuard` type that disables preemption for its lifetime. Add a `PerCpuAllocation` type used to track dynamic allocations. Add a `define_per_cpu!` macro to create static per-CPU allocations. Add `DynamicPerCpu` and `StaticPerCpu` to provide a high-level API. Add a `PerCpu` trait to unify the dynamic and static cases. Co-developed-by: Boqun Feng Signed-off-by: Boqun Feng Signed-off-by: Mitchell Levy --- rust/helpers/helpers.c | 2 + rust/helpers/percpu.c | 9 ++ rust/helpers/preempt.c | 14 +++ rust/kernel/lib.rs | 3 + rust/kernel/percpu.rs | 223 ++++++++++++++++++++++++++++++++++++++++ rust/kernel/percpu/cpu_guard.rs | 35 +++++++ rust/kernel/percpu/dynamic.rs | 83 +++++++++++++++ rust/kernel/percpu/static_.rs | 132 ++++++++++++++++++++++++ 8 files changed, 501 insertions(+) diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 7cf7fe95e41d..2fc8d26cfe66 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -31,9 +31,11 @@ #include "of.c" #include "page.c" #include "pci.c" +#include "percpu.c" #include "pid_namespace.c" #include "platform.c" #include "poll.c" +#include "preempt.c" #include "property.c" #include "rbtree.c" #include "rcu.c" diff --git a/rust/helpers/percpu.c b/rust/helpers/percpu.c new file mode 100644 index 000000000000..a091389f730f --- /dev/null +++ b/rust/helpers/percpu.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +void __percpu *rust_helper_alloc_percpu(size_t sz, size_t align) +{ + return __alloc_percpu(sz, align); +} + diff --git a/rust/helpers/preempt.c b/rust/helpers/preempt.c new file mode 100644 index 000000000000..2c7529528ddd --- /dev/null +++ b/rust/helpers/preempt.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +void rust_helper_preempt_disable(void) +{ + preempt_disable(); +} + +void rust_helper_preempt_enable(void) +{ + preempt_enable(); +} + diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index ed53169e795c..ed0d5756dc55 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -106,6 +106,9 @@ pub mod page; #[cfg(CONFIG_PCI)] pub mod pci; +// Only x86_64 is supported by percpu for now +#[cfg(CONFIG_X86_64)] +pub mod percpu; pub mod pid_namespace; pub mod platform; pub mod prelude; diff --git a/rust/kernel/percpu.rs b/rust/kernel/percpu.rs new file mode 100644 index 000000000000..35afcdba3ccd --- /dev/null +++ b/rust/kernel/percpu.rs @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0 +//! This module contains abstractions for creating and using per-CPU variables from Rust. +//! See the define_per_cpu! macro and the DynamicPerCpu type, as well as the PerCpu trait. + +pub mod cpu_guard; +mod dynamic; +mod static_; + +#[doc(inline)] +pub use dynamic::*; +#[doc(inline)] +pub use static_::*; + +use bindings::{alloc_percpu, free_percpu}; + +use crate::alloc::Flags; +use crate::percpu::cpu_guard::CpuGuard; +use crate::prelude::*; +use crate::sync::Arc; +use crate::types::Opaque; +use crate::{declare_extern_per_cpu, get_static_per_cpu}; + +use core::arch::asm; +use core::cell::{Cell, RefCell, UnsafeCell}; +use core::mem::{align_of, size_of, MaybeUninit}; + +use ffi::c_void; + +/// A per-CPU pointer; that is, an offset into the per-CPU area. Note that this type is NOT a smart +/// pointer, it does not manage the allocation. +pub struct PerCpuPtr(*mut MaybeUninit); + +/// Represents exclusive access to the memory location pointed at by a particular PerCpu. +pub struct PerCpuToken<'a, T> { + // INVARIANT: the current CPU's memory location associated with the per-CPU variable pointed at + // by `ptr` (i.e., the entry in the per-CPU area on the current CPU) has been initialized. + _guard: CpuGuard, + ptr: &'a PerCpuPtr, +} + +/// Represents access to the memory location pointed at by a particular PerCpu where the type +/// `T` manages access to the underlying memory to avoid aliaising troubles. (For example, `T` +/// might be a `Cell` or `RefCell`.) +pub struct CheckedPerCpuToken<'a, T> { + // INVARIANT: the current CPU's memory location associated with the per-CPU variable pointed at + // by `ptr` (i.e., the entry in the per-CPU area on the current CPU) has been initialized. + _guard: CpuGuard, + ptr: &'a PerCpuPtr, +} + +impl PerCpuPtr { + /// Makes a new PerCpuPtr from a raw per-CPU pointer. + /// + /// # Safety + /// `ptr` must be a valid per-CPU pointer. + pub unsafe fn new(ptr: *mut MaybeUninit) -> Self { + Self(ptr) + } + + /// Get a `&mut MaybeUninit` to the per-CPU variable on the current CPU represented by `&self` + /// + /// # Safety + /// The returned `&mut T` must follow Rust's aliasing rules. That is, no other `&(mut) T` may + /// exist that points to the same location in memory. In practice, this means that `get_(mut_)ref` + /// must not be called on another `PerCpuPtr` that is a copy/clone of `&self` for as long as + /// the returned reference lives. + /// + /// CPU preemption must be disabled before calling this function and for the lifetime of the + /// returned reference. Otherwise, the returned reference might end up being a reference to a + /// different CPU's per-CPU area, causing the potential for a data race. + #[allow(clippy::mut_from_ref)] // Safety requirements prevent aliasing issues + pub unsafe fn get_mut_ref(&self) -> &mut MaybeUninit { + // SAFETY: `self.get_ptr()` returns a valid pointer to a `MaybeUninit` by its contract, + // and the safety requirements of this function ensure that the returned reference is + // exclusive. + unsafe { &mut *(self.get_ptr()) } + } + + /// Get a `&MaybeUninit` to the per-CPU variable on the current CPU represented by `&self` + /// + /// # Safety + /// The returned `&T` must follow Rust's aliasing rules. That is, no `&mut T` may exist that + /// points to the same location in memory. In practice, this means that `get_mut_ref` must not + /// be called on another `PerCpuPtr` that is a copy/clone of `&self` for as long as the + /// returned reference lives. + /// + /// CPU preemption must be disabled before calling this function and for the lifetime of the + /// returned reference. Otherwise, the returned reference might end up being a reference to a + /// different CPU's per-CPU area, causing the potential for a data race. + pub unsafe fn get_ref(&self) -> &MaybeUninit { + // SAFETY: `self.get_ptr()` returns a valid pointer to a `MaybeUninit` by its contract. + // The safety requirements of this function ensure that the returned reference isn't + // aliased by a `&mut MaybeUninit`. + unsafe { &*self.get_ptr() } + } + + /// Get a `*mut MaybeUninit` to the per-CPU variable on the current CPU represented by + /// `&self`. Note that if CPU preemption is not disabled before calling this function, use of + /// the returned pointer may cause a data race without some other synchronization mechanism. + /// Buyer beware! + pub fn get_ptr(&self) -> *mut MaybeUninit { + let this_cpu_off_pcpu: PerCpuPtr<*mut c_void> = get_static_per_cpu!(this_cpu_off).0; + let mut this_cpu_area: *mut c_void; + // SAFETY: gs + this_cpu_off_pcpu is guaranteed to be a valid pointer because `gs` points + // to the per-CPU area and this_cpu_off_pcpu is a valid per-CPU allocation. + unsafe { + asm!( + "mov {out}, gs:[{off_val}]", + off_val = in(reg) this_cpu_off_pcpu.0, + out = out(reg) this_cpu_area, + ) + }; + + // This_cpu_area + self.0 is guaranteed to be a valid pointer by the per-CPU subsystem and + // the invariant that self.0 is a valid offset into the per-CPU area. + (this_cpu_area).wrapping_add(self.0 as usize).cast() + } +} + +// SAFETY: Sending a `PerCpuPtr` to another thread is safe because as soon as it's sent, the +// pointer is logically referring to a different place in memory in the other CPU's per-CPU area. +// In particular, this means that there are no restrictions on the type `T`. +unsafe impl Send for PerCpuPtr {} + +// SAFETY: Two threads concurrently making use of a `PerCpuPtr` will each see the `T` in their +// own per-CPU area, so there's no potential for a data race (regardless of whether `T` is itself +// `Sync`). +unsafe impl Sync for PerCpuPtr {} + +impl Clone for PerCpuPtr { + fn clone(&self) -> Self { + *self + } +} + +/// `PerCpuPtr` is just a wrapper around a pointer. +impl Copy for PerCpuPtr {} + +/// A trait representing a per-CPU variable. This is implemented for both `StaticPerCpu` and +/// `DynamicPerCpu`. The main usage of this trait is to call `get_mut` to get a `PerCpuToken` +/// that can be used to access the underlying per-CPU variable. See `PerCpuToken::with`. +pub trait PerCpu { + /// Produces a token, asserting that the holder has exclusive access to the underlying memory + /// pointed to by `self` + /// + /// # Safety + /// No other `PerCpuToken` or `CheckedPerCpuToken` may exist on the current CPU (which is a + /// sensible notion, since we keep a `CpuGuard` around) that is derived from the same + /// `PerCpu` or a clone thereof. + unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T>; +} + +/// A marker trait for types that are interior mutable. Types that implement this trait can be used +/// to create "checked" per-CPU variables. See `CheckedPerCpu`. +pub trait InteriorMutable {} + +impl InteriorMutable for Cell {} +impl InteriorMutable for RefCell {} +impl InteriorMutable for UnsafeCell {} +impl InteriorMutable for Opaque {} + +/// A trait representing a per-CPU variable that is usable via a `&T`. The unsafety of `PerCpu` +/// stems from the fact that the holder of a `PerCpuToken` can use it to get a `&mut T` to the +/// underlying per-CPU variable. This is problematic because the existence of aliaising `&mut T` is +/// undefined behavior in Rust. This type avoids that issue by only allowing access via a `&T`, +/// with the tradeoff that then `T` must be interior mutable or the underlying per-CPU variable +/// must be a constant for the lifetime of the corresponding `CheckedPerCpu`. +/// +/// Currently, only the case where `T` is interior mutable has first-class support, though a custom +/// implementation of `PerCpu`/`CheckedPerCpu` could be created for the const case. +pub trait CheckedPerCpu: PerCpu { + /// Produces a token via which the holder can access the underlying per-CPU variable. + fn get(&mut self, guard: CpuGuard) -> CheckedPerCpuToken<'_, T>; +} + +impl<'a, T> PerCpuToken<'a, T> { + /// # Safety + /// No other `PerCpuToken` or `CheckedPerCpuToken` may exist on the current CPU (which is a + /// sensible notion, since we keep a `CpuGuard` around) that uses the same `PerCpuPtr`. + /// + /// The current CPU's memory location associated with the per-CPU variable pointed at by `ptr` + /// (i.e., the entry in the per-CPU area on this CPU) must be initialized. + pub unsafe fn new(guard: CpuGuard, ptr: &'a PerCpuPtr) -> PerCpuToken<'a, T> { + Self { _guard: guard, ptr } + } + + /// Immediately invokes `func` with a `&mut T` that points at the underlying per-CPU variable + /// that `&mut self` represents. + pub fn with(&mut self, func: U) + where + U: FnOnce(&mut T), + { + // SAFETY: The existence of a PerCpuToken means that the requirements for get_mut_ref are + // satisfied. Likewise, the requirements for assume_init_mut are satisfied because the + // invariants of this type ensure that on the current CPU (which is a sensible notion + // because we have a CpuGuard), the memory location pointed to by `ptr` is initialized. + func(unsafe { self.ptr.get_mut_ref().assume_init_mut() }); + } +} + +impl<'a, T> CheckedPerCpuToken<'a, T> { + /// # Safety + /// The current CPU's memory location associated with the per-CPU variable pointed at by `ptr` + /// (i.e., the entry in the per-CPU area on this CPU) must be initialized. + pub unsafe fn new(guard: CpuGuard, ptr: &'a PerCpuPtr) -> CheckedPerCpuToken<'a, T> { + Self { _guard: guard, ptr } + } + + /// Immediately invokes `func` with a `&T` that points at the underlying per-CPU variable that + /// `&mut self` represents. + pub fn with(&mut self, func: U) + where + U: FnOnce(&T), + { + // SAFETY: The existence of a CheckedPerCpuToken means that the requirements for get_ref + // are satisfied. Likewise, the requirements for assume_init_ref are satisfied because the + // invariants of this type ensure that on the current CPU (which is a sensible notion + // because we have a CpuGuard), the memory location pointed to by `ptr` is initialized. + func(unsafe { self.ptr.get_ref().assume_init_ref() }); + } +} + +declare_extern_per_cpu!(this_cpu_off: *mut c_void); diff --git a/rust/kernel/percpu/cpu_guard.rs b/rust/kernel/percpu/cpu_guard.rs new file mode 100644 index 000000000000..14c04b12e7f0 --- /dev/null +++ b/rust/kernel/percpu/cpu_guard.rs @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 +//! Contains abstractions for disabling CPU preemption. See `CpuGuard`. + +/// A RAII guard for bindings::preempt_disable and bindings::preempt_enable. Guarantees preemption +/// is disabled for as long as this object exists. +pub struct CpuGuard { + // Don't make one without using new() + _phantom: (), +} + +impl CpuGuard { + /// Create a new CpuGuard. Disables preemption for its lifetime. + pub fn new() -> Self { + // SAFETY: There are no preconditions required to call preempt_disable + unsafe { + bindings::preempt_disable(); + } + CpuGuard { _phantom: () } + } +} + +impl Default for CpuGuard { + fn default() -> Self { + Self::new() + } +} + +impl Drop for CpuGuard { + fn drop(&mut self) { + // SAFETY: There are no preconditions required to call preempt_enable + unsafe { + bindings::preempt_enable(); + } + } +} diff --git a/rust/kernel/percpu/dynamic.rs b/rust/kernel/percpu/dynamic.rs new file mode 100644 index 000000000000..ce95e420f943 --- /dev/null +++ b/rust/kernel/percpu/dynamic.rs @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0 +//! Dynamically allocated per-CPU variables. + +use super::*; + +/// Represents a dynamic allocation of a per-CPU variable via alloc_percpu. Calls free_percpu when +/// dropped. +pub struct PerCpuAllocation(PerCpuPtr); + +impl PerCpuAllocation { + /// Dynamically allocates a space in the per-CPU area suitably sized and aligned to hold a `T`, + /// initially filled with the zero value for `T`. + /// + /// Returns `None` under the same circumstances the C function `alloc_percpu` returns `NULL`. + pub fn new_zero() -> Option> { + let ptr: *mut MaybeUninit = + // SAFETY: No preconditions to call alloc_percpu; MaybeUninit is + // `#[repr(transparent)]`, so we can cast a `*mut T` to it. + unsafe { alloc_percpu(size_of::(), align_of::()) }.cast(); + if ptr.is_null() { + return None; + } + + // alloc_percpu returns zero'ed memory + Some(Self(PerCpuPtr(ptr))) + } +} + +impl PerCpuAllocation { + /// Makes a per-CPU allocation sized and aligned to hold a `T`. + /// + /// Returns `None` under the same circumstances the C function `alloc_percpu` returns `NULL`. + pub fn new_uninit() -> Option> { + let ptr: *mut MaybeUninit = + // SAFETY: No preconditions to call alloc_percpu; MaybeUninit is + // `#[repr(transparent)]`, so we can cast a `*mut T` to it. + unsafe { alloc_percpu(size_of::(), align_of::()) }.cast(); + if ptr.is_null() { + return None; + } + + Some(Self(PerCpuPtr(ptr))) + } +} + +impl Drop for PerCpuAllocation { + fn drop(&mut self) { + // SAFETY: self.0.0 was returned by alloc_percpu, and so was a valid pointer into + // the percpu area, and has remained valid by the invariants of PerCpuAllocation. + unsafe { free_percpu(self.0 .0.cast()) } + } +} + +/// Holds a dynamically-allocated per-CPU variable. +#[derive(Clone)] +pub struct DynamicPerCpu { + // INVARIANT: The memory location in each CPU's per-CPU area pointed at by `alloc.0` has been + // initialized. + pub(super) alloc: Arc>, +} + +impl DynamicPerCpu { + /// Allocates a new per-CPU variable + /// + /// # Arguments + /// * `flags` - Flags used to allocate an `Arc` that keeps track of the underlying + /// `PerCpuAllocation`. + pub fn new_zero(flags: Flags) -> Option { + let alloc: PerCpuAllocation = PerCpuAllocation::new_zero()?; + + let arc = Arc::new(alloc, flags).ok()?; + + Some(Self { alloc: arc }) + } +} + +impl PerCpu for DynamicPerCpu { + unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T> { + // SAFETY: The requirements of `PerCpu::get_mut` and this type's invariant ensure that the + // requirements of `PerCpuToken::new` are met. + unsafe { PerCpuToken::new(guard, &self.alloc.0) } + } +} diff --git a/rust/kernel/percpu/static_.rs b/rust/kernel/percpu/static_.rs new file mode 100644 index 000000000000..be226dd2c3aa --- /dev/null +++ b/rust/kernel/percpu/static_.rs @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0 +//! Statically allocated per-CPU variables. + +use super::*; + +/// A wrapper used for declaring static per-CPU variables. These symbols are "virtual" in that the +/// linker uses them to generate offsets into each CPU's per-CPU area, but shouldn't be read +/// from/written to directly. The fact that the statics are immutable prevents them being written +/// to (generally), this struct having _val be non-public prevents reading from them. +/// +/// The end-user of the per-CPU API should make use of the define_per_cpu! macro instead of +/// declaring variables of this type directly. All instances of this type must be `static` and +/// `#[link_section = ".data..percpu"]` (which the macro handles). +#[repr(transparent)] +pub struct StaticPerCpuSymbol { + _val: T, // generate a correctly sized type +} + +/// Holds a statically-allocated per-CPU variable. +#[derive(Clone)] +pub struct StaticPerCpu(pub(super) PerCpuPtr); + +impl StaticPerCpu { + /// Creates a `StaticPerCpu` from a `StaticPerCpuSymbol`. You should probably be using + /// `get_static_per_cpu!` instead. + pub fn new(ptr: *const StaticPerCpuSymbol) -> StaticPerCpu { + // SAFETY: `StaticPerCpuSymbol` is `#[repr(transparent)]`, so we can safely cast a + // pointer to it into a pointer to `MaybeUninit`. The validity of it as a per-CPU + // pointer is guaranteed by the per-CPU subsystem and invariants of the StaticPerCpuSymbol + // type. + let pcpu_ptr = unsafe { PerCpuPtr::new(ptr.cast_mut().cast()) }; + Self(pcpu_ptr) + } +} + +impl PerCpu for StaticPerCpu { + unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T> { + // SAFETY: The requirements of `PerCpu::get_mut` and the fact that statically-allocated + // per-CPU variables are initialized by the per-CPU subsystem ensure that the requirements + // of `PerCpuToken::new` are met. + unsafe { PerCpuToken::new(guard, &self.0) } + } +} + +impl CheckedPerCpu for StaticPerCpu { + fn get(&mut self, guard: CpuGuard) -> CheckedPerCpuToken<'_, T> { + // SAFETY: The per-CPU subsystem guarantees that each CPU's instance of a + // statically allocated variable begins with a copy of the contents of the + // corresponding symbol in `.data..percpu`. Thus, the requirements of + // `CheckedPerCpuToken::new` are met. + unsafe { CheckedPerCpuToken::new(guard, &self.0) } + } +} + +/// Gets a `StaticPerCpu` from a symbol declared with `define_per_cpu!` or +/// `declare_extern_per_cpu!`. +/// +/// # Arguments +/// * `ident` - The identifier declared +#[macro_export] +macro_rules! get_static_per_cpu { + ($id:ident) => { + $crate::percpu::StaticPerCpu::new((&raw const $id).cast()) + }; +} + +/// Declares a StaticPerCpuSymbol corresponding to a per-CPU variable defined in C. Be sure to read +/// the safety requirements of `PerCpu::get`. +#[macro_export] +macro_rules! declare_extern_per_cpu { + ($id:ident: $ty:ty) => { + extern "C" { + static $id: StaticPerCpuSymbol<$ty>; + } + }; +} + +/// define_per_cpu! is analogous to the C DEFINE_PER_CPU macro in that it lets you create a +/// statically allocated per-CPU variable. +/// +/// # Example +/// ``` +/// use kernel::define_per_cpu; +/// use kernel::percpu::StaticPerCpuSymbol; +/// +/// define_per_cpu!(pub MY_PERCPU: u64 = 0); +/// ``` +#[macro_export] +macro_rules! define_per_cpu { + ($vis:vis $id:ident: $ty:ty = $expr:expr) => { + $crate::macros::paste! { + // We might want to have a per-CPU variable that doesn't implement `Sync` (not paying + // sync overhead costs is part of the point), but Rust won't let us declare a static of + // a `!Sync` type. Of course, we don't actually have any synchronization issues, since + // each CPU will see its own copy of the variable, so we cheat a little bit and tell + // Rust it's fine. + #[doc(hidden)] + #[allow(non_camel_case_types)] + #[repr(transparent)] // It needs to be the same size as $ty + struct [<__PRIVATE_TYPE_ $id>]($ty); + + impl [<__PRIVATE_TYPE_ $id>] { + #[doc(hidden)] + const fn new(val: $ty) -> Self { + Self(val) + } + } + + // Expand $expr outside of the unsafe block to avoid silently allowing unsafe code to be + // used without a user-facing unsafe block + #[doc(hidden)] + static [<__INIT_ $id>]: [<__PRIVATE_TYPE_ $id>] = [<__PRIVATE_TYPE_ $id>]::new($expr); + + // SAFETY: This type will ONLY ever be used to declare a `StaticPerCpuSymbol` + // (which we then only ever use as input to `&raw`). Reading from the symbol is + // already UB, so we won't ever actually have any variables of this type where + // synchronization is a concern. + #[doc(hidden)] + unsafe impl Sync for [<__PRIVATE_TYPE_ $id>] {} + + // SAFETY: StaticPerCpuSymbol is #[repr(transparent)], so we can freely convert from + // [<__PRIVATE_TYPE_ $id>], which is also `#[repr(transparent)]` (i.e., everything is + // just a `$ty` from a memory layout perspective). + #[link_section = ".data..percpu"] + $vis static $id: StaticPerCpuSymbol<[<__PRIVATE_TYPE_ $id>]> = unsafe { + core::mem::transmute_copy::< + [<__PRIVATE_TYPE_ $id>], StaticPerCpuSymbol<[<__PRIVATE_TYPE_ $id>]> + >(&[<__INIT_ $id>]) + }; + } + }; +} -- 2.34.1 Add a short exercise for Rust's per-CPU variable API, modelled after lib/percpu_test.c Signed-off-by: Mitchell Levy --- rust/helpers/percpu.c | 6 ++ samples/rust/Kconfig | 9 +++ samples/rust/Makefile | 1 + samples/rust/rust_percpu.rs | 163 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 179 insertions(+) diff --git a/rust/helpers/percpu.c b/rust/helpers/percpu.c index a091389f730f..8cc01d094752 100644 --- a/rust/helpers/percpu.c +++ b/rust/helpers/percpu.c @@ -1,9 +1,15 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include void __percpu *rust_helper_alloc_percpu(size_t sz, size_t align) { return __alloc_percpu(sz, align); } +void rust_helper_on_each_cpu(smp_call_func_t func, void *info, int wait) +{ + on_each_cpu(func, info, wait); +} + diff --git a/samples/rust/Kconfig b/samples/rust/Kconfig index 7f7371a004ee..23e35d64ac78 100644 --- a/samples/rust/Kconfig +++ b/samples/rust/Kconfig @@ -105,6 +105,15 @@ config SAMPLE_RUST_DRIVER_AUXILIARY If unsure, say N. +config SAMPLE_RUST_PERCPU + tristate "Per-CPU support" + depends on m + help + Enable this option to build a module which demonstrates Rust per-CPU + operations. + + If unsure, say N. + config SAMPLE_RUST_HOSTPROGS bool "Host programs" help diff --git a/samples/rust/Makefile b/samples/rust/Makefile index bd2faad63b4f..8a34d9d74754 100644 --- a/samples/rust/Makefile +++ b/samples/rust/Makefile @@ -10,6 +10,7 @@ obj-$(CONFIG_SAMPLE_RUST_DRIVER_PLATFORM) += rust_driver_platform.o obj-$(CONFIG_SAMPLE_RUST_DRIVER_FAUX) += rust_driver_faux.o obj-$(CONFIG_SAMPLE_RUST_DRIVER_AUXILIARY) += rust_driver_auxiliary.o obj-$(CONFIG_SAMPLE_RUST_CONFIGFS) += rust_configfs.o +obj-$(CONFIG_SAMPLE_RUST_PERCPU) += rust_percpu.o rust_print-y := rust_print_main.o rust_print_events.o diff --git a/samples/rust/rust_percpu.rs b/samples/rust/rust_percpu.rs new file mode 100644 index 000000000000..98ca1c781b6b --- /dev/null +++ b/samples/rust/rust_percpu.rs @@ -0,0 +1,163 @@ +// SPDX-License-Identifier: GPL-2.0 +//! A simple demonstration of the rust per-CPU API. + +use core::cell::RefCell; +use core::ffi::c_void; + +use kernel::{ + bindings::on_each_cpu, + cpu::CpuId, + define_per_cpu, get_static_per_cpu, + percpu::{cpu_guard::*, *}, + pr_info, + prelude::*, +}; + +module! { + type: PerCpuMod, + name: "rust_percpu", + authors: ["Mitchell Levy"], + description: "Sample to demonstrate the Rust per-CPU API", + license: "GPL v2", +} + +struct PerCpuMod; + +define_per_cpu!(PERCPU: i64 = 0); +define_per_cpu!(UPERCPU: u64 = 0); +define_per_cpu!(CHECKED: RefCell = RefCell::new(0)); + +impl kernel::Module for PerCpuMod { + fn init(_module: &'static ThisModule) -> Result { + pr_info!("rust percpu test start\n"); + + let mut native: i64 = 0; + let mut pcpu: StaticPerCpu = get_static_per_cpu!(PERCPU); + + // SAFETY: We only have one PerCpu that points at PERCPU + unsafe { pcpu.get_mut(CpuGuard::new()) }.with(|val: &mut i64| { + pr_info!("The contents of pcpu are {}\n", val); + + native += -1; + *val += -1; + pr_info!("Native: {}, *pcpu: {}\n", native, val); + assert!(native == *val && native == -1); + + native += 1; + *val += 1; + pr_info!("Native: {}, *pcpu: {}\n", native, val); + assert!(native == *val && native == 0); + }); + + let mut unative: u64 = 0; + let mut upcpu: StaticPerCpu = get_static_per_cpu!(UPERCPU); + + // SAFETY: We only have one PerCpu pointing at UPERCPU + unsafe { upcpu.get_mut(CpuGuard::new()) }.with(|val: &mut u64| { + unative += 1; + *val += 1; + pr_info!("Unative: {}, *upcpu: {}\n", unative, val); + assert!(unative == *val && unative == 1); + + unative = unative.wrapping_add((-1i64) as u64); + *val = val.wrapping_add((-1i64) as u64); + pr_info!("Unative: {}, *upcpu: {}\n", unative, val); + assert!(unative == *val && unative == 0); + + unative = unative.wrapping_add((-1i64) as u64); + *val = val.wrapping_add((-1i64) as u64); + pr_info!("Unative: {}, *upcpu: {}\n", unative, val); + assert!(unative == *val && unative == (-1i64) as u64); + + unative = 0; + *val = 0; + + unative = unative.wrapping_sub(1); + *val = val.wrapping_sub(1); + pr_info!("Unative: {}, *upcpu: {}\n", unative, val); + assert!(unative == *val && unative == (-1i64) as u64); + assert!(unative == *val && unative == u64::MAX); + }); + + let mut checked_native: u64 = 0; + let mut checked: StaticPerCpu> = get_static_per_cpu!(CHECKED); + checked.get(CpuGuard::new()).with(|val: &RefCell| { + checked_native += 1; + *val.borrow_mut() += 1; + pr_info!( + "Checked native: {}, *checked: {}\n", + checked_native, + val.borrow() + ); + assert!(checked_native == *val.borrow() && checked_native == 1); + + checked_native = checked_native.wrapping_add((-1i64) as u64); + val.replace_with(|old: &mut u64| old.wrapping_add((-1i64) as u64)); + pr_info!( + "Checked native: {}, *checked: {}\n", + checked_native, + val.borrow() + ); + assert!(checked_native == *val.borrow() && checked_native == 0); + + checked_native = checked_native.wrapping_add((-1i64) as u64); + val.replace_with(|old: &mut u64| old.wrapping_add((-1i64) as u64)); + pr_info!( + "Checked native: {}, *checked: {}\n", + checked_native, + val.borrow() + ); + assert!(checked_native == *val.borrow() && checked_native == (-1i64) as u64); + + checked_native = 0; + *val.borrow_mut() = 0; + + checked_native = checked_native.wrapping_sub(1); + val.replace_with(|old: &mut u64| old.wrapping_sub(1)); + pr_info!( + "Checked native: {}, *checked: {}\n", + checked_native, + val.borrow() + ); + assert!(checked_native == *val.borrow() && checked_native == (-1i64) as u64); + assert!(checked_native == *val.borrow() && checked_native == u64::MAX); + }); + + pr_info!("rust static percpu test done\n"); + + pr_info!("rust dynamic percpu test start\n"); + let mut test: DynamicPerCpu = DynamicPerCpu::new_zero(GFP_KERNEL).unwrap(); + + // SAFETY: No prerequisites for on_each_cpu. + unsafe { + on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 0); + on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 0); + on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 0); + on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 1); + on_each_cpu(Some(check_percpu), (&raw mut test).cast(), 1); + } + + pr_info!("rust dynamic percpu test done\n"); + + // Return Err to unload the module + Result::Err(EINVAL) + } +} + +extern "C" fn inc_percpu(info: *mut c_void) { + // SAFETY: We know that info is a void *const DynamicPerCpu and DynamicPerCpu is Send. + let mut pcpu = unsafe { (*(info as *const DynamicPerCpu)).clone() }; + pr_info!("Incrementing on {}\n", CpuId::current().as_u32()); + + // SAFETY: We don't have multiple clones of pcpu in scope + unsafe { pcpu.get_mut(CpuGuard::new()) }.with(|val: &mut u64| *val += 1); +} + +extern "C" fn check_percpu(info: *mut c_void) { + // SAFETY: We know that info is a void *const DynamicPerCpu and DynamicPerCpu is Send. + let mut pcpu = unsafe { (*(info as *const DynamicPerCpu)).clone() }; + pr_info!("Asserting on {}\n", CpuId::current().as_u32()); + + // SAFETY: We don't have multiple clones of pcpu in scope + unsafe { pcpu.get_mut(CpuGuard::new()) }.with(|val: &mut u64| assert!(*val == 4)); +} -- 2.34.1 Add an iterator for `Cpumask` making use of C's `cpumask_next`. Signed-off-by: Mitchell Levy --- rust/helpers/cpumask.c | 5 +++++ rust/kernel/cpumask.rs | 48 +++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/rust/helpers/cpumask.c b/rust/helpers/cpumask.c index eb10598a0242..d95bfa111191 100644 --- a/rust/helpers/cpumask.c +++ b/rust/helpers/cpumask.c @@ -42,6 +42,11 @@ bool rust_helper_cpumask_full(struct cpumask *srcp) return cpumask_full(srcp); } +unsigned int rust_helper_cpumask_next(int n, struct cpumask *srcp) +{ + return cpumask_next(n, srcp); +} + unsigned int rust_helper_cpumask_weight(struct cpumask *srcp) { return cpumask_weight(srcp); diff --git a/rust/kernel/cpumask.rs b/rust/kernel/cpumask.rs index 3fcbff438670..b7401848f59e 100644 --- a/rust/kernel/cpumask.rs +++ b/rust/kernel/cpumask.rs @@ -6,7 +6,7 @@ use crate::{ alloc::{AllocError, Flags}, - cpu::CpuId, + cpu::{self, CpuId}, prelude::*, types::Opaque, }; @@ -161,6 +161,52 @@ pub fn copy(&self, dstp: &mut Self) { } } +/// Iterator for a `Cpumask`. +pub struct CpumaskIter<'a> { + mask: &'a Cpumask, + last: Option, +} + +impl<'a> CpumaskIter<'a> { + /// Creates a new `CpumaskIter` for the given `Cpumask`. + fn new(mask: &'a Cpumask) -> CpumaskIter<'a> { + Self { mask, last: None } + } +} + +impl<'a> Iterator for CpumaskIter<'a> { + type Item = CpuId; + + fn next(&mut self) -> Option { + // SAFETY: By the type invariant, `self.mask.as_raw` is a `struct cpumask *`. + let next = unsafe { + bindings::cpumask_next( + if let Some(last) = self.last { + last.try_into().unwrap() + } else { + -1 + }, + self.mask.as_raw(), + ) + }; + + if next == cpu::nr_cpu_ids() { + None + } else { + self.last = Some(next); + // SAFETY: `cpumask_next` returns either `nr_cpu_ids` or a valid CPU ID. + unsafe { Some(CpuId::from_u32_unchecked(next)) } + } + } +} + +impl Cpumask { + /// Returns an iterator over the set bits in the cpumask. + pub fn iter(&self) -> CpumaskIter<'_> { + CpumaskIter::new(self) + } +} + /// A CPU Mask pointer. /// /// Rust abstraction for the C `struct cpumask_var_t`. -- 2.34.1 Add getters for the global cpumasks documented in `include/linux/cpumask.h`, specifically: - cpu_possible_mask - cpu_online_mask - cpu_enabled_mask - cpu_present_mask - cpu_active_mask Signed-off-by: Mitchell Levy --- rust/kernel/cpumask.rs | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/rust/kernel/cpumask.rs b/rust/kernel/cpumask.rs index b7401848f59e..ca9aa6875b4a 100644 --- a/rust/kernel/cpumask.rs +++ b/rust/kernel/cpumask.rs @@ -77,6 +77,52 @@ pub unsafe fn as_ref<'a>(ptr: *const bindings::cpumask) -> &'a Self { unsafe { &*ptr.cast() } } + /// Get a CPU mask representing possible CPUs; has bit `cpu` set iff cpu is populatable + #[inline] + pub fn possible() -> &'static Self { + // SAFETY: `__cpu_possible_mask` is a valid global provided by the kernel that lives + // forever. + unsafe { Cpumask::as_ref(&raw const bindings::__cpu_possible_mask) } + } + + /// Get a CPU mask representing online CPUs; has bit `cpu` set iff cpu available to the + /// scheduler + #[inline] + pub fn online() -> &'static Self { + // SAFETY: `__cpu_online_mask` is a valid global provided by the kernel that lives forever. + // Since we wrap the returned pointer in an `Opaque`, it's ok that `__cpu_online_mask` + // may change its value. + unsafe { Cpumask::as_ref(&raw const bindings::__cpu_online_mask) } + } + + /// Get a CPU mask representing enabled CPUs; has bit `cpu` set iff cpu can be brought online + #[inline] + pub fn enabled() -> &'static Self { + // SAFETY: `__cpu_enabled_mask` is a valid global provided by the kernel that lives forever. + // Since we wrap the returned pointer in an `Opaque`, it's ok that `__cpu_enabled_mask` + // may change its value. + unsafe { Cpumask::as_ref(&raw const bindings::__cpu_enabled_mask) } + } + + /// Get a CPU mask representing present CPUs; has bit `cpu` set iff cpu is populated + #[inline] + pub fn present() -> &'static Self { + // SAFETY: `__cpu_present_mask` is a valid global provided by the kernel that lives + // forever. Since we wrap the returned pointer in an `Opaque`, it's ok that + // `__cpu_present_mask` may change its value. + unsafe { Cpumask::as_ref(&raw const bindings::__cpu_present_mask) } + } + + /// Get a CPU mask representing active CPUs; has bit `cpu` set iff cpu is available to + /// migration. + #[inline] + pub fn active() -> &'static Self { + // SAFETY: `__cpu_active_mask` is a valid global provided by the kernel that lives forever. + // Since we wrap the returned pointer in an `Opaque`, it's ok that `__cpu_active_mask` + // may change its value. + unsafe { Cpumask::as_ref(&raw const bindings::__cpu_active_mask) } + } + /// Obtain the raw `struct cpumask` pointer. pub fn as_raw(&self) -> *mut bindings::cpumask { let this: *const Self = self; -- 2.34.1 Add functionality to `PerCpuPtr` to compute pointers to per-CPU variable slots on other CPUs. Use this facility to initialize per-CPU variables on all possible CPUs when a dynamic per-CPU variable is created with a non-zeroable type. Since `RefCell` and other `Cell`-like types fall into this category, `impl CheckedPerCpu` on `DynamicPerCpu` for these `InteriorMutable` types since they can now be used. Add examples of these usages to `samples/rust/rust_percpu.rs`. Signed-off-by: Mitchell Levy --- rust/helpers/percpu.c | 5 +++ rust/kernel/percpu.rs | 15 +++++++ rust/kernel/percpu/dynamic.rs | 40 +++++++++++++++++ samples/rust/rust_percpu.rs | 99 ++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 152 insertions(+), 7 deletions(-) diff --git a/rust/helpers/percpu.c b/rust/helpers/percpu.c index 8cc01d094752..8d83b6b86106 100644 --- a/rust/helpers/percpu.c +++ b/rust/helpers/percpu.c @@ -8,6 +8,11 @@ void __percpu *rust_helper_alloc_percpu(size_t sz, size_t align) return __alloc_percpu(sz, align); } +void *rust_helper_per_cpu_ptr(void __percpu *ptr, unsigned int cpu) +{ + return per_cpu_ptr(ptr, cpu); +} + void rust_helper_on_each_cpu(smp_call_func_t func, void *info, int wait) { on_each_cpu(func, info, wait); diff --git a/rust/kernel/percpu.rs b/rust/kernel/percpu.rs index 35afcdba3ccd..c68c7520b67f 100644 --- a/rust/kernel/percpu.rs +++ b/rust/kernel/percpu.rs @@ -14,6 +14,7 @@ use bindings::{alloc_percpu, free_percpu}; use crate::alloc::Flags; +use crate::cpu::CpuId; use crate::percpu::cpu_guard::CpuGuard; use crate::prelude::*; use crate::sync::Arc; @@ -115,6 +116,20 @@ pub fn get_ptr(&self) -> *mut MaybeUninit { // the invariant that self.0 is a valid offset into the per-CPU area. (this_cpu_area).wrapping_add(self.0 as usize).cast() } + + /// Get a `*mut MaybeUninit` to the per-CPU variable on the CPU represented by `cpu`. Note + /// that without some kind of synchronization, use of the returned pointer may cause a data + /// race. It is the caller's responsibility to use the returned pointer in a reasonable way. + /// + /// # Safety + /// - The returned pointer is valid only if `self` is (that is, it points to a live allocation + /// correctly sized and aligned to hold a `T`) + /// - The returned pointer is valid only if the bit corresponding to `cpu` is set in + /// `Cpumask::possible()`. + pub unsafe fn get_remote_ptr(&self, cpu: CpuId) -> *mut MaybeUninit { + // SAFETY: The requirements of this function ensure this call is safe. + unsafe { bindings::per_cpu_ptr(self.0.cast(), cpu.as_u32()) }.cast() + } } // SAFETY: Sending a `PerCpuPtr` to another thread is safe because as soon as it's sent, the diff --git a/rust/kernel/percpu/dynamic.rs b/rust/kernel/percpu/dynamic.rs index ce95e420f943..64f04cef3705 100644 --- a/rust/kernel/percpu/dynamic.rs +++ b/rust/kernel/percpu/dynamic.rs @@ -3,6 +3,8 @@ use super::*; +use crate::cpumask::Cpumask; + /// Represents a dynamic allocation of a per-CPU variable via alloc_percpu. Calls free_percpu when /// dropped. pub struct PerCpuAllocation(PerCpuPtr); @@ -74,6 +76,36 @@ pub fn new_zero(flags: Flags) -> Option { } } +impl DynamicPerCpu { + /// Allocates a new per-CPU variable + /// + /// # Arguments + /// * `val` - The initial value of the per-CPU variable on all CPUs. + /// * `flags` - Flags used to allocate an `Arc` that keeps track of the underlying + /// `PerCpuAllocation`. + pub fn new_with(val: T, flags: Flags) -> Option { + let alloc: PerCpuAllocation = PerCpuAllocation::new_uninit()?; + let ptr = alloc.0; + + for cpu in Cpumask::possible().iter() { + // SAFETY: `ptr` is a valid allocation, and `cpu` appears in `Cpumask::possible()` + let remote_ptr = unsafe { ptr.get_remote_ptr(cpu) }; + // SAFETY: Each CPU's slot corresponding to `ptr` is currently uninitialized, and no + // one else has a reference to it. Therefore, we can freely write to it without + // worrying about the need to drop what was there or whether we're racing with someone + // else. `PerCpuPtr::get_remote_ptr` guarantees that the pointer is valid since we + // derived it from a valid allocation and `cpu`. + unsafe { + (*remote_ptr).write(val.clone()); + } + } + + let arc = Arc::new(alloc, flags).ok()?; + + Some(Self { alloc: arc }) + } +} + impl PerCpu for DynamicPerCpu { unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T> { // SAFETY: The requirements of `PerCpu::get_mut` and this type's invariant ensure that the @@ -81,3 +113,11 @@ unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T> { unsafe { PerCpuToken::new(guard, &self.alloc.0) } } } + +impl CheckedPerCpu for DynamicPerCpu { + fn get(&mut self, guard: CpuGuard) -> CheckedPerCpuToken<'_, T> { + // SAFETY: By the invariant of `DynamicPerCpu`, the memory location in each CPU's + // per-CPU area corresponding to this variable has been initialized. + unsafe { CheckedPerCpuToken::new(guard, &self.alloc.0) } + } +} diff --git a/samples/rust/rust_percpu.rs b/samples/rust/rust_percpu.rs index 98ca1c781b6b..06b322019134 100644 --- a/samples/rust/rust_percpu.rs +++ b/samples/rust/rust_percpu.rs @@ -130,13 +130,72 @@ fn init(_module: &'static ThisModule) -> Result { // SAFETY: No prerequisites for on_each_cpu. unsafe { - on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 0); - on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 0); - on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 0); - on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 1); - on_each_cpu(Some(check_percpu), (&raw mut test).cast(), 1); + on_each_cpu(Some(inc_percpu_u64), (&raw mut test).cast(), 0); + on_each_cpu(Some(inc_percpu_u64), (&raw mut test).cast(), 0); + on_each_cpu(Some(inc_percpu_u64), (&raw mut test).cast(), 0); + on_each_cpu(Some(inc_percpu_u64), (&raw mut test).cast(), 1); + on_each_cpu(Some(check_percpu_u64), (&raw mut test).cast(), 1); } + let mut checked: DynamicPerCpu> = + DynamicPerCpu::new_with(RefCell::new(100), GFP_KERNEL).unwrap(); + + // SAFETY: No prerequisites for on_each_cpu. + unsafe { + on_each_cpu(Some(inc_percpu_refcell_u64), (&raw mut checked).cast(), 0); + on_each_cpu(Some(inc_percpu_refcell_u64), (&raw mut checked).cast(), 0); + on_each_cpu(Some(inc_percpu_refcell_u64), (&raw mut checked).cast(), 0); + on_each_cpu(Some(inc_percpu_refcell_u64), (&raw mut checked).cast(), 1); + on_each_cpu(Some(check_percpu_refcell_u64), (&raw mut checked).cast(), 1); + } + + checked.get(CpuGuard::new()).with(|val: &RefCell| { + assert!(*val.borrow() == 104); + + let mut checked_native = 0; + *val.borrow_mut() = 0; + + checked_native += 1; + *val.borrow_mut() += 1; + pr_info!( + "Checked native: {}, *checked: {}\n", + checked_native, + val.borrow() + ); + assert!(checked_native == *val.borrow() && checked_native == 1); + + checked_native = checked_native.wrapping_add((-1i64) as u64); + val.replace_with(|old: &mut u64| old.wrapping_add((-1i64) as u64)); + pr_info!( + "Checked native: {}, *checked: {}\n", + checked_native, + val.borrow() + ); + assert!(checked_native == *val.borrow() && checked_native == 0); + + checked_native = checked_native.wrapping_add((-1i64) as u64); + val.replace_with(|old: &mut u64| old.wrapping_add((-1i64) as u64)); + pr_info!( + "Checked native: {}, *checked: {}\n", + checked_native, + val.borrow() + ); + assert!(checked_native == *val.borrow() && checked_native == (-1i64) as u64); + + checked_native = 0; + *val.borrow_mut() = 0; + + checked_native = checked_native.wrapping_sub(1); + val.replace_with(|old: &mut u64| old.wrapping_sub(1)); + pr_info!( + "Checked native: {}, *checked: {}\n", + checked_native, + val.borrow() + ); + assert!(checked_native == *val.borrow() && checked_native == (-1i64) as u64); + assert!(checked_native == *val.borrow() && checked_native == u64::MAX); + }); + pr_info!("rust dynamic percpu test done\n"); // Return Err to unload the module @@ -144,7 +203,7 @@ fn init(_module: &'static ThisModule) -> Result { } } -extern "C" fn inc_percpu(info: *mut c_void) { +extern "C" fn inc_percpu_u64(info: *mut c_void) { // SAFETY: We know that info is a void *const DynamicPerCpu and DynamicPerCpu is Send. let mut pcpu = unsafe { (*(info as *const DynamicPerCpu)).clone() }; pr_info!("Incrementing on {}\n", CpuId::current().as_u32()); @@ -153,7 +212,7 @@ extern "C" fn inc_percpu(info: *mut c_void) { unsafe { pcpu.get_mut(CpuGuard::new()) }.with(|val: &mut u64| *val += 1); } -extern "C" fn check_percpu(info: *mut c_void) { +extern "C" fn check_percpu_u64(info: *mut c_void) { // SAFETY: We know that info is a void *const DynamicPerCpu and DynamicPerCpu is Send. let mut pcpu = unsafe { (*(info as *const DynamicPerCpu)).clone() }; pr_info!("Asserting on {}\n", CpuId::current().as_u32()); @@ -161,3 +220,29 @@ extern "C" fn check_percpu(info: *mut c_void) { // SAFETY: We don't have multiple clones of pcpu in scope unsafe { pcpu.get_mut(CpuGuard::new()) }.with(|val: &mut u64| assert!(*val == 4)); } + +extern "C" fn inc_percpu_refcell_u64(info: *mut c_void) { + // SAFETY: We know that info is a void *const DynamicPerCpu> and + // DynamicPerCpu> is Send. + let mut pcpu = unsafe { (*(info as *const DynamicPerCpu>)).clone() }; + // SAFETY: smp_processor_id has no preconditions + pr_info!("Incrementing on {}\n", CpuId::current().as_u32()); + + pcpu.get(CpuGuard::new()).with(|val: &RefCell| { + let mut val = val.borrow_mut(); + *val += 1; + }); +} + +extern "C" fn check_percpu_refcell_u64(info: *mut c_void) { + // SAFETY: We know that info is a void *const DynamicPerCpu> and + // DynamicPerCpu> is Send. + let mut pcpu = unsafe { (*(info as *const DynamicPerCpu>)).clone() }; + // SAFETY: smp_processor_id has no preconditions + pr_info!("Asserting on {}\n", CpuId::current().as_u32()); + + pcpu.get(CpuGuard::new()).with(|val: &RefCell| { + let val = val.borrow(); + assert!(*val == 104); + }); +} -- 2.34.1 The C implementations of `this_cpu_add`, `this_cpu_sub`, etc., are optimized to save an instruction by avoiding having to compute `this_cpu_ptr(&x)` for some per-CPU variable `x`. For example, rather than u64 *x_ptr = this_cpu_ptr(&x); *x_ptr += 5; the implementation of `this_cpu_add` is clever enough to make use of the fact that per-CPU variables are implemented on x86 via segment registers, and so we can use only a single instruction (where we assume `&x` is already in `rax`) add gs:[rax], 5 Add this optimization via a `PerCpuNumeric` type to enable code-reuse between `DynamicPerCpu` and `StaticPerCpu`. Signed-off-by: Mitchell Levy --- rust/kernel/percpu.rs | 1 + rust/kernel/percpu/dynamic.rs | 2 +- rust/kernel/percpu/numeric.rs | 128 ++++++++++++++++++++++++++++++++++++++++++ samples/rust/rust_percpu.rs | 36 ++++++++++++ 4 files changed, 166 insertions(+), 1 deletion(-) diff --git a/rust/kernel/percpu.rs b/rust/kernel/percpu.rs index c68c7520b67f..c693d16518d2 100644 --- a/rust/kernel/percpu.rs +++ b/rust/kernel/percpu.rs @@ -4,6 +4,7 @@ pub mod cpu_guard; mod dynamic; +pub mod numeric; mod static_; #[doc(inline)] diff --git a/rust/kernel/percpu/dynamic.rs b/rust/kernel/percpu/dynamic.rs index 64f04cef3705..aad08e4b4251 100644 --- a/rust/kernel/percpu/dynamic.rs +++ b/rust/kernel/percpu/dynamic.rs @@ -7,7 +7,7 @@ /// Represents a dynamic allocation of a per-CPU variable via alloc_percpu. Calls free_percpu when /// dropped. -pub struct PerCpuAllocation(PerCpuPtr); +pub struct PerCpuAllocation(pub(super) PerCpuPtr); impl PerCpuAllocation { /// Dynamically allocates a space in the per-CPU area suitably sized and aligned to hold a `T`, diff --git a/rust/kernel/percpu/numeric.rs b/rust/kernel/percpu/numeric.rs new file mode 100644 index 000000000000..4de93f653f0e --- /dev/null +++ b/rust/kernel/percpu/numeric.rs @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: GPL-2.0 +//! Pin-hole optimizations for PerCpu where T is a numeric type. + +use super::*; +use core::arch::asm; + +/// Represents a per-CPU variable that can be manipulated with machine-intrinsic numeric +/// operations. +pub struct PerCpuNumeric<'a, T> { + // INVARIANT: `ptr.0` is a valid offset into the per-CPU area and is initialized on all CPUs + // (since we don't have a CPU guard, we have to be pessimistic and assume we could be on any + // CPU). + ptr: &'a PerCpuPtr, +} + +macro_rules! impl_ops { + ($ty:ty, $reg:tt) => { + impl DynamicPerCpu<$ty> { + /// Returns a `PerCpuNumeric` that can be used to manipulate the underlying per-CPU variable. + pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> { + // The invariant is satisfied because `DynamicPerCpu`'s invariant guarantees that + // this pointer is valid and initialized on all CPUs. + PerCpuNumeric { ptr: &self.alloc.0 } + } + } + impl StaticPerCpu<$ty> { + /// Returns a `PerCpuNumeric` that can be used to manipulate the underlying per-CPU variable. + pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> { + // The invariant is satisfied because `StaticPerCpu`'s invariant guarantees that + // this pointer is valid and initialized on all CPUs. + PerCpuNumeric { ptr: &self.0 } + } + } + + impl PerCpuNumeric<'_, $ty> { + /// Adds `rhs` to the per-CPU variable. + pub fn add(&mut self, rhs: $ty) { + // SAFETY: `self.ptr.0` is a valid offset into the per-CPU area (i.e., valid as a + // pointer relative to the `gs` segment register) by the invariants of this type. + unsafe { + asm!( + concat!("add gs:[{off}], {val:", $reg, "}"), + off = in(reg) self.ptr.0.cast::<*mut $ty>(), + val = in(reg) rhs, + ); + } + } + } + impl PerCpuNumeric<'_, $ty> { + /// Subtracts `rhs` from the per-CPU variable. + pub fn sub(&mut self, rhs: $ty) { + // SAFETY: `self.ptr.0` is a valid offset into the per-CPU area (i.e., valid as a + // pointer relative to the `gs` segment register) by the invariants of this type. + unsafe { + asm!( + concat!("sub gs:[{off}], {val:", $reg, "}"), + off = in(reg) self.ptr.0.cast::<*mut $ty>(), + val = in(reg) rhs, + ); + } + } + } + }; +} + +macro_rules! impl_ops_byte { + ($ty:ty) => { + impl DynamicPerCpu<$ty> { + /// Returns a `PerCpuNumeric` that can be used to manipulate the underlying per-CPU + /// variable. + pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> { + // The invariant is satisfied because `DynamicPerCpu`'s invariant guarantees that + // this pointer is valid and initialized on all CPUs. + PerCpuNumeric { ptr: &self.alloc.0 } + } + } + impl StaticPerCpu<$ty> { + /// Returns a `PerCpuNumeric` that can be used to manipulate the underlying per-CPU + /// variable. + pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> { + // The invariant is satisfied because `DynamicPerCpu`'s invariant guarantees that + // this pointer is valid and initialized on all CPUs. + PerCpuNumeric { ptr: &self.0 } + } + } + + impl PerCpuNumeric<'_, $ty> { + /// Adds `rhs` to the per-CPU variable. + pub fn add(&mut self, rhs: $ty) { + // SAFETY: `self.ptr.0` is a valid offset into the per-CPU area (i.e., valid as a + // pointer relative to the `gs` segment register) by the invariants of this type. + unsafe { + asm!( + concat!("add gs:[{off}], {val}"), + off = in(reg) self.ptr.0.cast::<*mut $ty>(), + val = in(reg_byte) rhs, + ); + } + } + } + impl PerCpuNumeric<'_, $ty> { + /// Subtracts `rhs` from the per-CPU variable. + pub fn sub(&mut self, rhs: $ty) { + // SAFETY: `self.ptr.0` is a valid offset into the per-CPU area (i.e., valid as a + // pointer relative to the `gs` segment register) by the invariants of this type. + unsafe { + asm!( + concat!("sub gs:[{off}], {val}"), + off = in(reg) self.ptr.0.cast::<*mut $ty>(), + val = in(reg_byte) rhs, + ); + } + } + } + }; +} + +impl_ops_byte!(i8); +impl_ops!(i16, "x"); +impl_ops!(i32, "e"); +impl_ops!(i64, "r"); +impl_ops!(isize, "r"); + +impl_ops_byte!(u8); +impl_ops!(u16, "x"); +impl_ops!(u32, "e"); +impl_ops!(u64, "r"); +impl_ops!(usize, "r"); diff --git a/samples/rust/rust_percpu.rs b/samples/rust/rust_percpu.rs index 06b322019134..e3a46a053b8d 100644 --- a/samples/rust/rust_percpu.rs +++ b/samples/rust/rust_percpu.rs @@ -27,6 +27,26 @@ define_per_cpu!(UPERCPU: u64 = 0); define_per_cpu!(CHECKED: RefCell = RefCell::new(0)); +macro_rules! make_optimization_test { + ($ty:ty) => { + let mut test: DynamicPerCpu<$ty> = DynamicPerCpu::new_zero(GFP_KERNEL).unwrap(); + { + let _ = CpuGuard::new(); + // SAFETY: No other usage of `test` + unsafe { test.get_mut(CpuGuard::new()) }.with(|val: &mut $ty| *val = 10); + test.num().add(1); + // SAFETY: No other usage of `test` + unsafe { test.get_mut(CpuGuard::new()) }.with(|val: &mut $ty| assert_eq!(*val, 11)); + test.num().add(10); + // SAFETY: No other usage of `test` + unsafe { test.get_mut(CpuGuard::new()) }.with(|val: &mut $ty| assert_eq!(*val, 21)); + test.num().sub(5); + // SAFETY: No other usage of `test` + unsafe { test.get_mut(CpuGuard::new()) }.with(|val: &mut $ty| assert_eq!(*val, 16)); + } + }; +} + impl kernel::Module for PerCpuMod { fn init(_module: &'static ThisModule) -> Result { pr_info!("rust percpu test start\n"); @@ -198,6 +218,22 @@ fn init(_module: &'static ThisModule) -> Result { pr_info!("rust dynamic percpu test done\n"); + pr_info!("rust numeric optimizations test start\n"); + + make_optimization_test!(u8); + make_optimization_test!(u16); + make_optimization_test!(u32); + make_optimization_test!(u64); + make_optimization_test!(usize); + + make_optimization_test!(i8); + make_optimization_test!(i16); + make_optimization_test!(i32); + make_optimization_test!(i64); + make_optimization_test!(isize); + + pr_info!("rust numeric optimizations test done\n"); + // Return Err to unload the module Result::Err(EINVAL) } -- 2.34.1 Currently, the creation of a `PerCpuNumeric` requires a memory read via the `Arc` managing the dynamic allocation. While the compiler might be clever enough to consolidate these reads in some cases, the read must happen *somewhere*, which, when we're concerning ourselves with individual instructions, is a very high burden. Instead, cache the `PerCpuPointer` inside the `DynamicPerCpu` structure; then, the `Arc` is used solely to manage the allocation. Signed-off-by: Mitchell Levy --- rust/kernel/percpu/dynamic.rs | 8 ++++++-- rust/kernel/percpu/numeric.rs | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/rust/kernel/percpu/dynamic.rs b/rust/kernel/percpu/dynamic.rs index aad08e4b4251..8683e94bcd9f 100644 --- a/rust/kernel/percpu/dynamic.rs +++ b/rust/kernel/percpu/dynamic.rs @@ -58,7 +58,10 @@ fn drop(&mut self) { pub struct DynamicPerCpu { // INVARIANT: The memory location in each CPU's per-CPU area pointed at by `alloc.0` has been // initialized. + // INVARIANT: `ptr` is the per-CPU pointer managed by `alloc`, which does not change for the + // lifetime of `self`. pub(super) alloc: Arc>, + pub(super) ptr: PerCpuPtr, } impl DynamicPerCpu { @@ -70,9 +73,10 @@ impl DynamicPerCpu { pub fn new_zero(flags: Flags) -> Option { let alloc: PerCpuAllocation = PerCpuAllocation::new_zero()?; + let ptr = alloc.0; let arc = Arc::new(alloc, flags).ok()?; - Some(Self { alloc: arc }) + Some(Self { alloc: arc, ptr }) } } @@ -102,7 +106,7 @@ pub fn new_with(val: T, flags: Flags) -> Option { let arc = Arc::new(alloc, flags).ok()?; - Some(Self { alloc: arc }) + Some(Self { alloc: arc, ptr }) } } diff --git a/rust/kernel/percpu/numeric.rs b/rust/kernel/percpu/numeric.rs index 4de93f653f0e..3412c2de85fd 100644 --- a/rust/kernel/percpu/numeric.rs +++ b/rust/kernel/percpu/numeric.rs @@ -20,7 +20,7 @@ impl DynamicPerCpu<$ty> { pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> { // The invariant is satisfied because `DynamicPerCpu`'s invariant guarantees that // this pointer is valid and initialized on all CPUs. - PerCpuNumeric { ptr: &self.alloc.0 } + PerCpuNumeric { ptr: &self.ptr } } } impl StaticPerCpu<$ty> { @@ -71,7 +71,7 @@ impl DynamicPerCpu<$ty> { pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> { // The invariant is satisfied because `DynamicPerCpu`'s invariant guarantees that // this pointer is valid and initialized on all CPUs. - PerCpuNumeric { ptr: &self.alloc.0 } + PerCpuNumeric { ptr: &self.ptr } } } impl StaticPerCpu<$ty> { -- 2.34.1