Add an iterator for `Cpumask` making use of C's `cpumask_next`. Acked-by: Viresh Kumar Signed-off-by: Mitchell Levy --- rust/helpers/cpumask.c | 5 +++++ rust/kernel/cpumask.rs | 48 +++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/rust/helpers/cpumask.c b/rust/helpers/cpumask.c index eb10598a0242..d95bfa111191 100644 --- a/rust/helpers/cpumask.c +++ b/rust/helpers/cpumask.c @@ -42,6 +42,11 @@ bool rust_helper_cpumask_full(struct cpumask *srcp) return cpumask_full(srcp); } +unsigned int rust_helper_cpumask_next(int n, struct cpumask *srcp) +{ + return cpumask_next(n, srcp); +} + unsigned int rust_helper_cpumask_weight(struct cpumask *srcp) { return cpumask_weight(srcp); diff --git a/rust/kernel/cpumask.rs b/rust/kernel/cpumask.rs index 3fcbff438670..b7401848f59e 100644 --- a/rust/kernel/cpumask.rs +++ b/rust/kernel/cpumask.rs @@ -6,7 +6,7 @@ use crate::{ alloc::{AllocError, Flags}, - cpu::CpuId, + cpu::{self, CpuId}, prelude::*, types::Opaque, }; @@ -161,6 +161,52 @@ pub fn copy(&self, dstp: &mut Self) { } } +/// Iterator for a `Cpumask`. +pub struct CpumaskIter<'a> { + mask: &'a Cpumask, + last: Option, +} + +impl<'a> CpumaskIter<'a> { + /// Creates a new `CpumaskIter` for the given `Cpumask`. + fn new(mask: &'a Cpumask) -> CpumaskIter<'a> { + Self { mask, last: None } + } +} + +impl<'a> Iterator for CpumaskIter<'a> { + type Item = CpuId; + + fn next(&mut self) -> Option { + // SAFETY: By the type invariant, `self.mask.as_raw` is a `struct cpumask *`. + let next = unsafe { + bindings::cpumask_next( + if let Some(last) = self.last { + last.try_into().unwrap() + } else { + -1 + }, + self.mask.as_raw(), + ) + }; + + if next == cpu::nr_cpu_ids() { + None + } else { + self.last = Some(next); + // SAFETY: `cpumask_next` returns either `nr_cpu_ids` or a valid CPU ID. + unsafe { Some(CpuId::from_u32_unchecked(next)) } + } + } +} + +impl Cpumask { + /// Returns an iterator over the set bits in the cpumask. + pub fn iter(&self) -> CpumaskIter<'_> { + CpumaskIter::new(self) + } +} + /// A CPU Mask pointer. /// /// Rust abstraction for the C `struct cpumask_var_t`. -- 2.34.1 Add getters for the global cpumasks documented in `include/linux/cpumask.h`, specifically: - cpu_possible_mask - cpu_online_mask - cpu_enabled_mask - cpu_present_mask - cpu_active_mask Acked-by: Viresh Kumar Suggested-by: Yury Norov Signed-off-by: Mitchell Levy --- rust/kernel/cpumask.rs | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/rust/kernel/cpumask.rs b/rust/kernel/cpumask.rs index b7401848f59e..a6a130092fcb 100644 --- a/rust/kernel/cpumask.rs +++ b/rust/kernel/cpumask.rs @@ -77,6 +77,52 @@ pub unsafe fn as_ref<'a>(ptr: *const bindings::cpumask) -> &'a Self { unsafe { &*ptr.cast() } } + /// Get a CPU mask representing possible CPUs; has bit `cpu` set iff cpu is populatable + #[inline] + pub fn possible_cpus() -> &'static Self { + // SAFETY: `__cpu_possible_mask` is a valid global provided by the kernel that lives + // forever. + unsafe { Cpumask::as_ref(&raw const bindings::__cpu_possible_mask) } + } + + /// Get a CPU mask representing online CPUs; has bit `cpu` set iff cpu available to the + /// scheduler + #[inline] + pub fn online_cpus() -> &'static Self { + // SAFETY: `__cpu_online_mask` is a valid global provided by the kernel that lives forever. + // Since we wrap the returned pointer in an `Opaque`, it's ok that `__cpu_online_mask` + // may change its value. + unsafe { Cpumask::as_ref(&raw const bindings::__cpu_online_mask) } + } + + /// Get a CPU mask representing enabled CPUs; has bit `cpu` set iff cpu can be brought online + #[inline] + pub fn enabled_cpus() -> &'static Self { + // SAFETY: `__cpu_enabled_mask` is a valid global provided by the kernel that lives forever. + // Since we wrap the returned pointer in an `Opaque`, it's ok that `__cpu_enabled_mask` + // may change its value. + unsafe { Cpumask::as_ref(&raw const bindings::__cpu_enabled_mask) } + } + + /// Get a CPU mask representing present CPUs; has bit `cpu` set iff cpu is populated + #[inline] + pub fn present_cpus() -> &'static Self { + // SAFETY: `__cpu_present_mask` is a valid global provided by the kernel that lives + // forever. Since we wrap the returned pointer in an `Opaque`, it's ok that + // `__cpu_present_mask` may change its value. + unsafe { Cpumask::as_ref(&raw const bindings::__cpu_present_mask) } + } + + /// Get a CPU mask representing active CPUs; has bit `cpu` set iff cpu is available to + /// migration. + #[inline] + pub fn active_cpus() -> &'static Self { + // SAFETY: `__cpu_active_mask` is a valid global provided by the kernel that lives forever. + // Since we wrap the returned pointer in an `Opaque`, it's ok that `__cpu_active_mask` + // may change its value. + unsafe { Cpumask::as_ref(&raw const bindings::__cpu_active_mask) } + } + /// Obtain the raw `struct cpumask` pointer. pub fn as_raw(&self) -> *mut bindings::cpumask { let this: *const Self = self; -- 2.34.1 Add bindings necessary to implement a Rust per-CPU variable API, specifically per-CPU variable allocation and management of CPU preemption. Signed-off-by: Mitchell Levy --- rust/helpers/helpers.c | 2 ++ rust/helpers/percpu.c | 9 +++++++++ rust/helpers/preempt.c | 14 ++++++++++++++ 3 files changed, 25 insertions(+) diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 7cf7fe95e41d..2fc8d26cfe66 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -31,9 +31,11 @@ #include "of.c" #include "page.c" #include "pci.c" +#include "percpu.c" #include "pid_namespace.c" #include "platform.c" #include "poll.c" +#include "preempt.c" #include "property.c" #include "rbtree.c" #include "rcu.c" diff --git a/rust/helpers/percpu.c b/rust/helpers/percpu.c new file mode 100644 index 000000000000..a091389f730f --- /dev/null +++ b/rust/helpers/percpu.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +void __percpu *rust_helper_alloc_percpu(size_t sz, size_t align) +{ + return __alloc_percpu(sz, align); +} + diff --git a/rust/helpers/preempt.c b/rust/helpers/preempt.c new file mode 100644 index 000000000000..2c7529528ddd --- /dev/null +++ b/rust/helpers/preempt.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +void rust_helper_preempt_disable(void) +{ + preempt_disable(); +} + +void rust_helper_preempt_enable(void) +{ + preempt_enable(); +} + -- 2.34.1 Per-CPU variables are an important tool for reducing lock contention, especially in systems with many processors. They also provide a convenient way to handle data that are logically associated with a particular CPU (e.g., the currently running task). Therefore, add a Rust API to make use of statically-allocated per-CPU variables. Add a RAII `CpuGuard` type for disabling CPU preemption. Introduce unifying abstractions that can be reused for a Rust API for dynamically-allocated per-CPU variables. Co-developed-by: Boqun Feng Signed-off-by: Boqun Feng Signed-off-by: Mitchell Levy --- Using `rustc` 1.83.0, I can confirm that the symbols `__INIT_$id` are optimized out by the compiler and do not appear in the final `.ko` file when compiling `samples/rust/rust_percpu.rs` (introduced in a later patch in this series). --- rust/kernel/lib.rs | 3 + rust/kernel/percpu.rs | 250 ++++++++++++++++++++++++++++++++++++++++ rust/kernel/percpu/cpu_guard.rs | 36 ++++++ rust/kernel/percpu/static_.rs | 218 +++++++++++++++++++++++++++++++++++ 4 files changed, 507 insertions(+) diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index f910a5ab80ba..2fdb194aa068 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -108,6 +108,9 @@ pub mod page; #[cfg(CONFIG_PCI)] pub mod pci; +// Only x86_64 is supported by percpu for now +#[cfg(CONFIG_X86_64)] +pub mod percpu; pub mod pid_namespace; pub mod platform; pub mod prelude; diff --git a/rust/kernel/percpu.rs b/rust/kernel/percpu.rs new file mode 100644 index 000000000000..2fba9a165636 --- /dev/null +++ b/rust/kernel/percpu.rs @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: GPL-2.0 +//! Per-CPU variables. +//! +//! See the [`crate::define_per_cpu!`] macro and the [`PerCpu`] trait. + +pub mod cpu_guard; +mod static_; + +#[doc(inline)] +pub use static_::*; + +use crate::declare_extern_per_cpu; +use crate::percpu::cpu_guard::CpuGuard; +use crate::types::Opaque; + +use core::arch::asm; +use core::cell::{Cell, RefCell, UnsafeCell}; +use core::mem::MaybeUninit; + +use ffi::c_void; + +/// A per-CPU pointer; that is, an offset into the per-CPU area. +/// +/// Note that this type is NOT a smart pointer, it does not manage the allocation. +pub struct PerCpuPtr(*mut MaybeUninit); + +/// Represents exclusive access to the memory location pointed at by a particular [`PerCpu`]. +pub struct PerCpuToken<'a, T> { + // INVARIANT: the current CPU's memory location associated with the per-CPU variable pointed at + // by `ptr` (i.e., the entry in the per-CPU area on the current CPU) has been initialized. + _guard: CpuGuard, + ptr: &'a PerCpuPtr, +} + +/// Represents access to the memory location pointed at by a particular [`PerCpu`] where the +/// type `T` manages access to the underlying memory to avoid aliaising troubles. +/// +/// For example, `T` might be a [`Cell`] or [`RefCell`]. +pub struct CheckedPerCpuToken<'a, T> { + // INVARIANT: the current CPU's memory location associated with the per-CPU variable pointed at + // by `ptr` (i.e., the entry in the per-CPU area on the current CPU) has been initialized. + _guard: CpuGuard, + ptr: &'a PerCpuPtr, +} + +impl PerCpuPtr { + /// Makes a new [`PerCpuPtr`] from a raw per-CPU pointer. + /// + /// Note that the returned [`PerCpuPtr`] is valid only as long as the given pointer is. This + /// also requires that the allocation pointed to by `ptr` must be correctly sized and aligned + /// to hold a `T`. + pub fn new(ptr: *mut MaybeUninit) -> Self { + Self(ptr) + } + + /// Get a [`&mut MaybeUninit`](MaybeUninit) to the per-CPU variable on the current CPU + /// represented by `&self` + /// + /// # Safety + /// + /// The returned `&mut T` must follow Rust's aliasing rules. That is, no other `&(mut) T` may + /// exist that points to the same location in memory. In practice, this means that + /// [`Self::get_ref`] and [`Self::get_mut_ref`] must not be called on another [`PerCpuPtr`] + /// that is a copy/clone of `&self` for as long as the returned reference lives. If the per-CPU + /// variable represented by `&self` is available to C, the caller of this function must ensure + /// that the C code does not modify the variable for as long as this reference lives. + /// + /// `self` must point to a live allocation correctly sized and aligned to hold a `T`. + /// + /// CPU preemption must be disabled before calling this function and for the lifetime of the + /// returned reference. Otherwise, the returned reference might end up being a reference to a + /// different CPU's per-CPU area, causing the potential for a data race. + #[allow(clippy::mut_from_ref)] // Safety requirements prevent aliasing issues + pub unsafe fn get_mut_ref(&self) -> &mut MaybeUninit { + // SAFETY: `self.get_ptr()` returns a valid pointer to a `MaybeUninit` by its contract, + // and the safety requirements of this function ensure that the returned reference is + // exclusive. + unsafe { &mut *(self.get_ptr()) } + } + + /// Get a [`&mut MaybeUninit`](MaybeUninit) to the per-CPU variable on the current CPU + /// represented by `&self` + /// + /// # Safety + /// + /// The returned `&T` must follow Rust's aliasing rules. That is, no `&mut T` may exist that + /// points to the same location in memory. In practice, this means that [`Self::get_mut_ref`] + /// must not be called on another [`PerCpuPtr`] that is a copy/clone of `&self` for as long + /// as the returned reference lives. If the per-CPU variable represented by `&self` is + /// available to C, the caller of this function must ensure that the C code does not modify the + /// variable for as long as this reference lives. + /// + /// `self` must point to a live allocation correctly sized and aligned to hold a `T`. + /// + /// CPU preemption must be disabled before calling this function and for the lifetime of the + /// returned reference. Otherwise, the returned reference might end up being a reference to a + /// different CPU's per-CPU area, causing the potential for a data race. + pub unsafe fn get_ref(&self) -> &MaybeUninit { + // SAFETY: `self.get_ptr()` returns a valid pointer to a `MaybeUninit` by its contract. + // The safety requirements of this function ensure that the returned reference isn't + // aliased by a `&mut MaybeUninit`. + unsafe { &*self.get_ptr() } + } + + /// Get a [`*mut MaybeUninit`](MaybeUninit) to the per-CPU variable on the current CPU + /// represented by `&self`. Note that if CPU preemption is not disabled before calling this + /// function, use of the returned pointer may cause a data race without some other + /// synchronization mechanism. Buyer beware! + pub fn get_ptr(&self) -> *mut MaybeUninit { + let this_cpu_off_pcpu = ExternStaticPerCpuSymbol::ptr(&raw const this_cpu_off); + let mut this_cpu_area: *mut c_void; + // SAFETY: gs + this_cpu_off_pcpu is guaranteed to be a valid pointer because `gs` points + // to the per-CPU area and this_cpu_off_pcpu is a valid per-CPU allocation. + unsafe { + asm!( + "mov {out}, gs:[{off_val}]", + off_val = in(reg) this_cpu_off_pcpu.0, + out = out(reg) this_cpu_area, + ) + }; + + // This_cpu_area + self.0 is guaranteed to be a valid pointer by the per-CPU subsystem and + // the invariant that self.0 is a valid offset into the per-CPU area. + (this_cpu_area).wrapping_add(self.0 as usize).cast() + } +} + +// SAFETY: Sending a [`PerCpuPtr`] to another thread is safe because as soon as it's sent, the +// pointer is logically referring to a different place in memory in the other CPU's per-CPU area. +// In particular, this means that there are no restrictions on the type `T`. +unsafe impl Send for PerCpuPtr {} + +// SAFETY: Two threads concurrently making use of a [`PerCpuPtr`] will each see the `T` in their +// own per-CPU area, so there's no potential for a data race (regardless of whether `T` is itself +// `Sync`). +unsafe impl Sync for PerCpuPtr {} + +impl Clone for PerCpuPtr { + fn clone(&self) -> Self { + *self + } +} + +/// [`PerCpuPtr`] is just a wrapper around a pointer. +impl Copy for PerCpuPtr {} + +/// A trait representing a per-CPU variable. +/// +/// This is implemented for [`StaticPerCpu`]. The main usage of this trait is to call +/// [`Self::get_mut`] to get a [`PerCpuToken`] that can be used to access the underlying per-CPU +/// variable. +/// +/// See [`PerCpuToken::with`]. +pub trait PerCpu { + /// Produces a token, asserting that the holder has exclusive access to the underlying memory + /// pointed to by `self` + /// + /// # Safety + /// + /// No other [`PerCpuToken`] or [`CheckedPerCpuToken`] may exist on the current CPU (which is a + /// sensible notion, since we keep a [`CpuGuard`] around) that is derived from the same + /// [`PerCpu`] or a clone thereof. + unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T>; +} + +/// A marker trait for types that are interior mutable. Types that implement this trait can be used +/// to create "checked" per-CPU variables. See [`CheckedPerCpu`]. +pub trait InteriorMutable {} + +impl InteriorMutable for Cell {} +impl InteriorMutable for RefCell {} +impl InteriorMutable for UnsafeCell {} +impl InteriorMutable for Opaque {} + +/// A trait representing a per-CPU variable that is usable via a `&T`. +/// +/// The unsafety of [`PerCpu`] stems from the fact that the holder of a [`PerCpuToken`] can use +/// it to get a `&mut T` to the underlying per-CPU variable. This is problematic because the +/// existence of aliaising `&mut T` is undefined behavior in Rust. This type avoids that issue by +/// only allowing access via a `&T`, with the tradeoff that then `T` must be interior mutable or +/// the underlying per-CPU variable must be a constant for the lifetime of any corresponding +/// [`CheckedPerCpuToken`]. +/// +/// Currently, only the case where `T` is interior mutable has first-class support, though a custom +/// implementation of [`PerCpu`]/[`CheckedPerCpu`] could be created for the const case. +pub trait CheckedPerCpu: PerCpu { + /// Produces a token via which the holder can access the underlying per-CPU variable. + fn get(&mut self, guard: CpuGuard) -> CheckedPerCpuToken<'_, T>; +} + +impl<'a, T> PerCpuToken<'a, T> { + /// Asserts that the holder has exclusive access to the initialized underlying memory pointed + /// to by `ptr` on the current CPU, producing a token. + /// + /// # Safety + /// + /// No other [`PerCpuToken`] or [`CheckedPerCpuToken`] may exist on the current CPU (which is a + /// sensible notion, since we keep a `CpuGuard` around) that uses the same [`PerCpuPtr`]. + /// + /// The current CPU's memory location associated with the per-CPU variable pointed at by `ptr` + /// (i.e., the entry in the per-CPU area on this CPU) must be initialized; `ptr` must be valid + /// (that is, pointing at a live per-CPU allocation correctly sized and aligned to hold a `T`) + /// for `'a`. + pub unsafe fn new(guard: CpuGuard, ptr: &'a PerCpuPtr) -> PerCpuToken<'a, T> { + Self { _guard: guard, ptr } + } + + /// Immediately invokes `func` with a `&mut T` that points at the underlying per-CPU variable + /// that `&mut self` represents. + pub fn with(&mut self, func: U) + where + U: FnOnce(&mut T), + { + // SAFETY: The existence of a PerCpuToken means that the requirements for get_mut_ref are + // satisfied. Likewise, the requirements for assume_init_mut are satisfied because the + // invariants of this type ensure that on the current CPU (which is a sensible notion + // because we have a CpuGuard), the memory location pointed to by `ptr` is initialized. + func(unsafe { self.ptr.get_mut_ref().assume_init_mut() }); + } +} + +impl<'a, T> CheckedPerCpuToken<'a, T> { + /// Asserts that the memory pointed to by `ptr` is initialized on the current CPU, producing a + /// token. + /// + /// # Safety + /// + /// The current CPU's memory location associated with the per-CPU variable pointed at by `ptr` + /// (i.e., the entry in the per-CPU area on this CPU) must be initialized; `ptr` must be valid + /// (that is, pointing at a live per-CPU allocation correctly sized and aligned to hold a `T`) + /// for `'a`. + pub unsafe fn new(guard: CpuGuard, ptr: &'a PerCpuPtr) -> CheckedPerCpuToken<'a, T> { + Self { _guard: guard, ptr } + } + + /// Immediately invokes `func` with a `&T` that points at the underlying per-CPU variable that + /// `&mut self` represents. + pub fn with(&mut self, func: U) + where + U: FnOnce(&T), + { + // SAFETY: The existence of a CheckedPerCpuToken means that the requirements for get_ref + // are satisfied. Likewise, the requirements for assume_init_ref are satisfied because the + // invariants of this type ensure that on the current CPU (which is a sensible notion + // because we have a CpuGuard), the memory location pointed to by `ptr` is initialized. + func(unsafe { self.ptr.get_ref().assume_init_ref() }); + } +} + +declare_extern_per_cpu!(this_cpu_off: *mut c_void); diff --git a/rust/kernel/percpu/cpu_guard.rs b/rust/kernel/percpu/cpu_guard.rs new file mode 100644 index 000000000000..2fb4c9218922 --- /dev/null +++ b/rust/kernel/percpu/cpu_guard.rs @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-2.0 +//! Contains abstractions for disabling CPU preemption. See [`CpuGuard`]. + +/// A RAII guard for `bindings::preempt_disable` and `bindings::preempt_enable`. +/// +/// Guarantees preemption is disabled for as long as this object exists. +pub struct CpuGuard { + // Don't make one without using new() + _phantom: (), +} + +impl CpuGuard { + /// Create a new [`CpuGuard`]. Disables preemption for its lifetime. + pub fn new() -> Self { + // SAFETY: There are no preconditions required to call preempt_disable + unsafe { + bindings::preempt_disable(); + } + CpuGuard { _phantom: () } + } +} + +impl Default for CpuGuard { + fn default() -> Self { + Self::new() + } +} + +impl Drop for CpuGuard { + fn drop(&mut self) { + // SAFETY: There are no preconditions required to call preempt_enable + unsafe { + bindings::preempt_enable(); + } + } +} diff --git a/rust/kernel/percpu/static_.rs b/rust/kernel/percpu/static_.rs new file mode 100644 index 000000000000..418fc2fa06f2 --- /dev/null +++ b/rust/kernel/percpu/static_.rs @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: GPL-2.0 +//! Statically allocated per-CPU variables. + +use super::*; + +/// A wrapper used for declaring static per-CPU variables. +/// +/// These symbols are "virtual" in that the linker uses them to generate offsets into each CPU's +/// per-CPU area, but shouldn't be read from/written to directly. The fact that the statics are +/// immutable prevents them being written to (generally), this struct having _val be non-public +/// prevents reading from them. +/// +/// The end-user of the per-CPU API should make use of the [`crate::define_per_cpu!`] macro instead +/// of declaring variables of this type directly. All instances of this type must be `static` and +/// `#[link_section = ".data..percpu"]` (which the macro handles). +#[repr(transparent)] +pub struct StaticPerCpuSymbol { + _val: T, // generate a correctly sized type +} + +/// A wrapper for per-CPU variables declared in C. +/// +/// As with [`StaticPerCpuSymbol`], this type should not be used directly. Instead, use the +/// [`crate::declare_extern_per_cpu!`] macro. +#[repr(transparent)] +pub struct ExternStaticPerCpuSymbol(StaticPerCpuSymbol); + +/// Holds a statically-allocated per-CPU variable. +#[derive(Clone)] +pub struct StaticPerCpu(pub(super) PerCpuPtr); + +impl StaticPerCpuSymbol { + /// Removes the per-CPU marker type from a static per-CPU symbol. + /// + /// To declare a static per-CPU variable, Rust requires that the variable be `Sync`. However, + /// in the case of per-CPU variables, this is silly. See [`PerCpuSyncMarkerType`]. Thus, our + /// statics are actually of type `StaticPerCpuSymbol>`, and this + /// produces a `StaticPerCpuSymbol`. + pub fn forward(ptr: *const Self) -> *const StaticPerCpuSymbol { + ptr.cast() + } +} + +impl ExternStaticPerCpuSymbol { + /// Gets a [`PerCpuPtr`] to the per-CPU variable. + /// + /// Usage of this [`PerCpuPtr`] must be very careful to keep in mind what's happening in C. In + /// particular, the C code cannot modify the variable while any reference derived from the + /// returned pointer is live. + pub fn ptr(ptr: *const Self) -> PerCpuPtr { + // These casts are OK because, ExternStaticPerCpuSymbol, StaticPerCpuSymbol, and + // MaybeUninit are transparent, everything is just a `T` from a memory layout perspective. + // Casting to `mut` is OK because any usage of the returned pointer must satisfy the + // soundness requirements for using it as such. + PerCpuPtr::new(ptr.cast::>().cast_mut().cast()) + } +} + +impl StaticPerCpu { + /// Creates a [`StaticPerCpu`] from a [`StaticPerCpuSymbol`]. + /// + /// Users should probably declare static per-CPU variables with [`crate::define_per_cpu!`] and + /// then get instances of [`StaticPerCpu`] using [`crate::get_static_per_cpu!`]. + /// + /// # Safety + /// You should probably be using [`crate::get_static_per_cpu!`] instead. + /// + /// `ptr` must be a valid offset into the per-CPU area, sized and aligned for access to a `T`; + /// typically this is the address of a static in the `.data..percpu` section, which is managed + /// by the per-CPU subsystem. + pub unsafe fn new(ptr: *const StaticPerCpuSymbol) -> StaticPerCpu { + let pcpu_ptr = PerCpuPtr::new(ptr.cast_mut().cast()); + Self(pcpu_ptr) + } +} + +impl PerCpu for StaticPerCpu { + unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T> { + // SAFETY: + // 1. By the requirements of `PerCpu::get_mut`, no other `[Checked]PerCpuToken` exists on + // the current CPU. + // 2. The per-CPU subsystem guarantees that each CPU's instance of a statically allocated + // variable begins with a copy of the contents of the corresponding symbol in + // `.data..percpu` and is therefore initialized. + // 3. The per-CPU subsystem guarantees that `self.0` is correctly aligned for a `T`. + // 4. The per-CPU subsystem guarantees that `self.0` is lives forever as a per-CPU + // allocation, and that this allocation is the proper size for a `T`. + unsafe { PerCpuToken::new(guard, &self.0) } + } +} + +impl CheckedPerCpu for StaticPerCpu { + fn get(&mut self, guard: CpuGuard) -> CheckedPerCpuToken<'_, T> { + // SAFETY: + // 1. The per-CPU subsystem guarantees that each CPU's instance of a statically allocated + // variable begins with a copy of the contents of the corresponding symbol in + // `.data..percpu` and is therefore initialized. + // 2. The per-CPU subsystem guarantees that `self.0` is correctly aligned for a `T`. + // 3. The per-CPU subsystem guarantees that `self.0` is lives forever as a per-CPU + // allocation, and that this allocation is the proper size for a `T`. + unsafe { CheckedPerCpuToken::new(guard, &self.0) } + } +} + +/// Gets a [`StaticPerCpu`] from a symbol declared with [`crate::define_per_cpu!`]. +/// +/// # Arguments +/// +/// * `ident` - The identifier declared +#[macro_export] +macro_rules! get_static_per_cpu { + ($id:ident) => { + unsafe { + // SAFETY: The signature of `StaticPerCpuSymbol::forward` guarantees that `&raw const + // $id` is a `*const StaticPerCpuSymbol>` if the macro + // invocation compiles. + // + // Values of type `StaticPerCpuSymbol` must be created via `define_per_cpu`, and so + // the per-CPU subsystem guarantees that the requirements for `StaticPerCpu::new` are + // satisfied. + $crate::percpu::StaticPerCpu::new($crate::percpu::StaticPerCpuSymbol::forward( + &raw const $id, + )) + } + }; +} + +/// Declares an [`ExternStaticPerCpuSymbol`] corresponding to a per-CPU variable defined in C. +#[macro_export] +macro_rules! declare_extern_per_cpu { + ($id:ident: $ty:ty) => { + extern "C" { + static $id: ExternStaticPerCpuSymbol<$ty>; + } + }; +} + +/// A trait implemented by static per-CPU wrapper types. +/// +/// Internally, static per-CPU variables are declared as `static` variables. However, Rust doesn't +/// allow you to declare statics of a `!Sync` type. This trait is implemented by the marker type +/// that is declared as `Sync` and used to declare the static. See [`crate::define_per_cpu!`] for +/// the gory details. +/// +/// # Safety +/// +/// Implementations must be `#[repr(transparent)]` wrappers around an `Inner`. This trait should +/// only be used from the [`crate::define_per_cpu!`] macro. +pub unsafe trait PerCpuSyncMarkerType { + /// The "true" type of the per-CPU variable. It must always be valid to cast a `*const Self` to + /// a `*const Self::Inner`, which is guarnateed by this trait's safety requirement. + type Inner; +} + +/// Declares and initializes a static per-CPU variable. +/// +/// Analogous to the C `DEFINE_PER_CPU` macro. +/// +/// See also [`crate::get_static_per_cpu!`] for how to get a [`StaticPerCpu`] from this +/// declaration. +/// +/// # Example +/// ``` +/// use kernel::define_per_cpu; +/// use kernel::percpu::StaticPerCpuSymbol; +/// +/// define_per_cpu!(pub MY_PERCPU: u64 = 0); +/// ``` +#[macro_export] +macro_rules! define_per_cpu { + ($vis:vis $id:ident: $ty:ty = $expr:expr) => { + $crate::macros::paste! { + // We might want to have a per-CPU variable that doesn't implement `Sync` (not paying + // sync overhead costs is part of the point), but Rust won't let us declare a static of + // a `!Sync` type. Of course, we don't actually have any synchronization issues, since + // each CPU will see its own copy of the variable, so we cheat a little bit and tell + // Rust it's fine. + #[doc(hidden)] + #[allow(non_camel_case_types)] + #[repr(transparent)] // It needs to be the same size as $ty + struct [<__PRIVATE_TYPE_ $id>]($ty); + + // SAFETY: [<__PRIVATE_TYPE_ $id>] is a `#[repr(transparent)]` wrapper around a `$ty`. + unsafe impl PerCpuSyncMarkerType for [<__PRIVATE_TYPE_ $id>] { + type Inner = $ty; + } + + impl [<__PRIVATE_TYPE_ $id>] { + #[doc(hidden)] + const fn new(val: $ty) -> Self { + Self(val) + } + } + + // Expand $expr outside of the unsafe block to avoid silently allowing unsafe code to be + // used without a user-facing unsafe block + #[doc(hidden)] + static [<__INIT_ $id>]: [<__PRIVATE_TYPE_ $id>] = [<__PRIVATE_TYPE_ $id>]::new($expr); + + // SAFETY: This type will ONLY ever be used to declare a `StaticPerCpuSymbol` + // (which we then only ever use as input to `&raw`). Reading from the symbol is + // already UB, so we won't ever actually have any variables of this type where + // synchronization is a concern. + #[doc(hidden)] + unsafe impl Sync for [<__PRIVATE_TYPE_ $id>] {} + + // SAFETY: StaticPerCpuSymbol is #[repr(transparent)], so we can freely convert from + // [<__PRIVATE_TYPE_ $id>], which is also `#[repr(transparent)]` (i.e., everything is + // just a `$ty` from a memory layout perspective). + #[link_section = ".data..percpu"] + $vis static $id: StaticPerCpuSymbol<[<__PRIVATE_TYPE_ $id>]> = unsafe { + core::mem::transmute_copy::< + [<__PRIVATE_TYPE_ $id>], StaticPerCpuSymbol<[<__PRIVATE_TYPE_ $id>]> + >(&[<__INIT_ $id>]) + }; + } + }; +} -- 2.34.1 Dynamically allocated per-CPU variables are core to many of the use-cases of per-CPU variables (e.g., ref counting). Add support for them using the core `PerCpuPtr` primitive, implementing the `PerCpu` trait. Co-developed-by: Boqun Feng Signed-off-by: Boqun Feng Signed-off-by: Mitchell Levy --- rust/helpers/percpu.c | 10 ++++ rust/kernel/percpu.rs | 30 ++++++++-- rust/kernel/percpu/dynamic.rs | 130 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 166 insertions(+), 4 deletions(-) diff --git a/rust/helpers/percpu.c b/rust/helpers/percpu.c index a091389f730f..35656333dfae 100644 --- a/rust/helpers/percpu.c +++ b/rust/helpers/percpu.c @@ -7,3 +7,13 @@ void __percpu *rust_helper_alloc_percpu(size_t sz, size_t align) return __alloc_percpu(sz, align); } +void *rust_helper_per_cpu_ptr(void __percpu *ptr, unsigned int cpu) +{ + return per_cpu_ptr(ptr, cpu); +} + +void rust_helper_on_each_cpu(smp_call_func_t func, void *info, int wait) +{ + on_each_cpu(func, info, wait); +} + diff --git a/rust/kernel/percpu.rs b/rust/kernel/percpu.rs index 2fba9a165636..294b8ffc4f62 100644 --- a/rust/kernel/percpu.rs +++ b/rust/kernel/percpu.rs @@ -1,14 +1,19 @@ // SPDX-License-Identifier: GPL-2.0 //! Per-CPU variables. //! -//! See the [`crate::define_per_cpu!`] macro and the [`PerCpu`] trait. +//! See the [`crate::define_per_cpu!`] macro, the [`DynamicPerCpu`] type, and the [`PerCpu`] +//! trait. pub mod cpu_guard; +mod dynamic; mod static_; +#[doc(inline)] +pub use dynamic::*; #[doc(inline)] pub use static_::*; +use crate::cpu::CpuId; use crate::declare_extern_per_cpu; use crate::percpu::cpu_guard::CpuGuard; use crate::types::Opaque; @@ -123,6 +128,23 @@ pub fn get_ptr(&self) -> *mut MaybeUninit { // the invariant that self.0 is a valid offset into the per-CPU area. (this_cpu_area).wrapping_add(self.0 as usize).cast() } + + /// Get a [`*mut MaybeUninit`](MaybeUninit) to the per-CPU variable on the CPU represented + /// by `cpu`. Note that without some kind of synchronization, use of the returned pointer may + /// cause a data race. It is the caller's responsibility to use the returned pointer in a + /// reasonable way. + /// + /// # Returns + /// - The returned pointer is valid only if `self` is (that is, it points to a live allocation + /// correctly sized and aligned to hold a `T`) + /// - The returned pointer is valid only if the bit corresponding to `cpu` is set in + /// [`kernel::cpumask::Cpumask::possible_cpus()`]. + pub fn get_remote_ptr(&self, cpu: CpuId) -> *mut MaybeUninit { + // SAFETY: `bindings::per_cpu_ptr` is just doing pointer arithmetic. The returned pointer + // may not be valid (under the conditions specified in this function's documentation), but + // the act of producing the pointer is safe. + unsafe { bindings::per_cpu_ptr(self.0.cast(), cpu.as_u32()) }.cast() + } } // SAFETY: Sending a [`PerCpuPtr`] to another thread is safe because as soon as it's sent, the @@ -146,9 +168,9 @@ impl Copy for PerCpuPtr {} /// A trait representing a per-CPU variable. /// -/// This is implemented for [`StaticPerCpu`]. The main usage of this trait is to call -/// [`Self::get_mut`] to get a [`PerCpuToken`] that can be used to access the underlying per-CPU -/// variable. +/// This is implemented for both [`StaticPerCpu`] and [`DynamicPerCpu`]. The main usage of +/// this trait is to call [`Self::get_mut`] to get a [`PerCpuToken`] that can be used to access the +/// underlying per-CPU variable. /// /// See [`PerCpuToken::with`]. pub trait PerCpu { diff --git a/rust/kernel/percpu/dynamic.rs b/rust/kernel/percpu/dynamic.rs new file mode 100644 index 000000000000..1863f31a2817 --- /dev/null +++ b/rust/kernel/percpu/dynamic.rs @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0 +//! Dynamically allocated per-CPU variables. + +use super::*; + +use crate::alloc::Flags; +use crate::bindings::{alloc_percpu, free_percpu}; +use crate::cpumask::Cpumask; +use crate::prelude::*; +use crate::sync::Arc; +use core::mem::{align_of, size_of, MaybeUninit}; + +/// Represents a dynamic allocation of a per-CPU variable via `alloc_percpu`. Calls `free_percpu` +/// when dropped. +/// +/// # Contents +/// Note that the allocated memory need not be initialized, and this type does not track when/if +/// the memory location on any particular CPU has been initialized. This means that it cannot tell +/// whether it should drop the *contents* of the allocation when it is dropped. It is up to the +/// user to do this via something like [`core::ptr::drop_in_place`]. +pub struct PerCpuAllocation(PerCpuPtr); + +impl PerCpuAllocation { + /// Dynamically allocates a space in the per-CPU area suitably sized and aligned to hold a `T`, + /// initially filled with the zero value for `T`. + /// + /// Returns [`None`] under the same circumstances the C function `alloc_percpu` returns `NULL`. + pub fn new_zero() -> Option> { + let ptr: *mut MaybeUninit = + // SAFETY: No preconditions to call `alloc_percpu`; `MaybeUninit` is + // `#[repr(transparent)]`, so we can cast a `*mut T` to it. + unsafe { alloc_percpu(size_of::(), align_of::()) }.cast(); + if ptr.is_null() { + return None; + } + + // alloc_percpu returns zero'ed memory + Some(Self(PerCpuPtr::new(ptr))) + } +} + +impl PerCpuAllocation { + /// Makes a per-CPU allocation sized and aligned to hold a `T`. + /// + /// Returns [`None`] under the same circumstances the C function `alloc_percpu` returns `NULL`. + pub fn new_uninit() -> Option> { + let ptr: *mut MaybeUninit = + // SAFETY: No preconditions to call `alloc_percpu`; `MaybeUninit` is + // `#[repr(transparent)]`, so we can cast a `*mut T` to it. + unsafe { alloc_percpu(size_of::(), align_of::()) }.cast(); + if ptr.is_null() { + return None; + } + + Some(Self(PerCpuPtr::new(ptr))) + } +} + +impl Drop for PerCpuAllocation { + fn drop(&mut self) { + // SAFETY: self.0.0 was returned by alloc_percpu, and so was a valid pointer into + // the percpu area, and has remained valid by the invariants of PerCpuAllocation. + unsafe { free_percpu(self.0 .0.cast()) } + } +} + +/// Holds a dynamically-allocated per-CPU variable. +#[derive(Clone)] +pub struct DynamicPerCpu { + // INVARIANT: `alloc` is `Some` unless this object is in the process of being dropped. + // INVARIANT: The allocation held by `alloc` is sized and aligned for a `T`. + // INVARIANT: The memory location in each CPU's per-CPU area pointed at by the alloc is + // initialized. + alloc: Option>>, +} + +impl DynamicPerCpu { + /// Allocates a new per-CPU variable + /// + /// # Arguments + /// * `flags` - [`Flags`] used to allocate an [`Arc`] that keeps track of the underlying + /// [`PerCpuAllocation`]. + pub fn new_zero(flags: Flags) -> Option { + let alloc: PerCpuAllocation = PerCpuAllocation::new_zero()?; + + let arc = Arc::new(alloc, flags).ok()?; + + Some(Self { alloc: Some(arc) }) + } +} + +impl PerCpu for DynamicPerCpu { + unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T> { + // SAFETY: + // 1. Invariants of this type assure that `alloc` is `Some`. + // 2. The requirements of `PerCpu::get_mut` ensure that no other `[Checked]PerCpuToken` + // exists on the current CPU. + // 3. The invariants of `DynamicPerCpu` ensure that the contents of the allocation are + // initialized on each CPU. + // 4. The existence of a reference to the `PerCpuAllocation` ensures that the allocation is + // live. + // 5. The invariants of `DynamicPerCpu` ensure that the allocation is sized and aligned for + // a `T`. + unsafe { PerCpuToken::new(guard, &self.alloc.as_ref().unwrap_unchecked().0) } + } +} + +impl Drop for DynamicPerCpu { + fn drop(&mut self) { + // SAFETY: This type's invariant ensures that `self.alloc` is `Some`. + let alloc = unsafe { self.alloc.take().unwrap_unchecked() }; + if let Some(unique_alloc) = alloc.into_unique_or_drop() { + let ptr = unique_alloc.0; + for cpu in Cpumask::possible_cpus().iter() { + let remote_ptr = ptr.get_remote_ptr(cpu); + // SAFETY: `remote_ptr` is valid because the allocation it points to is still live, + // `cpu` appears in `Cpumask::possible_cpus()`, and the original allocation was + // sized and aligned for a `T`. + // + // This type's invariant ensures that the memory location in each CPU's per-CPU + // area pointed at by `alloc.0` has been initialized. We have a `UniqueArc`, so we + // know we're the only ones with a reference to the memory. These two facts + // together satisfy the requirements for `assume_init_drop`. + unsafe { + (*remote_ptr).assume_init_drop(); + } + } + } + } +} -- 2.34.1 Add a short exercise for Rust's per-CPU variable API, modelled after lib/percpu_test.c Signed-off-by: Mitchell Levy --- rust/helpers/percpu.c | 1 + rust/kernel/percpu.rs | 2 +- samples/rust/Kconfig | 9 +++ samples/rust/Makefile | 1 + samples/rust/rust_percpu.rs | 163 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 175 insertions(+), 1 deletion(-) diff --git a/rust/helpers/percpu.c b/rust/helpers/percpu.c index 35656333dfae..8d83b6b86106 100644 --- a/rust/helpers/percpu.c +++ b/rust/helpers/percpu.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include void __percpu *rust_helper_alloc_percpu(size_t sz, size_t align) { diff --git a/rust/kernel/percpu.rs b/rust/kernel/percpu.rs index 294b8ffc4f62..2db670c87fae 100644 --- a/rust/kernel/percpu.rs +++ b/rust/kernel/percpu.rs @@ -2,7 +2,7 @@ //! Per-CPU variables. //! //! See the [`crate::define_per_cpu!`] macro, the [`DynamicPerCpu`] type, and the [`PerCpu`] -//! trait. +//! trait. Example usage can be found in `samples/rust/rust_percpu.rs`. pub mod cpu_guard; mod dynamic; diff --git a/samples/rust/Kconfig b/samples/rust/Kconfig index 7f7371a004ee..23e35d64ac78 100644 --- a/samples/rust/Kconfig +++ b/samples/rust/Kconfig @@ -105,6 +105,15 @@ config SAMPLE_RUST_DRIVER_AUXILIARY If unsure, say N. +config SAMPLE_RUST_PERCPU + tristate "Per-CPU support" + depends on m + help + Enable this option to build a module which demonstrates Rust per-CPU + operations. + + If unsure, say N. + config SAMPLE_RUST_HOSTPROGS bool "Host programs" help diff --git a/samples/rust/Makefile b/samples/rust/Makefile index bd2faad63b4f..8a34d9d74754 100644 --- a/samples/rust/Makefile +++ b/samples/rust/Makefile @@ -10,6 +10,7 @@ obj-$(CONFIG_SAMPLE_RUST_DRIVER_PLATFORM) += rust_driver_platform.o obj-$(CONFIG_SAMPLE_RUST_DRIVER_FAUX) += rust_driver_faux.o obj-$(CONFIG_SAMPLE_RUST_DRIVER_AUXILIARY) += rust_driver_auxiliary.o obj-$(CONFIG_SAMPLE_RUST_CONFIGFS) += rust_configfs.o +obj-$(CONFIG_SAMPLE_RUST_PERCPU) += rust_percpu.o rust_print-y := rust_print_main.o rust_print_events.o diff --git a/samples/rust/rust_percpu.rs b/samples/rust/rust_percpu.rs new file mode 100644 index 000000000000..98ca1c781b6b --- /dev/null +++ b/samples/rust/rust_percpu.rs @@ -0,0 +1,163 @@ +// SPDX-License-Identifier: GPL-2.0 +//! A simple demonstration of the rust per-CPU API. + +use core::cell::RefCell; +use core::ffi::c_void; + +use kernel::{ + bindings::on_each_cpu, + cpu::CpuId, + define_per_cpu, get_static_per_cpu, + percpu::{cpu_guard::*, *}, + pr_info, + prelude::*, +}; + +module! { + type: PerCpuMod, + name: "rust_percpu", + authors: ["Mitchell Levy"], + description: "Sample to demonstrate the Rust per-CPU API", + license: "GPL v2", +} + +struct PerCpuMod; + +define_per_cpu!(PERCPU: i64 = 0); +define_per_cpu!(UPERCPU: u64 = 0); +define_per_cpu!(CHECKED: RefCell = RefCell::new(0)); + +impl kernel::Module for PerCpuMod { + fn init(_module: &'static ThisModule) -> Result { + pr_info!("rust percpu test start\n"); + + let mut native: i64 = 0; + let mut pcpu: StaticPerCpu = get_static_per_cpu!(PERCPU); + + // SAFETY: We only have one PerCpu that points at PERCPU + unsafe { pcpu.get_mut(CpuGuard::new()) }.with(|val: &mut i64| { + pr_info!("The contents of pcpu are {}\n", val); + + native += -1; + *val += -1; + pr_info!("Native: {}, *pcpu: {}\n", native, val); + assert!(native == *val && native == -1); + + native += 1; + *val += 1; + pr_info!("Native: {}, *pcpu: {}\n", native, val); + assert!(native == *val && native == 0); + }); + + let mut unative: u64 = 0; + let mut upcpu: StaticPerCpu = get_static_per_cpu!(UPERCPU); + + // SAFETY: We only have one PerCpu pointing at UPERCPU + unsafe { upcpu.get_mut(CpuGuard::new()) }.with(|val: &mut u64| { + unative += 1; + *val += 1; + pr_info!("Unative: {}, *upcpu: {}\n", unative, val); + assert!(unative == *val && unative == 1); + + unative = unative.wrapping_add((-1i64) as u64); + *val = val.wrapping_add((-1i64) as u64); + pr_info!("Unative: {}, *upcpu: {}\n", unative, val); + assert!(unative == *val && unative == 0); + + unative = unative.wrapping_add((-1i64) as u64); + *val = val.wrapping_add((-1i64) as u64); + pr_info!("Unative: {}, *upcpu: {}\n", unative, val); + assert!(unative == *val && unative == (-1i64) as u64); + + unative = 0; + *val = 0; + + unative = unative.wrapping_sub(1); + *val = val.wrapping_sub(1); + pr_info!("Unative: {}, *upcpu: {}\n", unative, val); + assert!(unative == *val && unative == (-1i64) as u64); + assert!(unative == *val && unative == u64::MAX); + }); + + let mut checked_native: u64 = 0; + let mut checked: StaticPerCpu> = get_static_per_cpu!(CHECKED); + checked.get(CpuGuard::new()).with(|val: &RefCell| { + checked_native += 1; + *val.borrow_mut() += 1; + pr_info!( + "Checked native: {}, *checked: {}\n", + checked_native, + val.borrow() + ); + assert!(checked_native == *val.borrow() && checked_native == 1); + + checked_native = checked_native.wrapping_add((-1i64) as u64); + val.replace_with(|old: &mut u64| old.wrapping_add((-1i64) as u64)); + pr_info!( + "Checked native: {}, *checked: {}\n", + checked_native, + val.borrow() + ); + assert!(checked_native == *val.borrow() && checked_native == 0); + + checked_native = checked_native.wrapping_add((-1i64) as u64); + val.replace_with(|old: &mut u64| old.wrapping_add((-1i64) as u64)); + pr_info!( + "Checked native: {}, *checked: {}\n", + checked_native, + val.borrow() + ); + assert!(checked_native == *val.borrow() && checked_native == (-1i64) as u64); + + checked_native = 0; + *val.borrow_mut() = 0; + + checked_native = checked_native.wrapping_sub(1); + val.replace_with(|old: &mut u64| old.wrapping_sub(1)); + pr_info!( + "Checked native: {}, *checked: {}\n", + checked_native, + val.borrow() + ); + assert!(checked_native == *val.borrow() && checked_native == (-1i64) as u64); + assert!(checked_native == *val.borrow() && checked_native == u64::MAX); + }); + + pr_info!("rust static percpu test done\n"); + + pr_info!("rust dynamic percpu test start\n"); + let mut test: DynamicPerCpu = DynamicPerCpu::new_zero(GFP_KERNEL).unwrap(); + + // SAFETY: No prerequisites for on_each_cpu. + unsafe { + on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 0); + on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 0); + on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 0); + on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 1); + on_each_cpu(Some(check_percpu), (&raw mut test).cast(), 1); + } + + pr_info!("rust dynamic percpu test done\n"); + + // Return Err to unload the module + Result::Err(EINVAL) + } +} + +extern "C" fn inc_percpu(info: *mut c_void) { + // SAFETY: We know that info is a void *const DynamicPerCpu and DynamicPerCpu is Send. + let mut pcpu = unsafe { (*(info as *const DynamicPerCpu)).clone() }; + pr_info!("Incrementing on {}\n", CpuId::current().as_u32()); + + // SAFETY: We don't have multiple clones of pcpu in scope + unsafe { pcpu.get_mut(CpuGuard::new()) }.with(|val: &mut u64| *val += 1); +} + +extern "C" fn check_percpu(info: *mut c_void) { + // SAFETY: We know that info is a void *const DynamicPerCpu and DynamicPerCpu is Send. + let mut pcpu = unsafe { (*(info as *const DynamicPerCpu)).clone() }; + pr_info!("Asserting on {}\n", CpuId::current().as_u32()); + + // SAFETY: We don't have multiple clones of pcpu in scope + unsafe { pcpu.get_mut(CpuGuard::new()) }.with(|val: &mut u64| assert!(*val == 4)); +} -- 2.34.1 Add functionality to `PerCpuPtr` to compute pointers to per-CPU variable slots on other CPUs. Use this facility to initialize per-CPU variables on all possible CPUs when a dynamic per-CPU variable is created with a non-zeroable type. Since `RefCell` and other `Cell`-like types fall into this category, `impl CheckedPerCpu` on `DynamicPerCpu` for these `InteriorMutable` types since they can now be used. Add examples of these usages to `samples/rust/rust_percpu.rs`. Add a test to ensure dynamic per-CPU variables properly drop their contents, done here since non-trivially dropped types often aren't `Zeroable`. Signed-off-by: Mitchell Levy --- rust/kernel/percpu/dynamic.rs | 44 +++++++++++++++++ samples/rust/rust_percpu.rs | 109 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 146 insertions(+), 7 deletions(-) diff --git a/rust/kernel/percpu/dynamic.rs b/rust/kernel/percpu/dynamic.rs index 1863f31a2817..a74c8841aeb2 100644 --- a/rust/kernel/percpu/dynamic.rs +++ b/rust/kernel/percpu/dynamic.rs @@ -89,6 +89,36 @@ pub fn new_zero(flags: Flags) -> Option { } } +impl DynamicPerCpu { + /// Allocates a new per-CPU variable + /// + /// # Arguments + /// * `val` - The initial value of the per-CPU variable on all CPUs. + /// * `flags` - Flags used to allocate an [`Arc`] that keeps track of the underlying + /// [`PerCpuAllocation`]. + pub fn new_with(val: &T, flags: Flags) -> Option { + let alloc: PerCpuAllocation = PerCpuAllocation::new_uninit()?; + let ptr = alloc.0; + + for cpu in Cpumask::possible_cpus().iter() { + let remote_ptr = ptr.get_remote_ptr(cpu); + // SAFETY: `remote_ptr` is valid because `ptr` points to a live allocation and `cpu` + // appears in `Cpumask::possible_cpus()`. + // + // Each CPU's slot corresponding to `ptr` is currently uninitialized, and no one else + // has a reference to it. Therefore, we can freely write to it without worrying about + // the need to drop what was there or whether we're racing with someone else. + unsafe { + (*remote_ptr).write(val.clone()); + } + } + + let arc = Arc::new(alloc, flags).ok()?; + + Some(Self { alloc: Some(arc) }) + } +} + impl PerCpu for DynamicPerCpu { unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T> { // SAFETY: @@ -105,6 +135,20 @@ unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T> { } } +impl CheckedPerCpu for DynamicPerCpu { + fn get(&mut self, guard: CpuGuard) -> CheckedPerCpuToken<'_, T> { + // SAFETY: + // 1. Invariants of this type assure that `alloc` is `Some`. + // 2. The invariants of `DynamicPerCpu` ensure that the contents of the allocation are + // initialized on each CPU. + // 3. The existence of a reference to the `PerCpuAllocation` ensures that the allocation is + // live. + // 4. The invariants of `DynamicPerCpu` ensure that the allocation is sized and aligned for + // a `T`. + unsafe { CheckedPerCpuToken::new(guard, &self.alloc.as_ref().unwrap_unchecked().0) } + } +} + impl Drop for DynamicPerCpu { fn drop(&mut self) { // SAFETY: This type's invariant ensures that `self.alloc` is `Some`. diff --git a/samples/rust/rust_percpu.rs b/samples/rust/rust_percpu.rs index 98ca1c781b6b..be70ee2e513f 100644 --- a/samples/rust/rust_percpu.rs +++ b/samples/rust/rust_percpu.rs @@ -11,6 +11,7 @@ percpu::{cpu_guard::*, *}, pr_info, prelude::*, + sync::Arc, }; module! { @@ -130,13 +131,81 @@ fn init(_module: &'static ThisModule) -> Result { // SAFETY: No prerequisites for on_each_cpu. unsafe { - on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 0); - on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 0); - on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 0); - on_each_cpu(Some(inc_percpu), (&raw mut test).cast(), 1); - on_each_cpu(Some(check_percpu), (&raw mut test).cast(), 1); + on_each_cpu(Some(inc_percpu_u64), (&raw mut test).cast(), 0); + on_each_cpu(Some(inc_percpu_u64), (&raw mut test).cast(), 0); + on_each_cpu(Some(inc_percpu_u64), (&raw mut test).cast(), 0); + on_each_cpu(Some(inc_percpu_u64), (&raw mut test).cast(), 1); + on_each_cpu(Some(check_percpu_u64), (&raw mut test).cast(), 1); } + let mut checked: DynamicPerCpu> = + DynamicPerCpu::new_with(&RefCell::new(100), GFP_KERNEL).unwrap(); + + // SAFETY: No prerequisites for on_each_cpu. + unsafe { + on_each_cpu(Some(inc_percpu_refcell_u64), (&raw mut checked).cast(), 0); + on_each_cpu(Some(inc_percpu_refcell_u64), (&raw mut checked).cast(), 0); + on_each_cpu(Some(inc_percpu_refcell_u64), (&raw mut checked).cast(), 0); + on_each_cpu(Some(inc_percpu_refcell_u64), (&raw mut checked).cast(), 1); + on_each_cpu(Some(check_percpu_refcell_u64), (&raw mut checked).cast(), 1); + } + + checked.get(CpuGuard::new()).with(|val: &RefCell| { + assert!(*val.borrow() == 104); + + let mut checked_native = 0; + *val.borrow_mut() = 0; + + checked_native += 1; + *val.borrow_mut() += 1; + pr_info!( + "Checked native: {}, *checked: {}\n", + checked_native, + val.borrow() + ); + assert!(checked_native == *val.borrow() && checked_native == 1); + + checked_native = checked_native.wrapping_add((-1i64) as u64); + val.replace_with(|old: &mut u64| old.wrapping_add((-1i64) as u64)); + pr_info!( + "Checked native: {}, *checked: {}\n", + checked_native, + val.borrow() + ); + assert!(checked_native == *val.borrow() && checked_native == 0); + + checked_native = checked_native.wrapping_add((-1i64) as u64); + val.replace_with(|old: &mut u64| old.wrapping_add((-1i64) as u64)); + pr_info!( + "Checked native: {}, *checked: {}\n", + checked_native, + val.borrow() + ); + assert!(checked_native == *val.borrow() && checked_native == (-1i64) as u64); + + checked_native = 0; + *val.borrow_mut() = 0; + + checked_native = checked_native.wrapping_sub(1); + val.replace_with(|old: &mut u64| old.wrapping_sub(1)); + pr_info!( + "Checked native: {}, *checked: {}\n", + checked_native, + val.borrow() + ); + assert!(checked_native == *val.borrow() && checked_native == (-1i64) as u64); + assert!(checked_native == *val.borrow() && checked_native == u64::MAX); + }); + + let arc = Arc::new(0, GFP_KERNEL).unwrap(); + { + let _arc_pcpu: DynamicPerCpu> = + DynamicPerCpu::new_with(&arc, GFP_KERNEL).unwrap(); + } + // `arc` should be unique, since all the clones on each CPU should be dropped when + // `_arc_pcpu` is dropped + assert!(arc.into_unique_or_drop().is_some()); + pr_info!("rust dynamic percpu test done\n"); // Return Err to unload the module @@ -144,7 +213,7 @@ fn init(_module: &'static ThisModule) -> Result { } } -extern "C" fn inc_percpu(info: *mut c_void) { +extern "C" fn inc_percpu_u64(info: *mut c_void) { // SAFETY: We know that info is a void *const DynamicPerCpu and DynamicPerCpu is Send. let mut pcpu = unsafe { (*(info as *const DynamicPerCpu)).clone() }; pr_info!("Incrementing on {}\n", CpuId::current().as_u32()); @@ -153,7 +222,7 @@ extern "C" fn inc_percpu(info: *mut c_void) { unsafe { pcpu.get_mut(CpuGuard::new()) }.with(|val: &mut u64| *val += 1); } -extern "C" fn check_percpu(info: *mut c_void) { +extern "C" fn check_percpu_u64(info: *mut c_void) { // SAFETY: We know that info is a void *const DynamicPerCpu and DynamicPerCpu is Send. let mut pcpu = unsafe { (*(info as *const DynamicPerCpu)).clone() }; pr_info!("Asserting on {}\n", CpuId::current().as_u32()); @@ -161,3 +230,29 @@ extern "C" fn check_percpu(info: *mut c_void) { // SAFETY: We don't have multiple clones of pcpu in scope unsafe { pcpu.get_mut(CpuGuard::new()) }.with(|val: &mut u64| assert!(*val == 4)); } + +extern "C" fn inc_percpu_refcell_u64(info: *mut c_void) { + // SAFETY: We know that info is a void *const DynamicPerCpu> and + // DynamicPerCpu> is Send. + let mut pcpu = unsafe { (*(info as *const DynamicPerCpu>)).clone() }; + // SAFETY: smp_processor_id has no preconditions + pr_info!("Incrementing on {}\n", CpuId::current().as_u32()); + + pcpu.get(CpuGuard::new()).with(|val: &RefCell| { + let mut val = val.borrow_mut(); + *val += 1; + }); +} + +extern "C" fn check_percpu_refcell_u64(info: *mut c_void) { + // SAFETY: We know that info is a void *const DynamicPerCpu> and + // DynamicPerCpu> is Send. + let mut pcpu = unsafe { (*(info as *const DynamicPerCpu>)).clone() }; + // SAFETY: smp_processor_id has no preconditions + pr_info!("Asserting on {}\n", CpuId::current().as_u32()); + + pcpu.get(CpuGuard::new()).with(|val: &RefCell| { + let val = val.borrow(); + assert!(*val == 104); + }); +} -- 2.34.1 The C implementations of `this_cpu_add`, `this_cpu_sub`, etc., are optimized to save an instruction by avoiding having to compute `this_cpu_ptr(&x)` for some per-CPU variable `x`. For example, rather than u64 *x_ptr = this_cpu_ptr(&x); *x_ptr += 5; the implementation of `this_cpu_add` is clever enough to make use of the fact that per-CPU variables are implemented on x86 via segment registers, and so we can use only a single instruction (where we assume `&x` is already in `rax`) add gs:[rax], 5 Add this optimization via a `PerCpuNumeric` type to enable code-reuse between `DynamicPerCpu` and `StaticPerCpu`. Signed-off-by: Mitchell Levy --- rust/kernel/percpu.rs | 1 + rust/kernel/percpu/dynamic.rs | 10 ++- rust/kernel/percpu/numeric.rs | 138 ++++++++++++++++++++++++++++++++++++++++++ samples/rust/rust_percpu.rs | 36 +++++++++++ 4 files changed, 184 insertions(+), 1 deletion(-) diff --git a/rust/kernel/percpu.rs b/rust/kernel/percpu.rs index 2db670c87fae..c1148cb36eff 100644 --- a/rust/kernel/percpu.rs +++ b/rust/kernel/percpu.rs @@ -6,6 +6,7 @@ pub mod cpu_guard; mod dynamic; +pub mod numeric; mod static_; #[doc(inline)] diff --git a/rust/kernel/percpu/dynamic.rs b/rust/kernel/percpu/dynamic.rs index a74c8841aeb2..99acbf6363f5 100644 --- a/rust/kernel/percpu/dynamic.rs +++ b/rust/kernel/percpu/dynamic.rs @@ -18,7 +18,7 @@ /// the memory location on any particular CPU has been initialized. This means that it cannot tell /// whether it should drop the *contents* of the allocation when it is dropped. It is up to the /// user to do this via something like [`core::ptr::drop_in_place`]. -pub struct PerCpuAllocation(PerCpuPtr); +pub struct PerCpuAllocation(pub(super) PerCpuPtr); impl PerCpuAllocation { /// Dynamically allocates a space in the per-CPU area suitably sized and aligned to hold a `T`, @@ -119,6 +119,14 @@ pub fn new_with(val: &T, flags: Flags) -> Option { } } +impl DynamicPerCpu { + /// Gets the allocation backing this per-CPU variable. + pub(crate) fn alloc(&self) -> &Arc> { + // SAFETY: This type's invariant ensures that `self.alloc` is `Some`. + unsafe { self.alloc.as_ref().unwrap_unchecked() } + } +} + impl PerCpu for DynamicPerCpu { unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T> { // SAFETY: diff --git a/rust/kernel/percpu/numeric.rs b/rust/kernel/percpu/numeric.rs new file mode 100644 index 000000000000..e76461f05c66 --- /dev/null +++ b/rust/kernel/percpu/numeric.rs @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0 +//! Pin-hole optimizations for [`PerCpu`] where T is a numeric type. + +use super::*; +use core::arch::asm; + +/// Represents a per-CPU variable that can be manipulated with machine-intrinsic numeric +/// operations. +pub struct PerCpuNumeric<'a, T> { + // INVARIANT: `ptr.0` is a valid offset into the per-CPU area and is initialized on all CPUs + // (since we don't have a CPU guard, we have to be pessimistic and assume we could be on any + // CPU). + ptr: &'a PerCpuPtr, +} + +macro_rules! impl_ops { + ($ty:ty, $reg:tt) => { + impl DynamicPerCpu<$ty> { + /// Returns a [`PerCpuNumeric`] that can be used to manipulate the underlying per-CPU + /// variable. + #[inline] + pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> { + // The invariant is satisfied because `DynamicPerCpu`'s invariant guarantees that + // this pointer is valid and initialized on all CPUs. + PerCpuNumeric { ptr: &self.alloc().0 } + } + } + impl StaticPerCpu<$ty> { + /// Returns a [`PerCpuNumeric`] that can be used to manipulate the underlying per-CPU + /// variable. + #[inline] + pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> { + // The invariant is satisfied because `StaticPerCpu`'s invariant guarantees that + // this pointer is valid and initialized on all CPUs. + PerCpuNumeric { ptr: &self.0 } + } + } + + impl PerCpuNumeric<'_, $ty> { + /// Adds `rhs` to the per-CPU variable. + #[inline] + pub fn add(&mut self, rhs: $ty) { + // SAFETY: `self.ptr.0` is a valid offset into the per-CPU area (i.e., valid as a + // pointer relative to the `gs` segment register) by the invariants of this type. + unsafe { + asm!( + concat!("add gs:[{off}], {val:", $reg, "}"), + off = in(reg) self.ptr.0.cast::<*mut $ty>(), + val = in(reg) rhs, + ); + } + } + } + impl PerCpuNumeric<'_, $ty> { + /// Subtracts `rhs` from the per-CPU variable. + #[inline] + pub fn sub(&mut self, rhs: $ty) { + // SAFETY: `self.ptr.0` is a valid offset into the per-CPU area (i.e., valid as a + // pointer relative to the `gs` segment register) by the invariants of this type. + unsafe { + asm!( + concat!("sub gs:[{off}], {val:", $reg, "}"), + off = in(reg) self.ptr.0.cast::<*mut $ty>(), + val = in(reg) rhs, + ); + } + } + } + }; +} + +macro_rules! impl_ops_byte { + ($ty:ty) => { + impl DynamicPerCpu<$ty> { + /// Returns a [`PerCpuNumeric`] that can be used to manipulate the underlying per-CPU + /// variable. + #[inline] + pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> { + // The invariant is satisfied because `DynamicPerCpu`'s invariant guarantees that + // this pointer is valid and initialized on all CPUs. + PerCpuNumeric { ptr: &self.alloc().0 } + } + } + impl StaticPerCpu<$ty> { + /// Returns a [`PerCpuNumeric`] that can be used to manipulate the underlying per-CPU + /// variable. + #[inline] + pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> { + // The invariant is satisfied because `DynamicPerCpu`'s invariant guarantees that + // this pointer is valid and initialized on all CPUs. + PerCpuNumeric { ptr: &self.0 } + } + } + + impl PerCpuNumeric<'_, $ty> { + /// Adds `rhs` to the per-CPU variable. + #[inline] + pub fn add(&mut self, rhs: $ty) { + // SAFETY: `self.ptr.0` is a valid offset into the per-CPU area (i.e., valid as a + // pointer relative to the `gs` segment register) by the invariants of this type. + unsafe { + asm!( + concat!("add gs:[{off}], {val}"), + off = in(reg) self.ptr.0.cast::<*mut $ty>(), + val = in(reg_byte) rhs, + ); + } + } + } + impl PerCpuNumeric<'_, $ty> { + /// Subtracts `rhs` from the per-CPU variable. + #[inline] + pub fn sub(&mut self, rhs: $ty) { + // SAFETY: `self.ptr.0` is a valid offset into the per-CPU area (i.e., valid as a + // pointer relative to the `gs` segment register) by the invariants of this type. + unsafe { + asm!( + concat!("sub gs:[{off}], {val}"), + off = in(reg) self.ptr.0.cast::<*mut $ty>(), + val = in(reg_byte) rhs, + ); + } + } + } + }; +} + +impl_ops_byte!(i8); +impl_ops!(i16, "x"); +impl_ops!(i32, "e"); +impl_ops!(i64, "r"); +impl_ops!(isize, "r"); + +impl_ops_byte!(u8); +impl_ops!(u16, "x"); +impl_ops!(u32, "e"); +impl_ops!(u64, "r"); +impl_ops!(usize, "r"); diff --git a/samples/rust/rust_percpu.rs b/samples/rust/rust_percpu.rs index be70ee2e513f..31ab3fcf5c6c 100644 --- a/samples/rust/rust_percpu.rs +++ b/samples/rust/rust_percpu.rs @@ -28,6 +28,26 @@ define_per_cpu!(UPERCPU: u64 = 0); define_per_cpu!(CHECKED: RefCell = RefCell::new(0)); +macro_rules! make_optimization_test { + ($ty:ty) => { + let mut test: DynamicPerCpu<$ty> = DynamicPerCpu::new_zero(GFP_KERNEL).unwrap(); + { + let _ = CpuGuard::new(); + // SAFETY: No other usage of `test` + unsafe { test.get_mut(CpuGuard::new()) }.with(|val: &mut $ty| *val = 10); + test.num().add(1); + // SAFETY: No other usage of `test` + unsafe { test.get_mut(CpuGuard::new()) }.with(|val: &mut $ty| assert_eq!(*val, 11)); + test.num().add(10); + // SAFETY: No other usage of `test` + unsafe { test.get_mut(CpuGuard::new()) }.with(|val: &mut $ty| assert_eq!(*val, 21)); + test.num().sub(5); + // SAFETY: No other usage of `test` + unsafe { test.get_mut(CpuGuard::new()) }.with(|val: &mut $ty| assert_eq!(*val, 16)); + } + }; +} + impl kernel::Module for PerCpuMod { fn init(_module: &'static ThisModule) -> Result { pr_info!("rust percpu test start\n"); @@ -208,6 +228,22 @@ fn init(_module: &'static ThisModule) -> Result { pr_info!("rust dynamic percpu test done\n"); + pr_info!("rust numeric optimizations test start\n"); + + make_optimization_test!(u8); + make_optimization_test!(u16); + make_optimization_test!(u32); + make_optimization_test!(u64); + make_optimization_test!(usize); + + make_optimization_test!(i8); + make_optimization_test!(i16); + make_optimization_test!(i32); + make_optimization_test!(i64); + make_optimization_test!(isize); + + pr_info!("rust numeric optimizations test done\n"); + // Return Err to unload the module Result::Err(EINVAL) } -- 2.34.1 Currently, the creation of a `PerCpuNumeric` requires a memory read via the `Arc` managing the dynamic allocation. While the compiler might be clever enough to consolidate these reads in some cases, the read must happen *somewhere*, which, when we're concerning ourselves with individual instructions, is a very high burden. Instead, cache the `PerCpuPointer` inside the `DynamicPerCpu` structure; then, the `Arc` is used solely to manage the allocation. Signed-off-by: Mitchell Levy --- rust/kernel/percpu/dynamic.rs | 22 ++++++++++++---------- rust/kernel/percpu/numeric.rs | 4 ++-- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/rust/kernel/percpu/dynamic.rs b/rust/kernel/percpu/dynamic.rs index 99acbf6363f5..dcf3e1c4f7a9 100644 --- a/rust/kernel/percpu/dynamic.rs +++ b/rust/kernel/percpu/dynamic.rs @@ -72,6 +72,9 @@ pub struct DynamicPerCpu { // INVARIANT: The memory location in each CPU's per-CPU area pointed at by the alloc is // initialized. alloc: Option>>, + // INVARIANT: `ptr` is the per-CPU pointer managed by `alloc`, which does not change for the + // lifetime of `self`. + pub(super) ptr: PerCpuPtr, } impl DynamicPerCpu { @@ -83,9 +86,13 @@ impl DynamicPerCpu { pub fn new_zero(flags: Flags) -> Option { let alloc: PerCpuAllocation = PerCpuAllocation::new_zero()?; + let ptr = alloc.0; let arc = Arc::new(alloc, flags).ok()?; - Some(Self { alloc: Some(arc) }) + Some(Self { + alloc: Some(arc), + ptr, + }) } } @@ -115,15 +122,10 @@ pub fn new_with(val: &T, flags: Flags) -> Option { let arc = Arc::new(alloc, flags).ok()?; - Some(Self { alloc: Some(arc) }) - } -} - -impl DynamicPerCpu { - /// Gets the allocation backing this per-CPU variable. - pub(crate) fn alloc(&self) -> &Arc> { - // SAFETY: This type's invariant ensures that `self.alloc` is `Some`. - unsafe { self.alloc.as_ref().unwrap_unchecked() } + Some(Self { + alloc: Some(arc), + ptr, + }) } } diff --git a/rust/kernel/percpu/numeric.rs b/rust/kernel/percpu/numeric.rs index e76461f05c66..23a7a09216d0 100644 --- a/rust/kernel/percpu/numeric.rs +++ b/rust/kernel/percpu/numeric.rs @@ -22,7 +22,7 @@ impl DynamicPerCpu<$ty> { pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> { // The invariant is satisfied because `DynamicPerCpu`'s invariant guarantees that // this pointer is valid and initialized on all CPUs. - PerCpuNumeric { ptr: &self.alloc().0 } + PerCpuNumeric { ptr: &self.ptr } } } impl StaticPerCpu<$ty> { @@ -78,7 +78,7 @@ impl DynamicPerCpu<$ty> { pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> { // The invariant is satisfied because `DynamicPerCpu`'s invariant guarantees that // this pointer is valid and initialized on all CPUs. - PerCpuNumeric { ptr: &self.alloc().0 } + PerCpuNumeric { ptr: &self.ptr } } } impl StaticPerCpu<$ty> { -- 2.34.1