Currently, the creation of a `PerCpuNumeric` requires a memory read via the `Arc` managing the dynamic allocation. While the compiler might be clever enough to consolidate these reads in some cases, the read must happen *somewhere*, which, when we're concerning ourselves with individual instructions, is a very high burden. Instead, cache the `PerCpuPointer` inside the `DynamicPerCpu` structure; then, the `Arc` is used solely to manage the allocation. Signed-off-by: Mitchell Levy --- rust/kernel/percpu/dynamic.rs | 8 ++++++-- rust/kernel/percpu/numeric.rs | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/rust/kernel/percpu/dynamic.rs b/rust/kernel/percpu/dynamic.rs index aad08e4b4251..8683e94bcd9f 100644 --- a/rust/kernel/percpu/dynamic.rs +++ b/rust/kernel/percpu/dynamic.rs @@ -58,7 +58,10 @@ fn drop(&mut self) { pub struct DynamicPerCpu { // INVARIANT: The memory location in each CPU's per-CPU area pointed at by `alloc.0` has been // initialized. + // INVARIANT: `ptr` is the per-CPU pointer managed by `alloc`, which does not change for the + // lifetime of `self`. pub(super) alloc: Arc>, + pub(super) ptr: PerCpuPtr, } impl DynamicPerCpu { @@ -70,9 +73,10 @@ impl DynamicPerCpu { pub fn new_zero(flags: Flags) -> Option { let alloc: PerCpuAllocation = PerCpuAllocation::new_zero()?; + let ptr = alloc.0; let arc = Arc::new(alloc, flags).ok()?; - Some(Self { alloc: arc }) + Some(Self { alloc: arc, ptr }) } } @@ -102,7 +106,7 @@ pub fn new_with(val: T, flags: Flags) -> Option { let arc = Arc::new(alloc, flags).ok()?; - Some(Self { alloc: arc }) + Some(Self { alloc: arc, ptr }) } } diff --git a/rust/kernel/percpu/numeric.rs b/rust/kernel/percpu/numeric.rs index 4de93f653f0e..3412c2de85fd 100644 --- a/rust/kernel/percpu/numeric.rs +++ b/rust/kernel/percpu/numeric.rs @@ -20,7 +20,7 @@ impl DynamicPerCpu<$ty> { pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> { // The invariant is satisfied because `DynamicPerCpu`'s invariant guarantees that // this pointer is valid and initialized on all CPUs. - PerCpuNumeric { ptr: &self.alloc.0 } + PerCpuNumeric { ptr: &self.ptr } } } impl StaticPerCpu<$ty> { @@ -71,7 +71,7 @@ impl DynamicPerCpu<$ty> { pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> { // The invariant is satisfied because `DynamicPerCpu`'s invariant guarantees that // this pointer is valid and initialized on all CPUs. - PerCpuNumeric { ptr: &self.alloc.0 } + PerCpuNumeric { ptr: &self.ptr } } } impl StaticPerCpu<$ty> { -- 2.34.1