Replace the out-of-line percpu_counter_tree_items_size() function with a static inline that loads the __ro_after_init variable __percpu_counter_tree_items_size directly. This eliminates a function call on paths that compute mm_struct flexible array layout offsets. Initialize the variable to SIZE_MAX as a sentinel value, and add a WARN_ON_ONCE check in the inline accessor to catch any accidental use before percpu_counter_tree_subsystem_init() sets the real value based on nr_cpu_ids. An early misuse would produce an absurd offset that triggers the warning with a stack trace pointing at the offending caller, rather than silently computing wrong memory offsets. percpu_counter_tree_subsystem_init() sets the variable to the runtime value immediately after deriving counter_config from nr_cpu_ids, before calculate_accuracy_topology(). After boot, __ro_after_init moves the variable into read-only memory. On !CONFIG_SMP, the existing inline returning 0 is unchanged and requires no sentinel. Signed-off-by: Mathieu Desnoyers Cc: "Paul E. McKenney" Cc: Steven Rostedt Cc: Masami Hiramatsu Cc: Dennis Zhou Cc: Tejun Heo Cc: Christoph Lameter Cc: Martin Liu Cc: David Rientjes Cc: christian.koenig@amd.com Cc: Shakeel Butt Cc: SeongJae Park Cc: Michal Hocko Cc: Johannes Weiner Cc: Sweet Tea Dorminy Cc: Lorenzo Stoakes Cc: Liam R. Howlett Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: Christian Brauner Cc: Wei Yang Cc: David Hildenbrand Cc: Miaohe Lin Cc: Al Viro Cc: Yu Zhao Cc: Roman Gushchin Cc: Mateusz Guzik Cc: Matthew Wilcox Cc: Baolin Wang Cc: Aboorva Devarajan Cc: David Carlier Cc: Josh Law Cc: Andrew Morton Cc: linux-mm@kvack.org --- include/linux/percpu_counter_tree.h | 19 ++++++++++++++++++- lib/percpu_counter_tree.c | 24 ++++++++---------------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/include/linux/percpu_counter_tree.h b/include/linux/percpu_counter_tree.h index 3e8a820e2d1d..4095acdc879b 100644 --- a/include/linux/percpu_counter_tree.h +++ b/include/linux/percpu_counter_tree.h @@ -101,7 +101,24 @@ struct percpu_counter_tree { } approx_accuracy_range; }; -size_t percpu_counter_tree_items_size(void); +extern size_t __percpu_counter_tree_items_size __ro_after_init; + +/* + * percpu_counter_tree_items_size - Query the size required for counter tree items. + * + * Query the size of the memory area required to hold the counter tree + * items. This depends on the hardware topology and is invariant after + * boot. + * + * Return: Size required to hold tree items. + */ +static inline +size_t percpu_counter_tree_items_size(void) +{ + WARN_ON_ONCE(__percpu_counter_tree_items_size == SIZE_MAX); + return __percpu_counter_tree_items_size; +} + int percpu_counter_tree_init_many(struct percpu_counter_tree *counters, struct percpu_counter_tree_level_item *items, unsigned int nr_counters, unsigned long batch_size, gfp_t gfp_flags); int percpu_counter_tree_init(struct percpu_counter_tree *counter, struct percpu_counter_tree_level_item *items, diff --git a/lib/percpu_counter_tree.c b/lib/percpu_counter_tree.c index 5c8fc2dcdc16..cfa5b04e05d7 100644 --- a/lib/percpu_counter_tree.c +++ b/lib/percpu_counter_tree.c @@ -653,22 +653,9 @@ void percpu_counter_tree_set(struct percpu_counter_tree *counter, long v) } EXPORT_SYMBOL_GPL(percpu_counter_tree_set); -/* - * percpu_counter_tree_items_size - Query the size required for counter tree items. - * - * Query the size of the memory area required to hold the counter tree - * items. This depends on the hardware topology and is invariant after - * boot. - * - * Return: Size required to hold tree items. - */ -size_t percpu_counter_tree_items_size(void) -{ - if (!nr_cpus_order) - return 0; - return counter_config->nr_items * sizeof(struct percpu_counter_tree_level_item); -} -EXPORT_SYMBOL_GPL(percpu_counter_tree_items_size); +/* Initialize to SIZE_MAX to catch early boot misuses. */ +size_t __percpu_counter_tree_items_size __ro_after_init = SIZE_MAX; +EXPORT_SYMBOL_GPL(__percpu_counter_tree_items_size); static void __init calculate_accuracy_topology(void) { @@ -697,6 +684,11 @@ int __init percpu_counter_tree_subsystem_init(void) return -1; } counter_config = &per_nr_cpu_order_config[nr_cpus_order]; + if (!nr_cpus_order) + __percpu_counter_tree_items_size = 0; + else + __percpu_counter_tree_items_size = counter_config->nr_items * + sizeof(struct percpu_counter_tree_level_item); calculate_accuracy_topology(); return 0; } -- 2.43.0