Tasks RCU exists so that ftrace, BPF and kprobes can free trampoline text once no task can still be executing in it. Today the only way a task tells Tasks RCU "I am not in a trampoline" is a voluntary context switch, so a preempted task is always assumed to be inside one. Add task_struct::rcu_tramp_nesting so that trampolines can say so directly: a trampoline increments it before calling out and decrements it before returning, and while it is non-zero the task must not be treated as Tasks-RCU quiescent. Provide rcu_tasks_trampoline_enter() and rcu_tasks_trampoline_exit() for C users, report the count in the Tasks RCU stall output, and, under CONFIG_PROVE_RCU, assert that it is zero on every return to userspace since no task can legitimately reach userspace with a trampoline on its stack. Only current ever writes the count and every nested user (interrupts running their own trampolines) is balanced, so plain accesses suffice. The callbacks reached from static trampolines (return_to_handler, the rethook and kretprobe trampolines) are covered by the preempt_disable() in the ftrace recursion protection rather than by the count; note that dependency in trace_recursion.h so it is not lost if the preempt_disable() is ever removed from there. Nothing increments the count and nothing consults it for quiescent-state decisions yet; both come in later patches. Assisted-by: LLM Signed-off-by: Josef Bacik --- include/linux/irq-entry-common.h | 2 ++ include/linux/rcupdate.h | 37 +++++++++++++++++++++++++++++++++++++ include/linux/sched.h | 1 + include/linux/trace_recursion.h | 11 +++++++++++ kernel/fork.c | 1 + kernel/rcu/tasks.h | 3 ++- 6 files changed, 54 insertions(+), 1 deletion(-) diff --git a/include/linux/irq-entry-common.h b/include/linux/irq-entry-common.h index 0bb6c03481fa..8da571622000 100644 --- a/include/linux/irq-entry-common.h +++ b/include/linux/irq-entry-common.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -214,6 +215,7 @@ static __always_inline void __exit_to_user_mode_validate(void) { /* Ensure that kernel state is sane for a return to userspace */ kmap_assert_nomap(); + rcu_tasks_trampoline_assert_none(); lockdep_assert_irqs_disabled(); lockdep_sys_exit(); } diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 44c07a66edff..b5c666c82479 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -180,6 +180,37 @@ static inline void rcu_nocb_flush_deferred_wakeup(void) { } #ifdef CONFIG_TASKS_RCU_GENERIC # ifdef CONFIG_TASKS_RCU + +/* + * Trampoline nesting: dynamically allocated text (ftrace trampolines, BPF + * trampoline images, kprobe optinsn slots) that relies on Tasks RCU for its + * lifetime brackets itself with an increment/decrement of + * current->rcu_tramp_nesting. While the count is non-zero the task is inside, + * or was called from, such text and an involuntary context switch must not be + * treated as a Tasks RCU quiescent state. + * + * Only current writes the count and only current (or an interrupt on the same + * CPU) reads it, so plain accesses suffice. + */ +static __always_inline void rcu_tasks_trampoline_enter(void) +{ + current->rcu_tramp_nesting++; + barrier(); +} + +static __always_inline void rcu_tasks_trampoline_exit(void) +{ + barrier(); + current->rcu_tramp_nesting--; +} + +/* A task must never reach userspace with a trampoline on its stack. */ +static __always_inline void rcu_tasks_trampoline_assert_none(void) +{ + if (IS_ENABLED(CONFIG_PROVE_RCU)) + WARN_ON_ONCE(current->rcu_tramp_nesting); +} + # define rcu_tasks_classic_qs(t, preempt) \ do { \ if (!(preempt) && READ_ONCE((t)->rcu_tasks_holdout)) \ @@ -192,6 +223,9 @@ void rcu_tasks_torture_stats_print(char *tt, char *tf); # define rcu_tasks_classic_qs(t, preempt) do { } while (0) # define call_rcu_tasks call_rcu # define synchronize_rcu_tasks synchronize_rcu +static inline void rcu_tasks_trampoline_enter(void) { } +static inline void rcu_tasks_trampoline_exit(void) { } +static inline void rcu_tasks_trampoline_assert_none(void) { } # endif #define rcu_tasks_qs(t, preempt) rcu_tasks_classic_qs((t), (preempt)) @@ -208,6 +242,9 @@ void exit_tasks_rcu_finish(void); #define rcu_tasks_classic_qs(t, preempt) do { } while (0) #define rcu_tasks_qs(t, preempt) do { } while (0) #define rcu_note_voluntary_context_switch(t) do { } while (0) +static inline void rcu_tasks_trampoline_enter(void) { } +static inline void rcu_tasks_trampoline_exit(void) { } +static inline void rcu_tasks_trampoline_assert_none(void) { } #define call_rcu_tasks call_rcu #define synchronize_rcu_tasks synchronize_rcu static inline void exit_tasks_rcu_start(void) { } diff --git a/include/linux/sched.h b/include/linux/sched.h index 8b3d47a325cc..d2e7b1b3c9d2 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -956,6 +956,7 @@ struct task_struct { unsigned long rcu_tasks_nvcsw; u8 rcu_tasks_holdout; u8 rcu_tasks_idx; + int rcu_tramp_nesting; int rcu_tasks_idle_cpu; struct list_head rcu_tasks_holdout_list; int rcu_tasks_exit_cpu; diff --git a/include/linux/trace_recursion.h b/include/linux/trace_recursion.h index e6ca052b2a85..2da23a52ca4a 100644 --- a/include/linux/trace_recursion.h +++ b/include/linux/trace_recursion.h @@ -153,6 +153,17 @@ static __always_inline int trace_test_and_set_recursion(unsigned long ip, unsign current->trace_recursion = val; barrier(); + /* + * Callbacks reached from static trampoline text (return_to_handler, + * the rethook and kretprobe trampolines) do not maintain + * current->rcu_tramp_nesting themselves; they rely on this + * preempt_disable() to keep the task from being preempted, and thus + * from reporting a Tasks RCU quiescent state, while an ftrace_ops or + * its data is in use. If the preempt_disable() is ever removed from + * the recursion protection, this must rcu_tasks_trampoline_enter() + * here and rcu_tasks_trampoline_exit() in trace_clear_recursion() + * instead. See CONFIG_RCU_TASKS_PREEMPT_QS. + */ preempt_disable_notrace(); return bit; diff --git a/kernel/fork.c b/kernel/fork.c index 416758c8a3d4..cfe3a8e53fbd 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1869,6 +1869,7 @@ static inline void rcu_copy_process(struct task_struct *p) #endif /* #ifdef CONFIG_PREEMPT_RCU */ #ifdef CONFIG_TASKS_RCU p->rcu_tasks_holdout = false; + p->rcu_tramp_nesting = 0; INIT_LIST_HEAD(&p->rcu_tasks_holdout_list); p->rcu_tasks_idle_cpu = -1; INIT_LIST_HEAD(&p->rcu_tasks_exit_list); diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 627295396cd9..1662ba18bf34 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1113,10 +1113,11 @@ static void check_holdout_task(struct task_struct *t, *firstreport = false; } cpu = task_cpu(t); - pr_alert("%p: %c%c nvcsw: %lu/%lu holdout: %d idle_cpu: %d/%d\n", + pr_alert("%p: %c%c nvcsw: %lu/%lu holdout: %d tramp_nesting: %d idle_cpu: %d/%d\n", t, ".I"[is_idle_task(t)], "N."[cpu < 0 || !tick_nohz_full_cpu(cpu)], t->rcu_tasks_nvcsw, t->nvcsw, t->rcu_tasks_holdout, + data_race(t->rcu_tramp_nesting), data_race(t->rcu_tasks_idle_cpu), cpu); sched_show_task(t); } -- 2.55.0