Bracket the call out to the ftrace_ops callback in ftrace_caller with an increment/decrement of current->rcu_tramp_nesting, using x12/w13 which are scratch there. The read-modify-write is not atomic, but only current modifies the count and every interrupting user is balanced, so nothing is lost. ftrace_caller itself, including the early CALL_OPS direct path and the late direct tail that carry a BPF trampoline address in x17 with the count at zero, is static kernel text: add an ftrace_static_tramp_end marker after ftrace_stub_direct_tramp and provide arch_rcu_tasks_ip_in_trampoline() covering [ftrace_caller, ftrace_static_tramp_end) so the irq-exit check treats a task interrupted anywhere in it as inside a trampoline. The hook is built only under CONFIG_RCU_TASKS_PREEMPT_QS, which arm64 does not select until a later patch. Assisted-by: LLM Signed-off-by: Josef Bacik --- arch/arm64/kernel/asm-offsets.c | 3 +++ arch/arm64/kernel/entry-ftrace.S | 35 +++++++++++++++++++++++++++++++++++ arch/arm64/kernel/ftrace.c | 16 ++++++++++++++++ 3 files changed, 54 insertions(+) diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index 9c853ed3ceab..f6655a284f18 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -39,6 +39,9 @@ int main(void) DEFINE(TSK_STACK, offsetof(struct task_struct, stack)); #ifdef CONFIG_STACKPROTECTOR DEFINE(TSK_STACK_CANARY, offsetof(struct task_struct, stack_canary)); +#endif +#ifdef CONFIG_TASKS_RCU + DEFINE(TSK_RCU_TRAMP_NESTING, offsetof(struct task_struct, rcu_tramp_nesting)); #endif BLANK(); DEFINE(THREAD_CPU_CONTEXT, offsetof(struct task_struct, thread.cpu_context)); diff --git a/arch/arm64/kernel/entry-ftrace.S b/arch/arm64/kernel/entry-ftrace.S index 025140caafe7..46a102e7199a 100644 --- a/arch/arm64/kernel/entry-ftrace.S +++ b/arch/arm64/kernel/entry-ftrace.S @@ -14,6 +14,33 @@ #include #ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS +/* + * Tasks RCU trampoline nesting, see rcu_tasks_trampoline_enter(). The whole + * of ftrace_caller is treated as trampoline text by the irq-exit IP check (see + * arch_rcu_tasks_ip_in_trampoline()), so these only need to bracket the call + * out to ops->func; everything before the increment and after the decrement, + * including the direct-call tails that carry a BPF trampoline address in x17, + * is covered by that. The count is only modified by current and every nested + * user (interrupts) is balanced, so a plain ldr/add/str is sufficient. + */ + .macro rcu_tasks_tramp_enter, tsk:req, tmp:req +#ifdef CONFIG_TASKS_RCU + mrs \tsk, sp_el0 + ldr \tmp, [\tsk, #TSK_RCU_TRAMP_NESTING] + add \tmp, \tmp, #1 + str \tmp, [\tsk, #TSK_RCU_TRAMP_NESTING] +#endif + .endm + + .macro rcu_tasks_tramp_exit, tsk:req, tmp:req +#ifdef CONFIG_TASKS_RCU + mrs \tsk, sp_el0 + ldr \tmp, [\tsk, #TSK_RCU_TRAMP_NESTING] + sub \tmp, \tmp, #1 + str \tmp, [\tsk, #TSK_RCU_TRAMP_NESTING] +#endif + .endm + /* * Due to -fpatchable-function-entry=2, the compiler has placed two NOPs before * the regular function prologue. For an enabled callsite, ftrace_init_nop() and @@ -94,6 +121,8 @@ SYM_CODE_START(ftrace_caller) stp x29, x30, [sp, #FREGS_SIZE] add x29, sp, #FREGS_SIZE + rcu_tasks_tramp_enter x12, w13 + /* Prepare arguments for the tracer func */ sub x0, x30, #AARCH64_INSN_SIZE // ip (callsite's BL insn) mov x1, x9 // parent_ip (callsite's LR) @@ -111,6 +140,8 @@ SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) bl ftrace_stub // func(ip, parent_ip, op, regs) #endif + rcu_tasks_tramp_exit x12, w13 + /* * At the callsite x0-x8 and x19-x30 were live. Any C code will have preserved * x19-x29 per the AAPCS, and we created frame records upon entry, so we need @@ -178,6 +209,10 @@ SYM_CODE_START(ftrace_stub_direct_tramp) SYM_CODE_END(ftrace_stub_direct_tramp) #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ +/* End of [ftrace_caller, ...) for arch_rcu_tasks_ip_in_trampoline(). */ +SYM_CODE_START(ftrace_static_tramp_end) +SYM_CODE_END(ftrace_static_tramp_end) + #else /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */ /* diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c index e1a3c0b3a051..1b7ac2afed0d 100644 --- a/arch/arm64/kernel/ftrace.c +++ b/arch/arm64/kernel/ftrace.c @@ -17,6 +17,22 @@ #include #include +#ifdef CONFIG_RCU_TASKS_PREEMPT_QS +extern void ftrace_static_tramp_end(void); + +/* + * See rcu_tasks_ip_in_trampoline(). ftrace_caller and ftrace_stub_direct_tramp + * are core kernel text but must be treated as trampolines: a task preempted in + * them may be carrying an ops pointer (x11) or a direct-call BPF trampoline + * address (x17) whose lifetime is guarded only by Tasks RCU. + */ +bool arch_rcu_tasks_ip_in_trampoline(unsigned long ip) +{ + return ip >= (unsigned long)ftrace_caller && + ip < (unsigned long)ftrace_static_tramp_end; +} +#endif + #ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS struct fregs_offset { const char *name; -- 2.55.0