Bracket the call out to the ftrace_ops callback in ftrace_caller and ftrace_regs_caller with an increment/decrement of current->rcu_tramp_nesting. The instructions sit inside the region that create_trampoline() copies for per-ops dynamic trampolines, so those inherit them; the %rip-relative per-CPU reference to current_task is fixed up by text_poke_apply_relocation() like CALL_DEPTH_ACCOUNT's. %rdx is dead at both points (about to be loaded with the ops pointer on entry, restored by restore_mcount_regs on exit). Two pieces of core text still run with the count at zero while holding the address of a Tasks-RCU-protected trampoline they are about to enter: the static stubs themselves, whose direct-call tails keep a BPF trampoline address on the stack until the final RET, and, under CONFIG_MITIGATION_RETHUNK, the return thunk that RET expands to. Add an ftrace_static_tramp_end marker after ftrace_stub_direct_tramp and linker symbols around .text..__x86.return_thunk and .text..__x86.rethunk_safe, and provide arch_rcu_tasks_ip_in_trampoline() covering [ftrace_caller, ftrace_static_tramp_end) and both thunk ranges so the irq-exit check treats a task interrupted there as still inside a trampoline. The hook is built only under CONFIG_RCU_TASKS_PREEMPT_QS, which x86 does not select until a later patch. Assisted-by: LLM Signed-off-by: Josef Bacik --- arch/x86/kernel/asm-offsets.c | 3 +++ arch/x86/kernel/ftrace.c | 37 +++++++++++++++++++++++++++++++++++++ arch/x86/kernel/ftrace_64.S | 43 +++++++++++++++++++++++++++++++++++++++++++ arch/x86/kernel/vmlinux.lds.S | 4 ++++ 4 files changed, 87 insertions(+) diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c index 081816888f7a..4f3b1caa5a30 100644 --- a/arch/x86/kernel/asm-offsets.c +++ b/arch/x86/kernel/asm-offsets.c @@ -46,6 +46,9 @@ static void __used common(void) #ifdef CONFIG_STACKPROTECTOR OFFSET(TASK_stack_canary, task_struct, stack_canary); #endif +#ifdef CONFIG_TASKS_RCU + OFFSET(TASK_rcu_tramp_nesting, task_struct, rcu_tramp_nesting); +#endif BLANK(); OFFSET(pbe_address, pbe, address); diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c index 17d6edfcb7e0..8f63cd4b543c 100644 --- a/arch/x86/kernel/ftrace.c +++ b/arch/x86/kernel/ftrace.c @@ -275,6 +275,43 @@ static inline void tramp_free(void *tramp) execmem_free(tramp); } +#ifdef CONFIG_RCU_TASKS_PREEMPT_QS +extern void ftrace_static_tramp_end(void); +extern char __return_thunk_start[], __return_thunk_end[]; +extern char __rethunk_safe_start[], __rethunk_safe_end[]; + +/* + * See rcu_tasks_ip_in_trampoline(). Some core kernel text behaves like a + * trampoline for Tasks RCU purposes because a task executing there with + * rcu_tramp_nesting == 0 may still be about to enter a Tasks-RCU-protected + * trampoline whose address it already holds: + * + * - the static ftrace_caller / ftrace_regs_caller / ftrace_stub_direct_tramp + * stubs, which carry a direct-call target on the stack until their final + * RET, and + * - the return thunks that RET expands to under CONFIG_MITIGATION_RETHUNK, + * which run after leaving the stubs above and before landing in that + * target. + */ +bool arch_rcu_tasks_ip_in_trampoline(unsigned long ip) +{ + if (ip >= (unsigned long)ftrace_caller && + ip < (unsigned long)ftrace_static_tramp_end) + return true; +#ifdef CONFIG_MITIGATION_RETPOLINE + if (ip >= (unsigned long)__return_thunk_start && + ip < (unsigned long)__return_thunk_end) + return true; +#endif +#ifdef CONFIG_MITIGATION_SRSO + if (ip >= (unsigned long)__rethunk_safe_start && + ip < (unsigned long)__rethunk_safe_end) + return true; +#endif + return false; +} +#endif /* CONFIG_RCU_TASKS_PREEMPT_QS */ + /* Defined as markers to the end of the ftrace default trampolines */ extern void ftrace_regs_caller_end(void); extern void ftrace_caller_end(void); diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S index 62c1c93aa1c6..902472c41798 100644 --- a/arch/x86/kernel/ftrace_64.S +++ b/arch/x86/kernel/ftrace_64.S @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -145,6 +146,27 @@ SYM_FUNC_END(ftrace_stub_graph) #ifdef CONFIG_DYNAMIC_FTRACE +/* + * Tasks RCU trampoline nesting, see rcu_tasks_trampoline_enter(). These live + * inside the region copied into dynamic trampolines; the %rip-relative per-CPU + * reference is fixed up by text_poke_apply_relocation() in create_trampoline(). + * The increment must precede the function_trace_op load: between that load and + * the call, the ops pointer in %rdx is protected only by Tasks RCU. + */ +.macro RCU_TASKS_TRAMP_ENTER reg:req +#ifdef CONFIG_TASKS_RCU + movq PER_CPU_VAR(current_task), \reg + incl TASK_rcu_tramp_nesting(\reg) +#endif +.endm + +.macro RCU_TASKS_TRAMP_EXIT reg:req +#ifdef CONFIG_TASKS_RCU + movq PER_CPU_VAR(current_task), \reg + decl TASK_rcu_tramp_nesting(\reg) +#endif +.endm + SYM_FUNC_START(__fentry__) ANNOTATE_NOENDBR CALL_DEPTH_ACCOUNT @@ -163,6 +185,8 @@ SYM_FUNC_START(ftrace_caller) leaq MCOUNT_REG_SIZE+8(%rsp), %rcx movq %rcx, RSP(%rsp) + RCU_TASKS_TRAMP_ENTER %rdx + SYM_INNER_LABEL(ftrace_caller_op_ptr, SYM_L_GLOBAL) ANNOTATE_NOENDBR /* Load the ftrace_ops into the 3rd parameter */ @@ -181,6 +205,8 @@ SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) ANNOTATE_NOENDBR call ftrace_stub + RCU_TASKS_TRAMP_EXIT %rdx + /* Handlers can change the RIP */ movq RIP(%rsp), %rax movq %rax, MCOUNT_REG_SIZE(%rsp) @@ -209,6 +235,8 @@ SYM_FUNC_START(ftrace_regs_caller) CALL_DEPTH_ACCOUNT + RCU_TASKS_TRAMP_ENTER %rdx + SYM_INNER_LABEL(ftrace_regs_caller_op_ptr, SYM_L_GLOBAL) ANNOTATE_NOENDBR /* Load the ftrace_ops into the 3rd parameter */ @@ -246,6 +274,8 @@ SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL) ANNOTATE_NOENDBR call ftrace_stub + RCU_TASKS_TRAMP_EXIT %rdx + /* Copy flags back to SS, to restore them */ movq EFLAGS(%rsp), %rax movq %rax, MCOUNT_REG_SIZE(%rsp) @@ -328,6 +358,19 @@ SYM_FUNC_START(ftrace_stub_direct_tramp) RET SYM_FUNC_END(ftrace_stub_direct_tramp) +/* + * [ftrace_caller, ftrace_static_tramp_end) is treated as trampoline text by + * rcu_tasks_ip_in_trampoline(): after RCU_TASKS_TRAMP_EXIT the stubs may + * still hold a direct-call target (a BPF trampoline) on the stack until the + * final RET, and that target's lifetime is guarded by Tasks RCU. With + * return thunks the RET itself runs elsewhere; arch_rcu_tasks_ip_in_trampoline() + * covers the thunk text too. + */ +SYM_CODE_START_NOALIGN(ftrace_static_tramp_end) + UNWIND_HINT_UNDEFINED + ANNOTATE_NOENDBR +SYM_CODE_END(ftrace_static_tramp_end) + #else /* ! CONFIG_DYNAMIC_FTRACE */ SYM_FUNC_START(__fentry__) diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index 2438b89a4620..e546283dc267 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -151,7 +151,9 @@ SECTIONS * definition. */ . = srso_alias_untrain_ret | (1 << 2) | (1 << 8) | (1 << 14) | (1 << 20); + __rethunk_safe_start = .; *(.text..__x86.rethunk_safe) + __rethunk_safe_end = .; #endif ALIGN_ENTRY_TEXT_END @@ -162,7 +164,9 @@ SECTIONS SOFTIRQENTRY_TEXT #ifdef CONFIG_MITIGATION_RETPOLINE *(.text..__x86.indirect_thunk) + __return_thunk_start = .; *(.text..__x86.return_thunk) + __return_thunk_end = .; #endif STATIC_CALL_TEXT *(.gnu.warning) -- 2.55.0