RISC-V does not provide architecture-specific this_cpu operations, so read-modify-write operations use generic fallbacks that disable local interrupts. Disabling preemption around each architecture-specific operation would avoid the interrupt-off region, but would still add preemption-count bookkeeping and a reschedule check to the common path. Use the recoverable-GPR scheme employed by arm64 so the operations can remain preemptible. Record the registers used for per-CPU address generation and repair them on exception return if the task migrated. Exception entry saves and clears the active state so handlers can use per-CPU operations independently. Keep this protocol in common assembly macros so NMI-like entry paths which bypass handle_exception, such as SBI Supervisor Software Events, can preserve an interrupted operation. The existing exception return path clears LR/SC reservations, so an interrupted LR/SC sequence retries after its saved address is repaired. Suggested-by: David Laight Link: https://lore.kernel.org/r/20261108213254.67805ef0@pumpkin Suggested-by: Jisheng Zhang Link: https://lore.kernel.org/r/avG07pmpcLGkq5i-@xhacker Signed-off-by: Yunhui Cui --- arch/riscv/include/asm/asm.h | 19 ++++++++++++ arch/riscv/include/asm/percpu.h | 43 ++++++++++++++++++++++++++++ arch/riscv/include/asm/ptrace.h | 15 ++++++++++ arch/riscv/include/asm/thread_info.h | 1 + arch/riscv/kernel/Makefile | 1 + arch/riscv/kernel/asm-offsets.c | 2 ++ arch/riscv/kernel/entry-common.c | 32 +++++++++++++++++++++ arch/riscv/kernel/entry.S | 10 +++++++ 8 files changed, 123 insertions(+) create mode 100644 arch/riscv/kernel/entry-common.c diff --git a/arch/riscv/include/asm/asm.h b/arch/riscv/include/asm/asm.h index 06ac38f733b0a..57af10a936d9b 100644 --- a/arch/riscv/include/asm/asm.h +++ b/arch/riscv/include/asm/asm.h @@ -81,6 +81,25 @@ /* Common assembly source macros */ +/* + * Save and clear the active per-CPU GPR state on exception entry. Entry + * paths which bypass handle_exception, such as SBI Supervisor Software + * Events (SSE), must pair this with pcpu_gprs_exit around their handlers. + */ +.macro pcpu_gprs_entry tsk regs tmp + lhu \tmp, TASK_TI_PCPU_GPRS(\tsk) + REG_S \tmp, PT_PCPU_GPRS(\regs) + sh zero, TASK_TI_PCPU_GPRS(\tsk) +.endm + +/* + * Restore the saved state before resuming the interrupted context. tmp must + * contain PT_PCPU_GPRS from the interrupted frame. + */ +.macro pcpu_gprs_exit tsk tmp + sh \tmp, TASK_TI_PCPU_GPRS(\tsk) +.endm + /* * NOP sequence */ diff --git a/arch/riscv/include/asm/percpu.h b/arch/riscv/include/asm/percpu.h index faf9a75ec66d7..59f7d44f7d3fe 100644 --- a/arch/riscv/include/asm/percpu.h +++ b/arch/riscv/include/asm/percpu.h @@ -3,12 +3,55 @@ #ifndef __ASM_PERCPU_H #define __ASM_PERCPU_H +#include +#include #include +#include #include +#include #define __my_cpu_offset (((struct thread_info *)current)->pcpu_offset) +#define PCPU_GPR_PCP_SHIFT 0 +#define PCPU_GPR_PCP GENMASK(4, 0) +#define PCPU_GPR_OFF_SHIFT 5 +#define PCPU_GPR_OFF GENMASK(9, 5) +#define PCPU_GPR_ADDR_SHIFT 10 +#define PCPU_GPR_ADDR GENMASK(14, 10) + +#define __PCPU_GPR_NUM(gpr) ".L__gpr_num_" gpr + +#define __PCPU_GPRS_VALUE \ + "(" \ + "(" __PCPU_GPR_NUM("%[pcp]") " << " __stringify(PCPU_GPR_PCP_SHIFT) ") | " \ + "(" __PCPU_GPR_NUM("%[off]") " << " __stringify(PCPU_GPR_OFF_SHIFT) ") | " \ + "(" __PCPU_GPR_NUM("%[addr]") " << " __stringify(PCPU_GPR_ADDR_SHIFT) ")" \ + ")" + +#define __PCPU_GPRS_ASSERT_DISTINCT \ + ".if ((" __PCPU_GPR_NUM("%[pcp]") " == " __PCPU_GPR_NUM("%[off]") ") || " \ + "(" __PCPU_GPR_NUM("%[pcp]") " == " __PCPU_GPR_NUM("%[addr]") ") || " \ + "(" __PCPU_GPR_NUM("%[off]") " == " __PCPU_GPR_NUM("%[addr]") "))\n" \ + ".error \"PCPU GPRs overlap\"\n" \ + ".endif\n" + +/* + * Mark a region where the per-CPU offset and address registers may need to + * be rewritten by exception return after migration. The three registers + * must remain distinct for the whole region. + */ +#define __PCPU_GPRS_BEGIN \ + __DEFINE_ASM_GPR_NUMS \ + __PCPU_GPRS_ASSERT_DISTINCT \ + "li %[off], " __PCPU_GPRS_VALUE "\n" \ + "sh %[off], %[gprs]\n" \ + REG_L " %[off], %[pcpu_offset]\n" \ + "add %[addr], %[pcp], %[off]\n" + +#define __PCPU_GPRS_END \ + "sh zero, %[gprs]\n" + #include #endif /* __ASM_PERCPU_H */ diff --git a/arch/riscv/include/asm/ptrace.h b/arch/riscv/include/asm/ptrace.h index addc8188152f7..ceff8f63bc866 100644 --- a/arch/riscv/include/asm/ptrace.h +++ b/arch/riscv/include/asm/ptrace.h @@ -53,6 +53,8 @@ struct pt_regs { unsigned long cause; /* a0 value before the syscall */ unsigned long orig_a0; + /* Active preemptible per-CPU operation, if any. */ + unsigned long pcpu_gprs; }; #define PTRACE_SYSEMU 0x1f @@ -145,6 +147,19 @@ static inline unsigned long regs_get_register(struct pt_regs *regs, return *(unsigned long *)((unsigned long)regs + offset); } +static inline void regs_set_register(struct pt_regs *regs, + unsigned int offset, unsigned long val) +{ + if (unlikely(offset > MAX_REG_OFFSET)) + return; + + /* Register x0 is represented by the first, read-only slot. */ + if (offset) + *(unsigned long *)((unsigned long)regs + offset) = val; +} + +void noinstr riscv_pcpu_regs_fixup(struct pt_regs *regs); + /** * regs_get_kernel_argument() - get Nth function argument in kernel * @regs: pt_regs of that context diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h index f10ba62b61016..51c4076763c1b 100644 --- a/arch/riscv/include/asm/thread_info.h +++ b/arch/riscv/include/asm/thread_info.h @@ -63,6 +63,7 @@ struct thread_info { long user_sp; /* User stack pointer */ unsigned long syscall_work; /* SYSCALL_WORK_ flags */ unsigned long pcpu_offset; + unsigned short pcpu_gprs; #ifdef CONFIG_SHADOW_CALL_STACK void *scs_base; void *scs_sp; diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index f54c336bf547c..bf786d5dd4de3 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -52,6 +52,7 @@ obj-y += alternative.o obj-y += cpu.o obj-y += cpufeature.o obj-y += entry.o +obj-y += entry-common.o obj-y += irq.o obj-y += process.o obj-y += ptrace.o diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c index f4f7ee976cd94..38f246a691c60 100644 --- a/arch/riscv/kernel/asm-offsets.c +++ b/arch/riscv/kernel/asm-offsets.c @@ -39,6 +39,7 @@ void asm_offsets(void) OFFSET(TASK_TI_CPU, task_struct, thread_info.cpu); OFFSET(TASK_TI_PCPU_OFFSET, task_struct, thread_info.pcpu_offset); + OFFSET(TASK_TI_PCPU_GPRS, task_struct, thread_info.pcpu_gprs); OFFSET(TASK_TI_PREEMPT_COUNT, task_struct, thread_info.preempt_count); OFFSET(TASK_TI_KERNEL_SP, task_struct, thread_info.kernel_sp); OFFSET(TASK_TI_USER_SP, task_struct, thread_info.user_sp); @@ -130,6 +131,7 @@ void asm_offsets(void) OFFSET(PT_STATUS, pt_regs, status); OFFSET(PT_BADADDR, pt_regs, badaddr); OFFSET(PT_CAUSE, pt_regs, cause); + OFFSET(PT_PCPU_GPRS, pt_regs, pcpu_gprs); OFFSET(SUSPEND_CONTEXT_REGS, suspend_context, regs); diff --git a/arch/riscv/kernel/entry-common.c b/arch/riscv/kernel/entry-common.c new file mode 100644 index 0000000000000..023d856b1679b --- /dev/null +++ b/arch/riscv/kernel/entry-common.c @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include + +#include +#include + +/* + * If the interrupted context had an active per-CPU operation, rewrite its + * saved offset and address registers for the CPU on which it will resume. + */ +void noinstr riscv_pcpu_regs_fixup(struct pt_regs *regs) +{ + unsigned long pcpu_offset = __my_cpu_offset; + unsigned long gprs = regs->pcpu_gprs; + unsigned long pcp; + unsigned int reg_pcp; + unsigned int reg_off; + unsigned int reg_addr; + + reg_pcp = FIELD_GET(PCPU_GPR_PCP, gprs); + reg_off = FIELD_GET(PCPU_GPR_OFF, gprs); + reg_addr = FIELD_GET(PCPU_GPR_ADDR, gprs); + + /* pt_regs offset zero represents x0, followed by x1 through x31. */ + pcp = regs_get_register(regs, reg_pcp * sizeof(unsigned long)); + regs_set_register(regs, reg_off * sizeof(unsigned long), pcpu_offset); + regs_set_register(regs, reg_addr * sizeof(unsigned long), + pcp + pcpu_offset); +} diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index 628b85e3915a7..65298ac828959 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -198,6 +198,8 @@ SYM_CODE_START(handle_exception) REG_S s4, PT_CAUSE(sp) REG_S s5, PT_TP(sp) + pcpu_gprs_entry tp sp s0 + /* * Set the scratch register to 0, so that if a recursive exception * occurs, the exception vector knows it came from the kernel @@ -281,6 +283,14 @@ SYM_CODE_START_NOALIGN(ret_from_exception) move a0, sp call riscv_v_context_nesting_end #endif + REG_L t0, PT_PCPU_GPRS(sp) + beqz t0, .Lpcpu_regs_restored + move a0, sp + call riscv_pcpu_regs_fixup + REG_L t0, PT_PCPU_GPRS(sp) +.Lpcpu_regs_restored: + pcpu_gprs_exit tp t0 + REG_L a0, PT_STATUS(sp) restore_userssp s3, a0 /* -- 2.39.5