Apply the recoverable per-CPU address handling to void read-modify-write operations. Use relaxed AMOs for native-width operations, and Zabha or a masked LR/SC sequence for subword operations. Use value-returning AMOs even when the old value is unused so CPU-local operations remain likely to execute near the CPU. Link: https://lore.kernel.org/r/20260826010748.zphu3wh2hlm4t4o2@offworld Signed-off-by: Yunhui Cui --- arch/riscv/include/asm/percpu.h | 127 ++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) diff --git a/arch/riscv/include/asm/percpu.h b/arch/riscv/include/asm/percpu.h index ae7c123a48093..46eafa1ef02a1 100644 --- a/arch/riscv/include/asm/percpu.h +++ b/arch/riscv/include/asm/percpu.h @@ -9,8 +9,10 @@ #include #include +#include #include #include +#include #define __my_cpu_offset (((struct thread_info *)current)->pcpu_offset) @@ -102,6 +104,105 @@ PERCPU_RW_OPS("ld", "sd", 64) PERCPU_RW_OPS("lw", "sw", 32) #endif +/* + * Use value-returning AMOs even when the old value is unused. Avoiding x0 + * as rd makes an AMO more likely to execute near the CPU, which better suits + * CPU-local data that is commonly accessed again by the same CPU. + */ +#define __PERCPU_AMO_OP_CASE(sfx, name, sz, amo_insn) \ +static __always_inline void \ +__percpu_##name##_amo_case_##sz(void __percpu *pcp, unsigned long val) \ +{ \ + __PCPU_OP_CONTEXT(); \ + u##sz tmp; \ + \ + asm volatile (__PCPU_GPRS_BEGIN \ + "amo" #amo_insn #sfx " %[tmp], %[val], 0(%[addr])\n" \ + __PCPU_GPRS_END \ + : [gprs] "=m" (__ti->pcpu_gprs), \ + [off] "=&r" (__off), [addr] "=&r" (__addr), \ + [tmp] "=&r" (tmp) \ + : [pcpu_offset] "m" (__ti->pcpu_offset), \ + [pcp] "r" (pcp), [val] "r" ((u##sz)val) \ + : "memory"); \ +} + +__PERCPU_AMO_OP_CASE(.w, add, 32, add) +__PERCPU_AMO_OP_CASE(.w, and, 32, and) +__PERCPU_AMO_OP_CASE(.w, or, 32, or) + +#ifdef CONFIG_64BIT +__PERCPU_AMO_OP_CASE(.d, add, 64, add) +__PERCPU_AMO_OP_CASE(.d, and, 64, and) +__PERCPU_AMO_OP_CASE(.d, or, 64, or) +#endif + +#define PERCPU_8_16_GET_SHIFT(pcp) \ + (((__force unsigned long)(pcp) & 0x3) * BITS_PER_BYTE) +#define PERCPU_8_16_GET_MASK(sz) GENMASK((sz) - 1, 0) +#define PERCPU_8_16_GET_PCP32(pcp) \ + ((void __percpu *)((__force unsigned long)(pcp) & ~0x3UL)) + +#define __PERCPU_8_16_OP_CASE(name, sz, sfx, amo_insn, op_insn) \ +static __always_inline void \ +__percpu_##name##_amo_case_##sz(void __percpu *pcp, unsigned long val) \ +{ \ + __PCPU_OP_CONTEXT(); \ + const u32 field_mask = PERCPU_8_16_GET_MASK(sz); \ + const unsigned long shift = PERCPU_8_16_GET_SHIFT(pcp); \ + const u32 mask = field_mask << shift; \ + const u32 val_trunc = val & field_mask; \ + void __percpu *pcp32 = PERCPU_8_16_GET_PCP32(pcp); \ + u32 old, new, field; \ + u##sz tmp; \ + \ + if (IS_ENABLED(CONFIG_RISCV_ISA_ZABHA) && \ + riscv_has_extension_unlikely(RISCV_ISA_EXT_ZABHA)) { \ + asm volatile (__PCPU_GPRS_BEGIN \ + ".option push\n" \ + ".option arch, +zabha\n" \ + "amo" #amo_insn #sfx " %[tmp], %[val], 0(%[addr])\n" \ + ".option pop\n" \ + __PCPU_GPRS_END \ + : [gprs] "=m" (__ti->pcpu_gprs), \ + [off] "=&r" (__off), [addr] "=&r" (__addr), \ + [tmp] "=&r" (tmp) \ + : [pcpu_offset] "m" (__ti->pcpu_offset), \ + [pcp] "r" (pcp), [val] "r" ((u##sz)val) \ + : "memory"); \ + } else { \ + asm volatile (__PCPU_GPRS_BEGIN \ + "0: lr.w %[old], 0(%[addr])\n" \ + "and %[field], %[old], %[mask]\n" \ + "srl %[field], %[field], %[shift]\n" \ + op_insn " %[field], %[field], %[val]\n" \ + "and %[field], %[field], %[field_mask]\n" \ + "sll %[field], %[field], %[shift]\n" \ + "and %[new], %[old], %[inv_mask]\n" \ + "or %[new], %[new], %[field]\n" \ + "sc.w %[new], %[new], 0(%[addr])\n" \ + "bnez %[new], 0b\n" \ + __PCPU_GPRS_END \ + : [gprs] "=m" (__ti->pcpu_gprs), \ + [off] "=&r" (__off), [addr] "=&r" (__addr), \ + [old] "=&r" (old), [new] "=&r" (new), \ + [field] "=&r" (field) \ + : [pcpu_offset] "m" (__ti->pcpu_offset), \ + [pcp] "r" (pcp32), [mask] "r" (mask), \ + [shift] "r" (shift), [val] "r" (val_trunc), \ + [field_mask] "r" (field_mask), \ + [inv_mask] "r" (~mask) \ + : "memory"); \ + } \ +} + +__PERCPU_8_16_OP_CASE(add, 8, .b, add, "add") +__PERCPU_8_16_OP_CASE(add, 16, .h, add, "add") +__PERCPU_8_16_OP_CASE(and, 8, .b, and, "and") +__PERCPU_8_16_OP_CASE(and, 16, .h, and, "and") +__PERCPU_8_16_OP_CASE(or, 8, .b, or, "or") +__PERCPU_8_16_OP_CASE(or, 16, .h, or, "or") + #define _pcp_wrap(op, pcp, ...) \ ({ \ op(&(pcp), __VA_ARGS__); \ @@ -134,7 +235,33 @@ PERCPU_RW_OPS("lw", "sw", 32) _pcp_wrap(__percpu_write_64, pcp, (unsigned long)(val)) #endif +#define this_cpu_add_1(pcp, val) _pcp_wrap(__percpu_add_amo_case_8, pcp, val) +#define this_cpu_add_2(pcp, val) _pcp_wrap(__percpu_add_amo_case_16, pcp, val) +#define this_cpu_add_4(pcp, val) _pcp_wrap(__percpu_add_amo_case_32, pcp, val) +#ifdef CONFIG_64BIT +#define this_cpu_add_8(pcp, val) _pcp_wrap(__percpu_add_amo_case_64, pcp, val) +#endif + +#define this_cpu_and_1(pcp, val) _pcp_wrap(__percpu_and_amo_case_8, pcp, val) +#define this_cpu_and_2(pcp, val) _pcp_wrap(__percpu_and_amo_case_16, pcp, val) +#define this_cpu_and_4(pcp, val) _pcp_wrap(__percpu_and_amo_case_32, pcp, val) +#ifdef CONFIG_64BIT +#define this_cpu_and_8(pcp, val) _pcp_wrap(__percpu_and_amo_case_64, pcp, val) +#endif + +#define this_cpu_or_1(pcp, val) _pcp_wrap(__percpu_or_amo_case_8, pcp, val) +#define this_cpu_or_2(pcp, val) _pcp_wrap(__percpu_or_amo_case_16, pcp, val) +#define this_cpu_or_4(pcp, val) _pcp_wrap(__percpu_or_amo_case_32, pcp, val) +#ifdef CONFIG_64BIT +#define this_cpu_or_8(pcp, val) _pcp_wrap(__percpu_or_amo_case_64, pcp, val) +#endif + #undef PERCPU_RW_OPS +#undef __PERCPU_AMO_OP_CASE +#undef __PERCPU_8_16_OP_CASE +#undef PERCPU_8_16_GET_SHIFT +#undef PERCPU_8_16_GET_MASK +#undef PERCPU_8_16_GET_PCP32 #undef __PCPU_OP_CONTEXT #include -- 2.39.5