Apply the recoverable per-CPU address handling to compare-and-exchange operations. Providing this_cpu_cmpxchg() also allows asm-generic to derive this_cpu_try_cmpxchg(). Use Zacas where available and LR/SC otherwise. Keep masked subword operands XLEN-wide so RV64 LR.W sign extension does not affect the comparison, and use typed helpers to separate asm-generic from the architecture cmpxchg macro expansion. Link: https://lore.kernel.org/r/20260826010748.zphu3wh2hlm4t4o2@offworld Signed-off-by: Yunhui Cui --- arch/riscv/include/asm/percpu.h | 140 ++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) diff --git a/arch/riscv/include/asm/percpu.h b/arch/riscv/include/asm/percpu.h index 810f4c5eaa508..4d2fc10cbdab1 100644 --- a/arch/riscv/include/asm/percpu.h +++ b/arch/riscv/include/asm/percpu.h @@ -401,6 +401,123 @@ __percpu_xchg_case_##sz(void __percpu *pcp, unsigned long val) \ __PERCPU_8_16_XCHG_CASE(8, .b) __PERCPU_8_16_XCHG_CASE(16, .h) + +#define __PERCPU_CMPXCHG_CASE(sz, lr_sfx, sc_sfx, cas_sfx) \ +static __always_inline unsigned long \ +__percpu_cmpxchg_case_##sz(void __percpu *pcp, unsigned long old, \ + unsigned long new) \ +{ \ + __PCPU_OP_CONTEXT(); \ + const long cmp = (long)(s##sz)(u##sz)old; \ + unsigned long rc; \ + u##sz ret = (u##sz)old; \ + \ + if (IS_ENABLED(CONFIG_RISCV_ISA_ZACAS) && \ + IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZACAS) && \ + riscv_has_extension_unlikely(RISCV_ISA_EXT_ZACAS)) { \ + asm volatile (__PCPU_GPRS_BEGIN \ + ".option push\n" \ + ".option arch, +zacas\n" \ + "amocas" #cas_sfx " %[ret], %[new], 0(%[addr])\n" \ + ".option pop\n" \ + __PCPU_GPRS_END \ + : [gprs] "=m" (__ti->pcpu_gprs), \ + [off] "=&r" (__off), [addr] "=&r" (__addr), \ + [ret] "+&r" (ret) \ + : [pcpu_offset] "m" (__ti->pcpu_offset), \ + [pcp] "r" (pcp), [new] "r" ((u##sz)new) \ + : "memory"); \ + } else { \ + asm volatile (__PCPU_GPRS_BEGIN \ + "0: lr" #lr_sfx " %[ret], 0(%[addr])\n" \ + "bne %[ret], %[old], 1f\n" \ + "sc" #sc_sfx " %[rc], %[new], 0(%[addr])\n" \ + "bnez %[rc], 0b\n" \ + "1:\n" \ + __PCPU_GPRS_END \ + : [gprs] "=m" (__ti->pcpu_gprs), \ + [off] "=&r" (__off), [addr] "=&r" (__addr), \ + [ret] "=&r" (ret), [rc] "=&r" (rc) \ + : [pcpu_offset] "m" (__ti->pcpu_offset), \ + [pcp] "r" (pcp), [old] "r" (cmp), \ + [new] "r" ((u##sz)new) \ + : "memory"); \ + } \ + \ + return ret; \ +} + +__PERCPU_CMPXCHG_CASE(32, .w, .w, .w) +#ifdef CONFIG_64BIT +__PERCPU_CMPXCHG_CASE(64, .d, .d, .d) +#endif + +/* + * Use XLEN-wide masks to clear the upper bits of an RV64 sign-extended + * LR.W result before comparing a high byte or halfword. + */ +#define __PERCPU_8_16_CMPXCHG_CASE(sz, sfx) \ +static __always_inline unsigned long \ +__percpu_cmpxchg_case_##sz(void __percpu *pcp, unsigned long old, \ + unsigned long new) \ +{ \ + __PCPU_OP_CONTEXT(); \ + const unsigned long shift = PERCPU_8_16_GET_SHIFT(pcp); \ + const unsigned long field_mask = PERCPU_8_16_GET_MASK(sz); \ + const unsigned long mask = field_mask << shift; \ + const unsigned long old_field = (old & field_mask) << shift; \ + const unsigned long new_field = (new & field_mask) << shift; \ + void __percpu *pcp32 = PERCPU_8_16_GET_PCP32(pcp); \ + unsigned long ret32, tmp, rc; \ + u##sz ret = (u##sz)old; \ + \ + if (IS_ENABLED(CONFIG_RISCV_ISA_ZABHA) && \ + IS_ENABLED(CONFIG_RISCV_ISA_ZACAS) && \ + IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZACAS) && \ + riscv_has_extension_unlikely(RISCV_ISA_EXT_ZABHA) && \ + riscv_has_extension_unlikely(RISCV_ISA_EXT_ZACAS)) { \ + asm volatile (__PCPU_GPRS_BEGIN \ + ".option push\n" \ + ".option arch, +zacas, +zabha\n" \ + "amocas" #sfx " %[ret], %[new], 0(%[addr])\n" \ + ".option pop\n" \ + __PCPU_GPRS_END \ + : [gprs] "=m" (__ti->pcpu_gprs), \ + [off] "=&r" (__off), [addr] "=&r" (__addr), \ + [ret] "+&r" (ret) \ + : [pcpu_offset] "m" (__ti->pcpu_offset), \ + [pcp] "r" (pcp), [new] "r" ((u##sz)new) \ + : "memory"); \ + } else { \ + asm volatile (__PCPU_GPRS_BEGIN \ + "0: lr.w %[ret], 0(%[addr])\n" \ + "and %[tmp], %[ret], %[mask]\n" \ + "bne %[tmp], %[old], 1f\n" \ + "and %[tmp], %[ret], %[inv_mask]\n" \ + "or %[tmp], %[tmp], %[new]\n" \ + "sc.w %[rc], %[tmp], 0(%[addr])\n" \ + "bnez %[rc], 0b\n" \ + "1:\n" \ + __PCPU_GPRS_END \ + : [gprs] "=m" (__ti->pcpu_gprs), \ + [off] "=&r" (__off), [addr] "=&r" (__addr), \ + [ret] "=&r" (ret32), [tmp] "=&r" (tmp), \ + [rc] "=&r" (rc) \ + : [pcpu_offset] "m" (__ti->pcpu_offset), \ + [pcp] "r" (pcp32), [mask] "r" (mask), \ + [old] "r" (old_field), \ + [new] "r" (new_field), \ + [inv_mask] "r" (~mask) \ + : "memory"); \ + \ + ret = (ret32 & mask) >> shift; \ + } \ + \ + return ret; \ +} + +__PERCPU_8_16_CMPXCHG_CASE(8, .b) +__PERCPU_8_16_CMPXCHG_CASE(16, .h) #define _pcp_wrap(op, pcp, ...) \ ({ \ op(&(pcp), __VA_ARGS__); \ @@ -421,6 +538,15 @@ __PERCPU_8_16_XCHG_CASE(16, .h) (unsigned long)(val)); \ }) +#define _pcp_wrap_cmpxchg(op, pcp, old, new) \ +({ \ + __auto_type __pcp_ptr = &(pcp); \ + \ + (TYPEOF_UNQUAL(*__pcp_ptr))op(__pcp_ptr, \ + (unsigned long)(old), \ + (unsigned long)(new)); \ +}) + #define this_cpu_read_1(pcp) _pcp_wrap_return(__percpu_read_8, pcp) #define this_cpu_read_2(pcp) _pcp_wrap_return(__percpu_read_16, pcp) #define this_cpu_read_4(pcp) _pcp_wrap_return(__percpu_read_32, pcp) @@ -480,11 +606,25 @@ __PERCPU_8_16_XCHG_CASE(16, .h) #define this_cpu_xchg_8(pcp, val) _pcp_wrap_xchg(__percpu_xchg_case_64, pcp, val) #endif +#define this_cpu_cmpxchg_1(pcp, o, n) \ + _pcp_wrap_cmpxchg(__percpu_cmpxchg_case_8, pcp, o, n) +#define this_cpu_cmpxchg_2(pcp, o, n) \ + _pcp_wrap_cmpxchg(__percpu_cmpxchg_case_16, pcp, o, n) +#define this_cpu_cmpxchg_4(pcp, o, n) \ + _pcp_wrap_cmpxchg(__percpu_cmpxchg_case_32, pcp, o, n) +#ifdef CONFIG_64BIT +#define this_cpu_cmpxchg_8(pcp, o, n) \ + _pcp_wrap_cmpxchg(__percpu_cmpxchg_case_64, pcp, o, n) +#define this_cpu_cmpxchg64(pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n) +#endif + #undef PERCPU_RW_OPS #undef __PERCPU_AMO_OP_CASE #undef __PERCPU_8_16_OP_CASE #undef __PERCPU_XCHG_AMO_CASE #undef __PERCPU_8_16_XCHG_CASE +#undef __PERCPU_CMPXCHG_CASE +#undef __PERCPU_8_16_CMPXCHG_CASE #undef PERCPU_8_16_GET_SHIFT #undef PERCPU_8_16_GET_MASK #undef PERCPU_8_16_GET_PCP32 -- 2.39.5