Apply the recoverable per-CPU address handling to add-return operations. Native-width AMOs return the old value, so derive the new value required by the percpu API from the operand. Use Zabha for subword operations when available, or a masked LR/SC sequence that returns the updated field. Signed-off-by: Yunhui Cui --- arch/riscv/include/asm/percpu.h | 134 ++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/arch/riscv/include/asm/percpu.h b/arch/riscv/include/asm/percpu.h index 46eafa1ef02a1..9ff02f3811550 100644 --- a/arch/riscv/include/asm/percpu.h +++ b/arch/riscv/include/asm/percpu.h @@ -203,6 +203,129 @@ __PERCPU_8_16_OP_CASE(and, 16, .h, and, "and") __PERCPU_8_16_OP_CASE(or, 8, .b, or, "or") __PERCPU_8_16_OP_CASE(or, 16, .h, or, "or") +static __always_inline u32 +__percpu_add_return_amo_case_32(void __percpu *pcp, unsigned long val) +{ + __PCPU_OP_CONTEXT(); + u32 ret; + + asm volatile (__PCPU_GPRS_BEGIN + "amoadd.w %[ret], %[val], 0(%[addr])\n" + __PCPU_GPRS_END + : [gprs] "=m" (__ti->pcpu_gprs), + [off] "=&r" (__off), [addr] "=&r" (__addr), + [ret] "=&r" (ret) + : [pcpu_offset] "m" (__ti->pcpu_offset), + [pcp] "r" (pcp), [val] "r" ((u32)val) + : "memory"); + + return ret + (u32)val; +} + +#ifdef CONFIG_64BIT +static __always_inline u64 +__percpu_add_return_amo_case_64(void __percpu *pcp, unsigned long val) +{ + __PCPU_OP_CONTEXT(); + u64 ret; + + asm volatile (__PCPU_GPRS_BEGIN + "amoadd.d %[ret], %[val], 0(%[addr])\n" + __PCPU_GPRS_END + : [gprs] "=m" (__ti->pcpu_gprs), + [off] "=&r" (__off), [addr] "=&r" (__addr), + [ret] "=&r" (ret) + : [pcpu_offset] "m" (__ti->pcpu_offset), + [pcp] "r" (pcp), [val] "r" ((u64)val) + : "memory"); + + return ret + (u64)val; +} +#endif + +static __always_inline unsigned long +__percpu_add_return_8_16(void __percpu *pcp, unsigned long val, + unsigned int bits) +{ + __PCPU_OP_CONTEXT(); + const u32 field_mask = GENMASK(bits - 1, 0); + const unsigned long shift = PERCPU_8_16_GET_SHIFT(pcp); + const u32 mask = field_mask << shift; + const u32 val_trunc = val & field_mask; + void __percpu *pcp32 = PERCPU_8_16_GET_PCP32(pcp); + u32 old, new, field; + unsigned long ret; + + if (IS_ENABLED(CONFIG_RISCV_ISA_ZABHA) && + riscv_has_extension_unlikely(RISCV_ISA_EXT_ZABHA)) { + if (bits == 8) { + asm volatile (__PCPU_GPRS_BEGIN + ".option push\n" + ".option arch, +zabha\n" + "amoadd.b %[ret], %[val], 0(%[addr])\n" + ".option pop\n" + __PCPU_GPRS_END + : [gprs] "=m" (__ti->pcpu_gprs), + [off] "=&r" (__off), [addr] "=&r" (__addr), + [ret] "=&r" (ret) + : [pcpu_offset] "m" (__ti->pcpu_offset), + [pcp] "r" (pcp), [val] "r" ((u8)val) + : "memory"); + return (u8)(ret + val); + } + + asm volatile (__PCPU_GPRS_BEGIN + ".option push\n" + ".option arch, +zabha\n" + "amoadd.h %[ret], %[val], 0(%[addr])\n" + ".option pop\n" + __PCPU_GPRS_END + : [gprs] "=m" (__ti->pcpu_gprs), + [off] "=&r" (__off), [addr] "=&r" (__addr), + [ret] "=&r" (ret) + : [pcpu_offset] "m" (__ti->pcpu_offset), + [pcp] "r" (pcp), [val] "r" ((u16)val) + : "memory"); + return (u16)(ret + val); + } + + asm volatile (__PCPU_GPRS_BEGIN + "0: lr.w %[old], 0(%[addr])\n" + "and %[field], %[old], %[mask]\n" + "srl %[field], %[field], %[shift]\n" + "add %[field], %[field], %[val]\n" + "and %[field], %[field], %[field_mask]\n" + "sll %[field], %[field], %[shift]\n" + "and %[new], %[old], %[inv_mask]\n" + "or %[new], %[new], %[field]\n" + "sc.w %[new], %[new], 0(%[addr])\n" + "bnez %[new], 0b\n" + __PCPU_GPRS_END + : [gprs] "=m" (__ti->pcpu_gprs), + [off] "=&r" (__off), [addr] "=&r" (__addr), + [old] "=&r" (old), [new] "=&r" (new), + [field] "=&r" (field) + : [pcpu_offset] "m" (__ti->pcpu_offset), + [pcp] "r" (pcp32), [mask] "r" (mask), + [shift] "r" (shift), [val] "r" (val_trunc), + [field_mask] "r" (field_mask), [inv_mask] "r" (~mask) + : "memory"); + + return field >> shift; +} + +static __always_inline u8 +__percpu_add_return_amo_case_8(void __percpu *pcp, unsigned long val) +{ + return __percpu_add_return_8_16(pcp, val, 8); +} + +static __always_inline u16 +__percpu_add_return_amo_case_16(void __percpu *pcp, unsigned long val) +{ + return __percpu_add_return_8_16(pcp, val, 16); +} + #define _pcp_wrap(op, pcp, ...) \ ({ \ op(&(pcp), __VA_ARGS__); \ @@ -256,6 +379,17 @@ __PERCPU_8_16_OP_CASE(or, 16, .h, or, "or") #define this_cpu_or_8(pcp, val) _pcp_wrap(__percpu_or_amo_case_64, pcp, val) #endif +#define this_cpu_add_return_1(pcp, val) \ + _pcp_wrap_return(__percpu_add_return_amo_case_8, pcp, val) +#define this_cpu_add_return_2(pcp, val) \ + _pcp_wrap_return(__percpu_add_return_amo_case_16, pcp, val) +#define this_cpu_add_return_4(pcp, val) \ + _pcp_wrap_return(__percpu_add_return_amo_case_32, pcp, val) +#ifdef CONFIG_64BIT +#define this_cpu_add_return_8(pcp, val) \ + _pcp_wrap_return(__percpu_add_return_amo_case_64, pcp, val) +#endif + #undef PERCPU_RW_OPS #undef __PERCPU_AMO_OP_CASE #undef __PERCPU_8_16_OP_CASE -- 2.39.5