When using WFE we can go beyond the timeout since, lacking an event we depend on the event-stream for wakeups when there's no event. So, prefer WFET where available to take advantage of precise wait. Handling of WFET is via __cmpwait_timeout() which behaves similarly to __cmpwait() calling out to the __cmpwait_case_timeout_##sz() or __cmpwait_case_##sz() functions. Cc: Arnd Bergmann Cc: Catalin Marinas Cc: Will Deacon Cc: linux-arm-kernel@lists.infradead.org Reviewed-by: Catalin Marinas Tested-by: Haris Okanovic Tested-by: Sahil Chandna Signed-off-by: Ankur Arora --- Notes: - use arm64_wfet_deadline() to calculate the deadline. arch/arm64/include/asm/barrier.h | 8 ++-- arch/arm64/include/asm/cmpxchg.h | 64 ++++++++++++++++++++++++++------ 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/arch/arm64/include/asm/barrier.h b/arch/arm64/include/asm/barrier.h index e2263c754a55..3b024058cf2f 100644 --- a/arch/arm64/include/asm/barrier.h +++ b/arch/arm64/include/asm/barrier.h @@ -224,8 +224,8 @@ do { \ extern bool arch_timer_evtstrm_available(void); /* - * In the common case, cpu_poll_relax() sits waiting in __cmpwait_relaxed() - * for @ptr value to change. + * In the common case, cpu_poll_relax() sits waiting in __cmpwait_relaxed()/ + * __cmpwait_relaxed_timeout() for the ptr value to change. * * State this by defining CPU_POLL_RELAX_WAITS. This enables a time-check * optimization in smp_cond_load_{relaxed,acquire}_timeout(). Note that this @@ -235,7 +235,9 @@ extern bool arch_timer_evtstrm_available(void); #define CPU_POLL_RELAX_WAITS #define cpu_poll_relax(ptr, val, timeout_ns) do { \ - if (arch_timer_evtstrm_available()) \ + if (alternative_has_cap_unlikely(ARM64_HAS_WFXT)) \ + __cmpwait_relaxed_timeout(ptr, val, timeout_ns); \ + else if (arch_timer_evtstrm_available()) \ __cmpwait_relaxed(ptr, val); \ else \ cpu_relax(); \ diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h index 6cf3cd6873f5..e543366801ce 100644 --- a/arch/arm64/include/asm/cmpxchg.h +++ b/arch/arm64/include/asm/cmpxchg.h @@ -212,7 +212,8 @@ __CMPXCHG_GEN(_mb) #define __CMPWAIT_CASE(w, sfx, sz) \ static inline void __cmpwait_case_##sz(volatile void *ptr, \ - unsigned long val) \ + unsigned long val, \ + u64 __maybe_unused timeout_ns) \ { \ unsigned long tmp; \ \ @@ -235,20 +236,55 @@ __CMPWAIT_CASE( , , 64); #undef __CMPWAIT_CASE -#define __CMPWAIT_GEN(sfx) \ -static __always_inline void __cmpwait##sfx(volatile void *ptr, \ - unsigned long val, \ - int size) \ +/* Declared locally to avoid recursive include dependency */ +u64 arm64_wfet_deadline(u64 timeout_ns); + +#define __CMPWAIT_TIMEOUT_CASE(w, sfx, sz) \ +static inline void __cmpwait_case_timeout_##sz(volatile void *ptr, \ + unsigned long val, \ + u64 timeout_ns) \ +{ \ + unsigned long tmp; \ + u64 ecycles = arm64_wfet_deadline(timeout_ns); \ + \ + asm volatile( \ + " sevl\n" \ + " wfe\n" \ + " ldxr" #sfx "\t%" #w "[tmp], %[v]\n" \ + " eor %" #w "[tmp], %" #w "[tmp], %" #w "[val]\n" \ + " cbnz %" #w "[tmp], 1f\n" \ + " msr s0_3_c1_c0_0, %[ecycles]\n" \ + "1:" \ + : [tmp] "=&r" (tmp), [v] "+Q" (*(u##sz *)ptr) \ + : [val] "r" (val), [ecycles] "r" (ecycles)); \ +} + +__CMPWAIT_TIMEOUT_CASE(w, b, 8); +__CMPWAIT_TIMEOUT_CASE(w, h, 16); +__CMPWAIT_TIMEOUT_CASE(w, , 32); +__CMPWAIT_TIMEOUT_CASE( , , 64); + +#undef __CMPWAIT_TIMEOUT_CASE + +#define __CMPWAIT_GEN(timeout, sfx) \ +static __always_inline void __cmpwait##timeout##sfx(volatile void *ptr, \ + unsigned long val, \ + u64 timeout_ns, \ + int size) \ { \ switch (size) { \ case 1: \ - return __cmpwait_case##sfx##_8(ptr, (u8)val); \ + return __cmpwait_case##timeout##sfx##_8(ptr, (u8)val, \ + timeout_ns); \ case 2: \ - return __cmpwait_case##sfx##_16(ptr, (u16)val); \ + return __cmpwait_case##timeout##sfx##_16(ptr, (u16)val, \ + timeout_ns); \ case 4: \ - return __cmpwait_case##sfx##_32(ptr, val); \ + return __cmpwait_case##timeout##sfx##_32(ptr, val, \ + timeout_ns); \ case 8: \ - return __cmpwait_case##sfx##_64(ptr, val); \ + return __cmpwait_case##timeout##sfx##_64(ptr, val, \ + timeout_ns); \ default: \ BUILD_BUG(); \ } \ @@ -256,11 +292,15 @@ static __always_inline void __cmpwait##sfx(volatile void *ptr, \ unreachable(); \ } -__CMPWAIT_GEN() +__CMPWAIT_GEN( , ) +__CMPWAIT_GEN(_timeout, ) #undef __CMPWAIT_GEN -#define __cmpwait_relaxed(ptr, val) \ - __cmpwait((ptr), (unsigned long)(val), sizeof(*(ptr))) +#define __cmpwait_relaxed_timeout(ptr, val, timeout_ns) \ + __cmpwait_timeout((ptr), (unsigned long)(val), timeout_ns, sizeof(*(ptr))) + +#define __cmpwait_relaxed(ptr, val) \ + __cmpwait((ptr), (unsigned long)(val), 0, sizeof(*(ptr))) #endif /* __ASM_CMPXCHG_H */ -- 2.43.7