Update xloops_to_cycles() to use mul_u64_u64_shr() instead of 32-bit fixed point arithmetic to avoid truncation for delay values larger than ~10s. This adds an extra mul operation but since xloops_to_cycles() is only used from delay loops so shouldn't matter too much. Also move some related constants and functions related to the cycles computation out to a new header (converting some from macros to static inlines). And finally make __delay_cycles() available outside of arch/arm64/lib/delay.c. Rename some macros in qcom/rpmh-rsc.c which were occupying the same namespace. Cc: Catalin Marinas Cc: Will Deacon Cc: Bjorn Andersson Cc: Konrad Dybcio Cc: linux-arm-kernel@lists.infradead.org Reviewed-by: Christoph Lameter Acked-by: Catalin Marinas Signed-off-by: Ankur Arora --- Notes: - convert the multiply, shift operation to use mul_u64_u64_shr() to avoid truncating large delay values. - makes usecs_to_cycles(), nsecs_to_cycles() static inlines. arch/arm64/include/asm/delay-const.h | 35 ++++++++++++++++++++++++++++ arch/arm64/lib/delay.c | 17 ++++---------- drivers/soc/qcom/rpmh-rsc.c | 8 +++---- 3 files changed, 44 insertions(+), 16 deletions(-) create mode 100644 arch/arm64/include/asm/delay-const.h diff --git a/arch/arm64/include/asm/delay-const.h b/arch/arm64/include/asm/delay-const.h new file mode 100644 index 000000000000..6ccdfe0a4130 --- /dev/null +++ b/arch/arm64/include/asm/delay-const.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _ASM_DELAY_CONST_H +#define _ASM_DELAY_CONST_H + +#include +#include +#include /* For HZ */ + +/* 2**32 / 1000000 (rounded up) */ +#define __usecs_to_xloops_mult 0x10C7UL + +/* 2**32 / 1000000000 (rounded up) */ +#define __nsecs_to_xloops_mult 0x5UL + +extern unsigned long loops_per_jiffy; +static inline u64 xloops_to_cycles(u64 xloops) +{ + u64 loops_per_sec = (u64)loops_per_jiffy * HZ; + + return mul_u64_u64_shr(xloops, loops_per_sec, 32); +} + +static inline u64 usecs_to_cycles(u64 time_usecs) +{ + return xloops_to_cycles(time_usecs * __usecs_to_xloops_mult); +} + +static inline u64 nsecs_to_cycles(u64 time_nsecs) +{ + return xloops_to_cycles((time_nsecs) * __nsecs_to_xloops_mult); +} + +u64 notrace __delay_cycles(void); + +#endif /* _ASM_DELAY_CONST_H */ diff --git a/arch/arm64/lib/delay.c b/arch/arm64/lib/delay.c index e278e060e78a..f08eacef2f0a 100644 --- a/arch/arm64/lib/delay.c +++ b/arch/arm64/lib/delay.c @@ -12,17 +12,10 @@ #include #include #include +#include #include -#define USECS_TO_CYCLES(time_usecs) \ - xloops_to_cycles((time_usecs) * 0x10C7UL) - -static inline unsigned long xloops_to_cycles(unsigned long xloops) -{ - return (xloops * loops_per_jiffy * HZ) >> 32; -} - /* * Force the use of CNTVCT_EL0 in order to have the same base as WFxT. * This avoids some annoying issues when CNTVOFF_EL2 is not reset 0 on a @@ -32,7 +25,7 @@ static inline unsigned long xloops_to_cycles(unsigned long xloops) * Note that userspace cannot change the offset behind our back either, * as the vcpu mutex is held as long as KVM_RUN is in progress. */ -static cycles_t notrace __delay_cycles(void) +u64 notrace __delay_cycles(void) { guard(preempt_notrace)(); return __arch_counter_get_cntvct_stable(); @@ -54,7 +47,7 @@ void __delay(unsigned long cycles) wfet(end); } else if (arch_timer_evtstrm_available()) { const cycles_t timer_evt_period = - USECS_TO_CYCLES(ARCH_TIMER_EVT_STREAM_PERIOD_US); + usecs_to_cycles(ARCH_TIMER_EVT_STREAM_PERIOD_US); while ((__delay_cycles() - start + timer_evt_period) < cycles) wfe(); @@ -73,12 +66,12 @@ EXPORT_SYMBOL(__const_udelay); void __udelay(unsigned long usecs) { - __const_udelay(usecs * 0x10C7UL); /* 2**32 / 1000000 (rounded up) */ + __const_udelay(usecs * __usecs_to_xloops_mult); } EXPORT_SYMBOL(__udelay); void __ndelay(unsigned long nsecs) { - __const_udelay(nsecs * 0x5UL); /* 2**32 / 1000000000 (rounded up) */ + __const_udelay(nsecs * __nsecs_to_xloops_mult); } EXPORT_SYMBOL(__ndelay); diff --git a/drivers/soc/qcom/rpmh-rsc.c b/drivers/soc/qcom/rpmh-rsc.c index f881af35ecfc..c9a814d9e06f 100644 --- a/drivers/soc/qcom/rpmh-rsc.c +++ b/drivers/soc/qcom/rpmh-rsc.c @@ -146,10 +146,10 @@ enum { * +---------------------------------------------------+ */ -#define USECS_TO_CYCLES(time_usecs) \ - xloops_to_cycles((time_usecs) * 0x10C7UL) +#define RPMH_USECS_TO_CYCLES(time_usecs) \ + rpmh_xloops_to_cycles((time_usecs) * 0x10C7UL) -static inline unsigned long xloops_to_cycles(u64 xloops) +static inline unsigned long rpmh_xloops_to_cycles(u64 xloops) { return (xloops * loops_per_jiffy * HZ) >> 32; } @@ -828,7 +828,7 @@ void rpmh_rsc_write_next_wakeup(struct rsc_drv *drv) wakeup_us = ktime_to_us(wakeup); /* Convert the wakeup to arch timer scale */ - wakeup_cycles = USECS_TO_CYCLES(wakeup_us); + wakeup_cycles = RPMH_USECS_TO_CYCLES(wakeup_us); wakeup_cycles += arch_timer_read_counter(); exit: -- 2.43.7