check_timeout() gets the current time value and depending on how much time has passed, checks for deadlock or times out, returning 0 or -errno on deadlock or timeout. Switch this out to a clock style interface, where it functions as a clock in the "lock-domain", returning the current time until a deadlock or timeout occurs. Once a deadlock or timeout has occurred, it stops functioning as a clock and returns error. Also adjust the RES_CHECK_TIMEOUT macro to discard the clock value when updating the explicit return status. Cc: bpf@vger.kernel.org Cc: Kumar Kartikeya Dwivedi Cc: Alexei Starovoitov Acked-by: Kumar Kartikeya Dwivedi Signed-off-by: Ankur Arora --- kernel/bpf/rqspinlock.c | 45 +++++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/kernel/bpf/rqspinlock.c b/kernel/bpf/rqspinlock.c index e4e338cdb437..0ec17ebb67c1 100644 --- a/kernel/bpf/rqspinlock.c +++ b/kernel/bpf/rqspinlock.c @@ -196,8 +196,12 @@ static noinline int check_deadlock_ABBA(rqspinlock_t *lock, u32 mask) return 0; } -static noinline int check_timeout(rqspinlock_t *lock, u32 mask, - struct rqspinlock_timeout *ts) +/* + * Returns current monotonic time in ns on success or, negative errno + * value on failure due to timeout expiration or detection of deadlock. + */ +static noinline s64 clock_deadlock(rqspinlock_t *lock, u32 mask, + struct rqspinlock_timeout *ts) { u64 prev = ts->cur; u64 time; @@ -207,7 +211,7 @@ static noinline int check_timeout(rqspinlock_t *lock, u32 mask, return -EDEADLK; ts->cur = ktime_get_mono_fast_ns(); ts->timeout_end = ts->cur + ts->duration; - return 0; + return (s64)ts->cur; } time = ktime_get_mono_fast_ns(); @@ -219,11 +223,15 @@ static noinline int check_timeout(rqspinlock_t *lock, u32 mask, * checks. */ if (prev + NSEC_PER_MSEC < time) { + int ret; ts->cur = time; - return check_deadlock_ABBA(lock, mask); + ret = check_deadlock_ABBA(lock, mask); + if (ret) + return ret; + } - return 0; + return (s64)time; } /* @@ -231,15 +239,22 @@ static noinline int check_timeout(rqspinlock_t *lock, u32 mask, * as the macro does internal amortization for us. */ #ifndef res_smp_cond_load_acquire -#define RES_CHECK_TIMEOUT(ts, ret, mask) \ - ({ \ - if (!(ts).spin++) \ - (ret) = check_timeout((lock), (mask), &(ts)); \ - (ret); \ +#define RES_CHECK_TIMEOUT(ts, ret, mask) \ + ({ \ + s64 __timeval_err = 0; \ + if (!(ts).spin++) \ + __timeval_err = clock_deadlock((lock), (mask), &(ts)); \ + (ret) = __timeval_err < 0 ? __timeval_err : 0; \ + __timeval_err; \ }) #else -#define RES_CHECK_TIMEOUT(ts, ret, mask) \ - ({ (ret) = check_timeout((lock), (mask), &(ts)); }) +#define RES_CHECK_TIMEOUT(ts, ret, mask) \ + ({ \ + s64 __timeval_err; \ + __timeval_err = clock_deadlock((lock), (mask), &(ts)); \ + (ret) = __timeval_err < 0 ? __timeval_err : 0; \ + __timeval_err; \ + }) #endif /* @@ -281,7 +296,7 @@ int __lockfunc resilient_tas_spin_lock(rqspinlock_t *lock) val = atomic_read(&lock->val); if (val || !atomic_try_cmpxchg(&lock->val, &val, 1)) { - if (RES_CHECK_TIMEOUT(ts, ret, ~0u)) + if (RES_CHECK_TIMEOUT(ts, ret, ~0u) < 0) goto out; cpu_relax(); goto retry; @@ -406,7 +421,7 @@ int __lockfunc resilient_queued_spin_lock_slowpath(rqspinlock_t *lock, u32 val) */ if (val & _Q_LOCKED_MASK) { RES_RESET_TIMEOUT(ts, RES_DEF_TIMEOUT); - res_smp_cond_load_acquire(&lock->locked, !VAL || RES_CHECK_TIMEOUT(ts, ret, _Q_LOCKED_MASK)); + res_smp_cond_load_acquire(&lock->locked, !VAL || RES_CHECK_TIMEOUT(ts, ret, _Q_LOCKED_MASK) < 0); } if (ret) { @@ -568,7 +583,7 @@ int __lockfunc resilient_queued_spin_lock_slowpath(rqspinlock_t *lock, u32 val) */ RES_RESET_TIMEOUT(ts, RES_DEF_TIMEOUT * 2); val = res_atomic_cond_read_acquire(&lock->val, !(VAL & _Q_LOCKED_PENDING_MASK) || - RES_CHECK_TIMEOUT(ts, ret, _Q_LOCKED_PENDING_MASK)); + RES_CHECK_TIMEOUT(ts, ret, _Q_LOCKED_PENDING_MASK) < 0); /* Disable queue destruction when we detect deadlocks. */ if (ret == -EDEADLK) { -- 2.31.1