From: "Guo Ren (Alibaba DAMO Academy)" The VM guests should fall back to a Test-and-Set spinlock when PARAVIRT_SPINLOCKS disabled, because fair locks have horrible lock 'holder' preemption issues. The virt_spin_lock_key would shortcut for the queued_spin_lock_- slowpath() function that allow virt_spin_lock to hijack it. ref: 43b3f02899f7 ("locking/qspinlock/x86: Fix performance regression under unaccelerated VMs"). Add a static key controlling whether virt_spin_lock() should be called or not. Add nopvspin support as x86. Signed-off-by: Guo Ren (Alibaba DAMO Academy) --- .../admin-guide/kernel-parameters.txt | 2 +- arch/riscv/include/asm/qspinlock.h | 24 +++++++++++++++++++ arch/riscv/kernel/qspinlock_paravirt.c | 8 +++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 6c42061ca20e..9e895e9ca655 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -4393,7 +4393,7 @@ as generic guest with no PV drivers. Currently support XEN HVM, KVM, HYPER_V and VMWARE guest. - nopvspin [X86,XEN,KVM,EARLY] + nopvspin [X86,RISCV,XEN,KVM,EARLY] Disables the qspinlock slow path using PV optimizations which allow the hypervisor to 'idle' the guest on lock contention. diff --git a/arch/riscv/include/asm/qspinlock.h b/arch/riscv/include/asm/qspinlock.h index b39f23415ec1..70ad7679fce0 100644 --- a/arch/riscv/include/asm/qspinlock.h +++ b/arch/riscv/include/asm/qspinlock.h @@ -14,6 +14,8 @@ /* How long a lock should spin before we consider blocking */ #define SPIN_THRESHOLD (1 << 15) +extern bool nopvspin; + void native_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); void __pv_init_lock_hash(void); void __pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); @@ -31,5 +33,27 @@ static inline void queued_spin_unlock(struct qspinlock *lock) #endif /* CONFIG_PARAVIRT_SPINLOCKS */ #include +#include + +/* + * The KVM guests fall back to a Test-and-Set spinlock, because fair locks + * have horrible lock 'holder' preemption issues. The test_and_set_spinlock_key + * would shortcut for the queued_spin_lock_slowpath() function that allow + * virt_spin_lock to hijack it. + */ +DECLARE_STATIC_KEY_FALSE(virt_spin_lock_key); + +#define virt_spin_lock rv_virt_spin_lock +static inline bool rv_virt_spin_lock(struct qspinlock *lock) +{ + if (!static_branch_likely(&virt_spin_lock_key)) + return false; + + do { + smp_cond_load_relaxed((s32 *)&lock->val, VAL == 0); + } while (atomic_cmpxchg(&lock->val, 0, _Q_LOCKED_VAL) != 0); + + return true; +} #endif /* _ASM_RISCV_QSPINLOCK_H */ diff --git a/arch/riscv/kernel/qspinlock_paravirt.c b/arch/riscv/kernel/qspinlock_paravirt.c index cae991139abe..b89f13d0d6e8 100644 --- a/arch/riscv/kernel/qspinlock_paravirt.c +++ b/arch/riscv/kernel/qspinlock_paravirt.c @@ -50,6 +50,8 @@ EXPORT_STATIC_CALL(pv_queued_spin_lock_slowpath); DEFINE_STATIC_CALL(pv_queued_spin_unlock, native_queued_spin_unlock); EXPORT_STATIC_CALL(pv_queued_spin_unlock); +DEFINE_STATIC_KEY_FALSE(virt_spin_lock_key); + bool __init pv_qspinlock_init(void) { if (num_possible_cpus() == 1) @@ -58,6 +60,12 @@ bool __init pv_qspinlock_init(void) if (!sbi_probe_extension(SBI_EXT_PVLOCK)) return false; + if (nopvspin) { + static_branch_enable(&virt_spin_lock_key); + pr_info("virt_spin_lock enabled by nopvspin\n"); + return true; + } + pr_info("PV qspinlocks enabled\n"); __pv_init_lock_hash(); -- 2.40.1