From: David Woodhouse Initialize the TSC generation and master clock state at VM creation time, rather than relying on kvm_update_masterclock() from kvm_arch_vcpu_postcreate(). Set cur_tsc_generation=1, cur_tsc_khz, cur_tsc_scaling_ratio, cur_tsc_offset, cur_tsc_nsec and kvmclock_offset from an atomic ktime/TSC snapshot. This establishes a coherent epoch so that: - The first vCPU synchronizes to the existing generation (getting the pre-computed TSC offset) rather than computing its own. - pvclock_update_vm_gtod_copy() succeeds immediately (no zero tsc_hz from uninitialized cur_tsc_scaling_ratio). - KVM_GET_CLOCK returns valid {host_tsc, realtime} fields as soon as any vCPU exists. - KVM_[GS]ET_CLOCK_GUEST works without a prior KVM_RUN. - No O(N) iteration of all vCPUs on each vCPU creation. This replaces the KVM_REQ_MASTERCLOCK_UPDATE check and kvm_update_masterclock() call in kvm_arch_vcpu_postcreate(), which worked around the missing initialization. A masterclock refresh is still performed (synchronously, so that it is valid even for userspace which queries the clock before ever running a vCPU) when the first vCPU is created, since the VM-creation snapshot can go stale while the VM has no vCPUs: a clocksource change notifies only VMs with vCPUs, and a change back to TSC leaves nothing to disagree with the stale snapshot. The generation's recorded offset in kvm_synchronize_tsc() can now only be joined by a vCPU with the same scaling ratio it was recorded with: the initial generation's offset is recorded with the default ratio before any vCPU exists, and a vCPU with a different ratio opens a new generation with its own offset, keeping the generation's ratio and offset describing the same scaling domain. Signed-off-by: David Woodhouse --- arch/x86/kvm/x86.c | 76 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 64 insertions(+), 12 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 940168528721..6dbdb60962fe 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1500,11 +1500,25 @@ void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *value, bool user_initiated) data = kvm->arch.cur_tsc_write + delta; - if (!kvm_check_tsc_unstable()) - offset = kvm->arch.cur_tsc_offset; - else + /* + * The generation's offset can only be joined by a vCPU which + * scales the TSC by the same ratio it was recorded with; the + * initial generation's offset is recorded at VM creation + * with the default ratio, before any vCPU exists. With a + * different ratio, open a new generation (recording this + * vCPU's own offset) rather than leave the generation's + * ratio and offset describing different scaling domains. + */ + if (vcpu->arch.l1_tsc_scaling_ratio != + kvm->arch.cur_tsc_scaling_ratio) { offset = kvm_compute_l1_tsc_offset(vcpu, host_tsc, data); - matched = true; + } else { + if (!kvm_check_tsc_unstable()) + offset = kvm->arch.cur_tsc_offset; + else + offset = kvm_compute_l1_tsc_offset(vcpu, host_tsc, data); + matched = true; + } } __kvm_synchronize_tsc(vcpu, offset, data, ns, matched, user_initiated); @@ -9645,8 +9659,15 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) return; vcpu_load(vcpu); kvm_synchronize_tsc(vcpu, NULL, false); - if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu)) + + /* + * Refresh the VM-creation masterclock snapshot (see + * kvm_arch_init_vm), synchronously so that it is valid even for + * userspace which queries the clock before ever running a vCPU. + */ + if (vcpu->vcpu_idx == 0) kvm_update_masterclock(vcpu->kvm, NULL); + vcpu_put(vcpu); /* poll control enabled by default */ @@ -10065,6 +10086,8 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) { int ret; unsigned long flags; + u64 kvmclock_host_tsc; + s64 kvmclock_ns; if (!kvm_is_vm_type_supported(type)) return -EINVAL; @@ -10096,9 +10119,13 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) seqcount_raw_spinlock_init(&kvm->arch.pvclock_sc, &kvm->arch.tsc_write_lock); ratelimit_state_init(&kvm->arch.kvmclock_update_rs, HZ, 10); ratelimit_set_flags(&kvm->arch.kvmclock_update_rs, RATELIMIT_MSG_ON_RELEASE); - kvm->arch.kvmclock_offset = -get_kvmclock_base_ns(); - kvm->arch.all_vcpus_matched_freq = true; + kvm->arch.default_tsc_khz = max_tsc_khz ? : tsc_khz; + kvm->arch.apic_bus_cycle_ns = APIC_BUS_CYCLE_NS_DEFAULT; + kvm->arch.guest_can_read_msr_platform_info = true; + kvm->arch.enable_pmu = enable_pmu && !kvm->arch.has_protected_pmu; + + kvm->arch.all_vcpus_matched_freq = true; /* * Start with all (zero) vCPUs considered matched, so that a VM whose * vCPUs never reach kvm_track_tsc_matching() — notably when the guest @@ -10106,16 +10133,41 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) * returns early — is still eligible for PVCLOCK_TSC_STABLE_BIT. */ kvm->arch.all_vcpus_matched_tsc = true; + kvm->arch.cur_tsc_scaling_ratio = kvm_caps.default_tsc_scaling_ratio; + kvm->arch.cur_tsc_khz = kvm->arch.default_tsc_khz; raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags); + + /* + * Establish the initial TSC generation for synchronization logic. + * The first vCPU will sync to this, starting its guest TSC at the + * number of cycles elapsed since VM creation rather than at zero. + * This ensures coherence with the kvmclock epoch (also set here), + * and the master clock snapshot taken below is immediately valid. + * It is refreshed again when the first vCPU is created, in case + * the host clocksource changed (and changed back) in between: + * pvclock_gtod_update_fn() does not notify a VM with no vCPUs. + */ +#ifdef CONFIG_X86_64 + if (!kvm_get_time_and_clockread(&kvmclock_ns, &kvmclock_host_tsc)) +#endif + { + /* + * If not simultaneous, we want the TSC to be zero slightly + * *after* the kvmclock is zero. + */ + kvmclock_ns = get_kvmclock_base_ns(); + kvmclock_host_tsc = rdtsc(); + } + kvm->arch.cur_tsc_write = 0; + kvm->arch.cur_tsc_offset = -(s64)kvmclock_host_tsc; + kvm->arch.cur_tsc_nsec = kvmclock_ns; + kvm->arch.cur_tsc_generation = 1; + kvm->arch.kvmclock_offset = -kvmclock_ns; + pvclock_update_vm_gtod_copy(kvm); raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags); - kvm->arch.default_tsc_khz = max_tsc_khz ? : tsc_khz; - kvm->arch.apic_bus_cycle_ns = APIC_BUS_CYCLE_NS_DEFAULT; - kvm->arch.guest_can_read_msr_platform_info = true; - kvm->arch.enable_pmu = enable_pmu && !kvm->arch.has_protected_pmu; - #if IS_ENABLED(CONFIG_HYPERV) spin_lock_init(&kvm->arch.hv_root_tdp_lock); kvm->arch.hv_root_tdp = INVALID_PAGE; -- 2.43.0