Move vmx_handle_exit_irqoff() and its helpers to common.h / main.c to capture that it's a common handler and to allow guarding against incorrectly using to_vmx(). Opportunistically use a "vt" prefix instead of "vmx" to communicate that it's a shared handler. No functional change intended. Cc: Rick Edgecombe Cc: Xiaoyao Li Cc: Binbin Wu Cc: Kai Huang Cc: Yan Zhao Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/common.h | 6 ++++ arch/x86/kvm/vmx/main.c | 71 +++++++++++++++++++++++++++++++++++- arch/x86/kvm/vmx/vmx.c | 74 -------------------------------------- arch/x86/kvm/vmx/x86_ops.h | 1 - 4 files changed, 76 insertions(+), 76 deletions(-) diff --git a/arch/x86/kvm/vmx/common.h b/arch/x86/kvm/vmx/common.h index 08005676702c..88f637c81353 100644 --- a/arch/x86/kvm/vmx/common.h +++ b/arch/x86/kvm/vmx/common.h @@ -74,6 +74,12 @@ static __always_inline bool is_td_vcpu(struct kvm_vcpu *vcpu) { return false; } #endif +static inline bool is_xfd_nm_fault(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.guest_fpu.fpstate->xfd && + !kvm_is_cr0_bit_set(vcpu, X86_CR0_TS); +} + static inline bool vt_is_tdx_private_gpa(struct kvm *kvm, gpa_t gpa) { /* For TDX the direct mask is the shared mask. */ diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 4c52ab8d0786..274200b0a307 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include "x86_ops.h" @@ -876,6 +877,74 @@ static int vt_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, #define vt_op_tdx_only(name) NULL #endif /* CONFIG_KVM_INTEL_TDX */ +static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu) +{ + /* + * Save xfd_err to guest_fpu before interrupt is enabled, so the + * MSR value is not clobbered by the host activity before the guest + * has chance to consume it. + * + * Update the guest's XFD_ERR if and only if XFD is enabled, as the #NM + * interception may have been caused by L1 interception. Per the SDM, + * XFD_ERR is not modified for non-XFD #NM, i.e. if CR0.TS=1. + * + * Note, XFD_ERR is updated _before_ the #NM interception check, i.e. + * unlike CR2 and DR6, the value is not a payload that is attached to + * the #NM exception. + */ + if (is_xfd_nm_fault(vcpu)) + rdmsrq(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); +} + +static void handle_exception_irqoff(struct kvm_vcpu *vcpu, u32 intr_info) +{ + /* if exit due to PF check for async PF */ + if (is_page_fault(intr_info)) + vcpu->arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags(); + /* if exit due to NM, handle before interrupts are enabled */ + else if (is_nm_fault(intr_info)) + handle_nm_fault_irqoff(vcpu); + /* Handle machine checks before interrupts are enabled */ + else if (is_machine_check(intr_info)) + kvm_machine_check(); +} + +static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu, + u32 intr_info) +{ + unsigned int vector = intr_info & INTR_INFO_VECTOR_MASK; + + if (KVM_BUG(!is_external_intr(intr_info), vcpu->kvm, + "unexpected VM-Exit interrupt info: 0x%x", intr_info)) + return; + + kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ); + x86_entry_from_kvm(EVENT_TYPE_EXTINT, vector); + kvm_after_interrupt(vcpu); + + vcpu->arch.at_instruction_boundary = true; +} + +static void vt_handle_exit_irqoff(struct kvm_vcpu *vcpu) +{ + if (to_vt(vcpu)->emulation_required) + return; + + switch (vmx_get_exit_reason(vcpu).basic) { + case EXIT_REASON_EXTERNAL_INTERRUPT: + handle_external_interrupt_irqoff(vcpu, vmx_get_intr_info(vcpu)); + break; + case EXIT_REASON_EXCEPTION_NMI: + handle_exception_irqoff(vcpu, vmx_get_intr_info(vcpu)); + break; + case EXIT_REASON_MCE_DURING_VMENTRY: + kvm_machine_check(); + break; + default: + break; + } +} + #define VMX_REQUIRED_APICV_INHIBITS \ (BIT(APICV_INHIBIT_REASON_DISABLED) | \ BIT(APICV_INHIBIT_REASON_ABSENT) | \ @@ -1000,7 +1069,7 @@ struct kvm_x86_ops vt_x86_ops __initdata = { .load_mmu_pgd = vt_op(load_mmu_pgd), .check_intercept = vmx_check_intercept, - .handle_exit_irqoff = vmx_handle_exit_irqoff, + .handle_exit_irqoff = vt_handle_exit_irqoff, .update_cpu_dirty_logging = vt_op(update_cpu_dirty_logging), diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 504630f0eb40..adf2bc13bed2 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -5379,12 +5379,6 @@ bool vmx_guest_inject_ac(struct kvm_vcpu *vcpu) (kvm_get_rflags(vcpu) & X86_EFLAGS_AC); } -static bool is_xfd_nm_fault(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.guest_fpu.fpstate->xfd && - !kvm_is_cr0_bit_set(vcpu, X86_CR0_TS); -} - static int vmx_handle_page_fault(struct kvm_vcpu *vcpu, u32 error_code) { unsigned long cr2 = vmx_get_exit_qual(vcpu); @@ -7143,74 +7137,6 @@ void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]); } -static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu) -{ - /* - * Save xfd_err to guest_fpu before interrupt is enabled, so the - * MSR value is not clobbered by the host activity before the guest - * has chance to consume it. - * - * Update the guest's XFD_ERR if and only if XFD is enabled, as the #NM - * interception may have been caused by L1 interception. Per the SDM, - * XFD_ERR is not modified for non-XFD #NM, i.e. if CR0.TS=1. - * - * Note, XFD_ERR is updated _before_ the #NM interception check, i.e. - * unlike CR2 and DR6, the value is not a payload that is attached to - * the #NM exception. - */ - if (is_xfd_nm_fault(vcpu)) - rdmsrq(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); -} - -static void handle_exception_irqoff(struct kvm_vcpu *vcpu, u32 intr_info) -{ - /* if exit due to PF check for async PF */ - if (is_page_fault(intr_info)) - vcpu->arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags(); - /* if exit due to NM, handle before interrupts are enabled */ - else if (is_nm_fault(intr_info)) - handle_nm_fault_irqoff(vcpu); - /* Handle machine checks before interrupts are enabled */ - else if (is_machine_check(intr_info)) - kvm_machine_check(); -} - -static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu, - u32 intr_info) -{ - unsigned int vector = intr_info & INTR_INFO_VECTOR_MASK; - - if (KVM_BUG(!is_external_intr(intr_info), vcpu->kvm, - "unexpected VM-Exit interrupt info: 0x%x", intr_info)) - return; - - kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ); - x86_entry_from_kvm(EVENT_TYPE_EXTINT, vector); - kvm_after_interrupt(vcpu); - - vcpu->arch.at_instruction_boundary = true; -} - -void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu) -{ - if (to_vt(vcpu)->emulation_required) - return; - - switch (vmx_get_exit_reason(vcpu).basic) { - case EXIT_REASON_EXTERNAL_INTERRUPT: - handle_external_interrupt_irqoff(vcpu, vmx_get_intr_info(vcpu)); - break; - case EXIT_REASON_EXCEPTION_NMI: - handle_exception_irqoff(vcpu, vmx_get_intr_info(vcpu)); - break; - case EXIT_REASON_MCE_DURING_VMENTRY: - kvm_machine_check(); - break; - default: - break; - } -} - /* * The kvm parameter can be NULL (module initialization, or invocation before * VM creation). Be sure to check the kvm parameter before using it. diff --git a/arch/x86/kvm/vmx/x86_ops.h b/arch/x86/kvm/vmx/x86_ops.h index 054fd14bb2e1..7ac02c68457f 100644 --- a/arch/x86/kvm/vmx/x86_ops.h +++ b/arch/x86/kvm/vmx/x86_ops.h @@ -27,7 +27,6 @@ void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event); void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu); void vmx_vcpu_put(struct kvm_vcpu *vcpu); int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath); -void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu); int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu); void vmx_update_emulated_instruction(struct kvm_vcpu *vcpu); bool vmx_unhandleable_emulation_required(struct kvm_vcpu *vcpu); -- 2.55.0.887.g758fc8c411-goog