Introduce the split_external_spt hook and call it within tdp_mmu_set_spte() for the mirror page table. tdp_mmu_set_spte() is invoked for SPTE transitions under write mmu_lock. For the mirror page table, in addition to the valid transitions from a shadow-present entry to !shadow-present entry, introduce a new valid transition case for splitting and propagate the transition to the external page table via the hook split_external_spt. Signed-off-by: Xiaoyao Li Signed-off-by: Isaku Yamahata Signed-off-by: Yan Zhao --- RFC v2: - Removed the KVM_BUG_ON() in split_external_spt(). (Rick) - Add a comment for the KVM_BUG_ON() in tdp_mmu_set_spte(). (Rick) - Use kvm_x86_call() instead of static_call(). (Binbin) RFC v1: - Split patch. - Dropped invoking hook zap_private_spte and kvm_flush_remote_tlbs() in KVM MMU core. --- arch/x86/include/asm/kvm-x86-ops.h | 1 + arch/x86/include/asm/kvm_host.h | 4 ++++ arch/x86/kvm/mmu/tdp_mmu.c | 29 +++++++++++++++++++++++++---- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index 18a5c3119e1a..7653a45ad5b2 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -98,6 +98,7 @@ KVM_X86_OP_OPTIONAL(link_external_spt) KVM_X86_OP_OPTIONAL(set_external_spte) KVM_X86_OP_OPTIONAL(free_external_spt) KVM_X86_OP_OPTIONAL(remove_external_spte) +KVM_X86_OP_OPTIONAL(split_external_spt) KVM_X86_OP(has_wbinvd_exit) KVM_X86_OP(get_l2_tsc_offset) KVM_X86_OP(get_l2_tsc_multiplier) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 823d1aeef2a8..e431ce0e3180 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1839,6 +1839,10 @@ struct kvm_x86_ops { int (*remove_external_spte)(struct kvm *kvm, gfn_t gfn, enum pg_level level, kvm_pfn_t pfn_for_gfn); + /* Split the external page table into smaller page tables */ + int (*split_external_spt)(struct kvm *kvm, gfn_t gfn, enum pg_level level, + void *external_spt); + bool (*has_wbinvd_exit)(void); u64 (*get_l2_tsc_offset)(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index 46b9f276bb6d..a2c6e6e4773f 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -325,6 +325,7 @@ static void handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn, bool shared); static struct kvm_mmu_page *tdp_mmu_alloc_sp_for_split(bool mirror); +static void *get_external_spt(gfn_t gfn, u64 new_spte, int level); static void tdp_account_mmu_page(struct kvm *kvm, struct kvm_mmu_page *sp) { @@ -384,6 +385,18 @@ static void remove_external_spte(struct kvm *kvm, gfn_t gfn, u64 old_spte, KVM_BUG_ON(ret, kvm); } +static int split_external_spt(struct kvm *kvm, gfn_t gfn, u64 old_spte, + u64 new_spte, int level) +{ + void *external_spt = get_external_spt(gfn, new_spte, level); + int ret; + + KVM_BUG_ON(!external_spt, kvm); + + ret = kvm_x86_call(split_external_spt)(kvm, gfn, level, external_spt); + + return ret; +} /** * handle_removed_pt() - handle a page table removed from the TDP structure * @@ -765,12 +778,20 @@ static u64 tdp_mmu_set_spte(struct kvm *kvm, int as_id, tdp_ptep_t sptep, handle_changed_spte(kvm, as_id, gfn, old_spte, new_spte, level, false); /* - * Users that do non-atomic setting of PTEs don't operate on mirror - * roots, so don't handle it and bug the VM if it's seen. + * Propagate changes of SPTE to the external page table under write + * mmu_lock. + * Current valid transitions: + * - present leaf to !present. + * - present non-leaf to !present. + * - present leaf to present non-leaf (splitting) */ if (is_mirror_sptep(sptep)) { - KVM_BUG_ON(is_shadow_present_pte(new_spte), kvm); - remove_external_spte(kvm, gfn, old_spte, level); + if (!is_shadow_present_pte(new_spte)) + remove_external_spte(kvm, gfn, old_spte, level); + else if (is_last_spte(old_spte, level) && !is_last_spte(new_spte, level)) + split_external_spt(kvm, gfn, old_spte, new_spte, level); + else + KVM_BUG_ON(1, kvm); } return old_spte; -- 2.43.2