Add the split page cache for dirty logging enablement and the KVM_CLEAR_DIRTY_LOG ioctl. Signed-off-by: Wang Yechao --- arch/riscv/include/asm/kvm_host.h | 1 + arch/riscv/kvm/mmu.c | 3 +++ 2 files changed, 4 insertions(+) diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h index e2d5808169e44..71e455d166905 100644 --- a/arch/riscv/include/asm/kvm_host.h +++ b/arch/riscv/include/asm/kvm_host.h @@ -86,6 +86,7 @@ struct kvm_arch { pgd_t *pgd; phys_addr_t pgd_phys; unsigned long pgd_levels; + struct kvm_mmu_memory_cache pgd_split_page_cache; /* Guest Timer */ struct kvm_guest_timer timer; diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c index 8a0aa5e0e216e..2017d292f4ca7 100644 --- a/arch/riscv/kvm/mmu.c +++ b/arch/riscv/kvm/mmu.c @@ -677,6 +677,7 @@ int kvm_riscv_mmu_alloc_pgd(struct kvm *kvm) kvm->arch.pgd = page_to_virt(pgd_page); kvm->arch.pgd_phys = page_to_phys(pgd_page); kvm->arch.pgd_levels = kvm_riscv_gstage_max_pgd_levels; + kvm->arch.pgd_split_page_cache.gfp_zero = __GFP_ZERO; return 0; } @@ -704,6 +705,8 @@ void kvm_riscv_mmu_free_pgd(struct kvm *kvm) if (pgd) free_pages((unsigned long)pgd, get_order(kvm_riscv_gstage_pgd_size)); + + kvm_mmu_free_memory_cache(&kvm->arch.pgd_split_page_cache); } void kvm_riscv_mmu_update_hgatp(struct kvm_vcpu *vcpu) -- 2.39.3 Split huge pages eagerly when enabling dirty logging. The goal is to avoid doing it while faulting on write-protected pages, which negatively impacts guest performance. The benefits of eager page splitting are the same as in x86 and arm64, added with commit a3fe5dbda0a4 ("KVM: x86/mmu: Split huge pages mapped by the TDP MMU when dirty logging is enabled") and commit e7bf7a490c68 ("KVM: arm64: Split huge pages when dirty logging is enabled") Signed-off-by: Wang Yechao Reviewed-by: Anup Patel --- arch/riscv/include/asm/kvm_gstage.h | 6 +-- arch/riscv/kvm/gstage.c | 23 ++++++--- arch/riscv/kvm/mmu.c | 76 +++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 10 deletions(-) diff --git a/arch/riscv/include/asm/kvm_gstage.h b/arch/riscv/include/asm/kvm_gstage.h index 21e2019df0cf5..f726279780177 100644 --- a/arch/riscv/include/asm/kvm_gstage.h +++ b/arch/riscv/include/asm/kvm_gstage.h @@ -64,9 +64,9 @@ int kvm_riscv_gstage_map_page(struct kvm_gstage *gstage, bool page_rdonly, bool page_exec, struct kvm_gstage_mapping *out_map); -int kvm_riscv_gstage_split_huge(struct kvm_gstage *gstage, - struct kvm_mmu_memory_cache *pcache, - gpa_t addr, u32 target_level, bool flush); +bool kvm_riscv_gstage_split_huge(struct kvm_gstage *gstage, + struct kvm_mmu_memory_cache *pcache, + gpa_t addr, u32 target_level, bool flush); enum kvm_riscv_gstage_op { GSTAGE_OP_NOP = 0, /* Nothing */ diff --git a/arch/riscv/kvm/gstage.c b/arch/riscv/kvm/gstage.c index b0474fcf065aa..01679aef53289 100644 --- a/arch/riscv/kvm/gstage.c +++ b/arch/riscv/kvm/gstage.c @@ -307,19 +307,20 @@ static inline unsigned long make_child_pte(unsigned long huge_pte, int index, return child_pte; } -int kvm_riscv_gstage_split_huge(struct kvm_gstage *gstage, - struct kvm_mmu_memory_cache *pcache, - gpa_t addr, u32 target_level, bool flush) +bool kvm_riscv_gstage_split_huge(struct kvm_gstage *gstage, + struct kvm_mmu_memory_cache *pcache, + gpa_t addr, u32 target_level, bool flush) { u32 current_level = gstage->pgd_levels - 1; pte_t *next_ptep = (pte_t *)gstage->pgd; unsigned long huge_pte, child_pte; unsigned long child_page_size; + bool need_flush = false; pte_t *ptep; int i, ret; if (!pcache) - return -ENOMEM; + return false; while(current_level > target_level) { ptep = (pte_t *)&next_ptep[gstage_pte_index(gstage, addr, current_level)]; @@ -337,27 +338,35 @@ int kvm_riscv_gstage_split_huge(struct kvm_gstage *gstage, ret = gstage_level_to_page_size(gstage, current_level - 1, &child_page_size); if (ret) - return ret; + return need_flush; next_ptep = kvm_mmu_memory_cache_alloc(pcache); if (!next_ptep) - return -ENOMEM; + return need_flush; for (i = 0; i < PTRS_PER_PTE; i++) { child_pte = make_child_pte(huge_pte, i, child_page_size); set_pte((pte_t *)&next_ptep[i], __pte(child_pte)); } + /* + * Ensure the writes to the child PTEs are visible before + * linking the new page table to the parent PTE. + */ + smp_wmb(); + set_pte(ptep, pfn_pte(PFN_DOWN(__pa(next_ptep)), __pgprot(_PAGE_TABLE))); if (flush) gstage_tlb_flush(gstage, current_level, addr); + else + need_flush = true; current_level--; } - return 0; + return need_flush; } bool kvm_riscv_gstage_op_pte(struct kvm_gstage *gstage, gpa_t addr, diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c index 2017d292f4ca7..2362eeb6c08a5 100644 --- a/arch/riscv/kvm/mmu.c +++ b/arch/riscv/kvm/mmu.c @@ -98,6 +98,62 @@ void kvm_riscv_mmu_iounmap(struct kvm *kvm, gpa_t gpa, unsigned long size) size >> PAGE_SHIFT); } +static bool need_topup_split_caches_or_resched(struct kvm *kvm, int count) +{ + struct kvm_mmu_memory_cache *cache; + + if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) + return true; + + cache = &kvm->arch.pgd_split_page_cache; + return kvm_mmu_memory_cache_nr_free_objects(cache) < count; +} + +static bool mmu_split_huge_pages(struct kvm_gstage *gstage, + phys_addr_t start, phys_addr_t end) +{ + struct kvm *kvm = gstage->kvm; + struct kvm_mmu_memory_cache *pcache = &kvm->arch.pgd_split_page_cache; + phys_addr_t addr = ALIGN_DOWN(start, PMD_SIZE); + phys_addr_t last_flush_gfn = addr >> PAGE_SHIFT; + int count = gstage->pgd_levels; + bool flush = false; + int ret; + + lockdep_assert_held_write(&kvm->mmu_lock); + + while (addr < end) { + if (need_topup_split_caches_or_resched(kvm, count)) { + if (flush) { + kvm_flush_remote_tlbs_range(kvm, last_flush_gfn, + (addr >> PAGE_SHIFT) - last_flush_gfn); + last_flush_gfn = addr >> PAGE_SHIFT; + flush = false; + } + + write_unlock(&kvm->mmu_lock); + cond_resched(); + + ret = kvm_mmu_topup_memory_cache(pcache, count); + if (ret) { + kvm_err("Failed to toup split page cache\n"); + write_lock(&kvm->mmu_lock); + return flush; + } + write_lock(&kvm->mmu_lock); + } + + if (!kvm->arch.pgd) + return flush; + + flush |= kvm_riscv_gstage_split_huge(gstage, pcache, addr, 0, false); + + addr += PMD_SIZE; + } + + return flush; +} + void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn_offset, @@ -152,6 +208,25 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm, size >> PAGE_SHIFT); } +static void mmu_split_memory_region(struct kvm *kvm, int slot) +{ + struct kvm_memslots *slots = kvm_memslots(kvm); + struct kvm_memory_slot *memslot = id_to_memslot(slots, slot); + phys_addr_t start = memslot->base_gfn << PAGE_SHIFT; + phys_addr_t end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT; + struct kvm_gstage gstage; + bool flush; + + kvm_riscv_gstage_init(&gstage, kvm); + + write_lock(&kvm->mmu_lock); + flush = mmu_split_huge_pages(&gstage, start, end); + write_unlock(&kvm->mmu_lock); + + if (flush) + kvm_flush_remote_tlbs_memslot(kvm, memslot); +} + void kvm_arch_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *old, const struct kvm_memory_slot *new, @@ -165,6 +240,7 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, if (kvm_dirty_log_manual_protect_and_init_set(kvm)) return; mmu_wp_memory_region(kvm, new->id); + mmu_split_memory_region(kvm, new->id); } } -- 2.39.3 The function kvm_arch_mmu_enable_log_dirty_pt_masked() is invoked from two distinct call paths: kvm_clear_dirty_log_protect() kvm_arch_mmu_enable_log_dirty_pt_masked() kvm_vm_ioctl_reset_dirty_pages() kvm_dirty_ring_reset() kvm_reset_dirty_gfn() kvm_arch_mmu_enable_log_dirty_pt_masked() In both scenarios, the caller already performs a remote TLB flush after dirty logging is enabled, so the TLB flush inside kvm_arch_mmu_enable_log_dirty_pt_masked() is unnecessary. Remove it. Signed-off-by: Wang Yechao Reviewed-by: Anup Patel --- arch/riscv/kvm/mmu.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c index 2362eeb6c08a5..5a93cffa9ad7f 100644 --- a/arch/riscv/kvm/mmu.c +++ b/arch/riscv/kvm/mmu.c @@ -163,14 +163,15 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, phys_addr_t start = (base_gfn + __ffs(mask)) << PAGE_SHIFT; phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT; struct kvm_gstage gstage; - bool flush; kvm_riscv_gstage_init(&gstage, kvm); - flush = kvm_riscv_gstage_wp_range(&gstage, start, end); - if (flush) - kvm_flush_remote_tlbs_range(kvm, start >> PAGE_SHIFT, - (end - start) >> PAGE_SHIFT); + kvm_riscv_gstage_wp_range(&gstage, start, end); + + /* + * remote TLB flush is not needed here since callers of + * kvm_arch_mmu_enable_log_dirty_pt_masked() already do it. + */ } void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) -- 2.39.3 Split huge pages on the range specified using KVM_CLEAR_DIRTY_LOG. And do not split when enabling dirty logging if KVM_DIRTY_LOG_INITIALLY_SET is set. Signed-off-by: Wang Yechao Reviewed-by: Anup Patel --- arch/riscv/kvm/mmu.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c index 5a93cffa9ad7f..722637a85bdf9 100644 --- a/arch/riscv/kvm/mmu.c +++ b/arch/riscv/kvm/mmu.c @@ -168,6 +168,9 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, kvm_riscv_gstage_wp_range(&gstage, start, end); + if (kvm_dirty_log_manual_protect_and_init_set(kvm)) + mmu_split_huge_pages(&gstage, start, end); + /* * remote TLB flush is not needed here since callers of * kvm_arch_mmu_enable_log_dirty_pt_masked() already do it. -- 2.39.3 Add an eager_page_split module parameter for RISC-V KVM, following the same approach as on x86. This parameter controls whether eager page splitting is enabled. The default value is on. When eager page splitting is enabled, KVM proactively splits large pages (huge pages) into smaller pages when needed for dirty logging or other operations. Disabling it can be beneficial for VM workloads that rarely perform writes, or that only write to a small region of memory, as it allows huge pages to remain intact for read accesses. Signed-off-by: Wang Yechao Reviewed-by: Anup Patel --- Documentation/admin-guide/kernel-parameters.txt | 7 +++++-- arch/riscv/kvm/mmu.c | 13 ++++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index b5493a7f8f228..d78eda7ee23a5 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3047,7 +3047,7 @@ Kernel parameters Default is 0 (don't ignore, but inject #GP) kvm.eager_page_split= - [KVM,X86] Controls whether or not KVM will try to + [KVM,X86,RISCV] Controls whether or not KVM will try to proactively split all huge pages during dirty logging. Eager page splitting reduces interruptions to vCPU execution by eliminating the write-protection faults @@ -3067,7 +3067,10 @@ Kernel parameters the KVM_CLEAR_DIRTY ioctl, and only for the pages being cleared. - Eager page splitting is only supported when kvm.tdp_mmu=Y. + On x86, eager page splitting is only supported when + kvm.tdp_mmu=Y. + + On RISCV, eager page splitting is supported by default. Default is Y (on). diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c index 722637a85bdf9..4314753fc47a8 100644 --- a/arch/riscv/kvm/mmu.c +++ b/arch/riscv/kvm/mmu.c @@ -16,6 +16,9 @@ #include #include +static bool __read_mostly eager_page_split = true; +module_param(eager_page_split, bool, 0644); + static void mmu_wp_memory_region(struct kvm *kvm, int slot) { struct kvm_memslots *slots = kvm_memslots(kvm); @@ -168,8 +171,10 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, kvm_riscv_gstage_wp_range(&gstage, start, end); - if (kvm_dirty_log_manual_protect_and_init_set(kvm)) - mmu_split_huge_pages(&gstage, start, end); + if (kvm_dirty_log_manual_protect_and_init_set(kvm)) { + if (READ_ONCE(eager_page_split)) + mmu_split_huge_pages(&gstage, start, end); + } /* * remote TLB flush is not needed here since callers of @@ -244,7 +249,9 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, if (kvm_dirty_log_manual_protect_and_init_set(kvm)) return; mmu_wp_memory_region(kvm, new->id); - mmu_split_memory_region(kvm, new->id); + + if (READ_ONCE(eager_page_split)) + mmu_split_memory_region(kvm, new->id); } } -- 2.39.3