Keep the original SIE control block (SCB) pinned and only lazily unpin it on reuse of the vsie_page for a different SCB. - Pinned pages are tracked and will be unpinned on VM destruction - Memory pressure is not significantly impacted as the number of pinned SCBs is bounded by the number of vCPUs - Reuse detection ensures stale pins are released when needed {,un}pin_scb() methods are extended to track the pin status in the vsie_page->flags. A new vsie_page_init() method is created to allow reuse for common tasks in following patches. Signed-off-by: Christoph Schlameuss --- arch/s390/kvm/s390/vsie.c | 129 +++++++++++++++++++++++++++++++--------------- 1 file changed, 88 insertions(+), 41 deletions(-) diff --git a/arch/s390/kvm/s390/vsie.c b/arch/s390/kvm/s390/vsie.c index 988370f491bf..24af48af89cd 100644 --- a/arch/s390/kvm/s390/vsie.c +++ b/arch/s390/kvm/s390/vsie.c @@ -29,6 +29,7 @@ enum vsie_page_flags { VSIE_PAGE_IN_USE = 0, + VSIE_PAGE_SCB_PINNED = 1, }; struct vsie_page { @@ -774,14 +775,18 @@ static int shadow_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) } /* unpin the scb provided by guest 2, marking it as dirty */ -static void unpin_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, - gpa_t gpa) +static void unpin_scb(struct kvm *kvm, struct vsie_page *vsie_page) { - hpa_t hpa = virt_to_phys(vsie_page->scb_o); + hpa_t hpa; + + if (!test_bit(VSIE_PAGE_SCB_PINNED, &vsie_page->flags)) + return; + hpa = virt_to_phys(vsie_page->scb_o); if (hpa) - unpin_guest_page(vcpu->kvm, gpa, hpa); + unpin_guest_page(kvm, vsie_page->scb_gpa, hpa); vsie_page->scb_o = NULL; + clear_bit(VSIE_PAGE_SCB_PINNED, &vsie_page->flags); } /* @@ -790,19 +795,22 @@ static void unpin_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, * Returns: - 0 if the scb was pinned. * - > 0 if control has to be given to guest 2 */ -static int pin_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, - gpa_t gpa) +static int pin_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) { hpa_t hpa; int rc; - rc = pin_guest_page(vcpu->kvm, gpa, &hpa); + if (test_bit(VSIE_PAGE_SCB_PINNED, &vsie_page->flags)) + return 0; + + rc = pin_guest_page(vcpu->kvm, vsie_page->scb_gpa, &hpa); if (rc) { rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); WARN_ON_ONCE(rc); return 1; } vsie_page->scb_o = phys_to_virt(hpa); + set_bit(VSIE_PAGE_SCB_PINNED, &vsie_page->flags); return 0; } @@ -1542,6 +1550,22 @@ static struct vsie_page *alloc_vsie_page(struct kvm *kvm) return vsie_page; } +static int init_vsie_page(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, unsigned long scb_gpa) +{ + int rc; + + vsie_page->scb_gpa = scb_gpa; + rc = pin_scb(vcpu, vsie_page); + if (rc) { + vsie_page->scb_gpa = ULONG_MAX; + return rc; + } + + vsie_page->sca_gpa = read_scao(vcpu->kvm, vsie_page->scb_o); + + return 0; +} + /* Reset shadow state after a vsie_page has been (re)initialised for a new SCB. */ static void reset_vsie_page(struct kvm *kvm, struct vsie_page *vsie_page) { @@ -1555,14 +1579,21 @@ static void reset_vsie_page(struct kvm *kvm, struct vsie_page *vsie_page) /* * Get or create a vsie page for a scb address. * + * Original control blocks are pinned when the vsie_page pointing to them is + * returned. + * Newly created vsie_pages only have vsie_page->scb_gpa and vsie_page->sca_gpa + * set. + * * Returns: - address of a vsie page (cached or new one) * - NULL if the same scb address is already used by another VCPU * - ERR_PTR(-ENOMEM) if out of memory */ -static struct vsie_page *get_vsie_page(struct kvm *kvm, unsigned long addr) +static struct vsie_page *get_vsie_page(struct kvm_vcpu *vcpu, unsigned long addr) { - struct vsie_page *vsie_page; - int nr_vcpus; + struct vsie_page *vsie_page, *vsie_page_new = NULL; + struct kvm *kvm = vcpu->kvm; + unsigned int max_vsie_page; + int rc, pages_idx; vsie_page = xa_load(&kvm->arch.vsie.addr_to_page, addr >> SCB_ALIGNMENT_SHIFT); if (vsie_page && try_get_vsie_page(vsie_page)) { @@ -1575,47 +1606,65 @@ static struct vsie_page *get_vsie_page(struct kvm *kvm, unsigned long addr) put_vsie_page(vsie_page); } - /* - * We want at least #online_vcpus shadows, so every VCPU can execute - * the VSIE in parallel. - */ - nr_vcpus = atomic_read(&kvm->online_vcpus); + max_vsie_page = atomic_read(&kvm->online_vcpus); + + /* allocate new vsie_page - we will likely need it */ + if (kvm->arch.vsie.page_count < max_vsie_page) { + vsie_page_new = alloc_vsie_page(kvm); + if (!vsie_page_new) + return ERR_PTR(-ENOMEM); + __set_bit(VSIE_PAGE_IN_USE, &vsie_page_new->flags); + } mutex_lock(&kvm->arch.vsie.mutex); - if (kvm->arch.vsie.page_count < nr_vcpus) { - vsie_page = alloc_vsie_page(kvm); - if (!vsie_page) { + vsie_page = xa_load(&kvm->arch.vsie.addr_to_page, addr >> SCB_ALIGNMENT_SHIFT); + if (vsie_page && try_get_vsie_page(vsie_page)) { + if (vsie_page->scb_gpa == addr) { mutex_unlock(&kvm->arch.vsie.mutex); - return ERR_PTR(-ENOMEM); + if (vsie_page_new) + free_vsie_page(vsie_page_new); + return vsie_page; } - __set_bit(VSIE_PAGE_IN_USE, &vsie_page->flags); - kvm->arch.vsie.pages[kvm->arch.vsie.page_count] = vsie_page; + /* + * We raced with someone reusing + putting this vsie + * page before we grabbed it. + */ + put_vsie_page(vsie_page); + } + + if (kvm->arch.vsie.page_count < max_vsie_page) { + pages_idx = kvm->arch.vsie.page_count; + vsie_page = vsie_page_new; + vsie_page_new = NULL; + WRITE_ONCE(kvm->arch.vsie.pages[kvm->arch.vsie.page_count], vsie_page); kvm->arch.vsie.page_count++; } else { /* reuse an existing entry that belongs to nobody */ while (true) { - vsie_page = kvm->arch.vsie.pages[kvm->arch.vsie.next]; + pages_idx = kvm->arch.vsie.next; + kvm->arch.vsie.next++; + kvm->arch.vsie.next %= kvm->arch.vsie.page_count; + vsie_page = kvm->arch.vsie.pages[pages_idx]; if (try_get_vsie_page(vsie_page)) break; - kvm->arch.vsie.next++; - kvm->arch.vsie.next %= nr_vcpus; } - if (vsie_page->scb_gpa != ULONG_MAX) - xa_erase(&kvm->arch.vsie.addr_to_page, - vsie_page->scb_gpa >> SCB_ALIGNMENT_SHIFT); - /* Mark it as invalid until it resides in the tree. */ - vsie_page->scb_gpa = ULONG_MAX; + + unpin_scb(kvm, vsie_page); } - /* Double use of the same address or allocation failure. */ - if (xa_insert(&kvm->arch.vsie.addr_to_page, addr >> SCB_ALIGNMENT_SHIFT, vsie_page, - GFP_KERNEL_ACCOUNT)) { + rc = init_vsie_page(vcpu, vsie_page, addr); + if (!rc) + xa_store(&kvm->arch.vsie.addr_to_page, addr >> SCB_ALIGNMENT_SHIFT, vsie_page, + GFP_KERNEL_ACCOUNT); + + mutex_unlock(&kvm->arch.vsie.mutex); + if (vsie_page_new) + free_vsie_page(vsie_page_new); + if (rc) { + vsie_page->scb_gpa = ULONG_MAX; put_vsie_page(vsie_page); - mutex_unlock(&kvm->arch.vsie.mutex); - return NULL; + return rc < 0 ? ERR_PTR(rc) : NULL; } - vsie_page->scb_gpa = addr; - mutex_unlock(&kvm->arch.vsie.mutex); reset_vsie_page(kvm, vsie_page); @@ -1645,7 +1694,7 @@ int kvm_s390_handle_vsie(struct kvm_vcpu *vcpu) return 0; } - vsie_page = get_vsie_page(vcpu->kvm, scb_addr); + vsie_page = get_vsie_page(vcpu, scb_addr); if (IS_ERR(vsie_page)) { return PTR_ERR(vsie_page); } else if (!vsie_page) { @@ -1654,12 +1703,12 @@ int kvm_s390_handle_vsie(struct kvm_vcpu *vcpu) return 0; } - rc = pin_scb(vcpu, vsie_page, scb_addr); + rc = pin_scb(vcpu, vsie_page); if (rc) goto out_put; rc = shadow_scb(vcpu, vsie_page); if (rc) - goto out_unpin_scb; + goto out_put; rc = pin_blocks(vcpu, vsie_page); if (rc) goto out_unshadow; @@ -1669,8 +1718,6 @@ int kvm_s390_handle_vsie(struct kvm_vcpu *vcpu) unpin_blocks(vcpu, vsie_page); out_unshadow: unshadow_scb(vcpu, vsie_page); -out_unpin_scb: - unpin_scb(vcpu, vsie_page, scb_addr); out_put: put_vsie_page(vsie_page); -- 2.55.0