Add page table management functions to be used for KVM guest (gmap) page tables. This patch adds functions to clear, replace or exchange DAT table entries. Signed-off-by: Claudio Imbrenda --- arch/s390/kvm/dat.c | 120 ++++++++++++++++++++++++++++++++++++++++++++ arch/s390/kvm/dat.h | 40 +++++++++++++++ 2 files changed, 160 insertions(+) diff --git a/arch/s390/kvm/dat.c b/arch/s390/kvm/dat.c index 326be78adcda..f26e3579bd77 100644 --- a/arch/s390/kvm/dat.c +++ b/arch/s390/kvm/dat.c @@ -89,3 +89,123 @@ void dat_free_level(struct crst_table *table, bool owns_ptes) } dat_free_crst(table); } + +/** + * dat_crstep_xchg - exchange a guest CRST with another + * @crstep: pointer to the CRST entry + * @new: replacement entry + * @gfn: the affected guest address + * @asce: the ASCE of the address space + * + * This function is assumed to be called with the guest_table_lock + * held. + */ +void dat_crstep_xchg(union crste *crstep, union crste new, gfn_t gfn, union asce asce) +{ + if (crstep->h.i) { + WRITE_ONCE(*crstep, new); + return; + } else if (cpu_has_edat2()) { + crdte_crste(crstep, *crstep, new, gfn, asce); + return; + } + + if (machine_has_tlb_guest()) + idte_crste(crstep, gfn, IDTE_GUEST_ASCE, asce, IDTE_GLOBAL); + else if (cpu_has_idte()) + idte_crste(crstep, gfn, 0, NULL_ASCE, IDTE_GLOBAL); + else + csp_invalidate_crste(crstep); + WRITE_ONCE(*crstep, new); +} + +/** + * dat_crstep_xchg_atomic - exchange a gmap pmd with another + * @crstep: pointer to the crste entry + * @old: expected old value + * @new: replacement entry + * @gfn: the affected guest address + * @asce: the asce of the address space + * + * This function should only be called on invalid crstes, or on crstes with + * FC = 1, as that guarantees the presence of CSPG. + * + * Return: true if the exchange was successful. + */ +bool dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new, gfn_t gfn, + union asce asce) +{ + if (old.h.i) + return arch_try_cmpxchg((long *)crstep, &old.val, new.val); + if (cpu_has_edat2()) + return crdte_crste(crstep, old, new, gfn, asce); + if (cpu_has_idte()) + return cspg_crste(crstep, old, new); + + WARN_ONCE(1, "Machine does not have CSPG and DAT table was not invalid."); + return false; +} + +static void dat_set_storage_key_from_pgste(union pte pte, union pgste pgste) +{ + union skey nkey = { .acc = pgste.acc, .fp = pgste.fp }; + + page_set_storage_key(pte_origin(pte), nkey.skey, 0); +} + +static void dat_move_storage_key(union pte old, union pte new) +{ + page_set_storage_key(pte_origin(new), page_get_storage_key(pte_origin(old)), 1); +} + +static union pgste dat_save_storage_key_into_pgste(union pte pte, union pgste pgste) +{ + union skey skey; + + skey.skey = page_get_storage_key(pte_origin(pte)); + + pgste.acc = skey.acc; + pgste.fp = skey.fp; + pgste.gr |= skey.r; + pgste.gc |= skey.c; + + return pgste; +} + +union pgste __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, gfn_t gfn, + union asce asce, bool has_skeys) +{ + union pte old = READ_ONCE(*ptep); + + /* Updating only the software bits while holding the pgste lock */ + if (!((ptep->val ^ new.val) & ~_PAGE_SW_BITS)) { + WRITE_ONCE(ptep->swbyte, new.swbyte); + return pgste; + } + + if (!old.h.i) { + unsigned long opts = IPTE_GUEST_ASCE | (pgste.nodat ? IPTE_NODAT : 0); + + if (machine_has_tlb_guest()) + __ptep_ipte(gfn_to_gpa(gfn), (void *)ptep, opts, asce.val, IPTE_GLOBAL); + else + __ptep_ipte(gfn_to_gpa(gfn), (void *)ptep, 0, 0, IPTE_GLOBAL); + } + + if (has_skeys) { + if (old.h.i && !new.h.i) + /* Invalid to valid: restore storage keys from PGSTE */ + dat_set_storage_key_from_pgste(new, pgste); + else if (!old.h.i && new.h.i) + /* Valid to invalid: save storage keys to PGSTE */ + pgste = dat_save_storage_key_into_pgste(new, pgste); + else if (!old.h.i && !new.h.i) + /* Valid to valid: move storage keys */ + if (old.h.pfra != new.h.pfra) + dat_move_storage_key(*ptep, new); + /* Invalid to invalid: nothing to do */ + } + + WRITE_ONCE(*ptep, new); + return pgste; +} diff --git a/arch/s390/kvm/dat.h b/arch/s390/kvm/dat.h index 5056cfa02619..9e23f6cdbf73 100644 --- a/arch/s390/kvm/dat.h +++ b/arch/s390/kvm/dat.h @@ -385,6 +385,12 @@ static inline union crste _crste_fc1(kvm_pfn_t pfn, int tt, bool w, bool d) return res; } +union pgste __must_check __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, + gfn_t gfn, union asce asce, bool has_skeys); +bool dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new, gfn_t gfn, + union asce asce); +void dat_crstep_xchg(union crste *crstep, union crste new, gfn_t gfn, union asce asce); + void dat_free_level(struct crst_table *table, bool owns_ptes); struct page_table *dat_alloc_pt(unsigned long pte_bits, unsigned long pgste_bits); struct crst_table *dat_alloc_crst(unsigned long init); @@ -677,6 +683,21 @@ static inline void pgste_set_unlock(union pte *ptep, union pgste pgste) WRITE_ONCE(*pgste_of(ptep), pgste); } +static inline void dat_ptep_xchg(union pte *ptep, union pte new, gfn_t gfn, union asce asce, + bool has_skeys) +{ + union pgste pgste; + + pgste = pgste_get_lock(ptep); + pgste = __dat_ptep_xchg(ptep, pgste, new, gfn, asce, has_skeys); + pgste_set_unlock(ptep, pgste); +} + +static inline void dat_ptep_clear(union pte *ptep, gfn_t gfn, union asce asce, bool has_skeys) +{ + dat_ptep_xchg(ptep, _PTE_EMPTY, gfn, asce, has_skeys); +} + static inline struct page_table *dat_alloc_empty_pt(void) { return dat_alloc_pt(_PAGE_INVALID, 0); @@ -694,4 +715,23 @@ static inline void _dat_free_crst(struct crst_table *table) #define dat_free_crst(x) _dat_free_crst(_CRSTP(x)) +static inline bool dat_pmdp_xchg_atomic(union pmd *pmdp, union pmd old, union pmd new, + gfn_t gfn, union asce asce) +{ + return dat_crstep_xchg_atomic(_CRSTEP(pmdp), _CRSTE(old), _CRSTE(new), gfn, asce); +} + +static inline bool dat_pudp_xchg_atomic(union pud *pudp, union pud old, union pud new, + gfn_t gfn, union asce asce) +{ + return dat_crstep_xchg_atomic(_CRSTEP(pudp), _CRSTE(old), _CRSTE(new), gfn, asce); +} + +static inline void dat_crstep_clear(union crste *crstep, gfn_t gfn, union asce asce) +{ + union crste newcrste = _CRSTE_EMPTY(crstep->h.tt); + + dat_crstep_xchg(crstep, newcrste, gfn, asce); +} + #endif /* __KVM_S390_DAT_H */ -- 2.51.0