From: Sean Christopherson Remove the completely unnecessary assumption that memory mapped into a TDX guest is backed by refcounted struct page memory. From KVM's point of view, TDH_MEM_PAGE_ADD and TDH_MEM_PAGE_AUG are glorified writes to PTEs, so they have no business placing requirements on how KVM and guest_memfd manage memory. Rip out the misguided struct page assumptions/constraints and instead have the two SEAMCALL wrapper APIs take PFN directly. This ensures that for future huge page support in S-EPT, the kernel doesn't pick up even worse assumptions like "a hugepage must be contained in a single folio". Use "kvm_pfn_t pfn" for type safety. Using this KVM type is appropriate since APIs tdh_mem_page_add() and tdh_mem_page_aug() are exported to KVM only. [ Yan: Replace "u64 pfn" with "kvm_pfn_t pfn" ] Signed-off-by: Sean Christopherson Signed-off-by: Yan Zhao --- arch/x86/include/asm/tdx.h | 5 +++-- arch/x86/kvm/vmx/tdx.c | 7 +++---- arch/x86/virt/vmx/tdx/tdx.c | 20 +++++++++++++------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h index a149740b24e8..f3f0b1872176 100644 --- a/arch/x86/include/asm/tdx.h +++ b/arch/x86/include/asm/tdx.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -195,10 +196,10 @@ static inline int pg_level_to_tdx_sept_level(enum pg_level level) u64 tdh_vp_enter(struct tdx_vp *vp, struct tdx_module_args *args); u64 tdh_mng_addcx(struct tdx_td *td, struct page *tdcs_page); -u64 tdh_mem_page_add(struct tdx_td *td, u64 gpa, struct page *page, struct page *source, u64 *ext_err1, u64 *ext_err2); +u64 tdh_mem_page_add(struct tdx_td *td, u64 gpa, kvm_pfn_t pfn, struct page *source, u64 *ext_err1, u64 *ext_err2); u64 tdh_mem_sept_add(struct tdx_td *td, u64 gpa, int level, struct page *page, u64 *ext_err1, u64 *ext_err2); u64 tdh_vp_addcx(struct tdx_vp *vp, struct page *tdcx_page); -u64 tdh_mem_page_aug(struct tdx_td *td, u64 gpa, int level, struct page *page, u64 *ext_err1, u64 *ext_err2); +u64 tdh_mem_page_aug(struct tdx_td *td, u64 gpa, int level, kvm_pfn_t pfn, u64 *ext_err1, u64 *ext_err2); u64 tdh_mem_range_block(struct tdx_td *td, u64 gpa, int level, u64 *ext_err1, u64 *ext_err2); u64 tdh_mng_key_config(struct tdx_td *td); u64 tdh_mng_create(struct tdx_td *td, u16 hkid); diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 1e47c194af53..1f1abc5b5655 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -1624,8 +1624,8 @@ static int tdx_mem_page_add(struct kvm *kvm, gfn_t gfn, enum pg_level level, KVM_BUG_ON(!kvm_tdx->page_add_src, kvm)) return -EIO; - err = tdh_mem_page_add(&kvm_tdx->td, gpa, pfn_to_page(pfn), - kvm_tdx->page_add_src, &entry, &level_state); + err = tdh_mem_page_add(&kvm_tdx->td, gpa, pfn, kvm_tdx->page_add_src, + &entry, &level_state); if (unlikely(tdx_operand_busy(err))) return -EBUSY; @@ -1640,12 +1640,11 @@ static int tdx_mem_page_aug(struct kvm *kvm, gfn_t gfn, { int tdx_level = pg_level_to_tdx_sept_level(level); struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); - struct page *page = pfn_to_page(pfn); gpa_t gpa = gfn_to_gpa(gfn); u64 entry, level_state; u64 err; - err = tdh_mem_page_aug(&kvm_tdx->td, gpa, tdx_level, page, &entry, &level_state); + err = tdh_mem_page_aug(&kvm_tdx->td, gpa, tdx_level, pfn, &entry, &level_state); if (unlikely(tdx_operand_busy(err))) return -EBUSY; diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index cb9b3210ab71..a9dd75190c67 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -30,7 +30,6 @@ #include #include #include -#include #include #include #include @@ -1568,6 +1567,11 @@ static void tdx_clflush_page(struct page *page) clflush_cache_range(page_to_virt(page), PAGE_SIZE); } +static void tdx_clflush_pfn(kvm_pfn_t pfn) +{ + clflush_cache_range(__va(PFN_PHYS(pfn)), PAGE_SIZE); +} + noinstr u64 tdh_vp_enter(struct tdx_vp *td, struct tdx_module_args *args) { args->rcx = td->tdvpr_pa; @@ -1588,17 +1592,18 @@ u64 tdh_mng_addcx(struct tdx_td *td, struct page *tdcs_page) } EXPORT_SYMBOL_FOR_KVM(tdh_mng_addcx); -u64 tdh_mem_page_add(struct tdx_td *td, u64 gpa, struct page *page, struct page *source, u64 *ext_err1, u64 *ext_err2) +u64 tdh_mem_page_add(struct tdx_td *td, u64 gpa, kvm_pfn_t pfn, struct page *source, + u64 *ext_err1, u64 *ext_err2) { struct tdx_module_args args = { .rcx = gpa, .rdx = tdx_tdr_pa(td), - .r8 = page_to_phys(page), + .r8 = PFN_PHYS(pfn), .r9 = page_to_phys(source), }; u64 ret; - tdx_clflush_page(page); + tdx_clflush_pfn(pfn); ret = seamcall_ret(TDH_MEM_PAGE_ADD, &args); *ext_err1 = args.rcx; @@ -1639,16 +1644,17 @@ u64 tdh_vp_addcx(struct tdx_vp *vp, struct page *tdcx_page) } EXPORT_SYMBOL_FOR_KVM(tdh_vp_addcx); -u64 tdh_mem_page_aug(struct tdx_td *td, u64 gpa, int level, struct page *page, u64 *ext_err1, u64 *ext_err2) +u64 tdh_mem_page_aug(struct tdx_td *td, u64 gpa, int level, kvm_pfn_t pfn, + u64 *ext_err1, u64 *ext_err2) { struct tdx_module_args args = { .rcx = gpa | level, .rdx = tdx_tdr_pa(td), - .r8 = page_to_phys(page), + .r8 = PFN_PHYS(pfn), }; u64 ret; - tdx_clflush_page(page); + tdx_clflush_pfn(pfn); ret = seamcall_ret(TDH_MEM_PAGE_AUG, &args); *ext_err1 = args.rcx; -- 2.43.2 From: Sean Christopherson Remove the completely unnecessary assumptions that memory unmapped from a TDX guest is backed by refcounted struct page memory. APIs tdh_phymem_page_wbinvd_hkid(), tdx_quirk_reset_page() are used when unmapping guest private memory from S-EPT. Since mapping of guest private memory places no requirements on how KVM and guest_memfd manage memory, neither does guest private memory unmapping. Rip out the misguided struct page assumptions/constraints by having the two APIs take PFN directly. This ensures that for future huge page support in S-EPT, the kernel doesn't pick up even worse assumptions like "a hugepage must be contained in a single folio". Use "kvm_pfn_t pfn" for type safety. Using this KVM type is appropriate since APIs tdh_phymem_page_wbinvd_hkid() and tdx_quirk_reset_page() are exported to KVM only. Update mk_keyed_paddr(), which is invoked by tdh_phymem_page_wbinvd_hkid(), to take PFN as parameter accordingly. Opportunistically, move mk_keyed_paddr() from tdx.h to tdx.c since there are no external users. Have tdx_reclaim_page() remain using struct page as parameter since it's currently not used for removing guest private memory yet. [Yan: Use kvm_pfn_t, drop reclaim API param update, move mk_keyed_paddr()] Signed-off-by: Sean Christopherson Signed-off-by: Yan Zhao --- arch/x86/include/asm/tdx.h | 15 ++------------- arch/x86/kvm/vmx/tdx.c | 10 +++++----- arch/x86/virt/vmx/tdx/tdx.c | 16 +++++++++++----- 3 files changed, 18 insertions(+), 23 deletions(-) diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h index f3f0b1872176..6ceb4cd9ff21 100644 --- a/arch/x86/include/asm/tdx.h +++ b/arch/x86/include/asm/tdx.h @@ -153,7 +153,7 @@ int tdx_guest_keyid_alloc(void); u32 tdx_get_nr_guest_keyids(void); void tdx_guest_keyid_free(unsigned int keyid); -void tdx_quirk_reset_page(struct page *page); +void tdx_quirk_reset_page(kvm_pfn_t pfn); struct tdx_td { /* TD root structure: */ @@ -177,17 +177,6 @@ struct tdx_vp { struct page **tdcx_pages; }; -static inline u64 mk_keyed_paddr(u16 hkid, struct page *page) -{ - u64 ret; - - ret = page_to_phys(page); - /* KeyID bits are just above the physical address bits: */ - ret |= (u64)hkid << boot_cpu_data.x86_phys_bits; - - return ret; -} - static inline int pg_level_to_tdx_sept_level(enum pg_level level) { WARN_ON_ONCE(level == PG_LEVEL_NONE); @@ -219,7 +208,7 @@ u64 tdh_mem_track(struct tdx_td *tdr); u64 tdh_mem_page_remove(struct tdx_td *td, u64 gpa, u64 level, u64 *ext_err1, u64 *ext_err2); u64 tdh_phymem_cache_wb(bool resume); u64 tdh_phymem_page_wbinvd_tdr(struct tdx_td *td); -u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, struct page *page); +u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, kvm_pfn_t pfn); #else static inline void tdx_init(void) { } static inline u32 tdx_get_nr_guest_keyids(void) { return 0; } diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 1f1abc5b5655..75ad3debcd84 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -343,7 +343,7 @@ static int tdx_reclaim_page(struct page *page) r = __tdx_reclaim_page(page); if (!r) - tdx_quirk_reset_page(page); + tdx_quirk_reset_page(page_to_pfn(page)); return r; } @@ -597,7 +597,7 @@ static void tdx_reclaim_td_control_pages(struct kvm *kvm) if (TDX_BUG_ON(err, TDH_PHYMEM_PAGE_WBINVD, kvm)) return; - tdx_quirk_reset_page(kvm_tdx->td.tdr_page); + tdx_quirk_reset_page(page_to_pfn(kvm_tdx->td.tdr_page)); __free_page(kvm_tdx->td.tdr_page); kvm_tdx->td.tdr_page = NULL; @@ -1776,9 +1776,9 @@ static int tdx_sept_free_private_spt(struct kvm *kvm, gfn_t gfn, static void tdx_sept_remove_private_spte(struct kvm *kvm, gfn_t gfn, enum pg_level level, u64 mirror_spte) { - struct page *page = pfn_to_page(spte_to_pfn(mirror_spte)); int tdx_level = pg_level_to_tdx_sept_level(level); struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); + kvm_pfn_t pfn = spte_to_pfn(mirror_spte); gpa_t gpa = gfn_to_gpa(gfn); u64 err, entry, level_state; @@ -1817,11 +1817,11 @@ static void tdx_sept_remove_private_spte(struct kvm *kvm, gfn_t gfn, if (TDX_BUG_ON_2(err, TDH_MEM_PAGE_REMOVE, entry, level_state, kvm)) return; - err = tdh_phymem_page_wbinvd_hkid((u16)kvm_tdx->hkid, page); + err = tdh_phymem_page_wbinvd_hkid((u16)kvm_tdx->hkid, pfn); if (TDX_BUG_ON(err, TDH_PHYMEM_PAGE_WBINVD, kvm)) return; - tdx_quirk_reset_page(page); + tdx_quirk_reset_page(pfn); } void tdx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index a9dd75190c67..2f9d07ad1a9a 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -730,9 +730,9 @@ static void tdx_quirk_reset_paddr(unsigned long base, unsigned long size) mb(); } -void tdx_quirk_reset_page(struct page *page) +void tdx_quirk_reset_page(kvm_pfn_t pfn) { - tdx_quirk_reset_paddr(page_to_phys(page), PAGE_SIZE); + tdx_quirk_reset_paddr(PFN_PHYS(pfn), PAGE_SIZE); } EXPORT_SYMBOL_FOR_KVM(tdx_quirk_reset_page); @@ -1907,21 +1907,27 @@ u64 tdh_phymem_cache_wb(bool resume) } EXPORT_SYMBOL_FOR_KVM(tdh_phymem_cache_wb); +static inline u64 mk_keyed_paddr(u16 hkid, kvm_pfn_t pfn) +{ + /* KeyID bits are just above the physical address bits. */ + return PFN_PHYS(pfn) | ((u64)hkid << boot_cpu_data.x86_phys_bits); +} + u64 tdh_phymem_page_wbinvd_tdr(struct tdx_td *td) { struct tdx_module_args args = {}; - args.rcx = mk_keyed_paddr(tdx_global_keyid, td->tdr_page); + args.rcx = mk_keyed_paddr(tdx_global_keyid, page_to_pfn(td->tdr_page)); return seamcall(TDH_PHYMEM_PAGE_WBINVD, &args); } EXPORT_SYMBOL_FOR_KVM(tdh_phymem_page_wbinvd_tdr); -u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, struct page *page) +u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, kvm_pfn_t pfn) { struct tdx_module_args args = {}; - args.rcx = mk_keyed_paddr(hkid, page); + args.rcx = mk_keyed_paddr(hkid, pfn); return seamcall(TDH_PHYMEM_PAGE_WBINVD, &args); } -- 2.43.2