AI Trajectory

Trajectory: (Expand/Collapse All)
Seq	Timestamp	Type	Name	Duration
0/0	2026/08/06 03:01	flow	patch-triage	0m Args: null Results: { "EnableConfigs": null, "FocusSymbols": [], "Reasoning": "The patch adds support for Dynamic PAMT in the TDX host (KVM) code. TDX host support requires specific bare-metal Intel hardware with Trust Domain Extensions enabled and the TDX module loaded. This hardware is not emulated in standard virtualized environments like QEMU or GCE, making the modified code paths unreachable during fuzzing.", "WorthFuzzing": false }
1/1	2026/08/06 03:01	action	read-patch-diff	0m Args: null Results: { "PatchDiff": "commit f00efc49131881f16a5b1580adf3f7c794263146\nAuthor: syz-cluster \u003ctriage@syzkaller.com\u003e\nDate: Thu Aug 6 03:00:59 2026 +0000\n\n syz-cluster: applied patch under review\n\ndiff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt\nindex b5493a7f8f228..a0a7670246bc5 100644\n--- a/Documentation/admin-guide/kernel-parameters.txt\n+++ b/Documentation/admin-guide/kernel-parameters.txt\n@@ -7512,6 +7512,16 @@ Kernel parameters\n \n \ttdfx=\t\t[HW,DRM]\n \n+\ttdx_dpamt=\n+\t\t\t[X86] Controls whether TDX will use Dynamic PAMT\n+\t\t\tto save memory, when supported.\n+\n+\t\t\tValid parameters: \"on\", \"off\"\n+\t\t\tDefault: \"off\"\n+\n+\t\t\tFor details see:\n+\t\t\tDocumentation/arch/x86/tdx.rst\n+\n \ttest_suspend=\t[SUSPEND]\n \t\t\tFormat: { \"mem\" \| \"standby\" \| \"freeze\" }[,N]\n \t\t\tSpecify \"mem\" (for Suspend-to-RAM) or \"standby\" (for\ndiff --git a/Documentation/arch/x86/tdx.rst b/Documentation/arch/x86/tdx.rst\nindex 3303499ad4c6f..a1c2309230580 100644\n--- a/Documentation/arch/x86/tdx.rst\n+++ b/Documentation/arch/x86/tdx.rst\n@@ -200,6 +200,34 @@ reflects the TCB of the currently running TDX module and therefore\n changes after an update. By contrast, TEE_TCB_SVN reflects the TCB at TD\n launch time and is not affected.\n \n+Dynamic PAMT\n+------------\n+\n+Physical Address Metadata Table (PAMT) is memory that the TDX module needs\n+to keep data about each page (think like struct page). It needs to be handed\n+to the TDX module for its exclusive use. For normal PAMT, this is installed\n+when the TDX module is first loaded and comes to about 0.4% of system memory.\n+\n+Dynamic PAMT is a TDX module feature that allows VMM to allocate part of the\n+PAMT as needed (the parts for tracking 4KB size pages). The other page sizes\n+(1GB and 2MB) are still allocated statically at the time of TDX module\n+initialization. This reduces the amount of memory that TDX uses while TDs are\n+not in use.\n+\n+When Dynamic PAMT is in use, dmesg shows it like::\n+\n+ [..] virt/tdx: Enable Dynamic PAMT\n+ [..] virt/tdx: 10092 KB allocated for PAMT\n+ [..] virt/tdx: TDX-Module initialized\n+\n+Dynamic PAMT is only enabled when supported and the ``tdx_dpamt=`` kernel\n+parameter is set to \"on\". The feature is off by default because TDX module\n+internal details prevent Dynamic PAMT from working on all keyid partitioning\n+configurations. When the TDX module is fixed to include these constraints in\n+its enumeration of Dynamic PAMT support, kernel support can be changed to\n+default on. For more information, consult the Intel TDX documentation about\n+Dynamic PAMT.\n+\n TDX Interaction to Other Kernel Components\n ------------------------------------------\n \ndiff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h\nindex 83dc5086138b3..588563dfe88d5 100644\n--- a/arch/x86/include/asm/kvm-x86-ops.h\n+++ b/arch/x86/include/asm/kvm-x86-ops.h\n@@ -98,6 +98,7 @@ KVM_X86_OP_OPTIONAL_RET0(tdp_has_smep)\n KVM_X86_OP(load_mmu_pgd)\n KVM_X86_OP_OPTIONAL_RET0(set_external_spte)\n KVM_X86_OP_OPTIONAL(free_external_spt)\n+KVM_X86_OP_OPTIONAL_RET0(topup_external_cache)\n KVM_X86_OP(has_wbinvd_exit)\n KVM_X86_OP(get_l2_tsc_offset)\n KVM_X86_OP(get_l2_tsc_multiplier)\ndiff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h\nindex 5f6c1ce9673b7..0dbf6564dd0c5 100644\n--- a/arch/x86/include/asm/kvm_host.h\n+++ b/arch/x86/include/asm/kvm_host.h\n@@ -1922,6 +1922,7 @@ struct kvm_x86_ops {\n \t/* Update external page tables for page table about to be freed. /\n \tvoid (free_external_spt)(struct kvm kvm, struct kvm_mmu_page sp);\n \n+\tint (topup_external_cache)(struct kvm_vcpu vcpu, int min_nr_spts);\n \n \tbool (has_wbinvd_exit)(void);\n \ndiff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h\nindex 89e97d5761d89..7910901a7ba21 100644\n--- a/arch/x86/include/asm/tdx.h\n+++ b/arch/x86/include/asm/tdx.h\n@@ -36,6 +36,7 @@\n / Bit definitions of TDX_FEATURES0 metadata field /\n #define TDX_FEATURES0_TD_PRESERVING\tBIT_ULL(1)\n #define TDX_FEATURES0_NO_RBP_MOD\tBIT_ULL(18)\n+#define TDX_FEATURES0_DYNAMIC_PAMT\tBIT_ULL(36)\n \n #ifndef __ASSEMBLER__\n \n@@ -118,12 +119,37 @@ static inline bool tdx_supports_runtime_update(const struct tdx_sys_info sysinf\n \treturn sysinfo-\u003efeatures.tdx_features0 \u0026 TDX_FEATURES0_TD_PRESERVING;\n }\n \n+bool tdx_supports_dynamic_pamt(const struct tdx_sys_info sysinfo);\n+\n+/ Simple structure for pre-allocating Dynamic PAMT pages outside of spinlocks. /\n+struct tdx_pamt_cache {\n+\tstruct list_head page_list;\n+\tint cnt;\n+};\n+\n+static inline void tdx_init_pamt_cache(struct tdx_pamt_cache cache)\n+{\n+\tINIT_LIST_HEAD(\u0026cache-\u003epage_list);\n+\tcache-\u003ecnt = 0;\n+}\n+\n+void tdx_free_pamt_cache(struct tdx_pamt_cache cache);\n+int tdx_topup_pamt_cache(struct tdx_pamt_cache cache, unsigned long npages);\n+int tdx_pamt_get(kvm_pfn_t pfn, struct tdx_pamt_cache cache);\n+void tdx_pamt_put(kvm_pfn_t pfn);\n+\n int tdx_guest_keyid_alloc(void);\n u32 tdx_get_nr_guest_keyids(void);\n void tdx_guest_keyid_free(unsigned int keyid);\n \n void tdx_quirk_reset_paddr(unsigned long base, unsigned long size);\n \n+/ Number of PAMT pages to be provided to TDX module per 2MB region of PA /\n+#define TDX_DPAMT_ENTRY_PAGE_CNT 2\n+\n+struct page tdx_alloc_control_page(void);\n+void tdx_free_control_page(struct page page);\n+\n struct tdx_td {\n \t/ TD root structure: /\n \tstruct page tdr_page;\ndiff --git a/arch/x86/include/asm/tdx_global_metadata.h b/arch/x86/include/asm/tdx_global_metadata.h\nindex 41150d546589c..2a42551fc33cd 100644\n--- a/arch/x86/include/asm/tdx_global_metadata.h\n+++ b/arch/x86/include/asm/tdx_global_metadata.h\n@@ -21,6 +21,9 @@ struct tdx_sys_info_tdmr {\n \tu16 pamt_4k_entry_size;\n \tu16 pamt_2m_entry_size;\n \tu16 pamt_1g_entry_size;\n+\n+\t/* Optional metadata, if Dynamic PAMT is supported /\n+\tu8 pamt_page_bitmap_entry_bits;\n };\n \n struct tdx_sys_info_td_ctrl {\ndiff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c\nindex 66e69d2a41b3c..2a628b8304409 100644\n--- a/arch/x86/kvm/mmu/mmu.c\n+++ b/arch/x86/kvm/mmu/mmu.c\n@@ -614,6 +614,10 @@ static int mmu_topup_memory_caches(struct kvm_vcpu vcpu, bool maybe_indirect)\n \t\t\t\t\t PT64_ROOT_MAX_LEVEL);\n \t\tif (r)\n \t\t\treturn r;\n+\n+\t\tr = kvm_x86_call(topup_external_cache)(vcpu, PT64_ROOT_MAX_LEVEL);\n+\t\tif (r)\n+\t\t\treturn r;\n \t}\n \tr = kvm_mmu_topup_memory_cache(\u0026vcpu-\u003earch.mmu_shadow_page_cache,\n \t\t\t\t PT64_ROOT_MAX_LEVEL);\ndiff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c\nindex 545b03d9d10b8..09573112540e9 100644\n--- a/arch/x86/kvm/vmx/tdx.c\n+++ b/arch/x86/kvm/vmx/tdx.c\n@@ -362,7 +362,7 @@ static void tdx_reclaim_control_page(struct page ctrl_page)\n \tif (tdx_reclaim_page(ctrl_page))\n \t\treturn;\n \n-\t__free_page(ctrl_page);\n+\ttdx_free_control_page(ctrl_page);\n }\n \n struct tdx_flush_vp_arg {\n@@ -589,7 +589,7 @@ static void tdx_reclaim_td_control_pages(struct kvm kvm)\n \n \ttdx_quirk_reset_paddr(page_to_phys(kvm_tdx-\u003etd.tdr_page), PAGE_SIZE);\n \n-\t__free_page(kvm_tdx-\u003etd.tdr_page);\n+\ttdx_free_control_page(kvm_tdx-\u003etd.tdr_page);\n \tkvm_tdx-\u003etd.tdr_page = NULL;\n }\n \n@@ -681,6 +681,8 @@ int tdx_vcpu_create(struct kvm_vcpu vcpu)\n \tif (!irqchip_split(vcpu-\u003ekvm))\n \t\treturn -EINVAL;\n \n+\ttdx_init_pamt_cache(\u0026tdx-\u003epamt_cache);\n+\n \tfpstate_set_confidential(\u0026vcpu-\u003earch.guest_fpu);\n \tvcpu-\u003earch.apic-\u003eguest_apic_protected = true;\n \tINIT_LIST_HEAD(\u0026tdx-\u003evt.pi_wakeup_list);\n@@ -866,6 +868,8 @@ void tdx_vcpu_free(struct kvm_vcpu vcpu)\n \tstruct vcpu_tdx tdx = to_tdx(vcpu);\n \tint i;\n \n+\ttdx_free_pamt_cache(\u0026tdx-\u003epamt_cache);\n+\n \tif (vcpu-\u003ecpu != -1) {\n \t\tKVM_BUG_ON(tdx-\u003estate == VCPU_TD_STATE_INITIALIZED, vcpu-\u003ekvm);\n \t\ttdx_flush_vp_on_cpu(vcpu);\n@@ -1621,6 +1625,17 @@ void tdx_load_mmu_pgd(struct kvm_vcpu vcpu, hpa_t root_hpa, int pgd_level)\n \ttd_vmcs_write64(to_tdx(vcpu), SHARED_EPT_POINTER, root_hpa);\n }\n \n+static int tdx_topup_external_pamt_cache(struct kvm_vcpu vcpu, int min_nr_spts)\n+{\n+\t/\n+\t * Minus one page to exclude the root SPT, but plus one page for a\n+\t * possible 4KB private mapping.\n+\t /\n+\tmin_nr_spts += -1 + 1;\n+\n+\treturn tdx_topup_pamt_cache(\u0026to_tdx(vcpu)-\u003epamt_cache, min_nr_spts);\n+}\n+\n static int tdx_mem_page_add(struct kvm kvm, gfn_t gfn, enum pg_level level,\n \t\t\t kvm_pfn_t pfn)\n {\n@@ -1679,16 +1694,28 @@ static struct page tdx_spte_to_sept_pt(struct kvm kvm, gfn_t gfn,\n static int tdx_sept_map_nonleaf_spte(struct kvm kvm, gfn_t gfn,\n \t\t\t\t enum pg_level level, u64 new_spte)\n {\n+\tstruct kvm_vcpu vcpu = kvm_get_running_vcpu();\n \tgpa_t gpa = gfn_to_gpa(gfn);\n \tu64 err, entry, level_state;\n \tstruct page sept_pt;\n+\tint ret;\n+\n+\tif (KVM_BUG_ON(!vcpu, kvm))\n+\t\treturn -EIO;\n \n \tsept_pt = tdx_spte_to_sept_pt(kvm, gfn, new_spte, level);\n \tif (!sept_pt)\n \t\treturn -EIO;\n \n+\tret = tdx_pamt_get(page_to_pfn(sept_pt), \u0026to_tdx(vcpu)-\u003epamt_cache);\n+\tif (KVM_BUG_ON(ret, kvm))\n+\t\treturn ret;\n+\n \terr = tdh_mem_sept_add(\u0026to_kvm_tdx(kvm)-\u003etd, gpa, level, sept_pt,\n \t\t\t \u0026entry, \u0026level_state);\n+\tif (err)\n+\t\ttdx_pamt_put(page_to_pfn(sept_pt));\n+\n \tif (unlikely(tdx_operand_busy(err)))\n \t\treturn -EBUSY;\n \n@@ -1701,8 +1728,13 @@ static int tdx_sept_map_nonleaf_spte(struct kvm kvm, gfn_t gfn,\n static int tdx_sept_map_leaf_spte(struct kvm kvm, gfn_t gfn, enum pg_level level,\n \t\t\t\t u64 new_spte)\n {\n+\tstruct kvm_vcpu vcpu = kvm_get_running_vcpu();\n \tstruct kvm_tdx kvm_tdx = to_kvm_tdx(kvm);\n \tkvm_pfn_t pfn = spte_to_pfn(new_spte);\n+\tint ret;\n+\n+\tif (KVM_BUG_ON(!vcpu, kvm))\n+\t\treturn -EIO;\n \n \t/ TODO: handle large pages. /\n \tif (KVM_BUG_ON(level != PG_LEVEL_4K, kvm))\n@@ -1710,6 +1742,10 @@ static int tdx_sept_map_leaf_spte(struct kvm kvm, gfn_t gfn, enum pg_level leve\n \n \tWARN_ON_ONCE((new_spte \u0026 VMX_EPT_RWX_MASK) != VMX_EPT_RWX_MASK);\n \n+\tret = tdx_pamt_get(pfn, \u0026to_tdx(vcpu)-\u003epamt_cache);\n+\tif (KVM_BUG_ON(ret, kvm))\n+\t\treturn ret;\n+\n \t/\n \t Ensure pre_fault_allowed is read by kvm_arch_vcpu_pre_fault_memory()\n \t * before kvm_tdx-\u003estate. Userspace must not be allowed to pre-fault\n@@ -1722,10 +1758,15 @@ static int tdx_sept_map_leaf_spte(struct kvm kvm, gfn_t gfn, enum pg_level leve\n \t If the TD isn't finalized/runnable, then userspace is initializing\n \t * the VM image via KVM_TDX_INIT_MEM_REGION; ADD the page to the TD.\n \t /\n-\tif (unlikely(kvm_tdx-\u003estate != TD_STATE_RUNNABLE))\n-\t\treturn tdx_mem_page_add(kvm, gfn, level, pfn);\n+\tif (likely(kvm_tdx-\u003estate == TD_STATE_RUNNABLE))\n+\t\tret = tdx_mem_page_aug(kvm, gfn, level, pfn);\n+\telse\n+\t\tret = tdx_mem_page_add(kvm, gfn, level, pfn);\n+\n+\tif (ret)\n+\t\ttdx_pamt_put(pfn);\n \n-\treturn tdx_mem_page_aug(kvm, gfn, level, pfn);\n+\treturn ret;\n }\n \n /\n@@ -1822,6 +1863,7 @@ static int tdx_sept_remove_leaf_spte(struct kvm kvm, gfn_t gfn,\n \t\treturn -EIO;\n \n \ttdx_quirk_reset_paddr(PFN_PHYS(pfn), PAGE_SIZE);\n+\ttdx_pamt_put(pfn);\n \treturn 0;\n }\n \n@@ -1865,6 +1907,8 @@ static int tdx_sept_set_private_spte(struct kvm kvm, gfn_t gfn, u64 old_spte,\n /\n static void tdx_sept_free_private_spt(struct kvm kvm, struct kvm_mmu_page sp)\n {\n+\tstruct page sept_pt = virt_to_page(sp-\u003eexternal_spt);\n+\n \t/\n \t KVM doesn't (yet) zap page table pages in mirror page table while\n \t * TD is active, though guest pages mapped in mirror page table could be\n@@ -1878,15 +1922,15 @@ static void tdx_sept_free_private_spt(struct kvm kvm, struct kvm_mmu_page sp)\n \t * the page to prevent the kernel from accessing the encrypted page.\n \t /\n \tif (KVM_BUG_ON(is_hkid_assigned(to_kvm_tdx(kvm)), kvm) \|\|\n-\t tdx_reclaim_page(virt_to_page(sp-\u003eexternal_spt)))\n+\t tdx_reclaim_page(sept_pt))\n \t\tgoto out;\n \n \t/\n-\t * Immediately free the S-EPT page because RCU-time free is unnecessary\n-\t * after TDH.PHYMEM.PAGE.RECLAIM ensures there are no outstanding\n-\t * readers.\n+\t * Immediately free the S-EPT page as the TDX subsystem doesn't support\n+\t * freeing pages from RCU callbacks, and more importantly because\n+\t * TDH.PHYMEM.PAGE.RECLAIM ensures there are no outstanding readers.\n \t /\n-\tfree_page((unsigned long)sp-\u003eexternal_spt);\n+\ttdx_free_control_page(sept_pt);\n out:\n \tsp-\u003eexternal_spt = NULL;\n }\n@@ -2459,7 +2503,7 @@ static int __tdx_td_init(struct kvm kvm, struct td_params td_params,\n \n \tret = -ENOMEM;\n \n-\ttdr_page = alloc_page(GFP_KERNEL_ACCOUNT);\n+\ttdr_page = tdx_alloc_control_page();\n \tif (!tdr_page)\n \t\tgoto free_hkid;\n \n@@ -2472,7 +2516,7 @@ static int __tdx_td_init(struct kvm kvm, struct td_params td_params,\n \t\tgoto free_tdr;\n \n \tfor (i = 0; i \u003c kvm_tdx-\u003etd.tdcs_nr_pages; i++) {\n-\t\ttdcs_pages[i] = alloc_page(GFP_KERNEL_ACCOUNT);\n+\t\ttdcs_pages[i] = tdx_alloc_control_page();\n \t\tif (!tdcs_pages[i])\n \t\t\tgoto free_tdcs;\n \t}\n@@ -2590,10 +2634,8 @@ static int __tdx_td_init(struct kvm kvm, struct td_params td_params,\n teardown:\n \t/ Only free pages not yet added, so start at 'i' /\n \tfor (; i \u003c kvm_tdx-\u003etd.tdcs_nr_pages; i++) {\n-\t\tif (tdcs_pages[i]) {\n-\t\t\t__free_page(tdcs_pages[i]);\n-\t\t\ttdcs_pages[i] = NULL;\n-\t\t}\n+\t\ttdx_free_control_page(tdcs_pages[i]);\n+\t\ttdcs_pages[i] = NULL;\n \t}\n \tif (!kvm_tdx-\u003etd.tdcs_pages)\n \t\tkfree(tdcs_pages);\n@@ -2608,16 +2650,13 @@ static int __tdx_td_init(struct kvm kvm, struct td_params td_params,\n \tfree_cpumask_var(packages);\n \n free_tdcs:\n-\tfor (i = 0; i \u003c kvm_tdx-\u003etd.tdcs_nr_pages; i++) {\n-\t\tif (tdcs_pages[i])\n-\t\t\t__free_page(tdcs_pages[i]);\n-\t}\n+\tfor (i = 0; i \u003c kvm_tdx-\u003etd.tdcs_nr_pages; i++)\n+\t\ttdx_free_control_page(tdcs_pages[i]);\n \tkfree(tdcs_pages);\n \tkvm_tdx-\u003etd.tdcs_pages = NULL;\n \n free_tdr:\n-\tif (tdr_page)\n-\t\t__free_page(tdr_page);\n+\ttdx_free_control_page(tdr_page);\n \tkvm_tdx-\u003etd.tdr_page = NULL;\n \n free_hkid:\n@@ -2951,7 +2990,7 @@ static int tdx_td_vcpu_init(struct kvm_vcpu vcpu, u64 vcpu_rcx)\n \tint ret, i;\n \tu64 err;\n \n-\tpage = alloc_page(GFP_KERNEL_ACCOUNT);\n+\tpage = tdx_alloc_control_page();\n \tif (!page)\n \t\treturn -ENOMEM;\n \ttdx-\u003evp.tdvpr_page = page;\n@@ -2971,7 +3010,7 @@ static int tdx_td_vcpu_init(struct kvm_vcpu vcpu, u64 vcpu_rcx)\n \t}\n \n \tfor (i = 0; i \u003c kvm_tdx-\u003etd.tdcx_nr_pages; i++) {\n-\t\tpage = alloc_page(GFP_KERNEL_ACCOUNT);\n+\t\tpage = tdx_alloc_control_page();\n \t\tif (!page) {\n \t\t\tret = -ENOMEM;\n \t\t\tgoto free_tdcx;\n@@ -2993,7 +3032,7 @@ static int tdx_td_vcpu_init(struct kvm_vcpu vcpu, u64 vcpu_rcx)\n \t\t\t * method, but the rest are freed here.\n \t\t\t /\n \t\t\tfor (; i \u003c kvm_tdx-\u003etd.tdcx_nr_pages; i++) {\n-\t\t\t\t__free_page(tdx-\u003evp.tdcx_pages[i]);\n+\t\t\t\ttdx_free_control_page(tdx-\u003evp.tdcx_pages[i]);\n \t\t\t\ttdx-\u003evp.tdcx_pages[i] = NULL;\n \t\t\t}\n \t\t\treturn -EIO;\n@@ -3021,16 +3060,14 @@ static int tdx_td_vcpu_init(struct kvm_vcpu vcpu, u64 vcpu_rcx)\n \n free_tdcx:\n \tfor (i = 0; i \u003c kvm_tdx-\u003etd.tdcx_nr_pages; i++) {\n-\t\tif (tdx-\u003evp.tdcx_pages[i])\n-\t\t\t__free_page(tdx-\u003evp.tdcx_pages[i]);\n+\t\ttdx_free_control_page(tdx-\u003evp.tdcx_pages[i]);\n \t\ttdx-\u003evp.tdcx_pages[i] = NULL;\n \t}\n \tkfree(tdx-\u003evp.tdcx_pages);\n \ttdx-\u003evp.tdcx_pages = NULL;\n \n free_tdvpr:\n-\tif (tdx-\u003evp.tdvpr_page)\n-\t\t__free_page(tdx-\u003evp.tdvpr_page);\n+\ttdx_free_control_page(tdx-\u003evp.tdvpr_page);\n \ttdx-\u003evp.tdvpr_page = NULL;\n \ttdx-\u003evp.tdvpr_pa = 0;\n \n@@ -3493,6 +3530,10 @@ int __init tdx_hardware_setup(void)\n \n \tvt_x86_ops.set_external_spte = tdx_sept_set_private_spte;\n \tvt_x86_ops.free_external_spt = tdx_sept_free_private_spt;\n+\n+\tif (tdx_supports_dynamic_pamt(tdx_sysinfo))\n+\t\tvt_x86_ops.topup_external_cache = tdx_topup_external_pamt_cache;\n+\n \tvt_x86_ops.protected_apic_has_interrupt = tdx_protected_apic_has_interrupt;\n \treturn 0;\n \ndiff --git a/arch/x86/kvm/vmx/tdx.h b/arch/x86/kvm/vmx/tdx.h\nindex ac8323a68b163..fd368e3ee060b 100644\n--- a/arch/x86/kvm/vmx/tdx.h\n+++ b/arch/x86/kvm/vmx/tdx.h\n@@ -72,6 +72,8 @@ struct vcpu_tdx {\n \n \tu64 map_gpa_next;\n \tu64 map_gpa_end;\n+\n+\tstruct tdx_pamt_cache pamt_cache;\n };\n \n void tdh_vp_rd_failed(struct vcpu_tdx tdx, char uclass, u32 field, u64 err);\ndiff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c\nindex 42df8ea464c47..dec45709e454a 100644\n--- a/arch/x86/virt/vmx/tdx/tdx.c\n+++ b/arch/x86/virt/vmx/tdx/tdx.c\n@@ -30,6 +30,7 @@\n #include \u003clinux/suspend.h\u003e\n #include \u003clinux/syscore_ops.h\u003e\n #include \u003clinux/idr.h\u003e\n+#include \u003clinux/vmalloc.h\u003e\n #include \u003casm/page.h\u003e\n #include \u003casm/special_insns.h\u003e\n #include \u003casm/msr-index.h\u003e\n@@ -46,6 +47,8 @@\n #include \"seamcall_internal.h\"\n #include \"tdx.h\"\n \n+static bool tdx_enable_dpamt __ro_after_init;\n+\n struct tdx_module_state {\n \tbool initialized;\n \tbool sysinit_done;\n@@ -63,6 +66,14 @@ static DEFINE_PER_CPU(bool, tdx_lp_initialized);\n \n static struct tdmr_info_list tdx_tdmr_list;\n \n+/\n+ On a machine with Dynamic PAMT, the kernel maintains a reference counter\n+ * for every 2MB range. The counter indicates how many users there are for\n+ * the PAMT memory of the 2MB range. The kernel allocates PAMT refcounts at\n+ * initialization.\n+ /\n+static atomic_t pamt_refcounts;\n+\n /* All TDX-usable memory regions. Protected by mem_hotplug_lock. /\n static LIST_HEAD(tdx_memlist);\n \n@@ -252,6 +263,42 @@ static struct syscore tdx_syscore = {\n \t.ops = \u0026tdx_syscore_ops,\n };\n \n+/\n+ * Allocate PAMT reference counters for all physical memory.\n+ \n+ It consumes 2MB for every 1TB of physical memory.\n+ /\n+static __init int init_pamt_refcounts(void)\n+{\n+\tsize_t size = DIV_ROUND_UP(max_pfn, PTRS_PER_PTE) sizeof(pamt_refcounts);\n+\n+\tif (!tdx_supports_dynamic_pamt(\u0026tdx_sysinfo))\n+\t\treturn 0;\n+\n+\tpamt_refcounts = vzalloc(size);\n+\tif (!pamt_refcounts)\n+\t\treturn -ENOMEM;\n+\n+\treturn 0;\n+}\n+\n+static __init void free_pamt_refcounts(void)\n+{\n+\tif (!tdx_supports_dynamic_pamt(\u0026tdx_sysinfo))\n+\t\treturn;\n+\n+\tvfree(pamt_refcounts);\n+\tpamt_refcounts = NULL;\n+}\n+\n+static atomic_t tdx_find_pamt_refcount(unsigned long pfn)\n+{\n+\t/* Find which PMD a PFN is in. /\n+\tunsigned long index = pfn \u003e\u003e (PMD_SHIFT - PAGE_SHIFT);\n+\n+\treturn \u0026pamt_refcounts[index];\n+}\n+\n /\n * Add a memory region as a TDX memory block. The caller must make sure\n * all memory regions are added in address ascending order and don't\n@@ -510,35 +557,37 @@ static __init int fill_out_tdmrs(struct list_head tmb_list,\n \treturn 0;\n }\n \n+static __init unsigned long tdmr_get_pamt_bitmap_sz(struct tdmr_info tdmr)\n+{\n+\tunsigned long pamt_sz, nr_pamt_entries;\n+\tint bits_per_entry;\n+\n+\tbits_per_entry = tdx_sysinfo.tdmr.pamt_page_bitmap_entry_bits;\n+\tnr_pamt_entries = tdmr-\u003esize \u003e\u003e PAGE_SHIFT;\n+\tpamt_sz = DIV_ROUND_UP(nr_pamt_entries * bits_per_entry, BITS_PER_BYTE);\n+\n+\treturn PAGE_ALIGN(pamt_sz);\n+}\n+\n /\n Calculate PAMT size given a TDMR and a page size. The returned\n * PAMT size is always aligned up to 4K page boundary.\n /\n-static __init unsigned long tdmr_get_pamt_sz(struct tdmr_info tdmr, int pgsz,\n-\t\t\t\t\t u16 pamt_entry_size)\n+static __init unsigned long tdmr_get_pamt_sz(struct tdmr_info tdmr, int pgsz)\n {\n \tunsigned long pamt_sz, nr_pamt_entries;\n+\tconst int tdx_pg_size_shift[TDX_PS_NR] = { PAGE_SHIFT, PMD_SHIFT, PUD_SHIFT };\n+\tconst u16 pamt_entry_size[TDX_PS_NR] = {\n+\t\ttdx_sysinfo.tdmr.pamt_4k_entry_size,\n+\t\ttdx_sysinfo.tdmr.pamt_2m_entry_size,\n+\t\ttdx_sysinfo.tdmr.pamt_1g_entry_size,\n+\t};\n \n-\tswitch (pgsz) {\n-\tcase TDX_PS_4K:\n-\t\tnr_pamt_entries = tdmr-\u003esize \u003e\u003e PAGE_SHIFT;\n-\t\tbreak;\n-\tcase TDX_PS_2M:\n-\t\tnr_pamt_entries = tdmr-\u003esize \u003e\u003e PMD_SHIFT;\n-\t\tbreak;\n-\tcase TDX_PS_1G:\n-\t\tnr_pamt_entries = tdmr-\u003esize \u003e\u003e PUD_SHIFT;\n-\t\tbreak;\n-\tdefault:\n-\t\tWARN_ON_ONCE(1);\n-\t\treturn 0;\n-\t}\n+\tnr_pamt_entries = tdmr-\u003esize \u003e\u003e tdx_pg_size_shift[pgsz];\n+\tpamt_sz = nr_pamt_entries pamt_entry_size[pgsz];\n \n-\tpamt_sz = nr_pamt_entries * pamt_entry_size;\n \t/* TDX requires PAMT size must be 4K aligned /\n-\tpamt_sz = ALIGN(pamt_sz, PAGE_SIZE);\n-\n-\treturn pamt_sz;\n+\treturn PAGE_ALIGN(pamt_sz);\n }\n \n /\n@@ -576,15 +625,11 @@ static __init int tdmr_get_nid(struct tdmr_info tdmr, struct list_head tmb_lis\n * within @tdmr, and set up PAMTs for @tdmr.\n /\n static __init int tdmr_set_up_pamt(struct tdmr_info tdmr,\n-\t\t\t\t struct list_head tmb_list,\n-\t\t\t\t u16 pamt_entry_size[])\n+\t\t\t\t struct list_head tmb_list)\n {\n-\tunsigned long pamt_base[TDX_PS_NR];\n-\tunsigned long pamt_size[TDX_PS_NR];\n-\tunsigned long tdmr_pamt_base;\n \tunsigned long tdmr_pamt_size;\n \tstruct page pamt;\n-\tint pgsz, nid;\n+\tint nid;\n \n \tnid = tdmr_get_nid(tdmr, tmb_list);\n \n@@ -592,13 +637,18 @@ static __init int tdmr_set_up_pamt(struct tdmr_info tdmr,\n \t * Calculate the PAMT size for each TDX supported page size\n \t * and the total PAMT size.\n \t /\n-\ttdmr_pamt_size = 0;\n-\tfor (pgsz = TDX_PS_4K; pgsz \u003c TDX_PS_NR; pgsz++) {\n-\t\tpamt_size[pgsz] = tdmr_get_pamt_sz(tdmr, pgsz,\n-\t\t\t\t\tpamt_entry_size[pgsz]);\n-\t\ttdmr_pamt_size += pamt_size[pgsz];\n+\ttdmr-\u003epamt_1g_size = tdmr_get_pamt_sz(tdmr, TDX_PS_1G);\n+\ttdmr-\u003epamt_2m_size = tdmr_get_pamt_sz(tdmr, TDX_PS_2M);\n+\n+\tif (tdx_supports_dynamic_pamt(\u0026tdx_sysinfo)) {\n+\t\t/ With Dynamic PAMT, PAMT_4K is replaced with a bitmap /\n+\t\ttdmr-\u003epamt_4k_size = tdmr_get_pamt_bitmap_sz(tdmr);\n+\t} else {\n+\t\ttdmr-\u003epamt_4k_size = tdmr_get_pamt_sz(tdmr, TDX_PS_4K);\n \t}\n \n+\ttdmr_pamt_size = tdmr-\u003epamt_4k_size + tdmr-\u003epamt_2m_size + tdmr-\u003epamt_1g_size;\n+\n \t/\n \t * Allocate one chunk of physically contiguous memory for all\n \t * PAMTs. This helps minimize the PAMT's use of reserved areas\n@@ -606,25 +656,17 @@ static __init int tdmr_set_up_pamt(struct tdmr_info tdmr,\n \t /\n \tpamt = alloc_contig_pages(tdmr_pamt_size \u003e\u003e PAGE_SHIFT, GFP_KERNEL,\n \t\t\tnid, \u0026node_online_map);\n-\tif (!pamt)\n-\t\treturn -ENOMEM;\n \n \t/\n-\t Break the contiguous allocation back up into the\n-\t * individual PAMTs for each page size.\n+\t * tdmr-\u003epamt_4k_base is still zero so the error\n+\t * path of the caller will skip freeing the PAMT.\n \t /\n-\ttdmr_pamt_base = page_to_pfn(pamt) \u003c\u003c PAGE_SHIFT;\n-\tfor (pgsz = TDX_PS_4K; pgsz \u003c TDX_PS_NR; pgsz++) {\n-\t\tpamt_base[pgsz] = tdmr_pamt_base;\n-\t\ttdmr_pamt_base += pamt_size[pgsz];\n-\t}\n+\tif (!pamt)\n+\t\treturn -ENOMEM;\n \n-\ttdmr-\u003epamt_4k_base = pamt_base[TDX_PS_4K];\n-\ttdmr-\u003epamt_4k_size = pamt_size[TDX_PS_4K];\n-\ttdmr-\u003epamt_2m_base = pamt_base[TDX_PS_2M];\n-\ttdmr-\u003epamt_2m_size = pamt_size[TDX_PS_2M];\n-\ttdmr-\u003epamt_1g_base = pamt_base[TDX_PS_1G];\n-\ttdmr-\u003epamt_1g_size = pamt_size[TDX_PS_1G];\n+\ttdmr-\u003epamt_4k_base = page_to_phys(pamt);\n+\ttdmr-\u003epamt_2m_base = tdmr-\u003epamt_4k_base + tdmr-\u003epamt_4k_size;\n+\ttdmr-\u003epamt_1g_base = tdmr-\u003epamt_2m_base + tdmr-\u003epamt_2m_size;\n \n \treturn 0;\n }\n@@ -655,10 +697,7 @@ static __init void tdmr_do_pamt_func(struct tdmr_info tdmr,\n \ttdmr_get_pamt(tdmr, \u0026pamt_base, \u0026pamt_size);\n \n \t/* Do nothing if PAMT hasn't been allocated for this TDMR /\n-\tif (!pamt_size)\n-\t\treturn;\n-\n-\tif (WARN_ON_ONCE(!pamt_base))\n+\tif (!pamt_base)\n \t\treturn;\n \n \tpamt_func(pamt_base, pamt_size);\n@@ -684,14 +723,12 @@ static __init void tdmrs_free_pamt_all(struct tdmr_info_list tdmr_list)\n \n /* Allocate and set up PAMTs for all TDMRs /\n static __init int tdmrs_set_up_pamt_all(struct tdmr_info_list tdmr_list,\n-\t\t\t\t\tstruct list_head tmb_list,\n-\t\t\t\t\tu16 pamt_entry_size[])\n+\t\t\t\t struct list_head tmb_list)\n {\n \tint i, ret = 0;\n \n \tfor (i = 0; i \u003c tdmr_list-\u003enr_consumed_tdmrs; i++) {\n-\t\tret = tdmr_set_up_pamt(tdmr_entry(tdmr_list, i), tmb_list,\n-\t\t\t\tpamt_entry_size);\n+\t\tret = tdmr_set_up_pamt(tdmr_entry(tdmr_list, i), tmb_list);\n \t\tif (ret)\n \t\t\tgoto err;\n \t}\n@@ -968,18 +1005,13 @@ static __init int construct_tdmrs(struct list_head tmb_list,\n \t\t\t\t struct tdmr_info_list tdmr_list,\n \t\t\t\t struct tdx_sys_info_tdmr sysinfo_tdmr)\n {\n-\tu16 pamt_entry_size[TDX_PS_NR] = {\n-\t\tsysinfo_tdmr-\u003epamt_4k_entry_size,\n-\t\tsysinfo_tdmr-\u003epamt_2m_entry_size,\n-\t\tsysinfo_tdmr-\u003epamt_1g_entry_size,\n-\t};\n \tint ret;\n \n \tret = fill_out_tdmrs(tmb_list, tdmr_list);\n \tif (ret)\n \t\treturn ret;\n \n-\tret = tdmrs_set_up_pamt_all(tdmr_list, tmb_list, pamt_entry_size);\n+\tret = tdmrs_set_up_pamt_all(tdmr_list, tmb_list);\n \tif (ret)\n \t\treturn ret;\n \n@@ -998,6 +1030,8 @@ static __init int construct_tdmrs(struct list_head tmb_list,\n \treturn ret;\n }\n \n+#define TDX_SYS_CONFIG_DYNAMIC_PAMT\tBIT(16)\n+\n static __init int config_tdx_module(struct tdmr_info_list tdmr_list,\n \t\t\t\t u64 global_keyid)\n {\n@@ -1026,6 +1060,12 @@ static __init int config_tdx_module(struct tdmr_info_list tdmr_list,\n \targs.rcx = __pa(tdmr_pa_array);\n \targs.rdx = tdmr_list-\u003enr_consumed_tdmrs;\n \targs.r8 = global_keyid;\n+\n+\tif (tdx_supports_dynamic_pamt(\u0026tdx_sysinfo)) {\n+\t\tpr_info(\"Enable Dynamic PAMT\\n\");\n+\t\targs.r8 \|= TDX_SYS_CONFIG_DYNAMIC_PAMT;\n+\t}\n+\n \tret = seamcall_prerr(TDH_SYS_CONFIG, \u0026args);\n \n \t/* Free the array as it is not required anymore. /\n@@ -1167,10 +1207,14 @@ static __init int init_tdx_module(void)\n \t /\n \tget_online_mems();\n \n-\tret = build_tdx_memlist(\u0026tdx_memlist);\n+\tret = init_pamt_refcounts();\n \tif (ret)\n \t\tgoto out_put_tdxmem;\n \n+\tret = build_tdx_memlist(\u0026tdx_memlist);\n+\tif (ret)\n+\t\tgoto err_free_pamt_refcounts;\n+\n \t/* Allocate enough space for constructing TDMRs /\n \tret = alloc_tdmr_list(\u0026tdx_tdmr_list, \u0026tdx_sysinfo.tdmr);\n \tif (ret)\n@@ -1220,6 +1264,8 @@ static __init int init_tdx_module(void)\n \tfree_tdmr_list(\u0026tdx_tdmr_list);\n err_free_tdxmem:\n \tfree_tdx_memlist(\u0026tdx_memlist);\n+err_free_pamt_refcounts:\n+\tfree_pamt_refcounts();\n \tgoto out_put_tdxmem;\n }\n \n@@ -2003,6 +2049,288 @@ u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, kvm_pfn_t pfn)\n }\n EXPORT_SYMBOL_FOR_KVM(tdh_phymem_page_wbinvd_hkid);\n \n+bool tdx_supports_dynamic_pamt(const struct tdx_sys_info sysinfo)\n+{\n+\treturn sysinfo-\u003efeatures.tdx_features0 \u0026 TDX_FEATURES0_DYNAMIC_PAMT \u0026\u0026\n+\t tdx_enable_dpamt;\n+}\n+EXPORT_SYMBOL_FOR_KVM(tdx_supports_dynamic_pamt);\n+\n+static struct page tdx_alloc_page_pamt_cache(struct tdx_pamt_cache cache)\n+{\n+\tstruct page page;\n+\n+\tpage = list_first_entry_or_null(\u0026cache-\u003epage_list, struct page, lru);\n+\tif (page) {\n+\t\tlist_del(\u0026page-\u003elru);\n+\t\tcache-\u003ecnt--;\n+\t}\n+\n+\treturn page;\n+}\n+\n+static struct page alloc_dpamt_page(struct tdx_pamt_cache cache)\n+{\n+\tif (cache)\n+\t\treturn tdx_alloc_page_pamt_cache(cache);\n+\n+\treturn alloc_page(GFP_KERNEL_ACCOUNT);\n+}\n+\n+static int alloc_pamt_array(struct page pamt_pages, struct tdx_pamt_cache cache)\n+{\n+\tint i, j;\n+\n+\tfor (i = 0; i \u003c TDX_DPAMT_ENTRY_PAGE_CNT; i++) {\n+\t\tpamt_pages[i] = alloc_dpamt_page(cache);\n+\t\tif (!pamt_pages[i])\n+\t\t\tgoto err;\n+\t}\n+\n+\treturn 0;\n+\n+err:\n+\tfor (j = 0; j \u003c i; j++)\n+\t\t__free_page(pamt_pages[j]);\n+\n+\treturn -ENOMEM;\n+}\n+\n+static void free_pamt_array(struct page *pamt_pages)\n+{\n+\tfor (int i = 0; i \u003c TDX_DPAMT_ENTRY_PAGE_CNT; i++) {\n+\t\t/\n+\t\t * Reset pages unconditionally to cover cases\n+\t\t * where they were passed to the TDX module.\n+\t\t /\n+\t\ttdx_quirk_reset_paddr(page_to_phys(pamt_pages[i]), PAGE_SIZE);\n+\n+\t\t__free_page(pamt_pages[i]);\n+\t}\n+}\n+\n+/\n+ * Calculate the arg needed for operating on the DPAMT backing for\n+ * a given 4KB page.\n+ /\n+static u64 pamt_2mb_arg(kvm_pfn_t pfn)\n+{\n+\t/ Arg value will specify a 2MB region of physical address space. /\n+\tunsigned long hpa_2mb = ALIGN_DOWN(pfn \u003c\u003c PAGE_SHIFT, PMD_SIZE);\n+\n+\treturn hpa_2mb \| TDX_PS_2M;\n+}\n+\n+/ Add PAMT backing for the 2MB region surrounding the given pfn. /\n+static u64 tdh_phymem_pamt_add(kvm_pfn_t pfn, struct page pamt_pages)\n+{\n+\tstruct tdx_module_args args = {\n+\t\t.rcx = pamt_2mb_arg(pfn),\n+\t\t.rdx = page_to_phys(pamt_pages[0]),\n+\t\t.r8 = page_to_phys(pamt_pages[1]),\n+\t};\n+\n+\treturn seamcall(TDH_PHYMEM_PAMT_ADD, \u0026args);\n+}\n+\n+/ Remove PAMT backing for the 2MB region surrounding the given pfn. /\n+static u64 tdh_phymem_pamt_remove(kvm_pfn_t pfn, struct page pamt_pages)\n+{\n+\tstruct tdx_module_args args = {\n+\t\t.rcx = pamt_2mb_arg(pfn),\n+\t};\n+\tu64 ret;\n+\n+\tret = seamcall_ret(TDH_PHYMEM_PAMT_REMOVE, \u0026args);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\t/ Copy PAMT pages out of the struct per the TDX ABI /\n+\tpamt_pages[0] = phys_to_page(args.rdx);\n+\tpamt_pages[1] = phys_to_page(args.r8);\n+\n+\treturn 0;\n+}\n+\n+/ Serializes adding/removing PAMT memory /\n+static DEFINE_SPINLOCK(pamt_lock);\n+\n+/ Bump PAMT refcount for the given pfn and allocate PAMT backing if needed. /\n+int tdx_pamt_get(kvm_pfn_t pfn, struct tdx_pamt_cache cache)\n+{\n+\tstruct page pamt_pages[TDX_DPAMT_ENTRY_PAGE_CNT];\n+\tatomic_t pamt_refcount;\n+\tu64 tdx_status;\n+\tint ret;\n+\n+\tif (!tdx_supports_dynamic_pamt(\u0026tdx_sysinfo))\n+\t\treturn 0;\n+\n+\tpamt_refcount = tdx_find_pamt_refcount(pfn);\n+\n+\t/\n+\t If the pamt page is already added (i.e. refcount \u003e= 1),\n+\t * then just increment the refcount.\n+\t /\n+\tif (atomic_inc_not_zero(pamt_refcount))\n+\t\treturn 0;\n+\n+\tret = alloc_pamt_array(pamt_pages, cache);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tspin_lock(\u0026pamt_lock);\n+\n+\t/\n+\t * Unlike tdx_pamt_put() which uses atomic_dec_and_lock() to\n+\t * atomically handle the 1-\u003e0 transition, the get side has no\n+\t * equivalent combined primitive for 0-\u003e1. Recheck under the\n+\t * lock since another get may have already done the 0-\u003e1\n+\t * transition after both saw atomic_inc_not_zero() fail.\n+\t /\n+\tif (atomic_inc_not_zero(pamt_refcount))\n+\t\tgoto out_free;\n+\n+\ttdx_status = tdh_phymem_pamt_add(pfn, pamt_pages);\n+\tif (WARN_ON_ONCE(tdx_status != TDX_SUCCESS)) {\n+\t\tret = -EIO;\n+\t\tgoto out_free;\n+\t}\n+\n+\t/\n+\t * The refcount is zero, and this locked path is the\n+\t * only way to increase it from 0-\u003e1.\n+\t /\n+\tatomic_set(pamt_refcount, 1);\n+\tspin_unlock(\u0026pamt_lock);\n+\treturn 0;\n+\n+out_free:\n+\tspin_unlock(\u0026pamt_lock);\n+\tfree_pamt_array(pamt_pages);\n+\n+\treturn ret;\n+}\n+EXPORT_SYMBOL_FOR_KVM(tdx_pamt_get);\n+\n+/ Drop PAMT refcount for the given pfn and free PAMT backing if needed. /\n+void tdx_pamt_put(kvm_pfn_t pfn)\n+{\n+\tstruct page pamt_pages[TDX_DPAMT_ENTRY_PAGE_CNT] = {};\n+\tatomic_t pamt_refcount;\n+\tu64 tdx_status;\n+\n+\tif (!tdx_supports_dynamic_pamt(\u0026tdx_sysinfo))\n+\t\treturn;\n+\n+\tpamt_refcount = tdx_find_pamt_refcount(pfn);\n+\n+\t/\n+\t * If there is more than 1 reference on the pamt page, don't\n+\t * remove it yet. Just decrement the refcount.\n+\t /\n+\tif (!atomic_dec_and_lock(pamt_refcount, \u0026pamt_lock))\n+\t\treturn;\n+\n+\ttdx_status = tdh_phymem_pamt_remove(pfn, pamt_pages);\n+\n+\t/\n+\t * Don't free pamt_pages as it could hold garbage when\n+\t * tdh_phymem_pamt_remove() fails. Don't panic/BUG_ON(), as\n+\t * there is no risk of data corruption, but do yell loudly as\n+\t * failure indicates a kernel bug, memory is being leaked, and\n+\t * the dangling PAMT entry may cause future operations to fail.\n+\t /\n+\tif (WARN_ON_ONCE(tdx_status != TDX_SUCCESS)) {\n+\t\t/\n+\t\t * atomic_dec_and_lock() already decremented it to 0,\n+\t\t * but the PAMT entry still exists since REMOVE failed.\n+\t\t /\n+\t\tatomic_set(pamt_refcount, 1);\n+\t\tgoto out_unlock;\n+\t}\n+\n+\tspin_unlock(\u0026pamt_lock);\n+\tfree_pamt_array(pamt_pages);\n+\treturn;\n+out_unlock:\n+\tspin_unlock(\u0026pamt_lock);\n+}\n+EXPORT_SYMBOL_FOR_KVM(tdx_pamt_put);\n+\n+void tdx_free_pamt_cache(struct tdx_pamt_cache cache)\n+{\n+\tstruct page page;\n+\n+\twhile ((page = tdx_alloc_page_pamt_cache(cache)))\n+\t\t__free_page(page);\n+}\n+EXPORT_SYMBOL_FOR_KVM(tdx_free_pamt_cache);\n+\n+int tdx_topup_pamt_cache(struct tdx_pamt_cache cache, unsigned long npages)\n+{\n+\tif (WARN_ON_ONCE(!tdx_supports_dynamic_pamt(\u0026tdx_sysinfo)))\n+\t\treturn 0;\n+\n+\tnpages = TDX_DPAMT_ENTRY_PAGE_CNT;\n+\n+\twhile (cache-\u003ecnt \u003c npages) {\n+\t\tstruct page page = alloc_page(GFP_KERNEL_ACCOUNT);\n+\n+\t\tif (!page)\n+\t\t\treturn -ENOMEM;\n+\n+\t\tlist_add(\u0026page-\u003elru, \u0026cache-\u003epage_list);\n+\t\tcache-\u003ecnt++;\n+\t}\n+\n+\treturn 0;\n+}\n+EXPORT_SYMBOL_FOR_KVM(tdx_topup_pamt_cache);\n+\n+/\n+ Return a page that can be gifted to the TDX-Module for use as a \"control\"\n+ * page, i.e. pages that are used for control structures for a given TDX\n+ * guest, and thus obtain TDX protections, including PAMT tracking.\n+ /\n+struct page tdx_alloc_control_page(void)\n+{\n+\tstruct page page;\n+\n+\tpage = alloc_page(GFP_KERNEL_ACCOUNT);\n+\tif (!page)\n+\t\treturn NULL;\n+\n+\tif (tdx_pamt_get(page_to_pfn(page), NULL)) {\n+\t\t__free_page(page);\n+\t\treturn NULL;\n+\t}\n+\n+\treturn page;\n+}\n+EXPORT_SYMBOL_FOR_KVM(tdx_alloc_control_page);\n+\n+/\n+ * Free a page that was gifted to the TDX-Module for use as a control\n+ * page. After this, the page is no longer protected by TDX.\n+ /\n+void tdx_free_control_page(struct page page)\n+{\n+\tif (!page)\n+\t\treturn;\n+\n+\ttdx_pamt_put(page_to_pfn(page));\n+\t__free_page(page);\n+}\n+EXPORT_SYMBOL_FOR_KVM(tdx_free_control_page);\n+\n+static int __init tdx_dpamt_setup(char str)\n+{\n+\treturn kstrtobool(str, \u0026tdx_enable_dpamt) == 0;\n+}\n+\n+__setup(\"tdx_dpamt=\", tdx_dpamt_setup);\n+\n void tdx_sys_disable(void)\n {\n \tstruct tdx_module_args args = {};\ndiff --git a/arch/x86/virt/vmx/tdx/tdx.h b/arch/x86/virt/vmx/tdx/tdx.h\nindex bdfd0e1e337ac..a886c54decaad 100644\n--- a/arch/x86/virt/vmx/tdx/tdx.h\n+++ b/arch/x86/virt/vmx/tdx/tdx.h\n@@ -48,6 +48,8 @@\n #define TDH_SYS_CONFIG\t\t\t45\n #define TDH_SYS_SHUTDOWN\t\t52\n #define TDH_SYS_UPDATE\t\t\t53\n+#define TDH_PHYMEM_PAMT_ADD\t\t58\n+#define TDH_PHYMEM_PAMT_REMOVE\t\t59\n #define TDH_SYS_DISABLE\t\t\t69\n \n /\ndiff --git a/arch/x86/virt/vmx/tdx/tdx_global_metadata.c b/arch/x86/virt/vmx/tdx/tdx_global_metadata.c\nindex e49c300f23d43..93b1f26252c63 100644\n--- a/arch/x86/virt/vmx/tdx/tdx_global_metadata.c\n+++ b/arch/x86/virt/vmx/tdx/tdx_global_metadata.c\n@@ -1,6 +1,6 @@\n // SPDX-License-Identifier: GPL-2.0\n /\n- Automatically generated functions to read TDX global metadata.\n+ * Functions to read TDX global metadata.\n \n This file doesn't compile on its own as it lacks of inclusion\n * of SEAMCALL wrapper primitive which reads global metadata.\n@@ -33,6 +33,18 @@ static __init int get_tdx_sys_info_features(struct tdx_sys_info_features sysinf\n \treturn ret;\n }\n \n+static __init int get_tdx_sys_info_tdmr_dpamt(struct tdx_sys_info_tdmr sysinfo_tdmr)\n+{\n+\tint ret;\n+\tu64 val;\n+\n+\tret = read_sys_metadata_field(0x9100000000000013, \u0026val);\n+\tif (!ret)\n+\t\tsysinfo_tdmr-\u003epamt_page_bitmap_entry_bits = val;\n+\n+\treturn ret;\n+}\n+\n static __init int get_tdx_sys_info_tdmr(struct tdx_sys_info_tdmr sysinfo_tdmr)\n {\n \tint ret = 0;\n@@ -129,5 +141,14 @@ static __init int get_tdx_sys_info(struct tdx_sys_info sysinfo)\n \tret = ret ?: get_tdx_sys_info_td_ctrl(\u0026sysinfo-\u003etd_ctrl);\n \tret = ret ?: get_tdx_sys_info_td_conf(\u0026sysinfo-\u003etd_conf);\n \n+\t/\n+\t The kernel supports using TDX without Dynamic PAMT, so\n+\t * avoid reporting failure if it's not supported. Don't try\n+\t * to support buggy TDX modules that advertise Dynamic PAMT\n+\t * but don't expose the metadata.\n+\t */\n+\tif (!ret \u0026\u0026 tdx_supports_dynamic_pamt(sysinfo))\n+\t\tret = get_tdx_sys_info_tdmr_dpamt(\u0026sysinfo-\u003etdmr);\n+\n \treturn ret;\n }\n" }
2/1	2026/08/06 03:01	agent	patch-evaluator	0m Model: best-expensive Args: null Results: { "EnableConfigs": null, "FocusSymbols": [], "Reasoning": "The patch adds support for Dynamic PAMT in the TDX host (KVM) code. TDX host support requires specific bare-metal Intel hardware with Trust Domain Extensions enabled and the TDX module loaded. This hardware is not emulated in standard virtualized environments like QEMU or GCE, making the modified code paths unreachable during fuzzing.", "WorthFuzzing": false } Instruction: You are an expert Linux kernel maintainer. Your job is to review a provided patch series and determine if it makes functional changes to the kernel that should be fuzzed. IMPORTANT: The changes have ALREADY been applied and committed as the HEAD commit in your workspace. Do NOT rely on your internal knowledge of the kernel. You must actively use your code access tools to examine the actual source code and confirm any assumptions. Return WorthFuzzing=false if the patch only contains: - Modifications to Documentation/, Kconfig files, or code comments. - Purely decorative changes, such as logging (e.g., pr_err, printk) or tracepoints. - Changes to numeric constants or macros that do not functionally alter execution flow. - Code paths that are impossible to reach in virtualized environments like GCE or QEMU, even when utilizing software-emulated hardware (e.g., usb gadget, mac80211_hwsim). - Code in vendor-specific PCIe switch, SmartNIC, or GPU drivers (e.g., mlxsw, pds_core, qed, ionic, amdgpu) that require physical PCIe hardware cards not emulated in standard QEMU. - Driver .remove, .shutdown, or pci_unregister_driver teardown callbacks (e.g., igb_remove) that are executed only during PCI hot-unplug or sysfs driver unbind operations. If it modifies reachable core kernel logic, drivers, or architectures, use your code search tools to verify the code can be executed, then return WorthFuzzing=true. When returning WorthFuzzing=true, you MUST ALSO: 1. Extract any specific kernel functions that should be heavily fuzzed into FocusSymbols. Avoid listing generic hot-path functions to prevent skewed test distributions. Prefer non-static, non-inlined API entrypoint functions over internal static helper functions (which are inlined by the compiler and do not have distinct symbol addresses). 2. Identify any specific CONFIG_ options required to properly test this new/modified feature. Go and look into the Kconfig files and check for ifdefs around the code, do not make assumptions. Also check "depends on" lines in Kconfig to include any non-standard parent subsystem configs needed for Kbuild to compile the code statically into vmlinux. List them in the EnableConfigs output array, and DO NOT add a 'CONFIG_' prefix (e.g., return "NET_IPV4" instead of "CONFIG_NET_IPV4"). Prefer calling several tools at the same time to save round-trips. Use set-results tool to provide results of the analysis. It must be called exactly once before the final reply. Ignore results of this tool. Prompt: For your convenience, here is the diff of the changes: commit f00efc49131881f16a5b1580adf3f7c794263146 Author: syz-cluster <triage@syzkaller.com> Date: Thu Aug 6 03:00:59 2026 +0000 syz-cluster: applied patch under review diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index b5493a7f8f228..a0a7670246bc5 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -7512,6 +7512,16 @@ Kernel parameters tdfx= [HW,DRM] + tdx_dpamt= + [X86] Controls whether TDX will use Dynamic PAMT + to save memory, when supported. + + Valid parameters: "on", "off" + Default: "off" + + For details see: + Documentation/arch/x86/tdx.rst + test_suspend= [SUSPEND] Format: { "mem" \| "standby" \| "freeze" }[,N] Specify "mem" (for Suspend-to-RAM) or "standby" (for diff --git a/Documentation/arch/x86/tdx.rst b/Documentation/arch/x86/tdx.rst index 3303499ad4c6f..a1c2309230580 100644 --- a/Documentation/arch/x86/tdx.rst +++ b/Documentation/arch/x86/tdx.rst @@ -200,6 +200,34 @@ reflects the TCB of the currently running TDX module and therefore changes after an update. By contrast, TEE_TCB_SVN reflects the TCB at TD launch time and is not affected. +Dynamic PAMT +------------ + +Physical Address Metadata Table (PAMT) is memory that the TDX module needs +to keep data about each page (think like struct page). It needs to be handed +to the TDX module for its exclusive use. For normal PAMT, this is installed +when the TDX module is first loaded and comes to about 0.4% of system memory. + +Dynamic PAMT is a TDX module feature that allows VMM to allocate part of the +PAMT as needed (the parts for tracking 4KB size pages). The other page sizes +(1GB and 2MB) are still allocated statically at the time of TDX module +initialization. This reduces the amount of memory that TDX uses while TDs are +not in use. + +When Dynamic PAMT is in use, dmesg shows it like:: + + [..] virt/tdx: Enable Dynamic PAMT + [..] virt/tdx: 10092 KB allocated for PAMT + [..] virt/tdx: TDX-Module initialized + +Dynamic PAMT is only enabled when supported and the ``tdx_dpamt=`` kernel +parameter is set to "on". The feature is off by default because TDX module +internal details prevent Dynamic PAMT from working on all keyid partitioning +configurations. When the TDX module is fixed to include these constraints in +its enumeration of Dynamic PAMT support, kernel support can be changed to +default on. For more information, consult the Intel TDX documentation about +Dynamic PAMT. + TDX Interaction to Other Kernel Components ------------------------------------------ diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index 83dc5086138b3..588563dfe88d5 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -98,6 +98,7 @@ KVM_X86_OP_OPTIONAL_RET0(tdp_has_smep) KVM_X86_OP(load_mmu_pgd) KVM_X86_OP_OPTIONAL_RET0(set_external_spte) KVM_X86_OP_OPTIONAL(free_external_spt) +KVM_X86_OP_OPTIONAL_RET0(topup_external_cache) KVM_X86_OP(has_wbinvd_exit) KVM_X86_OP(get_l2_tsc_offset) KVM_X86_OP(get_l2_tsc_multiplier) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 5f6c1ce9673b7..0dbf6564dd0c5 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1922,6 +1922,7 @@ struct kvm_x86_ops { /* Update external page tables for page table about to be freed. / void (free_external_spt)(struct kvm kvm, struct kvm_mmu_page sp); + int (topup_external_cache)(struct kvm_vcpu vcpu, int min_nr_spts); bool (has_wbinvd_exit)(void); diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h index 89e97d5761d89..7910901a7ba21 100644 --- a/arch/x86/include/asm/tdx.h +++ b/arch/x86/include/asm/tdx.h @@ -36,6 +36,7 @@ / Bit definitions of TDX_FEATURES0 metadata field / #define TDX_FEATURES0_TD_PRESERVING BIT_ULL(1) #define TDX_FEATURES0_NO_RBP_MOD BIT_ULL(18) +#define TDX_FEATURES0_DYNAMIC_PAMT BIT_ULL(36) #ifndef __ASSEMBLER__ @@ -118,12 +119,37 @@ static inline bool tdx_supports_runtime_update(const struct tdx_sys_info sysinf return sysinfo->features.tdx_features0 & TDX_FEATURES0_TD_PRESERVING; } +bool tdx_supports_dynamic_pamt(const struct tdx_sys_info sysinfo); + +/ Simple structure for pre-allocating Dynamic PAMT pages outside of spinlocks. / +struct tdx_pamt_cache { + struct list_head page_list; + int cnt; +}; + +static inline void tdx_init_pamt_cache(struct tdx_pamt_cache cache) +{ + INIT_LIST_HEAD(&cache->page_list); + cache->cnt = 0; +} + +void tdx_free_pamt_cache(struct tdx_pamt_cache cache); +int tdx_topup_pamt_cache(struct tdx_pamt_cache cache, unsigned long npages); +int tdx_pamt_get(kvm_pfn_t pfn, struct tdx_pamt_cache cache); +void tdx_pamt_put(kvm_pfn_t pfn); + int tdx_guest_keyid_alloc(void); u32 tdx_get_nr_guest_keyids(void); void tdx_guest_keyid_free(unsigned int keyid); void tdx_quirk_reset_paddr(unsigned long base, unsigned long size); +/ Number of PAMT pages to be provided to TDX module per 2MB region of PA / +#define TDX_DPAMT_ENTRY_PAGE_CNT 2 + +struct page tdx_alloc_control_page(void); +void tdx_free_control_page(struct page page); + struct tdx_td { / TD root structure: / struct page tdr_page; diff --git a/arch/x86/include/asm/tdx_global_metadata.h b/arch/x86/include/asm/tdx_global_metadata.h index 41150d546589c..2a42551fc33cd 100644 --- a/arch/x86/include/asm/tdx_global_metadata.h +++ b/arch/x86/include/asm/tdx_global_metadata.h @@ -21,6 +21,9 @@ struct tdx_sys_info_tdmr { u16 pamt_4k_entry_size; u16 pamt_2m_entry_size; u16 pamt_1g_entry_size; + + /* Optional metadata, if Dynamic PAMT is supported / + u8 pamt_page_bitmap_entry_bits; }; struct tdx_sys_info_td_ctrl { diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 66e69d2a41b3c..2a628b8304409 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -614,6 +614,10 @@ static int mmu_topup_memory_caches(struct kvm_vcpu vcpu, bool maybe_indirect) PT64_ROOT_MAX_LEVEL); if (r) return r; + + r = kvm_x86_call(topup_external_cache)(vcpu, PT64_ROOT_MAX_LEVEL); + if (r) + return r; } r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_shadow_page_cache, PT64_ROOT_MAX_LEVEL); diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 545b03d9d10b8..09573112540e9 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -362,7 +362,7 @@ static void tdx_reclaim_control_page(struct page ctrl_page) if (tdx_reclaim_page(ctrl_page)) return; - __free_page(ctrl_page); + tdx_free_control_page(ctrl_page); } struct tdx_flush_vp_arg { @@ -589,7 +589,7 @@ static void tdx_reclaim_td_control_pages(struct kvm kvm) tdx_quirk_reset_paddr(page_to_phys(kvm_tdx->td.tdr_page), PAGE_SIZE); - __free_page(kvm_tdx->td.tdr_page); + tdx_free_control_page(kvm_tdx->td.tdr_page); kvm_tdx->td.tdr_page = NULL; } @@ -681,6 +681,8 @@ int tdx_vcpu_create(struct kvm_vcpu vcpu) if (!irqchip_split(vcpu->kvm)) return -EINVAL; + tdx_init_pamt_cache(&tdx->pamt_cache); + fpstate_set_confidential(&vcpu->arch.guest_fpu); vcpu->arch.apic->guest_apic_protected = true; INIT_LIST_HEAD(&tdx->vt.pi_wakeup_list); @@ -866,6 +868,8 @@ void tdx_vcpu_free(struct kvm_vcpu vcpu) struct vcpu_tdx tdx = to_tdx(vcpu); int i; + tdx_free_pamt_cache(&tdx->pamt_cache); + if (vcpu->cpu != -1) { KVM_BUG_ON(tdx->state == VCPU_TD_STATE_INITIALIZED, vcpu->kvm); tdx_flush_vp_on_cpu(vcpu); @@ -1621,6 +1625,17 @@ void tdx_load_mmu_pgd(struct kvm_vcpu vcpu, hpa_t root_hpa, int pgd_level) td_vmcs_write64(to_tdx(vcpu), SHARED_EPT_POINTER, root_hpa); } +static int tdx_topup_external_pamt_cache(struct kvm_vcpu vcpu, int min_nr_spts) +{ + / + * Minus one page to exclude the root SPT, but plus one page for a + * possible 4KB private mapping. + / + min_nr_spts += -1 + 1; + + return tdx_topup_pamt_cache(&to_tdx(vcpu)->pamt_cache, min_nr_spts); +} + static int tdx_mem_page_add(struct kvm kvm, gfn_t gfn, enum pg_level level, kvm_pfn_t pfn) { @@ -1679,16 +1694,28 @@ static struct page tdx_spte_to_sept_pt(struct kvm kvm, gfn_t gfn, static int tdx_sept_map_nonleaf_spte(struct kvm kvm, gfn_t gfn, enum pg_level level, u64 new_spte) { + struct kvm_vcpu vcpu = kvm_get_running_vcpu(); gpa_t gpa = gfn_to_gpa(gfn); u64 err, entry, level_state; struct page sept_pt; + int ret; + + if (KVM_BUG_ON(!vcpu, kvm)) + return -EIO; sept_pt = tdx_spte_to_sept_pt(kvm, gfn, new_spte, level); if (!sept_pt) return -EIO; + ret = tdx_pamt_get(page_to_pfn(sept_pt), &to_tdx(vcpu)->pamt_cache); + if (KVM_BUG_ON(ret, kvm)) + return ret; + err = tdh_mem_sept_add(&to_kvm_tdx(kvm)->td, gpa, level, sept_pt, &entry, &level_state); + if (err) + tdx_pamt_put(page_to_pfn(sept_pt)); + if (unlikely(tdx_operand_busy(err))) return -EBUSY; @@ -1701,8 +1728,13 @@ static int tdx_sept_map_nonleaf_spte(struct kvm kvm, gfn_t gfn, static int tdx_sept_map_leaf_spte(struct kvm kvm, gfn_t gfn, enum pg_level level, u64 new_spte) { + struct kvm_vcpu vcpu = kvm_get_running_vcpu(); struct kvm_tdx kvm_tdx = to_kvm_tdx(kvm); kvm_pfn_t pfn = spte_to_pfn(new_spte); + int ret; + + if (KVM_BUG_ON(!vcpu, kvm)) + return -EIO; / TODO: handle large pages. / if (KVM_BUG_ON(level != PG_LEVEL_4K, kvm)) @@ -1710,6 +1742,10 @@ static int tdx_sept_map_leaf_spte(struct kvm kvm, gfn_t gfn, enum pg_level leve WARN_ON_ONCE((new_spte & VMX_EPT_RWX_MASK) != VMX_EPT_RWX_MASK); + ret = tdx_pamt_get(pfn, &to_tdx(vcpu)->pamt_cache); + if (KVM_BUG_ON(ret, kvm)) + return ret; + /* * Ensure pre_fault_allowed is read by kvm_arch_vcpu_pre_fault_memory() * before kvm_tdx->state. Userspace must not be allowed to pre-fault @@ -1722,10 +1758,15 @@ static int tdx_sept_map_leaf_spte(struct kvm kvm, gfn_t gfn, enum pg_level leve If the TD isn't finalized/runnable, then userspace is initializing * the VM image via KVM_TDX_INIT_MEM_REGION; ADD the page to the TD. / - if (unlikely(kvm_tdx->state != TD_STATE_RUNNABLE)) - return tdx_mem_page_add(kvm, gfn, level, pfn); + if (likely(kvm_tdx->state == TD_STATE_RUNNABLE)) + ret = tdx_mem_page_aug(kvm, gfn, level, pfn); + else + ret = tdx_mem_page_add(kvm, gfn, level, pfn); + + if (ret) + tdx_pamt_put(pfn); - return tdx_mem_page_aug(kvm, gfn, level, pfn); + return ret; } / @@ -1822,6 +1863,7 @@ static int tdx_sept_remove_leaf_spte(struct kvm kvm, gfn_t gfn, return -EIO; tdx_quirk_reset_paddr(PFN_PHYS(pfn), PAGE_SIZE); + tdx_pamt_put(pfn); return 0; } @@ -1865,6 +1907,8 @@ static int tdx_sept_set_private_spte(struct kvm kvm, gfn_t gfn, u64 old_spte, / static void tdx_sept_free_private_spt(struct kvm kvm, struct kvm_mmu_page sp) { + struct page sept_pt = virt_to_page(sp->external_spt); + /* * KVM doesn't (yet) zap page table pages in mirror page table while * TD is active, though guest pages mapped in mirror page table could be @@ -1878,15 +1922,15 @@ static void tdx_sept_free_private_spt(struct kvm kvm, struct kvm_mmu_page sp) * the page to prevent the kernel from accessing the encrypted page. / if (KVM_BUG_ON(is_hkid_assigned(to_kvm_tdx(kvm)), kvm) \|\| - tdx_reclaim_page(virt_to_page(sp->external_spt))) + tdx_reclaim_page(sept_pt)) goto out; / - * Immediately free the S-EPT page because RCU-time free is unnecessary - * after TDH.PHYMEM.PAGE.RECLAIM ensures there are no outstanding - * readers. + * Immediately free the S-EPT page as the TDX subsystem doesn't support + * freeing pages from RCU callbacks, and more importantly because + * TDH.PHYMEM.PAGE.RECLAIM ensures there are no outstanding readers. / - free_page((unsigned long)sp->external_spt); + tdx_free_control_page(sept_pt); out: sp->external_spt = NULL; } @@ -2459,7 +2503,7 @@ static int __tdx_td_init(struct kvm kvm, struct td_params td_params, ret = -ENOMEM; - tdr_page = alloc_page(GFP_KERNEL_ACCOUNT); + tdr_page = tdx_alloc_control_page(); if (!tdr_page) goto free_hkid; @@ -2472,7 +2516,7 @@ static int __tdx_td_init(struct kvm kvm, struct td_params td_params, goto free_tdr; for (i = 0; i < kvm_tdx->td.tdcs_nr_pages; i++) { - tdcs_pages[i] = alloc_page(GFP_KERNEL_ACCOUNT); + tdcs_pages[i] = tdx_alloc_control_page(); if (!tdcs_pages[i]) goto free_tdcs; } @@ -2590,10 +2634,8 @@ static int __tdx_td_init(struct kvm kvm, struct td_params td_params, teardown: / Only free pages not yet added, so start at 'i' / for (; i < kvm_tdx->td.tdcs_nr_pages; i++) { - if (tdcs_pages[i]) { - __free_page(tdcs_pages[i]); - tdcs_pages[i] = NULL; - } + tdx_free_control_page(tdcs_pages[i]); + tdcs_pages[i] = NULL; } if (!kvm_tdx->td.tdcs_pages) kfree(tdcs_pages); @@ -2608,16 +2650,13 @@ static int __tdx_td_init(struct kvm kvm, struct td_params td_params, free_cpumask_var(packages); free_tdcs: - for (i = 0; i < kvm_tdx->td.tdcs_nr_pages; i++) { - if (tdcs_pages[i]) - __free_page(tdcs_pages[i]); - } + for (i = 0; i < kvm_tdx->td.tdcs_nr_pages; i++) + tdx_free_control_page(tdcs_pages[i]); kfree(tdcs_pages); kvm_tdx->td.tdcs_pages = NULL; free_tdr: - if (tdr_page) - __free_page(tdr_page); + tdx_free_control_page(tdr_page); kvm_tdx->td.tdr_page = NULL; free_hkid: @@ -2951,7 +2990,7 @@ static int tdx_td_vcpu_init(struct kvm_vcpu vcpu, u64 vcpu_rcx) int ret, i; u64 err; - page = alloc_page(GFP_KERNEL_ACCOUNT); + page = tdx_alloc_control_page(); if (!page) return -ENOMEM; tdx->vp.tdvpr_page = page; @@ -2971,7 +3010,7 @@ static int tdx_td_vcpu_init(struct kvm_vcpu vcpu, u64 vcpu_rcx) } for (i = 0; i < kvm_tdx->td.tdcx_nr_pages; i++) { - page = alloc_page(GFP_KERNEL_ACCOUNT); + page = tdx_alloc_control_page(); if (!page) { ret = -ENOMEM; goto free_tdcx; @@ -2993,7 +3032,7 @@ static int tdx_td_vcpu_init(struct kvm_vcpu vcpu, u64 vcpu_rcx) * method, but the rest are freed here. / for (; i < kvm_tdx->td.tdcx_nr_pages; i++) { - __free_page(tdx->vp.tdcx_pages[i]); + tdx_free_control_page(tdx->vp.tdcx_pages[i]); tdx->vp.tdcx_pages[i] = NULL; } return -EIO; @@ -3021,16 +3060,14 @@ static int tdx_td_vcpu_init(struct kvm_vcpu vcpu, u64 vcpu_rcx) free_tdcx: for (i = 0; i < kvm_tdx->td.tdcx_nr_pages; i++) { - if (tdx->vp.tdcx_pages[i]) - __free_page(tdx->vp.tdcx_pages[i]); + tdx_free_control_page(tdx->vp.tdcx_pages[i]); tdx->vp.tdcx_pages[i] = NULL; } kfree(tdx->vp.tdcx_pages); tdx->vp.tdcx_pages = NULL; free_tdvpr: - if (tdx->vp.tdvpr_page) - __free_page(tdx->vp.tdvpr_page); + tdx_free_control_page(tdx->vp.tdvpr_page); tdx->vp.tdvpr_page = NULL; tdx->vp.tdvpr_pa = 0; @@ -3493,6 +3530,10 @@ int __init tdx_hardware_setup(void) vt_x86_ops.set_external_spte = tdx_sept_set_private_spte; vt_x86_ops.free_external_spt = tdx_sept_free_private_spt; + + if (tdx_supports_dynamic_pamt(tdx_sysinfo)) + vt_x86_ops.topup_external_cache = tdx_topup_external_pamt_cache; + vt_x86_ops.protected_apic_has_interrupt = tdx_protected_apic_has_interrupt; return 0; diff --git a/arch/x86/kvm/vmx/tdx.h b/arch/x86/kvm/vmx/tdx.h index ac8323a68b163..fd368e3ee060b 100644 --- a/arch/x86/kvm/vmx/tdx.h +++ b/arch/x86/kvm/vmx/tdx.h @@ -72,6 +72,8 @@ struct vcpu_tdx { u64 map_gpa_next; u64 map_gpa_end; + + struct tdx_pamt_cache pamt_cache; }; void tdh_vp_rd_failed(struct vcpu_tdx tdx, char uclass, u32 field, u64 err); diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index 42df8ea464c47..dec45709e454a 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -30,6 +30,7 @@ #include <linux/suspend.h> #include <linux/syscore_ops.h> #include <linux/idr.h> +#include <linux/vmalloc.h> #include <asm/page.h> #include <asm/special_insns.h> #include <asm/msr-index.h> @@ -46,6 +47,8 @@ #include "seamcall_internal.h" #include "tdx.h" +static bool tdx_enable_dpamt __ro_after_init; + struct tdx_module_state { bool initialized; bool sysinit_done; @@ -63,6 +66,14 @@ static DEFINE_PER_CPU(bool, tdx_lp_initialized); static struct tdmr_info_list tdx_tdmr_list; +/* + * On a machine with Dynamic PAMT, the kernel maintains a reference counter + * for every 2MB range. The counter indicates how many users there are for + * the PAMT memory of the 2MB range. The kernel allocates PAMT refcounts at + * initialization. + / +static atomic_t pamt_refcounts; + /* All TDX-usable memory regions. Protected by mem_hotplug_lock. / static LIST_HEAD(tdx_memlist); @@ -252,6 +263,42 @@ static struct syscore tdx_syscore = { .ops = &tdx_syscore_ops, }; +/ + * Allocate PAMT reference counters for all physical memory. + * + * It consumes 2MB for every 1TB of physical memory. + / +static __init int init_pamt_refcounts(void) +{ + size_t size = DIV_ROUND_UP(max_pfn, PTRS_PER_PTE) sizeof(pamt_refcounts); + + if (!tdx_supports_dynamic_pamt(&tdx_sysinfo)) + return 0; + + pamt_refcounts = vzalloc(size); + if (!pamt_refcounts) + return -ENOMEM; + + return 0; +} + +static __init void free_pamt_refcounts(void) +{ + if (!tdx_supports_dynamic_pamt(&tdx_sysinfo)) + return; + + vfree(pamt_refcounts); + pamt_refcounts = NULL; +} + +static atomic_t tdx_find_pamt_refcount(unsigned long pfn) +{ + /* Find which PMD a PFN is in. / + unsigned long index = pfn >> (PMD_SHIFT - PAGE_SHIFT); + + return &pamt_refcounts[index]; +} + / * Add a memory region as a TDX memory block. The caller must make sure * all memory regions are added in address ascending order and don't @@ -510,35 +557,37 @@ static __init int fill_out_tdmrs(struct list_head tmb_list, return 0; } +static __init unsigned long tdmr_get_pamt_bitmap_sz(struct tdmr_info tdmr) +{ + unsigned long pamt_sz, nr_pamt_entries; + int bits_per_entry; + + bits_per_entry = tdx_sysinfo.tdmr.pamt_page_bitmap_entry_bits; + nr_pamt_entries = tdmr->size >> PAGE_SHIFT; + pamt_sz = DIV_ROUND_UP(nr_pamt_entries * bits_per_entry, BITS_PER_BYTE); + + return PAGE_ALIGN(pamt_sz); +} + /* * Calculate PAMT size given a TDMR and a page size. The returned * PAMT size is always aligned up to 4K page boundary. / -static __init unsigned long tdmr_get_pamt_sz(struct tdmr_info tdmr, int pgsz, - u16 pamt_entry_size) +static __init unsigned long tdmr_get_pamt_sz(struct tdmr_info tdmr, int pgsz) { unsigned long pamt_sz, nr_pamt_entries; + const int tdx_pg_size_shift[TDX_PS_NR] = { PAGE_SHIFT, PMD_SHIFT, PUD_SHIFT }; + const u16 pamt_entry_size[TDX_PS_NR] = { + tdx_sysinfo.tdmr.pamt_4k_entry_size, + tdx_sysinfo.tdmr.pamt_2m_entry_size, + tdx_sysinfo.tdmr.pamt_1g_entry_size, + }; - switch (pgsz) { - case TDX_PS_4K: - nr_pamt_entries = tdmr->size >> PAGE_SHIFT; - break; - case TDX_PS_2M: - nr_pamt_entries = tdmr->size >> PMD_SHIFT; - break; - case TDX_PS_1G: - nr_pamt_entries = tdmr->size >> PUD_SHIFT; - break; - default: - WARN_ON_ONCE(1); - return 0; - } + nr_pamt_entries = tdmr->size >> tdx_pg_size_shift[pgsz]; + pamt_sz = nr_pamt_entries pamt_entry_size[pgsz]; - pamt_sz = nr_pamt_entries * pamt_entry_size; /* TDX requires PAMT size must be 4K aligned / - pamt_sz = ALIGN(pamt_sz, PAGE_SIZE); - - return pamt_sz; + return PAGE_ALIGN(pamt_sz); } / @@ -576,15 +625,11 @@ static __init int tdmr_get_nid(struct tdmr_info tdmr, struct list_head tmb_lis * within @tdmr, and set up PAMTs for @tdmr. / static __init int tdmr_set_up_pamt(struct tdmr_info tdmr, - struct list_head tmb_list, - u16 pamt_entry_size[]) + struct list_head tmb_list) { - unsigned long pamt_base[TDX_PS_NR]; - unsigned long pamt_size[TDX_PS_NR]; - unsigned long tdmr_pamt_base; unsigned long tdmr_pamt_size; struct page pamt; - int pgsz, nid; + int nid; nid = tdmr_get_nid(tdmr, tmb_list); @@ -592,13 +637,18 @@ static __init int tdmr_set_up_pamt(struct tdmr_info tdmr, * Calculate the PAMT size for each TDX supported page size * and the total PAMT size. / - tdmr_pamt_size = 0; - for (pgsz = TDX_PS_4K; pgsz < TDX_PS_NR; pgsz++) { - pamt_size[pgsz] = tdmr_get_pamt_sz(tdmr, pgsz, - pamt_entry_size[pgsz]); - tdmr_pamt_size += pamt_size[pgsz]; + tdmr->pamt_1g_size = tdmr_get_pamt_sz(tdmr, TDX_PS_1G); + tdmr->pamt_2m_size = tdmr_get_pamt_sz(tdmr, TDX_PS_2M); + + if (tdx_supports_dynamic_pamt(&tdx_sysinfo)) { + / With Dynamic PAMT, PAMT_4K is replaced with a bitmap / + tdmr->pamt_4k_size = tdmr_get_pamt_bitmap_sz(tdmr); + } else { + tdmr->pamt_4k_size = tdmr_get_pamt_sz(tdmr, TDX_PS_4K); } + tdmr_pamt_size = tdmr->pamt_4k_size + tdmr->pamt_2m_size + tdmr->pamt_1g_size; + / * Allocate one chunk of physically contiguous memory for all * PAMTs. This helps minimize the PAMT's use of reserved areas @@ -606,25 +656,17 @@ static __init int tdmr_set_up_pamt(struct tdmr_info tdmr, / pamt = alloc_contig_pages(tdmr_pamt_size >> PAGE_SHIFT, GFP_KERNEL, nid, &node_online_map); - if (!pamt) - return -ENOMEM; /* - * Break the contiguous allocation back up into the - * individual PAMTs for each page size. + * tdmr->pamt_4k_base is still zero so the error + * path of the caller will skip freeing the PAMT. / - tdmr_pamt_base = page_to_pfn(pamt) << PAGE_SHIFT; - for (pgsz = TDX_PS_4K; pgsz < TDX_PS_NR; pgsz++) { - pamt_base[pgsz] = tdmr_pamt_base; - tdmr_pamt_base += pamt_size[pgsz]; - } + if (!pamt) + return -ENOMEM; - tdmr->pamt_4k_base = pamt_base[TDX_PS_4K]; - tdmr->pamt_4k_size = pamt_size[TDX_PS_4K]; - tdmr->pamt_2m_base = pamt_base[TDX_PS_2M]; - tdmr->pamt_2m_size = pamt_size[TDX_PS_2M]; - tdmr->pamt_1g_base = pamt_base[TDX_PS_1G]; - tdmr->pamt_1g_size = pamt_size[TDX_PS_1G]; + tdmr->pamt_4k_base = page_to_phys(pamt); + tdmr->pamt_2m_base = tdmr->pamt_4k_base + tdmr->pamt_4k_size; + tdmr->pamt_1g_base = tdmr->pamt_2m_base + tdmr->pamt_2m_size; return 0; } @@ -655,10 +697,7 @@ static __init void tdmr_do_pamt_func(struct tdmr_info tdmr, tdmr_get_pamt(tdmr, &pamt_base, &pamt_size); /* Do nothing if PAMT hasn't been allocated for this TDMR / - if (!pamt_size) - return; - - if (WARN_ON_ONCE(!pamt_base)) + if (!pamt_base) return; pamt_func(pamt_base, pamt_size); @@ -684,14 +723,12 @@ static __init void tdmrs_free_pamt_all(struct tdmr_info_list tdmr_list) /* Allocate and set up PAMTs for all TDMRs / static __init int tdmrs_set_up_pamt_all(struct tdmr_info_list tdmr_list, - struct list_head tmb_list, - u16 pamt_entry_size[]) + struct list_head tmb_list) { int i, ret = 0; for (i = 0; i < tdmr_list->nr_consumed_tdmrs; i++) { - ret = tdmr_set_up_pamt(tdmr_entry(tdmr_list, i), tmb_list, - pamt_entry_size); + ret = tdmr_set_up_pamt(tdmr_entry(tdmr_list, i), tmb_list); if (ret) goto err; } @@ -968,18 +1005,13 @@ static __init int construct_tdmrs(struct list_head tmb_list, struct tdmr_info_list tdmr_list, struct tdx_sys_info_tdmr sysinfo_tdmr) { - u16 pamt_entry_size[TDX_PS_NR] = { - sysinfo_tdmr->pamt_4k_entry_size, - sysinfo_tdmr->pamt_2m_entry_size, - sysinfo_tdmr->pamt_1g_entry_size, - }; int ret; ret = fill_out_tdmrs(tmb_list, tdmr_list); if (ret) return ret; - ret = tdmrs_set_up_pamt_all(tdmr_list, tmb_list, pamt_entry_size); + ret = tdmrs_set_up_pamt_all(tdmr_list, tmb_list); if (ret) return ret; @@ -998,6 +1030,8 @@ static __init int construct_tdmrs(struct list_head tmb_list, return ret; } +#define TDX_SYS_CONFIG_DYNAMIC_PAMT BIT(16) + static __init int config_tdx_module(struct tdmr_info_list tdmr_list, u64 global_keyid) { @@ -1026,6 +1060,12 @@ static __init int config_tdx_module(struct tdmr_info_list tdmr_list, args.rcx = __pa(tdmr_pa_array); args.rdx = tdmr_list->nr_consumed_tdmrs; args.r8 = global_keyid; + + if (tdx_supports_dynamic_pamt(&tdx_sysinfo)) { + pr_info("Enable Dynamic PAMT\n"); + args.r8 \|= TDX_SYS_CONFIG_DYNAMIC_PAMT; + } + ret = seamcall_prerr(TDH_SYS_CONFIG, &args); /* Free the array as it is not required anymore. / @@ -1167,10 +1207,14 @@ static __init int init_tdx_module(void) / get_online_mems(); - ret = build_tdx_memlist(&tdx_memlist); + ret = init_pamt_refcounts(); if (ret) goto out_put_tdxmem; + ret = build_tdx_memlist(&tdx_memlist); + if (ret) + goto err_free_pamt_refcounts; + /* Allocate enough space for constructing TDMRs / ret = alloc_tdmr_list(&tdx_tdmr_list, &tdx_sysinfo.tdmr); if (ret) @@ -1220,6 +1264,8 @@ static __init int init_tdx_module(void) free_tdmr_list(&tdx_tdmr_list); err_free_tdxmem: free_tdx_memlist(&tdx_memlist); +err_free_pamt_refcounts: + free_pamt_refcounts(); goto out_put_tdxmem; } @@ -2003,6 +2049,288 @@ u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, kvm_pfn_t pfn) } EXPORT_SYMBOL_FOR_KVM(tdh_phymem_page_wbinvd_hkid); +bool tdx_supports_dynamic_pamt(const struct tdx_sys_info sysinfo) +{ + return sysinfo->features.tdx_features0 & TDX_FEATURES0_DYNAMIC_PAMT && + tdx_enable_dpamt; +} +EXPORT_SYMBOL_FOR_KVM(tdx_supports_dynamic_pamt); + +static struct page tdx_alloc_page_pamt_cache(struct tdx_pamt_cache cache) +{ + struct page page; + + page = list_first_entry_or_null(&cache->page_list, struct page, lru); + if (page) { + list_del(&page->lru); + cache->cnt--; + } + + return page; +} + +static struct page alloc_dpamt_page(struct tdx_pamt_cache cache) +{ + if (cache) + return tdx_alloc_page_pamt_cache(cache); + + return alloc_page(GFP_KERNEL_ACCOUNT); +} + +static int alloc_pamt_array(struct page pamt_pages, struct tdx_pamt_cache cache) +{ + int i, j; + + for (i = 0; i < TDX_DPAMT_ENTRY_PAGE_CNT; i++) { + pamt_pages[i] = alloc_dpamt_page(cache); + if (!pamt_pages[i]) + goto err; + } + + return 0; + +err: + for (j = 0; j < i; j++) + __free_page(pamt_pages[j]); + + return -ENOMEM; +} + +static void free_pamt_array(struct page *pamt_pages) +{ + for (int i = 0; i < TDX_DPAMT_ENTRY_PAGE_CNT; i++) { + / + * Reset pages unconditionally to cover cases + * where they were passed to the TDX module. + / + tdx_quirk_reset_paddr(page_to_phys(pamt_pages[i]), PAGE_SIZE); + + __free_page(pamt_pages[i]); + } +} + +/ + * Calculate the arg needed for operating on the DPAMT backing for + * a given 4KB page. + / +static u64 pamt_2mb_arg(kvm_pfn_t pfn) +{ + / Arg value will specify a 2MB region of physical address space. / + unsigned long hpa_2mb = ALIGN_DOWN(pfn << PAGE_SHIFT, PMD_SIZE); + + return hpa_2mb \| TDX_PS_2M; +} + +/ Add PAMT backing for the 2MB region surrounding the given pfn. / +static u64 tdh_phymem_pamt_add(kvm_pfn_t pfn, struct page pamt_pages) +{ + struct tdx_module_args args = { + .rcx = pamt_2mb_arg(pfn), + .rdx = page_to_phys(pamt_pages[0]), + .r8 = page_to_phys(pamt_pages[1]), + }; + + return seamcall(TDH_PHYMEM_PAMT_ADD, &args); +} + +/ Remove PAMT backing for the 2MB region surrounding the given pfn. / +static u64 tdh_phymem_pamt_remove(kvm_pfn_t pfn, struct page pamt_pages) +{ + struct tdx_module_args args = { + .rcx = pamt_2mb_arg(pfn), + }; + u64 ret; + + ret = seamcall_ret(TDH_PHYMEM_PAMT_REMOVE, &args); + if (ret) + return ret; + + / Copy PAMT pages out of the struct per the TDX ABI / + pamt_pages[0] = phys_to_page(args.rdx); + pamt_pages[1] = phys_to_page(args.r8); + + return 0; +} + +/ Serializes adding/removing PAMT memory / +static DEFINE_SPINLOCK(pamt_lock); + +/ Bump PAMT refcount for the given pfn and allocate PAMT backing if needed. / +int tdx_pamt_get(kvm_pfn_t pfn, struct tdx_pamt_cache cache) +{ + struct page pamt_pages[TDX_DPAMT_ENTRY_PAGE_CNT]; + atomic_t pamt_refcount; + u64 tdx_status; + int ret; + + if (!tdx_supports_dynamic_pamt(&tdx_sysinfo)) + return 0; + + pamt_refcount = tdx_find_pamt_refcount(pfn); + + /* + * If the pamt page is already added (i.e. refcount >= 1), + * then just increment the refcount. + / + if (atomic_inc_not_zero(pamt_refcount)) + return 0; + + ret = alloc_pamt_array(pamt_pages, cache); + if (ret) + return ret; + + spin_lock(&pamt_lock); + + / + * Unlike tdx_pamt_put() which uses atomic_dec_and_lock() to + * atomically handle the 1->0 transition, the get side has no + * equivalent combined primitive for 0->1. Recheck under the + * lock since another get may have already done the 0->1 + * transition after both saw atomic_inc_not_zero() fail. + / + if (atomic_inc_not_zero(pamt_refcount)) + goto out_free; + + tdx_status = tdh_phymem_pamt_add(pfn, pamt_pages); + if (WARN_ON_ONCE(tdx_status != TDX_SUCCESS)) { + ret = -EIO; + goto out_free; + } + + / + * The refcount is zero, and this locked path is the + * only way to increase it from 0->1. + / + atomic_set(pamt_refcount, 1); + spin_unlock(&pamt_lock); + return 0; + +out_free: + spin_unlock(&pamt_lock); + free_pamt_array(pamt_pages); + + return ret; +} +EXPORT_SYMBOL_FOR_KVM(tdx_pamt_get); + +/ Drop PAMT refcount for the given pfn and free PAMT backing if needed. / +void tdx_pamt_put(kvm_pfn_t pfn) +{ + struct page pamt_pages[TDX_DPAMT_ENTRY_PAGE_CNT] = {}; + atomic_t pamt_refcount; + u64 tdx_status; + + if (!tdx_supports_dynamic_pamt(&tdx_sysinfo)) + return; + + pamt_refcount = tdx_find_pamt_refcount(pfn); + + / + * If there is more than 1 reference on the pamt page, don't + * remove it yet. Just decrement the refcount. + / + if (!atomic_dec_and_lock(pamt_refcount, &pamt_lock)) + return; + + tdx_status = tdh_phymem_pamt_remove(pfn, pamt_pages); + + / + * Don't free pamt_pages as it could hold garbage when + * tdh_phymem_pamt_remove() fails. Don't panic/BUG_ON(), as + * there is no risk of data corruption, but do yell loudly as + * failure indicates a kernel bug, memory is being leaked, and + * the dangling PAMT entry may cause future operations to fail. + / + if (WARN_ON_ONCE(tdx_status != TDX_SUCCESS)) { + / + * atomic_dec_and_lock() already decremented it to 0, + * but the PAMT entry still exists since REMOVE failed. + / + atomic_set(pamt_refcount, 1); + goto out_unlock; + } + + spin_unlock(&pamt_lock); + free_pamt_array(pamt_pages); + return; +out_unlock: + spin_unlock(&pamt_lock); +} +EXPORT_SYMBOL_FOR_KVM(tdx_pamt_put); + +void tdx_free_pamt_cache(struct tdx_pamt_cache cache) +{ + struct page page; + + while ((page = tdx_alloc_page_pamt_cache(cache))) + __free_page(page); +} +EXPORT_SYMBOL_FOR_KVM(tdx_free_pamt_cache); + +int tdx_topup_pamt_cache(struct tdx_pamt_cache cache, unsigned long npages) +{ + if (WARN_ON_ONCE(!tdx_supports_dynamic_pamt(&tdx_sysinfo))) + return 0; + + npages = TDX_DPAMT_ENTRY_PAGE_CNT; + + while (cache->cnt < npages) { + struct page page = alloc_page(GFP_KERNEL_ACCOUNT); + + if (!page) + return -ENOMEM; + + list_add(&page->lru, &cache->page_list); + cache->cnt++; + } + + return 0; +} +EXPORT_SYMBOL_FOR_KVM(tdx_topup_pamt_cache); + +/* + * Return a page that can be gifted to the TDX-Module for use as a "control" + * page, i.e. pages that are used for control structures for a given TDX + * guest, and thus obtain TDX protections, including PAMT tracking. + / +struct page tdx_alloc_control_page(void) +{ + struct page page; + + page = alloc_page(GFP_KERNEL_ACCOUNT); + if (!page) + return NULL; + + if (tdx_pamt_get(page_to_pfn(page), NULL)) { + __free_page(page); + return NULL; + } + + return page; +} +EXPORT_SYMBOL_FOR_KVM(tdx_alloc_control_page); + +/ + * Free a page that was gifted to the TDX-Module for use as a control + * page. After this, the page is no longer protected by TDX. + / +void tdx_free_control_page(struct page page) +{ + if (!page) + return; + + tdx_pamt_put(page_to_pfn(page)); + __free_page(page); +} +EXPORT_SYMBOL_FOR_KVM(tdx_free_control_page); + +static int __init tdx_dpamt_setup(char str) +{ + return kstrtobool(str, &tdx_enable_dpamt) == 0; +} + +__setup("tdx_dpamt=", tdx_dpamt_setup); + void tdx_sys_disable(void) { struct tdx_module_args args = {}; diff --git a/arch/x86/virt/vmx/tdx/tdx.h b/arch/x86/virt/vmx/tdx/tdx.h index bdfd0e1e337ac..a886c54decaad 100644 --- a/arch/x86/virt/vmx/tdx/tdx.h +++ b/arch/x86/virt/vmx/tdx/tdx.h @@ -48,6 +48,8 @@ #define TDH_SYS_CONFIG 45 #define TDH_SYS_SHUTDOWN 52 #define TDH_SYS_UPDATE 53 +#define TDH_PHYMEM_PAMT_ADD 58 +#define TDH_PHYMEM_PAMT_REMOVE 59 #define TDH_SYS_DISABLE 69 / diff --git a/arch/x86/virt/vmx/tdx/tdx_global_metadata.c b/arch/x86/virt/vmx/tdx/tdx_global_metadata.c index e49c300f23d43..93b1f26252c63 100644 --- a/arch/x86/virt/vmx/tdx/tdx_global_metadata.c +++ b/arch/x86/virt/vmx/tdx/tdx_global_metadata.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Automatically generated functions to read TDX global metadata. + * Functions to read TDX global metadata. * * This file doesn't compile on its own as it lacks of inclusion * of SEAMCALL wrapper primitive which reads global metadata. @@ -33,6 +33,18 @@ static __init int get_tdx_sys_info_features(struct tdx_sys_info_features sysinf return ret; } +static __init int get_tdx_sys_info_tdmr_dpamt(struct tdx_sys_info_tdmr sysinfo_tdmr) +{ + int ret; + u64 val; + + ret = read_sys_metadata_field(0x9100000000000013, &val); + if (!ret) + sysinfo_tdmr->pamt_page_bitmap_entry_bits = val; + + return ret; +} + static __init int get_tdx_sys_info_tdmr(struct tdx_sys_info_tdmr sysinfo_tdmr) { int ret = 0; @@ -129,5 +141,14 @@ static __init int get_tdx_sys_info(struct tdx_sys_info sysinfo) ret = ret ?: get_tdx_sys_info_td_ctrl(&sysinfo->td_ctrl); ret = ret ?: get_tdx_sys_info_td_conf(&sysinfo->td_conf); + /* + * The kernel supports using TDX without Dynamic PAMT, so + * avoid reporting failure if it's not supported. Don't try + * to support buggy TDX modules that advertise Dynamic PAMT + * but don't expose the metadata. + */ + if (!ret && tdx_supports_dynamic_pamt(sysinfo)) + ret = get_tdx_sys_info_tdmr_dpamt(&sysinfo->tdmr); + return ret; }
3/2	2026/08/06 03:01	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 17223 output: 20 thoughts: 620 Thoughts: Analyzing the Reachability of Dynamic PAMT TDX Patch Okay, so I'm looking at this patch adding "Dynamic PAMT" support within the TDX module for x86. It's touching several core files in the KVM and TDX infrastructure: `tdx.c` files, the MMU, etc. This is activated with the `tdx_dpamt=on` kernel parameter, and it introduces new functions like `tdx_pamt_get`, `tdx_pamt_put`, all that good stuff, plus it utilizes `TDH_PHYMEM_PAMT_ADD` and `TDH_PHYMEM_PAMT_REMOVE` SEAMCALLs. The initial question is whether this code is even reachable in a virtualized environment like GCE or QEMU. Immediately, the TDX requirement pops to mind - specific Intel TDX hardware. It's a virtualization feature, but the core of this patch seems focused on the host side, the KVM support, not the guest itself, at least that's my understanding. My immediate thought is, "Can we really test this in QEMU?". Wait, a further thought strikes: Isn't TDX guest support separate from this KVM host-side work? I need to be precise, this patch is targeting TDX host-side KVM functionality. Now I'm questioning whether TDX host support in KVM is even fully upstreamed, or if it is testable with standard QEMU. Realistically, TDX host support typically demands bare metal with TDX-capable hardware. Nested virtualization, even if it could be nested (which it can't right now), is a no-go. The prompt itself is very explicit about unreachable code paths in virtualized environments like GCE/QEMU when emulating hardware, it's referring to the TDX host code. So, the question becomes, can a fuzzer like syzkaller reach this TDX host code in something like GCE or QEMU? And the answer is likely a firm no. This TDX host functionality relies on the underlying Intel CPU with TDX support. QEMU, in its standard configuration, doesn't emulate that hardware, and standard GCE VMs don't typically provide it without specific hardware configurations, which is unlikely to be available for general use. QEMU can be a VMM that runs TDX guests, using KVM, but the KVM part (the host kernel) still needs to be on TDX-capable hardware, and if I try to run a kernel in QEMU, KVM will check for the TDX hardware. If the hardware isn't present, the initialization will fail. I should verify that, I'm going to look at `init_tdx_module` in `arch/x86/virt/vmx/tdx/tdx.c` and search for the key identifiers like `tdx_supports_dynamic_pamt` and `init_tdx_module` to confirm.
4/2	2026/08/06 03:01	tool	grepper	0m Args: { "Expression": "init_tdx_module" } Results: { "Output": "arch/x86/virt/vmx/tdx/tdx.c=1166=static __init int init_tdmrs(struct tdmr_info_list *tdmr_list)\n--\narch/x86/virt/vmx/tdx/tdx.c-1184-\narch/x86/virt/vmx/tdx/tdx.c:1185:static __init int init_tdx_module(void)\narch/x86/virt/vmx/tdx/tdx.c-1186-{\n--\narch/x86/virt/vmx/tdx/tdx.c=1272=static __init int tdx_enable(void)\n--\narch/x86/virt/vmx/tdx/tdx.c-1301-\narch/x86/virt/vmx/tdx/tdx.c:1302:\tret = init_tdx_module();\narch/x86/virt/vmx/tdx/tdx.c-1303-\tif (ret) {\n" }
5/2	2026/08/06 03:01	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 18799 output: 18 thoughts: 23 Thoughts: Okay, here's my interpretation of those thoughts, crafted with the expectation that I'm discussing this with a fellow expert: Investigating the Activation of `tdx_enable` Alright, I need to get to the bottom of this. My immediate question is: when does `tdx_enable` actually get called? Is it baked into the early boot sequence, or is it exclusively triggered by the KVM loading process? Understanding this has some major implications for how we architect the rest of the system, and what dependencies we need to consider. I suspect there may be some implications for memory initialization as well. I need to figure out exactly when this key function hits.
6/2	2026/08/06 03:01	tool	grepper	0m Args: { "Expression": "tdx_enable" } Results: { "Output": "arch/x86/virt/vmx/tdx/tdx.c-49-\narch/x86/virt/vmx/tdx/tdx.c:50:static bool tdx_enable_dpamt __ro_after_init;\narch/x86/virt/vmx/tdx/tdx.c-51-\n--\narch/x86/virt/vmx/tdx/tdx.c=1185=static __init int init_tdx_module(void)\n--\narch/x86/virt/vmx/tdx/tdx.c-1271-\narch/x86/virt/vmx/tdx/tdx.c:1272:static __init int tdx_enable(void)\narch/x86/virt/vmx/tdx/tdx.c-1273-{\n--\narch/x86/virt/vmx/tdx/tdx.c-1314-}\narch/x86/virt/vmx/tdx/tdx.c:1315:subsys_initcall(tdx_enable);\narch/x86/virt/vmx/tdx/tdx.c-1316-\n--\narch/x86/virt/vmx/tdx/tdx.c=2052=bool tdx_supports_dynamic_pamt(const struct tdx_sys_info sysinfo)\n--\narch/x86/virt/vmx/tdx/tdx.c-2054-\treturn sysinfo-\u003efeatures.tdx_features0 \u0026 TDX_FEATURES0_DYNAMIC_PAMT \u0026\u0026\narch/x86/virt/vmx/tdx/tdx.c:2055:\t tdx_enable_dpamt;\narch/x86/virt/vmx/tdx/tdx.c-2056-}\n--\narch/x86/virt/vmx/tdx/tdx.c=2327=static int __init tdx_dpamt_setup(char str)\narch/x86/virt/vmx/tdx/tdx.c-2328-{\narch/x86/virt/vmx/tdx/tdx.c:2329:\treturn kstrtobool(str, \u0026tdx_enable_dpamt) == 0;\narch/x86/virt/vmx/tdx/tdx.c-2330-}\n" }
7/2	2026/08/06 03:01	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 19595 output: 47 thoughts: 24 Thoughts: Here's my take on those thoughts, tailored for an expert audience: Diving into the `tdx_enable` Initialization Okay, so `tdx_enable` is getting hooked up as a `subsys_initcall`. That immediately piques my interest. It means this is a low-level, early-stage component, likely crucial for setting up the Trusted Domain Extensions (TDX) environment within the system, presuming this is a TDX-aware kernel. My gut tells me this is going to be the entry point for configuring hardware-level TDX functionality. I need to understand its exact role in the initialization sequence. I'll need to dissect the function's internals to see how it's interacting with the CPU, memory controllers, and any relevant platform-specific registers or ACPI tables. Is it attempting to discover and validate TDX support, probing for the presence of the required hardware features like the TDX Module (TDX-M)? Is it enabling specific CPU flags or MSRs? Perhaps it's creating or registering any kernel data structures required for TDX operation later on. I'll be looking for interaction with the TDX-related firmware components, the Intel TXT or the AMD SEV/SEV-SNP, assuming that is in effect. Let's dig in and see what's actually going on inside `tdx_enable`.
8/2	2026/08/06 03:01	tool	read-file	0m Args: { "File": "arch/x86/virt/vmx/tdx/tdx.c", "FirstLine": 1270, "LineCount": 50 } Results: { "Contents": "1270:\t}\n1271:\t\n1272:\tstatic __init int tdx_enable(void)\n1273:\t{\n1274:\t\tenum cpuhp_state state;\n1275:\t\tint ret;\n1276:\t\n1277:\t\tif (!cpu_feature_enabled(X86_FEATURE_TDX_HOST_PLATFORM)) {\n1278:\t\t\tpr_err(\"TDX not supported by the host platform\\n\");\n1279:\t\t\treturn -ENODEV;\n1280:\t\t}\n1281:\t\n1282:\t\tif (!cpu_feature_enabled(X86_FEATURE_XSAVE)) {\n1283:\t\t\tpr_err(\"XSAVE is required for TDX\\n\");\n1284:\t\t\treturn -EINVAL;\n1285:\t\t}\n1286:\t\n1287:\t\tif (!cpu_feature_enabled(X86_FEATURE_MOVDIR64B)) {\n1288:\t\t\tpr_err(\"MOVDIR64B is required for TDX\\n\");\n1289:\t\t\treturn -EINVAL;\n1290:\t\t}\n1291:\t\n1292:\t\tif (!cpu_feature_enabled(X86_FEATURE_SELFSNOOP)) {\n1293:\t\t\tpr_err(\"Self-snoop is required for TDX\\n\");\n1294:\t\t\treturn -ENODEV;\n1295:\t\t}\n1296:\t\n1297:\t\tstate = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, \"virt/tdx:online\",\n1298:\t\t\t\t\t tdx_online_cpu, tdx_offline_cpu);\n1299:\t\tif (state \u003c 0)\n1300:\t\t\treturn state;\n1301:\t\n1302:\t\tret = init_tdx_module();\n1303:\t\tif (ret) {\n1304:\t\t\tpr_err(\"TDX-Module initialization failed (%d)\\n\", ret);\n1305:\t\t\tcpuhp_remove_state(state);\n1306:\t\t\treturn ret;\n1307:\t\t}\n1308:\t\n1309:\t\tregister_syscore(\u0026tdx_syscore);\n1310:\t\n1311:\t\ttdx_module_state.initialized = true;\n1312:\t\tpr_info(\"TDX-Module initialized\\n\");\n1313:\t\treturn 0;\n1314:\t}\n1315:\tsubsys_initcall(tdx_enable);\n1316:\t\n1317:\tint tdx_module_shutdown(void)\n1318:\t{\n1319:\t\tstruct tdx_sys_info_handoff handoff = {};\n" }
9/2	2026/08/06 03:01	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 20687 output: 24 thoughts: 578 Thoughts: Analyzing TDX Host Code Reachability for Fuzzing Okay, so `tdx_enable` checks for `X86_FEATURE_TDX_HOST_PLATFORM` and returns `-ENODEV` if it's not enabled. Naturally, my first thought is: is this platform feature emulated by QEMU/KVM? No, standard QEMU doesn't emulate the TDX host platform. It requires actual Intel hardware with TDX enabled in the BIOS, and the SEAM module loaded, and so on. Now I'm thinking, can syzkaller even run on TDX hardware? The prompt is hinting at: "Code paths that are impossible to reach in virtualized environments like GCE or QEMU, even when utilizing software-emulated hardware". Since TDX host code is impossible to reach in standard QEMU, and there is no SEAM module emulation, I need to double-check if any software emulation exists for the TDX host. I am fairly confident in my knowledge here, but a quick confirmation is warranted. No, it's not emulated. TDX is a hardware feature, and `SEAMCALLs` are handled by the TDX module, which is a secure enclave. QEMU does not emulate the TDX module for nested virtualization, or at least not in the standard upstream version. So, this confirms that TDX host code is unreachable in standard virtualized environments. This brings me back to the prompt, which states to "Return WorthFuzzing=false if the patch only contains: ... Code paths that are impossible to reach in virtualized environments like GCE or QEMU, even when utilizing software-emulated hardware". TDX host code fits that description perfectly. But wait, I need to be thorough. Is there any part of this patch that is reachable without TDX hardware? Let me check the actual code changes. Okay, the patch modifies `arch/x86/kvm/mmu/mmu.c`. The change is adding a call to `kvm_x86_call(topup_external_cache)`. That looks like an optional call. I need to understand its behavior. Now I need to check the definition of `kvm_x86_call`. This function could be key.
10/2	2026/08/06 03:01	tool	grepper	0m Args: { "Expression": "#define kvm_x86_call" } Results: { "Output": "arch/x86/include/asm/kvm_host.h=2063=extern struct kvm_x86_ops kvm_x86_ops;\narch/x86/include/asm/kvm_host.h-2064-\narch/x86/include/asm/kvm_host.h:2065:#define kvm_x86_call(func) static_call(kvm_x86_##func)\narch/x86/include/asm/kvm_host.h-2066-\n" }
11/2	2026/08/06 03:01	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 21930 output: 102 thoughts: 519 Thoughts: TDX Host Code Unreachability Assessment Okay, let me break this down. My initial observation is that `topup_external_cache` is defined using `KVM_X86_OP_OPTIONAL_RET0`. That means if the underlying virtualization technology (like VMX or SVM) doesn't implement it, it simply returns zero. Crucially, this function is only assigned a non-zero value within `tdx_hardware_setup`. And `tdx_hardware_setup` is, and this is the critical point, only called if TDX support is detected. So, in the absence of a TDX-enabled environment, which is the norm for standard virtualized setups, `topup_external_cache` will always return zero. The rest of the patch, as I've seen, is confined to `arch/x86/kvm/vmx/tdx.c` and `arch/x86/virt/vmx/tdx/tdx.c`. Consequently, this TDX-specific code simply won't be executed in a standard QEMU or GCE environment. It's totally unreachable for Syzkaller. Now, let's just make sure there are no other potential escape routes. The `tdx_dpamt_setup` function is an `__setup` function, which simply flips a boolean, it's not relevant. `tdx_supports_dynamic_pamt` is exported, but is only used within the KVM TDX infrastructure. Similarly, functions like `tdx_pamt_get`, `tdx_pamt_put`, etc., are exported, but they're all firmly within the KVM TDX domain. And that `mmu_topup_memory_caches` call to `topup_external_cache` is a static call that effectively becomes a no-op that's designed to return 0. Therefore, without TDX hardware, there are no functional changes that can be triggered. So, the reasoning I'll formulate is this: "The patch introduces Dynamic PAMT support within the TDX host (KVM) code. Functionality dependent on this support requires specific bare-metal Intel hardware with Trust Domain Extensions enabled and the TDX module loaded. Since this hardware isn't emulated in standard virtualized environments like QEMU or GCE, the modified code paths are completely unreachable during fuzzing."
12/2	2026/08/06 03:01	tool	set-results	0m Args: { "EnableConfigs": [], "FocusSymbols": [], "Reasoning": "The patch adds support for Dynamic PAMT in the TDX host (KVM) code. TDX host support requires specific bare-metal Intel hardware with Trust Domain Extensions enabled and the TDX module loaded. This hardware is not emulated in standard virtualized environments like QEMU or GCE, making the modified code paths unreachable during fuzzing.", "WorthFuzzing": false } Results: { "EnableConfigs": null, "FocusSymbols": [], "Reasoning": "The patch adds support for Dynamic PAMT in the TDX host (KVM) code. TDX host support requires specific bare-metal Intel hardware with Trust Domain Extensions enabled and the TDX module loaded. This hardware is not emulated in standard virtualized environments like QEMU or GCE, making the modified code paths unreachable during fuzzing.", "WorthFuzzing": false }

{
  "PatchDiff": "commit f00efc49131881f16a5b1580adf3f7c794263146\nAuthor: syz-cluster \u003ctriage@syzkaller.com\u003e\nDate:   Thu Aug 6 03:00:59 2026 +0000\n\n    syz-cluster: applied patch under review\n\ndiff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt\nindex b5493a7f8f228..a0a7670246bc5 100644\n--- a/Documentation/admin-guide/kernel-parameters.txt\n+++ b/Documentation/admin-guide/kernel-parameters.txt\n@@ -7512,6 +7512,16 @@ Kernel parameters\n \n \ttdfx=\t\t[HW,DRM]\n \n+\ttdx_dpamt=\n+\t\t\t[X86] Controls whether TDX will use Dynamic PAMT\n+\t\t\tto save memory, when supported.\n+\n+\t\t\tValid parameters: \"on\", \"off\"\n+\t\t\tDefault: \"off\"\n+\n+\t\t\tFor details see:\n+\t\t\tDocumentation/arch/x86/tdx.rst\n+\n \ttest_suspend=\t[SUSPEND]\n \t\t\tFormat: { \"mem\" | \"standby\" | \"freeze\" }[,N]\n \t\t\tSpecify \"mem\" (for Suspend-to-RAM) or \"standby\" (for\ndiff --git a/Documentation/arch/x86/tdx.rst b/Documentation/arch/x86/tdx.rst\nindex 3303499ad4c6f..a1c2309230580 100644\n--- a/Documentation/arch/x86/tdx.rst\n+++ b/Documentation/arch/x86/tdx.rst\n@@ -200,6 +200,34 @@ reflects the TCB of the currently running TDX module and therefore\n changes after an update. By contrast, TEE_TCB_SVN reflects the TCB at TD\n launch time and is not affected.\n \n+Dynamic PAMT\n+------------\n+\n+Physical Address Metadata Table (PAMT) is memory that the TDX module needs\n+to keep data about each page (think like struct page). It needs to be handed\n+to the TDX module for its exclusive use. For normal PAMT, this is installed\n+when the TDX module is first loaded and comes to about 0.4% of system memory.\n+\n+Dynamic PAMT is a TDX module feature that allows VMM to allocate part of the\n+PAMT as needed (the parts for tracking 4KB size pages). The other page sizes\n+(1GB and 2MB) are still allocated statically at the time of TDX module\n+initialization. This reduces the amount of memory that TDX uses while TDs are\n+not in use.\n+\n+When Dynamic PAMT is in use, dmesg shows it like::\n+\n+  [..] virt/tdx: Enable Dynamic PAMT\n+  [..] virt/tdx: 10092 KB allocated for PAMT\n+  [..] virt/tdx: TDX-Module initialized\n+\n+Dynamic PAMT is only enabled when supported and the ``tdx_dpamt=`` kernel\n+parameter is set to \"on\". The feature is off by default because TDX module\n+internal details prevent Dynamic PAMT from working on all keyid partitioning\n+configurations. When the TDX module is fixed to include these constraints in\n+its enumeration of Dynamic PAMT support, kernel support can be changed to\n+default on. For more information, consult the Intel TDX documentation about\n+Dynamic PAMT.\n+\n TDX Interaction to Other Kernel Components\n ------------------------------------------\n \ndiff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h\nindex 83dc5086138b3..588563dfe88d5 100644\n--- a/arch/x86/include/asm/kvm-x86-ops.h\n+++ b/arch/x86/include/asm/kvm-x86-ops.h\n@@ -98,6 +98,7 @@ KVM_X86_OP_OPTIONAL_RET0(tdp_has_smep)\n KVM_X86_OP(load_mmu_pgd)\n KVM_X86_OP_OPTIONAL_RET0(set_external_spte)\n KVM_X86_OP_OPTIONAL(free_external_spt)\n+KVM_X86_OP_OPTIONAL_RET0(topup_external_cache)\n KVM_X86_OP(has_wbinvd_exit)\n KVM_X86_OP(get_l2_tsc_offset)\n KVM_X86_OP(get_l2_tsc_multiplier)\ndiff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h\nindex 5f6c1ce9673b7..0dbf6564dd0c5 100644\n--- a/arch/x86/include/asm/kvm_host.h\n+++ b/arch/x86/include/asm/kvm_host.h\n@@ -1922,6 +1922,7 @@ struct kvm_x86_ops {\n \t/* Update external page tables for page table about to be freed. */\n \tvoid (*free_external_spt)(struct kvm *kvm, struct kvm_mmu_page *sp);\n \n+\tint (*topup_external_cache)(struct kvm_vcpu *vcpu, int min_nr_spts);\n \n \tbool (*has_wbinvd_exit)(void);\n \ndiff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h\nindex 89e97d5761d89..7910901a7ba21 100644\n--- a/arch/x86/include/asm/tdx.h\n+++ b/arch/x86/include/asm/tdx.h\n@@ -36,6 +36,7 @@\n /* Bit definitions of TDX_FEATURES0 metadata field */\n #define TDX_FEATURES0_TD_PRESERVING\tBIT_ULL(1)\n #define TDX_FEATURES0_NO_RBP_MOD\tBIT_ULL(18)\n+#define TDX_FEATURES0_DYNAMIC_PAMT\tBIT_ULL(36)\n \n #ifndef __ASSEMBLER__\n \n@@ -118,12 +119,37 @@ static inline bool tdx_supports_runtime_update(const struct tdx_sys_info *sysinf\n \treturn sysinfo-\u003efeatures.tdx_features0 \u0026 TDX_FEATURES0_TD_PRESERVING;\n }\n \n+bool tdx_supports_dynamic_pamt(const struct tdx_sys_info *sysinfo);\n+\n+/* Simple structure for pre-allocating Dynamic PAMT pages outside of spinlocks. */\n+struct tdx_pamt_cache {\n+\tstruct list_head page_list;\n+\tint cnt;\n+};\n+\n+static inline void tdx_init_pamt_cache(struct tdx_pamt_cache *cache)\n+{\n+\tINIT_LIST_HEAD(\u0026cache-\u003epage_list);\n+\tcache-\u003ecnt = 0;\n+}\n+\n+void tdx_free_pamt_cache(struct tdx_pamt_cache *cache);\n+int tdx_topup_pamt_cache(struct tdx_pamt_cache *cache, unsigned long npages);\n+int tdx_pamt_get(kvm_pfn_t pfn, struct tdx_pamt_cache *cache);\n+void tdx_pamt_put(kvm_pfn_t pfn);\n+\n int tdx_guest_keyid_alloc(void);\n u32 tdx_get_nr_guest_keyids(void);\n void tdx_guest_keyid_free(unsigned int keyid);\n \n void tdx_quirk_reset_paddr(unsigned long base, unsigned long size);\n \n+/* Number of PAMT pages to be provided to TDX module per 2MB region of PA */\n+#define TDX_DPAMT_ENTRY_PAGE_CNT 2\n+\n+struct page *tdx_alloc_control_page(void);\n+void tdx_free_control_page(struct page *page);\n+\n struct tdx_td {\n \t/* TD root structure: */\n \tstruct page *tdr_page;\ndiff --git a/arch/x86/include/asm/tdx_global_metadata.h b/arch/x86/include/asm/tdx_global_metadata.h\nindex 41150d546589c..2a42551fc33cd 100644\n--- a/arch/x86/include/asm/tdx_global_metadata.h\n+++ b/arch/x86/include/asm/tdx_global_metadata.h\n@@ -21,6 +21,9 @@ struct tdx_sys_info_tdmr {\n \tu16 pamt_4k_entry_size;\n \tu16 pamt_2m_entry_size;\n \tu16 pamt_1g_entry_size;\n+\n+\t/* Optional metadata, if Dynamic PAMT is supported */\n+\tu8  pamt_page_bitmap_entry_bits;\n };\n \n struct tdx_sys_info_td_ctrl {\ndiff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c\nindex 66e69d2a41b3c..2a628b8304409 100644\n--- a/arch/x86/kvm/mmu/mmu.c\n+++ b/arch/x86/kvm/mmu/mmu.c\n@@ -614,6 +614,10 @@ static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu, bool maybe_indirect)\n \t\t\t\t\t       PT64_ROOT_MAX_LEVEL);\n \t\tif (r)\n \t\t\treturn r;\n+\n+\t\tr = kvm_x86_call(topup_external_cache)(vcpu, PT64_ROOT_MAX_LEVEL);\n+\t\tif (r)\n+\t\t\treturn r;\n \t}\n \tr = kvm_mmu_topup_memory_cache(\u0026vcpu-\u003earch.mmu_shadow_page_cache,\n \t\t\t\t       PT64_ROOT_MAX_LEVEL);\ndiff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c\nindex 545b03d9d10b8..09573112540e9 100644\n--- a/arch/x86/kvm/vmx/tdx.c\n+++ b/arch/x86/kvm/vmx/tdx.c\n@@ -362,7 +362,7 @@ static void tdx_reclaim_control_page(struct page *ctrl_page)\n \tif (tdx_reclaim_page(ctrl_page))\n \t\treturn;\n \n-\t__free_page(ctrl_page);\n+\ttdx_free_control_page(ctrl_page);\n }\n \n struct tdx_flush_vp_arg {\n@@ -589,7 +589,7 @@ static void tdx_reclaim_td_control_pages(struct kvm *kvm)\n \n \ttdx_quirk_reset_paddr(page_to_phys(kvm_tdx-\u003etd.tdr_page), PAGE_SIZE);\n \n-\t__free_page(kvm_tdx-\u003etd.tdr_page);\n+\ttdx_free_control_page(kvm_tdx-\u003etd.tdr_page);\n \tkvm_tdx-\u003etd.tdr_page = NULL;\n }\n \n@@ -681,6 +681,8 @@ int tdx_vcpu_create(struct kvm_vcpu *vcpu)\n \tif (!irqchip_split(vcpu-\u003ekvm))\n \t\treturn -EINVAL;\n \n+\ttdx_init_pamt_cache(\u0026tdx-\u003epamt_cache);\n+\n \tfpstate_set_confidential(\u0026vcpu-\u003earch.guest_fpu);\n \tvcpu-\u003earch.apic-\u003eguest_apic_protected = true;\n \tINIT_LIST_HEAD(\u0026tdx-\u003evt.pi_wakeup_list);\n@@ -866,6 +868,8 @@ void tdx_vcpu_free(struct kvm_vcpu *vcpu)\n \tstruct vcpu_tdx *tdx = to_tdx(vcpu);\n \tint i;\n \n+\ttdx_free_pamt_cache(\u0026tdx-\u003epamt_cache);\n+\n \tif (vcpu-\u003ecpu != -1) {\n \t\tKVM_BUG_ON(tdx-\u003estate == VCPU_TD_STATE_INITIALIZED, vcpu-\u003ekvm);\n \t\ttdx_flush_vp_on_cpu(vcpu);\n@@ -1621,6 +1625,17 @@ void tdx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, int pgd_level)\n \ttd_vmcs_write64(to_tdx(vcpu), SHARED_EPT_POINTER, root_hpa);\n }\n \n+static int tdx_topup_external_pamt_cache(struct kvm_vcpu *vcpu, int min_nr_spts)\n+{\n+\t/*\n+\t * Minus one page to exclude the root SPT, but plus one page for a\n+\t * possible 4KB private mapping.\n+\t */\n+\tmin_nr_spts += -1 + 1;\n+\n+\treturn tdx_topup_pamt_cache(\u0026to_tdx(vcpu)-\u003epamt_cache, min_nr_spts);\n+}\n+\n static int tdx_mem_page_add(struct kvm *kvm, gfn_t gfn, enum pg_level level,\n \t\t\t    kvm_pfn_t pfn)\n {\n@@ -1679,16 +1694,28 @@ static struct page *tdx_spte_to_sept_pt(struct kvm *kvm, gfn_t gfn,\n static int tdx_sept_map_nonleaf_spte(struct kvm *kvm, gfn_t gfn,\n \t\t\t\t     enum pg_level level, u64 new_spte)\n {\n+\tstruct kvm_vcpu *vcpu = kvm_get_running_vcpu();\n \tgpa_t gpa = gfn_to_gpa(gfn);\n \tu64 err, entry, level_state;\n \tstruct page *sept_pt;\n+\tint ret;\n+\n+\tif (KVM_BUG_ON(!vcpu, kvm))\n+\t\treturn -EIO;\n \n \tsept_pt = tdx_spte_to_sept_pt(kvm, gfn, new_spte, level);\n \tif (!sept_pt)\n \t\treturn -EIO;\n \n+\tret = tdx_pamt_get(page_to_pfn(sept_pt), \u0026to_tdx(vcpu)-\u003epamt_cache);\n+\tif (KVM_BUG_ON(ret, kvm))\n+\t\treturn ret;\n+\n \terr = tdh_mem_sept_add(\u0026to_kvm_tdx(kvm)-\u003etd, gpa, level, sept_pt,\n \t\t\t       \u0026entry, \u0026level_state);\n+\tif (err)\n+\t\ttdx_pamt_put(page_to_pfn(sept_pt));\n+\n \tif (unlikely(tdx_operand_busy(err)))\n \t\treturn -EBUSY;\n \n@@ -1701,8 +1728,13 @@ static int tdx_sept_map_nonleaf_spte(struct kvm *kvm, gfn_t gfn,\n static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level level,\n \t\t\t\t  u64 new_spte)\n {\n+\tstruct kvm_vcpu *vcpu = kvm_get_running_vcpu();\n \tstruct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);\n \tkvm_pfn_t pfn = spte_to_pfn(new_spte);\n+\tint ret;\n+\n+\tif (KVM_BUG_ON(!vcpu, kvm))\n+\t\treturn -EIO;\n \n \t/* TODO: handle large pages. */\n \tif (KVM_BUG_ON(level != PG_LEVEL_4K, kvm))\n@@ -1710,6 +1742,10 @@ static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level leve\n \n \tWARN_ON_ONCE((new_spte \u0026 VMX_EPT_RWX_MASK) != VMX_EPT_RWX_MASK);\n \n+\tret = tdx_pamt_get(pfn, \u0026to_tdx(vcpu)-\u003epamt_cache);\n+\tif (KVM_BUG_ON(ret, kvm))\n+\t\treturn ret;\n+\n \t/*\n \t * Ensure pre_fault_allowed is read by kvm_arch_vcpu_pre_fault_memory()\n \t * before kvm_tdx-\u003estate.  Userspace must not be allowed to pre-fault\n@@ -1722,10 +1758,15 @@ static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level leve\n \t * If the TD isn't finalized/runnable, then userspace is initializing\n \t * the VM image via KVM_TDX_INIT_MEM_REGION; ADD the page to the TD.\n \t */\n-\tif (unlikely(kvm_tdx-\u003estate != TD_STATE_RUNNABLE))\n-\t\treturn tdx_mem_page_add(kvm, gfn, level, pfn);\n+\tif (likely(kvm_tdx-\u003estate == TD_STATE_RUNNABLE))\n+\t\tret = tdx_mem_page_aug(kvm, gfn, level, pfn);\n+\telse\n+\t\tret = tdx_mem_page_add(kvm, gfn, level, pfn);\n+\n+\tif (ret)\n+\t\ttdx_pamt_put(pfn);\n \n-\treturn tdx_mem_page_aug(kvm, gfn, level, pfn);\n+\treturn ret;\n }\n \n /*\n@@ -1822,6 +1863,7 @@ static int tdx_sept_remove_leaf_spte(struct kvm *kvm, gfn_t gfn,\n \t\treturn -EIO;\n \n \ttdx_quirk_reset_paddr(PFN_PHYS(pfn), PAGE_SIZE);\n+\ttdx_pamt_put(pfn);\n \treturn 0;\n }\n \n@@ -1865,6 +1907,8 @@ static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn, u64 old_spte,\n  */\n static void tdx_sept_free_private_spt(struct kvm *kvm, struct kvm_mmu_page *sp)\n {\n+\tstruct page *sept_pt = virt_to_page(sp-\u003eexternal_spt);\n+\n \t/*\n \t * KVM doesn't (yet) zap page table pages in mirror page table while\n \t * TD is active, though guest pages mapped in mirror page table could be\n@@ -1878,15 +1922,15 @@ static void tdx_sept_free_private_spt(struct kvm *kvm, struct kvm_mmu_page *sp)\n \t * the page to prevent the kernel from accessing the encrypted page.\n \t */\n \tif (KVM_BUG_ON(is_hkid_assigned(to_kvm_tdx(kvm)), kvm) ||\n-\t    tdx_reclaim_page(virt_to_page(sp-\u003eexternal_spt)))\n+\t    tdx_reclaim_page(sept_pt))\n \t\tgoto out;\n \n \t/*\n-\t * Immediately free the S-EPT page because RCU-time free is unnecessary\n-\t * after TDH.PHYMEM.PAGE.RECLAIM ensures there are no outstanding\n-\t * readers.\n+\t * Immediately free the S-EPT page as the TDX subsystem doesn't support\n+\t * freeing pages from RCU callbacks, and more importantly because\n+\t * TDH.PHYMEM.PAGE.RECLAIM ensures there are no outstanding readers.\n \t */\n-\tfree_page((unsigned long)sp-\u003eexternal_spt);\n+\ttdx_free_control_page(sept_pt);\n out:\n \tsp-\u003eexternal_spt = NULL;\n }\n@@ -2459,7 +2503,7 @@ static int __tdx_td_init(struct kvm *kvm, struct td_params *td_params,\n \n \tret = -ENOMEM;\n \n-\ttdr_page = alloc_page(GFP_KERNEL_ACCOUNT);\n+\ttdr_page = tdx_alloc_control_page();\n \tif (!tdr_page)\n \t\tgoto free_hkid;\n \n@@ -2472,7 +2516,7 @@ static int __tdx_td_init(struct kvm *kvm, struct td_params *td_params,\n \t\tgoto free_tdr;\n \n \tfor (i = 0; i \u003c kvm_tdx-\u003etd.tdcs_nr_pages; i++) {\n-\t\ttdcs_pages[i] = alloc_page(GFP_KERNEL_ACCOUNT);\n+\t\ttdcs_pages[i] = tdx_alloc_control_page();\n \t\tif (!tdcs_pages[i])\n \t\t\tgoto free_tdcs;\n \t}\n@@ -2590,10 +2634,8 @@ static int __tdx_td_init(struct kvm *kvm, struct td_params *td_params,\n teardown:\n \t/* Only free pages not yet added, so start at 'i' */\n \tfor (; i \u003c kvm_tdx-\u003etd.tdcs_nr_pages; i++) {\n-\t\tif (tdcs_pages[i]) {\n-\t\t\t__free_page(tdcs_pages[i]);\n-\t\t\ttdcs_pages[i] = NULL;\n-\t\t}\n+\t\ttdx_free_control_page(tdcs_pages[i]);\n+\t\ttdcs_pages[i] = NULL;\n \t}\n \tif (!kvm_tdx-\u003etd.tdcs_pages)\n \t\tkfree(tdcs_pages);\n@@ -2608,16 +2650,13 @@ static int __tdx_td_init(struct kvm *kvm, struct td_params *td_params,\n \tfree_cpumask_var(packages);\n \n free_tdcs:\n-\tfor (i = 0; i \u003c kvm_tdx-\u003etd.tdcs_nr_pages; i++) {\n-\t\tif (tdcs_pages[i])\n-\t\t\t__free_page(tdcs_pages[i]);\n-\t}\n+\tfor (i = 0; i \u003c kvm_tdx-\u003etd.tdcs_nr_pages; i++)\n+\t\ttdx_free_control_page(tdcs_pages[i]);\n \tkfree(tdcs_pages);\n \tkvm_tdx-\u003etd.tdcs_pages = NULL;\n \n free_tdr:\n-\tif (tdr_page)\n-\t\t__free_page(tdr_page);\n+\ttdx_free_control_page(tdr_page);\n \tkvm_tdx-\u003etd.tdr_page = NULL;\n \n free_hkid:\n@@ -2951,7 +2990,7 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx)\n \tint ret, i;\n \tu64 err;\n \n-\tpage = alloc_page(GFP_KERNEL_ACCOUNT);\n+\tpage = tdx_alloc_control_page();\n \tif (!page)\n \t\treturn -ENOMEM;\n \ttdx-\u003evp.tdvpr_page = page;\n@@ -2971,7 +3010,7 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx)\n \t}\n \n \tfor (i = 0; i \u003c kvm_tdx-\u003etd.tdcx_nr_pages; i++) {\n-\t\tpage = alloc_page(GFP_KERNEL_ACCOUNT);\n+\t\tpage = tdx_alloc_control_page();\n \t\tif (!page) {\n \t\t\tret = -ENOMEM;\n \t\t\tgoto free_tdcx;\n@@ -2993,7 +3032,7 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx)\n \t\t\t * method, but the rest are freed here.\n \t\t\t */\n \t\t\tfor (; i \u003c kvm_tdx-\u003etd.tdcx_nr_pages; i++) {\n-\t\t\t\t__free_page(tdx-\u003evp.tdcx_pages[i]);\n+\t\t\t\ttdx_free_control_page(tdx-\u003evp.tdcx_pages[i]);\n \t\t\t\ttdx-\u003evp.tdcx_pages[i] = NULL;\n \t\t\t}\n \t\t\treturn -EIO;\n@@ -3021,16 +3060,14 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx)\n \n free_tdcx:\n \tfor (i = 0; i \u003c kvm_tdx-\u003etd.tdcx_nr_pages; i++) {\n-\t\tif (tdx-\u003evp.tdcx_pages[i])\n-\t\t\t__free_page(tdx-\u003evp.tdcx_pages[i]);\n+\t\ttdx_free_control_page(tdx-\u003evp.tdcx_pages[i]);\n \t\ttdx-\u003evp.tdcx_pages[i] = NULL;\n \t}\n \tkfree(tdx-\u003evp.tdcx_pages);\n \ttdx-\u003evp.tdcx_pages = NULL;\n \n free_tdvpr:\n-\tif (tdx-\u003evp.tdvpr_page)\n-\t\t__free_page(tdx-\u003evp.tdvpr_page);\n+\ttdx_free_control_page(tdx-\u003evp.tdvpr_page);\n \ttdx-\u003evp.tdvpr_page = NULL;\n \ttdx-\u003evp.tdvpr_pa = 0;\n \n@@ -3493,6 +3530,10 @@ int __init tdx_hardware_setup(void)\n \n \tvt_x86_ops.set_external_spte = tdx_sept_set_private_spte;\n \tvt_x86_ops.free_external_spt = tdx_sept_free_private_spt;\n+\n+\tif (tdx_supports_dynamic_pamt(tdx_sysinfo))\n+\t\tvt_x86_ops.topup_external_cache = tdx_topup_external_pamt_cache;\n+\n \tvt_x86_ops.protected_apic_has_interrupt = tdx_protected_apic_has_interrupt;\n \treturn 0;\n \ndiff --git a/arch/x86/kvm/vmx/tdx.h b/arch/x86/kvm/vmx/tdx.h\nindex ac8323a68b163..fd368e3ee060b 100644\n--- a/arch/x86/kvm/vmx/tdx.h\n+++ b/arch/x86/kvm/vmx/tdx.h\n@@ -72,6 +72,8 @@ struct vcpu_tdx {\n \n \tu64 map_gpa_next;\n \tu64 map_gpa_end;\n+\n+\tstruct tdx_pamt_cache pamt_cache;\n };\n \n void tdh_vp_rd_failed(struct vcpu_tdx *tdx, char *uclass, u32 field, u64 err);\ndiff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c\nindex 42df8ea464c47..dec45709e454a 100644\n--- a/arch/x86/virt/vmx/tdx/tdx.c\n+++ b/arch/x86/virt/vmx/tdx/tdx.c\n@@ -30,6 +30,7 @@\n #include \u003clinux/suspend.h\u003e\n #include \u003clinux/syscore_ops.h\u003e\n #include \u003clinux/idr.h\u003e\n+#include \u003clinux/vmalloc.h\u003e\n #include \u003casm/page.h\u003e\n #include \u003casm/special_insns.h\u003e\n #include \u003casm/msr-index.h\u003e\n@@ -46,6 +47,8 @@\n #include \"seamcall_internal.h\"\n #include \"tdx.h\"\n \n+static bool tdx_enable_dpamt __ro_after_init;\n+\n struct tdx_module_state {\n \tbool initialized;\n \tbool sysinit_done;\n@@ -63,6 +66,14 @@ static DEFINE_PER_CPU(bool, tdx_lp_initialized);\n \n static struct tdmr_info_list tdx_tdmr_list;\n \n+/*\n+ * On a machine with Dynamic PAMT, the kernel maintains a reference counter\n+ * for every 2MB range. The counter indicates how many users there are for\n+ * the PAMT memory of the 2MB range. The kernel allocates PAMT refcounts at\n+ * initialization.\n+ */\n+static atomic_t *pamt_refcounts;\n+\n /* All TDX-usable memory regions.  Protected by mem_hotplug_lock. */\n static LIST_HEAD(tdx_memlist);\n \n@@ -252,6 +263,42 @@ static struct syscore tdx_syscore = {\n \t.ops = \u0026tdx_syscore_ops,\n };\n \n+/*\n+ * Allocate PAMT reference counters for all physical memory.\n+ *\n+ * It consumes 2MB for every 1TB of physical memory.\n+ */\n+static __init int init_pamt_refcounts(void)\n+{\n+\tsize_t size = DIV_ROUND_UP(max_pfn, PTRS_PER_PTE) * sizeof(*pamt_refcounts);\n+\n+\tif (!tdx_supports_dynamic_pamt(\u0026tdx_sysinfo))\n+\t\treturn 0;\n+\n+\tpamt_refcounts = vzalloc(size);\n+\tif (!pamt_refcounts)\n+\t\treturn -ENOMEM;\n+\n+\treturn 0;\n+}\n+\n+static __init void free_pamt_refcounts(void)\n+{\n+\tif (!tdx_supports_dynamic_pamt(\u0026tdx_sysinfo))\n+\t\treturn;\n+\n+\tvfree(pamt_refcounts);\n+\tpamt_refcounts = NULL;\n+}\n+\n+static atomic_t *tdx_find_pamt_refcount(unsigned long pfn)\n+{\n+\t/* Find which PMD a PFN is in. */\n+\tunsigned long index = pfn \u003e\u003e (PMD_SHIFT - PAGE_SHIFT);\n+\n+\treturn \u0026pamt_refcounts[index];\n+}\n+\n /*\n  * Add a memory region as a TDX memory block.  The caller must make sure\n  * all memory regions are added in address ascending order and don't\n@@ -510,35 +557,37 @@ static __init int fill_out_tdmrs(struct list_head *tmb_list,\n \treturn 0;\n }\n \n+static __init unsigned long tdmr_get_pamt_bitmap_sz(struct tdmr_info *tdmr)\n+{\n+\tunsigned long pamt_sz, nr_pamt_entries;\n+\tint bits_per_entry;\n+\n+\tbits_per_entry = tdx_sysinfo.tdmr.pamt_page_bitmap_entry_bits;\n+\tnr_pamt_entries = tdmr-\u003esize \u003e\u003e PAGE_SHIFT;\n+\tpamt_sz = DIV_ROUND_UP(nr_pamt_entries * bits_per_entry, BITS_PER_BYTE);\n+\n+\treturn PAGE_ALIGN(pamt_sz);\n+}\n+\n /*\n  * Calculate PAMT size given a TDMR and a page size.  The returned\n  * PAMT size is always aligned up to 4K page boundary.\n  */\n-static __init unsigned long tdmr_get_pamt_sz(struct tdmr_info *tdmr, int pgsz,\n-\t\t\t\t\t     u16 pamt_entry_size)\n+static __init unsigned long tdmr_get_pamt_sz(struct tdmr_info *tdmr, int pgsz)\n {\n \tunsigned long pamt_sz, nr_pamt_entries;\n+\tconst int tdx_pg_size_shift[TDX_PS_NR] = { PAGE_SHIFT, PMD_SHIFT, PUD_SHIFT };\n+\tconst u16 pamt_entry_size[TDX_PS_NR] = {\n+\t\ttdx_sysinfo.tdmr.pamt_4k_entry_size,\n+\t\ttdx_sysinfo.tdmr.pamt_2m_entry_size,\n+\t\ttdx_sysinfo.tdmr.pamt_1g_entry_size,\n+\t};\n \n-\tswitch (pgsz) {\n-\tcase TDX_PS_4K:\n-\t\tnr_pamt_entries = tdmr-\u003esize \u003e\u003e PAGE_SHIFT;\n-\t\tbreak;\n-\tcase TDX_PS_2M:\n-\t\tnr_pamt_entries = tdmr-\u003esize \u003e\u003e PMD_SHIFT;\n-\t\tbreak;\n-\tcase TDX_PS_1G:\n-\t\tnr_pamt_entries = tdmr-\u003esize \u003e\u003e PUD_SHIFT;\n-\t\tbreak;\n-\tdefault:\n-\t\tWARN_ON_ONCE(1);\n-\t\treturn 0;\n-\t}\n+\tnr_pamt_entries = tdmr-\u003esize \u003e\u003e tdx_pg_size_shift[pgsz];\n+\tpamt_sz = nr_pamt_entries * pamt_entry_size[pgsz];\n \n-\tpamt_sz = nr_pamt_entries * pamt_entry_size;\n \t/* TDX requires PAMT size must be 4K aligned */\n-\tpamt_sz = ALIGN(pamt_sz, PAGE_SIZE);\n-\n-\treturn pamt_sz;\n+\treturn PAGE_ALIGN(pamt_sz);\n }\n \n /*\n@@ -576,15 +625,11 @@ static __init int tdmr_get_nid(struct tdmr_info *tdmr, struct list_head *tmb_lis\n  * within @tdmr, and set up PAMTs for @tdmr.\n  */\n static __init int tdmr_set_up_pamt(struct tdmr_info *tdmr,\n-\t\t\t\t   struct list_head *tmb_list,\n-\t\t\t\t   u16 pamt_entry_size[])\n+\t\t\t\t   struct list_head *tmb_list)\n {\n-\tunsigned long pamt_base[TDX_PS_NR];\n-\tunsigned long pamt_size[TDX_PS_NR];\n-\tunsigned long tdmr_pamt_base;\n \tunsigned long tdmr_pamt_size;\n \tstruct page *pamt;\n-\tint pgsz, nid;\n+\tint nid;\n \n \tnid = tdmr_get_nid(tdmr, tmb_list);\n \n@@ -592,13 +637,18 @@ static __init int tdmr_set_up_pamt(struct tdmr_info *tdmr,\n \t * Calculate the PAMT size for each TDX supported page size\n \t * and the total PAMT size.\n \t */\n-\ttdmr_pamt_size = 0;\n-\tfor (pgsz = TDX_PS_4K; pgsz \u003c TDX_PS_NR; pgsz++) {\n-\t\tpamt_size[pgsz] = tdmr_get_pamt_sz(tdmr, pgsz,\n-\t\t\t\t\tpamt_entry_size[pgsz]);\n-\t\ttdmr_pamt_size += pamt_size[pgsz];\n+\ttdmr-\u003epamt_1g_size = tdmr_get_pamt_sz(tdmr, TDX_PS_1G);\n+\ttdmr-\u003epamt_2m_size = tdmr_get_pamt_sz(tdmr, TDX_PS_2M);\n+\n+\tif (tdx_supports_dynamic_pamt(\u0026tdx_sysinfo)) {\n+\t\t/* With Dynamic PAMT, PAMT_4K is replaced with a bitmap */\n+\t\ttdmr-\u003epamt_4k_size = tdmr_get_pamt_bitmap_sz(tdmr);\n+\t} else {\n+\t\ttdmr-\u003epamt_4k_size = tdmr_get_pamt_sz(tdmr, TDX_PS_4K);\n \t}\n \n+\ttdmr_pamt_size = tdmr-\u003epamt_4k_size + tdmr-\u003epamt_2m_size + tdmr-\u003epamt_1g_size;\n+\n \t/*\n \t * Allocate one chunk of physically contiguous memory for all\n \t * PAMTs.  This helps minimize the PAMT's use of reserved areas\n@@ -606,25 +656,17 @@ static __init int tdmr_set_up_pamt(struct tdmr_info *tdmr,\n \t */\n \tpamt = alloc_contig_pages(tdmr_pamt_size \u003e\u003e PAGE_SHIFT, GFP_KERNEL,\n \t\t\tnid, \u0026node_online_map);\n-\tif (!pamt)\n-\t\treturn -ENOMEM;\n \n \t/*\n-\t * Break the contiguous allocation back up into the\n-\t * individual PAMTs for each page size.\n+\t * tdmr-\u003epamt_4k_base is still zero so the error\n+\t * path of the caller will skip freeing the PAMT.\n \t */\n-\ttdmr_pamt_base = page_to_pfn(pamt) \u003c\u003c PAGE_SHIFT;\n-\tfor (pgsz = TDX_PS_4K; pgsz \u003c TDX_PS_NR; pgsz++) {\n-\t\tpamt_base[pgsz] = tdmr_pamt_base;\n-\t\ttdmr_pamt_base += pamt_size[pgsz];\n-\t}\n+\tif (!pamt)\n+\t\treturn -ENOMEM;\n \n-\ttdmr-\u003epamt_4k_base = pamt_base[TDX_PS_4K];\n-\ttdmr-\u003epamt_4k_size = pamt_size[TDX_PS_4K];\n-\ttdmr-\u003epamt_2m_base = pamt_base[TDX_PS_2M];\n-\ttdmr-\u003epamt_2m_size = pamt_size[TDX_PS_2M];\n-\ttdmr-\u003epamt_1g_base = pamt_base[TDX_PS_1G];\n-\ttdmr-\u003epamt_1g_size = pamt_size[TDX_PS_1G];\n+\ttdmr-\u003epamt_4k_base = page_to_phys(pamt);\n+\ttdmr-\u003epamt_2m_base = tdmr-\u003epamt_4k_base + tdmr-\u003epamt_4k_size;\n+\ttdmr-\u003epamt_1g_base = tdmr-\u003epamt_2m_base + tdmr-\u003epamt_2m_size;\n \n \treturn 0;\n }\n@@ -655,10 +697,7 @@ static __init void tdmr_do_pamt_func(struct tdmr_info *tdmr,\n \ttdmr_get_pamt(tdmr, \u0026pamt_base, \u0026pamt_size);\n \n \t/* Do nothing if PAMT hasn't been allocated for this TDMR */\n-\tif (!pamt_size)\n-\t\treturn;\n-\n-\tif (WARN_ON_ONCE(!pamt_base))\n+\tif (!pamt_base)\n \t\treturn;\n \n \tpamt_func(pamt_base, pamt_size);\n@@ -684,14 +723,12 @@ static __init void tdmrs_free_pamt_all(struct tdmr_info_list *tdmr_list)\n \n /* Allocate and set up PAMTs for all TDMRs */\n static __init int tdmrs_set_up_pamt_all(struct tdmr_info_list *tdmr_list,\n-\t\t\t\t\tstruct list_head *tmb_list,\n-\t\t\t\t\tu16 pamt_entry_size[])\n+\t\t\t\t struct list_head *tmb_list)\n {\n \tint i, ret = 0;\n \n \tfor (i = 0; i \u003c tdmr_list-\u003enr_consumed_tdmrs; i++) {\n-\t\tret = tdmr_set_up_pamt(tdmr_entry(tdmr_list, i), tmb_list,\n-\t\t\t\tpamt_entry_size);\n+\t\tret = tdmr_set_up_pamt(tdmr_entry(tdmr_list, i), tmb_list);\n \t\tif (ret)\n \t\t\tgoto err;\n \t}\n@@ -968,18 +1005,13 @@ static __init int construct_tdmrs(struct list_head *tmb_list,\n \t\t\t\t  struct tdmr_info_list *tdmr_list,\n \t\t\t\t  struct tdx_sys_info_tdmr *sysinfo_tdmr)\n {\n-\tu16 pamt_entry_size[TDX_PS_NR] = {\n-\t\tsysinfo_tdmr-\u003epamt_4k_entry_size,\n-\t\tsysinfo_tdmr-\u003epamt_2m_entry_size,\n-\t\tsysinfo_tdmr-\u003epamt_1g_entry_size,\n-\t};\n \tint ret;\n \n \tret = fill_out_tdmrs(tmb_list, tdmr_list);\n \tif (ret)\n \t\treturn ret;\n \n-\tret = tdmrs_set_up_pamt_all(tdmr_list, tmb_list, pamt_entry_size);\n+\tret = tdmrs_set_up_pamt_all(tdmr_list, tmb_list);\n \tif (ret)\n \t\treturn ret;\n \n@@ -998,6 +1030,8 @@ static __init int construct_tdmrs(struct list_head *tmb_list,\n \treturn ret;\n }\n \n+#define TDX_SYS_CONFIG_DYNAMIC_PAMT\tBIT(16)\n+\n static __init int config_tdx_module(struct tdmr_info_list *tdmr_list,\n \t\t\t\t    u64 global_keyid)\n {\n@@ -1026,6 +1060,12 @@ static __init int config_tdx_module(struct tdmr_info_list *tdmr_list,\n \targs.rcx = __pa(tdmr_pa_array);\n \targs.rdx = tdmr_list-\u003enr_consumed_tdmrs;\n \targs.r8 = global_keyid;\n+\n+\tif (tdx_supports_dynamic_pamt(\u0026tdx_sysinfo)) {\n+\t\tpr_info(\"Enable Dynamic PAMT\\n\");\n+\t\targs.r8 |= TDX_SYS_CONFIG_DYNAMIC_PAMT;\n+\t}\n+\n \tret = seamcall_prerr(TDH_SYS_CONFIG, \u0026args);\n \n \t/* Free the array as it is not required anymore. */\n@@ -1167,10 +1207,14 @@ static __init int init_tdx_module(void)\n \t */\n \tget_online_mems();\n \n-\tret = build_tdx_memlist(\u0026tdx_memlist);\n+\tret = init_pamt_refcounts();\n \tif (ret)\n \t\tgoto out_put_tdxmem;\n \n+\tret = build_tdx_memlist(\u0026tdx_memlist);\n+\tif (ret)\n+\t\tgoto err_free_pamt_refcounts;\n+\n \t/* Allocate enough space for constructing TDMRs */\n \tret = alloc_tdmr_list(\u0026tdx_tdmr_list, \u0026tdx_sysinfo.tdmr);\n \tif (ret)\n@@ -1220,6 +1264,8 @@ static __init int init_tdx_module(void)\n \tfree_tdmr_list(\u0026tdx_tdmr_list);\n err_free_tdxmem:\n \tfree_tdx_memlist(\u0026tdx_memlist);\n+err_free_pamt_refcounts:\n+\tfree_pamt_refcounts();\n \tgoto out_put_tdxmem;\n }\n \n@@ -2003,6 +2049,288 @@ u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, kvm_pfn_t pfn)\n }\n EXPORT_SYMBOL_FOR_KVM(tdh_phymem_page_wbinvd_hkid);\n \n+bool tdx_supports_dynamic_pamt(const struct tdx_sys_info *sysinfo)\n+{\n+\treturn sysinfo-\u003efeatures.tdx_features0 \u0026 TDX_FEATURES0_DYNAMIC_PAMT \u0026\u0026\n+\t       tdx_enable_dpamt;\n+}\n+EXPORT_SYMBOL_FOR_KVM(tdx_supports_dynamic_pamt);\n+\n+static struct page *tdx_alloc_page_pamt_cache(struct tdx_pamt_cache *cache)\n+{\n+\tstruct page *page;\n+\n+\tpage = list_first_entry_or_null(\u0026cache-\u003epage_list, struct page, lru);\n+\tif (page) {\n+\t\tlist_del(\u0026page-\u003elru);\n+\t\tcache-\u003ecnt--;\n+\t}\n+\n+\treturn page;\n+}\n+\n+static struct page *alloc_dpamt_page(struct tdx_pamt_cache *cache)\n+{\n+\tif (cache)\n+\t\treturn tdx_alloc_page_pamt_cache(cache);\n+\n+\treturn alloc_page(GFP_KERNEL_ACCOUNT);\n+}\n+\n+static int alloc_pamt_array(struct page **pamt_pages, struct tdx_pamt_cache *cache)\n+{\n+\tint i, j;\n+\n+\tfor (i = 0; i \u003c TDX_DPAMT_ENTRY_PAGE_CNT; i++) {\n+\t\tpamt_pages[i] = alloc_dpamt_page(cache);\n+\t\tif (!pamt_pages[i])\n+\t\t\tgoto err;\n+\t}\n+\n+\treturn 0;\n+\n+err:\n+\tfor (j = 0; j \u003c i; j++)\n+\t\t__free_page(pamt_pages[j]);\n+\n+\treturn -ENOMEM;\n+}\n+\n+static void free_pamt_array(struct page **pamt_pages)\n+{\n+\tfor (int i = 0; i \u003c TDX_DPAMT_ENTRY_PAGE_CNT; i++) {\n+\t\t/*\n+\t\t * Reset pages unconditionally to cover cases\n+\t\t * where they were passed to the TDX module.\n+\t\t */\n+\t\ttdx_quirk_reset_paddr(page_to_phys(pamt_pages[i]), PAGE_SIZE);\n+\n+\t\t__free_page(pamt_pages[i]);\n+\t}\n+}\n+\n+/*\n+ * Calculate the arg needed for operating on the DPAMT backing for\n+ * a given 4KB page.\n+ */\n+static u64 pamt_2mb_arg(kvm_pfn_t pfn)\n+{\n+\t/* Arg value will specify a 2MB region of physical address space. */\n+\tunsigned long hpa_2mb = ALIGN_DOWN(pfn \u003c\u003c PAGE_SHIFT, PMD_SIZE);\n+\n+\treturn hpa_2mb | TDX_PS_2M;\n+}\n+\n+/* Add PAMT backing for the 2MB region surrounding the given pfn. */\n+static u64 tdh_phymem_pamt_add(kvm_pfn_t pfn, struct page **pamt_pages)\n+{\n+\tstruct tdx_module_args args = {\n+\t\t.rcx = pamt_2mb_arg(pfn),\n+\t\t.rdx = page_to_phys(pamt_pages[0]),\n+\t\t.r8 = page_to_phys(pamt_pages[1]),\n+\t};\n+\n+\treturn seamcall(TDH_PHYMEM_PAMT_ADD, \u0026args);\n+}\n+\n+/* Remove PAMT backing for the 2MB region surrounding the given pfn. */\n+static u64 tdh_phymem_pamt_remove(kvm_pfn_t pfn, struct page **pamt_pages)\n+{\n+\tstruct tdx_module_args args = {\n+\t\t.rcx = pamt_2mb_arg(pfn),\n+\t};\n+\tu64 ret;\n+\n+\tret = seamcall_ret(TDH_PHYMEM_PAMT_REMOVE, \u0026args);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\t/* Copy PAMT pages out of the struct per the TDX ABI */\n+\tpamt_pages[0] = phys_to_page(args.rdx);\n+\tpamt_pages[1] = phys_to_page(args.r8);\n+\n+\treturn 0;\n+}\n+\n+/* Serializes adding/removing PAMT memory */\n+static DEFINE_SPINLOCK(pamt_lock);\n+\n+/* Bump PAMT refcount for the given pfn and allocate PAMT backing if needed. */\n+int tdx_pamt_get(kvm_pfn_t pfn, struct tdx_pamt_cache *cache)\n+{\n+\tstruct page *pamt_pages[TDX_DPAMT_ENTRY_PAGE_CNT];\n+\tatomic_t *pamt_refcount;\n+\tu64 tdx_status;\n+\tint ret;\n+\n+\tif (!tdx_supports_dynamic_pamt(\u0026tdx_sysinfo))\n+\t\treturn 0;\n+\n+\tpamt_refcount = tdx_find_pamt_refcount(pfn);\n+\n+\t/*\n+\t * If the pamt page is already added (i.e. refcount \u003e= 1),\n+\t * then just increment the refcount.\n+\t */\n+\tif (atomic_inc_not_zero(pamt_refcount))\n+\t\treturn 0;\n+\n+\tret = alloc_pamt_array(pamt_pages, cache);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tspin_lock(\u0026pamt_lock);\n+\n+\t/*\n+\t * Unlike tdx_pamt_put() which uses atomic_dec_and_lock() to\n+\t * atomically handle the 1-\u003e0 transition, the get side has no\n+\t * equivalent combined primitive for 0-\u003e1. Recheck under the\n+\t * lock since another get may have already done the 0-\u003e1\n+\t * transition after both saw atomic_inc_not_zero() fail.\n+\t */\n+\tif (atomic_inc_not_zero(pamt_refcount))\n+\t\tgoto out_free;\n+\n+\ttdx_status = tdh_phymem_pamt_add(pfn, pamt_pages);\n+\tif (WARN_ON_ONCE(tdx_status != TDX_SUCCESS)) {\n+\t\tret = -EIO;\n+\t\tgoto out_free;\n+\t}\n+\n+\t/*\n+\t * The refcount is zero, and this locked path is the\n+\t * only way to increase it from 0-\u003e1.\n+\t */\n+\tatomic_set(pamt_refcount, 1);\n+\tspin_unlock(\u0026pamt_lock);\n+\treturn 0;\n+\n+out_free:\n+\tspin_unlock(\u0026pamt_lock);\n+\tfree_pamt_array(pamt_pages);\n+\n+\treturn ret;\n+}\n+EXPORT_SYMBOL_FOR_KVM(tdx_pamt_get);\n+\n+/* Drop PAMT refcount for the given pfn and free PAMT backing if needed. */\n+void tdx_pamt_put(kvm_pfn_t pfn)\n+{\n+\tstruct page *pamt_pages[TDX_DPAMT_ENTRY_PAGE_CNT] = {};\n+\tatomic_t *pamt_refcount;\n+\tu64 tdx_status;\n+\n+\tif (!tdx_supports_dynamic_pamt(\u0026tdx_sysinfo))\n+\t\treturn;\n+\n+\tpamt_refcount = tdx_find_pamt_refcount(pfn);\n+\n+\t/*\n+\t * If there is more than 1 reference on the pamt page, don't\n+\t * remove it yet. Just decrement the refcount.\n+\t */\n+\tif (!atomic_dec_and_lock(pamt_refcount, \u0026pamt_lock))\n+\t\treturn;\n+\n+\ttdx_status = tdh_phymem_pamt_remove(pfn, pamt_pages);\n+\n+\t/*\n+\t * Don't free pamt_pages as it could hold garbage when\n+\t * tdh_phymem_pamt_remove() fails.  Don't panic/BUG_ON(), as\n+\t * there is no risk of data corruption, but do yell loudly as\n+\t * failure indicates a kernel bug, memory is being leaked, and\n+\t * the dangling PAMT entry may cause future operations to fail.\n+\t */\n+\tif (WARN_ON_ONCE(tdx_status != TDX_SUCCESS)) {\n+\t\t/*\n+\t\t * atomic_dec_and_lock() already decremented it to 0,\n+\t\t * but the PAMT entry still exists since REMOVE failed.\n+\t\t */\n+\t\tatomic_set(pamt_refcount, 1);\n+\t\tgoto out_unlock;\n+\t}\n+\n+\tspin_unlock(\u0026pamt_lock);\n+\tfree_pamt_array(pamt_pages);\n+\treturn;\n+out_unlock:\n+\tspin_unlock(\u0026pamt_lock);\n+}\n+EXPORT_SYMBOL_FOR_KVM(tdx_pamt_put);\n+\n+void tdx_free_pamt_cache(struct tdx_pamt_cache *cache)\n+{\n+\tstruct page *page;\n+\n+\twhile ((page = tdx_alloc_page_pamt_cache(cache)))\n+\t\t__free_page(page);\n+}\n+EXPORT_SYMBOL_FOR_KVM(tdx_free_pamt_cache);\n+\n+int tdx_topup_pamt_cache(struct tdx_pamt_cache *cache, unsigned long npages)\n+{\n+\tif (WARN_ON_ONCE(!tdx_supports_dynamic_pamt(\u0026tdx_sysinfo)))\n+\t\treturn 0;\n+\n+\tnpages *= TDX_DPAMT_ENTRY_PAGE_CNT;\n+\n+\twhile (cache-\u003ecnt \u003c npages) {\n+\t\tstruct page *page = alloc_page(GFP_KERNEL_ACCOUNT);\n+\n+\t\tif (!page)\n+\t\t\treturn -ENOMEM;\n+\n+\t\tlist_add(\u0026page-\u003elru, \u0026cache-\u003epage_list);\n+\t\tcache-\u003ecnt++;\n+\t}\n+\n+\treturn 0;\n+}\n+EXPORT_SYMBOL_FOR_KVM(tdx_topup_pamt_cache);\n+\n+/*\n+ * Return a page that can be gifted to the TDX-Module for use as a \"control\"\n+ * page, i.e. pages that are used for control structures for a given TDX\n+ * guest, and thus obtain TDX protections, including PAMT tracking.\n+ */\n+struct page *tdx_alloc_control_page(void)\n+{\n+\tstruct page *page;\n+\n+\tpage = alloc_page(GFP_KERNEL_ACCOUNT);\n+\tif (!page)\n+\t\treturn NULL;\n+\n+\tif (tdx_pamt_get(page_to_pfn(page), NULL)) {\n+\t\t__free_page(page);\n+\t\treturn NULL;\n+\t}\n+\n+\treturn page;\n+}\n+EXPORT_SYMBOL_FOR_KVM(tdx_alloc_control_page);\n+\n+/*\n+ * Free a page that was gifted to the TDX-Module for use as a control\n+ * page. After this, the page is no longer protected by TDX.\n+ */\n+void tdx_free_control_page(struct page *page)\n+{\n+\tif (!page)\n+\t\treturn;\n+\n+\ttdx_pamt_put(page_to_pfn(page));\n+\t__free_page(page);\n+}\n+EXPORT_SYMBOL_FOR_KVM(tdx_free_control_page);\n+\n+static int __init tdx_dpamt_setup(char *str)\n+{\n+\treturn kstrtobool(str, \u0026tdx_enable_dpamt) == 0;\n+}\n+\n+__setup(\"tdx_dpamt=\", tdx_dpamt_setup);\n+\n void tdx_sys_disable(void)\n {\n \tstruct tdx_module_args args = {};\ndiff --git a/arch/x86/virt/vmx/tdx/tdx.h b/arch/x86/virt/vmx/tdx/tdx.h\nindex bdfd0e1e337ac..a886c54decaad 100644\n--- a/arch/x86/virt/vmx/tdx/tdx.h\n+++ b/arch/x86/virt/vmx/tdx/tdx.h\n@@ -48,6 +48,8 @@\n #define TDH_SYS_CONFIG\t\t\t45\n #define TDH_SYS_SHUTDOWN\t\t52\n #define TDH_SYS_UPDATE\t\t\t53\n+#define TDH_PHYMEM_PAMT_ADD\t\t58\n+#define TDH_PHYMEM_PAMT_REMOVE\t\t59\n #define TDH_SYS_DISABLE\t\t\t69\n \n /*\ndiff --git a/arch/x86/virt/vmx/tdx/tdx_global_metadata.c b/arch/x86/virt/vmx/tdx/tdx_global_metadata.c\nindex e49c300f23d43..93b1f26252c63 100644\n--- a/arch/x86/virt/vmx/tdx/tdx_global_metadata.c\n+++ b/arch/x86/virt/vmx/tdx/tdx_global_metadata.c\n@@ -1,6 +1,6 @@\n // SPDX-License-Identifier: GPL-2.0\n /*\n- * Automatically generated functions to read TDX global metadata.\n+ * Functions to read TDX global metadata.\n  *\n  * This file doesn't compile on its own as it lacks of inclusion\n  * of SEAMCALL wrapper primitive which reads global metadata.\n@@ -33,6 +33,18 @@ static __init int get_tdx_sys_info_features(struct tdx_sys_info_features *sysinf\n \treturn ret;\n }\n \n+static __init int get_tdx_sys_info_tdmr_dpamt(struct tdx_sys_info_tdmr *sysinfo_tdmr)\n+{\n+\tint ret;\n+\tu64 val;\n+\n+\tret = read_sys_metadata_field(0x9100000000000013, \u0026val);\n+\tif (!ret)\n+\t\tsysinfo_tdmr-\u003epamt_page_bitmap_entry_bits = val;\n+\n+\treturn ret;\n+}\n+\n static __init int get_tdx_sys_info_tdmr(struct tdx_sys_info_tdmr *sysinfo_tdmr)\n {\n \tint ret = 0;\n@@ -129,5 +141,14 @@ static __init int get_tdx_sys_info(struct tdx_sys_info *sysinfo)\n \tret = ret ?: get_tdx_sys_info_td_ctrl(\u0026sysinfo-\u003etd_ctrl);\n \tret = ret ?: get_tdx_sys_info_td_conf(\u0026sysinfo-\u003etd_conf);\n \n+\t/*\n+\t * The kernel supports using TDX without Dynamic PAMT, so\n+\t * avoid reporting failure if it's not supported. Don't try\n+\t * to support buggy TDX modules that advertise Dynamic PAMT\n+\t * but don't expose the metadata.\n+\t */\n+\tif (!ret \u0026\u0026 tdx_supports_dynamic_pamt(sysinfo))\n+\t\tret = get_tdx_sys_info_tdmr_dpamt(\u0026sysinfo-\u003etdmr);\n+\n \treturn ret;\n }\n"
}

For your convenience, here is the diff of the changes:
commit f00efc49131881f16a5b1580adf3f7c794263146
Author: syz-cluster <triage@syzkaller.com>
Date:   Thu Aug 6 03:00:59 2026 +0000

    syz-cluster: applied patch under review

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index b5493a7f8f228..a0a7670246bc5 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -7512,6 +7512,16 @@ Kernel parameters
 
 	tdfx=		[HW,DRM]
 
+	tdx_dpamt=
+			[X86] Controls whether TDX will use Dynamic PAMT
+			to save memory, when supported.
+
+			Valid parameters: "on", "off"
+			Default: "off"
+
+			For details see:
+			Documentation/arch/x86/tdx.rst
+
 	test_suspend=	[SUSPEND]
 			Format: { "mem" | "standby" | "freeze" }[,N]
 			Specify "mem" (for Suspend-to-RAM) or "standby" (for
diff --git a/Documentation/arch/x86/tdx.rst b/Documentation/arch/x86/tdx.rst
index 3303499ad4c6f..a1c2309230580 100644
--- a/Documentation/arch/x86/tdx.rst
+++ b/Documentation/arch/x86/tdx.rst
@@ -200,6 +200,34 @@ reflects the TCB of the currently running TDX module and therefore
 changes after an update. By contrast, TEE_TCB_SVN reflects the TCB at TD
 launch time and is not affected.
 
+Dynamic PAMT
+------------
+
+Physical Address Metadata Table (PAMT) is memory that the TDX module needs
+to keep data about each page (think like struct page). It needs to be handed
+to the TDX module for its exclusive use. For normal PAMT, this is installed
+when the TDX module is first loaded and comes to about 0.4% of system memory.
+
+Dynamic PAMT is a TDX module feature that allows VMM to allocate part of the
+PAMT as needed (the parts for tracking 4KB size pages). The other page sizes
+(1GB and 2MB) are still allocated statically at the time of TDX module
+initialization. This reduces the amount of memory that TDX uses while TDs are
+not in use.
+
+When Dynamic PAMT is in use, dmesg shows it like::
+
+  [..] virt/tdx: Enable Dynamic PAMT
+  [..] virt/tdx: 10092 KB allocated for PAMT
+  [..] virt/tdx: TDX-Module initialized
+
+Dynamic PAMT is only enabled when supported and the ``tdx_dpamt=`` kernel
+parameter is set to "on". The feature is off by default because TDX module
+internal details prevent Dynamic PAMT from working on all keyid partitioning
+configurations. When the TDX module is fixed to include these constraints in
+its enumeration of Dynamic PAMT support, kernel support can be changed to
+default on. For more information, consult the Intel TDX documentation about
+Dynamic PAMT.
+
 TDX Interaction to Other Kernel Components
 ------------------------------------------
 
diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h
index 83dc5086138b3..588563dfe88d5 100644
--- a/arch/x86/include/asm/kvm-x86-ops.h
+++ b/arch/x86/include/asm/kvm-x86-ops.h
@@ -98,6 +98,7 @@ KVM_X86_OP_OPTIONAL_RET0(tdp_has_smep)
 KVM_X86_OP(load_mmu_pgd)
 KVM_X86_OP_OPTIONAL_RET0(set_external_spte)
 KVM_X86_OP_OPTIONAL(free_external_spt)
+KVM_X86_OP_OPTIONAL_RET0(topup_external_cache)
 KVM_X86_OP(has_wbinvd_exit)
 KVM_X86_OP(get_l2_tsc_offset)
 KVM_X86_OP(get_l2_tsc_multiplier)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 5f6c1ce9673b7..0dbf6564dd0c5 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -1922,6 +1922,7 @@ struct kvm_x86_ops {
 	/* Update external page tables for page table about to be freed. */
 	void (*free_external_spt)(struct kvm *kvm, struct kvm_mmu_page *sp);
 
+	int (*topup_external_cache)(struct kvm_vcpu *vcpu, int min_nr_spts);
 
 	bool (*has_wbinvd_exit)(void);
 
diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h
index 89e97d5761d89..7910901a7ba21 100644
--- a/arch/x86/include/asm/tdx.h
+++ b/arch/x86/include/asm/tdx.h
@@ -36,6 +36,7 @@
 /* Bit definitions of TDX_FEATURES0 metadata field */
 #define TDX_FEATURES0_TD_PRESERVING	BIT_ULL(1)
 #define TDX_FEATURES0_NO_RBP_MOD	BIT_ULL(18)
+#define TDX_FEATURES0_DYNAMIC_PAMT	BIT_ULL(36)
 
 #ifndef __ASSEMBLER__
 
@@ -118,12 +119,37 @@ static inline bool tdx_supports_runtime_update(const struct tdx_sys_info *sysinf
 	return sysinfo->features.tdx_features0 & TDX_FEATURES0_TD_PRESERVING;
 }
 
+bool tdx_supports_dynamic_pamt(const struct tdx_sys_info *sysinfo);
+
+/* Simple structure for pre-allocating Dynamic PAMT pages outside of spinlocks. */
+struct tdx_pamt_cache {
+	struct list_head page_list;
+	int cnt;
+};
+
+static inline void tdx_init_pamt_cache(struct tdx_pamt_cache *cache)
+{
+	INIT_LIST_HEAD(&cache->page_list);
+	cache->cnt = 0;
+}
+
+void tdx_free_pamt_cache(struct tdx_pamt_cache *cache);
+int tdx_topup_pamt_cache(struct tdx_pamt_cache *cache, unsigned long npages);
+int tdx_pamt_get(kvm_pfn_t pfn, struct tdx_pamt_cache *cache);
+void tdx_pamt_put(kvm_pfn_t pfn);
+
 int tdx_guest_keyid_alloc(void);
 u32 tdx_get_nr_guest_keyids(void);
 void tdx_guest_keyid_free(unsigned int keyid);
 
 void tdx_quirk_reset_paddr(unsigned long base, unsigned long size);
 
+/* Number of PAMT pages to be provided to TDX module per 2MB region of PA */
+#define TDX_DPAMT_ENTRY_PAGE_CNT 2
+
+struct page *tdx_alloc_control_page(void);
+void tdx_free_control_page(struct page *page);
+
 struct tdx_td {
 	/* TD root structure: */
 	struct page *tdr_page;
diff --git a/arch/x86/include/asm/tdx_global_metadata.h b/arch/x86/include/asm/tdx_global_metadata.h
index 41150d546589c..2a42551fc33cd 100644
--- a/arch/x86/include/asm/tdx_global_metadata.h
+++ b/arch/x86/include/asm/tdx_global_metadata.h
@@ -21,6 +21,9 @@ struct tdx_sys_info_tdmr {
 	u16 pamt_4k_entry_size;
 	u16 pamt_2m_entry_size;
 	u16 pamt_1g_entry_size;
+
+	/* Optional metadata, if Dynamic PAMT is supported */
+	u8  pamt_page_bitmap_entry_bits;
 };
 
 struct tdx_sys_info_td_ctrl {
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 66e69d2a41b3c..2a628b8304409 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -614,6 +614,10 @@ static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu, bool maybe_indirect)
 					       PT64_ROOT_MAX_LEVEL);
 		if (r)
 			return r;
+
+		r = kvm_x86_call(topup_external_cache)(vcpu, PT64_ROOT_MAX_LEVEL);
+		if (r)
+			return r;
 	}
 	r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_shadow_page_cache,
 				       PT64_ROOT_MAX_LEVEL);
diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c
index 545b03d9d10b8..09573112540e9 100644
--- a/arch/x86/kvm/vmx/tdx.c
+++ b/arch/x86/kvm/vmx/tdx.c
@@ -362,7 +362,7 @@ static void tdx_reclaim_control_page(struct page *ctrl_page)
 	if (tdx_reclaim_page(ctrl_page))
 		return;
 
-	__free_page(ctrl_page);
+	tdx_free_control_page(ctrl_page);
 }
 
 struct tdx_flush_vp_arg {
@@ -589,7 +589,7 @@ static void tdx_reclaim_td_control_pages(struct kvm *kvm)
 
 	tdx_quirk_reset_paddr(page_to_phys(kvm_tdx->td.tdr_page), PAGE_SIZE);
 
-	__free_page(kvm_tdx->td.tdr_page);
+	tdx_free_control_page(kvm_tdx->td.tdr_page);
 	kvm_tdx->td.tdr_page = NULL;
 }
 
@@ -681,6 +681,8 @@ int tdx_vcpu_create(struct kvm_vcpu *vcpu)
 	if (!irqchip_split(vcpu->kvm))
 		return -EINVAL;
 
+	tdx_init_pamt_cache(&tdx->pamt_cache);
+
 	fpstate_set_confidential(&vcpu->arch.guest_fpu);
 	vcpu->arch.apic->guest_apic_protected = true;
 	INIT_LIST_HEAD(&tdx->vt.pi_wakeup_list);
@@ -866,6 +868,8 @@ void tdx_vcpu_free(struct kvm_vcpu *vcpu)
 	struct vcpu_tdx *tdx = to_tdx(vcpu);
 	int i;
 
+	tdx_free_pamt_cache(&tdx->pamt_cache);
+
 	if (vcpu->cpu != -1) {
 		KVM_BUG_ON(tdx->state == VCPU_TD_STATE_INITIALIZED, vcpu->kvm);
 		tdx_flush_vp_on_cpu(vcpu);
@@ -1621,6 +1625,17 @@ void tdx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, int pgd_level)
 	td_vmcs_write64(to_tdx(vcpu), SHARED_EPT_POINTER, root_hpa);
 }
 
+static int tdx_topup_external_pamt_cache(struct kvm_vcpu *vcpu, int min_nr_spts)
+{
+	/*
+	 * Minus one page to exclude the root SPT, but plus one page for a
+	 * possible 4KB private mapping.
+	 */
+	min_nr_spts += -1 + 1;
+
+	return tdx_topup_pamt_cache(&to_tdx(vcpu)->pamt_cache, min_nr_spts);
+}
+
 static int tdx_mem_page_add(struct kvm *kvm, gfn_t gfn, enum pg_level level,
 			    kvm_pfn_t pfn)
 {
@@ -1679,16 +1694,28 @@ static struct page *tdx_spte_to_sept_pt(struct kvm *kvm, gfn_t gfn,
 static int tdx_sept_map_nonleaf_spte(struct kvm *kvm, gfn_t gfn,
 				     enum pg_level level, u64 new_spte)
 {
+	struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
 	gpa_t gpa = gfn_to_gpa(gfn);
 	u64 err, entry, level_state;
 	struct page *sept_pt;
+	int ret;
+
+	if (KVM_BUG_ON(!vcpu, kvm))
+		return -EIO;
 
 	sept_pt = tdx_spte_to_sept_pt(kvm, gfn, new_spte, level);
 	if (!sept_pt)
 		return -EIO;
 
+	ret = tdx_pamt_get(page_to_pfn(sept_pt), &to_tdx(vcpu)->pamt_cache);
+	if (KVM_BUG_ON(ret, kvm))
+		return ret;
+
 	err = tdh_mem_sept_add(&to_kvm_tdx(kvm)->td, gpa, level, sept_pt,
 			       &entry, &level_state);
+	if (err)
+		tdx_pamt_put(page_to_pfn(sept_pt));
+
 	if (unlikely(tdx_operand_busy(err)))
 		return -EBUSY;
 
@@ -1701,8 +1728,13 @@ static int tdx_sept_map_nonleaf_spte(struct kvm *kvm, gfn_t gfn,
 static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level level,
 				  u64 new_spte)
 {
+	struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
 	struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
 	kvm_pfn_t pfn = spte_to_pfn(new_spte);
+	int ret;
+
+	if (KVM_BUG_ON(!vcpu, kvm))
+		return -EIO;
 
 	/* TODO: handle large pages. */
 	if (KVM_BUG_ON(level != PG_LEVEL_4K, kvm))
@@ -1710,6 +1742,10 @@ static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level leve
 
 	WARN_ON_ONCE((new_spte & VMX_EPT_RWX_MASK) != VMX_EPT_RWX_MASK);
 
+	ret = tdx_pamt_get(pfn, &to_tdx(vcpu)->pamt_cache);
+	if (KVM_BUG_ON(ret, kvm))
+		return ret;
+
 	/*
 	 * Ensure pre_fault_allowed is read by kvm_arch_vcpu_pre_fault_memory()
 	 * before kvm_tdx->state.  Userspace must not be allowed to pre-fault
@@ -1722,10 +1758,15 @@ static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level leve
 	 * If the TD isn't finalized/runnable, then userspace is initializing
 	 * the VM image via KVM_TDX_INIT_MEM_REGION; ADD the page to the TD.
 	 */
-	if (unlikely(kvm_tdx->state != TD_STATE_RUNNABLE))
-		return tdx_mem_page_add(kvm, gfn, level, pfn);
+	if (likely(kvm_tdx->state == TD_STATE_RUNNABLE))
+		ret = tdx_mem_page_aug(kvm, gfn, level, pfn);
+	else
+		ret = tdx_mem_page_add(kvm, gfn, level, pfn);
+
+	if (ret)
+		tdx_pamt_put(pfn);
 
-	return tdx_mem_page_aug(kvm, gfn, level, pfn);
+	return ret;
 }
 
 /*
@@ -1822,6 +1863,7 @@ static int tdx_sept_remove_leaf_spte(struct kvm *kvm, gfn_t gfn,
 		return -EIO;
 
 	tdx_quirk_reset_paddr(PFN_PHYS(pfn), PAGE_SIZE);
+	tdx_pamt_put(pfn);
 	return 0;
 }
 
@@ -1865,6 +1907,8 @@ static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn, u64 old_spte,
  */
 static void tdx_sept_free_private_spt(struct kvm *kvm, struct kvm_mmu_page *sp)
 {
+	struct page *sept_pt = virt_to_page(sp->external_spt);
+
 	/*
 	 * KVM doesn't (yet) zap page table pages in mirror page table while
 	 * TD is active, though guest pages mapped in mirror page table could be
@@ -1878,15 +1922,15 @@ static void tdx_sept_free_private_spt(struct kvm *kvm, struct kvm_mmu_page *sp)
 	 * the page to prevent the kernel from accessing the encrypted page.
 	 */
 	if (KVM_BUG_ON(is_hkid_assigned(to_kvm_tdx(kvm)), kvm) ||
-	    tdx_reclaim_page(virt_to_page(sp->external_spt)))
+	    tdx_reclaim_page(sept_pt))
 		goto out;
 
 	/*
-	 * Immediately free the S-EPT page because RCU-time free is unnecessary
-	 * after TDH.PHYMEM.PAGE.RECLAIM ensures there are no outstanding
-	 * readers.
+	 * Immediately free the S-EPT page as the TDX subsystem doesn't support
+	 * freeing pages from RCU callbacks, and more importantly because
+	 * TDH.PHYMEM.PAGE.RECLAIM ensures there are no outstanding readers.
 	 */
-	free_page((unsigned long)sp->external_spt);
+	tdx_free_control_page(sept_pt);
 out:
 	sp->external_spt = NULL;
 }
@@ -2459,7 +2503,7 @@ static int __tdx_td_init(struct kvm *kvm, struct td_params *td_params,
 
 	ret = -ENOMEM;
 
-	tdr_page = alloc_page(GFP_KERNEL_ACCOUNT);
+	tdr_page = tdx_alloc_control_page();
 	if (!tdr_page)
 		goto free_hkid;
 
@@ -2472,7 +2516,7 @@ static int __tdx_td_init(struct kvm *kvm, struct td_params *td_params,
 		goto free_tdr;
 
 	for (i = 0; i < kvm_tdx->td.tdcs_nr_pages; i++) {
-		tdcs_pages[i] = alloc_page(GFP_KERNEL_ACCOUNT);
+		tdcs_pages[i] = tdx_alloc_control_page();
 		if (!tdcs_pages[i])
 			goto free_tdcs;
 	}
@@ -2590,10 +2634,8 @@ static int __tdx_td_init(struct kvm *kvm, struct td_params *td_params,
 teardown:
 	/* Only free pages not yet added, so start at 'i' */
 	for (; i < kvm_tdx->td.tdcs_nr_pages; i++) {
-		if (tdcs_pages[i]) {
-			__free_page(tdcs_pages[i]);
-			tdcs_pages[i] = NULL;
-		}
+		tdx_free_control_page(tdcs_pages[i]);
+		tdcs_pages[i] = NULL;
 	}
 	if (!kvm_tdx->td.tdcs_pages)
 		kfree(tdcs_pages);
@@ -2608,16 +2650,13 @@ static int __tdx_td_init(struct kvm *kvm, struct td_params *td_params,
 	free_cpumask_var(packages);
 
 free_tdcs:
-	for (i = 0; i < kvm_tdx->td.tdcs_nr_pages; i++) {
-		if (tdcs_pages[i])
-			__free_page(tdcs_pages[i]);
-	}
+	for (i = 0; i < kvm_tdx->td.tdcs_nr_pages; i++)
+		tdx_free_control_page(tdcs_pages[i]);
 	kfree(tdcs_pages);
 	kvm_tdx->td.tdcs_pages = NULL;
 
 free_tdr:
-	if (tdr_page)
-		__free_page(tdr_page);
+	tdx_free_control_page(tdr_page);
 	kvm_tdx->td.tdr_page = NULL;
 
 free_hkid:
@@ -2951,7 +2990,7 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx)
 	int ret, i;
 	u64 err;
 
-	page = alloc_page(GFP_KERNEL_ACCOUNT);
+	page = tdx_alloc_control_page();
 	if (!page)
 		return -ENOMEM;
 	tdx->vp.tdvpr_page = page;
@@ -2971,7 +3010,7 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx)
 	}
 
 	for (i = 0; i < kvm_tdx->td.tdcx_nr_pages; i++) {
-		page = alloc_page(GFP_KERNEL_ACCOUNT);
+		page = tdx_alloc_control_page();
 		if (!page) {
 			ret = -ENOMEM;
 			goto free_tdcx;
@@ -2993,7 +3032,7 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx)
 			 * method, but the rest are freed here.
 			 */
 			for (; i < kvm_tdx->td.tdcx_nr_pages; i++) {
-				__free_page(tdx->vp.tdcx_pages[i]);
+				tdx_free_control_page(tdx->vp.tdcx_pages[i]);
 				tdx->vp.tdcx_pages[i] = NULL;
 			}
 			return -EIO;
@@ -3021,16 +3060,14 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx)
 
 free_tdcx:
 	for (i = 0; i < kvm_tdx->td.tdcx_nr_pages; i++) {
-		if (tdx->vp.tdcx_pages[i])
-			__free_page(tdx->vp.tdcx_pages[i]);
+		tdx_free_control_page(tdx->vp.tdcx_pages[i]);
 		tdx->vp.tdcx_pages[i] = NULL;
 	}
 	kfree(tdx->vp.tdcx_pages);
 	tdx->vp.tdcx_pages = NULL;
 
 free_tdvpr:
-	if (tdx->vp.tdvpr_page)
-		__free_page(tdx->vp.tdvpr_page);
+	tdx_free_control_page(tdx->vp.tdvpr_page);
 	tdx->vp.tdvpr_page = NULL;
 	tdx->vp.tdvpr_pa = 0;
 
@@ -3493,6 +3530,10 @@ int __init tdx_hardware_setup(void)
 
 	vt_x86_ops.set_external_spte = tdx_sept_set_private_spte;
 	vt_x86_ops.free_external_spt = tdx_sept_free_private_spt;
+
+	if (tdx_supports_dynamic_pamt(tdx_sysinfo))
+		vt_x86_ops.topup_external_cache = tdx_topup_external_pamt_cache;
+
 	vt_x86_ops.protected_apic_has_interrupt = tdx_protected_apic_has_interrupt;
 	return 0;
 
diff --git a/arch/x86/kvm/vmx/tdx.h b/arch/x86/kvm/vmx/tdx.h
index ac8323a68b163..fd368e3ee060b 100644
--- a/arch/x86/kvm/vmx/tdx.h
+++ b/arch/x86/kvm/vmx/tdx.h
@@ -72,6 +72,8 @@ struct vcpu_tdx {
 
 	u64 map_gpa_next;
 	u64 map_gpa_end;
+
+	struct tdx_pamt_cache pamt_cache;
 };
 
 void tdh_vp_rd_failed(struct vcpu_tdx *tdx, char *uclass, u32 field, u64 err);
diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c
index 42df8ea464c47..dec45709e454a 100644
--- a/arch/x86/virt/vmx/tdx/tdx.c
+++ b/arch/x86/virt/vmx/tdx/tdx.c
@@ -30,6 +30,7 @@
 #include <linux/suspend.h>
 #include <linux/syscore_ops.h>
 #include <linux/idr.h>
+#include <linux/vmalloc.h>
 #include <asm/page.h>
 #include <asm/special_insns.h>
 #include <asm/msr-index.h>
@@ -46,6 +47,8 @@
 #include "seamcall_internal.h"
 #include "tdx.h"
 
+static bool tdx_enable_dpamt __ro_after_init;
+
 struct tdx_module_state {
 	bool initialized;
 	bool sysinit_done;
@@ -63,6 +66,14 @@ static DEFINE_PER_CPU(bool, tdx_lp_initialized);
 
 static struct tdmr_info_list tdx_tdmr_list;
 
+/*
+ * On a machine with Dynamic PAMT, the kernel maintains a reference counter
+ * for every 2MB range. The counter indicates how many users there are for
+ * the PAMT memory of the 2MB range. The kernel allocates PAMT refcounts at
+ * initialization.
+ */
+static atomic_t *pamt_refcounts;
+
 /* All TDX-usable memory regions.  Protected by mem_hotplug_lock. */
 static LIST_HEAD(tdx_memlist);
 
@@ -252,6 +263,42 @@ static struct syscore tdx_syscore = {
 	.ops = &tdx_syscore_ops,
 };
 
+/*
+ * Allocate PAMT reference counters for all physical memory.
+ *
+ * It consumes 2MB for every 1TB of physical memory.
+ */
+static __init int init_pamt_refcounts(void)
+{
+	size_t size = DIV_ROUND_UP(max_pfn, PTRS_PER_PTE) * sizeof(*pamt_refcounts);
+
+	if (!tdx_supports_dynamic_pamt(&tdx_sysinfo))
+		return 0;
+
+	pamt_refcounts = vzalloc(size);
+	if (!pamt_refcounts)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static __init void free_pamt_refcounts(void)
+{
+	if (!tdx_supports_dynamic_pamt(&tdx_sysinfo))
+		return;
+
+	vfree(pamt_refcounts);
+	pamt_refcounts = NULL;
+}
+
+static atomic_t *tdx_find_pamt_refcount(unsigned long pfn)
+{
+	/* Find which PMD a PFN is in. */
+	unsigned long index = pfn >> (PMD_SHIFT - PAGE_SHIFT);
+
+	return &pamt_refcounts[index];
+}
+
 /*
  * Add a memory region as a TDX memory block.  The caller must make sure
  * all memory regions are added in address ascending order and don't
@@ -510,35 +557,37 @@ static __init int fill_out_tdmrs(struct list_head *tmb_list,
 	return 0;
 }
 
+static __init unsigned long tdmr_get_pamt_bitmap_sz(struct tdmr_info *tdmr)
+{
+	unsigned long pamt_sz, nr_pamt_entries;
+	int bits_per_entry;
+
+	bits_per_entry = tdx_sysinfo.tdmr.pamt_page_bitmap_entry_bits;
+	nr_pamt_entries = tdmr->size >> PAGE_SHIFT;
+	pamt_sz = DIV_ROUND_UP(nr_pamt_entries * bits_per_entry, BITS_PER_BYTE);
+
+	return PAGE_ALIGN(pamt_sz);
+}
+
 /*
  * Calculate PAMT size given a TDMR and a page size.  The returned
  * PAMT size is always aligned up to 4K page boundary.
  */
-static __init unsigned long tdmr_get_pamt_sz(struct tdmr_info *tdmr, int pgsz,
-					     u16 pamt_entry_size)
+static __init unsigned long tdmr_get_pamt_sz(struct tdmr_info *tdmr, int pgsz)
 {
 	unsigned long pamt_sz, nr_pamt_entries;
+	const int tdx_pg_size_shift[TDX_PS_NR] = { PAGE_SHIFT, PMD_SHIFT, PUD_SHIFT };
+	const u16 pamt_entry_size[TDX_PS_NR] = {
+		tdx_sysinfo.tdmr.pamt_4k_entry_size,
+		tdx_sysinfo.tdmr.pamt_2m_entry_size,
+		tdx_sysinfo.tdmr.pamt_1g_entry_size,
+	};
 
-	switch (pgsz) {
-	case TDX_PS_4K:
-		nr_pamt_entries = tdmr->size >> PAGE_SHIFT;
-		break;
-	case TDX_PS_2M:
-		nr_pamt_entries = tdmr->size >> PMD_SHIFT;
-		break;
-	case TDX_PS_1G:
-		nr_pamt_entries = tdmr->size >> PUD_SHIFT;
-		break;
-	default:
-		WARN_ON_ONCE(1);
-		return 0;
-	}
+	nr_pamt_entries = tdmr->size >> tdx_pg_size_shift[pgsz];
+	pamt_sz = nr_pamt_entries * pamt_entry_size[pgsz];
 
-	pamt_sz = nr_pamt_entries * pamt_entry_size;
 	/* TDX requires PAMT size must be 4K aligned */
-	pamt_sz = ALIGN(pamt_sz, PAGE_SIZE);
-
-	return pamt_sz;
+	return PAGE_ALIGN(pamt_sz);
 }
 
 /*
@@ -576,15 +625,11 @@ static __init int tdmr_get_nid(struct tdmr_info *tdmr, struct list_head *tmb_lis
  * within @tdmr, and set up PAMTs for @tdmr.
  */
 static __init int tdmr_set_up_pamt(struct tdmr_info *tdmr,
-				   struct list_head *tmb_list,
-				   u16 pamt_entry_size[])
+				   struct list_head *tmb_list)
 {
-	unsigned long pamt_base[TDX_PS_NR];
-	unsigned long pamt_size[TDX_PS_NR];
-	unsigned long tdmr_pamt_base;
 	unsigned long tdmr_pamt_size;
 	struct page *pamt;
-	int pgsz, nid;
+	int nid;
 
 	nid = tdmr_get_nid(tdmr, tmb_list);
 
@@ -592,13 +637,18 @@ static __init int tdmr_set_up_pamt(struct tdmr_info *tdmr,
 	 * Calculate the PAMT size for each TDX supported page size
 	 * and the total PAMT size.
 	 */
-	tdmr_pamt_size = 0;
-	for (pgsz = TDX_PS_4K; pgsz < TDX_PS_NR; pgsz++) {
-		pamt_size[pgsz] = tdmr_get_pamt_sz(tdmr, pgsz,
-					pamt_entry_size[pgsz]);
-		tdmr_pamt_size += pamt_size[pgsz];
+	tdmr->pamt_1g_size = tdmr_get_pamt_sz(tdmr, TDX_PS_1G);
+	tdmr->pamt_2m_size = tdmr_get_pamt_sz(tdmr, TDX_PS_2M);
+
+	if (tdx_supports_dynamic_pamt(&tdx_sysinfo)) {
+		/* With Dynamic PAMT, PAMT_4K is replaced with a bitmap */
+		tdmr->pamt_4k_size = tdmr_get_pamt_bitmap_sz(tdmr);
+	} else {
+		tdmr->pamt_4k_size = tdmr_get_pamt_sz(tdmr, TDX_PS_4K);
 	}
 
+	tdmr_pamt_size = tdmr->pamt_4k_size + tdmr->pamt_2m_size + tdmr->pamt_1g_size;
+
 	/*
 	 * Allocate one chunk of physically contiguous memory for all
 	 * PAMTs.  This helps minimize the PAMT's use of reserved areas
@@ -606,25 +656,17 @@ static __init int tdmr_set_up_pamt(struct tdmr_info *tdmr,
 	 */
 	pamt = alloc_contig_pages(tdmr_pamt_size >> PAGE_SHIFT, GFP_KERNEL,
 			nid, &node_online_map);
-	if (!pamt)
-		return -ENOMEM;
 
 	/*
-	 * Break the contiguous allocation back up into the
-	 * individual PAMTs for each page size.
+	 * tdmr->pamt_4k_base is still zero so the error
+	 * path of the caller will skip freeing the PAMT.
 	 */
-	tdmr_pamt_base = page_to_pfn(pamt) << PAGE_SHIFT;
-	for (pgsz = TDX_PS_4K; pgsz < TDX_PS_NR; pgsz++) {
-		pamt_base[pgsz] = tdmr_pamt_base;
-		tdmr_pamt_base += pamt_size[pgsz];
-	}
+	if (!pamt)
+		return -ENOMEM;
 
-	tdmr->pamt_4k_base = pamt_base[TDX_PS_4K];
-	tdmr->pamt_4k_size = pamt_size[TDX_PS_4K];
-	tdmr->pamt_2m_base = pamt_base[TDX_PS_2M];
-	tdmr->pamt_2m_size = pamt_size[TDX_PS_2M];
-	tdmr->pamt_1g_base = pamt_base[TDX_PS_1G];
-	tdmr->pamt_1g_size = pamt_size[TDX_PS_1G];
+	tdmr->pamt_4k_base = page_to_phys(pamt);
+	tdmr->pamt_2m_base = tdmr->pamt_4k_base + tdmr->pamt_4k_size;
+	tdmr->pamt_1g_base = tdmr->pamt_2m_base + tdmr->pamt_2m_size;
 
 	return 0;
 }
@@ -655,10 +697,7 @@ static __init void tdmr_do_pamt_func(struct tdmr_info *tdmr,
 	tdmr_get_pamt(tdmr, &pamt_base, &pamt_size);
 
 	/* Do nothing if PAMT hasn't been allocated for this TDMR */
-	if (!pamt_size)
-		return;
-
-	if (WARN_ON_ONCE(!pamt_base))
+	if (!pamt_base)
 		return;
 
 	pamt_func(pamt_base, pamt_size);
@@ -684,14 +723,12 @@ static __init void tdmrs_free_pamt_all(struct tdmr_info_list *tdmr_list)
 
 /* Allocate and set up PAMTs for all TDMRs */
 static __init int tdmrs_set_up_pamt_all(struct tdmr_info_list *tdmr_list,
-					struct list_head *tmb_list,
-					u16 pamt_entry_size[])
+				 struct list_head *tmb_list)
 {
 	int i, ret = 0;
 
 	for (i = 0; i < tdmr_list->nr_consumed_tdmrs; i++) {
-		ret = tdmr_set_up_pamt(tdmr_entry(tdmr_list, i), tmb_list,
-				pamt_entry_size);
+		ret = tdmr_set_up_pamt(tdmr_entry(tdmr_list, i), tmb_list);
 		if (ret)
 			goto err;
 	}
@@ -968,18 +1005,13 @@ static __init int construct_tdmrs(struct list_head *tmb_list,
 				  struct tdmr_info_list *tdmr_list,
 				  struct tdx_sys_info_tdmr *sysinfo_tdmr)
 {
-	u16 pamt_entry_size[TDX_PS_NR] = {
-		sysinfo_tdmr->pamt_4k_entry_size,
-		sysinfo_tdmr->pamt_2m_entry_size,
-		sysinfo_tdmr->pamt_1g_entry_size,
-	};
 	int ret;
 
 	ret = fill_out_tdmrs(tmb_list, tdmr_list);
 	if (ret)
 		return ret;
 
-	ret = tdmrs_set_up_pamt_all(tdmr_list, tmb_list, pamt_entry_size);
+	ret = tdmrs_set_up_pamt_all(tdmr_list, tmb_list);
 	if (ret)
 		return ret;
 
@@ -998,6 +1030,8 @@ static __init int construct_tdmrs(struct list_head *tmb_list,
 	return ret;
 }
 
+#define TDX_SYS_CONFIG_DYNAMIC_PAMT	BIT(16)
+
 static __init int config_tdx_module(struct tdmr_info_list *tdmr_list,
 				    u64 global_keyid)
 {
@@ -1026,6 +1060,12 @@ static __init int config_tdx_module(struct tdmr_info_list *tdmr_list,
 	args.rcx = __pa(tdmr_pa_array);
 	args.rdx = tdmr_list->nr_consumed_tdmrs;
 	args.r8 = global_keyid;
+
+	if (tdx_supports_dynamic_pamt(&tdx_sysinfo)) {
+		pr_info("Enable Dynamic PAMT\n");
+		args.r8 |= TDX_SYS_CONFIG_DYNAMIC_PAMT;
+	}
+
 	ret = seamcall_prerr(TDH_SYS_CONFIG, &args);
 
 	/* Free the array as it is not required anymore. */
@@ -1167,10 +1207,14 @@ static __init int init_tdx_module(void)
 	 */
 	get_online_mems();
 
-	ret = build_tdx_memlist(&tdx_memlist);
+	ret = init_pamt_refcounts();
 	if (ret)
 		goto out_put_tdxmem;
 
+	ret = build_tdx_memlist(&tdx_memlist);
+	if (ret)
+		goto err_free_pamt_refcounts;
+
 	/* Allocate enough space for constructing TDMRs */
 	ret = alloc_tdmr_list(&tdx_tdmr_list, &tdx_sysinfo.tdmr);
 	if (ret)
@@ -1220,6 +1264,8 @@ static __init int init_tdx_module(void)
 	free_tdmr_list(&tdx_tdmr_list);
 err_free_tdxmem:
 	free_tdx_memlist(&tdx_memlist);
+err_free_pamt_refcounts:
+	free_pamt_refcounts();
 	goto out_put_tdxmem;
 }
 
@@ -2003,6 +2049,288 @@ u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, kvm_pfn_t pfn)
 }
 EXPORT_SYMBOL_FOR_KVM(tdh_phymem_page_wbinvd_hkid);
 
+bool tdx_supports_dynamic_pamt(const struct tdx_sys_info *sysinfo)
+{
+	return sysinfo->features.tdx_features0 & TDX_FEATURES0_DYNAMIC_PAMT &&
+	       tdx_enable_dpamt;
+}
+EXPORT_SYMBOL_FOR_KVM(tdx_supports_dynamic_pamt);
+
+static struct page *tdx_alloc_page_pamt_cache(struct tdx_pamt_cache *cache)
+{
+	struct page *page;
+
+	page = list_first_entry_or_null(&cache->page_list, struct page, lru);
+	if (page) {
+		list_del(&page->lru);
+		cache->cnt--;
+	}
+
+	return page;
+}
+
+static struct page *alloc_dpamt_page(struct tdx_pamt_cache *cache)
+{
+	if (cache)
+		return tdx_alloc_page_pamt_cache(cache);
+
+	return alloc_page(GFP_KERNEL_ACCOUNT);
+}
+
+static int alloc_pamt_array(struct page **pamt_pages, struct tdx_pamt_cache *cache)
+{
+	int i, j;
+
+	for (i = 0; i < TDX_DPAMT_ENTRY_PAGE_CNT; i++) {
+		pamt_pages[i] = alloc_dpamt_page(cache);
+		if (!pamt_pages[i])
+			goto err;
+	}
+
+	return 0;
+
+err:
+	for (j = 0; j < i; j++)
+		__free_page(pamt_pages[j]);
+
+	return -ENOMEM;
+}
+
+static void free_pamt_array(struct page **pamt_pages)
+{
+	for (int i = 0; i < TDX_DPAMT_ENTRY_PAGE_CNT; i++) {
+		/*
+		 * Reset pages unconditionally to cover cases
+		 * where they were passed to the TDX module.
+		 */
+		tdx_quirk_reset_paddr(page_to_phys(pamt_pages[i]), PAGE_SIZE);
+
+		__free_page(pamt_pages[i]);
+	}
+}
+
+/*
+ * Calculate the arg needed for operating on the DPAMT backing for
+ * a given 4KB page.
+ */
+static u64 pamt_2mb_arg(kvm_pfn_t pfn)
+{
+	/* Arg value will specify a 2MB region of physical address space. */
+	unsigned long hpa_2mb = ALIGN_DOWN(pfn << PAGE_SHIFT, PMD_SIZE);
+
+	return hpa_2mb | TDX_PS_2M;
+}
+
+/* Add PAMT backing for the 2MB region surrounding the given pfn. */
+static u64 tdh_phymem_pamt_add(kvm_pfn_t pfn, struct page **pamt_pages)
+{
+	struct tdx_module_args args = {
+		.rcx = pamt_2mb_arg(pfn),
+		.rdx = page_to_phys(pamt_pages[0]),
+		.r8 = page_to_phys(pamt_pages[1]),
+	};
+
+	return seamcall(TDH_PHYMEM_PAMT_ADD, &args);
+}
+
+/* Remove PAMT backing for the 2MB region surrounding the given pfn. */
+static u64 tdh_phymem_pamt_remove(kvm_pfn_t pfn, struct page **pamt_pages)
+{
+	struct tdx_module_args args = {
+		.rcx = pamt_2mb_arg(pfn),
+	};
+	u64 ret;
+
+	ret = seamcall_ret(TDH_PHYMEM_PAMT_REMOVE, &args);
+	if (ret)
+		return ret;
+
+	/* Copy PAMT pages out of the struct per the TDX ABI */
+	pamt_pages[0] = phys_to_page(args.rdx);
+	pamt_pages[1] = phys_to_page(args.r8);
+
+	return 0;
+}
+
+/* Serializes adding/removing PAMT memory */
+static DEFINE_SPINLOCK(pamt_lock);
+
+/* Bump PAMT refcount for the given pfn and allocate PAMT backing if needed. */
+int tdx_pamt_get(kvm_pfn_t pfn, struct tdx_pamt_cache *cache)
+{
+	struct page *pamt_pages[TDX_DPAMT_ENTRY_PAGE_CNT];
+	atomic_t *pamt_refcount;
+	u64 tdx_status;
+	int ret;
+
+	if (!tdx_supports_dynamic_pamt(&tdx_sysinfo))
+		return 0;
+
+	pamt_refcount = tdx_find_pamt_refcount(pfn);
+
+	/*
+	 * If the pamt page is already added (i.e. refcount >= 1),
+	 * then just increment the refcount.
+	 */
+	if (atomic_inc_not_zero(pamt_refcount))
+		return 0;
+
+	ret = alloc_pamt_array(pamt_pages, cache);
+	if (ret)
+		return ret;
+
+	spin_lock(&pamt_lock);
+
+	/*
+	 * Unlike tdx_pamt_put() which uses atomic_dec_and_lock() to
+	 * atomically handle the 1->0 transition, the get side has no
+	 * equivalent combined primitive for 0->1. Recheck under the
+	 * lock since another get may have already done the 0->1
+	 * transition after both saw atomic_inc_not_zero() fail.
+	 */
+	if (atomic_inc_not_zero(pamt_refcount))
+		goto out_free;
+
+	tdx_status = tdh_phymem_pamt_add(pfn, pamt_pages);
+	if (WARN_ON_ONCE(tdx_status != TDX_SUCCESS)) {
+		ret = -EIO;
+		goto out_free;
+	}
+
+	/*
+	 * The refcount is zero, and this locked path is the
+	 * only way to increase it from 0->1.
+	 */
+	atomic_set(pamt_refcount, 1);
+	spin_unlock(&pamt_lock);
+	return 0;
+
+out_free:
+	spin_unlock(&pamt_lock);
+	free_pamt_array(pamt_pages);
+
+	return ret;
+}
+EXPORT_SYMBOL_FOR_KVM(tdx_pamt_get);
+
+/* Drop PAMT refcount for the given pfn and free PAMT backing if needed. */
+void tdx_pamt_put(kvm_pfn_t pfn)
+{
+	struct page *pamt_pages[TDX_DPAMT_ENTRY_PAGE_CNT] = {};
+	atomic_t *pamt_refcount;
+	u64 tdx_status;
+
+	if (!tdx_supports_dynamic_pamt(&tdx_sysinfo))
+		return;
+
+	pamt_refcount = tdx_find_pamt_refcount(pfn);
+
+	/*
+	 * If there is more than 1 reference on the pamt page, don't
+	 * remove it yet. Just decrement the refcount.
+	 */
+	if (!atomic_dec_and_lock(pamt_refcount, &pamt_lock))
+		return;
+
+	tdx_status = tdh_phymem_pamt_remove(pfn, pamt_pages);
+
+	/*
+	 * Don't free pamt_pages as it could hold garbage when
+	 * tdh_phymem_pamt_remove() fails.  Don't panic/BUG_ON(), as
+	 * there is no risk of data corruption, but do yell loudly as
+	 * failure indicates a kernel bug, memory is being leaked, and
+	 * the dangling PAMT entry may cause future operations to fail.
+	 */
+	if (WARN_ON_ONCE(tdx_status != TDX_SUCCESS)) {
+		/*
+		 * atomic_dec_and_lock() already decremented it to 0,
+		 * but the PAMT entry still exists since REMOVE failed.
+		 */
+		atomic_set(pamt_refcount, 1);
+		goto out_unlock;
+	}
+
+	spin_unlock(&pamt_lock);
+	free_pamt_array(pamt_pages);
+	return;
+out_unlock:
+	spin_unlock(&pamt_lock);
+}
+EXPORT_SYMBOL_FOR_KVM(tdx_pamt_put);
+
+void tdx_free_pamt_cache(struct tdx_pamt_cache *cache)
+{
+	struct page *page;
+
+	while ((page = tdx_alloc_page_pamt_cache(cache)))
+		__free_page(page);
+}
+EXPORT_SYMBOL_FOR_KVM(tdx_free_pamt_cache);
+
+int tdx_topup_pamt_cache(struct tdx_pamt_cache *cache, unsigned long npages)
+{
+	if (WARN_ON_ONCE(!tdx_supports_dynamic_pamt(&tdx_sysinfo)))
+		return 0;
+
+	npages *= TDX_DPAMT_ENTRY_PAGE_CNT;
+
+	while (cache->cnt < npages) {
+		struct page *page = alloc_page(GFP_KERNEL_ACCOUNT);
+
+		if (!page)
+			return -ENOMEM;
+
+		list_add(&page->lru, &cache->page_list);
+		cache->cnt++;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_FOR_KVM(tdx_topup_pamt_cache);
+
+/*
+ * Return a page that can be gifted to the TDX-Module for use as a "control"
+ * page, i.e. pages that are used for control structures for a given TDX
+ * guest, and thus obtain TDX protections, including PAMT tracking.
+ */
+struct page *tdx_alloc_control_page(void)
+{
+	struct page *page;
+
+	page = alloc_page(GFP_KERNEL_ACCOUNT);
+	if (!page)
+		return NULL;
+
+	if (tdx_pamt_get(page_to_pfn(page), NULL)) {
+		__free_page(page);
+		return NULL;
+	}
+
+	return page;
+}
+EXPORT_SYMBOL_FOR_KVM(tdx_alloc_control_page);
+
+/*
+ * Free a page that was gifted to the TDX-Module for use as a control
+ * page. After this, the page is no longer protected by TDX.
+ */
+void tdx_free_control_page(struct page *page)
+{
+	if (!page)
+		return;
+
+	tdx_pamt_put(page_to_pfn(page));
+	__free_page(page);
+}
+EXPORT_SYMBOL_FOR_KVM(tdx_free_control_page);
+
+static int __init tdx_dpamt_setup(char *str)
+{
+	return kstrtobool(str, &tdx_enable_dpamt) == 0;
+}
+
+__setup("tdx_dpamt=", tdx_dpamt_setup);
+
 void tdx_sys_disable(void)
 {
 	struct tdx_module_args args = {};
diff --git a/arch/x86/virt/vmx/tdx/tdx.h b/arch/x86/virt/vmx/tdx/tdx.h
index bdfd0e1e337ac..a886c54decaad 100644
--- a/arch/x86/virt/vmx/tdx/tdx.h
+++ b/arch/x86/virt/vmx/tdx/tdx.h
@@ -48,6 +48,8 @@
 #define TDH_SYS_CONFIG			45
 #define TDH_SYS_SHUTDOWN		52
 #define TDH_SYS_UPDATE			53
+#define TDH_PHYMEM_PAMT_ADD		58
+#define TDH_PHYMEM_PAMT_REMOVE		59
 #define TDH_SYS_DISABLE			69
 
 /*
diff --git a/arch/x86/virt/vmx/tdx/tdx_global_metadata.c b/arch/x86/virt/vmx/tdx/tdx_global_metadata.c
index e49c300f23d43..93b1f26252c63 100644
--- a/arch/x86/virt/vmx/tdx/tdx_global_metadata.c
+++ b/arch/x86/virt/vmx/tdx/tdx_global_metadata.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * Automatically generated functions to read TDX global metadata.
+ * Functions to read TDX global metadata.
  *
  * This file doesn't compile on its own as it lacks of inclusion
  * of SEAMCALL wrapper primitive which reads global metadata.
@@ -33,6 +33,18 @@ static __init int get_tdx_sys_info_features(struct tdx_sys_info_features *sysinf
 	return ret;
 }
 
+static __init int get_tdx_sys_info_tdmr_dpamt(struct tdx_sys_info_tdmr *sysinfo_tdmr)
+{
+	int ret;
+	u64 val;
+
+	ret = read_sys_metadata_field(0x9100000000000013, &val);
+	if (!ret)
+		sysinfo_tdmr->pamt_page_bitmap_entry_bits = val;
+
+	return ret;
+}
+
 static __init int get_tdx_sys_info_tdmr(struct tdx_sys_info_tdmr *sysinfo_tdmr)
 {
 	int ret = 0;
@@ -129,5 +141,14 @@ static __init int get_tdx_sys_info(struct tdx_sys_info *sysinfo)
 	ret = ret ?: get_tdx_sys_info_td_ctrl(&sysinfo->td_ctrl);
 	ret = ret ?: get_tdx_sys_info_td_conf(&sysinfo->td_conf);
 
+	/*
+	 * The kernel supports using TDX without Dynamic PAMT, so
+	 * avoid reporting failure if it's not supported. Don't try
+	 * to support buggy TDX modules that advertise Dynamic PAMT
+	 * but don't expose the metadata.
+	 */
+	if (!ret && tdx_supports_dynamic_pamt(sysinfo))
+		ret = get_tdx_sys_info_tdmr_dpamt(&sysinfo->tdmr);
+
 	return ret;
 }