Decrypted per-CPU data is gated by AMD_MEM_ENCRYPT. With INTEL_TDX_GUEST=y and AMD_MEM_ENCRYPT=n it falls back to ordinary per-CPU storage, so converting a variable to shared memory can expose unrelated data on the same page. Gate the declarations and linker section on X86_MEM_ENCRYPT, which both AMD_MEM_ENCRYPT and INTEL_TDX_GUEST select. Existing AMD configurations are unchanged. Keep the decrypted subsection page-aligned at both ends. The per-CPU template and each allocated per-CPU unit are also page-aligned, so page conversion cannot expose ordinary per-CPU data. TDX-only configurations may need additional space for this alignment and isolation. Signed-off-by: Zack Rusin --- include/asm-generic/vmlinux.lds.h | 2 +- include/linux/percpu-defs.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index b2988aa12f66..1448e791773e 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -1013,7 +1013,7 @@ * Note: We use a separate section so that only this section gets * decrypted to avoid exposing more than we wish. */ -#ifdef CONFIG_AMD_MEM_ENCRYPT +#ifdef CONFIG_X86_MEM_ENCRYPT #define PERCPU_DECRYPTED_SECTION \ . = ALIGN(PAGE_SIZE); \ *(.data..percpu..decrypted) \ diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h index dbe3267a0a13..54ce1442d918 100644 --- a/include/linux/percpu-defs.h +++ b/include/linux/percpu-defs.h @@ -172,7 +172,7 @@ * Declaration/definition used for per-CPU variables that should be accessed * as decrypted when memory encryption is enabled in the guest. */ -#ifdef CONFIG_AMD_MEM_ENCRYPT +#ifdef CONFIG_X86_MEM_ENCRYPT #define DECLARE_PER_CPU_DECRYPTED(type, name) \ DECLARE_PER_CPU_SECTION(type, name, "..decrypted") -- 2.53.0 VMware's steal-time counter must be in shared memory so the host can update it. Encrypted guests currently register its address without decrypting the storage. Move their setup to an early initcall, when the memory allocator is available for page-table splitting, but before secondary CPUs start. Leave ordinary guests' setup unchanged. Convert every possible CPU's storage before registering any address. Zero each object after conversion, since its contents may not survive conversion and the host initializes only the counter. Register the boot CPU with preemption disabled; the existing hotplug callbacks handle the others. If conversion or boot-CPU registration fails, attempt to re-encrypt all affected ranges, including a partially converted failing range, and disable steal time. TDX's conversion path requires directly mapped memory. Check all per-CPU objects before converting anything, and disable steal time if a TDX guest uses the page per-CPU allocator, which supplies vmalloc mappings. This also covers automatic fallback from the embedded allocator. AMD guests are unaffected by this restriction because their conversion path supports those mappings. Link: https://lore.kernel.org/r/20260309235250.2611115-5-alexey.makhalov@broadcom.com Co-developed-by: Bo Gan Signed-off-by: Bo Gan Co-developed-by: Alexey Makhalov Signed-off-by: Alexey Makhalov Signed-off-by: Zack Rusin --- arch/x86/kernel/cpu/vmware.c | 96 +++++++++++++++++++++++++++++++++++- 1 file changed, 95 insertions(+), 1 deletion(-) diff --git a/arch/x86/kernel/cpu/vmware.c b/arch/x86/kernel/cpu/vmware.c index 34b73573b108..f49898275d87 100644 --- a/arch/x86/kernel/cpu/vmware.c +++ b/arch/x86/kernel/cpu/vmware.c @@ -25,14 +25,19 @@ #include #include #include +#include #include #include +#include +#include #include +#include #include #include #include #include #include +#include #include #include #include @@ -147,6 +152,7 @@ static struct cyc2ns_data vmware_cyc2ns __ro_after_init; static bool vmw_sched_clock __initdata = true; static DEFINE_PER_CPU_DECRYPTED(struct vmware_steal_time, vmw_steal_time) __aligned(64); static bool has_steal_clock; +static bool vmw_steal_time_ready; static bool steal_acc __initdata = true; /* steal time accounting */ static __init int setup_vmw_sched_clock(char *s) @@ -280,7 +286,9 @@ static void vmware_disable_steal_time(void) static void vmware_guest_cpu_init(void) { - if (has_steal_clock) + if (has_steal_clock && + (!cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT) || + vmw_steal_time_ready)) vmware_register_steal_time(); } @@ -325,6 +333,92 @@ static int vmware_cpu_down_prepare(unsigned int cpu) } #endif +static void __init vmware_steal_time_range(int cpu, unsigned long *start, + int *numpages) +{ + unsigned long addr = (unsigned long)per_cpu_ptr(&vmw_steal_time, cpu); + + *start = addr & PAGE_MASK; + *numpages = DIV_ROUND_UP(offset_in_page(addr) + + sizeof(struct vmware_steal_time), PAGE_SIZE); +} + +static int __init vmware_decrypt_steal_time(void) +{ + int cpu, failed_cpu, numpages, ret; + unsigned long start; + + if (!has_steal_clock || + !cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) + return 0; + + /* + * TDX's conversion callback derives the physical range with __pa(), + * so the storage must live in the direct map. The page per-CPU + * allocator, which is also the automatic fallback when the embedding + * allocator fails, hands out vmalloc addresses instead. Reject those + * before converting anything, so no page is left converted. + */ + if (cpu_feature_enabled(X86_FEATURE_TDX_GUEST)) { + for_each_possible_cpu(cpu) { + if (!is_vmalloc_addr(per_cpu_ptr(&vmw_steal_time, cpu))) + continue; + + pr_warn("steal time disabled: TDX requires a direct mapping\n"); + has_steal_clock = false; + return 0; + } + } + + for_each_possible_cpu(cpu) { + vmware_steal_time_range(cpu, &start, &numpages); + ret = set_memory_decrypted(start, numpages); + if (ret) { + failed_cpu = cpu; + goto rollback; + } + + /* + * Conversion need not preserve the zeroes. The host + * initializes only the counter on enable, so the guest + * must initialize the reserved words itself. + */ + memset(per_cpu_ptr(&vmw_steal_time, cpu), 0, + sizeof(struct vmware_steal_time)); + } + + vmw_steal_time_ready = true; + preempt_disable(); + vmware_guest_cpu_init(); + preempt_enable(); + if (!has_steal_clock) { + pr_warn("failed to register boot CPU steal-time memory\n"); + failed_cpu = nr_cpu_ids; + goto rollback_pages; + } + return 0; + +rollback: + pr_warn("failed to decrypt steal-time memory for CPU %d: %d\n", + failed_cpu, ret); + +rollback_pages: + for_each_possible_cpu(cpu) { + vmware_steal_time_range(cpu, &start, &numpages); + ret = set_memory_encrypted(start, numpages); + if (ret) + pr_warn("failed to re-encrypt steal-time memory for CPU %d: %d\n", + cpu, ret); + if (cpu == failed_cpu) + break; + } + + vmw_steal_time_ready = false; + has_steal_clock = false; + return 0; +} +early_initcall(vmware_decrypt_steal_time); + static __init int activate_jump_labels(void) { if (has_steal_clock) { -- 2.53.0