From: Ashish Kalra When SNP is enabled, all writes to memory are checked to ensure memory integrity. This imposes performance overhead on the whole system. RMPOPT is a new instruction that minimizes the performance overhead of RMP checks on the hypervisor and on non-SNP guests by allowing RMP checks to be skipped for 1GB regions of memory that are known not to contain any SNP guest memory. Add support for performing RMP optimizations asynchronously using a dedicated workqueue. At RMP initialization time, run an optimization pass over all physical memory (up to 2TB of system RAM, starting from the lowest physical memory address aligned down to a 1GB boundary), skipping RMP checks for 1GB regions that do not contain SNP guest memory (excluding preassigned pages such as the RMP table and firmware pages). As SNP guests are launched, RMPUPDATE assigns their private pages to guest-owned state; when such a page falls within an optimized 1GB region, the hardware clears that region's RMPOPT optimization and RMP checks resume there to protect the guest memory. Since launching SNP guests clears these optimizations, perform them again asynchronously using the dedicated workqueue. Suggested-by: Thomas Lendacky Suggested-by: Dave Hansen Suggested-by: K Prateek Nayak Suggested-by: Borislav Petkov (AMD) Reviewed-by: Ackerley Tng Reviewed-by: Tom Lendacky Signed-off-by: Ashish Kalra --- arch/x86/virt/svm/sev.c | 113 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 111 insertions(+), 2 deletions(-) diff --git a/arch/x86/virt/svm/sev.c b/arch/x86/virt/svm/sev.c index 07f178eb76c7..a4c0fe49b9ec 100644 --- a/arch/x86/virt/svm/sev.c +++ b/arch/x86/virt/svm/sev.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -124,7 +125,16 @@ static void *rmp_bookkeeping __ro_after_init; static u64 probed_rmp_base, probed_rmp_size; -static phys_addr_t rmpopt_pa_start; +static u64 rmpopt_pa_start, rmpopt_pa_end; + +enum rmpopt_op_type { + RMPOPT_OP_VERIFY_AND_REPORT_STATUS, + RMPOPT_OP_REPORT_STATUS +}; + +static struct workqueue_struct *rmpopt_wq; +static struct delayed_work rmpopt_delayed_work; +static DEFINE_MUTEX(rmpopt_wq_mutex); static LIST_HEAD(snp_leaked_pages_list); static DEFINE_SPINLOCK(snp_leaked_pages_list_lock); @@ -561,10 +571,26 @@ static void rmpopt_disable(void) { int cpu; + guard(mutex)(&rmpopt_wq_mutex); + + /* + * rmpopt_wq is non-NULL only after RMPOPT has been fully set up: the + * workqueue is allocated and the RMPOPT_BASE MSRs are programmed. + * snp_setup_rmpopt() resets it to NULL if any of those steps fail, so a + * NULL rmpopt_wq means nothing was set up and there is nothing to tear + * down. + */ + if (!rmpopt_wq) + return; + + cancel_delayed_work_sync(&rmpopt_delayed_work); + destroy_workqueue(rmpopt_wq); + for_each_cpu(cpu, cpu_primary_thread_mask) wrmsrq_on_cpu(cpu, MSR_AMD64_RMPOPT_BASE, 0); - rmpopt_pa_start = 0; + rmpopt_pa_start = rmpopt_pa_end = 0; + rmpopt_wq = NULL; } void snp_shutdown(void) @@ -595,6 +621,44 @@ static bool rmpopt_capable(void) cc_platform_has(CC_ATTR_HOST_SEV_SNP); } +/* + * RMPOPT optimizations skip RMP checks at 1GB granularity if this range of + * memory does not contain any SNP guest memory. + * + * @pa is a system physical address; RMPOPT operates on the containing 1GB. + */ +static void rmpopt(u64 pa) +{ + enum rmpopt_op_type op = RMPOPT_OP_VERIFY_AND_REPORT_STATUS; + u64 pa_start = ALIGN_DOWN(pa, SZ_1G); + + asm volatile(".byte 0xf2, 0x0f, 0x01, 0xfc" + :: "a" (pa_start), "c" (op) + : "memory", "cc"); +} + +/* on_each_cpu() callback: optimize the whole RMPOPT range on this CPU. */ +static void rmpopt_scan_range(void *arg) +{ + u64 pa; + + for (pa = rmpopt_pa_start; pa < rmpopt_pa_end; pa += SZ_1G) + rmpopt(pa); +} + +static void do_rmpopt_work(struct work_struct *work) +{ + /* + * RMPOPT caches the results of a RMP table scan in reserved processor + * memory, allowing future invocations to skip such costly operations. + */ + migrate_disable(); + rmpopt_scan_range(NULL); + migrate_enable(); + + on_each_cpu_mask(cpu_primary_thread_mask, rmpopt_scan_range, NULL, true); +} + void snp_setup_rmpopt(void) { u64 rmpopt_base; @@ -603,6 +667,34 @@ void snp_setup_rmpopt(void) if (!rmpopt_capable()) return; + guard(mutex)(&rmpopt_wq_mutex); + + /* + * On re-initialization after a legacy SNP shutdown (SNP_SHUTDOWN_EX + * with x86_snp_shutdown=0), snp_shutdown() and thus rmpopt_disable() are + * skipped, so the workqueue, delayed work and per-CPU RMPOPT_BASE MSRs + * are still set up and valid (SnpEn stayed set and CPU hotplug stayed + * disabled). Rather than re-doing the setup, which would leak the + * existing state, just re-queue the optimization pass to re-optimize any + * memory the previous SNP session de-optimized. + */ + if (rmpopt_wq) { + queue_delayed_work(rmpopt_wq, &rmpopt_delayed_work, 0); + return; + } + + /* + * Create an RMPOPT-specific workqueue to avoid scheduling + * RMPOPT workitem on the global system workqueue. + */ + rmpopt_wq = alloc_workqueue("rmpopt_wq", WQ_UNBOUND, 1); + if (!rmpopt_wq) { + pr_err("Failed to allocate RMPOPT workqueue\n"); + return; + } + + INIT_DELAYED_WORK(&rmpopt_delayed_work, do_rmpopt_work); + rmpopt_pa_start = ALIGN_DOWN(PFN_PHYS(min_low_pfn), SZ_1G); rmpopt_base = rmpopt_pa_start | MSR_AMD64_RMPOPT_ENABLE; @@ -612,6 +704,23 @@ void snp_setup_rmpopt(void) */ for_each_cpu(cpu, cpu_primary_thread_mask) wrmsrq_on_cpu(cpu, MSR_AMD64_RMPOPT_BASE, rmpopt_base); + + rmpopt_pa_end = ALIGN(PFN_PHYS(max_pfn), SZ_1G); + + /* Limit memory scanning to 2TB of RAM */ + if ((rmpopt_pa_end - rmpopt_pa_start) > SZ_2T) { + pr_info("RMPOPT coverage limited to 2TB; memory above 0x%llx not optimized\n", + rmpopt_pa_start + SZ_2T); + rmpopt_pa_end = rmpopt_pa_start + SZ_2T; + } + + /* + * Once all per-CPU RMPOPT tables have been configured, enable RMPOPT + * optimizations on all physical memory. + */ + queue_delayed_work(rmpopt_wq, &rmpopt_delayed_work, 0); + + pr_info("RMPOPT optimizations enabled\n"); } EXPORT_SYMBOL_FOR_MODULES(snp_setup_rmpopt, "ccp"); -- 2.43.0