From: Ashish Kalra When SEV-SNP is enabled, all writes to memory are checked to ensure integrity of SNP guest memory. This imposes performance overhead on the whole system. RMPOPT is a new instruction that minimizes the performance overhead of RMP checks on the hypervisor and on non-SNP guests by allowing RMP checks to be skipped for 1GB regions of memory that are known not to contain any SEV-SNP guest memory. Add support for performing RMP optimizations asynchronously using a dedicated workqueue. Enable RMPOPT optimizations globally for all system RAM up to 2TB at RMP initialization time. RMP checks can initially be skipped for 1GB memory ranges that do not contain SEV-SNP guest memory (excluding preassigned pages such as the RMP table and firmware pages). As SNP guests are launched, RMPUPDATE will disable the corresponding RMPOPT optimizations. Suggested-by: Thomas Lendacky Suggested-by: Dave Hansen Reviewed-by: Ackerley Tng Signed-off-by: Ashish Kalra --- arch/x86/virt/svm/sev.c | 167 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 164 insertions(+), 3 deletions(-) diff --git a/arch/x86/virt/svm/sev.c b/arch/x86/virt/svm/sev.c index 82f9dc7a57c3..8876cac052d5 100644 --- a/arch/x86/virt/svm/sev.c +++ b/arch/x86/virt/svm/sev.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -125,7 +126,18 @@ static void *rmp_bookkeeping __ro_after_init; static u64 probed_rmp_base, probed_rmp_size; static cpumask_t rmpopt_cpumask; -static phys_addr_t rmpopt_pa_start; +static phys_addr_t rmpopt_pa_start, rmpopt_pa_end; + +enum rmpopt_function { + RMPOPT_FUNC_VERIFY_AND_REPORT_STATUS, + RMPOPT_FUNC_REPORT_STATUS +}; + +#define RMPOPT_WORK_TIMEOUT 10000 + +static struct workqueue_struct *rmpopt_wq; +static struct delayed_work rmpopt_delayed_work; +static DEFINE_MUTEX(rmpopt_wq_mutex); static LIST_HEAD(snp_leaked_pages_list); static DEFINE_SPINLOCK(snp_leaked_pages_list_lock); @@ -564,12 +576,21 @@ EXPORT_SYMBOL_FOR_MODULES(snp_prepare, "ccp"); static void rmpopt_cleanup(void) { + guard(mutex)(&rmpopt_wq_mutex); + + if (!rmpopt_wq) + return; + + cancel_delayed_work_sync(&rmpopt_delayed_work); + destroy_workqueue(rmpopt_wq); + cpus_read_lock(); wrmsrq_on_cpus(&rmpopt_cpumask, MSR_AMD64_RMPOPT_BASE, 0); cpus_read_unlock(); cpumask_clear(&rmpopt_cpumask); - rmpopt_pa_start = 0; + rmpopt_pa_start = rmpopt_pa_end = 0; + rmpopt_wq = NULL; } void snp_shutdown(void) @@ -587,6 +608,105 @@ void snp_shutdown(void) } EXPORT_SYMBOL_FOR_MODULES(snp_shutdown, "ccp"); +static inline bool __rmpopt(u64 rax, u64 rcx) +{ + bool optimized; + + asm volatile(".byte 0xf2, 0x0f, 0x01, 0xfc" + : "=@ccc" (optimized) + : "a" (rax), "c" (rcx) + : "memory", "cc"); + + return optimized; +} + +static void rmpopt(u64 pa) +{ + u64 rax = ALIGN_DOWN(pa, SZ_1G); + u64 rcx = RMPOPT_FUNC_VERIFY_AND_REPORT_STATUS; + + __rmpopt(rax, rcx); +} + +/* + * 'val' is a system physical address. + */ +static void rmpopt_smp(void *val) +{ + rmpopt((u64)val); +} + +/* + * RMPOPT optimizations skip RMP checks at 1GB granularity if this + * range of memory does not contain any SNP guest memory. + */ +static void rmpopt_work_handler(struct work_struct *work) +{ + bool current_cpu_cleared = false; + phys_addr_t pa; + int this_cpu; + + pr_info("Attempt RMP optimizations on physical address range @1GB alignment [0x%016llx - 0x%016llx]\n", + rmpopt_pa_start, rmpopt_pa_end); + + /* + * RMPOPT scans the RMP table, stores the result of the scan in the + * reserved processor memory. The RMP scan is the most expensive + * part. If a second RMPOPT occurs, it can skip the expensive scan + * if they can see a cached result in the reserved processor memory. + * + * Do RMPOPT on one CPU alone. Then, follow that up with RMPOPT + * on every other primary thread. This potentially allows the + * followers to use the "cached" scan results to avoid repeating + * full scans. + */ + + /* + * Pin the worker to the current CPU for the leader loop so that + * this_cpu remains valid and the RMPOPT instruction executes on + * the CPU that was cleared from the cpumask. The workqueue is + * WQ_UNBOUND, so without pinning, the scheduler could migrate + * the worker between the cpumask manipulation and the leader + * loop, causing the leader to run on a different CPU while + * this_cpu's core is skipped entirely. + * + * Use migrate_disable() rather than get_cpu() to prevent + * migration while still allowing preemption. + * + * Note: rmpopt_cpumask is modified here without holding + * rmpopt_wq_mutex. This is safe because the delayed_work + * mechanism guarantees single-threaded execution of this + * handler, and rmpopt_cleanup() calls cancel_delayed_work_sync() + * to ensure handler completion before tearing down the cpumask. + */ + migrate_disable(); + this_cpu = smp_processor_id(); + if (cpumask_test_cpu(this_cpu, &rmpopt_cpumask)) { + cpumask_clear_cpu(this_cpu, &rmpopt_cpumask); + current_cpu_cleared = true; + } + + /* Leader: prime the RMPOPT cache on this CPU */ + for (pa = rmpopt_pa_start; pa < rmpopt_pa_end; pa += SZ_1G) + rmpopt(pa); + + migrate_enable(); + + /* Followers: run RMPOPT on all other cores */ + cpus_read_lock(); + for (pa = rmpopt_pa_start; pa < rmpopt_pa_end; pa += SZ_1G) { + on_each_cpu_mask(&rmpopt_cpumask, rmpopt_smp, + (void *)pa, true); + + /* Give a chance for other threads to run */ + cond_resched(); + } + cpus_read_unlock(); + + if (current_cpu_cleared) + cpumask_set_cpu(this_cpu, &rmpopt_cpumask); +} + void snp_setup_rmpopt(void) { u64 rmpopt_base; @@ -595,11 +715,35 @@ void snp_setup_rmpopt(void) if (!cpu_feature_enabled(X86_FEATURE_RMPOPT)) return; + guard(mutex)(&rmpopt_wq_mutex); + + /* + * Guard against re-initialization. When SNP_SHUTDOWN_EX is issued + * with x86_snp_shutdown=0, snp_shutdown() is not called and + * rmpopt_cleanup() is skipped, but snp_initialized is still cleared. + * A subsequent __sev_snp_init_locked() would call snp_setup_rmpopt() + * again, leaking the existing workqueue, delayed work, debugfs + * entries, and cpumask state. + */ + if (rmpopt_wq) + return; + + /* + * Create an RMPOPT-specific workqueue to avoid scheduling + * RMPOPT workitem on the global system workqueue. + */ + rmpopt_wq = alloc_workqueue("rmpopt_wq", WQ_UNBOUND, 1); + if (!rmpopt_wq) { + pr_err("Failed to allocate RMPOPT workqueue\n"); + return; + } + cpus_read_lock(); /* * The RMPOPT_BASE MSR is per-core, so only one thread per core needs - * to set up the RMPOPT_BASE MSR. + * to set up the RMPOPT_BASE MSR. Likewise, only one thread per core + * needs to issue the RMPOPT instruction. * * Note: only online primary threads are included. If a core's * primary thread is offline, that core is not covered. CPU hotplug @@ -622,6 +766,23 @@ void snp_setup_rmpopt(void) wrmsrq_on_cpus(&rmpopt_cpumask, MSR_AMD64_RMPOPT_BASE, rmpopt_base); cpus_read_unlock(); + + INIT_DELAYED_WORK(&rmpopt_delayed_work, rmpopt_work_handler); + + rmpopt_pa_end = ALIGN(PFN_PHYS(max_pfn), SZ_1G); + + /* Limit memory scanning to 2TB of RAM */ + if ((rmpopt_pa_end - rmpopt_pa_start) > SZ_2T) { + pr_info("RMPOPT coverage limited to 2TB; memory above 0x%llx not optimized\n", + rmpopt_pa_start + SZ_2T); + rmpopt_pa_end = rmpopt_pa_start + SZ_2T; + } + + /* + * Once all per-CPU RMPOPT tables have been configured, enable RMPOPT + * optimizations on all physical memory. + */ + queue_delayed_work(rmpopt_wq, &rmpopt_delayed_work, 0); } EXPORT_SYMBOL_FOR_MODULES(snp_setup_rmpopt, "ccp"); -- 2.43.0