From: Ashish Kalra As SEV-SNP is enabled by default on boot when an RMP table is allocated by BIOS, the hypervisor and non-SNP guests are subject to RMP write checks to provide integrity of SNP guest memory. RMPOPT is a new instruction that minimizes the performance overhead of RMP checks on the hypervisor and on non-SNP guests by allowing RMP checks to be skipped for 1GB regions of memory that are known not to contain any SEV-SNP guest memory. Enable RMPOPT optimizations globally for all system RAM at RMP initialization time. RMP checks can initially be skipped for 1GB memory ranges that do not contain SEV-SNP guest memory (excluding preassigned pages such as the RMP table and firmware pages). As SNP guests are launched, RMPUPDATE will disable the corresponding RMPOPT optimizations. Suggested-by: Thomas Lendacky Signed-off-by: Ashish Kalra --- arch/x86/virt/svm/sev.c | 84 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/arch/x86/virt/svm/sev.c b/arch/x86/virt/svm/sev.c index e6b784d26c33..a0d38fc50698 100644 --- a/arch/x86/virt/svm/sev.c +++ b/arch/x86/virt/svm/sev.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -127,10 +128,17 @@ static DEFINE_SPINLOCK(snp_leaked_pages_list_lock); static unsigned long snp_nr_leaked_pages; +enum rmpopt_function { + RMPOPT_FUNC_VERIFY_AND_REPORT_STATUS, + RMPOPT_FUNC_REPORT_STATUS +}; + #define RMPOPT_TABLE_MAX_LIMIT_IN_TB 2 #define NUM_TB(pfn_min, pfn_max) \ (((pfn_max) - (pfn_min)) / (1 << (40 - PAGE_SHIFT))) +static struct task_struct *rmpopt_task; + struct rmpopt_socket_config { unsigned long start_pfn, end_pfn; cpumask_var_t cpulist; @@ -527,6 +535,66 @@ static void get_cpumask_of_primary_threads(cpumask_var_t cpulist) } } +/* + * 'val' is a system physical address aligned to 1GB OR'ed with + * a function selection. Currently supported functions are 0 + * (verify and report status) and 1 (report status). + */ +static void rmpopt(void *val) +{ + asm volatile(".byte 0xf2, 0x0f, 0x01, 0xfc\n\t" + : : "a" ((u64)val & PUD_MASK), "c" ((u64)val & 0x1) + : "memory", "cc"); +} + +static int rmpopt_kthread(void *__unused) +{ + phys_addr_t pa_start, pa_end; + cpumask_var_t cpus; + + if (!zalloc_cpumask_var(&cpus, GFP_KERNEL)) + return -ENOMEM; + + pa_start = ALIGN_DOWN(PFN_PHYS(min_low_pfn), PUD_SIZE); + pa_end = ALIGN(PFN_PHYS(max_pfn), PUD_SIZE); + + while (!kthread_should_stop()) { + phys_addr_t pa; + + pr_info("RMP optimizations enabled on physical address range @1GB alignment [0x%016llx - 0x%016llx]\n", + pa_start, pa_end); + + /* Only one thread per core needs to issue RMPOPT instruction */ + get_cpumask_of_primary_threads(cpus); + + /* + * RMPOPT optimizations skip RMP checks at 1GB granularity if this range of + * memory does not contain any SNP guest memory. + */ + for (pa = pa_start; pa < pa_end; pa += PUD_SIZE) { + /* Bit zero passes the function to the RMPOPT instruction. */ + on_each_cpu_mask(cpus, rmpopt, + (void *)(pa | RMPOPT_FUNC_VERIFY_AND_REPORT_STATUS), + true); + + /* Give a chance for other threads to run */ + cond_resched(); + } + + set_current_state(TASK_INTERRUPTIBLE); + schedule(); + } + + free_cpumask_var(cpus); + return 0; +} + +static void rmpopt_all_physmem(void) +{ + if (rmpopt_task) + wake_up_process(rmpopt_task); +} + static void __configure_rmpopt(void *val) { u64 rmpopt_base = ((u64)val & PUD_MASK) | MSR_AMD64_RMPOPT_ENABLE; @@ -687,6 +755,22 @@ static __init void configure_and_enable_rmpopt(void) else configure_rmpopt_large_physmem(primary_threads_cpulist); + rmpopt_task = kthread_create(rmpopt_kthread, NULL, "rmpopt_kthread"); + if (IS_ERR(rmpopt_task)) { + pr_warn("Unable to start RMPOPT kernel thread\n"); + rmpopt_task = NULL; + goto free_cpumask; + } + + pr_info("RMPOPT worker thread created with PID %d\n", task_pid_nr(rmpopt_task)); + + /* + * Once all per-CPU RMPOPT tables have been configured, enable RMPOPT + * optimizations on all physical memory. + */ + rmpopt_all_physmem(); + +free_cpumask: free_cpumask_var(primary_threads_cpulist); } -- 2.43.0