From: Ashish Kalra The new RMPOPT instruction helps manage per-CPU RMP optimization structures inside the CPU. It takes a 1GB-aligned physical address and either returns the status of the optimizations or tries to enable the optimizations. Per-CPU RMPOPT tables support at most 2 TB of addressable memory for RMP optimizations. Initialize the per-CPU RMPOPT table base to the starting physical address. This enables RMP optimization for up to 2 TB of system RAM on all CPUs. Additionally, add support to setup and enable RMPOPT once SNP is enabled and initialized. Suggested-by: Thomas Lendacky Suggested-by: Dave Hansen Reviewed-by: Dave Hansen Signed-off-by: Ashish Kalra --- arch/x86/coco/core.c | 1 + arch/x86/include/asm/msr-index.h | 3 ++ arch/x86/include/asm/sev.h | 2 + arch/x86/virt/svm/sev.c | 65 +++++++++++++++++++++++++++++++- drivers/crypto/ccp/sev-dev.c | 3 ++ 5 files changed, 73 insertions(+), 1 deletion(-) diff --git a/arch/x86/coco/core.c b/arch/x86/coco/core.c index 989ca9f72ba3..7fdef00ca8f2 100644 --- a/arch/x86/coco/core.c +++ b/arch/x86/coco/core.c @@ -172,6 +172,7 @@ static void amd_cc_platform_clear(enum cc_attr attr) switch (attr) { case CC_ATTR_HOST_SEV_SNP: cc_flags.host_sev_snp = 0; + setup_clear_cpu_cap(X86_FEATURE_RMPOPT); break; default: break; diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 86554de9a3f5..28540744f1eb 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -761,6 +761,9 @@ #define MSR_AMD64_SEG_RMP_ENABLED_BIT 0 #define MSR_AMD64_SEG_RMP_ENABLED BIT_ULL(MSR_AMD64_SEG_RMP_ENABLED_BIT) #define MSR_AMD64_RMP_SEGMENT_SHIFT(x) (((x) & GENMASK_ULL(13, 8)) >> 8) +#define MSR_AMD64_RMPOPT_BASE 0xc0010139 +#define MSR_AMD64_RMPOPT_ENABLE_BIT 0 +#define MSR_AMD64_RMPOPT_ENABLE BIT_ULL(MSR_AMD64_RMPOPT_ENABLE_BIT) #define MSR_SVSM_CAA 0xc001f000 diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h index 594cfa19cbd4..6fd72a44a51e 100644 --- a/arch/x86/include/asm/sev.h +++ b/arch/x86/include/asm/sev.h @@ -662,6 +662,7 @@ static inline void snp_leak_pages(u64 pfn, unsigned int pages) __snp_leak_pages(pfn, pages, true); } int snp_prepare(void); +void snp_setup_rmpopt(void); void snp_shutdown(void); #else static inline bool snp_probe_rmptable_info(void) { return false; } @@ -680,6 +681,7 @@ static inline void snp_leak_pages(u64 pfn, unsigned int npages) {} static inline void kdump_sev_callback(void) { } static inline void snp_fixup_e820_tables(void) {} static inline int snp_prepare(void) { return -ENODEV; } +static inline void snp_setup_rmpopt(void) {} static inline void snp_shutdown(void) {} #endif diff --git a/arch/x86/virt/svm/sev.c b/arch/x86/virt/svm/sev.c index 8bcdce98f6dc..089c9a14edc7 100644 --- a/arch/x86/virt/svm/sev.c +++ b/arch/x86/virt/svm/sev.c @@ -124,6 +124,9 @@ static void *rmp_bookkeeping __ro_after_init; static u64 probed_rmp_base, probed_rmp_size; +static cpumask_t rmpopt_cpumask; +static phys_addr_t rmpopt_pa_start; + static LIST_HEAD(snp_leaked_pages_list); static DEFINE_SPINLOCK(snp_leaked_pages_list_lock); @@ -488,9 +491,13 @@ static bool __init setup_segmented_rmptable(void) static bool __init setup_rmptable(void) { if (rmp_cfg & MSR_AMD64_SEG_RMP_ENABLED) { - if (!setup_segmented_rmptable()) + if (!setup_segmented_rmptable()) { + setup_clear_cpu_cap(X86_FEATURE_RMPOPT); return false; + } } else { + /* Note that Segmented RMP must be enabled to enable RMPOPT. */ + setup_clear_cpu_cap(X86_FEATURE_RMPOPT); if (!setup_contiguous_rmptable()) return false; } @@ -555,6 +562,21 @@ int snp_prepare(void) } EXPORT_SYMBOL_FOR_MODULES(snp_prepare, "ccp"); +static void rmpopt_cleanup(void) +{ + int cpu; + + cpus_read_lock(); + + for_each_cpu(cpu, &rmpopt_cpumask) + wrmsrq_on_cpu(cpu, MSR_AMD64_RMPOPT_BASE, 0); + + cpus_read_unlock(); + + cpumask_clear(&rmpopt_cpumask); + rmpopt_pa_start = 0; +} + void snp_shutdown(void) { u64 syscfg; @@ -563,11 +585,52 @@ void snp_shutdown(void) if (syscfg & MSR_AMD64_SYSCFG_SNP_EN) return; + rmpopt_cleanup(); + clear_rmp(); on_each_cpu(mfd_reconfigure, NULL, 1); } EXPORT_SYMBOL_FOR_MODULES(snp_shutdown, "ccp"); +void snp_setup_rmpopt(void) +{ + u64 rmpopt_base; + int cpu; + + if (!cpu_feature_enabled(X86_FEATURE_RMPOPT)) + return; + + cpus_read_lock(); + + /* + * The RMPOPT_BASE MSR is per-core, so only one thread per core needs + * to set up the RMPOPT_BASE MSR. + * + * Note: only online primary threads are included. If a core's + * primary thread is offline, that core is not covered. CPU hotplug + * is not currently supported with SNP enabled. + */ + + for_each_online_cpu(cpu) + if (topology_is_primary_thread(cpu)) + cpumask_set_cpu(cpu, &rmpopt_cpumask); + + rmpopt_pa_start = ALIGN_DOWN(PFN_PHYS(min_low_pfn), SZ_1G); + rmpopt_base = rmpopt_pa_start | MSR_AMD64_RMPOPT_ENABLE; + + /* + * Per-CPU RMPOPT tables support at most 2 TB of addressable memory + * for RMP optimizations. Initialize the per-CPU RMPOPT table base + * to the starting physical address to enable RMP optimizations for + * up to 2 TB of system RAM on all CPUs. + */ + for_each_cpu(cpu, &rmpopt_cpumask) + wrmsrq_on_cpu(cpu, MSR_AMD64_RMPOPT_BASE, rmpopt_base); + + cpus_read_unlock(); +} +EXPORT_SYMBOL_FOR_MODULES(snp_setup_rmpopt, "ccp"); + /* * Do the necessary preparations which are verified by the firmware as * described in the SNP_INIT_EX firmware command description in the SNP diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 78f98aee7a66..217b6b19802e 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -1478,6 +1478,9 @@ static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid) } snp_hv_fixed_pages_state_update(sev, HV_FIXED); + + snp_setup_rmpopt(); + sev->snp_initialized = true; dev_dbg(sev->dev, "SEV-SNP firmware initialized, SEV-TIO is %s\n", data.tio_en ? "enabled" : "disabled"); -- 2.43.0