When a single CPU is brought online at runtime (cache_aps_delayed_init == false), cache_ap_online() unconditionally calls stop_machine_from_inactive_cpu() to rendezvous all online CPUs while the incoming AP runs cache_cpu_init() to disable caches (CR0.CD = 1, plus wbinvd() when X86_FEATURE_SELFSNOOP is not set), reprogram MTRRs via mtrr_generic_set_state(), and re-enable caches. During initial boot (cache_aps_init()) and across warm CPU offline/online cycles, the AP's hardware MTRR MSRs have already been programmed to match the saved mtrr_state. Stopping all online CPUs in stop_machine (and flushing caches on the AP) when the AP's MTRRs already match mtrr_state is unnecessary. Add mtrr_generic_state_matches() to compare the local CPU's MSR_MTRRdefType, variable MTRR ranges, and fixed MTRR ranges against mtrr_state (including k8_check_syscfg_dram_mod_en() on AMD), and check it on the AP in cache_ap_online() before invoking stop_machine_from_inactive_cpu(). If the hardware MTRRs already match mtrr_state, initialize PAT locally on the AP and return. 2-socket, 480-CPU Intel Xeon 6985P-C (N=479, mean of 10 runs): Offline: 9.63 s -> 9.85 s Online: 11.11 s -> 8.69 s (-21.8%) Total: 20.74 s -> 18.54 s (-10.6%) cache_ap_online() (cpu479 online): 7,704.0 us -> 4.7 us (-99.94%) 2-socket, 512-CPU AMD EPYC 9B45 (N=511, mean of 10 runs): Offline: 4.53 s -> 4.51 s (-0.4%) Online: 3.92 s -> 3.15 s (-19.6%) Total: 8.45 s -> 7.66 s (-9.3%) cache_ap_online() (cpu511 online): 1,354.6 us -> 6.4 us (-99.53%) Signed-off-by: Pasha Tatashin --- arch/x86/include/asm/mtrr.h | 2 ++ arch/x86/kernel/cpu/cacheinfo.c | 7 +++++ arch/x86/kernel/cpu/mtrr/generic.c | 49 ++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+) diff --git a/arch/x86/include/asm/mtrr.h b/arch/x86/include/asm/mtrr.h index 76b95bd1a4058..0527e216808c4 100644 --- a/arch/x86/include/asm/mtrr.h +++ b/arch/x86/include/asm/mtrr.h @@ -61,6 +61,7 @@ extern int amd_special_default_mtrr(void); void mtrr_disable(void); void mtrr_enable(void); void mtrr_generic_set_state(void); +bool mtrr_generic_state_matches(void); # else static inline void guest_force_mtrr_state(struct mtrr_var_range *var, unsigned int num_var, @@ -106,6 +107,7 @@ static inline int mtrr_trim_uncached_memory(unsigned long end_pfn) #define mtrr_disable() do {} while (0) #define mtrr_enable() do {} while (0) #define mtrr_generic_set_state() do {} while (0) +static inline bool mtrr_generic_state_matches(void) { return true; } # endif #ifdef CONFIG_COMPAT diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index 13ed16527905f..64796d5b34fdc 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -770,6 +770,13 @@ static int cache_ap_online(unsigned int cpu) if (!memory_caching_control || get_cache_aps_delayed_init()) return 0; + if (!(memory_caching_control & CACHE_MTRR) || + mtrr_generic_state_matches()) { + if (memory_caching_control & CACHE_PAT) + pat_cpu_init(); + return 0; + } + /* * Ideally we should hold mtrr_mutex here to avoid MTRR entries * changed, but this routine will be called in CPU boot time, diff --git a/arch/x86/kernel/cpu/mtrr/generic.c b/arch/x86/kernel/cpu/mtrr/generic.c index 67cf69f24b005..8760a7b4dc38b 100644 --- a/arch/x86/kernel/cpu/mtrr/generic.c +++ b/arch/x86/kernel/cpu/mtrr/generic.c @@ -981,6 +981,55 @@ void mtrr_generic_set_state(void) } } +bool mtrr_generic_state_matches(void) +{ + struct msr val; + unsigned int i; + + if (!cpu_feature_enabled(X86_FEATURE_MTRR)) + return true; + + rdmsrq(MSR_MTRRdefType, val.q); + if ((val.l & MTRR_DEF_TYPE_TYPE) != mtrr_state.def_type || + ((val.l & MTRR_DEF_TYPE_ENABLE) >> MTRR_STATE_SHIFT) != mtrr_state.enabled) + return false; + + for (i = 0; i < num_var_ranges; i++) { + struct mtrr_var_range *vr = &mtrr_state.var_ranges[i]; + + rdmsrq(MTRRphysBase_MSR(i), val.q); + if ((vr->base_lo & ~MTRR_PHYSBASE_RSVD) != (val.l & ~MTRR_PHYSBASE_RSVD) || + (vr->base_hi & ~phys_hi_rsvd) != (val.h & ~phys_hi_rsvd)) + return false; + + rdmsrq(MTRRphysMask_MSR(i), val.q); + if ((vr->mask_lo & ~MTRR_PHYSMASK_RSVD) != (val.l & ~MTRR_PHYSMASK_RSVD) || + (vr->mask_hi & ~phys_hi_rsvd) != (val.h & ~phys_hi_rsvd)) + return false; + } + + if (mtrr_state.have_fixed) { + u64 *saved = (u64 *)mtrr_state.fixed_ranges; + int block = -1, range; + + /* + * Ensure MtrrFixDramModEn is clear so fixed MTRR reads do not + * expose K8 RdMem/WrMem bits and cause false mismatches. + */ + k8_check_syscfg_dram_mod_en(); + + while (fixed_range_blocks[++block].ranges) { + for (range = 0; range < fixed_range_blocks[block].ranges; range++) { + rdmsrq(fixed_range_blocks[block].base_msr + range, val.q); + if (val.q != *saved++) + return false; + } + } + } + + return true; +} + /** * generic_set_mtrr - set variable MTRR register on the local CPU. * -- 2.56.0.385.gd3acb90ef8-goog