When a CPU goes offline, cmci_clear() disables CMCI on any MCA banks owned by that CPU, and mce_cpu_dead() calls cmci_rediscover(). Currently cmci_rediscover() invokes on_each_cpu(cmci_rediscover_work_func, NULL, 1), sending a synchronous broadcast IPI to every online CPU in the system, all of which contend on the global cmci_discover_lock while reading up to 32 MSR_IA32_MCx_CTL2 MSRs. Any bank disabled by cmci_clear() is either thread-private, core-scoped (shared between SMT threads of the same physical core), L2-cluster-scoped, die-scoped, or package-scoped: 1. Any bank index that is already owned by the target peer in its own mce_banks_owned bitmap is private to a narrower scope than the shared level and will be skipped by cmci_skip_bank() on the target anyway. 2. Core- or cluster-shared MCA banks can only be claimed by an online SMT sibling or L2 cluster peer. 3. Shared die-scoped or package-scoped uncore banks can be claimed in a single pass by one online peer on the same die or package. Record the bitmap of banks disabled by __cmci_disable_bank() in cmci_orphaned_banks, select a single online topology peer cmci_orphaned_target (SMT sibling, L2 cluster peer, die peer, or package peer) in cmci_clear(), and mask out any bank indices already owned by that target peer. In cmci_rediscover(), return immediately when cmci_orphaned_banks is empty; otherwise probe only cmci_orphaned_target via a single smp_call_function_single() call. 2-socket, 480-CPU Intel Xeon 6985P-C (N=479, mean of 10 runs): Offline: 11.42 s -> 9.63 s (-15.7%) Online: 11.08 s -> 11.11 s Total: 22.50 s -> 20.74 s (-7.8%) cmci_rediscover() (cpu479 offline): 7,720.6 us -> 0.6 us (-99.99%) Signed-off-by: Pasha Tatashin --- arch/x86/kernel/cpu/mce/intel.c | 57 +++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/cpu/mce/intel.c b/arch/x86/kernel/cpu/mce/intel.c index 4655223ba5606..f57718b097d9b 100644 --- a/arch/x86/kernel/cpu/mce/intel.c +++ b/arch/x86/kernel/cpu/mce/intel.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -40,6 +41,8 @@ * owned by the offlined cpu. */ static DEFINE_PER_CPU(mce_banks_t, mce_banks_owned); +static mce_banks_t cmci_orphaned_banks; +static unsigned int cmci_orphaned_target; /* * cmci_discover_lock protects against parallel discovery attempts @@ -244,6 +247,7 @@ static void cmci_claim_bank(int bank, u64 val, int bios_zero_thresh, int *bios_w /* This CPU successfully set the enable bit. */ set_bit(bank, (void *)this_cpu_ptr(&mce_banks_owned)); + __clear_bit(bank, cmci_orphaned_banks); if ((val & MCI_CTL2_CMCI_THRESHOLD_MASK) == CMCI_STORM_THRESHOLD) { pr_notice("CPU%d BANK%d CMCI inherited storm\n", smp_processor_id(), bank); @@ -328,6 +332,7 @@ static void __cmci_disable_bank(int bank) val &= ~MCI_CTL2_CMCI_EN; wrmsrq(MSR_IA32_MCx_CTL2(bank), val); __clear_bit(bank, this_cpu_ptr(mce_banks_owned)); + __set_bit(bank, cmci_orphaned_banks); if ((val & MCI_CTL2_CMCI_THRESHOLD_MASK) == CMCI_STORM_THRESHOLD) cmci_storm_end(bank); @@ -339,6 +344,8 @@ static void __cmci_disable_bank(int bank) */ void cmci_clear(void) { + unsigned int cpu = smp_processor_id(); + unsigned int target; unsigned long flags; int i; int banks; @@ -346,8 +353,31 @@ void cmci_clear(void) if (!cmci_supported(&banks)) return; raw_spin_lock_irqsave(&cmci_discover_lock, flags); + bitmap_zero(cmci_orphaned_banks, MAX_NR_BANKS); + /* + * A bank is shared by the CPUs of one SMT core, L2 cluster, die or + * package, and these scopes nest. The first online peer found from + * the innermost scope outwards is therefore part of every sharing + * scope that still has an online CPU, so it alone can claim every + * orphaned bank that is still claimable. If it already owns an index, + * the instance released here is not shared with it, hence not with + * any online CPU, and that index can be dropped from the rediscovery. + */ + target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu); + if (target >= nr_cpu_ids) + target = cpumask_any_but(topology_cluster_cpumask(cpu), cpu); + if (target >= nr_cpu_ids) + target = cpumask_any_but(topology_die_cpumask(cpu), cpu); + if (target >= nr_cpu_ids) + target = cpumask_any_but(topology_core_cpumask(cpu), cpu); + cmci_orphaned_target = target; for (i = 0; i < banks; i++) __cmci_disable_bank(i); + if (target < nr_cpu_ids) + bitmap_andnot(cmci_orphaned_banks, cmci_orphaned_banks, + per_cpu(mce_banks_owned, target), MAX_NR_BANKS); + else + bitmap_zero(cmci_orphaned_banks, MAX_NR_BANKS); raw_spin_unlock_irqrestore(&cmci_discover_lock, flags); } @@ -360,15 +390,38 @@ static void cmci_rediscover_work_func(void *arg) cmci_discover(banks); } -/* After a CPU went down cycle through all the others and rediscover */ +/* + * After a CPU went down, rediscover orphaned shared banks on a single + * topology peer (SMT sibling, L2 cluster peer, die peer, or package peer) + * selected in cmci_clear(). Core-private banks whose core is now completely + * offline cannot be claimed by any other core, so probing more than one + * peer is redundant. + */ void cmci_rediscover(void) { + unsigned int target; + unsigned long flags; + bool orphaned; int banks; if (!cmci_supported(&banks)) return; - on_each_cpu(cmci_rediscover_work_func, NULL, 1); + raw_spin_lock_irqsave(&cmci_discover_lock, flags); + orphaned = !bitmap_empty(cmci_orphaned_banks, MAX_NR_BANKS); + target = cmci_orphaned_target; + raw_spin_unlock_irqrestore(&cmci_discover_lock, flags); + + if (!orphaned) + return; + + if (target < nr_cpu_ids && cpu_online(target)) + smp_call_function_single(target, cmci_rediscover_work_func, + NULL, 1); + + raw_spin_lock_irqsave(&cmci_discover_lock, flags); + bitmap_zero(cmci_orphaned_banks, MAX_NR_BANKS); + raw_spin_unlock_irqrestore(&cmci_discover_lock, flags); } /* -- 2.56.0.385.gd3acb90ef8-goog