When running as a guest with a vPMU exposed, the host may expose zero general-purpose counters or zero fixed counters. In that case, GENMASK_ULL() is called with a negative high-bit argument, resulting in an out-of-range shift and undefined behavior. On systems with PMU partitioning enabled, this configuration is more likely. Change GENMASK_ULL() to BIT_ULL() to guard mask generation against zero counters. Fixes: 722e42e45c2f ("perf/x86: Support counter mask") Cc: stable@vger.kernel.org Signed-off-by: Zide Chen --- arch/x86/events/intel/core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 2b35483e2b70..8b13bcc5259c 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -7905,7 +7905,7 @@ __init int intel_pmu_init(void) x86_pmu = intel_pmu; x86_pmu.version = version; - x86_pmu.cntr_mask64 = GENMASK_ULL(eax.split.num_counters - 1, 0); + x86_pmu.cntr_mask64 = BIT_ULL(eax.split.num_counters) - 1; x86_pmu.cntval_bits = eax.split.bit_width; x86_pmu.cntval_mask = (1ULL << eax.split.bit_width) - 1; @@ -7924,7 +7924,7 @@ __init int intel_pmu_init(void) int assume = 3 * !boot_cpu_has(X86_FEATURE_HYPERVISOR); x86_pmu.fixed_cntr_mask64 = - GENMASK_ULL(max((int)edx.split.num_counters_fixed, assume) - 1, 0); + BIT_ULL(max((int)edx.split.num_counters_fixed, assume)) - 1; } else if (version >= 5) x86_pmu.fixed_cntr_mask64 = fixed_mask; -- 2.55.0 x86 PMU partitioning support requires mediated vPMU and is supported only on Intel platforms. KVM is responsible for determining if PMU partitioning can be enabled, and if so, it passes the mask to perf core when creating a mediated PMU VM. The mask is host-wide state shared by all mediated PMU guests. Perf core passes the mask down to perf/x86 via the new callback arch_perf_set_pmu_partition_mask(), which sanitizes it before applying it to the newly added x86_pmu. The mask is cleared when the last mediated PMU VM is released. Note: PMU partitioning technically allows the host to run certain !exclude_guest events while partitioned guests are running, as long as such events don't use exclusive resources reserved for the guests. The current behavior is kept: reject the creation of a partitioned guest as long as any !exclude_guest host events exist. Introduce PERF_PMU_CAP_PMU_PARTITION to indicate that the pmu supports PMU partitioning. Note that virtualization of Intel PT and BTS is currently unsupported; even if it were, this flag would not apply to those PMUs, since their exclusive resources can't be shared between host and guest, and their events are not subject to PMU partitioning scheduling. perf_create_mediated_pmu() is called with 0 for now; this will be replaced with the actual partition_mask in a later patch. Signed-off-by: Zide Chen --- arch/x86/events/core.c | 58 ++++++++++++++++++++++++++++++++++++ arch/x86/events/perf_event.h | 9 ++++++ arch/x86/kvm/x86.c | 2 +- include/linux/perf_event.h | 4 ++- kernel/events/core.c | 19 +++++++++--- 5 files changed, 86 insertions(+), 6 deletions(-) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 9b6df8bc9059..8032311c0a47 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -2773,6 +2773,64 @@ static bool x86_pmu_filter(struct pmu *pmu, int cpu) return ret; } +/** + * arch_perf_set_pmu_partition_mask - Validate and set the VM-owned counter + * mask for mediated vPMU + * @partition_mask: Bitmask of PMU counters or other hardware resources + * to hand over to mediated vPMU guests. 0 disables PMU + * partitioning, as in the legacy model. + * + * Called via perf_create_mediated_pmu() to validate @partition_mask and, + * if valid, record it in x86_pmu.partition_mask for use by the + * scheduler, and set PERF_PMU_CAP_PMU_PARTITION on the generic PMU so + * perf core can check it without reaching into x86-private state. + * + * Return: 0 on success, -errno otherwise. + */ +int arch_perf_set_pmu_partition_mask(u64 partition_mask) +{ + u64 current_mask = READ_ONCE(x86_pmu.partition_mask); + struct pmu *pmu; + + /* Non-paritioned mediated guests fall into this case. */ + if (current_mask == partition_mask) + return 0; + + /* + * AMD does not yet implement the hardware support for PMU partitioning + * between host and guest. Thus limit it to Intel platforms with the + * PerfMon masking VMX extension. Leave it to KVM to check the VMX + * feature. KVM doesn't support vPMU on Hybrid CPUs at all. + */ + if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL || is_hybrid()) + return -EOPNOTSUPP; + + pmu = x86_get_pmu(raw_smp_processor_id()); + if (!(pmu->capabilities & PERF_PMU_CAP_MEDIATED_VPMU)) + return -EOPNOTSUPP; + + /* + * If a mediated-PMU VM was already created, the configured mask + * cannot be changed. + */ + if (current_mask && partition_mask) + return -EINVAL; + + /* + * perf/core guarantees that this path is reached only after all + * partitioned guests have been released. + */ + if (!partition_mask) { + WRITE_ONCE(x86_pmu.partition_mask, 0); + pmu->capabilities &= ~PERF_PMU_CAP_PMU_PARTITION; + return 0; + } + + WRITE_ONCE(x86_pmu.partition_mask, partition_mask); + pmu->capabilities |= PERF_PMU_CAP_PMU_PARTITION; + return 0; +} + static struct pmu pmu = { .pmu_enable = x86_pmu_enable, .pmu_disable = x86_pmu_disable, diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index eae24bb35dc1..19beb16baa8e 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -883,6 +883,15 @@ struct x86_pmu { int events_mask_len; int apic; u64 max_period; + + /* + * Bitmask of PMU resources that may be assigned to a guest. + * + * The mask is set when the first mediated vPMU is created and is + * cleared when the last mediated vPMU is torn down. + */ + u64 partition_mask; + struct event_constraint * (*get_event_constraints)(struct cpu_hw_events *cpuc, int idx, diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index d349224d2734..5f3215915c76 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -9328,7 +9328,7 @@ int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) if (enable_mediated_pmu && kvm->arch.enable_pmu && !kvm->arch.created_mediated_pmu) { if (irqchip_in_kernel(kvm)) { - r = perf_create_mediated_pmu(); + r = perf_create_mediated_pmu(0); if (r) { pr_warn_ratelimited(PERF_MEDIATED_PMU_MSG); return r; diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 48d851fbd8ea..bd952e09055d 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -306,6 +306,7 @@ struct perf_event_pmu_context; #define PERF_PMU_CAP_AUX_PAUSE 0x0200 #define PERF_PMU_CAP_AUX_PREFER_LARGE 0x0400 #define PERF_PMU_CAP_MEDIATED_VPMU 0x0800 +#define PERF_PMU_CAP_PMU_PARTITION 0x1000 /** * pmu::scope @@ -1931,10 +1932,11 @@ extern int perf_event_period(struct perf_event *event, u64 value); extern u64 perf_event_pause(struct perf_event *event, bool reset); #ifdef CONFIG_PERF_GUEST_MEDIATED_PMU -int perf_create_mediated_pmu(void); +int perf_create_mediated_pmu(u64 pmu_partition_mask); void perf_release_mediated_pmu(void); void perf_load_guest_context(void); void perf_put_guest_context(void); +int arch_perf_set_pmu_partition_mask(u64 pmu_partition_mask); #endif #else /* !CONFIG_PERF_EVENTS: */ diff --git a/kernel/events/core.c b/kernel/events/core.c index d7f3e2c2ecb1..9ce27cd83e04 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -6340,6 +6340,11 @@ static atomic_t nr_include_guest_events __read_mostly; static atomic_t nr_mediated_pmu_vms __read_mostly; static DEFINE_MUTEX(perf_mediated_pmu_mutex); +int __weak arch_perf_set_pmu_partition_mask(u64 partition_mask) +{ + return partition_mask ? -EOPNOTSUPP : 0; +} + /* !exclude_guest event of PMU with PERF_PMU_CAP_MEDIATED_VPMU */ static inline bool is_include_guest_event(struct perf_event *event) { @@ -6387,15 +6392,18 @@ static void mediated_pmu_unaccount_event(struct perf_event *event) * No impact for the PMU without PERF_PMU_CAP_MEDIATED_VPMU. The perf * still owns all the PMU resources. */ -int perf_create_mediated_pmu(void) +int perf_create_mediated_pmu(u64 partition_mask) { - if (atomic_inc_not_zero(&nr_mediated_pmu_vms)) - return 0; + int ret; guard(mutex)(&perf_mediated_pmu_mutex); if (atomic_read(&nr_include_guest_events)) return -EBUSY; + ret = arch_perf_set_pmu_partition_mask(partition_mask); + if (ret) + return ret; + atomic_inc(&nr_mediated_pmu_vms); return 0; } @@ -6403,10 +6411,13 @@ EXPORT_SYMBOL_FOR_KVM(perf_create_mediated_pmu); void perf_release_mediated_pmu(void) { + guard(mutex)(&perf_mediated_pmu_mutex); + if (WARN_ON_ONCE(!atomic_read(&nr_mediated_pmu_vms))) return; - atomic_dec(&nr_mediated_pmu_vms); + if (atomic_dec_and_test(&nr_mediated_pmu_vms)) + arch_perf_set_pmu_partition_mask(0); } EXPORT_SYMBOL_FOR_KVM(perf_release_mediated_pmu); -- 2.55.0 Currently, KVM loads a guest's mediated PMU context in a strict order: perf_load_guest_context() runs first (it must execute in host context), then PERF_GLOBAL_CTRL is cleared (to avoid spurious PMIs), then perf_load_guest_lvtpc() switches the LVTPC hardware vector, and finally the guest's PMCs are loaded. Under PMU partitioning, host-owned counters remain available to host events. When loading guest context, the perf scheduler must reschedule host events off guest-owned counters. Add GUEST_PMU_PARTITION_PRELOAD to represent the intermediate state in which PMU partitioning constraints are needed for scheduling, but PMU partition PMI handling is not yet active. Enter this state through the new perf_pmu_partition_preload() helper. Add GUEST_PMU_PARTITION_NMI for the state after the above _PRELOAD state, i.e. the host context transition is complete and guest context loading may proceed. LVTPC remains routed to NMI, and the PMU partition mask can be applied in the PMI handler. Add GUEST_PMU_MEDIATED to indicate that the LVTPC is set to the fixed PERF_GUEST_MEDIATED_PMI_VECTOR, either in the PMU non-partitioned case, or the partition setup where no host events are scheduled and no need to share NMI with the host. Signed-off-by: Zide Chen --- arch/x86/events/core.c | 94 ++++++++++++++++++++++++++++--- arch/x86/events/perf_event.h | 2 + arch/x86/include/asm/perf_event.h | 1 + arch/x86/kvm/pmu.c | 2 + 4 files changed, 91 insertions(+), 8 deletions(-) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 8032311c0a47..ba441f4d5f3e 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -57,7 +57,38 @@ DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { .pmu = &pmu, }; -static DEFINE_PER_CPU(bool, guest_lvtpc_loaded); +/* + * GUEST_PMU_NONE - No guest mediated PMU context is loaded. + * + * GUEST_PMU_MEDIATED - LVTPC is routed to the dedicated mediated PMI vector + * instead of NMI, so any PMI in this state can only be guest-induced. + * This covers both the non-partitioned mediated vPMU model and the PMU + * partitioning case where the host currently has no events scheduled on + * this CPU (see perf_load_guest_lvtpc()). + * + * GUEST_PMU_PARTITION_PRELOAD - Entering PMU partitioning guest mode, the host + * context is still loaded. PMU partitioning constraints must be applied + * so that host events can be rescheduled onto host-owned counters. + * + * GUEST_PMU_PARTITION_NMI - PMU partitioning is enabled, host event + * rescheduling is complete, and the host currently has events scheduled + * on this CPU, so LVTPC is routed to NMI and shared between host- and + * guest-owned counters: a PMI in this state may be host- or + * guest-induced. + * + * _PARTITION_PRELOAD and _PARTITION_NMI are distinct because PMU + * partitioning constraints must be visible to the scheduler before host + * events are rescheduled, while PMU partition masking can be applied to PMI + * handling only after the rescheduling completes. + */ +enum guest_pmu_mode { + GUEST_PMU_NONE, + GUEST_PMU_MEDIATED, + GUEST_PMU_PARTITION_PRELOAD, + GUEST_PMU_PARTITION_NMI, +}; + +static DEFINE_PER_CPU(enum guest_pmu_mode, guest_pmu_state); DEFINE_STATIC_KEY_FALSE(rdpmc_never_available_key); DEFINE_STATIC_KEY_FALSE(rdpmc_always_available_key); @@ -1769,21 +1800,63 @@ void perf_events_lapic_init(void) apic_write(APIC_LVTPC, APIC_DM_NMI); } +bool pmu_partition_configured(void) +{ + return READ_ONCE(x86_pmu.partition_mask) != 0; +} + #ifdef CONFIG_PERF_GUEST_MEDIATED_PMU +/* + * Mark this CPU as running a PMU partitioned guest. Guest PMU partition + * constraints apply from this point, even if host PMU context remains loaded. + */ +void perf_pmu_partition_preload(void) +{ + if (pmu_partition_configured()) + this_cpu_write(guest_pmu_state, GUEST_PMU_PARTITION_PRELOAD); +} +EXPORT_SYMBOL_FOR_KVM(perf_pmu_partition_preload); + void perf_load_guest_lvtpc(u32 guest_lvtpc) { - u32 masked = guest_lvtpc & APIC_LVT_MASKED; + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + bool is_pmu_partitioned = pmu_partition_configured(); + bool use_nmi; - apic_write(APIC_LVTPC, - APIC_DM_FIXED | PERF_GUEST_MEDIATED_PMI_VECTOR | masked); - this_cpu_write(guest_lvtpc_loaded, true); + if (is_pmu_partitioned) + WARN_ON_ONCE(this_cpu_read(guest_pmu_state) != + GUEST_PMU_PARTITION_PRELOAD); + + /* + * If the host has events scheduled on this CPU, a PMI could be host- + * or guest-induced, so share NMI with the guest. Otherwise, route + * LVTPC to the dedicated mediated PMI vector for better + * performance and simpler handling. + */ + use_nmi = is_pmu_partitioned && cpuc->n_events; + if (!use_nmi) + apic_write(APIC_LVTPC, APIC_DM_FIXED | + PERF_GUEST_MEDIATED_PMI_VECTOR | + (guest_lvtpc & APIC_LVT_MASKED)); + + this_cpu_write(guest_pmu_state, + use_nmi ? GUEST_PMU_PARTITION_NMI : GUEST_PMU_MEDIATED); } EXPORT_SYMBOL_FOR_KVM(perf_load_guest_lvtpc); void perf_put_guest_lvtpc(void) { - this_cpu_write(guest_lvtpc_loaded, false); - apic_write(APIC_LVTPC, APIC_DM_NMI); + enum guest_pmu_mode state = this_cpu_read(guest_pmu_state); + + this_cpu_write(guest_pmu_state, GUEST_PMU_NONE); + + /* + * LVTPC needs restoring to NMI unless it's already routed there, i.e. + * unless LVTPC was left routed to the dedicated mediated PMI vector + * (see perf_load_guest_lvtpc()). + */ + if (state == GUEST_PMU_MEDIATED) + apic_write(APIC_LVTPC, APIC_DM_NMI); } EXPORT_SYMBOL_FOR_KVM(perf_put_guest_lvtpc); #endif /* CONFIG_PERF_GUEST_MEDIATED_PMU */ @@ -1802,8 +1875,13 @@ perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs) * loaded will generate false positives and clobber guest state. Note, * the LVTPC is switched to/from the dedicated mediated PMI IRQ vector * while host events are quiesced. + * + * GUEST_PMU_PARTITION_NMI is intentionally excluded here: LVTPC stays + * routed to NMI in that state, and an NMI there can be host- or + * guest-induced. GUEST_PMU_PARTITION_PRELOAD is likewise excluded, as + * PMU is still loaded with host context. */ - if (this_cpu_read(guest_lvtpc_loaded)) + if (this_cpu_read(guest_pmu_state) == GUEST_PMU_MEDIATED) return NMI_DONE; /* diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index 19beb16baa8e..29ea11421874 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -1602,6 +1602,8 @@ static inline int is_pebs_pt(struct perf_event *event) return !!(event->hw.flags & PERF_X86_EVENT_PEBS_VIA_PT); } +bool pmu_partition_configured(void); + #ifdef CONFIG_CPU_SUP_INTEL static inline bool intel_pmu_has_bts_period(struct perf_event *event, u64 period) diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h index 17b0bc7dfce7..18f1ac5e008b 100644 --- a/arch/x86/include/asm/perf_event.h +++ b/arch/x86/include/asm/perf_event.h @@ -790,6 +790,7 @@ static inline void perf_check_microcode(void) { } #endif #ifdef CONFIG_PERF_GUEST_MEDIATED_PMU +extern void perf_pmu_partition_preload(void); extern void perf_load_guest_lvtpc(u32 guest_lvtpc); extern void perf_put_guest_lvtpc(void); #endif diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index b5a9fbd415d1..7f619a99a152 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -1388,6 +1388,8 @@ void kvm_mediated_pmu_load(struct kvm_vcpu *vcpu) lockdep_assert_irqs_disabled(); + perf_pmu_partition_preload(); + perf_load_guest_context(); /* -- 2.55.0 If PMU partitioning is enabled, LVTPC could route to NMI if any host events are scheduled in this CPU. Thus, PMIs that fire in guest context can be host- or guest-induced. The host NMI handler processes host-owned PMIs as usual. Guest-induced PMIs are marked as handled to avoid unknown NMI warnings, and their GLOBAL_STATUS bits deliberately remain untouched in hardware; it's KVM's responsibility to inject the corresponding PMIs into the guest. Other places that write to global MSRs in NMI context also need to distinguish host-owned and guest-owned bits, so that they won't clobber guest-owned bits. The GLOBAL_CTRL is an exception: its guest value is expected to be saved by VMX on VM exit and restored by KVM on VM Entry. Add x86_pmu_partition_nmi_active() to identify when PMU partitioning is active and LVTPC is routed to NMI, to help determine when the partition mask needs to be applied. The effective mask may differ across guests. Add a per-cpu partition_mask field to struct cpu_hw_events, to be updated by KVM whenever the effective mask changes on the CPU. This could differ from the static x86_pmu.partition_mask, of which the effective mask is a subset. Signed-off-by: Zide Chen --- arch/x86/events/core.c | 19 +++++++++++- arch/x86/events/intel/core.c | 58 ++++++++++++++++++++++++++++++++++-- arch/x86/events/perf_event.h | 8 +++++ 3 files changed, 82 insertions(+), 3 deletions(-) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index ba441f4d5f3e..bbae68c49063 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -1805,6 +1805,19 @@ bool pmu_partition_configured(void) return READ_ONCE(x86_pmu.partition_mask) != 0; } +bool x86_pmu_partition_nmi_active(void) +{ + enum guest_pmu_mode state = this_cpu_read(guest_pmu_state); + + return pmu_partition_configured() && + state == GUEST_PMU_PARTITION_NMI; +} + +u64 x86_pmu_current_partition_mask(void) +{ + return this_cpu_ptr(&cpu_hw_events)->partition_mask; +} + #ifdef CONFIG_PERF_GUEST_MEDIATED_PMU /* * Mark this CPU as running a PMU partitioned guest. Guest PMU partition @@ -1887,8 +1900,12 @@ perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs) /* * All PMUs/events that share this PMI handler should make sure to * increment active_events for their events. + * + * If PMU partitioning is enabled, guest-induced PMIs need to be marked + * as handled to avoid unknown NMI warnings. */ - if (!atomic_read(&active_events)) + if (!atomic_read(&active_events) && + !x86_pmu_partition_nmi_active()) return NMI_DONE; start_clock = sched_clock(); diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 8b13bcc5259c..c595c86ecf90 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -2774,15 +2774,41 @@ static __always_inline void intel_pmu_disable_all(void) intel_pmu_lbr_disable_all(); } +static inline u64 intel_pmu_guest_fixed_ctrl_mask(void) +{ + u64 partition_mask = x86_pmu_current_partition_mask(); + int i = INTEL_PMC_IDX_FIXED; + u64 mask = 0; + + for_each_set_bit_from(i, (unsigned long *)&partition_mask, + INTEL_PMC_IDX_FIXED + INTEL_PMC_MAX_FIXED) { + mask |= intel_fixed_bits_by_idx(i - INTEL_PMC_IDX_FIXED, + INTEL_FIXED_BITS_MASK); + } + + return mask; +} + static void __intel_pmu_enable_all(int added, bool pmi) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); u64 intel_ctrl = hybrid(cpuc->pmu, intel_ctrl); + u64 guest_mask, fixed_ctrl; intel_pmu_lbr_enable_all(pmi); if (cpuc->fixed_ctrl_val != cpuc->active_fixed_ctrl_val) { - wrmsrq(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, cpuc->fixed_ctrl_val); + if (x86_pmu_partition_nmi_active() && pmi) { + guest_mask = intel_pmu_guest_fixed_ctrl_mask(); + + rdmsrq(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed_ctrl); + fixed_ctrl = (fixed_ctrl & guest_mask) | + (cpuc->fixed_ctrl_val & ~guest_mask); + } else { + fixed_ctrl = cpuc->fixed_ctrl_val; + } + + wrmsrq(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed_ctrl); cpuc->active_fixed_ctrl_val = cpuc->fixed_ctrl_val; } @@ -3700,23 +3726,31 @@ static void intel_pmu_reset(void) struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); unsigned long *cntr_mask = hybrid(cpuc->pmu, cntr_mask); unsigned long *fixed_cntr_mask = hybrid(cpuc->pmu, fixed_cntr_mask); + u64 guest_owned_mask = 0; unsigned long flags; int idx; if (!*(u64 *)cntr_mask) return; + if (x86_pmu_partition_nmi_active()) + guest_owned_mask = x86_pmu_current_partition_mask(); + local_irq_save(flags); pr_info("clearing PMU state on CPU#%d\n", smp_processor_id()); for_each_set_bit(idx, cntr_mask, INTEL_PMC_MAX_GENERIC) { + if (BIT_ULL(idx) & guest_owned_mask) + continue; wrmsrq_safe(x86_pmu_config_addr(idx), 0ull); wrmsrq_safe(x86_pmu_event_addr(idx), 0ull); } for_each_set_bit(idx, fixed_cntr_mask, INTEL_PMC_MAX_FIXED) { if (fixed_counter_disabled(idx, cpuc->pmu)) continue; + if (BIT_ULL(INTEL_PMC_IDX_FIXED + idx) & guest_owned_mask) + continue; wrmsrq_safe(x86_pmu_fixed_ctr_addr(idx), 0ull); } @@ -3730,7 +3764,7 @@ static void intel_pmu_reset(void) } /* Reset LBRs and LBR freezing */ - if (x86_pmu.lbr_nr) { + if (x86_pmu.lbr_nr && !(guest_owned_mask & GLOBAL_STATUS_LBRS_FROZEN)) { update_debugctlmsr(get_debugctlmsr() & ~(DEBUGCTLMSR_FREEZE_LBRS_ON_PMI|DEBUGCTLMSR_LBR)); } @@ -3944,11 +3978,22 @@ static int intel_pmu_handle_irq(struct pt_regs *regs) struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); bool late_ack = hybrid_bit(cpuc->pmu, late_ack); bool mid_ack = hybrid_bit(cpuc->pmu, mid_ack); + u64 guest_owned_mask = 0; int loops; u64 status; int handled; int pmu_enabled; + /* + * When PMU partitioning is enabled, PMIs fired in non-root mode could + * be either host- or guest-induced. + * + * The host won't clear guest-owned bits from IA32_PERF_GLOBAL_STATUS, + * and leaves them for KVM to inject into the guest. + */ + if (x86_pmu_partition_nmi_active()) + guest_owned_mask = x86_pmu_current_partition_mask(); + /* * Save the PMU state. * It needs to be restored when leaving the handler. @@ -3970,6 +4015,8 @@ static int intel_pmu_handle_irq(struct pt_regs *regs) handled = intel_pmu_drain_bts_buffer(); handled += intel_bts_interrupt(); status = intel_pmu_get_status(); + handled += hweight64(status & guest_owned_mask); + status &= ~guest_owned_mask; if (!status) goto done; @@ -3995,6 +4042,13 @@ static int intel_pmu_handle_irq(struct pt_regs *regs) * Repeat if there is more work to be done: */ status = intel_pmu_get_status(); + + /* + * Guest-owned bits were already counted into "handled" on the + * initial read and are never acked, so no hweight64() is needed + * here; just mask them out to avoid an infinite "goto again" loop. + */ + status &= ~guest_owned_mask; if (status) goto again; diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index 29ea11421874..d9875f3e6c8c 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -300,6 +300,12 @@ struct cpu_hw_events { unsigned int txn_flags; int is_fake; + /* + * The PMU resources owned by the vCPU currently scheduled on this + * CPU, which is a subset of x86_pmu.partition_mask. + */ + u64 partition_mask; + /* * Intel DebugStore bits */ @@ -1603,6 +1609,8 @@ static inline int is_pebs_pt(struct perf_event *event) } bool pmu_partition_configured(void); +bool x86_pmu_partition_nmi_active(void); +u64 x86_pmu_current_partition_mask(void); #ifdef CONFIG_CPU_SUP_INTEL -- 2.55.0 On Intel, exclude_host events are prevented from running by masking the Global Ctrl bits with ~cpuc->intel_ctrl_guest_mask. When PMU partitioning is enabled, the host is allowed to run exclude_host events in non-root mode, while such events still need to be masked in root mode, since there are no other checkpoints to stop them from being scheduled in. Add x86_pmu_partition_loaded() to distinguish scheduling constraints from x86_pmu_partition_nmi_active(), as PMU partitioning constraints apply in both the _PARTITION_PRELOAD and _PARTITION_NMI states. Signed-off-by: Zide Chen --- arch/x86/events/core.c | 11 +++++++++++ arch/x86/events/intel/core.c | 6 ++++-- arch/x86/events/perf_event.h | 1 + 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index bbae68c49063..76af9cfaf0ad 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -1813,6 +1813,17 @@ bool x86_pmu_partition_nmi_active(void) state == GUEST_PMU_PARTITION_NMI; } +/* + * Skip for "fake" cpuc used during event validation, where + * per-CPU state like guest_pmu_state is meaningless. + */ +bool x86_pmu_partition_loaded(struct cpu_hw_events *cpuc) +{ + return !cpuc->is_fake && + pmu_partition_configured() && + this_cpu_read(guest_pmu_state) != GUEST_PMU_NONE; +} + u64 x86_pmu_current_partition_mask(void) { return this_cpu_ptr(&cpu_hw_events)->partition_mask; diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index c595c86ecf90..894f98eb871a 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -2812,8 +2812,10 @@ static void __intel_pmu_enable_all(int added, bool pmi) cpuc->active_fixed_ctrl_val = cpuc->fixed_ctrl_val; } - wrmsrq(MSR_CORE_PERF_GLOBAL_CTRL, - intel_ctrl & ~cpuc->intel_ctrl_guest_mask); + if (!x86_pmu_partition_loaded(cpuc)) + intel_ctrl &= ~cpuc->intel_ctrl_guest_mask; + + wrmsrq(MSR_CORE_PERF_GLOBAL_CTRL, intel_ctrl); if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) { struct perf_event *event = diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index d9875f3e6c8c..fa3023c355d5 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -1610,6 +1610,7 @@ static inline int is_pebs_pt(struct perf_event *event) bool pmu_partition_configured(void); bool x86_pmu_partition_nmi_active(void); +bool x86_pmu_partition_loaded(struct cpu_hw_events *cpuc); u64 x86_pmu_current_partition_mask(void); #ifdef CONFIG_CPU_SUP_INTEL -- 2.55.0 From: Kan Liang When PMU partitioning is enabled and a guest is running, !exclude_guest events must be scheduled only on host-owned counters. After guest exit, PMU partition constraints are no longer applied and host events may use the full counter set. Signed-off-by: Kan Liang Co-developed-by: Zide Chen Signed-off-by: Zide Chen --- arch/x86/events/intel/core.c | 45 +++++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 894f98eb871a..7951accfcf2c 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -2814,6 +2814,13 @@ static void __intel_pmu_enable_all(int added, bool pmi) if (!x86_pmu_partition_loaded(cpuc)) intel_ctrl &= ~cpuc->intel_ctrl_guest_mask; + else if (pmi) + /* + * Guest-owned bits are excluded here, not because of clobbering + * global ctrl guest bits in non-root mode, but because guest + * counter MSRs have not yet been saved by KVM in NMI context. + */ + intel_ctrl &= ~x86_pmu_current_partition_mask(); wrmsrq(MSR_CORE_PERF_GLOBAL_CTRL, intel_ctrl); @@ -4448,6 +4455,31 @@ dyn_constraint(struct cpu_hw_events *cpuc, struct event_constraint *c, int idx) return c; } +/* + * Mask out guest-owned counters from a constraint when PMU partition has been + * entered, so !exclude_guest host events are not scheduled onto them while + * the CPU is in non-root mode. + */ +static struct event_constraint * +part_constraint(struct cpu_hw_events *cpuc, int idx, + struct perf_event *event, struct event_constraint *c) +{ + /* + * Skip &emptyconstraint: dyn_constraint() would clone it, breaking + * the "c == &emptyconstraint" pointer checks callers rely on. + */ + if (!c->weight || c == &emptyconstraint) + return c; + + if (x86_pmu_partition_loaded(cpuc)) { + c = dyn_constraint(cpuc, c, idx); + c->idxmsk64 &= ~x86_pmu_current_partition_mask(); + c->weight = hweight64(c->idxmsk64); + } + + return c; +} + static struct event_constraint * intel_get_excl_constraints(struct cpu_hw_events *cpuc, struct perf_event *event, int idx, struct event_constraint *c) @@ -4569,8 +4601,14 @@ intel_get_event_constraints(struct cpu_hw_events *cpuc, int idx, c2 = c1; } - if (cpuc->excl_cntrs) + /* + * No platform supports both PMU_FL_EXCL_CNTRS and PerfMon masking + * simultaneously, so part_constraint() is not needed on this path. + */ + if (cpuc->excl_cntrs) { + WARN_ON_ONCE(x86_pmu_partition_loaded(cpuc)); return intel_get_excl_constraints(cpuc, event, idx, c2); + } if (event->hw.dyn_constraint != ~0ULL) { c2 = dyn_constraint(cpuc, c2, idx); @@ -4578,7 +4616,7 @@ intel_get_event_constraints(struct cpu_hw_events *cpuc, int idx, c2->weight = hweight64(c2->idxmsk64); } - return c2; + return part_constraint(cpuc, idx, event, c2); } static void intel_put_excl_constraints(struct cpu_hw_events *cpuc, @@ -5948,7 +5986,8 @@ int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu) goto err; } - if (x86_pmu.flags & (PMU_FL_EXCL_CNTRS | PMU_FL_TFA | PMU_FL_DYN_CONSTRAINT)) { + if (x86_pmu.flags & (PMU_FL_EXCL_CNTRS | PMU_FL_TFA | PMU_FL_DYN_CONSTRAINT) || + (x86_get_pmu(cpu)->capabilities & PERF_PMU_CAP_MEDIATED_VPMU)) { size_t sz = X86_PMC_IDX_MAX * sizeof(struct event_constraint); cpuc->constraint_list = kzalloc_node(sz, GFP_KERNEL, cpu_to_node(cpu)); -- 2.55.0 Static, counter-specific constraints used for attr.precise_ip == 3, or other special cases bypass intel_get_event_constraints(), which is where partition_mask is applied via dyn_constraint(). As a result, such a host !exclude_guest event can still be scheduled onto a counter that partition_mask reserves for the guest, causing host and guest to share the same hardware counter. Some of these static constraint paths, e.g. glp or cmt, are currently only reachable on platforms that don't support PerfMon masking. Still, apply part_constraint() to all of them uniformly, so future platforms that combine PerfMon masking with these constraint paths are not silently exposed to this bug. Signed-off-by: Zide Chen --- arch/x86/events/intel/core.c | 39 +++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 7951accfcf2c..0f76e56fd2db 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -4459,6 +4459,9 @@ dyn_constraint(struct cpu_hw_events *cpuc, struct event_constraint *c, int idx) * Mask out guest-owned counters from a constraint when PMU partition has been * entered, so !exclude_guest host events are not scheduled onto them while * the CPU is in non-root mode. + * + * This is also used by PMU-specific get_event_constraints() wrappers + * that hard-code a static, counter-specific constraint. */ static struct event_constraint * part_constraint(struct cpu_hw_events *cpuc, int idx, @@ -5568,7 +5571,7 @@ hsw_get_event_constraints(struct cpu_hw_events *cpuc, int idx, /* Handle special quirk on in_tx_checkpointed only in counter 2 */ if (event->hw.config & HSW_IN_TX_CHECKPOINTED) { if (c->idxmsk64 & (1U << 2)) - return &counter2_constraint; + return part_constraint(cpuc, idx, event, &counter2_constraint); return &emptyconstraint; } @@ -5585,7 +5588,7 @@ icl_get_event_constraints(struct cpu_hw_events *cpuc, int idx, */ if ((event->attr.precise_ip == 3) && constraint_match(&fixed0_constraint, event->hw.config)) - return &fixed0_constraint; + return part_constraint(cpuc, idx, event, &fixed0_constraint); return hsw_get_event_constraints(cpuc, idx, event); } @@ -5607,7 +5610,7 @@ glc_get_event_constraints(struct cpu_hw_events *cpuc, int idx, if ((event->attr.precise_ip == 3) && !constraint_match(&fixed0_constraint, event->hw.config)) { if (c->idxmsk64 & BIT_ULL(0)) - return &counter0_constraint; + return part_constraint(cpuc, idx, event, &counter0_constraint); return &emptyconstraint; } @@ -5623,7 +5626,7 @@ glp_get_event_constraints(struct cpu_hw_events *cpuc, int idx, /* :ppp means to do reduced skid PEBS which is PMC0 only. */ if (event->attr.precise_ip == 3) - return &counter0_constraint; + return part_constraint(cpuc, idx, event, &counter0_constraint); c = intel_get_event_constraints(cpuc, idx, event); @@ -5645,9 +5648,9 @@ tnt_get_event_constraints(struct cpu_hw_events *cpuc, int idx, if (event->attr.precise_ip == 3) { /* Force instruction:ppp on PMC0 and Fixed counter 0 */ if (constraint_match(&fixed0_constraint, event->hw.config)) - return &fixed0_counter0_constraint; + return part_constraint(cpuc, idx, event, &fixed0_counter0_constraint); - return &counter0_constraint; + return part_constraint(cpuc, idx, event, &counter0_constraint); } return c; @@ -5705,22 +5708,30 @@ cmt_get_event_constraints(struct cpu_hw_events *cpuc, int idx, if (event->attr.precise_ip == 3) { /* Force instruction:ppp on PMC0, 1 and Fixed counter 0 */ if (constraint_match(&fixed0_constraint, event->hw.config)) { + c = &fixed0_counter0_1_constraint; + /* The fixed counter 0 doesn't support LBR event logging. */ if (branch_sample_counters(event)) - return &counter0_1_constraint; - else - return &fixed0_counter0_1_constraint; + c = &counter0_1_constraint; + + return part_constraint(cpuc, idx, event, c); } switch (c->idxmsk64 & 0x3ull) { case 0x1: - return &counter0_constraint; + c = &counter0_constraint; + break; case 0x2: - return &counter1_constraint; + c = &counter1_constraint; + break; case 0x3: - return &counter0_1_constraint; + c = &counter0_1_constraint; + break; + default: + c = &emptyconstraint; + break; } - return &emptyconstraint; + return part_constraint(cpuc, idx, event, c); } return c; @@ -5744,7 +5755,7 @@ rwc_get_event_constraints(struct cpu_hw_events *cpuc, int idx, */ if (event->attr.precise_ip == 3) return &emptyconstraint; - return &counters_1_7_constraint; + return part_constraint(cpuc, idx, event, &counters_1_7_constraint); } return c; -- 2.55.0 With PMU partitioning support, the set of PMU counters available to a PMU can vary from the architectural maximum. While this information is available through CPUID, it is not easily consumable from scripts. Add two new sysfs ABI files, gp_counters and fixed_counters, under the PMU caps group to export the available counters. The fixed_counters file is omitted when the PMU does not support fixed-function counters. $ grep . /sys/devices/cpu/caps/*_counters /sys/devices/cpu/caps/fixed_counters:0-3 /sys/devices/cpu/caps/gp_counters:0-7 Suggested-by: Andi Kleen Signed-off-by: Zide Chen --- .../sysfs-bus-event_source-devices-caps | 5 +++ arch/x86/events/core.c | 37 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/Documentation/ABI/testing/sysfs-bus-event_source-devices-caps b/Documentation/ABI/testing/sysfs-bus-event_source-devices-caps index a5f506f7d481..385ad7a23dd6 100644 --- a/Documentation/ABI/testing/sysfs-bus-event_source-devices-caps +++ b/Documentation/ABI/testing/sysfs-bus-event_source-devices-caps @@ -22,3 +22,8 @@ Description: of PERF_SAMPLE_BRANCH_COUNTERS, while the "branch_counter_width" exposes the width of each counter. Both of them can be used by the perf tool to parse the logged counters in each branch. + + The "gp_counters" and "fixed_counters" attributes expose the + available general-purpose and fixed-function PMU counters as + a range list, e.g. "0-7". The "fixed_counters" attribute is + omitted if the PMU does not support fixed-function counters. diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 76af9cfaf0ad..282170f7c1e1 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -2814,14 +2814,51 @@ static ssize_t max_precise_show(struct device *cdev, static DEVICE_ATTR_RO(max_precise); +static ssize_t gp_counters_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct pmu *pmu = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%*pbl\n", INTEL_PMC_MAX_GENERIC, + hybrid(pmu, cntr_mask)); +} + +static ssize_t fixed_counters_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct pmu *pmu = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%*pbl\n", INTEL_PMC_MAX_FIXED, + hybrid(pmu, fixed_cntr_mask)); +} + +static DEVICE_ATTR_RO(gp_counters); +static DEVICE_ATTR_RO(fixed_counters); + +static umode_t x86_pmu_caps_is_visible(struct kobject *kobj, + struct attribute *attr, int n) +{ + struct device *dev = kobj_to_dev(kobj); + struct pmu *pmu = dev_get_drvdata(dev); + + if (attr == &dev_attr_fixed_counters.attr && + !hybrid(pmu, fixed_cntr_mask64)) + return 0; + + return attr->mode; +} + static struct attribute *x86_pmu_caps_attrs[] = { &dev_attr_max_precise.attr, + &dev_attr_gp_counters.attr, + &dev_attr_fixed_counters.attr, NULL }; static struct attribute_group x86_pmu_caps_group __ro_after_init = { .name = "caps", .attrs = x86_pmu_caps_attrs, + .is_visible = x86_pmu_caps_is_visible, }; static const struct attribute_group *x86_pmu_attr_groups[] = { -- 2.55.0 From: Kan Liang Without PMU partitioning, a PMU with PERF_PMU_CAP_MEDIATED_VPMU blocks host event scheduling while the guest PMU context is loaded. When PMU partitioning is enabled, host-owned counters remain available. Skip exclude_guest events, but continue to allow other events to be scheduled on the remaining host-owned counters. Signed-off-by: Kan Liang Co-developed-by: Zide Chen Signed-off-by: Zide Chen --- kernel/events/core.c | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/kernel/events/core.c b/kernel/events/core.c index 9ce27cd83e04..23375f8d2261 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -4105,6 +4105,32 @@ struct merge_sched_data { enum event_type_t event_type; }; +static bool +perf_mediated_pmu_skip_event(struct perf_event *event, enum event_type_t event_type) +{ + if (event->pmu_ctx->pmu->capabilities & PERF_PMU_CAP_PMU_PARTITION) { + /* + * EVENT_GUEST && !guest_ctx_loaded: perf_load_guest_context() + * reschedules !exclude_guest events. + * + * !EVENT_GUEST && guest_ctx_loaded: Guest context loaded. + */ + if (event->attr.exclude_guest && + !!(event_type & EVENT_GUEST) != is_guest_mediated_pmu_loaded()) + return true; + } else { + /* + * Don't schedule in any host events because no counters + * are available. + */ + if (!(event_type & EVENT_GUEST) && + is_guest_mediated_pmu_loaded()) + return true; + } + + return false; +} + static int merge_sched_in(struct perf_event *event, void *data) { struct perf_event_context *ctx = event->ctx; @@ -4116,13 +4142,8 @@ static int merge_sched_in(struct perf_event *event, void *data) if (!event_filter_match(event)) return 0; - /* - * Don't schedule in any host events from PMU with - * PERF_PMU_CAP_MEDIATED_VPMU, while a guest is running. - */ - if (is_guest_mediated_pmu_loaded() && - event->pmu_ctx->pmu->capabilities & PERF_PMU_CAP_MEDIATED_VPMU && - !(msd->event_type & EVENT_GUEST)) + if ((event->pmu_ctx->pmu->capabilities & PERF_PMU_CAP_MEDIATED_VPMU) && + perf_mediated_pmu_skip_event(event, msd->event_type)) return 0; if (group_can_go_on(event, msd->can_add_hw)) { -- 2.55.0 From: Kan Liang When entering a guest with PMU partitioning enabled, some counters remain available to the host. Similar to the non-partitioning setup, schedule out all events because a counter may become guest-owned. Unlike the non-partitioned case, host events must then be rescheduled so that !exclude_guest events can be scheduled onto the remaining host-owned counters, while keeping exclude_guest events off. Since PMU partitioning shrinks the counters available to host events, add a lightweight PMU-context reschedule cpuctx_sched_in_all() instead of rebuilding the entire CPU perf scheduling state. When exiting the guest, schedule out host events so that they can be rescheduled against the expanded set of host-owned counters. Similar to guest entry, only PMU-level event placement needs to be rebuilt, so add ctx_sched_out_all() instead of using the existing full CPU-context scheduling API. These context switches rely on the architectural PMU partition mask being configured for the target context. On guest entry, the guest PMU partition mask is expected to be active before events are rescheduled. On guest exit, it is expected to be disabled so that events are rescheduled with host counter constraints. This differs from the perf core guest context, where guest_ctx_loaded is updated at the end of the load/put guest context. As a result, guest_ctx_loaded does not yet reflect the target context when the rescheduling occurs. Add pmu_partition_enabled to perf_{load,put}_guest_context(), and hardcode to false temporarily until later patches. Signed-off-by: Kan Liang Co-developed-by: Zide Chen Signed-off-by: Zide Chen --- arch/x86/kvm/pmu.c | 4 +-- include/linux/perf_event.h | 4 +-- kernel/events/core.c | 52 +++++++++++++++++++++++++++++++++++--- 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index 7f619a99a152..c022337d0bec 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -1390,7 +1390,7 @@ void kvm_mediated_pmu_load(struct kvm_vcpu *vcpu) perf_pmu_partition_preload(); - perf_load_guest_context(); + perf_load_guest_context(false); /* * Explicitly clear PERF_GLOBAL_CTRL, as "loading" the guest's context @@ -1463,5 +1463,5 @@ void kvm_mediated_pmu_put(struct kvm_vcpu *vcpu) perf_put_guest_lvtpc(); - perf_put_guest_context(); + perf_put_guest_context(false); } diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index bd952e09055d..9f66a4c49256 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1934,8 +1934,8 @@ extern u64 perf_event_pause(struct perf_event *event, bool reset); #ifdef CONFIG_PERF_GUEST_MEDIATED_PMU int perf_create_mediated_pmu(u64 pmu_partition_mask); void perf_release_mediated_pmu(void); -void perf_load_guest_context(void); -void perf_put_guest_context(void); +void perf_load_guest_context(bool pmu_partition_enabled); +void perf_put_guest_context(bool pmu_partition_enabled); int arch_perf_set_pmu_partition_mask(u64 pmu_partition_mask); #endif diff --git a/kernel/events/core.c b/kernel/events/core.c index 23375f8d2261..c9e7a2f0edc8 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -6442,8 +6442,25 @@ void perf_release_mediated_pmu(void) } EXPORT_SYMBOL_FOR_KVM(perf_release_mediated_pmu); -/* When loading a guest's mediated PMU, schedule out all exclude_guest events. */ -void perf_load_guest_context(void) +static void cpuctx_sched_in_all(struct perf_cpu_context *cpuctx, + enum event_type_t type) +{ + struct perf_event_pmu_context *pmu_ctx; + + for_each_epc(pmu_ctx, &cpuctx->ctx, NULL, EVENT_GUEST) + __pmu_ctx_sched_in(pmu_ctx, type | EVENT_GUEST); + + if (cpuctx->task_ctx) { + for_each_epc(pmu_ctx, cpuctx->task_ctx, NULL, EVENT_GUEST) + __pmu_ctx_sched_in(pmu_ctx, type | EVENT_GUEST); + } +} + +/* + * When loading a guest's mediated PMU, schedule out all exclude_guest events. + * In PMU partitioning, reschedule host events onto host-owned counters. + */ +void perf_load_guest_context(bool pmu_partition_enabled) { struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); @@ -6461,6 +6478,12 @@ void perf_load_guest_context(void) task_ctx_sched_out(cpuctx->task_ctx, NULL, EVENT_GUEST); } + /* Reschedule !exclude_guest events onto host-owned counters. */ + if (pmu_partition_enabled) { + cpuctx_sched_in_all(cpuctx, EVENT_PINNED); + cpuctx_sched_in_all(cpuctx, EVENT_FLEXIBLE); + } + perf_ctx_enable(&cpuctx->ctx, EVENT_GUEST); if (cpuctx->task_ctx) perf_ctx_enable(cpuctx->task_ctx, EVENT_GUEST); @@ -6469,7 +6492,21 @@ void perf_load_guest_context(void) } EXPORT_SYMBOL_GPL(perf_load_guest_context); -void perf_put_guest_context(void) +static void ctx_sched_out_all(struct perf_event_context *ctx) +{ + struct perf_event_pmu_context *pmu_ctx; + + if (!ctx) + return; + + list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) { + if (perf_skip_pmu_ctx(pmu_ctx, EVENT_GUEST)) + continue; + __pmu_ctx_sched_out(pmu_ctx, EVENT_ALL); + } +} + +void perf_put_guest_context(bool pmu_partition_enabled) { struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); @@ -6484,6 +6521,15 @@ void perf_put_guest_context(void) if (cpuctx->task_ctx) perf_ctx_disable(cpuctx->task_ctx, EVENT_GUEST); + if (pmu_partition_enabled) { + ctx_time_update(cpuctx, &cpuctx->ctx); + if (cpuctx->task_ctx) + ctx_time_update(cpuctx, cpuctx->task_ctx); + + ctx_sched_out_all(&cpuctx->ctx); + ctx_sched_out_all(cpuctx->task_ctx); + } + perf_event_sched_in(cpuctx, cpuctx->task_ctx, NULL, EVENT_GUEST); if (cpuctx->task_ctx) -- 2.55.0 With PMU partitioning, the host is allowed to create !exclude_guest events because it no longer yields all PMU resources to the guest while the guest is running. However, Perf Metrics, LBR, BTS, and PEBS cannot be shared between host and guest. Host events that rely on these exclusive facilities must not be scheduled in if the facilities are guest-owned. Intel PT PMU is special: Since it can't be partitioned, supporting Intel PT passthrough requires heterogeneous mediated vPMUs. Additional work is needed to handle nr_include_guest_events accounting. Signed-off-by: Zide Chen --- arch/x86/events/intel/core.c | 35 ++++++++++++++++++++++++++++++- arch/x86/include/asm/perf_event.h | 1 + kernel/events/core.c | 16 ++++++++++++-- 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 0f76e56fd2db..2928c8262fef 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -4455,10 +4455,40 @@ dyn_constraint(struct cpu_hw_events *cpuc, struct event_constraint *c, int idx) return c; } +static bool event_uses_guest_owned_facility(struct perf_event *event) +{ + /* + * For Intel platforms, PMU partition mask shares the same bit layout + * as IA32_PERF_GLOBAL_STATUS. + */ + u64 partition_mask = x86_pmu_current_partition_mask(); + + if ((partition_mask & GLOBAL_STATUS_PERF_METRICS_OVF) && + is_topdown_event(event)) + return true; + + if ((partition_mask & GLOBAL_STATUS_LBRS_FROZEN) && + needs_branch_stack(event)) + return true; + + if (partition_mask & + (GLOBAL_STATUS_BUFFER_OVF | GLOBAL_STATUS_ARCH_PEBS_THRESHOLD)) { + if (event->attr.precise_ip || is_pebs_counter_event_group(event)) + return true; + + if ((partition_mask & GLOBAL_STATUS_BUFFER_OVF) && + intel_pmu_has_bts(event)) + return true; + } + + return false; +} + /* * Mask out guest-owned counters from a constraint when PMU partition has been * entered, so !exclude_guest host events are not scheduled onto them while - * the CPU is in non-root mode. + * the CPU is in non-root mode. Reject the event when a guest is currently + * loaded and it needs a guest-owned exclusive facility. * * This is also used by PMU-specific get_event_constraints() wrappers * that hard-code a static, counter-specific constraint. @@ -4475,6 +4505,9 @@ part_constraint(struct cpu_hw_events *cpuc, int idx, return c; if (x86_pmu_partition_loaded(cpuc)) { + if (event_uses_guest_owned_facility(event)) + return &emptyconstraint; + c = dyn_constraint(cpuc, c, idx); c->idxmsk64 &= ~x86_pmu_current_partition_mask(); c->weight = hweight64(c->idxmsk64); diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h index 18f1ac5e008b..aaaa34062f8c 100644 --- a/arch/x86/include/asm/perf_event.h +++ b/arch/x86/include/asm/perf_event.h @@ -453,6 +453,7 @@ static inline bool is_topdown_idx(int idx) #define GLOBAL_STATUS_ARCH_PEBS_THRESHOLD_BIT 54 #define GLOBAL_STATUS_ARCH_PEBS_THRESHOLD BIT_ULL(GLOBAL_STATUS_ARCH_PEBS_THRESHOLD_BIT) #define GLOBAL_STATUS_PERF_METRICS_OVF_BIT 48 +#define GLOBAL_STATUS_PERF_METRICS_OVF BIT_ULL(GLOBAL_STATUS_PERF_METRICS_OVF_BIT) #define GLOBAL_CTRL_EN_PERF_METRICS BIT_ULL(48) /* diff --git a/kernel/events/core.c b/kernel/events/core.c index c9e7a2f0edc8..1ae52a0ce234 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -6381,11 +6381,23 @@ static int mediated_pmu_account_event(struct perf_event *event) if (!is_include_guest_event(event)) return 0; + /* + * This lockless fast path assumes that heterogeneous mediated vPMUs + * are not supported, i.e. a mix of PERF_PMU_CAP_MEDIATED_VPMU PMUs + * with and without PERF_PMU_CAP_PMU_PARTITION. + */ if (atomic_inc_not_zero(&nr_include_guest_events)) return 0; guard(mutex)(&perf_mediated_pmu_mutex); - if (atomic_read(&nr_mediated_pmu_vms)) + + /* + * PMU partitioning allows scheduling !exclude_guest events while a + * guest is running. However, it is up to the PMU driver to validate + * whether the facilities needed by the event are available on the host. + */ + if (atomic_read(&nr_mediated_pmu_vms) && + !(event->pmu->capabilities & PERF_PMU_CAP_PMU_PARTITION)) return -EOPNOTSUPP; atomic_inc(&nr_include_guest_events); @@ -6418,7 +6430,7 @@ int perf_create_mediated_pmu(u64 partition_mask) int ret; guard(mutex)(&perf_mediated_pmu_mutex); - if (atomic_read(&nr_include_guest_events)) + if (atomic_read(&nr_include_guest_events) && !partition_mask) return -EBUSY; ret = arch_perf_set_pmu_partition_mask(partition_mask); -- 2.55.0 Only Intel CPUs support PerfMon masking, so this new parameter is Intel-specific. Mediated vPMU must be enabled for PerfMon masking. PerfMon masking lets a VMM partition PMU resources between host and guest: each mask bit determines whether the guest (set) or host (clear) owns a counter's MSR(s), RDPMC access, and the corresponding bit in the global MSRs (e.g. IA32_PERF_GLOBAL_CTRL) in non-root mode. The setting is system-wide and caps what any individual guest may be given; each guest may configure a subset of it. The parameter is configured with a text-based, semicolon-separated list of terms. Currently supported for a mediated vPMU guest: guest_gp= general purpose counters (0-based) owned by the guest guest_fixed= fixed counters (0-based) owned by the guest perf_metrics PERF_METRICS is owned by the guest For example: kvm-intel.perfmon_mask=guest_gp=0-3;guest_fixed=0,2-3;perf_metrics Internally, the parsed terms are folded into a 64-bit perfmon_mask variable that uses the same layout as the PERFMON_MASK VMCS field, i.e. the IA32_PERF_GLOBAL_STATUS layout. An empty (default) value behaves the same as a plain mediated vPMU. Using IA32_PERF_GLOBAL_CTRL in the generic VM-exit MSR-store area while the PerfMon masking VM-execution control is set is undefined, so PerfMon masking requires the dedicated Save-IA32_PERF_GLOBAL_CTRL VM-exit control. cpu_has_vmx_perfmon_mask() is hardcoded to false temporarily until later patches. Signed-off-by: Zide Chen --- .../admin-guide/kernel-parameters.txt | 30 ++++++++ arch/x86/kvm/pmu.c | 3 + arch/x86/kvm/pmu.h | 1 + arch/x86/kvm/vmx/capabilities.h | 5 ++ arch/x86/kvm/vmx/pmu_intel.c | 68 +++++++++++++++++++ arch/x86/kvm/vmx/vmx.c | 54 +++++++++++++++ arch/x86/kvm/vmx/vmx.h | 1 + arch/x86/kvm/x86.c | 2 +- 8 files changed, 163 insertions(+), 1 deletion(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index b5493a7f8f22..e0ff16746a08 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3284,6 +3284,36 @@ Kernel parameters [KVM,Intel] Control nested virtualization feature in KVM/VMX. Default is 1 (enabled). + kvm-intel.perfmon_mask= + [KVM,Intel] Defines the host-wide PMU resource + partition between guest and host. Resources assigned + to the guest are unavailable to the host, and vice + versa, while a guest is running. + + The value is a semicolon-separated list of terms: + + guest_gp= General purpose counters + (0-based) assigned to the guest. + guest_fixed= Fixed counters (0-based) assigned + to the guest. + perf_metrics PERF_METRICS is assigned to the + guest. Requires fixed counter 3 + to also be assigned to the + guest. + + is a comma-separated list of numbers and/or + ranges, e.g. "0-2,5". Terms and the resources they + don't mention default to being host-owned. For + example: + + kvm-intel.perfmon_mask=guest_gp=0-3;guest_fixed=0-1,3;perf_metrics + + assigns general purpose counters 0-3, fixed counters + 0, 1 and 3, and PERF_METRICS to the guest, while all + other resources remain with the host. + + Default is "" (disabled). + kvm-intel.unrestricted_guest= [KVM,Intel] Control KVM's use of unrestricted guest feature (virtualized real and unpaged mode). Default diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index c022337d0bec..92ff685d11b3 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -43,6 +43,9 @@ module_param(enable_pmu, bool, 0444); bool __read_mostly enable_mediated_pmu; EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_mediated_pmu); +u64 __read_mostly perfmon_mask; +EXPORT_SYMBOL_FOR_KVM_INTERNAL(perfmon_mask); + struct kvm_x86_pmu_event_filter { __u32 action; __u32 nevents; diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h index 8322bbed2d64..2dc12e3f3af0 100644 --- a/arch/x86/kvm/pmu.h +++ b/arch/x86/kvm/pmu.h @@ -64,6 +64,7 @@ struct kvm_pmu_ops { extern bool enable_pmu; extern bool enable_mediated_pmu; +extern u64 perfmon_mask; void kvm_pmu_ops_update(const struct kvm_pmu_ops *pmu_ops); diff --git a/arch/x86/kvm/vmx/capabilities.h b/arch/x86/kvm/vmx/capabilities.h index 810119167f79..d4c362093966 100644 --- a/arch/x86/kvm/vmx/capabilities.h +++ b/arch/x86/kvm/vmx/capabilities.h @@ -296,6 +296,11 @@ static inline bool cpu_has_vmx_ipiv(void) return vmcs_config.cpu_based_3rd_exec_ctrl & TERTIARY_EXEC_IPI_VIRT; } +static inline bool cpu_has_vmx_perfmon_mask(void) +{ + return false; +} + static inline bool cpu_has_vmx_flexpriority(void) { return cpu_has_vmx_tpr_shadow() && diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index e2e51006ca47..62e542eac05e 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -955,6 +955,74 @@ static void intel_mediated_pmu_put(struct kvm_vcpu *vcpu) } } +static bool intel_pmu_validate_perfmon_mask(void) +{ + u64 guest_fixed_mask, guest_gp_mask; + + /* + * Combining VM-exit MSR-store with PerfMon masking produces + * undefined behavior, so it requires hardware support for this + * dedicated save control. + */ + if (!cpu_has_save_perf_global_ctrl()) + return false; + + guest_gp_mask = perfmon_mask & GENMASK_ULL(INTEL_PMC_MAX_GENERIC - 1, 0); + guest_fixed_mask = perfmon_mask >> INTEL_PMC_IDX_FIXED; + guest_fixed_mask &= GENMASK_ULL(INTEL_PMC_MAX_FIXED - 1, 0); + + if ((guest_fixed_mask & ~kvm_pmu_cap.fixed_cntr_mask64) || + (guest_gp_mask & ~kvm_pmu_cap.cntr_mask64)) + return false; + + /* + * Without KVM Arch PerfMon extension support, the guest cannot own + * non-contiguous GP counters. + */ + if (guest_gp_mask & (guest_gp_mask + 1)) + return false; + + if ((perfmon_mask & GLOBAL_STATUS_PERF_METRICS_OVF) && + !(kvm_host.perf_capabilities & PERF_CAP_PERF_METRICS)) + return false; + + /* + * PERF_METRICS and fixed counter 3 must both be host-owned or both + * guest-owned. + */ + if (!!(perfmon_mask & BIT_ULL(GLOBAL_STATUS_PERF_METRICS_OVF_BIT)) != + !!(perfmon_mask & BIT_ULL(INTEL_PMC_IDX_FIXED + 3))) + return false; + + /* + * The guest must not own all PMU counters. Otherwise, the configuration + * degenerates into plain mediated vPMU and adds unnecessary complexity + * to the perf scheduler. + */ + if ((guest_fixed_mask == kvm_host_pmu.fixed_cntr_mask64) && + (guest_gp_mask == kvm_host_pmu.cntr_mask64)) + return false; + + return true; +} + +void intel_pmu_perfmon_mask_setup(void) +{ + if (!perfmon_mask) + return; + + if (!enable_mediated_pmu || !cpu_has_vmx_perfmon_mask()) { + perfmon_mask = 0; + return; + } + + if (!intel_pmu_validate_perfmon_mask()) { + pr_warn("Invalid perfmon_mask=%#llx, disabling PerfMon masking\n", + perfmon_mask); + perfmon_mask = 0; + } +} + struct kvm_pmu_ops intel_pmu_ops __initdata = { .emulate_rdpmc = intel_emulate_rdpmc, .msr_idx_to_pmc = intel_msr_idx_to_pmc, diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index ded63e2e39ce..cdd141d22efa 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -14,6 +14,7 @@ */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include #include #include @@ -25,6 +26,7 @@ #include #include #include +#include #include #include @@ -164,6 +166,56 @@ module_param(allow_smaller_maxphyaddr, bool, S_IRUGO); module_param(enable_mediated_pmu, bool, 0444); +/* + * See the "kvm-intel.perfmon_mask" entry in + * Documentation/admin-guide/kernel-parameters.txt for the full syntax. + * Example: kvm-intel.perfmon_mask=guest_gp=0-3;guest_fixed=0-1,3;perf_metrics + */ +static int perfmon_mask_set(const char *val, const struct kernel_param *kp) +{ + unsigned long gp_bitmap = 0, fixed_bitmap = 0; + char *buf, *orig, *tok; + size_t prefix_len; + u64 mask = 0; + int r = 0; + + buf = orig = kstrdup(val, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + while ((tok = strsep(&buf, ";")) != NULL) { + if (!*tok) + continue; + + if (!strcmp(tok, "perf_metrics")) { + mask |= GLOBAL_STATUS_PERF_METRICS_OVF; + } else if ((prefix_len = str_has_prefix(tok, "guest_gp="))) { + r = bitmap_parselist(tok + prefix_len, &gp_bitmap, + INTEL_PMC_MAX_GENERIC); + } else if ((prefix_len = str_has_prefix(tok, "guest_fixed="))) { + r = bitmap_parselist(tok + prefix_len, &fixed_bitmap, + INTEL_PMC_MAX_FIXED); + } else { + r = -EINVAL; + } + + if (r) + goto out; + } + + mask |= gp_bitmap | ((u64)fixed_bitmap << INTEL_PMC_IDX_FIXED); + *(u64 *)kp->arg = mask; +out: + kfree(orig); + return r; +} + +static const struct kernel_param_ops perfmon_mask_ops = { + .set = perfmon_mask_set, + .get = param_get_ullong, +}; +module_param_cb(perfmon_mask, &perfmon_mask_ops, &perfmon_mask, 0444); + #define KVM_VM_CR0_ALWAYS_OFF (X86_CR0_NW | X86_CR0_CD) #define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR0_NE #define KVM_VM_CR0_ALWAYS_ON \ @@ -8824,6 +8876,8 @@ __init int vmx_hardware_setup(void) else vt_init_ops.handle_intel_pt_intr = NULL; + intel_pmu_perfmon_mask_setup(); + setup_default_sgx_lepubkeyhash(); vmx_set_cpu_caps(); diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index dc8517f15bc4..ccda5c5c8c2c 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -667,6 +667,7 @@ static __always_inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu) void intel_pmu_cross_mapped_check(struct kvm_pmu *pmu); int intel_pmu_create_guest_lbr_event(struct kvm_vcpu *vcpu); void vmx_passthrough_lbr_msrs(struct kvm_vcpu *vcpu); +void intel_pmu_perfmon_mask_setup(void); struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags); void free_vmcs(struct vmcs *vmcs); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 5f3215915c76..26a3b7a267b5 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -9328,7 +9328,7 @@ int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) if (enable_mediated_pmu && kvm->arch.enable_pmu && !kvm->arch.created_mediated_pmu) { if (irqchip_in_kernel(kvm)) { - r = perf_create_mediated_pmu(0); + r = perf_create_mediated_pmu(perfmon_mask); if (r) { pr_warn_ratelimited(PERF_MEDIATED_PMU_MSG); return r; -- 2.55.0 Program the VMCS PERFMON_MASK field when PerfMon masking is enabled. The field uses the same bit definitions as IA32_PERF_GLOBAL_STATUS and can therefore be derived from ~pmu->global_status_rsvd. Expose IA32_PERF_CAPABILITIES.PERF_METRICS_AVAILABLE[15] only when PERFMON_MASK[48] is available to the guest. When PerfMon masking is enabled, guest RDMSR/WRMSR accesses to PMU global MSRs are filtered by the mask, writes outside the mask cause an #GP(0), and RDPMC returns only guest-owned counter values. Intentionally defer enabling PerfMon masking in VM-execution control until the remaining PerfMon masking support is in place, so that intermediate commits remain functional during bisection. Signed-off-by: Zide Chen --- arch/x86/include/asm/kvm_host.h | 1 + arch/x86/include/asm/vmx.h | 2 ++ arch/x86/kvm/pmu.c | 4 ++-- arch/x86/kvm/pmu.h | 6 ++++++ arch/x86/kvm/vmx/pmu_intel.c | 26 ++++++++++++++++++++++++++ arch/x86/kvm/vmx/vmx.c | 6 +++++- 6 files changed, 42 insertions(+), 3 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 262553f95793..467090bf0ab2 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -582,6 +582,7 @@ struct kvm_pmu { u64 eventsel_rsvd; u64 raw_event_mask; u64 perf_metrics; + u64 perfmon_mask; struct kvm_pmc gp_counters[KVM_MAX_NR_GP_COUNTERS]; struct kvm_pmc fixed_counters[KVM_MAX_NR_FIXED_COUNTERS]; diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h index 3f1b3096ff04..1cb092d86955 100644 --- a/arch/x86/include/asm/vmx.h +++ b/arch/x86/include/asm/vmx.h @@ -275,6 +275,8 @@ enum vmcs_field { SHARED_EPT_POINTER = 0x0000203C, PID_POINTER_TABLE = 0x00002042, PID_POINTER_TABLE_HIGH = 0x00002043, + PERFMON_MASK = 0x00002054, + PERFMON_MASK_HIGH = 0x00002055, GUEST_PHYSICAL_ADDRESS = 0x00002400, GUEST_PHYSICAL_ADDRESS_HIGH = 0x00002401, VMCS_LINK_POINTER = 0x00002800, diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index 92ff685d11b3..f944a15160cb 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -1393,7 +1393,7 @@ void kvm_mediated_pmu_load(struct kvm_vcpu *vcpu) perf_pmu_partition_preload(); - perf_load_guest_context(false); + perf_load_guest_context(kvm_vcpu_has_perfmon_mask(vcpu)); /* * Explicitly clear PERF_GLOBAL_CTRL, as "loading" the guest's context @@ -1466,5 +1466,5 @@ void kvm_mediated_pmu_put(struct kvm_vcpu *vcpu) perf_put_guest_lvtpc(); - perf_put_guest_context(false); + perf_put_guest_context(kvm_vcpu_has_perfmon_mask(vcpu)); } diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h index 2dc12e3f3af0..057e3258e473 100644 --- a/arch/x86/kvm/pmu.h +++ b/arch/x86/kvm/pmu.h @@ -89,6 +89,12 @@ static inline bool kvm_vcpu_has_mediated_pmu(struct kvm_vcpu *vcpu) return enable_mediated_pmu && vcpu_to_pmu(vcpu)->version; } +static inline bool kvm_vcpu_has_perfmon_mask(struct kvm_vcpu *vcpu) +{ + return kvm_vcpu_has_mediated_pmu(vcpu) && + vcpu_to_pmu(vcpu)->perfmon_mask; +} + static inline unsigned long kvm_gp_pmc_mask(struct kvm_pmu *pmu) { return pmu->pmc_exists64 & diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index 62e542eac05e..19ccc7cd319c 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -703,6 +703,21 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu) pmu->pebs_enable_rsvd = ~kvm_gp_pmc_mask(pmu); } } + + if (kvm_vcpu_has_mediated_pmu(vcpu) && perfmon_mask) { + pmu->perfmon_mask = ~pmu->global_status_rsvd; + + /* + * The PerfMon mask for a particular guest must be a subset + * of the module-wide mask. This masks out the global bits + * (e.g. GLOBAL_STATUS_COND_CHG) that must be handled by the + * host and were removed from global_status_rsvd without + * checking perfmon_mask, and defends in depth against any + * other bits inadvertently granted to the guest. + */ + pmu->perfmon_mask &= perfmon_mask; + vmcs_write64(PERFMON_MASK, pmu->perfmon_mask); + } } static void intel_pmu_init(struct kvm_vcpu *vcpu) @@ -739,6 +754,7 @@ static void intel_pmu_reset(struct kvm_vcpu *vcpu) struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); pmu->perf_metrics = 0; + pmu->perfmon_mask = 0; intel_pmu_release_guest_lbr_event(vcpu); } @@ -1021,6 +1037,16 @@ void intel_pmu_perfmon_mask_setup(void) perfmon_mask); perfmon_mask = 0; } + + /* + * perfmon_mask represents the maximum resources that any guest may + * have. KVM chooses to expose fewer hardware resources to guests. + */ + if (perfmon_mask) { + kvm_pmu_cap.cntr_mask64 &= perfmon_mask; + kvm_pmu_cap.fixed_cntr_mask64 &= + (perfmon_mask >> INTEL_PMC_IDX_FIXED); + } } struct kvm_pmu_ops intel_pmu_ops __initdata = { diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index cdd141d22efa..1b42c9d6f168 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -4294,6 +4294,9 @@ static void vmx_recalc_pmu_msr_intercepts(struct kvm_vcpu *vcpu) if (!cpu_has_save_perf_global_ctrl()) { vm_exit_controls_bits &= ~VM_EXIT_SAVE_IA32_PERF_GLOBAL_CTRL; + /* Module parameter validation should already prevent this. */ + WARN_ON_ONCE(kvm_vcpu_has_perfmon_mask(vcpu)); + if (has_mediated_pmu) vmx_add_autostore_msr(vmx, MSR_CORE_PERF_GLOBAL_CTRL); else @@ -8133,7 +8136,8 @@ static __init u64 vmx_get_perf_capabilities(void) perf_cap &= ~PERF_CAP_PEBS_BASELINE; } - if (enable_mediated_pmu) + if (enable_mediated_pmu && + (!perfmon_mask || (perfmon_mask & GLOBAL_STATUS_PERF_METRICS_OVF))) perf_cap |= kvm_host.perf_capabilities & PERF_CAP_PERF_METRICS; return perf_cap; -- 2.55.0 The static system-wide mask x86_pmu.partition_mask defines the global maximum guest capability, while a guest may have its own mask, which is a subset of x86_pmu.partition_mask. The per-guest mask, which on Intel is backed by the PERFMON_MASK VMCS field, is the effective mask while a vCPU is loaded. Whenever it changes -- e.g. during kvm_arch_vcpu_{load,put}() -- perf/x86 needs to be told so it can keep its own per-CPU copy in sync. Suggested-by: Andi Kleen Signed-off-by: Zide Chen --- arch/x86/events/core.c | 8 ++++++++ arch/x86/include/asm/perf_event.h | 1 + arch/x86/kvm/pmu.c | 13 +++++++++++++ arch/x86/kvm/pmu.h | 3 +++ arch/x86/kvm/x86.c | 4 ++++ 5 files changed, 29 insertions(+) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 282170f7c1e1..f485f927967a 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -1883,6 +1883,14 @@ void perf_put_guest_lvtpc(void) apic_write(APIC_LVTPC, APIC_DM_NMI); } EXPORT_SYMBOL_FOR_KVM(perf_put_guest_lvtpc); + +void perf_set_current_partition_mask(u64 mask) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + + cpuc->partition_mask = mask & READ_ONCE(x86_pmu.partition_mask); +} +EXPORT_SYMBOL_FOR_KVM(perf_set_current_partition_mask); #endif /* CONFIG_PERF_GUEST_MEDIATED_PMU */ static int diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h index aaaa34062f8c..5be4f6a93b14 100644 --- a/arch/x86/include/asm/perf_event.h +++ b/arch/x86/include/asm/perf_event.h @@ -794,6 +794,7 @@ static inline void perf_check_microcode(void) { } extern void perf_pmu_partition_preload(void); extern void perf_load_guest_lvtpc(u32 guest_lvtpc); extern void perf_put_guest_lvtpc(void); +extern void perf_set_current_partition_mask(u64 mask); #endif #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL) diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index f944a15160cb..c447f32c1cb1 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -46,6 +46,19 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_mediated_pmu); u64 __read_mostly perfmon_mask; EXPORT_SYMBOL_FOR_KVM_INTERNAL(perfmon_mask); +void kvm_pmu_vcpu_load(struct kvm_vcpu *vcpu) +{ + struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); + + perf_set_current_partition_mask(kvm_vcpu_has_perfmon_mask(vcpu) ? + pmu->perfmon_mask : 0); +} + +void kvm_pmu_vcpu_put(struct kvm_vcpu *vcpu) +{ + perf_set_current_partition_mask(0); +} + struct kvm_x86_pmu_event_filter { __u32 action; __u32 nevents; diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h index 057e3258e473..c5feeb60bcf6 100644 --- a/arch/x86/kvm/pmu.h +++ b/arch/x86/kvm/pmu.h @@ -70,6 +70,9 @@ void kvm_pmu_ops_update(const struct kvm_pmu_ops *pmu_ops); void kvm_handle_guest_mediated_pmi(void); +void kvm_pmu_vcpu_load(struct kvm_vcpu *vcpu); +void kvm_pmu_vcpu_put(struct kvm_vcpu *vcpu); + static inline bool kvm_pmu_has_perf_global_ctrl(struct kvm_pmu *pmu) { /* diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 26a3b7a267b5..07e3fe5b53f8 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -2538,6 +2538,8 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) kvm_request_l1tf_flush_l1d(); + kvm_pmu_vcpu_load(vcpu); + if (vcpu->scheduled_out && pmu->version && pmu->event_count) { pmu->need_cleanup = true; kvm_make_request(KVM_REQ_PMU, vcpu); @@ -2686,6 +2688,8 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) kvm_x86_call(vcpu_put)(vcpu); vcpu->arch.last_host_tsc = rdtsc(); + + kvm_pmu_vcpu_put(vcpu); } static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu, -- 2.55.0 When a PerfMon mask is configured for the guest, hardware already delivers #GP(0) on RDPMC for any counter or PERF_METRICS whose corresponding PerfMon mask bit is clear, so KVM does not need to intercept RDPMC, nor separately gate access in software, to enforce the same restriction. No changes are needed in intel_emulate_rdpmc(): the existing checks already fully reflect the PerfMon mask restrictions for GP counters, fixed counters, and PERF_METRICS respectively. Similarly, guest reads of IA32_PERF_GLOBAL_CTRL are subject to a logical AND with the PerfMon mask, and writes cause a #GP(0) if the guest attempts to set a bit whose corresponding bit in the PerfMon mask is clear. Signed-off-by: Zide Chen --- arch/x86/kvm/pmu.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index c447f32c1cb1..337d2f55a216 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -808,6 +808,14 @@ static bool kvm_need_any_pmc_intercept(struct kvm_vcpu *vcpu) if (!kvm_vcpu_has_mediated_pmu(vcpu)) return true; + /* + * When PerfMon mask is enabled, KVM need not intercept RDPMC or + * accesses to IA32_PERF_GLOBAL_CTRL, as hardware blocks or filters + * out access to non-guest-owned RDPMC indices or GLOBAL_CTRL bits. + */ + if (kvm_vcpu_has_perfmon_mask(vcpu)) + return false; + /* * Note! Check *host* PMU capabilities, not KVM's PMU capabilities, as * KVM's capabilities are constrained based on KVM support, i.e. KVM's -- 2.55.0 PerfMon masking affects reads from and writes to this MSR. The effective mask is defined as bits 4n+3:4n and bits 4n+35:4n+32 being set if and only if bit 32+n of the PerfMon mask is set (for 0 <= n <= 7). KVM doesn't support some fixed-counter features, and pmu->fixed_ctr_ctrl_rsvd is a subset of the effective mask above, so the existing gating in intel_pmu_set_msr() is sufficient even under PerfMon masking. Upon guest PMU context load, OR the guest-owned bits with whatever the hardware currently holds for every fixed-counter index the guest does not own, before writing the result to the MSR. During PMU context put, host-owned fixed counters may still be scheduled for system-wide host events, so the host-owned bits should remain unchanged. Signed-off-by: Zide Chen --- arch/x86/kvm/vmx/pmu_intel.c | 40 +++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index 19ccc7cd319c..9236bfa15c41 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -920,6 +920,33 @@ static void intel_pmu_write_global_ctrl(u64 global_ctrl) vmcs_write64(GUEST_IA32_PERF_GLOBAL_CTRL, global_ctrl); } +static u64 intel_fixed_ctrl_host_bits(struct kvm_pmu *pmu) +{ + unsigned long fixed_mask; + u64 fixed_ctl; + int i; + + if (!kvm_vcpu_has_perfmon_mask(pmu_to_vcpu(pmu))) + return 0; + + fixed_mask = kvm_fixed_pmc_mask(pmu); + + rdmsrq(MSR_CORE_PERF_FIXED_CTR_CTRL, fixed_ctl); + + /* + * Use the full per-counter nibbles (bits 4n+3:4n and 4n+35:4n+32) to + * strip all bits belonging to guest-owned counters. + * + * pmu->fixed_ctr_ctrl_rsvd can't be used here since it can't gate bits + * that are not supported by KVM. + */ + kvm_for_each_fixed_counter(i, fixed_mask) + fixed_ctl &= ~intel_fixed_bits_by_idx(i, GENMASK_ULL(3, 0) | + GENMASK_ULL(35, 32)); + + return fixed_ctl; +} + static void intel_mediated_pmu_load(struct kvm_vcpu *vcpu) { struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); @@ -935,7 +962,8 @@ static void intel_mediated_pmu_load(struct kvm_vcpu *vcpu) if (pmu->global_status & toggle) wrmsrq(MSR_CORE_PERF_GLOBAL_STATUS_SET, pmu->global_status & toggle); - wrmsrq(MSR_CORE_PERF_FIXED_CTR_CTRL, pmu->fixed_ctr_ctrl_hw); + wrmsrq(MSR_CORE_PERF_FIXED_CTR_CTRL, + pmu->fixed_ctr_ctrl_hw | intel_fixed_ctrl_host_bits(pmu)); } static void intel_mediated_pmu_put(struct kvm_vcpu *vcpu) @@ -953,9 +981,15 @@ static void intel_mediated_pmu_put(struct kvm_vcpu *vcpu) * Clear hardware FIXED_CTR_CTRL MSR to avoid information leakage and * also to avoid accidentally enabling fixed counters (based on guest * state) while running in the host, e.g. when setting global ctrl. + * + * Keep the host-owned counters unchanged. */ - if (pmu->fixed_ctr_ctrl_hw) - wrmsrq(MSR_CORE_PERF_FIXED_CTR_CTRL, 0); + if (pmu->fixed_ctr_ctrl_hw) { + u64 fixed_ctl = intel_fixed_ctrl_host_bits(pmu); + + fixed_ctl &= ~pmu->fixed_ctr_ctrl_hw; + wrmsrq(MSR_CORE_PERF_FIXED_CTR_CTRL, fixed_ctl); + } if (kvm_vcpu_has_perf_metrics(vcpu)) { pmu->perf_metrics = rdpmc(INTEL_PMC_FIXED_RDPMC_METRICS); -- 2.55.0 Guest writes must be gated, and guest reads masked, with pmu->global_ctrl_rsvd rather than pmu->perfmon_mask, because bits 63:52 in global_ctrl_rsvd are guaranteed to be cleared, and bits 51:0 are the complement between the two masks. Since the host could schedule !exclude_guest events on host-owned resources in non-root mode, OR the host-owned bits into GUEST_IA32_PERF_GLOBAL_CTRL when loading guest PMU state. The host- owned bits are supposed to be set during perf_load_guest_context(). Add a host_global_ctrl parameter to the mediated_load() callback so that the host value can be available before the register is cleared. This allows intel_mediated_pmu_load() to preserve host-owned bits. When caching pmu->global_ctrl at VM exit, mask the guest value with pmu->global_ctrl_rsvd to retain guest-owned bits only. Signed-off-by: Zide Chen --- arch/x86/kvm/pmu.c | 7 ++++++- arch/x86/kvm/pmu.h | 2 +- arch/x86/kvm/svm/pmu.c | 2 +- arch/x86/kvm/vmx/pmu_intel.c | 13 ++++++++++++- arch/x86/kvm/vmx/vmx.c | 4 ++++ 5 files changed, 24 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index 337d2f55a216..b051ba66edce 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -1406,6 +1406,8 @@ static void kvm_pmu_load_guest_pmcs(struct kvm_vcpu *vcpu) void kvm_mediated_pmu_load(struct kvm_vcpu *vcpu) { + u64 host_global_ctrl = 0; + if (!kvm_vcpu_has_mediated_pmu(vcpu) || KVM_BUG_ON(!lapic_in_kernel(vcpu), vcpu->kvm)) return; @@ -1429,13 +1431,16 @@ void kvm_mediated_pmu_load(struct kvm_vcpu *vcpu) * even for SVM to minimize the damage if a perf event is left enabled, * and to ensure a consistent starting state. */ + if (kvm_vcpu_has_perfmon_mask(vcpu)) + rdmsrq(kvm_pmu_ops.PERF_GLOBAL_CTRL, host_global_ctrl); + wrmsrq(kvm_pmu_ops.PERF_GLOBAL_CTRL, 0); perf_load_guest_lvtpc(kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVTPC)); kvm_pmu_load_guest_pmcs(vcpu); - kvm_pmu_call(mediated_load)(vcpu); + kvm_pmu_call(mediated_load)(vcpu, host_global_ctrl); } static void kvm_pmu_put_guest_pmcs(struct kvm_vcpu *vcpu) diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h index c5feeb60bcf6..30e456302c65 100644 --- a/arch/x86/kvm/pmu.h +++ b/arch/x86/kvm/pmu.h @@ -39,7 +39,7 @@ struct kvm_pmu_ops { bool (*pmc_is_disabled_in_current_mode)(struct kvm_pmc *pmc); bool (*is_mediated_pmu_supported)(struct x86_pmu_capability *host_pmu); - void (*mediated_load)(struct kvm_vcpu *vcpu); + void (*mediated_load)(struct kvm_vcpu *vcpu, u64 host_global_ctrl); void (*mediated_put)(struct kvm_vcpu *vcpu); void (*write_global_ctrl)(u64 global_ctrl); diff --git a/arch/x86/kvm/svm/pmu.c b/arch/x86/kvm/svm/pmu.c index f81817606baa..d9cd2a5ab411 100644 --- a/arch/x86/kvm/svm/pmu.c +++ b/arch/x86/kvm/svm/pmu.c @@ -246,7 +246,7 @@ static bool amd_pmu_is_mediated_pmu_supported(struct x86_pmu_capability *host_pm return host_pmu->version >= 2; } -static void amd_mediated_pmu_load(struct kvm_vcpu *vcpu) +static void amd_mediated_pmu_load(struct kvm_vcpu *vcpu, u64 host_global_ctrl) { struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); u64 global_status; diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index 9236bfa15c41..d0373de951d5 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -947,11 +947,22 @@ static u64 intel_fixed_ctrl_host_bits(struct kvm_pmu *pmu) return fixed_ctl; } -static void intel_mediated_pmu_load(struct kvm_vcpu *vcpu) +static void intel_mediated_pmu_load(struct kvm_vcpu *vcpu, u64 host_global_ctrl) { struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); u64 global_status, toggle; + /* + * Preserve host-owned bits: perf may schedule !exclude_guest events on + * host-owned counters in non-root mode. + * PerfMon masking requires VM_EXIT_SAVE_IA32_PERF_GLOBAL_CTRL, so the + * MSR-store/load path does not apply here. + */ + if (kvm_vcpu_has_perfmon_mask(vcpu)) { + host_global_ctrl &= pmu->global_ctrl_rsvd; + intel_pmu_write_global_ctrl(pmu->global_ctrl | host_global_ctrl); + } + if (kvm_vcpu_has_perf_metrics(vcpu)) wrmsrq(MSR_PERF_METRICS, pmu->perf_metrics); diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 1b42c9d6f168..a241efb99b3d 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -7467,6 +7467,10 @@ static void vmx_refresh_guest_perf_global_control(struct kvm_vcpu *vcpu) } pmu->global_ctrl = vmcs_read64(GUEST_IA32_PERF_GLOBAL_CTRL); + + /* Strip host-owned bits that were ORed into the VMCS on VMX entry. */ + if (kvm_vcpu_has_perfmon_mask(vcpu)) + pmu->global_ctrl &= ~pmu->global_ctrl_rsvd; } void noinstr vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp) -- 2.55.0 For IA32_PERF_GLOBAL_STATUS{,_SET,_RESET}, the effective mask under PerfMon masking is perfmon_mask. Writes that set bits outside the effective mask must #GP, so use ~perfmon_mask as the effective reserved-bit mask instead of global_status_rsvd. At guest context load, keep host-owned GLOBAL_STATUS bits unchanged because system-wide events may be scheduled on host-owned resources while running in VMX non-root mode. Likewise, at guest context put, preserve host-owned bits and clear only the guest-owned subset from hardware GLOBAL_STATUS, leaving pmu->global_status containing only guest-owned bits. Signed-off-by: Zide Chen --- arch/x86/kvm/pmu.c | 13 ++++++++----- arch/x86/kvm/vmx/pmu_intel.c | 17 ++++++++++++++++- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index b051ba66edce..23177c38f286 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -922,12 +922,15 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); u32 msr = msr_info->index; u64 data = msr_info->data; - u64 diff; + u64 global_status_rsvd, diff; /* * Note, AMD ignores writes to reserved bits and read-only PMU MSRs, * whereas Intel generates #GP on attempts to write reserved/RO MSRs. */ + global_status_rsvd = kvm_vcpu_has_perfmon_mask(vcpu) ? + ~pmu->perfmon_mask : pmu->global_status_rsvd; + switch (msr) { case MSR_CORE_PERF_GLOBAL_STATUS: if (!msr_info->host_initiated) @@ -938,7 +941,7 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) if (!msr_info->host_initiated) break; - if (data & pmu->global_status_rsvd) + if (data & global_status_rsvd) return 1; pmu->global_status = data; @@ -967,7 +970,7 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) * GLOBAL_OVF_CTRL, a.k.a. GLOBAL STATUS_RESET, clears bits in * GLOBAL_STATUS, and so the set of reserved bits is the same. */ - if (data & pmu->global_status_rsvd) + if (data & global_status_rsvd) return 1; fallthrough; case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR: @@ -975,14 +978,14 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) pmu->global_status &= ~data; break; case MSR_CORE_PERF_GLOBAL_STATUS_SET: - if (data & pmu->global_status_rsvd) + if (data & global_status_rsvd) return 1; if (!msr_info->host_initiated) pmu->global_status |= data; break; case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET: if (!msr_info->host_initiated) - pmu->global_status |= data & ~pmu->global_status_rsvd; + pmu->global_status |= data & ~global_status_rsvd; break; default: kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index); diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index d0373de951d5..e43380f79c72 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -968,6 +968,14 @@ static void intel_mediated_pmu_load(struct kvm_vcpu *vcpu, u64 host_global_ctrl) rdmsrq(MSR_CORE_PERF_GLOBAL_STATUS, global_status); toggle = pmu->global_status ^ global_status; + + /* + * Restrict OVF_CTRL/STATUS_SET writes to guest-owned bits under + * PerfMon masking. + */ + if (kvm_vcpu_has_perfmon_mask(vcpu)) + toggle &= pmu->perfmon_mask; + if (global_status & toggle) wrmsrq(MSR_CORE_PERF_GLOBAL_OVF_CTRL, global_status & toggle); if (pmu->global_status & toggle) @@ -984,7 +992,14 @@ static void intel_mediated_pmu_put(struct kvm_vcpu *vcpu) /* MSR_CORE_PERF_GLOBAL_CTRL is already saved at VM-exit. */ rdmsrq(MSR_CORE_PERF_GLOBAL_STATUS, pmu->global_status); - /* Clear hardware MSR_CORE_PERF_GLOBAL_STATUS MSR, if non-zero. */ + /* + * Clear only the guest-owned bits from the hardware GLOBAL_STATUS + * if any are set. pmu->global_status is then left holding just the + * guest-owned subset. + */ + if (kvm_vcpu_has_perfmon_mask(vcpu)) + pmu->global_status &= pmu->perfmon_mask; + if (pmu->global_status) wrmsrq(MSR_CORE_PERF_GLOBAL_OVF_CTRL, pmu->global_status); -- 2.55.0 Unlike IA32_PERF_GLOBAL_CTRL, hardware does not filter GLOBAL_INUSE reads based on the PerfMon mask at all, so always intercept this MSR and filter out host-owned bits in KVM. Signed-off-by: Zide Chen --- arch/x86/kvm/vmx/vmx.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index a241efb99b3d..2bd1ffe65510 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -4339,8 +4339,15 @@ static void vmx_recalc_pmu_msr_intercepts(struct kvm_vcpu *vcpu) MSR_TYPE_RW, intercept); vmx_set_intercept_for_msr(vcpu, MSR_CORE_PERF_GLOBAL_STATUS_SET, MSR_TYPE_RW, intercept || pmu->version < 4); + + /* + * IA32_PERF_GLOBAL_INUSE reads are not subject to PerfMon mask + * filtering. + */ vmx_set_intercept_for_msr(vcpu, MSR_CORE_PERF_GLOBAL_INUSE, - MSR_TYPE_RW, intercept || pmu->version < 4); + MSR_TYPE_RW, + intercept || kvm_vcpu_has_perfmon_mask(vcpu) || + pmu->version < 4); intercept = !has_mediated_pmu || !kvm_vcpu_has_perf_metrics(vcpu); vmx_set_intercept_for_msr(vcpu, MSR_PERF_METRICS, -- 2.55.0 When PMU partitioning is enabled, guest-induced PMIs that occur in guest mode may be delivered to the host perf PMI handler, resulting in a VM-Exit. intel_mediated_pmu_put() reads the hardware GLOBAL_STATUS MSR into pmu->global_status and masks it to the subset owned by the guest. Request a guest PMI when the saved status indicates a pending guest PMI, i.e. when - an overflow status bit (47:0) is set for a counter whose PMI-enable bit is enabled; or - a miscellaneous status bit without a corresponding PMI-enable control is set. Signed-off-by: Zide Chen --- arch/x86/kvm/pmu.c | 3 ++- arch/x86/kvm/pmu.h | 1 + arch/x86/kvm/vmx/pmu_intel.c | 42 ++++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index 23177c38f286..f99ddd6549f7 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -1112,7 +1112,7 @@ void kvm_pmu_destroy(struct kvm_vcpu *vcpu) kvm_pmu_reset(vcpu); } -static bool pmc_is_pmi_enabled(struct kvm_pmc *pmc) +bool pmc_is_pmi_enabled(struct kvm_pmc *pmc) { u8 fixed_ctr_ctrl; @@ -1123,6 +1123,7 @@ static bool pmc_is_pmi_enabled(struct kvm_pmc *pmc) pmc->idx - KVM_FIXED_PMC_BASE_IDX); return fixed_ctr_ctrl & INTEL_FIXED_0_ENABLE_PMI; } +EXPORT_SYMBOL_FOR_KVM_INTERNAL(pmc_is_pmi_enabled); static void kvm_pmu_incr_counter(struct kvm_pmc *pmc) { diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h index 30e456302c65..4228eb49c6bf 100644 --- a/arch/x86/kvm/pmu.h +++ b/arch/x86/kvm/pmu.h @@ -336,6 +336,7 @@ static inline bool kvm_vcpu_has_perf_metrics(struct kvm_vcpu *vcpu) return kvm_vcpu_get_perf_caps(vcpu) & PERF_CAP_PERF_METRICS; } +bool pmc_is_pmi_enabled(struct kvm_pmc *pmc); void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu); int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned pmc, u64 *data); int kvm_pmu_check_rdpmc_early(struct kvm_vcpu *vcpu, unsigned int idx); diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index e43380f79c72..7fab735252d7 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -985,6 +985,46 @@ static void intel_mediated_pmu_load(struct kvm_vcpu *vcpu, u64 host_global_ctrl) pmu->fixed_ctr_ctrl_hw | intel_fixed_ctrl_host_bits(pmu)); } +static void intel_perfmon_mask_request_pmi(struct kvm_vcpu *vcpu) +{ + struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); + struct kvm_pmc *pmc; + int i; + + if (!kvm_vcpu_has_perfmon_mask(vcpu) || !pmu->global_status) + return; + + /* + * pmu->global_status should never carry a bit that isn't + * guest-owned per pmu->perfmon_mask; such a bit would mean a + * host-owned resource is being (mis)reported to the guest. + */ + WARN_ON_ONCE(pmu->global_status & ~pmu->perfmon_mask); + + /* + * Bit 48 is currently the only miscellaneous status bit (63:48) that + * can be guest-owned; it indicates that a PMI is triggered, regardless + * of fixed counter 3's PMI-enable state. + */ + if (pmu->global_status & GLOBAL_STATUS_PERF_METRICS_OVF) { + kvm_make_request(KVM_REQ_PMI, vcpu); + return; + } + + /* + * Match bare-metal behavior for counter bits (47:0): request a guest + * PMI if any overflowed counter actually has its PMI-enable bit set. + */ + for_each_set_bit(i, (unsigned long *)&pmu->global_status, + GLOBAL_STATUS_PERF_METRICS_OVF_BIT) { + pmc = kvm_pmc_idx_to_pmc(pmu, i); + if (pmc && pmc_is_pmi_enabled(pmc)) { + kvm_make_request(KVM_REQ_PMI, vcpu); + break; + } + } +} + static void intel_mediated_pmu_put(struct kvm_vcpu *vcpu) { struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); @@ -1029,6 +1069,8 @@ static void intel_mediated_pmu_put(struct kvm_vcpu *vcpu) if (pmu->perf_metrics) wrmsrq(MSR_PERF_METRICS, 0); } + + intel_perfmon_mask_request_pmi(vcpu); } static bool intel_pmu_validate_perfmon_mask(void) -- 2.55.0 PerfMon masking is enumerated by bit [17] of IA32_VMX_PROCBASED_CTLS3 (MSR 0x492), and it can be enabled by the new tertiary VM-execution control bit 17. Add TERTIARY_EXEC_PERFMON_MASK_ENABLE to KVM's optional tertiary VM-execution controls, allowing cpu_has_vmx_perfmon_mask() to return true on PerfMon masking capable platforms. With this in place, a valid perfmon_mask module parameter is no longer forced to zero, pmu->perfmon_mask can be configured as intended, and PerfMon masking can be enabled in hardware. perf_create_mediated_pmu() can now pass a valid PerfMon mask to the perf subsystem, enabling PMU partitioning aware event scheduling. Signed-off-by: Zide Chen --- arch/x86/include/asm/vmx.h | 1 + arch/x86/include/asm/vmxfeatures.h | 1 + arch/x86/kvm/vmx/capabilities.h | 3 ++- arch/x86/kvm/vmx/vmx.c | 13 +++++++++++++ arch/x86/kvm/vmx/vmx.h | 2 +- 5 files changed, 18 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h index 1cb092d86955..652322163f1b 100644 --- a/arch/x86/include/asm/vmx.h +++ b/arch/x86/include/asm/vmx.h @@ -96,6 +96,7 @@ struct vmcs { * Definitions of Tertiary Processor-Based VM-Execution Controls. */ #define TERTIARY_EXEC_IPI_VIRT VMCS_CONTROL_BIT(IPI_VIRT) +#define TERTIARY_EXEC_PERFMON_MASK_ENABLE VMCS_CONTROL_BIT(PERFMON_MASK_ENABLE) #define PIN_BASED_EXT_INTR_MASK VMCS_CONTROL_BIT(INTR_EXITING) #define PIN_BASED_NMI_EXITING VMCS_CONTROL_BIT(NMI_EXITING) diff --git a/arch/x86/include/asm/vmxfeatures.h b/arch/x86/include/asm/vmxfeatures.h index 09b1d7e607c1..5efbd35ba5ad 100644 --- a/arch/x86/include/asm/vmxfeatures.h +++ b/arch/x86/include/asm/vmxfeatures.h @@ -90,4 +90,5 @@ /* Tertiary Processor-Based VM-Execution Controls, word 3 */ #define VMX_FEATURE_IPI_VIRT ( 3*32+ 4) /* "ipi_virt" Enable IPI virtualization */ +#define VMX_FEATURE_PERFMON_MASK_ENABLE ( 3*32+ 17) /* "perfmon_mask_enable" Enable PERFMON_MASK */ #endif /* _ASM_X86_VMXFEATURES_H */ diff --git a/arch/x86/kvm/vmx/capabilities.h b/arch/x86/kvm/vmx/capabilities.h index d4c362093966..3852a4a9da88 100644 --- a/arch/x86/kvm/vmx/capabilities.h +++ b/arch/x86/kvm/vmx/capabilities.h @@ -298,7 +298,8 @@ static inline bool cpu_has_vmx_ipiv(void) static inline bool cpu_has_vmx_perfmon_mask(void) { - return false; + return vmcs_config.cpu_based_3rd_exec_ctrl & + TERTIARY_EXEC_PERFMON_MASK_ENABLE; } static inline bool cpu_has_vmx_flexpriority(void) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 2bd1ffe65510..64266d94ab76 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -4288,6 +4288,14 @@ static void vmx_recalc_pmu_msr_intercepts(struct kvm_vcpu *vcpu) bool intercept = !has_mediated_pmu; int i; + /* + * Keep the PerfMon masking tertiary VM-execution control in sync + * regardless of enable_mediated_pmu. + */ + if (cpu_has_vmx_perfmon_mask()) + tertiary_exec_controls_changebit(vmx, TERTIARY_EXEC_PERFMON_MASK_ENABLE, + kvm_vcpu_has_perfmon_mask(vcpu)); + if (!enable_mediated_pmu) return; @@ -4766,6 +4774,9 @@ static u64 vmx_tertiary_exec_control(struct vcpu_vmx *vmx) if (!enable_ipiv || !kvm_vcpu_apicv_active(&vmx->vcpu)) exec_control &= ~TERTIARY_EXEC_IPI_VIRT; + if (!kvm_vcpu_has_perfmon_mask(&vmx->vcpu)) + exec_control &= ~TERTIARY_EXEC_PERFMON_MASK_ENABLE; + return exec_control; } @@ -6754,6 +6765,8 @@ void dump_vmcs(struct kvm_vcpu *vcpu) if (secondary_exec_control & SECONDARY_EXEC_ENABLE_VPID) pr_err("Virtual processor ID = 0x%04x\n", vmcs_read16(VIRTUAL_PROCESSOR_ID)); + if (tertiary_exec_control & TERTIARY_EXEC_PERFMON_MASK_ENABLE) + pr_err("PERFMON_MASK = 0x%016llx\n", vmcs_read64(PERFMON_MASK)); if (secondary_exec_control & SECONDARY_EXEC_EPT_VIOLATION_VE) { struct vmx_ve_information *ve_info = vmx->ve_info; u64 ve_info_pa = vmcs_read64(VE_INFORMATION_ADDRESS); diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index ccda5c5c8c2c..0530e0fa0076 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -577,7 +577,7 @@ static inline u8 vmx_get_rvi(void) #define KVM_REQUIRED_VMX_TERTIARY_VM_EXEC_CONTROL 0 #define KVM_OPTIONAL_VMX_TERTIARY_VM_EXEC_CONTROL \ - (TERTIARY_EXEC_IPI_VIRT) + (TERTIARY_EXEC_IPI_VIRT | TERTIARY_EXEC_PERFMON_MASK_ENABLE) #define BUILD_CONTROLS_SHADOW(lname, uname, bits) \ static inline void lname##_controls_set(struct vcpu_vmx *vmx, u##bits val) \ -- 2.55.0 IA32_PERF_CAPABILITIES.PERF_METRICS_AVAILABLE[bit 15] is not tied to a particular PMU version, and userspace can enable it as long as it is advertised. On a PERF_METRICS capable host that supports fixed counter 3 through a non-contiguous fixed counter bitmap, userspace may enable this capability while configuring a guest PMU version lower than 5. This scenario is unlikely on bare-metal systems, but it can occur with PMU partitioning. For example, when kvm-intel is loaded with: perfmon_mask=guest_fixed=1-3;perf_metrics CPUID.0AH:ECX[4:0] is 0, and thus fixed counter 3 is not available when the guest PMU version is below 5. Skip the test in this case. Signed-off-by: Zide Chen --- tools/testing/selftests/kvm/x86/pmu_counters_test.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/x86/pmu_counters_test.c b/tools/testing/selftests/kvm/x86/pmu_counters_test.c index 2797d7e0e3a3..edcd04b4111c 100644 --- a/tools/testing/selftests/kvm/x86/pmu_counters_test.c +++ b/tools/testing/selftests/kvm/x86/pmu_counters_test.c @@ -678,7 +678,8 @@ static void __guest_test_perf_metrics(void) if ((guest_get_pmu_version() < 2) || /* Does guest have GLOBAL_CTRL? */ !this_cpu_has(X86_FEATURE_PDCM) || - !(rdmsr(MSR_IA32_PERF_CAPABILITIES) & PERF_CAP_PERF_METRICS)) + !(rdmsr(MSR_IA32_PERF_CAPABILITIES) & PERF_CAP_PERF_METRICS) || + !this_pmu_has(X86_PMU_FEATURE_TOPDOWN_SLOTS_FIXED)) return; wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0); -- 2.55.0 Do not GUEST_ASSERT(nr_gp_counters). Instead, skip the architectural event tests that require general-purpose counters when none are available. In theory, guest CPUID.0AH:EAX[15:08] can be configured to report zero when the selftest runs on L1. In practice, this scenario is more likely to occur when PMU partitioning limits the number of available general-purpose counters. Signed-off-by: Zide Chen --- .../selftests/kvm/x86/pmu_counters_test.c | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/tools/testing/selftests/kvm/x86/pmu_counters_test.c b/tools/testing/selftests/kvm/x86/pmu_counters_test.c index edcd04b4111c..b38e802e753b 100644 --- a/tools/testing/selftests/kvm/x86/pmu_counters_test.c +++ b/tools/testing/selftests/kvm/x86/pmu_counters_test.c @@ -282,19 +282,16 @@ static void __guest_test_arch_event(u8 idx, u32 pmc, u32 pmc_msr, GUEST_TEST_EVENT(idx, pmc, pmc_msr, ctrl_msr, ctrl_msr_value, KVM_FEP); } -static void guest_test_arch_event(u8 idx) +static void guest_test_arch_gp_event(u8 idx, bool guest_has_perf_global_ctrl) { u32 nr_gp_counters = this_cpu_property(X86_PROPERTY_PMU_NR_GP_COUNTERS); - u32 pmu_version = guest_get_pmu_version(); - /* PERF_GLOBAL_CTRL exists only for Architectural PMU Version 2+. */ - bool guest_has_perf_global_ctrl = pmu_version >= 2; - struct kvm_x86_pmu_feature gp_event, fixed_event; + struct kvm_x86_pmu_feature gp_event; u32 base_pmc_msr; unsigned int i; u64 eventsel; - /* The host side shouldn't invoke this without a guest PMU. */ - GUEST_ASSERT(pmu_version); + if (!nr_gp_counters) + return; if (this_cpu_has(X86_FEATURE_PDCM) && rdmsr(MSR_IA32_PERF_CAPABILITIES) & PMU_CAP_FW_WRITES) @@ -305,7 +302,6 @@ static void guest_test_arch_event(u8 idx) gp_event = intel_event_to_feature(idx).gp_event; GUEST_ASSERT_EQ(idx, gp_event.f.bit); - GUEST_ASSERT(nr_gp_counters); i = kvm_random_u32_in_range(&kvm_rng, 0, nr_gp_counters - 1); eventsel = ARCH_PERFMON_EVENTSEL_OS | ARCH_PERFMON_EVENTSEL_ENABLE | @@ -316,6 +312,20 @@ static void guest_test_arch_event(u8 idx) wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, BIT_ULL(i)); __guest_test_arch_event(idx, i, base_pmc_msr + i, MSR_P6_EVNTSEL0 + i, eventsel); +} + +static void guest_test_arch_event(u8 idx) +{ + u32 pmu_version = guest_get_pmu_version(); + /* PERF_GLOBAL_CTRL exists only for Architectural PMU Version 2+. */ + bool guest_has_perf_global_ctrl = pmu_version >= 2; + struct kvm_x86_pmu_feature fixed_event; + unsigned int i; + + /* The host side shouldn't invoke this without a guest PMU. */ + GUEST_ASSERT(pmu_version); + + guest_test_arch_gp_event(idx, guest_has_perf_global_ctrl); if (!guest_has_perf_global_ctrl) return; -- 2.55.0