Because ARM hardware does not directly inject PMU PPIs into guests, overflows on partitioned guest counters raise physical PMU interrupts that enter the host's armv8pmu_handle_irq() handler. Update armv8pmu_handle_irq() to read PMOVSCLR_EL0 without blindly clearing guest overflow bits: - Pass pmovsr to kvm_pmu_handle_guest_irq(), which clears guest-reserved overflow bits in hardware and latches them into the running vCPU's shadow PMOVSSET_EL0 register. - Mask out guest-reserved counter bits before clearing and processing host perf overflows, returning IRQ_HANDLED early when only guest counters overflowed or no host events are active. Also add kvm_pmu_host_start() and kvm_pmu_host_stop() so armv8pmu_start() and armv8pmu_stop() toggle MDCR_EL2.HPME rather than PMCR_EL0.E while a partitioned guest's PMU state is loaded on the CPU (leaving guest counters 0..HPMN-1 and PMCCNTR_EL0 under guest PMCR_EL0.E control), guarding local_daif_save() with !in_nmi() for GICv3 pseudo-NMI interrupt handlers. Signed-off-by: Colton Lewis --- arch/arm/include/asm/arm_pmuv3.h | 8 +++ arch/arm64/include/asm/arm_pmuv3.h | 5 ++ arch/arm64/kvm/pmu-direct.c | 102 +++++++++++++++++++++++++++++ drivers/perf/arm_pmuv3.c | 34 +++++----- include/kvm/arm_pmu.h | 7 ++ 5 files changed, 141 insertions(+), 15 deletions(-) diff --git a/arch/arm/include/asm/arm_pmuv3.h b/arch/arm/include/asm/arm_pmuv3.h index 71cf03ab58cbb..d8051ea261481 100644 --- a/arch/arm/include/asm/arm_pmuv3.h +++ b/arch/arm/include/asm/arm_pmuv3.h @@ -180,6 +180,11 @@ static inline void write_pmintenset(u32 val) write_sysreg(val, PMINTENSET); } +static inline u32 read_pmintenset(void) +{ + return read_sysreg(PMINTENSET); +} + static inline void write_pmintenclr(u32 val) { write_sysreg(val, PMINTENCLR); @@ -231,6 +236,9 @@ static inline u64 kvm_pmu_guest_counter_mask(void) { return 0; } +static inline void kvm_pmu_handle_guest_irq(u64 pmovsr) {} +static inline bool kvm_pmu_host_start(void) { return false; } +static inline bool kvm_pmu_host_stop(void) { return false; } /* PMU Version in DFR Register */ #define ARMV8_PMU_DFR_VER_NI 0 diff --git a/arch/arm64/include/asm/arm_pmuv3.h b/arch/arm64/include/asm/arm_pmuv3.h index cf2b2212e00a2..9037a28365951 100644 --- a/arch/arm64/include/asm/arm_pmuv3.h +++ b/arch/arm64/include/asm/arm_pmuv3.h @@ -110,6 +110,11 @@ static inline void write_pmintenset(u64 val) write_sysreg(val, pmintenset_el1); } +static inline u64 read_pmintenset(void) +{ + return read_sysreg(pmintenset_el1); +} + static inline void write_pmintenclr(u64 val) { write_sysreg(val, pmintenclr_el1); diff --git a/arch/arm64/kvm/pmu-direct.c b/arch/arm64/kvm/pmu-direct.c index c2ea1ce5efe74..21357decce214 100644 --- a/arch/arm64/kvm/pmu-direct.c +++ b/arch/arm64/kvm/pmu-direct.c @@ -800,3 +800,105 @@ static void kvm_pmu_set_guest_owned(struct kvm_vcpu *vcpu) __kvm_pmu_load(vcpu, true); } } + +/** + * kvm_pmu_handle_guest_irq() - Record IRQs in guest counters + * @pmovsr: Overflow flags reported by driver + * + * Set overflow flags in guest-reserved counters in the VCPU register + * for the guest to clear later. + */ +void kvm_pmu_handle_guest_irq(u64 pmovsr) +{ + struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); + u64 mask = kvm_pmu_guest_counter_mask(); + u64 govf = pmovsr & mask; + + if (!govf) + return; + + write_pmovsclr(govf); + + if (!vcpu || !vcpu->arch.pmu.loaded_on_cpu) + return; + + __vcpu_rmw_sys_reg(vcpu, PMOVSSET_EL0, |=, govf); +} + +/** + * kvm_pmu_host_start() - Enable host PMU counters while guest owns PMU + * + * When a partitioned guest owns the PMU, host counters (HPMN..N-1) are + * gated by MDCR_EL2.HPME rather than PMCR_EL0.E. + * + * Return: True if handled via MDCR_EL2.HPME, false if caller should use PMCR_EL0.E + */ +bool kvm_pmu_host_start(void) +{ + struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); + unsigned long daif = 0; + bool nmi; + u64 mdcr; + + if (!vcpu || !kvm_pmu_is_partitioned(vcpu->kvm) || + !vcpu->arch.pmu.loaded_on_cpu) + return false; + + nmi = in_nmi(); + if (!nmi) + daif = local_daif_save(); + + if (!vcpu->arch.pmu.loaded_on_cpu) { + if (!nmi) + local_daif_restore(daif); + return false; + } + + mdcr = READ_ONCE(vcpu->arch.mdcr_el2) | MDCR_EL2_HPME; + WRITE_ONCE(vcpu->arch.mdcr_el2, mdcr); + if (has_vhe()) + write_sysreg(mdcr, mdcr_el2); + isb(); + if (!nmi) + local_daif_restore(daif); + return true; +} + +/** + * kvm_pmu_host_stop() - Disable host PMU counters while guest owns PMU + * + * When a partitioned guest owns the PMU, host counters (HPMN..N-1) are + * gated by MDCR_EL2.HPME rather than PMCR_EL0.E. + * + * Return: True if handled via MDCR_EL2.HPME, false if caller should use PMCR_EL0.E + */ +bool kvm_pmu_host_stop(void) +{ + struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); + unsigned long daif = 0; + bool nmi; + u64 mdcr; + + if (!vcpu || !kvm_pmu_is_partitioned(vcpu->kvm) || + !vcpu->arch.pmu.loaded_on_cpu) + return false; + + nmi = in_nmi(); + if (!nmi) + daif = local_daif_save(); + + if (!vcpu->arch.pmu.loaded_on_cpu) { + if (!nmi) + local_daif_restore(daif); + return false; + } + + mdcr = READ_ONCE(vcpu->arch.mdcr_el2) & ~MDCR_EL2_HPME; + WRITE_ONCE(vcpu->arch.mdcr_el2, mdcr); + if (has_vhe()) + write_sysreg(mdcr, mdcr_el2); + isb(); + if (!nmi) + local_daif_restore(daif); + return true; +} diff --git a/drivers/perf/arm_pmuv3.c b/drivers/perf/arm_pmuv3.c index 1676f13cc7c38..440f77cf0a2fc 100644 --- a/drivers/perf/arm_pmuv3.c +++ b/drivers/perf/arm_pmuv3.c @@ -777,18 +777,9 @@ static void armv8pmu_disable_event_irq(struct perf_event *event) armv8pmu_disable_intens(BIT(event->hw.idx)); } -static u64 armv8pmu_getreset_flags(void) +static u64 armv8pmu_getovf_flags(void) { - u64 value; - - /* Read */ - value = read_pmovsclr(); - - /* Write to clear flags */ - value &= ARMV8_PMU_CNT_MASK_ALL; - write_pmovsclr(value); - - return value; + return read_pmovsclr() & ARMV8_PMU_CNT_MASK_ALL; } static void update_pmuserenr(u64 val) @@ -878,7 +869,8 @@ static void armv8pmu_start(struct arm_pmu *cpu_pmu) brbe_enable(cpu_pmu); /* Enable all counters */ - armv8pmu_pmcr_write(armv8pmu_pmcr_read() | ARMV8_PMU_PMCR_E); + if (!kvm_pmu_host_start()) + armv8pmu_pmcr_write(armv8pmu_pmcr_read() | ARMV8_PMU_PMCR_E); } static void armv8pmu_stop(struct arm_pmu *cpu_pmu) @@ -889,7 +881,8 @@ static void armv8pmu_stop(struct arm_pmu *cpu_pmu) brbe_disable(); /* Disable all counters */ - armv8pmu_pmcr_write(armv8pmu_pmcr_read() & ~ARMV8_PMU_PMCR_E); + if (!kvm_pmu_host_stop()) + armv8pmu_pmcr_write(armv8pmu_pmcr_read() & ~ARMV8_PMU_PMCR_E); } static void read_branch_records(struct pmu_hw_events *cpuc, @@ -911,9 +904,9 @@ static irqreturn_t armv8pmu_handle_irq(struct arm_pmu *cpu_pmu) int idx; /* - * Get and reset the IRQ flags + * Get the IRQ flags */ - pmovsr = armv8pmu_getreset_flags(); + pmovsr = armv8pmu_getovf_flags(); /* * Did an overflow occur? @@ -921,6 +914,17 @@ static irqreturn_t armv8pmu_handle_irq(struct arm_pmu *cpu_pmu) if (!armv8pmu_has_overflowed(pmovsr)) return IRQ_NONE; + kvm_pmu_handle_guest_irq(pmovsr); + + pmovsr &= ~kvm_pmu_guest_counter_mask(); + if (!pmovsr) + return IRQ_HANDLED; + + write_pmovsclr(pmovsr); + + if (bitmap_empty(cpuc->used_mask, ARMPMU_MAX_HWEVENTS)) + return IRQ_HANDLED; + /* * Handle the counter(s) overflow(s) */ diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h index e3f5b9a1a4817..9800514019630 100644 --- a/include/kvm/arm_pmu.h +++ b/include/kvm/arm_pmu.h @@ -108,6 +108,9 @@ void kvm_pmu_reload_part(struct kvm_vcpu *vcpu); u64 kvm_pmu_guest_counter_mask(void); void kvm_pmu_load(struct kvm_vcpu *vcpu); void kvm_pmu_put(struct kvm_vcpu *vcpu); +void kvm_pmu_handle_guest_irq(u64 pmovsr); +bool kvm_pmu_host_start(void); +bool kvm_pmu_host_stop(void); #define kvm_pmu_get_access(vcpu) ((vcpu)->arch.pmu.access) #define kvm_pmu_reset_access(vcpu) ((vcpu)->arch.pmu.access = VCPU_PMU_ACCESS_FREE) @@ -277,6 +280,10 @@ static inline u64 kvm_pmu_guest_counter_mask(void) return 0; } +static inline void kvm_pmu_handle_guest_irq(u64 pmovsr) {} +static inline bool kvm_pmu_host_start(void) { return false; } +static inline bool kvm_pmu_host_stop(void) { return false; } + static inline bool has_kvm_pmu_partition_support(void) { return false; -- 2.56.1.708.g77570a6a1d-goog