This change introduces GICv5 load/put. Additionally, it plumbs in save/restore for: * PPIs (ICH_PPI_x_EL2 regs) * ICH_VMCR_EL2 * ICH_APR_EL2 * ICC_ICSR_EL1 A GICv5-specific enable bit is added to struct vgic_vmcr as this differs from previous GICs. On GICv5-native systems, the VMCR only contains the enable bit (driven by the guest via ICC_CR0_EL1.EN) and the priority mask (PCR). A struct gicv5_vpe is also introduced. This currently only contains a single field - bool resident - which is used to track if a VPE is currently running or not, and is used to avoid a case of double load or double put on the WFI path for a vCPU. This struct will be extended as additional GICv5 support is merged, specifically for VPE doorbells. Co-authored-by: Timothy Hayes Signed-off-by: Timothy Hayes Signed-off-by: Sascha Bischoff --- arch/arm64/kvm/hyp/nvhe/switch.c | 10 ++++ arch/arm64/kvm/vgic/vgic-mmio.c | 28 +++++++---- arch/arm64/kvm/vgic/vgic-v5.c | 77 ++++++++++++++++++++++++++++++ arch/arm64/kvm/vgic/vgic.c | 32 ++++++++----- arch/arm64/kvm/vgic/vgic.h | 7 +++ include/kvm/arm_vgic.h | 2 + include/linux/irqchip/arm-gic-v5.h | 9 ++++ 7 files changed, 146 insertions(+), 19 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c index c23e22ffac080..e25f1f000536b 100644 --- a/arch/arm64/kvm/hyp/nvhe/switch.c +++ b/arch/arm64/kvm/hyp/nvhe/switch.c @@ -113,6 +113,11 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu) /* Save VGICv3 state on non-VHE systems */ static void __hyp_vgic_save_state(struct kvm_vcpu *vcpu) { + if (kern_hyp_va(vcpu->kvm)->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V5) { + __vgic_v5_save_icsr(&vcpu->arch.vgic_cpu.vgic_v5); + __vgic_v5_save_ppi_state(&vcpu->arch.vgic_cpu.vgic_v5); + } + if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) { __vgic_v3_save_state(&vcpu->arch.vgic_cpu.vgic_v3); __vgic_v3_deactivate_traps(&vcpu->arch.vgic_cpu.vgic_v3); @@ -122,6 +127,11 @@ static void __hyp_vgic_save_state(struct kvm_vcpu *vcpu) /* Restore VGICv3 state on non-VHE systems */ static void __hyp_vgic_restore_state(struct kvm_vcpu *vcpu) { + if (kern_hyp_va(vcpu->kvm)->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V5) { + __vgic_v5_restore_icsr(&vcpu->arch.vgic_cpu.vgic_v5); + __vgic_v5_restore_ppi_state(&vcpu->arch.vgic_cpu.vgic_v5); + } + if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) { __vgic_v3_activate_traps(&vcpu->arch.vgic_cpu.vgic_v3); __vgic_v3_restore_state(&vcpu->arch.vgic_cpu.vgic_v3); diff --git a/arch/arm64/kvm/vgic/vgic-mmio.c b/arch/arm64/kvm/vgic/vgic-mmio.c index a573b1f0c6cbe..675c2844f5e5c 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio.c +++ b/arch/arm64/kvm/vgic/vgic-mmio.c @@ -842,18 +842,30 @@ vgic_find_mmio_region(const struct vgic_register_region *regions, void vgic_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr) { - if (kvm_vgic_global_state.type == VGIC_V2) - vgic_v2_set_vmcr(vcpu, vmcr); - else - vgic_v3_set_vmcr(vcpu, vmcr); + const struct vgic_dist *dist = &vcpu->kvm->arch.vgic; + + if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V5) { + vgic_v5_set_vmcr(vcpu, vmcr); + } else { + if (kvm_vgic_global_state.type == VGIC_V2) + vgic_v2_set_vmcr(vcpu, vmcr); + else + vgic_v3_set_vmcr(vcpu, vmcr); + } } void vgic_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr) { - if (kvm_vgic_global_state.type == VGIC_V2) - vgic_v2_get_vmcr(vcpu, vmcr); - else - vgic_v3_get_vmcr(vcpu, vmcr); + const struct vgic_dist *dist = &vcpu->kvm->arch.vgic; + + if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V5) { + vgic_v5_get_vmcr(vcpu, vmcr); + } else { + if (kvm_vgic_global_state.type == VGIC_V2) + vgic_v2_get_vmcr(vcpu, vmcr); + else + vgic_v3_get_vmcr(vcpu, vmcr); + } } /* diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index 2d3811f4e1174..2fb2db23ed39a 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -1,4 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2025 Arm Ltd. + */ #include #include @@ -50,3 +53,77 @@ int vgic_v5_probe(const struct gic_kvm_info *info) return 0; } + +void vgic_v5_load(struct kvm_vcpu *vcpu) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + /* + * On the WFI path, vgic_load is called a second time. The first is when + * scheduling in the vcpu thread again, and the second is when leaving + * WFI. Skip the second instance as it serves no purpose and just + * restores the same state again. + */ + if (READ_ONCE(cpu_if->gicv5_vpe.resident)) + return; + + kvm_call_hyp(__vgic_v5_restore_vmcr_aprs, cpu_if); + + WRITE_ONCE(cpu_if->gicv5_vpe.resident, true); +} + +void vgic_v5_put(struct kvm_vcpu *vcpu) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + /* + * Do nothing if we're not resident. This can happen in the WFI path + * where we do a vgic_put in the WFI path and again later when + * descheduling the thread. We risk losing VMCR state if we sync it + * twice, so instead return early in this case. + */ + if (!READ_ONCE(cpu_if->gicv5_vpe.resident)) + return; + + kvm_call_hyp(__vgic_v5_save_vmcr_aprs, cpu_if); + + WRITE_ONCE(cpu_if->gicv5_vpe.resident, false); +} + +void vgic_v5_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcrp) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + u64 vmcr = cpu_if->vgic_vmcr; + + vmcrp->en = FIELD_GET(FEAT_GCIE_ICH_VMCR_EL2_EN, vmcr); + vmcrp->pmr = FIELD_GET(FEAT_GCIE_ICH_VMCR_EL2_VPMR, vmcr); +} + +void vgic_v5_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcrp) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + u64 vmcr; + + vmcr = FIELD_PREP(FEAT_GCIE_ICH_VMCR_EL2_VPMR, vmcrp->pmr) | + FIELD_PREP(FEAT_GCIE_ICH_VMCR_EL2_EN, vmcrp->en); + + cpu_if->vgic_vmcr = vmcr; +} + +void vgic_v5_restore_state(struct kvm_vcpu *vcpu) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + __vgic_v5_restore_icsr(cpu_if); + kvm_call_hyp(__vgic_v5_restore_ppi_state, cpu_if); + dsb(sy); +} + +void vgic_v5_save_state(struct kvm_vcpu *vcpu) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + __vgic_v5_save_icsr(cpu_if); + kvm_call_hyp(__vgic_v5_save_ppi_state, cpu_if); + dsb(sy); +} diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index 2c0e8803342e2..1005ff5f36235 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -996,7 +996,9 @@ static inline bool can_access_vgic_from_kernel(void) static inline void vgic_save_state(struct kvm_vcpu *vcpu) { - if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) + if (vgic_is_v5(vcpu->kvm)) + vgic_v5_save_state(vcpu); + else if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) vgic_v2_save_state(vcpu); else __vgic_v3_save_state(&vcpu->arch.vgic_cpu.vgic_v3); @@ -1005,14 +1007,16 @@ static inline void vgic_save_state(struct kvm_vcpu *vcpu) /* Sync back the hardware VGIC state into our emulation after a guest's run. */ void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu) { - /* If nesting, emulate the HW effect from L0 to L1 */ - if (vgic_state_is_nested(vcpu)) { - vgic_v3_sync_nested(vcpu); - return; - } + if (!vgic_is_v5(vcpu->kvm)) { + /* If nesting, emulate the HW effect from L0 to L1 */ + if (vgic_state_is_nested(vcpu)) { + vgic_v3_sync_nested(vcpu); + return; + } - if (vcpu_has_nv(vcpu)) - vgic_v3_nested_update_mi(vcpu); + if (vcpu_has_nv(vcpu)) + vgic_v3_nested_update_mi(vcpu); + } if (can_access_vgic_from_kernel()) vgic_save_state(vcpu); @@ -1034,7 +1038,9 @@ void kvm_vgic_process_async_update(struct kvm_vcpu *vcpu) static inline void vgic_restore_state(struct kvm_vcpu *vcpu) { - if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) + if (vgic_is_v5(vcpu->kvm)) + vgic_v5_restore_state(vcpu); + else if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) vgic_v2_restore_state(vcpu); else __vgic_v3_restore_state(&vcpu->arch.vgic_cpu.vgic_v3); @@ -1094,7 +1100,9 @@ void kvm_vgic_load(struct kvm_vcpu *vcpu) return; } - if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) + if (vgic_is_v5(vcpu->kvm)) + vgic_v5_load(vcpu); + else if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) vgic_v2_load(vcpu); else vgic_v3_load(vcpu); @@ -1108,7 +1116,9 @@ void kvm_vgic_put(struct kvm_vcpu *vcpu) return; } - if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) + if (vgic_is_v5(vcpu->kvm)) + vgic_v5_put(vcpu); + else if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) vgic_v2_put(vcpu); else vgic_v3_put(vcpu); diff --git a/arch/arm64/kvm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h index bf5bae023751b..6e1f386dffade 100644 --- a/arch/arm64/kvm/vgic/vgic.h +++ b/arch/arm64/kvm/vgic/vgic.h @@ -187,6 +187,7 @@ static inline u64 vgic_ich_hcr_trap_bits(void) * registers regardless of the hardware backed GIC used. */ struct vgic_vmcr { + u32 en; /* GICv5-specific */ u32 grpen0; u32 grpen1; @@ -362,6 +363,12 @@ void vgic_debug_init(struct kvm *kvm); void vgic_debug_destroy(struct kvm *kvm); int vgic_v5_probe(const struct gic_kvm_info *info); +void vgic_v5_load(struct kvm_vcpu *vcpu); +void vgic_v5_put(struct kvm_vcpu *vcpu); +void vgic_v5_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr); +void vgic_v5_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr); +void vgic_v5_restore_state(struct kvm_vcpu *vcpu); +void vgic_v5_save_state(struct kvm_vcpu *vcpu); static inline int vgic_v3_max_apr_idx(struct kvm_vcpu *vcpu) { diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index 525c8b83e83c9..45d83f45b065d 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -388,6 +388,8 @@ struct vgic_v5_cpu_if { * it is the hyp's responsibility to keep the state constistent. */ u64 vgic_icsr; + + struct gicv5_vpe gicv5_vpe; }; struct vgic_cpu { diff --git a/include/linux/irqchip/arm-gic-v5.h b/include/linux/irqchip/arm-gic-v5.h index 68ddcdb1cec5a..ff10d6c7be2ae 100644 --- a/include/linux/irqchip/arm-gic-v5.h +++ b/include/linux/irqchip/arm-gic-v5.h @@ -354,6 +354,15 @@ int gicv5_spi_irq_set_type(struct irq_data *d, unsigned int type); int gicv5_irs_iste_alloc(u32 lpi); void gicv5_irs_syncr(void); +#ifdef CONFIG_KVM + +/* Embedded in kvm.arch */ +struct gicv5_vpe { + bool resident; +}; + +#endif // CONFIG_KVM + struct gicv5_its_devtab_cfg { union { struct { -- 2.34.1