6.18-stable review patch. If anyone has any objections, please let me know. ------------------ From: Marc Zyngier commit 34af2c3e31f91a739dc175459fdbd99ed952b457 upstream. Our TLB invalidation by VA code is based on comparing two ranges, one defined by the TLB, and one defined by the TLBI instruction. Each range is defined by a start and a size. However, the way the comparison is done doesn't account for address rollover, as it compares an address with (base + size). This works nicely until this expression represent the last page/block in the TTBR1 VA space, as the result is a big fat 0. And a failed TLB invalidation. Rewrite the comparison in a way that is immune to the address rollover (making the end address inclusive instead of exclusive), and move this into a common helper that is used by both VA and IPA invalidations, as suggested by Hyunwoo Kim (although the IPA version didn't suffer from this particular problem, obviously). Fixes: 4ffa72ad8f37e ("KVM: arm64: nv: Add S1 TLB invalidation primitive for VNCR_EL2") Reviewed-by: Yuan Yao Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260806091026.620700-5-maz@kernel.org Signed-off-by: Oliver Upton Signed-off-by: Greg Kroah-Hartman --- arch/arm64/kvm/nested.c | 43 +++++++++++++++++++------------------------ 1 file changed, 19 insertions(+), 24 deletions(-) --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -868,6 +868,20 @@ static void invalidate_vncr(struct vncr_ clear_fixmap(vncr_fixmap(vt->cpu)); } +static bool vncr_tlb_intersects(struct vncr_tlb *vt, u64 addr, + u64 scope_start, u64 scope_size) +{ + u64 tlb_size, tlb_start, tlb_end, scope_end; + + tlb_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, vt->wr.level)); + + tlb_start = addr & ~(tlb_size - 1); + tlb_end = tlb_start + tlb_size - 1; + scope_end = scope_start + scope_size - 1; + + return !(tlb_end < scope_start || tlb_start > scope_end); +} + /* * VNCR TLB invalidation occurs from MMU notifiers or TLBI instructions, and * either can race against a vcpu not being onlined yet (no pseudo-TLB @@ -890,19 +904,9 @@ static void kvm_invalidate_vncr_ipa(stru if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)) return; - kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) { - u64 ipa_start, ipa_end, ipa_size; - - ipa_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, - vt->wr.level)); - ipa_start = vt->wr.pa & ~(ipa_size - 1); - ipa_end = ipa_start + ipa_size; - - if (ipa_end <= start || ipa_start >= end) - continue; - - invalidate_vncr(vt); - } + kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) + if (vncr_tlb_intersects(vt, vt->wr.pa, start, end - start)) + invalidate_vncr(vt); } struct s1e2_tlbi_scope { @@ -928,28 +932,19 @@ static void invalidate_vncr_va(struct kv lockdep_assert_held_write(&kvm->mmu_lock); kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) { - u64 va_start, va_end, va_size; - - va_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, - vt->wr.level)); - va_start = vt->gva & ~(va_size - 1); - va_end = va_start + va_size; - switch (scope->type) { case TLBI_ALL: break; case TLBI_VA: - if (va_end <= scope->va || - va_start >= (scope->va + scope->size)) + if (!vncr_tlb_intersects(vt, vt->gva, scope->va, scope->size)) continue; if (vt->wr.nG && vt->wr.asid != scope->asid) continue; break; case TLBI_VAA: - if (va_end <= scope->va || - va_start >= (scope->va + scope->size)) + if (!vncr_tlb_intersects(vt, vt->gva, scope->va, scope->size)) continue; break;