The local HFENCE range helpers build the loop end with addr + size and then step through the range by the requested page order. If addr + size wraps, a small range near the top of the address space can be treated as empty and skip the intended fence. The final loop step can also wrap before the loop condition is evaluated again. Treat wrapped ranges conservatively as full-range HFENCE requests and stop each bounded loop before the next step can wrap past the computed end. This keeps the existing large-range fallback and only changes local GVMA/VVMA range construction. Fixes: 2415e46e3a9a ("RISC-V: KVM: Introduce range based local HFENCE functions") Signed-off-by: Pengpeng Hou --- arch/riscv/kvm/tlb.c | 71 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 59 insertions(+), 12 deletions(-) diff --git a/arch/riscv/kvm/tlb.c b/arch/riscv/kvm/tlb.c index 993b25ea94d67..b0c83e3ccc15a 100644 --- a/arch/riscv/kvm/tlb.c +++ b/arch/riscv/kvm/tlb.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -24,7 +25,12 @@ void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid, gpa_t gpa, gpa_t gpsz, unsigned long order) { - gpa_t pos; + gpa_t end, pos, step = BIT(order); + + if (check_add_overflow(gpa, gpsz, &end)) { + kvm_riscv_local_hfence_gvma_vmid_all(vmid); + return; + } if (PTRS_PER_PTE < (gpsz >> order)) { kvm_riscv_local_hfence_gvma_vmid_all(vmid); @@ -33,14 +39,20 @@ void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid, if (has_svinval()) { asm volatile (SFENCE_W_INVAL() ::: "memory"); - for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) + for (pos = gpa; pos < end; pos += step) { asm volatile (HINVAL_GVMA(%0, %1) : : "r" (pos >> 2), "r" (vmid) : "memory"); + if (end - pos <= step) + break; + } asm volatile (SFENCE_INVAL_IR() ::: "memory"); } else { - for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) + for (pos = gpa; pos < end; pos += step) { asm volatile (HFENCE_GVMA(%0, %1) : : "r" (pos >> 2), "r" (vmid) : "memory"); + if (end - pos <= step) + break; + } } } @@ -52,7 +64,12 @@ void kvm_riscv_local_hfence_gvma_vmid_all(unsigned long vmid) void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz, unsigned long order) { - gpa_t pos; + gpa_t end, pos, step = BIT(order); + + if (check_add_overflow(gpa, gpsz, &end)) { + kvm_riscv_local_hfence_gvma_all(); + return; + } if (PTRS_PER_PTE < (gpsz >> order)) { kvm_riscv_local_hfence_gvma_all(); @@ -61,14 +78,20 @@ void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz, if (has_svinval()) { asm volatile (SFENCE_W_INVAL() ::: "memory"); - for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) + for (pos = gpa; pos < end; pos += step) { asm volatile(HINVAL_GVMA(%0, zero) : : "r" (pos >> 2) : "memory"); + if (end - pos <= step) + break; + } asm volatile (SFENCE_INVAL_IR() ::: "memory"); } else { - for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) + for (pos = gpa; pos < end; pos += step) { asm volatile(HFENCE_GVMA(%0, zero) : : "r" (pos >> 2) : "memory"); + if (end - pos <= step) + break; + } } } @@ -83,7 +106,13 @@ void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid, unsigned long gvsz, unsigned long order) { - unsigned long pos, hgatp; + unsigned long end, pos, step = BIT(order); + unsigned long hgatp; + + if (check_add_overflow(gva, gvsz, &end)) { + kvm_riscv_local_hfence_vvma_asid_all(vmid, asid); + return; + } if (PTRS_PER_PTE < (gvsz >> order)) { kvm_riscv_local_hfence_vvma_asid_all(vmid, asid); @@ -94,14 +123,20 @@ void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid, if (has_svinval()) { asm volatile (SFENCE_W_INVAL() ::: "memory"); - for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) + for (pos = gva; pos < end; pos += step) { asm volatile(HINVAL_VVMA(%0, %1) : : "r" (pos), "r" (asid) : "memory"); + if (end - pos <= step) + break; + } asm volatile (SFENCE_INVAL_IR() ::: "memory"); } else { - for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) + for (pos = gva; pos < end; pos += step) { asm volatile(HFENCE_VVMA(%0, %1) : : "r" (pos), "r" (asid) : "memory"); + if (end - pos <= step) + break; + } } csr_write(CSR_HGATP, hgatp); @@ -123,7 +158,13 @@ void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid, unsigned long gva, unsigned long gvsz, unsigned long order) { - unsigned long pos, hgatp; + unsigned long end, pos, step = BIT(order); + unsigned long hgatp; + + if (check_add_overflow(gva, gvsz, &end)) { + kvm_riscv_local_hfence_vvma_all(vmid); + return; + } if (PTRS_PER_PTE < (gvsz >> order)) { kvm_riscv_local_hfence_vvma_all(vmid); @@ -134,14 +175,20 @@ void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid, if (has_svinval()) { asm volatile (SFENCE_W_INVAL() ::: "memory"); - for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) + for (pos = gva; pos < end; pos += step) { asm volatile(HINVAL_VVMA(%0, zero) : : "r" (pos) : "memory"); + if (end - pos <= step) + break; + } asm volatile (SFENCE_INVAL_IR() ::: "memory"); } else { - for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) + for (pos = gva; pos < end; pos += step) { asm volatile(HFENCE_VVMA(%0, zero) : : "r" (pos) : "memory"); + if (end - pos <= step) + break; + } } csr_write(CSR_HGATP, hgatp); -- 2.43.0