From: David Woodhouse Rename the local 'longmode' variable and function parameter to 'is_64bit' throughout the Xen hypercall handling code. This distinguishes it from the VM-wide kvm->arch.xen.long_mode which represents the Xen shared_info layout mode. The 'is_64bit' parameter indicates whether the vCPU was in 64-bit mode when it made the hypercall, which determines how to parse the hypercall arguments. The UAPI field name (vcpu->run->xen.u.hcall.longmode) is unchanged. Assisted-by: Kiro:claude-opus-4.6-1m Signed-off-by: David Woodhouse --- arch/x86/kvm/xen.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index f2c6757fc1fa..09f4e154e6b8 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -1469,7 +1469,7 @@ static bool wait_pending_event(struct kvm_vcpu *vcpu, int nr_ports, return ret; } -static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool longmode, +static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool is_64bit, u64 param, u64 *r) { struct sched_poll sched_poll; @@ -1481,7 +1481,7 @@ static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool longmode, !(vcpu->kvm->arch.xen.hvm_config.flags & KVM_XEN_HVM_CONFIG_EVTCHN_SEND)) return false; - if (IS_ENABLED(CONFIG_64BIT) && !longmode) { + if (IS_ENABLED(CONFIG_64BIT) && !is_64bit) { struct compat_sched_poll sp32; /* Sanity check that the compat struct definition is correct */ @@ -1578,12 +1578,12 @@ static void cancel_evtchn_poll(struct timer_list *t) kvm_vcpu_kick(vcpu); } -static bool kvm_xen_hcall_sched_op(struct kvm_vcpu *vcpu, bool longmode, +static bool kvm_xen_hcall_sched_op(struct kvm_vcpu *vcpu, bool is_64bit, int cmd, u64 param, u64 *r) { switch (cmd) { case SCHEDOP_poll: - if (kvm_xen_schedop_poll(vcpu, longmode, param, r)) + if (kvm_xen_schedop_poll(vcpu, is_64bit, param, r)) return true; fallthrough; case SCHEDOP_yield: @@ -1602,7 +1602,7 @@ struct compat_vcpu_set_singleshot_timer { uint32_t flags; } __attribute__((packed)); -static bool kvm_xen_hcall_vcpu_op(struct kvm_vcpu *vcpu, bool longmode, int cmd, +static bool kvm_xen_hcall_vcpu_op(struct kvm_vcpu *vcpu, bool is_64bit, int cmd, int vcpu_id, u64 param, u64 *r) { struct vcpu_set_singleshot_timer oneshot; @@ -1646,7 +1646,7 @@ static bool kvm_xen_hcall_vcpu_op(struct kvm_vcpu *vcpu, bool longmode, int cmd, BUILD_BUG_ON(sizeof_field(struct compat_vcpu_set_singleshot_timer, flags) != sizeof_field(struct vcpu_set_singleshot_timer, flags)); - if (kvm_read_guest_virt(vcpu, param, &oneshot, longmode ? sizeof(oneshot) : + if (kvm_read_guest_virt(vcpu, param, &oneshot, is_64bit ? sizeof(oneshot) : sizeof(struct compat_vcpu_set_singleshot_timer), &e)) { *r = -EFAULT; return true; @@ -1678,7 +1678,7 @@ static bool kvm_xen_hcall_set_timer_op(struct kvm_vcpu *vcpu, uint64_t timeout, int kvm_xen_hypercall(struct kvm_vcpu *vcpu) { - bool longmode; + bool is_64bit; u64 input, params[6], r = -ENOSYS; bool handled = false; u8 cpl; @@ -1688,8 +1688,8 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu) kvm_hv_hypercall_enabled(vcpu)) return kvm_hv_hypercall(vcpu); - longmode = is_64_bit_hypercall(vcpu); - if (!longmode) { + is_64bit = is_64_bit_hypercall(vcpu); + if (!is_64bit) { input = kvm_eax_read(vcpu); params[0] = kvm_ebx_read(vcpu); params[1] = kvm_ecx_read(vcpu); @@ -1735,17 +1735,17 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu) handled = kvm_xen_hcall_evtchn_send(vcpu, params[1], &r); break; case __HYPERVISOR_sched_op: - handled = kvm_xen_hcall_sched_op(vcpu, longmode, params[0], + handled = kvm_xen_hcall_sched_op(vcpu, is_64bit, params[0], params[1], &r); break; case __HYPERVISOR_vcpu_op: - handled = kvm_xen_hcall_vcpu_op(vcpu, longmode, params[0], params[1], + handled = kvm_xen_hcall_vcpu_op(vcpu, is_64bit, params[0], params[1], params[2], &r); break; case __HYPERVISOR_set_timer_op: { u64 timeout = params[0]; /* In 32-bit mode, the 64-bit timeout is in two 32-bit params. */ - if (!longmode) + if (!is_64bit) timeout |= params[1] << 32; handled = kvm_xen_hcall_set_timer_op(vcpu, timeout, &r); break; @@ -1760,7 +1760,7 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu) handle_in_userspace: vcpu->run->exit_reason = KVM_EXIT_XEN; vcpu->run->xen.type = KVM_EXIT_XEN_HCALL; - vcpu->run->xen.u.hcall.longmode = longmode; + vcpu->run->xen.u.hcall.longmode = is_64bit; vcpu->run->xen.u.hcall.cpl = cpl; vcpu->run->xen.u.hcall.input = input; vcpu->run->xen.u.hcall.params[0] = params[0]; base-commit: 51ba04112e93ac6e04c627eddcea5caf9cbc0134 -- 2.55.0 From: David Woodhouse Add a kvm_xen_has_64bit_shinfo() helper macro to replace the repeated pattern of 'IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode' throughout the Xen emulation code. The macro uses READ_ONCE() to ensure a consistent snapshot of the flag, which can be changed by another vCPU at any time. This is the KVM equivalent of Xen's !has_32bit_shinfo(). Assisted-by: Kiro:claude-opus-4.6-1m Signed-off-by: David Woodhouse --- arch/x86/kvm/xen.c | 16 ++++++++-------- arch/x86/kvm/xen.h | 5 +++++ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 09f4e154e6b8..e234c2a192c3 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -73,7 +73,7 @@ static int kvm_xen_shared_info_init(struct kvm *kvm) BUILD_BUG_ON(offsetof(struct shared_info, wc) != 0xc00); BUILD_BUG_ON(offsetof(struct shared_info, wc_sec_hi) != 0xc0c); - if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) { + if (kvm_xen_has_64bit_shinfo(kvm)) { struct shared_info *shinfo = gpc->khva; wc_sec_hi = &shinfo->wc_sec_hi; @@ -389,7 +389,7 @@ static void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, bool atomic) BUILD_BUG_ON(sizeof_field(struct vcpu_runstate_info, time) != sizeof(vx->runstate_times)); - if (IS_ENABLED(CONFIG_64BIT) && v->kvm->arch.xen.long_mode) { + if (kvm_xen_has_64bit_shinfo(v->kvm)) { user_len = sizeof(struct vcpu_runstate_info); times_ofs = offsetof(struct vcpu_runstate_info, state_entry_time); @@ -660,7 +660,7 @@ void kvm_xen_inject_pending_events(struct kvm_vcpu *v) } /* Now gpc->khva is a valid kernel address for the vcpu_info */ - if (IS_ENABLED(CONFIG_64BIT) && v->kvm->arch.xen.long_mode) { + if (kvm_xen_has_64bit_shinfo(v->kvm)) { struct vcpu_info *vi = gpc->khva; asm volatile(LOCK_PREFIX "orq %0, %1\n" @@ -977,7 +977,7 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) * address, that's actually OK. kvm_xen_update_runstate_guest() * will cope. */ - if (IS_ENABLED(CONFIG_64BIT) && vcpu->kvm->arch.xen.long_mode) + if (kvm_xen_has_64bit_shinfo(vcpu->kvm)) sz = sizeof(struct vcpu_runstate_info); else sz = sizeof(struct compat_vcpu_runstate_info); @@ -1425,7 +1425,7 @@ static int kvm_xen_hypercall_complete_userspace(struct kvm_vcpu *vcpu) static inline int max_evtchn_port(struct kvm *kvm) { - if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) + if (kvm_xen_has_64bit_shinfo(kvm)) return EVTCHN_2L_NR_CHANNELS; else return COMPAT_EVTCHN_2L_NR_CHANNELS; @@ -1447,7 +1447,7 @@ static bool wait_pending_event(struct kvm_vcpu *vcpu, int nr_ports, goto out_rcu; ret = false; - if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) { + if (kvm_xen_has_64bit_shinfo(kvm)) { struct shared_info *shinfo = gpc->khva; pending_bits = (unsigned long *)&shinfo->evtchn_pending; } else { @@ -1828,7 +1828,7 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) if (!kvm_gpc_check(gpc, PAGE_SIZE)) goto out_rcu; - if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) { + if (kvm_xen_has_64bit_shinfo(kvm)) { struct shared_info *shinfo = gpc->khva; pending_bits = (unsigned long *)&shinfo->evtchn_pending; mask_bits = (unsigned long *)&shinfo->evtchn_mask; @@ -1869,7 +1869,7 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) goto out_rcu; } - if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) { + if (kvm_xen_has_64bit_shinfo(kvm)) { struct vcpu_info *vcpu_info = gpc->khva; if (!test_and_set_bit(port_word_bit, &vcpu_info->evtchn_pending_sel)) { WRITE_ONCE(vcpu_info->evtchn_upcall_pending, 1); diff --git a/arch/x86/kvm/xen.h b/arch/x86/kvm/xen.h index 59e6128a7bd3..3e8c9306eb89 100644 --- a/arch/x86/kvm/xen.h +++ b/arch/x86/kvm/xen.h @@ -248,6 +248,11 @@ struct compat_shared_info { #define COMPAT_EVTCHN_2L_NR_CHANNELS (8 * \ sizeof_field(struct compat_shared_info, \ evtchn_pending)) + +/* Latched VM-wide mode; the KVM equivalent of Xen's !has_32bit_shinfo(). */ +#define kvm_xen_has_64bit_shinfo(kvm) \ + (IS_ENABLED(CONFIG_64BIT) && READ_ONCE((kvm)->arch.xen.long_mode)) + struct compat_vcpu_runstate_info { int state; uint64_t state_entry_time; -- 2.55.0 From: David Woodhouse Rename in preparation for adding a variant that takes a latched bool argument for use in paths that need a consistent snapshot of the shinfo mode. No functional change. Assisted-by: Kiro:claude-opus-4.6-1m Signed-off-by: David Woodhouse --- arch/x86/kvm/xen.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index e234c2a192c3..00894afea9be 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -1423,7 +1423,7 @@ static int kvm_xen_hypercall_complete_userspace(struct kvm_vcpu *vcpu) return kvm_xen_hypercall_set_result(vcpu, run->xen.u.hcall.result); } -static inline int max_evtchn_port(struct kvm *kvm) +static inline int kvm_max_evtchn_port(struct kvm *kvm) { if (kvm_xen_has_64bit_shinfo(kvm)) return EVTCHN_2L_NR_CHANNELS; @@ -1530,7 +1530,7 @@ static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool is_64bit, } for (i = 0; i < sched_poll.nr_ports; i++) { - if (ports[i] >= max_evtchn_port(vcpu->kvm)) { + if (ports[i] >= kvm_max_evtchn_port(vcpu->kvm)) { *r = -EINVAL; goto out; } @@ -1817,7 +1817,7 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) WRITE_ONCE(xe->vcpu_idx, vcpu->vcpu_idx); } - if (xe->port >= max_evtchn_port(kvm)) + if (xe->port >= kvm_max_evtchn_port(kvm)) return -EINVAL; rc = -EWOULDBLOCK; @@ -1979,7 +1979,7 @@ int kvm_xen_setup_evtchn(struct kvm *kvm, struct kvm_vcpu *vcpu; /* - * Don't check for the port being within range of max_evtchn_port(). + * Don't check for the port being within range of kvm_max_evtchn_port(). * Userspace can configure what ever targets it likes; events just won't * be delivered if/while the target is invalid, just like userspace can * configure MSIs which target non-existent APICs. @@ -1988,8 +1988,8 @@ int kvm_xen_setup_evtchn(struct kvm *kvm, * can be restored *independently* of other things like creating vCPUs, * without imposing an ordering dependency on userspace. In this * particular case, the problematic ordering would be with setting the - * Xen 'long mode' flag, which changes max_evtchn_port() to allow 4096 - * instead of 1024 event channels. + * Xen 'long mode' flag, which changes kvm_max_evtchn_port() to allow + * 4096 instead of 1024 event channels. */ /* We only support 2 level event channels for now */ @@ -2026,7 +2026,7 @@ int kvm_xen_hvm_evtchn_send(struct kvm *kvm, struct kvm_irq_routing_xen_evtchn * struct kvm_xen_evtchn e; int ret; - if (!uxe->port || uxe->port >= max_evtchn_port(kvm)) + if (!uxe->port || uxe->port >= kvm_max_evtchn_port(kvm)) return -EINVAL; /* We only support 2 level event channels for now */ @@ -2136,7 +2136,7 @@ static int kvm_xen_eventfd_assign(struct kvm *kvm, case EVTCHNSTAT_interdomain: if (data->u.evtchn.deliver.port.port) { - if (data->u.evtchn.deliver.port.port >= max_evtchn_port(kvm)) + if (data->u.evtchn.deliver.port.port >= kvm_max_evtchn_port(kvm)) goto out_noeventfd; /* -EINVAL */ } else { eventfd = eventfd_ctx_fdget(data->u.evtchn.deliver.eventfd.fd); @@ -2254,7 +2254,7 @@ static int kvm_xen_setattr_evtchn(struct kvm *kvm, struct kvm_xen_hvm_attr *data if (data->u.evtchn.flags == KVM_XEN_EVTCHN_RESET) return kvm_xen_eventfd_reset(kvm); - if (!port || port >= max_evtchn_port(kvm)) + if (!port || port >= kvm_max_evtchn_port(kvm)) return -EINVAL; if (data->u.evtchn.flags == KVM_XEN_EVTCHN_DEASSIGN) -- 2.55.0 From: Hyunwoo Kim kvm_xen_set_evtchn_fast() assumes the port range check in max_evtchn_port() and the bitmap layout selection observe the same shinfo mode, but each calls kvm_xen_has_64bit_shinfo() separately. If the guest changes the mode in between, a port accepted by the 64-bit range check is handled with the 32-bit layout, and port_word_bit falls outside evtchn_pending_sel. Latch kvm_xen_has_64bit_shinfo() once on entry so the range check and both layout computations use the same value. In practice this is harmless: the evtchn_pending bitmap is at the same offset in both native and compat shared_info layouts, so a stale mode just results in setting a bit in what the guest (in its new compat mode) considers the evtchn_mask, wallclock, or the arch_shared_info fields which follow it — all of which are in the guest's own page. Even with this fix, the same corruption can occur if 64-bit mode is latched and the guest switches to 32-bit mode immediately afterward. Like Xen, KVM makes no attempt to *convert* when shinfo mode is changed. Only the wallclock field is updated in the new location. This fix is for internal consistency rather than correcting any observable bug. Fixes: 14243b387137 ("KVM: x86/xen: Add KVM_IRQ_ROUTING_XEN_EVTCHN and event channel delivery") Reported-by: Hyunwoo Kim Closes: https://lore.kernel.org/all/aiHPPUk5DY7rH-zL@v4bel/ Signed-off-by: Hyunwoo Kim [dwmw2: Rework on top of long_mode/has_64bit_shinfo cleanups] Signed-off-by: David Woodhouse --- arch/x86/kvm/xen.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 00894afea9be..9edfdc585337 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -1423,14 +1423,19 @@ static int kvm_xen_hypercall_complete_userspace(struct kvm_vcpu *vcpu) return kvm_xen_hypercall_set_result(vcpu, run->xen.u.hcall.result); } -static inline int kvm_max_evtchn_port(struct kvm *kvm) +static inline int max_evtchn_port(bool has_64bit_shinfo) { - if (kvm_xen_has_64bit_shinfo(kvm)) + if (has_64bit_shinfo) return EVTCHN_2L_NR_CHANNELS; else return COMPAT_EVTCHN_2L_NR_CHANNELS; } +static inline int kvm_max_evtchn_port(struct kvm *kvm) +{ + return max_evtchn_port(kvm_xen_has_64bit_shinfo(kvm)); +} + static bool wait_pending_event(struct kvm_vcpu *vcpu, int nr_ports, evtchn_port_t *ports) { @@ -1800,8 +1805,9 @@ static void kvm_xen_check_poller(struct kvm_vcpu *vcpu, int port) int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) { struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache; - struct kvm_vcpu *vcpu; + bool has_64bit_shinfo = kvm_xen_has_64bit_shinfo(kvm); unsigned long *pending_bits, *mask_bits; + struct kvm_vcpu *vcpu; unsigned long flags; int port_word_bit; bool kick_vcpu = false; @@ -1817,7 +1823,7 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) WRITE_ONCE(xe->vcpu_idx, vcpu->vcpu_idx); } - if (xe->port >= kvm_max_evtchn_port(kvm)) + if (xe->port >= max_evtchn_port(has_64bit_shinfo)) return -EINVAL; rc = -EWOULDBLOCK; @@ -1828,7 +1834,7 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) if (!kvm_gpc_check(gpc, PAGE_SIZE)) goto out_rcu; - if (kvm_xen_has_64bit_shinfo(kvm)) { + if (has_64bit_shinfo) { struct shared_info *shinfo = gpc->khva; pending_bits = (unsigned long *)&shinfo->evtchn_pending; mask_bits = (unsigned long *)&shinfo->evtchn_mask; @@ -1869,7 +1875,7 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) goto out_rcu; } - if (kvm_xen_has_64bit_shinfo(kvm)) { + if (has_64bit_shinfo) { struct vcpu_info *vcpu_info = gpc->khva; if (!test_and_set_bit(port_word_bit, &vcpu_info->evtchn_pending_sel)) { WRITE_ONCE(vcpu_info->evtchn_upcall_pending, 1); -- 2.55.0 From: David Woodhouse kvm_xen_schedop_poll() validates port numbers against kvm_max_evtchn_port() and then calls wait_pending_event() which reads the shinfo mode again to select the bitmap layout. Latch kvm_xen_has_64bit_shinfo() once and pass it to both max_evtchn_port() and wait_pending_event(). As with the previous fix to kvm_xen_set_evtchn_fast(), this is harmless in practice for the same reasons: the inconsistency can only corrupt fields in the guest's own shared_info page, and the same corruption can occur anyway if the mode changes immediately after the latch. Fixes: d518b9d0fc80 ("KVM: x86/xen: handle PV spinlocks slowpath") Assisted-by: Kiro:claude-opus-4.6-1m Signed-off-by: David Woodhouse --- arch/x86/kvm/xen.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 9edfdc585337..e249a1b1d446 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -1436,8 +1436,8 @@ static inline int kvm_max_evtchn_port(struct kvm *kvm) return max_evtchn_port(kvm_xen_has_64bit_shinfo(kvm)); } -static bool wait_pending_event(struct kvm_vcpu *vcpu, int nr_ports, - evtchn_port_t *ports) +static bool wait_pending_event(struct kvm_vcpu *vcpu, bool has_64bit_shinfo, + int nr_ports, evtchn_port_t *ports) { struct kvm *kvm = vcpu->kvm; struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache; @@ -1452,7 +1452,7 @@ static bool wait_pending_event(struct kvm_vcpu *vcpu, int nr_ports, goto out_rcu; ret = false; - if (kvm_xen_has_64bit_shinfo(kvm)) { + if (has_64bit_shinfo) { struct shared_info *shinfo = gpc->khva; pending_bits = (unsigned long *)&shinfo->evtchn_pending; } else { @@ -1477,6 +1477,7 @@ static bool wait_pending_event(struct kvm_vcpu *vcpu, int nr_ports, static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool is_64bit, u64 param, u64 *r) { + bool has_64bit_shinfo = kvm_xen_has_64bit_shinfo(vcpu->kvm); struct sched_poll sched_poll; evtchn_port_t port, *ports; struct x86_exception e; @@ -1535,7 +1536,7 @@ static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool is_64bit, } for (i = 0; i < sched_poll.nr_ports; i++) { - if (ports[i] >= kvm_max_evtchn_port(vcpu->kvm)) { + if (ports[i] >= max_evtchn_port(has_64bit_shinfo)) { *r = -EINVAL; goto out; } @@ -1548,7 +1549,7 @@ static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool is_64bit, set_bit(vcpu->vcpu_idx, vcpu->kvm->arch.xen.poll_mask); - if (!wait_pending_event(vcpu, sched_poll.nr_ports, ports)) { + if (!wait_pending_event(vcpu, has_64bit_shinfo, sched_poll.nr_ports, ports)) { kvm_set_mp_state(vcpu, KVM_MP_STATE_HALTED); if (sched_poll.timeout) -- 2.55.0 From: David Woodhouse Xen's map_guest_area() enforces that vcpu_info is aligned to sizeof(xen_ulong_t). KVM has no such check, allowing a guest to register vcpu_info at an arbitrary byte alignment. Enforce unconditional 4-byte alignment regardless of the current shinfo mode. This is sufficient because subsequent commits ensure that all locked atomic operations on vcpu_info fields use at most 32-bit accesses. Return -ENXIO on failure, matching Xen's map_guest_area() behaviour for unaligned requests. Originally observed in review of an unrelated patch: https://lore.kernel.org/all/20260604193554.1BA311F00893@smtp.kernel.org/ Cc: stable@vger.kernel.org Fixes: 73e69a86347a ("KVM: x86/xen: register vcpu info") Reported-by: sashiko-bot@kernel.org Closes: https://lore.kernel.org/all/20260604193554.1BA311F00893@smtp.kernel.org Assisted-by: Kiro:claude-opus-4.6-1m Signed-off-by: David Woodhouse --- arch/x86/kvm/xen.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index e249a1b1d446..959d79eef0ce 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -925,6 +925,10 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) break; } + r = -ENXIO; + if (!IS_ALIGNED(data->u.gpa, sizeof(u32))) + break; + r = kvm_gpc_activate(&vcpu->arch.xen.vcpu_info_cache, data->u.gpa, sizeof(struct vcpu_info)); } else { @@ -934,6 +938,10 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) break; } + r = -ENXIO; + if (!IS_ALIGNED(data->u.hva, sizeof(u32))) + break; + r = kvm_gpc_activate_hva(&vcpu->arch.xen.vcpu_info_cache, data->u.hva, sizeof(struct vcpu_info)); } -- 2.55.0 From: David Woodhouse Replace test_and_set_bit() on vcpu_info->evtchn_pending_sel with an explicit 'lock btsl' in kvm_xen_set_evtchn_fast(). The generic test_and_set_bit() uses a 64-bit locked operation ('lock btsq') on x86-64, and the address of the per-vCPU info is guest-controlled and only required to be 32-bit aligned, so an 8-byte access can generate a split-lock #AC exception. Since evtchn_pending_sel is at most 64 bits wide and port_word_bit ranges 0-63, a 32-bit 'lock btsl' suffices for both native and compat vcpu_info layouts, and only requires the 4-byte alignment that is already guaranteed by the registration path. This also eliminates the bogus cast of compat_vcpu_info's 32-bit evtchn_pending_sel to 'unsigned long *' which was the original source of the split-lock hazard. Note, KVM reuses the local gpc. The atomic accesses to pending_bits is on the page-aligned per-VM shared_info structure, i.e. doesn't need the same treatment as the access is guaranteed to be 64-bit aligned. Fixes: 14243b387137 ("KVM: x86/xen: Add KVM_IRQ_ROUTING_XEN_EVTCHN and event channel delivery") Reported-by: sashiko-bot@kernel.org Closes: https://lore.kernel.org/all/20260604193554.1BA311F00893@smtp.kernel.org Suggested-by: Sean Christopherson Signed-off-by: David Woodhouse Assisted-by: Claude:claude-mythos-5 --- arch/x86/kvm/xen.c | 47 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 959d79eef0ce..c935651906ec 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -1815,7 +1815,7 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) { struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache; bool has_64bit_shinfo = kvm_xen_has_64bit_shinfo(kvm); - unsigned long *pending_bits, *mask_bits; + unsigned long *pending_bits, *mask_bits, vi_pending_sel_ofs; struct kvm_vcpu *vcpu; unsigned long flags; int port_word_bit; @@ -1848,11 +1848,18 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) pending_bits = (unsigned long *)&shinfo->evtchn_pending; mask_bits = (unsigned long *)&shinfo->evtchn_mask; port_word_bit = xe->port / 64; + + vi_pending_sel_ofs = offsetof(struct vcpu_info, evtchn_pending_sel); } else { struct compat_shared_info *shinfo = gpc->khva; pending_bits = (unsigned long *)&shinfo->evtchn_pending; mask_bits = (unsigned long *)&shinfo->evtchn_mask; port_word_bit = xe->port / 32; + + vi_pending_sel_ofs = offsetof(struct compat_vcpu_info, evtchn_pending_sel); + + /* test_and_set_bit() needs 64-bit alignment, but that's OK */ + BUILD_BUG_ON(offsetof(struct compat_shared_info, evtchn_pending) & 7); } /* @@ -1868,6 +1875,8 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) rc = -ENOTCONN; /* Masked */ kvm_xen_check_poller(vcpu, xe->port); } else { + bool old; + rc = 1; /* Delivered to the bitmap in shared_info. */ /* Now switch to the vCPU's vcpu_info to set the index and pending_sel */ read_unlock_irqrestore(&gpc->lock, flags); @@ -1884,19 +1893,29 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) goto out_rcu; } - if (has_64bit_shinfo) { - struct vcpu_info *vcpu_info = gpc->khva; - if (!test_and_set_bit(port_word_bit, &vcpu_info->evtchn_pending_sel)) { - WRITE_ONCE(vcpu_info->evtchn_upcall_pending, 1); - kick_vcpu = true; - } - } else { - struct compat_vcpu_info *vcpu_info = gpc->khva; - if (!test_and_set_bit(port_word_bit, - (unsigned long *)&vcpu_info->evtchn_pending_sel)) { - WRITE_ONCE(vcpu_info->evtchn_upcall_pending, 1); - kick_vcpu = true; - } + /* + * Explicitly use a 32-bit btsl instead of test_and_set_bit(), + * which would use btsq on x86-64. The vcpu_info is guest- + * controlled and only required to be 32-bit aligned, so a + * 64-bit access could generate a split-lock #AC. + * + * Note, this does not apply to the test_and_set_bit() on + * pending_bits above: that is in the per-VM shared_info, which + * is page aligned, so the access is guaranteed to be 64-bit + * aligned. + */ + old = GEN_BINARY_RMWcc(LOCK_PREFIX "btsl", + *(u32 *)(gpc->khva + vi_pending_sel_ofs), + c, "Ir", port_word_bit); + if (!old) { + struct vcpu_info *vi = gpc->khva; + + /* No need for compat handling */ + BUILD_BUG_ON(offsetof(struct vcpu_info, evtchn_upcall_pending) != + offsetof(struct compat_vcpu_info, evtchn_upcall_pending)); + + WRITE_ONCE(vi->evtchn_upcall_pending, 1); + kick_vcpu = true; } /* For the per-vCPU lapic vector, deliver it as MSI. */ -- 2.55.0 From: Sean Christopherson When propagating pending Xen events from KVM's "cache" to the guest-visible structure, use two 32-bit atomic operations to do the bitwise-OR into the guest-controlled structure if the structure isn't 64-bit aligned, i.e. if the guest registered its vcpu_info in 32-bit mode and then switched to 64-bit mode, in which case using a 64-bit atomic OR will generate a split-lock #AC (if enabled). Opportunistically isolate the clearing of the bits from KVM's cache, as that structure is KVM-controlled, i.e. is guaranteed to be 64-bit aligned. This will allow dropping the open-coded inline asm blobs in the future. [dwmw2: Cast to u64 before shifting; evtchn_pending_sel is unsigned long, so >> 32 is undefined on 32-bit even though the branch is unreachable there] Fixes: 14243b387137 ("KVM: x86/xen: Add KVM_IRQ_ROUTING_XEN_EVTCHN and event channel delivery") Reported-by: sashiko-bot@kernel.org Closes: https://lore.kernel.org/all/20260604193554.1BA311F00893@smtp.kernel.org Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202608071502.rYOi3Pg8-lkp@intel.com/ Signed-off-by: Sean Christopherson Signed-off-by: David Woodhouse --- arch/x86/kvm/xen.c | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index c935651906ec..419b07fdaa3a 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -663,13 +663,28 @@ void kvm_xen_inject_pending_events(struct kvm_vcpu *v) if (kvm_xen_has_64bit_shinfo(v->kvm)) { struct vcpu_info *vi = gpc->khva; - asm volatile(LOCK_PREFIX "orq %0, %1\n" - "notq %0\n" - LOCK_PREFIX "andq %0, %2\n" - : "=r" (evtchn_pending_sel), - "+m" (vi->evtchn_pending_sel), - "+m" (v->arch.xen.evtchn_pending_sel) - : "0" (evtchn_pending_sel)); + if (IS_ALIGNED((unsigned long)&vi->evtchn_pending_sel, sizeof(u64))) + asm volatile(LOCK_PREFIX "orq %[src], %[dst]\n" + : [dst] "+m" (vi->evtchn_pending_sel) + : [src] "r" (evtchn_pending_sel)); + else + /* + * The cast keeps the shift well-defined on 32-bit, + * where evtchn_pending_sel is 32 bits wide and this + * branch is unreachable anyway (this is inside + * kvm_xen_has_64bit_shinfo(), which is gated on + * IS_ENABLED(CONFIG_64BIT)). + */ + asm volatile(LOCK_PREFIX "orl %[src_lo], %[dst_lo]\n" + LOCK_PREFIX "orl %[src_hi], %[dst_hi]\n" + : [dst_lo] "+m" (vi->evtchn_pending_sel), + [dst_hi] "+m" (*(((u32 *)&vi->evtchn_pending_sel) + 1)) + : [src_lo] "r" ((u32)evtchn_pending_sel), + [src_hi] "r" ((u32)((u64)evtchn_pending_sel >> 32))); + + asm volatile(LOCK_PREFIX "andq %1, %0\n" + : "+m" (v->arch.xen.evtchn_pending_sel) + : "r" (~evtchn_pending_sel)); WRITE_ONCE(vi->evtchn_upcall_pending, 1); } else { u32 evtchn_pending_sel32 = evtchn_pending_sel; -- 2.55.0 From: Sean Christopherson Replace the open coded atomic asm blobs in the Xen event injection code with equivalent atomic{,64}_xxx() operations. Casting the event channel to atomic types is ugly, but not as ugly as asm blobs. No functional change intended. Signed-off-by: Sean Christopherson Signed-off-by: David Woodhouse --- arch/x86/kvm/xen.c | 35 ++++++++++++----------------------- 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 419b07fdaa3a..99f3ffa64fdb 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -662,12 +662,12 @@ void kvm_xen_inject_pending_events(struct kvm_vcpu *v) /* Now gpc->khva is a valid kernel address for the vcpu_info */ if (kvm_xen_has_64bit_shinfo(v->kvm)) { struct vcpu_info *vi = gpc->khva; + void *vi_pending_sel = &vi->evtchn_pending_sel; - if (IS_ALIGNED((unsigned long)&vi->evtchn_pending_sel, sizeof(u64))) - asm volatile(LOCK_PREFIX "orq %[src], %[dst]\n" - : [dst] "+m" (vi->evtchn_pending_sel) - : [src] "r" (evtchn_pending_sel)); - else + if (IS_ALIGNED((unsigned long)vi_pending_sel, sizeof(u64))) { + atomic64_or(evtchn_pending_sel, vi_pending_sel); + } else { + atomic_or(evtchn_pending_sel, vi_pending_sel); /* * The cast keeps the shift well-defined on 32-bit, * where evtchn_pending_sel is 32 bits wide and this @@ -675,28 +675,17 @@ void kvm_xen_inject_pending_events(struct kvm_vcpu *v) * kvm_xen_has_64bit_shinfo(), which is gated on * IS_ENABLED(CONFIG_64BIT)). */ - asm volatile(LOCK_PREFIX "orl %[src_lo], %[dst_lo]\n" - LOCK_PREFIX "orl %[src_hi], %[dst_hi]\n" - : [dst_lo] "+m" (vi->evtchn_pending_sel), - [dst_hi] "+m" (*(((u32 *)&vi->evtchn_pending_sel) + 1)) - : [src_lo] "r" ((u32)evtchn_pending_sel), - [src_hi] "r" ((u32)((u64)evtchn_pending_sel >> 32))); - - asm volatile(LOCK_PREFIX "andq %1, %0\n" - : "+m" (v->arch.xen.evtchn_pending_sel) - : "r" (~evtchn_pending_sel)); + atomic_or((u64)evtchn_pending_sel >> 32, + vi_pending_sel + 4); + } + + atomic64_andnot(evtchn_pending_sel, (void *)&v->arch.xen.evtchn_pending_sel); WRITE_ONCE(vi->evtchn_upcall_pending, 1); } else { - u32 evtchn_pending_sel32 = evtchn_pending_sel; struct compat_vcpu_info *vi = gpc->khva; - asm volatile(LOCK_PREFIX "orl %0, %1\n" - "notl %0\n" - LOCK_PREFIX "andl %0, %2\n" - : "=r" (evtchn_pending_sel32), - "+m" (vi->evtchn_pending_sel), - "+m" (v->arch.xen.evtchn_pending_sel) - : "0" (evtchn_pending_sel32)); + atomic_or(evtchn_pending_sel, (void *)&vi->evtchn_pending_sel); + atomic_andnot(evtchn_pending_sel, (void *)&v->arch.xen.evtchn_pending_sel); WRITE_ONCE(vi->evtchn_upcall_pending, 1); } -- 2.55.0 From: David Woodhouse kvm_gpc_check() checks the cached memslot generation against the current one, which dereferences kvm->memslots and therefore requires kvm->srcu to be held. __kvm_xen_has_interrupt() does not take it. Most callers do happen to hold kvm->srcu, but not all of them: - kvm_emulate_halt() on the VM-Exit path, via kvm_vcpu_has_events() and kvm_cpu_has_extint(). vcpu_enter_guest() drops the vCPU's SRCU lock before entering the guest, so it is not held on the way back out. - kvm_vcpu_block() -> kvm_vcpu_check_block() -> kvm_arch_vcpu_runnable(), which is the case the existing comment in this function describes. On a PROVE_RCU kernel the former produces: WARNING: suspicious RCU usage include/linux/kvm_host.h:1092 suspicious rcu_dereference_check() usage! ... kvm_gpc_check+0x344/0x3e0 [kvm] __kvm_xen_has_interrupt+0x83/0x310 [kvm] kvm_cpu_has_extint+0x1ff/0x370 [kvm] kvm_cpu_has_interrupt+0x16/0x100 [kvm] kvm_vcpu_has_events+0x4ce/0x690 [kvm] kvm_emulate_halt+0x52/0x1f0 [kvm] vmx_vcpu_run+0x988/0x2630 [kvm_intel] Use guard(srcu) so that the three existing early returns don't each need an explicit unlock. SRCU read sections nest, so this is harmless on the paths which already hold it, and srcu_read_lock() does not sleep, so it is also safe in the atomic case which this function already handles. Fixes: 7caf9571563e ("KVM: x86/xen: Use gfn_to_pfn_cache for vcpu_info") Cc: stable@vger.kernel.org Signed-off-by: David Woodhouse Assisted-by: Claude:claude-mythos-5 --- arch/x86/kvm/xen.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 99f3ffa64fdb..ff55ff290afb 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -716,6 +716,16 @@ int __kvm_xen_has_interrupt(struct kvm_vcpu *v) BUILD_BUG_ON(sizeof(rc) != sizeof_field(struct compat_vcpu_info, evtchn_upcall_pending)); + /* + * kvm_gpc_check() checks the memslot generation, so kvm->srcu must be + * held. Most callers hold it already, but this is also reached from + * kvm_emulate_halt() on the VM-Exit path and from kvm_vcpu_block(), + * where vcpu_enter_guest() has already dropped the vCPU's SRCU lock. + * Taking SRCU does not sleep, so it is safe even in the atomic case + * which is handled below. + */ + guard(srcu)(&v->kvm->srcu); + read_lock_irqsave(&gpc->lock, flags); while (!kvm_gpc_check(gpc, sizeof(struct vcpu_info))) { read_unlock_irqrestore(&gpc->lock, flags); -- 2.55.0 From: David Woodhouse The gfn_to_pfn_cache refresh path guards against mmu notifier invalidations which complete while it has dropped gpc->lock for the HVA->PFN lookup: hva_to_pfn_retry() samples kvm->mmu_invalidate_seq and retries if it changed, or if mn_active_invalidate_count is still elevated. That is insufficient for HVA-based caches. mmu_invalidate_seq is only advanced by kvm_mmu_invalidate_end() when the invalidated range overlaps a memslot, and an HVA-based cache (e.g. the Xen shared_info page mapped with KVM_XEN_ATTR_TYPE_SHARED_INFO_HVA) need not be backed by any memslot at all. An invalidation of the cached HVA which starts and ends entirely within the lookup window is thus invisible to the retry check: mn_active_invalidate_count is back to zero and the sequence never moved. The refresh then publishes a mapping of a page which has already been freed, and the next reader dereferences it: BUG: KASAN: use-after-free in kvm_xen_shared_info_init+0x3c6/0x440 Read of size 4 at addr ffff8880599c2900 by task syz.2.383/7257 Since gfn_to_pfn_cache_invalidate_start() deliberately skips caches which are not currently valid (including one whose refresh is in progress, as the refresh clears the valid flag before dropping the lock), the retry check is the only line of defence, and it must fire for *any* invalidation, not just those hitting a memslot. Add a dedicated kvm->gpc_invalidate_seq, incremented by every kvm_mmu_notifier_invalidate_range_end() under mn_invalidate_lock before mn_active_invalidate_count is decremented, and check it in hva_to_pfn_retry() instead of mmu_invalidate_seq. Incrementing in range_end() in the same critical section as the in-progress count also closes the variant where the cache is activated with the contested HVA only after invalidate_range_start() has run. The same bug is also reachable through the per-vCPU vcpu_info cache (KVM_XEN_VCPU_ATTR_TYPE_VCPU_INFO_HVA), where the stale mapping is then dereferenced by kvm_setup_guest_pvclock() on the next KVM_RUN: BUG: KASAN: use-after-free in kvm_setup_guest_pvclock+0x5bf/0x660 This intentionally makes refresh retry on *unrelated* mmu notifier events; restoring precision (and reworking the GPC locking more generally) is left for a subsequent series. Reproducers: https://david.woodhou.se/xen_shinfo_race.c https://david.woodhou.se/vcpu_info_race.c Suggested-by: Sean Christopherson Reported-by: syzbot+0948c82180d475ad24e2@syzkaller.appspotmail.com Closes: https://lore.kernel.org/all/6a0c5f2c.a00a0220.2c7954.0000.GAE@google.com/ Tested-by: syzbot+0948c82180d475ad24e2@syzkaller.appspotmail.com Reported-by: syzbot+fb7c2dd166d3ea63df2a@syzkaller.appspotmail.com Closes: https://lore.kernel.org/all/6a426dd2.854d4ab9.360e1d.0008.GAE@google.com/ Fixes: b9220d32799a ("KVM: x86/xen: allow shared_info to be mapped by fixed HVA") Cc: stable@vger.kernel.org Signed-off-by: David Woodhouse Assisted-by: Claude:claude-mythos-5 --- include/linux/kvm_host.h | 2 ++ virt/kvm/kvm_main.c | 10 ++++++++++ virt/kvm/pfncache.c | 18 +++++++++--------- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 03bfc92864b6..3dd04605f2e5 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -855,6 +855,8 @@ struct kvm { gfn_t mmu_invalidate_range_start; gfn_t mmu_invalidate_range_end; + unsigned long gpc_invalidate_seq; + struct list_head devices; u64 manual_dirty_log_protect; struct dentry *debugfs_dentry; diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 345d56a15fa4..41c88a8ade95 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -812,6 +812,16 @@ static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, /* Pairs with the increment in range_start(). */ spin_lock(&kvm->mn_invalidate_lock); + kvm->gpc_invalidate_seq++; + + /* + * As with the MMU sequence counter and mmu_invalidate_in_progress, the + * GPC sequence increase must be visible before the invalidate count + * goes to zero. Pairs with the smp_rmb() in + * mmu_notifier_retry_cache(). + */ + smp_wmb(); + if (!WARN_ON_ONCE(!kvm->mn_active_invalidate_count)) --kvm->mn_active_invalidate_count; wake = !kvm->mn_active_invalidate_count; diff --git a/virt/kvm/pfncache.c b/virt/kvm/pfncache.c index 728d2c1b488a..3659686b97c2 100644 --- a/virt/kvm/pfncache.c +++ b/virt/kvm/pfncache.c @@ -124,7 +124,7 @@ static void gpc_unmap(kvm_pfn_t pfn, void *khva) #endif } -static inline bool mmu_notifier_retry_cache(struct kvm *kvm, unsigned long mmu_seq) +static inline bool mmu_notifier_retry_cache(struct kvm *kvm, unsigned long gpc_seq) { /* * mn_active_invalidate_count acts for all intents and purposes @@ -136,20 +136,20 @@ static inline bool mmu_notifier_retry_cache(struct kvm *kvm, unsigned long mmu_s * Note, it does not matter that mn_active_invalidate_count * is not protected by gpc->lock. It is guaranteed to * be elevated before the mmu_notifier acquires gpc->lock, and - * isn't dropped until after mmu_invalidate_seq is updated. + * isn't dropped until after gpc_invalidate_seq is updated. */ if (kvm->mn_active_invalidate_count) return true; /* * Ensure mn_active_invalidate_count is read before - * mmu_invalidate_seq. This pairs with the smp_wmb() in - * mmu_notifier_invalidate_range_end() to guarantee either the + * gpc_invalidate_seq. This pairs with the smp_wmb() in + * kvm_mmu_notifier_invalidate_range_end() to guarantee either the * old (non-zero) value of mn_active_invalidate_count or the - * new (incremented) value of mmu_invalidate_seq is observed. + * new (incremented) value of gpc_invalidate_seq is observed. */ smp_rmb(); - return kvm->mmu_invalidate_seq != mmu_seq; + return kvm->gpc_invalidate_seq != gpc_seq; } static kvm_pfn_t hva_to_pfn_retry(struct gfn_to_pfn_cache *gpc) @@ -158,7 +158,7 @@ static kvm_pfn_t hva_to_pfn_retry(struct gfn_to_pfn_cache *gpc) void *old_khva = (void *)PAGE_ALIGN_DOWN((uintptr_t)gpc->khva); kvm_pfn_t new_pfn = KVM_PFN_ERR_FAULT; void *new_khva = NULL; - unsigned long mmu_seq; + unsigned long gpc_seq; struct page *page; struct kvm_follow_pfn kfp = { @@ -181,7 +181,7 @@ static kvm_pfn_t hva_to_pfn_retry(struct gfn_to_pfn_cache *gpc) gpc->valid = false; do { - mmu_seq = gpc->kvm->mmu_invalidate_seq; + gpc_seq = gpc->kvm->gpc_invalidate_seq; smp_rmb(); write_unlock_irq(&gpc->lock); @@ -232,7 +232,7 @@ static kvm_pfn_t hva_to_pfn_retry(struct gfn_to_pfn_cache *gpc) * attempting to refresh. */ WARN_ON_ONCE(gpc->valid); - } while (mmu_notifier_retry_cache(gpc->kvm, mmu_seq)); + } while (mmu_notifier_retry_cache(gpc->kvm, gpc_seq)); gpc->valid = true; gpc->pfn = new_pfn; -- 2.55.0