Call prepare_attributes() before invalidating guest mappings or changing file attributes. Remove host mappings first when converting to private, so folio validation can check for outstanding references. Signed-off-by: Aneesh Kumar K.V (Arm) --- include/linux/guest_memfd.h | 9 ++++++++ virt/kvm/guest_memfd.c | 43 ++++++++++++++++++++++++------------- 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/include/linux/guest_memfd.h b/include/linux/guest_memfd.h index d3b153a3d891..06f094a2fd49 100644 --- a/include/linux/guest_memfd.h +++ b/include/linux/guest_memfd.h @@ -42,6 +42,13 @@ struct guest_memfd_handle { * @unlock: Release the provider lock taken by lock(). Called after guest * invalidation ends and any successful attribute publication, with * the inode invalidate lock still held. + * @prepare_attributes: Optional memory validation before conversion, under + * the inode invalidate lock for write, before taking + * the provider lock or starting guest invalidation. Host mappings have + * been removed for private conversion. Set @err_index on + * failure; it initially points to the range start. NULL + * selects folio validation for folio providers; for + * other providers, NULL skips this validation. * @alloc_folio: Allocate an uninserted folio for a given index and NUMA policy. * Must return a pointer to a struct folio on success, or an * ERR_PTR() error code on failure. Must never return NULL. @@ -77,6 +84,8 @@ struct guest_memfd_provider_operations { void (*release)(void *provider); void (*lock)(void *provider); void (*unlock)(void *provider); + int (*prepare_attributes)(struct inode *inode, void *provider, pgoff_t start, + size_t nr_pages, u64 memory_attrs, pgoff_t *err_index); struct folio *(*alloc_folio)(void *provider, pgoff_t index, struct mempolicy *mpol); void (*invalidate_folio)(void *provider, struct folio *folio); diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index ad9cc1bbe5d7..c69a8b7d046c 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -54,6 +54,8 @@ static int gmem_folio_get_pfn(struct inode *inode, void *provider, pgoff_t index struct guest_memfd_handle *handle); static int gmem_folio_lookup_pfn_range(struct inode *inode, void *provider, pgoff_t index, unsigned long *pfn, size_t *nr_pages, u64 *pfn_attrs); +static int gmem_folio_prepare_attributes(struct inode *inode, void *provider, + pgoff_t start, size_t nr_pages, u64 attrs, pgoff_t *err_index); static __always_inline struct gmem_inode *GMEM_I(struct inode *inode) { @@ -91,6 +93,7 @@ static const struct guest_memfd_provider_operations gmem_native_provider_ops = { .alloc_folio = gmem_native_alloc_folio, .get_pfn = gmem_folio_get_pfn, .lookup_pfn_range = gmem_folio_lookup_pfn_range, + .prepare_attributes = gmem_folio_prepare_attributes, }; static bool gmem_provider_ops_valid(struct gmem_inode *gi, @@ -136,6 +139,8 @@ static void gmem_provider_init_ops(struct gmem_inode *gi, gi->provider_operations.get_pfn = gmem_folio_get_pfn; gi->provider_operations.lookup_pfn_range = gmem_folio_lookup_pfn_range; } + if (!ops->prepare_attributes) + gi->provider_operations.prepare_attributes = gmem_folio_prepare_attributes; } gi->provider_ops = &gi->provider_operations; } @@ -1095,6 +1100,16 @@ static bool kvm_gmem_has_outstanding_references(struct inode *inode, return false; } +static int gmem_folio_prepare_attributes(struct inode *inode, void *provider, + pgoff_t start, size_t nr_pages, u64 attrs, pgoff_t *err_index) +{ + if ((attrs & KVM_MEMORY_ATTRIBUTE_PRIVATE) && + kvm_gmem_has_outstanding_references(inode, start, nr_pages, err_index)) + return -EAGAIN; + + return 0; +} + #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT static void kvm_gmem_make_shared(struct inode *inode, pgoff_t start, pgoff_t end) { @@ -1169,6 +1184,7 @@ static int __kvm_gmem_set_attributes(struct inode *inode, pgoff_t start, mt = &gi->attributes; + *err_index = start; filemap_invalidate_lock(mapping); if (kvm_gmem_range_has_attributes(inode, start, nr_pages, attrs)) @@ -1176,26 +1192,23 @@ static int __kvm_gmem_set_attributes(struct inode *inode, pgoff_t start, mas_init(&mas, mt, start); r = kvm_gmem_mas_preallocate(&mas, attrs, start, nr_pages); - if (r) { - *err_index = start; + if (r) goto out; - } - if (to_private) { - /* - * Forcefully unmap the pages from all userspace page tables, - * and then verify there are no outstanding references, e.g. - * acquired via GUP or similar. Tell userspace to try again if - * there are outstanding references and hope that whatever has - * pinned the page will put its reference "soon". - */ + /* + * Before private conversion, forcefully unmap the pages from all userspace + * page tables. The folio provider then checks for outstanding references, + * e.g. from GUP, and returns -EAGAIN so userspace can retry after those + * references are released. + */ + if (to_private) unmap_mapping_pages(mapping, start, nr_pages, false); - if (!kvm_gmem_is_pfn_mode(gi) && - kvm_gmem_has_outstanding_references(inode, start, nr_pages, - err_index)) { + if (gi->provider_ops->prepare_attributes) { + r = gi->provider_ops->prepare_attributes(inode, gi->provider, start, + nr_pages, attrs, err_index); + if (r) { mas_destroy(&mas); - r = -EAGAIN; goto out; } } -- 2.43.0