From: Matthew Brost The core MM splits the folio before calling folio_free, restoring the zone pages associated with the folio to an initialized state (e.g., non-compound, pgmap valid, etc...). The order argument represents the folio’s order prior to the split which can be used driver side to know how many pages are being freed. Fixes: 3a5a06554566 ("mm/zone_device: rename page_free callback to folio_free") Cc: Zi Yan Cc: Madhavan Srinivasan Cc: Nicholas Piggin Cc: Michael Ellerman Cc: "Christophe Leroy (CS GROUP)" Cc: Felix Kuehling Cc: Alex Deucher Cc: "Christian König" Cc: David Airlie Cc: Simona Vetter Cc: Maarten Lankhorst Cc: Maxime Ripard Cc: Thomas Zimmermann Cc: Lyude Paul Cc: Danilo Krummrich Cc: Bjorn Helgaas Cc: Logan Gunthorpe Cc: David Hildenbrand Cc: Oscar Salvador Cc: Andrew Morton Cc: Jason Gunthorpe Cc: Leon Romanovsky Cc: Balbir Singh Cc: Lorenzo Stoakes Cc: Liam R. Howlett Cc: Vlastimil Babka Cc: Mike Rapoport Cc: Suren Baghdasaryan Cc: Michal Hocko Cc: Alistair Popple Cc: linuxppc-dev@lists.ozlabs.org Cc: kvm@vger.kernel.org Cc: linux-kernel@vger.kernel.org Cc: amd-gfx@lists.freedesktop.org Cc: dri-devel@lists.freedesktop.org Cc: nouveau@lists.freedesktop.org Cc: linux-pci@vger.kernel.org Cc: linux-mm@kvack.org Cc: linux-cxl@vger.kernel.org Signed-off-by: Matthew Brost Signed-off-by: Francois Dugast --- arch/powerpc/kvm/book3s_hv_uvmem.c | 2 +- drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 2 +- drivers/gpu/drm/drm_pagemap.c | 3 ++- drivers/gpu/drm/nouveau/nouveau_dmem.c | 4 ++-- drivers/pci/p2pdma.c | 2 +- include/linux/memremap.h | 7 ++++++- lib/test_hmm.c | 4 +--- mm/memremap.c | 5 +++-- 8 files changed, 17 insertions(+), 12 deletions(-) diff --git a/arch/powerpc/kvm/book3s_hv_uvmem.c b/arch/powerpc/kvm/book3s_hv_uvmem.c index e5000bef90f2..b58f34eec6e5 100644 --- a/arch/powerpc/kvm/book3s_hv_uvmem.c +++ b/arch/powerpc/kvm/book3s_hv_uvmem.c @@ -1014,7 +1014,7 @@ static vm_fault_t kvmppc_uvmem_migrate_to_ram(struct vm_fault *vmf) * to a normal PFN during H_SVM_PAGE_OUT. * Gets called with kvm->arch.uvmem_lock held. */ -static void kvmppc_uvmem_folio_free(struct folio *folio) +static void kvmppc_uvmem_folio_free(struct folio *folio, unsigned int order) { struct page *page = &folio->page; unsigned long pfn = page_to_pfn(page) - diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c index af53e796ea1b..a26e3c448e47 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c @@ -567,7 +567,7 @@ svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc, return r < 0 ? r : 0; } -static void svm_migrate_folio_free(struct folio *folio) +static void svm_migrate_folio_free(struct folio *folio, unsigned int order) { struct page *page = &folio->page; struct svm_range_bo *svm_bo = page->zone_device_data; diff --git a/drivers/gpu/drm/drm_pagemap.c b/drivers/gpu/drm/drm_pagemap.c index 03ee39a761a4..df253b13cf85 100644 --- a/drivers/gpu/drm/drm_pagemap.c +++ b/drivers/gpu/drm/drm_pagemap.c @@ -1144,11 +1144,12 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas, /** * drm_pagemap_folio_free() - Put GPU SVM zone device data associated with a folio * @folio: Pointer to the folio + * @order: Order of the folio prior to being split by core MM * * This function is a callback used to put the GPU SVM zone device data * associated with a page when it is being released. */ -static void drm_pagemap_folio_free(struct folio *folio) +static void drm_pagemap_folio_free(struct folio *folio, unsigned int order) { drm_pagemap_zdd_put(folio->page.zone_device_data); } diff --git a/drivers/gpu/drm/nouveau/nouveau_dmem.c b/drivers/gpu/drm/nouveau/nouveau_dmem.c index 58071652679d..545f316fca14 100644 --- a/drivers/gpu/drm/nouveau/nouveau_dmem.c +++ b/drivers/gpu/drm/nouveau/nouveau_dmem.c @@ -115,14 +115,14 @@ unsigned long nouveau_dmem_page_addr(struct page *page) return chunk->bo->offset + off; } -static void nouveau_dmem_folio_free(struct folio *folio) +static void nouveau_dmem_folio_free(struct folio *folio, unsigned int order) { struct page *page = &folio->page; struct nouveau_dmem_chunk *chunk = nouveau_page_to_chunk(page); struct nouveau_dmem *dmem = chunk->drm->dmem; spin_lock(&dmem->lock); - if (folio_order(folio)) { + if (order) { page->zone_device_data = dmem->free_folios; dmem->free_folios = folio; } else { diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c index 4a2fc7ab42c3..a6fa7610f8a8 100644 --- a/drivers/pci/p2pdma.c +++ b/drivers/pci/p2pdma.c @@ -200,7 +200,7 @@ static const struct attribute_group p2pmem_group = { .name = "p2pmem", }; -static void p2pdma_folio_free(struct folio *folio) +static void p2pdma_folio_free(struct folio *folio, unsigned int order) { struct page *page = &folio->page; struct pci_p2pdma_pagemap *pgmap = to_p2p_pgmap(page_pgmap(page)); diff --git a/include/linux/memremap.h b/include/linux/memremap.h index 713ec0435b48..97fcffeb1c1e 100644 --- a/include/linux/memremap.h +++ b/include/linux/memremap.h @@ -79,8 +79,13 @@ struct dev_pagemap_ops { * Called once the folio refcount reaches 0. The reference count will be * reset to one by the core code after the method is called to prepare * for handing out the folio again. + * + * The core MM splits the folio before calling folio_free, restoring the + * zone pages associated with the folio to an initialized state (e.g., + * non-compound, pgmap valid, etc...). The order argument represents the + * folio’s order prior to the split. */ - void (*folio_free)(struct folio *folio); + void (*folio_free)(struct folio *folio, unsigned int order); /* * Used for private (un-addressable) device memory only. Must migrate diff --git a/lib/test_hmm.c b/lib/test_hmm.c index 8af169d3873a..e17c71d02a3a 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -1580,13 +1580,11 @@ static const struct file_operations dmirror_fops = { .owner = THIS_MODULE, }; -static void dmirror_devmem_free(struct folio *folio) +static void dmirror_devmem_free(struct folio *folio, unsigned int order) { struct page *page = &folio->page; struct page *rpage = BACKING_PAGE(page); struct dmirror_device *mdevice; - struct folio *rfolio = page_folio(rpage); - unsigned int order = folio_order(rfolio); if (rpage != page) { if (order) diff --git a/mm/memremap.c b/mm/memremap.c index 63c6ab4fdf08..39dc4bd190d0 100644 --- a/mm/memremap.c +++ b/mm/memremap.c @@ -417,6 +417,7 @@ void free_zone_device_folio(struct folio *folio) { struct dev_pagemap *pgmap = folio->pgmap; unsigned long nr = folio_nr_pages(folio); + unsigned int order = folio_order(folio); int i; if (WARN_ON_ONCE(!pgmap)) @@ -453,7 +454,7 @@ void free_zone_device_folio(struct folio *folio) case MEMORY_DEVICE_COHERENT: if (WARN_ON_ONCE(!pgmap->ops || !pgmap->ops->folio_free)) break; - pgmap->ops->folio_free(folio); + pgmap->ops->folio_free(folio, order); percpu_ref_put_many(&folio->pgmap->ref, nr); break; @@ -472,7 +473,7 @@ void free_zone_device_folio(struct folio *folio) case MEMORY_DEVICE_PCI_P2PDMA: if (WARN_ON_ONCE(!pgmap->ops || !pgmap->ops->folio_free)) break; - pgmap->ops->folio_free(folio); + pgmap->ops->folio_free(folio, order); break; } } -- 2.43.0