IOMMU pages are allocated/freed using APIs using struct ioptdesc. For the proper preservation and restoration of ioptdesc add helper functions. Signed-off-by: Samiullah Khawaja --- drivers/iommu/iommu-pages.c | 108 ++++++++++++++++++++++++++++++++++-- drivers/iommu/iommu-pages.h | 30 ++++++++++ 2 files changed, 134 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/iommu-pages.c b/drivers/iommu/iommu-pages.c index 3bab175d8557..b1ffeb930e6d 100644 --- a/drivers/iommu/iommu-pages.c +++ b/drivers/iommu/iommu-pages.c @@ -6,6 +6,7 @@ #include "iommu-pages.h" #include #include +#include #include #define IOPTDESC_MATCH(pg_elm, elm) \ @@ -28,6 +29,13 @@ static inline size_t ioptdesc_mem_size(struct ioptdesc *desc) return 1UL << (folio_order(ioptdesc_folio(desc)) + PAGE_SHIFT); } +static inline void iommu_folio_update_stats(struct folio *folio, + unsigned long nr_pages) +{ + mod_node_page_state(folio_pgdat(folio), NR_IOMMU_PAGES, nr_pages); + lruvec_stat_mod_folio(folio, NR_SECONDARY_PAGETABLE, nr_pages); +} + /** * iommu_alloc_pages_node_sz - Allocate a zeroed page of a given size from * specific NUMA node @@ -80,8 +88,7 @@ void *iommu_alloc_pages_node_sz(int nid, gfp_t gfp, size_t size) * rather large, i.e. multiple gigabytes in size. */ pgcnt = 1UL << order; - mod_node_page_state(folio_pgdat(folio), NR_IOMMU_PAGES, pgcnt); - lruvec_stat_mod_folio(folio, NR_SECONDARY_PAGETABLE, pgcnt); + iommu_folio_update_stats(folio, pgcnt); return folio_address(folio); } @@ -95,8 +102,7 @@ static void __iommu_free_desc(struct ioptdesc *iopt) if (IOMMU_PAGES_USE_DMA_API) WARN_ON_ONCE(iopt->incoherent); - mod_node_page_state(folio_pgdat(folio), NR_IOMMU_PAGES, -pgcnt); - lruvec_stat_mod_folio(folio, NR_SECONDARY_PAGETABLE, -pgcnt); + iommu_folio_update_stats(folio, -pgcnt); folio_put(folio); } @@ -131,6 +137,100 @@ void iommu_put_pages_list(struct iommu_pages_list *list) } EXPORT_SYMBOL_GPL(iommu_put_pages_list); +#if IS_ENABLED(CONFIG_IOMMU_LIVEUPDATE) +/** + * iommu_unpreserve_page - Unpreserve a page that was preserved in KHO + * @virt: Virtual address of a page + */ +void iommu_unpreserve_page(void *virt) +{ + kho_unpreserve_folio(ioptdesc_folio(virt_to_ioptdesc(virt))); +} +EXPORT_SYMBOL_GPL(iommu_unpreserve_page); + +/** + * iommu_preserve_page - Preserve a page during kexec handover + * @virt: Virtual address of the page to preserve + * + * Returns 0 on success, negative error on failure + */ +int iommu_preserve_page(void *virt) +{ + return kho_preserve_folio(ioptdesc_folio(virt_to_ioptdesc(virt))); +} +EXPORT_SYMBOL_GPL(iommu_preserve_page); + +/** + * iommu_unpreserve_pages - Unpreserve pages that were preserved in KHO + * @list: List of pages to unpreserve + */ +void iommu_unpreserve_pages(struct iommu_pages_list *list) +{ + struct ioptdesc *iopt; + + list_for_each_entry(iopt, &list->pages, iopt_freelist_elm) + kho_unpreserve_folio(ioptdesc_folio(iopt)); +} +EXPORT_SYMBOL_GPL(iommu_unpreserve_pages); + +/** + * iommu_restore_page - Restore a page that was preserved in KHO + * @phys: Physical address of a page + */ +void iommu_restore_page(u64 phys) +{ + struct ioptdesc *iopt; + struct folio *folio; + unsigned long pgcnt; + unsigned int order; + + folio = kho_restore_folio(phys); + BUG_ON(!folio); + + iopt = folio_ioptdesc(folio); + + /* + * For the restored pages incoherent is set to false as these are not + * mapped using the DMA_API. The remapping of these pages using DMA_API + * is not needed as these are not going to be written to by the new + * kernel. + */ + iopt->incoherent = false; + + order = folio_order(folio); + pgcnt = 1UL << order; + iommu_folio_update_stats(folio, pgcnt); +} +EXPORT_SYMBOL_GPL(iommu_restore_page); + +/** + * iommu_preserve_pages - Preserve pages during kexec handover + * @list: List of pages to preserve + * + * Returns 0 on success, negative error on failure + */ +int iommu_preserve_pages(struct iommu_pages_list *list) +{ + struct ioptdesc *iopt; + int ret; + + list_for_each_entry(iopt, &list->pages, iopt_freelist_elm) { + ret = kho_preserve_folio(ioptdesc_folio(iopt)); + if (ret) + goto err; + } + + return 0; + +err: + list_for_each_entry_continue_reverse(iopt, &list->pages, iopt_freelist_elm) + kho_unpreserve_folio(ioptdesc_folio(iopt)); + + return ret; +} +EXPORT_SYMBOL_GPL(iommu_preserve_pages); +#endif + /** * iommu_pages_start_incoherent - Setup the page for cache incoherent operation * @virt: The page to setup diff --git a/drivers/iommu/iommu-pages.h b/drivers/iommu/iommu-pages.h index ae9da4f571f6..7b9b6bb504b2 100644 --- a/drivers/iommu/iommu-pages.h +++ b/drivers/iommu/iommu-pages.h @@ -53,6 +53,36 @@ void *iommu_alloc_pages_node_sz(int nid, gfp_t gfp, size_t size); void iommu_free_pages(void *virt); void iommu_put_pages_list(struct iommu_pages_list *list); +#if IS_ENABLED(CONFIG_IOMMU_LIVEUPDATE) +int iommu_preserve_page(void *virt); +void iommu_unpreserve_page(void *virt); +int iommu_preserve_pages(struct iommu_pages_list *list); +void iommu_unpreserve_pages(struct iommu_pages_list *list); +void iommu_restore_page(u64 phys); +#else +static inline int iommu_preserve_page(void *virt) +{ + return -EOPNOTSUPP; +} + +static inline void iommu_unpreserve_page(void *virt) +{ +} + +static inline int iommu_preserve_pages(struct iommu_pages_list *list) +{ + return -EOPNOTSUPP; +} + +static inline void iommu_unpreserve_pages(struct iommu_pages_list *list, int count) +{ +} + +static inline void iommu_restore_page(u64 phys) +{ +} +#endif + /** * iommu_pages_list_add - add the page to a iommu_pages_list * @list: List to add the page to -- 2.54.0.545.g6539524ca2-goog