From: Mika Penttilä Do the preparations in hmm_range_fault() and pagewalk callbacks to do the "collecting" part of migration, needed for migration. These steps include locking for pmd/pte if migrating, and calling the still dummy hmm_vma_handle_migrate_prepare_pmd() and hmm_vma_handle_migrate_prepare() functions in the pagewalk. When doing migration, have to have pmd/pte table locked for the duration of both hmm_vma_handle*() and hmm_vma_handle_migrate_prepare*() walks, and unlock when handling the faults. Cc: David Hildenbrand Cc: Jason Gunthorpe Cc: Leon Romanovsky Cc: Alistair Popple Cc: Balbir Singh Cc: Zi Yan Cc: Matthew Brost Suggested-by: Alistair Popple Signed-off-by: Mika Penttilä --- mm/hmm.c | 360 +++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 309 insertions(+), 51 deletions(-) diff --git a/mm/hmm.c b/mm/hmm.c index 637c00188b3e..75e19be6ef76 100644 --- a/mm/hmm.c +++ b/mm/hmm.c @@ -39,8 +39,33 @@ struct hmm_vma_walk { unsigned long start; unsigned long end; unsigned long last; + /* + * For migration we need pte/pmd + * locked for the handle_* and + * prepare_* regions. While faulting + * we have to drop the locks and + * start again. + * ptelocked and pmdlocked + * hold the state and tells if need + * to drop locks before faulting. + * ptl is the lock held for pte or pmd. + * + */ + bool ptelocked; + bool pmdlocked; + spinlock_t *ptl; }; +#define HMM_ASSERT_PTE_LOCKED(hmm_vma_walk, locked) \ + WARN_ON_ONCE((hmm_vma_walk)->ptelocked != locked) + +#define HMM_ASSERT_PMD_LOCKED(hmm_vma_walk, locked) \ + WARN_ON_ONCE((hmm_vma_walk)->pmdlocked != locked) + +#define HMM_ASSERT_UNLOCKED(hmm_vma_walk) \ + WARN_ON_ONCE((hmm_vma_walk)->ptelocked || \ + (hmm_vma_walk)->pmdlocked) + enum { HMM_NEED_FAULT = 1 << 0, HMM_NEED_WRITE_FAULT = 1 << 1, @@ -54,14 +79,38 @@ enum { }; static int hmm_pfns_fill(unsigned long addr, unsigned long end, - struct hmm_range *range, unsigned long cpu_flags) + struct hmm_vma_walk *hmm_vma_walk, unsigned long cpu_flags) { + struct hmm_range *range = hmm_vma_walk->range; unsigned long i = (addr - range->start) >> PAGE_SHIFT; + enum migrate_vma_info minfo; + bool migrate = false; + + minfo = hmm_select_migrate(range); + if (cpu_flags != HMM_PFN_ERROR) { + if (minfo && (vma_is_anonymous(hmm_vma_walk->vma))) { + cpu_flags |= HMM_PFN_MIGRATE; + migrate = true; + } + } + + if (migrate && thp_migration_supported() && + (minfo & MIGRATE_VMA_SELECT_COMPOUND) && + IS_ALIGNED(addr, HPAGE_PMD_SIZE) && + IS_ALIGNED(end, HPAGE_PMD_SIZE) && + end-addr == HPAGE_PMD_SIZE) { + range->hmm_pfns[i] &= HMM_PFN_INOUT_FLAGS; + range->hmm_pfns[i] |= cpu_flags | HMM_PFN_COMPOUND; + addr += PAGE_SIZE; + i++; + cpu_flags = 0; + } for (; addr < end; addr += PAGE_SIZE, i++) { range->hmm_pfns[i] &= HMM_PFN_INOUT_FLAGS; range->hmm_pfns[i] |= cpu_flags; } + return 0; } @@ -84,6 +133,7 @@ static int hmm_vma_fault(unsigned long addr, unsigned long end, unsigned int fault_flags = FAULT_FLAG_REMOTE; WARN_ON_ONCE(!required_fault); + HMM_ASSERT_UNLOCKED(hmm_vma_walk); hmm_vma_walk->last = addr; if (required_fault & HMM_NEED_WRITE_FAULT) { @@ -177,11 +227,16 @@ static int hmm_vma_walk_hole(unsigned long addr, unsigned long end, if (!walk->vma) { if (required_fault) return -EFAULT; - return hmm_pfns_fill(addr, end, range, HMM_PFN_ERROR); + return hmm_pfns_fill(addr, end, hmm_vma_walk, HMM_PFN_ERROR); } - if (required_fault) + if (required_fault) { + if (hmm_vma_walk->pmdlocked) { + spin_unlock(hmm_vma_walk->ptl); + hmm_vma_walk->pmdlocked = false; + } return hmm_vma_fault(addr, end, required_fault, walk); - return hmm_pfns_fill(addr, end, range, 0); + } + return hmm_pfns_fill(addr, end, hmm_vma_walk, 0); } static inline unsigned long hmm_pfn_flags_order(unsigned long order) @@ -214,8 +269,13 @@ static int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr, cpu_flags = pmd_to_hmm_pfn_flags(range, pmd); required_fault = hmm_range_need_fault(hmm_vma_walk, hmm_pfns, npages, cpu_flags); - if (required_fault) + if (required_fault) { + if (hmm_vma_walk->pmdlocked) { + spin_unlock(hmm_vma_walk->ptl); + hmm_vma_walk->pmdlocked = false; + } return hmm_vma_fault(addr, end, required_fault, walk); + } pfn = pmd_pfn(pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++) { @@ -295,14 +355,25 @@ static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr, goto fault; if (softleaf_is_migration(entry)) { - pte_unmap(ptep); - hmm_vma_walk->last = addr; - migration_entry_wait(walk->mm, pmdp, addr); - return -EBUSY; + if (!hmm_select_migrate(range)) { + HMM_ASSERT_UNLOCKED(hmm_vma_walk); + pte_unmap(ptep); + hmm_vma_walk->last = addr; + migration_entry_wait(walk->mm, pmdp, addr); + return -EBUSY; + } + return 0; } /* Report error for everything else */ - pte_unmap(ptep); + + if (hmm_vma_walk->ptelocked) { + pte_unmap_unlock(ptep, hmm_vma_walk->ptl); + hmm_vma_walk->ptelocked = false; + } else { + pte_unmap(ptep); + } + return -EFAULT; } @@ -319,7 +390,13 @@ static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr, if (!vm_normal_page(walk->vma, addr, pte) && !is_zero_pfn(pte_pfn(pte))) { if (hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, 0)) { - pte_unmap(ptep); + if (hmm_vma_walk->ptelocked) { + pte_unmap_unlock(ptep, hmm_vma_walk->ptl); + hmm_vma_walk->ptelocked = false; + } else { + pte_unmap(ptep); + } + return -EFAULT; } new_pfn_flags = HMM_PFN_ERROR; @@ -332,7 +409,12 @@ static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr, return 0; fault: - pte_unmap(ptep); + if (hmm_vma_walk->ptelocked) { + pte_unmap_unlock(ptep, hmm_vma_walk->ptl); + hmm_vma_walk->ptelocked = false; + } else { + pte_unmap(ptep); + } /* Fault any virtual address we were asked to fault */ return hmm_vma_fault(addr, end, required_fault, walk); } @@ -376,13 +458,18 @@ static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start, required_fault = hmm_range_need_fault(hmm_vma_walk, hmm_pfns, npages, 0); if (required_fault) { - if (softleaf_is_device_private(entry)) + if (softleaf_is_device_private(entry)) { + if (hmm_vma_walk->pmdlocked) { + spin_unlock(hmm_vma_walk->ptl); + hmm_vma_walk->pmdlocked = false; + } return hmm_vma_fault(addr, end, required_fault, walk); + } else return -EFAULT; } - return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR); + return hmm_pfns_fill(start, end, hmm_vma_walk, HMM_PFN_ERROR); } #else static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start, @@ -390,15 +477,72 @@ static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start, pmd_t pmd) { struct hmm_vma_walk *hmm_vma_walk = walk->private; - struct hmm_range *range = hmm_vma_walk->range; unsigned long npages = (end - start) >> PAGE_SHIFT; if (hmm_range_need_fault(hmm_vma_walk, hmm_pfns, npages, 0)) return -EFAULT; - return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR); + return hmm_pfns_fill(start, end, hmm_vma_walk, HMM_PFN_ERROR); } #endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#ifdef CONFIG_DEVICE_MIGRATION +static int hmm_vma_handle_migrate_prepare_pmd(const struct mm_walk *walk, + pmd_t *pmdp, + unsigned long start, + unsigned long end, + unsigned long *hmm_pfn) +{ + // TODO: implement migration entry insertion + return 0; +} + +static int hmm_vma_handle_migrate_prepare(const struct mm_walk *walk, + pmd_t *pmdp, + pte_t *ptep, + unsigned long addr, + unsigned long *hmm_pfn, + bool *unmapped) +{ + // TODO: implement migration entry insertion + return 0; +} + +static int hmm_vma_walk_split(pmd_t *pmdp, + unsigned long addr, + struct mm_walk *walk) +{ + // TODO : implement split + return 0; +} + +#else +static int hmm_vma_handle_migrate_prepare_pmd(const struct mm_walk *walk, + pmd_t *pmdp, + unsigned long start, + unsigned long end, + unsigned long *hmm_pfn) +{ + return 0; +} + +static int hmm_vma_handle_migrate_prepare(const struct mm_walk *walk, + pmd_t *pmdp, + pte_t *ptep, + unsigned long addr, + unsigned long *hmm_pfn, + bool *unmapped) +{ + return 0; +} + +static int hmm_vma_walk_split(pmd_t *pmdp, + unsigned long addr, + struct mm_walk *walk) +{ + return 0; +} +#endif + static int hmm_vma_capture_migrate_range(unsigned long start, unsigned long end, struct mm_walk *walk) @@ -457,46 +601,128 @@ static int hmm_vma_walk_pmd(pmd_t *pmdp, { struct hmm_vma_walk *hmm_vma_walk = walk->private; struct hmm_range *range = hmm_vma_walk->range; - unsigned long *hmm_pfns = - &range->hmm_pfns[(start - range->start) >> PAGE_SHIFT]; unsigned long npages = (end - start) >> PAGE_SHIFT; + struct mm_struct *mm = walk->vma->vm_mm; + unsigned long *hmm_pfns, *hmm_pfns_start; + enum migrate_vma_info minfo; unsigned long addr = start; + bool unmapped = false; + unsigned long i; pte_t *ptep; pmd_t pmd; + int r = 0; + minfo = hmm_select_migrate(range); + hmm_pfns_start = &range->hmm_pfns[(start - range->start) >> PAGE_SHIFT]; again: - pmd = pmdp_get_lockless(pmdp); - if (pmd_none(pmd)) - return hmm_vma_walk_hole(start, end, -1, walk); + hmm_pfns = &range->hmm_pfns[(addr - range->start) >> PAGE_SHIFT]; + hmm_vma_walk->ptelocked = false; + hmm_vma_walk->pmdlocked = false; + + if (minfo) { + hmm_vma_walk->ptl = pmd_lock(mm, pmdp); + hmm_vma_walk->pmdlocked = true; + pmd = pmdp_get(pmdp); + } else + pmd = pmdp_get_lockless(pmdp); + + if (pmd_none(pmd)) { + r = hmm_vma_walk_hole(start, end, -1, walk); + + if (hmm_vma_walk->pmdlocked) { + spin_unlock(hmm_vma_walk->ptl); + hmm_vma_walk->pmdlocked = false; + } + return r; + } if (thp_migration_supported() && pmd_is_migration_entry(pmd)) { - if (hmm_range_need_fault(hmm_vma_walk, hmm_pfns, npages, 0)) { - hmm_vma_walk->last = addr; - pmd_migration_entry_wait(walk->mm, pmdp); - return -EBUSY; + if (!minfo) { + if (hmm_range_need_fault(hmm_vma_walk, hmm_pfns_start, npages, 0)) { + hmm_vma_walk->last = addr; + pmd_migration_entry_wait(walk->mm, pmdp); + return -EBUSY; + } + for (i = 0; start < end; start += PAGE_SIZE, i++) + hmm_pfns_start[i] &= HMM_PFN_INOUT_FLAGS; } - return hmm_pfns_fill(start, end, range, 0); + + if (hmm_vma_walk->pmdlocked) { + spin_unlock(hmm_vma_walk->ptl); + hmm_vma_walk->pmdlocked = false; + } + + return 0; } - if (!pmd_present(pmd)) - return hmm_vma_handle_absent_pmd(walk, start, end, hmm_pfns, - pmd); + if (pmd_trans_huge(pmd) || !pmd_present(pmd)) { + if (!pmd_present(pmd)) { + r = hmm_vma_handle_absent_pmd(walk, start, end, hmm_pfns_start, + pmd); + // If not migrating we are done + if (r || !minfo) { + if (hmm_vma_walk->pmdlocked) { + spin_unlock(hmm_vma_walk->ptl); + hmm_vma_walk->pmdlocked = false; + } + return r; + } + } - if (pmd_trans_huge(pmd)) { - /* - * No need to take pmd_lock here, even if some other thread - * is splitting the huge pmd we will get that event through - * mmu_notifier callback. - * - * So just read pmd value and check again it's a transparent - * huge or device mapping one and compute corresponding pfn - * values. - */ - pmd = pmdp_get_lockless(pmdp); - if (!pmd_trans_huge(pmd)) - goto again; + if (pmd_trans_huge(pmd)) { + /* + * No need to take pmd_lock here if not migrating, + * even if some other thread is splitting the huge + * pmd we will get that event through mmu_notifier callback. + * + * So just read pmd value and check again it's a transparent + * huge or device mapping one and compute corresponding pfn + * values. + */ + + if (!minfo) { + pmd = pmdp_get_lockless(pmdp); + if (!pmd_trans_huge(pmd)) + goto again; + } + + r = hmm_vma_handle_pmd(walk, start, end, hmm_pfns_start, pmd); + + // If not migrating we are done + if (r || !minfo) { + if (hmm_vma_walk->pmdlocked) { + spin_unlock(hmm_vma_walk->ptl); + hmm_vma_walk->pmdlocked = false; + } + return r; + } + } + + r = hmm_vma_handle_migrate_prepare_pmd(walk, pmdp, start, end, hmm_pfns_start); + + if (hmm_vma_walk->pmdlocked) { + spin_unlock(hmm_vma_walk->ptl); + hmm_vma_walk->pmdlocked = false; + } + + if (r == -ENOENT) { + r = hmm_vma_walk_split(pmdp, addr, walk); + if (r) { + /* Split not successful, skip */ + return hmm_pfns_fill(start, end, hmm_vma_walk, HMM_PFN_ERROR); + } + + /* Split successful, reloop */ + hmm_vma_walk->last = addr; + return -EBUSY; + } + return r; - return hmm_vma_handle_pmd(walk, addr, end, hmm_pfns, pmd); + } + + if (hmm_vma_walk->pmdlocked) { + spin_unlock(hmm_vma_walk->ptl); + hmm_vma_walk->pmdlocked = false; } /* @@ -506,24 +732,56 @@ static int hmm_vma_walk_pmd(pmd_t *pmdp, * recover. */ if (pmd_bad(pmd)) { - if (hmm_range_need_fault(hmm_vma_walk, hmm_pfns, npages, 0)) + if (hmm_range_need_fault(hmm_vma_walk, hmm_pfns_start, npages, 0)) return -EFAULT; - return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR); + return hmm_pfns_fill(start, end, hmm_vma_walk, HMM_PFN_ERROR); } - ptep = pte_offset_map(pmdp, addr); - if (!ptep) + if (minfo) { + ptep = pte_offset_map_lock(mm, pmdp, addr, &hmm_vma_walk->ptl); + if (ptep) + hmm_vma_walk->ptelocked = true; + } else { + ptep = pte_offset_map(pmdp, addr); + } + if (!ptep) { + addr = start; goto again; + } + for (; addr < end; addr += PAGE_SIZE, ptep++, hmm_pfns++) { - int r; r = hmm_vma_handle_pte(walk, addr, end, pmdp, ptep, hmm_pfns); if (r) { - /* hmm_vma_handle_pte() did pte_unmap() */ + /* hmm_vma_handle_pte() did pte_unmap() / pte_unmap_unlock */ return r; } + + r = hmm_vma_handle_migrate_prepare(walk, pmdp, ptep, addr, hmm_pfns, &unmapped); + if (r == -EAGAIN) { + HMM_ASSERT_UNLOCKED(hmm_vma_walk); + if (unmapped) { + flush_tlb_range(walk->vma, start, addr); + unmapped = false; + } + goto again; + } + if (r) { + hmm_pfns_fill(addr, end, hmm_vma_walk, HMM_PFN_ERROR); + break; + } + } + + if (unmapped) + flush_tlb_range(walk->vma, start, addr); + + if (hmm_vma_walk->ptelocked) { + pte_unmap_unlock(ptep - 1, hmm_vma_walk->ptl); + hmm_vma_walk->ptelocked = false; + } else { + pte_unmap(ptep - 1); } - pte_unmap(ptep - 1); + return 0; } @@ -684,7 +942,7 @@ static int hmm_vma_walk_test(unsigned long start, unsigned long end, (end - start) >> PAGE_SHIFT, 0)) return -EFAULT; - hmm_pfns_fill(start, end, range, HMM_PFN_ERROR); + hmm_pfns_fill(start, end, hmm_vma_walk, HMM_PFN_ERROR); /* Skip this vma and continue processing the next vma. */ return 1; -- 2.55.0