From: "Kiryl Shutsemau (Meta)" MADV_COLLAPSE is a madvise operation, but its implementation sat in khugepaged.c. The daemon's file therefore also held a syscall's worth of code that has nothing to do with the daemon: the walk over the user's range, the per-PMD loop, and the errno translation. Move it to madvise.c, among the operations it belongs with, along with the errno map and the policy it states for itself. It takes a struct madvise_behavior like every one of those operations, which is where the range, the VMA and the lock-dropped flag it used to be handed separately already live. It stays a caller of the interface khugepaged uses, so nothing about the collapse changes. The !CONFIG_TRANSPARENT_HUGEPAGE stub moves in with it. Assisted-by: LLM Signed-off-by: Kiryl Shutsemau (Meta) --- include/linux/huge_mm.h | 9 --- mm/khugepaged.c | 157 +------------------------------------ mm/madvise.c | 169 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 169 insertions(+), 166 deletions(-) diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index c745f7ad2298..8ca0fa3be2ac 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -510,8 +510,6 @@ change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, int hugepage_madvise(struct vm_area_struct *vma, vm_flags_t *vm_flags, int advice); -int madvise_collapse(struct vm_area_struct *vma, unsigned long start, - unsigned long end, bool *lock_dropped); void vma_adjust_trans_huge(struct vm_area_struct *vma, unsigned long start, unsigned long end, struct vm_area_struct *next); spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma); @@ -715,13 +713,6 @@ static inline int hugepage_madvise(struct vm_area_struct *vma, return -EINVAL; } -static inline int madvise_collapse(struct vm_area_struct *vma, - unsigned long start, - unsigned long end, bool *lock_dropped) -{ - return -EINVAL; -} - static inline void vma_adjust_trans_huge(struct vm_area_struct *vma, unsigned long start, unsigned long end, diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 77f34aedf0f2..1deb74cf28af 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -969,21 +969,6 @@ static void collapse_policy_khugepaged(struct collapse_policy *p) p->tva_type = TVA_KHUGEPAGED; } -/* MADV_COLLAPSE was asked for explicitly, so it is not held to those */ -static void collapse_policy_forced(struct collapse_policy *p) -{ - p->max_ptes_none = HPAGE_PMD_NR; - p->max_ptes_swap = HPAGE_PMD_NR; - p->max_ptes_shared = HPAGE_PMD_NR; - p->strict_sub_pmd = false; - p->skip_lazyfree = false; - p->require_referenced = false; - p->install_pmd = true; - p->writeback_dirty = true; - p->gfp = GFP_TRANSHUGE; - p->tva_type = TVA_FORCED_COLLAPSE; -} - #ifdef CONFIG_NUMA static int collapse_find_target_node(struct collapse_control *cc) { @@ -2802,8 +2787,8 @@ enum scan_result collapse_scan_pmd(struct vm_area_struct *vma, return result; } -enum scan_result collapse_run_pmd(struct mm_struct *mm, - unsigned long addr, struct collapse_control *cc) +enum scan_result collapse_run_pmd(struct mm_struct *mm, unsigned long addr, + struct collapse_control *cc) { struct file *file = cc->scan_file; bool triggered_wb = false; @@ -3179,141 +3164,3 @@ bool current_is_khugepaged(void) { return kthread_func(current) == khugepaged; } - -static int madvise_collapse_errno(enum scan_result r) -{ - /* - * MADV_COLLAPSE breaks from existing madvise(2) conventions to provide - * actionable feedback to caller, so they may take an appropriate - * fallback measure depending on the nature of the failure. - */ - switch (r) { - case SCAN_ALLOC_HUGE_PAGE_FAIL: - return -ENOMEM; - case SCAN_CGROUP_CHARGE_FAIL: - case SCAN_EXCEED_NONE_PTE: - return -EBUSY; - /* Resource temporary unavailable - trying again might succeed */ - case SCAN_PAGE_COUNT: - case SCAN_PAGE_LOCK: - case SCAN_PAGE_LRU: - case SCAN_DEL_PAGE_LRU: - case SCAN_PAGE_FILLED: - case SCAN_PAGE_HAS_PRIVATE: - case SCAN_PAGE_DIRTY_OR_WRITEBACK: - return -EAGAIN; - /* - * Other: Trying again likely not to succeed / error intrinsic to - * specified memory range. khugepaged likely won't be able to collapse - * either. - */ - default: - return -EINVAL; - } -} - -int madvise_collapse(struct vm_area_struct *vma, unsigned long start, - unsigned long end, bool *lock_dropped) -{ - struct collapse_control *cc; - struct mm_struct *mm = vma->vm_mm; - unsigned long hstart, hend, addr, orders; - enum scan_result last_fail = SCAN_FAIL; - int thps = 0; - - BUG_ON(vma->vm_start > start); - BUG_ON(vma->vm_end < end); - - orders = collapse_possible_orders(vma, vma->vm_flags, - TVA_FORCED_COLLAPSE); - if (!orders) - return -EINVAL; - - hstart = ALIGN(start, HPAGE_PMD_SIZE); - hend = ALIGN_DOWN(end, HPAGE_PMD_SIZE); - - if (hstart >= hend) - return 0; - - cc = kmalloc_obj(*cc); - if (!cc) - return -ENOMEM; - collapse_control_init(cc); - collapse_policy_forced(&cc->policy); - - lru_add_drain_all(); - - for (addr = hstart; addr < hend; addr += HPAGE_PMD_SIZE) { - struct vm_area_struct *found; - enum scan_result result; - - /* - * A collapse gives the lock up, so the VMA has to be found - * again after one: it can shrink while nothing is held. A scan - * that finds nothing to collapse leaves the lock alone, so a - * range that is already collapsed walks on without relocking. - */ - if (!vma) { - cond_resched(); - mmap_read_lock(mm); - result = collapse_vma_revalidate(mm, addr, false, &found, - cc, HPAGE_PMD_ORDER); - if (result != SCAN_SUCCEED) { - last_fail = result; - goto out_locked; - } - vma = found; - hend = min(hend, vma->vm_end & HPAGE_PMD_MASK); - orders = collapse_possible_orders(vma, vma->vm_flags, - cc->policy.tva_type); - } - - result = collapse_scan_pmd(vma, addr, cc, orders); - /* Nothing to collapse here, and the lock is still ours */ - if (result != SCAN_SUCCEED) - goto tally; - - /* The collapse takes its own locks, so give this up */ - mmap_read_unlock(mm); - *lock_dropped = true; - vma = NULL; - - result = collapse_run_pmd(mm, addr, cc); -tally: - switch (result) { - case SCAN_SUCCEED: - case SCAN_PMD_MAPPED: - ++thps; - break; - /* Whitelisted set of results where continuing OK */ - case SCAN_NO_PTE_TABLE: - case SCAN_PTE_NON_PRESENT: - case SCAN_PTE_UFFD: - case SCAN_LACK_REFERENCED_PAGE: - case SCAN_PAGE_NULL: - case SCAN_PAGE_COUNT: - case SCAN_PAGE_LOCK: - case SCAN_PAGE_COMPOUND: - case SCAN_PAGE_LRU: - case SCAN_DEL_PAGE_LRU: - last_fail = result; - break; - default: - last_fail = result; - /* Other error, exit */ - goto out; - } - } - -out: - /* Caller expects us to hold mmap_lock on return */ - if (!vma) - mmap_read_lock(mm); -out_locked: - mmap_assert_locked(mm); - collapse_control_release(cc); - kfree(cc); - - return thps == ((hend - hstart) >> HPAGE_PMD_SHIFT) ? 0 - : madvise_collapse_errno(last_fail); -} diff --git a/mm/madvise.c b/mm/madvise.c index 963337f93a7a..f75a9d139980 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -38,6 +38,7 @@ #include "internal.h" #include "swap.h" +#include "collapse.h" #define __MADV_SET_ANON_VMA_NAME (-1) @@ -906,6 +907,171 @@ bool madvise_dontneed_free_valid_vma(struct madvise_behavior *madv_behavior) return true; } +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + +/* MADV_COLLAPSE was asked for explicitly, so it is not held to those */ +static void collapse_policy_forced(struct collapse_policy *p) +{ + p->max_ptes_none = HPAGE_PMD_NR; + p->max_ptes_swap = HPAGE_PMD_NR; + p->max_ptes_shared = HPAGE_PMD_NR; + p->strict_sub_pmd = false; + p->skip_lazyfree = false; + p->require_referenced = false; + p->install_pmd = true; + p->writeback_dirty = true; + p->gfp = GFP_TRANSHUGE; + p->tva_type = TVA_FORCED_COLLAPSE; +} + +static int madvise_collapse_errno(enum scan_result r) +{ + /* + * MADV_COLLAPSE breaks from existing madvise(2) conventions to provide + * actionable feedback to caller, so they may take an appropriate + * fallback measure depending on the nature of the failure. + */ + switch (r) { + case SCAN_ALLOC_HUGE_PAGE_FAIL: + return -ENOMEM; + case SCAN_CGROUP_CHARGE_FAIL: + case SCAN_EXCEED_NONE_PTE: + return -EBUSY; + /* Resource temporary unavailable - trying again might succeed */ + case SCAN_PAGE_COUNT: + case SCAN_PAGE_LOCK: + case SCAN_PAGE_LRU: + case SCAN_DEL_PAGE_LRU: + case SCAN_PAGE_FILLED: + case SCAN_PAGE_HAS_PRIVATE: + case SCAN_PAGE_DIRTY_OR_WRITEBACK: + return -EAGAIN; + /* + * Other: Trying again likely not to succeed / error intrinsic to + * specified memory range. khugepaged likely won't be able to collapse + * either. + */ + default: + return -EINVAL; + } +} + +static int madvise_collapse(struct madvise_behavior *madv_behavior) +{ + struct madvise_behavior_range *range = &madv_behavior->range; + struct vm_area_struct *vma = madv_behavior->vma; + struct mm_struct *mm = madv_behavior->mm; + struct collapse_control *cc; + unsigned long hstart, hend, addr, orders; + enum scan_result last_fail = SCAN_FAIL; + int thps = 0; + + BUG_ON(vma->vm_start > range->start); + BUG_ON(vma->vm_end < range->end); + + orders = collapse_possible_orders(vma, vma->vm_flags, + TVA_FORCED_COLLAPSE); + if (!orders) + return -EINVAL; + + hstart = ALIGN(range->start, HPAGE_PMD_SIZE); + hend = ALIGN_DOWN(range->end, HPAGE_PMD_SIZE); + + if (hstart >= hend) + return 0; + + cc = kmalloc_obj(*cc); + if (!cc) + return -ENOMEM; + collapse_control_init(cc); + collapse_policy_forced(&cc->policy); + + lru_add_drain_all(); + + for (addr = hstart; addr < hend; addr += HPAGE_PMD_SIZE) { + struct vm_area_struct *found; + enum scan_result result; + + /* + * A collapse gives the lock up, so the VMA has to be found + * again after one: it can shrink while nothing is held. A scan + * that finds nothing to collapse leaves the lock alone, so a + * range that is already collapsed walks on without relocking. + */ + if (!vma) { + cond_resched(); + mmap_read_lock(mm); + result = collapse_vma_revalidate(mm, addr, false, &found, + cc, HPAGE_PMD_ORDER); + if (result != SCAN_SUCCEED) { + last_fail = result; + goto out_locked; + } + vma = found; + hend = min(hend, vma->vm_end & HPAGE_PMD_MASK); + orders = collapse_possible_orders(vma, vma->vm_flags, + cc->policy.tva_type); + } + + result = collapse_scan_pmd(vma, addr, cc, orders); + /* Nothing to collapse here, and the lock is still ours */ + if (result != SCAN_SUCCEED) + goto tally; + + /* The collapse takes its own locks, so give this up */ + mmap_read_unlock(mm); + mark_mmap_lock_dropped(madv_behavior); + vma = NULL; + + result = collapse_run_pmd(mm, addr, cc); +tally: + switch (result) { + case SCAN_SUCCEED: + case SCAN_PMD_MAPPED: + ++thps; + break; + /* Whitelisted set of results where continuing OK */ + case SCAN_NO_PTE_TABLE: + case SCAN_PTE_NON_PRESENT: + case SCAN_PTE_UFFD: + case SCAN_LACK_REFERENCED_PAGE: + case SCAN_PAGE_NULL: + case SCAN_PAGE_COUNT: + case SCAN_PAGE_LOCK: + case SCAN_PAGE_COMPOUND: + case SCAN_PAGE_LRU: + case SCAN_DEL_PAGE_LRU: + last_fail = result; + break; + default: + last_fail = result; + /* Other error, exit */ + goto out; + } + } + +out: + /* Caller expects us to hold mmap_lock on return */ + if (!vma) + mmap_read_lock(mm); +out_locked: + mmap_assert_locked(mm); + collapse_control_release(cc); + kfree(cc); + + return thps == ((hend - hstart) >> HPAGE_PMD_SHIFT) ? 0 + : madvise_collapse_errno(last_fail); +} + +#else /* CONFIG_TRANSPARENT_HUGEPAGE */ + +static int madvise_collapse(struct madvise_behavior *madv_behavior) +{ + return -EINVAL; +} + +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + static long madvise_dontneed_free(struct madvise_behavior *madv_behavior) { struct mm_struct *mm = madv_behavior->mm; @@ -1373,8 +1539,7 @@ static int madvise_vma_behavior(struct madvise_behavior *madv_behavior) case MADV_DONTNEED_LOCKED: return madvise_dontneed_free(madv_behavior); case MADV_COLLAPSE: - return madvise_collapse(vma, range->start, range->end, - &madv_behavior->lock_dropped); + return madvise_collapse(madv_behavior); case MADV_GUARD_INSTALL: return madvise_guard_install(madv_behavior); case MADV_GUARD_REMOVE: -- 2.54.0