This partially reverts commit b4c9ffb54b32 ("mm/khugepaged: remove definition of struct khugepaged_mm_slot"), as a subsequent commit needs to resurrect this type to store additional data alongside the mm_slot. The stated intent of the commit was to address a NULL check in __khugepaged_exit(): mm_slot = mm_slot_entry(slot, struct khugepaged_mm_slot, slot); slot can be NULL and mm_slot_entry() is an alias for container_of(), so this is container_of() a NULL value. This was benign as there was only 1 field in khugepaged_mm_slot (offsetting NULL at 0 results in NULL), however it was right to fix this, so retain the fix. It also unwrapped KMEM_CACHE() in khugepaged_init(). This was a mistake, and was itself reverted in commit c14bdcc9f274 ("mm/khugepaged: use KMEM_CACHE()"). Therefore, retain the KMEM_CACHE() form in this revert. Reviewed-by: David Hildenbrand (Arm) Reviewed-by: Lance Yang Signed-off-by: Lorenzo Stoakes (ARM) --- mm/khugepaged.c | 57 ++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/mm/khugepaged.c b/mm/khugepaged.c index da91762f6358..aad8797f8d5a 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -66,6 +66,14 @@ static DEFINE_READ_MOSTLY_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS); static struct kmem_cache *mm_slot_cache __ro_after_init; +/** + * struct khugepaged_mm_slot - khugepaged information per mm that is being scanned + * @slot: hash lookup from mm to mm_slot + */ +struct khugepaged_mm_slot { + struct mm_slot slot; +}; + /** * struct khugepaged_scan - cursor for scanning * @mm_head: the head of the mm list to scan @@ -76,7 +84,7 @@ static struct kmem_cache *mm_slot_cache __ro_after_init; */ struct khugepaged_scan { struct list_head mm_head; - struct mm_slot *mm_slot; + struct khugepaged_mm_slot *mm_slot; unsigned long address; }; @@ -386,7 +394,7 @@ int hugepage_madvise(struct vm_area_struct *vma, int __init khugepaged_init(void) { - mm_slot_cache = KMEM_CACHE(mm_slot, 0); + mm_slot_cache = KMEM_CACHE(khugepaged_mm_slot, 0); if (!mm_slot_cache) return -ENOMEM; @@ -446,20 +454,22 @@ static bool hugepage_enabled(void) void __khugepaged_enter(struct mm_struct *mm) { + struct khugepaged_mm_slot *mm_slot; struct mm_slot *slot; int wakeup; /* __khugepaged_exit() must not run from under us */ VM_BUG_ON_MM(collapse_test_exit(mm), mm); - slot = mm_slot_alloc(mm_slot_cache); - if (!slot) + mm_slot = mm_slot_alloc(mm_slot_cache); + if (!mm_slot) return; if (unlikely(mm_flags_test_and_set(MMF_VM_HUGEPAGE, mm))) { - mm_slot_free(mm_slot_cache, slot); + mm_slot_free(mm_slot_cache, mm_slot); return; } + slot = &mm_slot->slot; spin_lock(&khugepaged_mm_lock); mm_slot_insert(mm_slots_hash, mm, slot); @@ -504,12 +514,15 @@ void khugepaged_enter_vma(struct vm_area_struct *vma, void __khugepaged_exit(struct mm_struct *mm) { + struct khugepaged_mm_slot *mm_slot; struct mm_slot *slot; int free = 0; spin_lock(&khugepaged_mm_lock); slot = mm_slot_lookup(mm_slots_hash, mm); - if (slot && khugepaged_scan.mm_slot != slot) { + + mm_slot = slot ? mm_slot_entry(slot, struct khugepaged_mm_slot, slot) : NULL; + if (mm_slot && khugepaged_scan.mm_slot != mm_slot) { mm_slot_remove(slot); free = 1; } @@ -517,9 +530,9 @@ void __khugepaged_exit(struct mm_struct *mm) if (free) { mm_flags_clear(MMF_VM_HUGEPAGE, mm); - mm_slot_free(mm_slot_cache, slot); + mm_slot_free(mm_slot_cache, mm_slot); mmdrop(mm); - } else if (slot) { + } else if (mm_slot) { /* * This is required to serialize against * collapse_test_exit() (which is guaranteed to run @@ -1773,8 +1786,9 @@ static enum scan_result collapse_scan_anon(struct vm_area_struct *vma, return result; } -static void collect_mm_slot(struct mm_slot *slot) +static void collect_mm_slot(struct khugepaged_mm_slot *mm_slot) { + struct mm_slot *slot = &mm_slot->slot; struct mm_struct *mm = slot->mm; lockdep_assert_held(&khugepaged_mm_lock); @@ -1790,7 +1804,7 @@ static void collect_mm_slot(struct mm_slot *slot) */ /* khugepaged_mm_lock actually not necessary for the below */ - mm_slot_free(mm_slot_cache, slot); + mm_slot_free(mm_slot_cache, mm_slot); mmdrop(mm); } } @@ -2891,6 +2905,7 @@ static void collapse_scan_mm_slot(unsigned int progress_max, __acquires(&khugepaged_mm_lock) { struct vma_iterator vmi; + struct khugepaged_mm_slot *mm_slot; struct mm_slot *slot; struct mm_struct *mm; struct vm_area_struct *vma; @@ -2900,12 +2915,14 @@ static void collapse_scan_mm_slot(unsigned int progress_max, *result = SCAN_FAIL; if (khugepaged_scan.mm_slot) { - slot = khugepaged_scan.mm_slot; + mm_slot = khugepaged_scan.mm_slot; + slot = &mm_slot->slot; } else { slot = list_first_entry(&khugepaged_scan.mm_head, struct mm_slot, mm_node); + mm_slot = mm_slot_entry(slot, struct khugepaged_mm_slot, slot); khugepaged_scan.address = 0; - khugepaged_scan.mm_slot = slot; + khugepaged_scan.mm_slot = mm_slot; } spin_unlock(&khugepaged_mm_lock); @@ -2981,7 +2998,7 @@ static void collapse_scan_mm_slot(unsigned int progress_max, breakouterloop_mmap_lock: spin_lock(&khugepaged_mm_lock); - VM_BUG_ON(khugepaged_scan.mm_slot != slot); + VM_BUG_ON(khugepaged_scan.mm_slot != mm_slot); /* * Release the current mm_slot if this mm is about to die, or * if we scanned all vmas of this mm, or THP got disabled. @@ -2993,14 +3010,16 @@ static void collapse_scan_mm_slot(unsigned int progress_max, * mm_slot not pointing to the exiting mm. */ if (!list_is_last(&slot->mm_node, &khugepaged_scan.mm_head)) { - khugepaged_scan.mm_slot = list_next_entry(slot, mm_node); + slot = list_next_entry(slot, mm_node); + khugepaged_scan.mm_slot = + mm_slot_entry(slot, struct khugepaged_mm_slot, slot); khugepaged_scan.address = 0; } else { khugepaged_scan.mm_slot = NULL; khugepaged_full_scans++; } - collect_mm_slot(slot); + collect_mm_slot(mm_slot); } trace_mm_khugepaged_scan(mm, cc->progress - progress_prev, @@ -3090,7 +3109,7 @@ static void khugepaged_wait_work(void) static int khugepaged(void *none) { - struct mm_slot *slot; + struct khugepaged_mm_slot *mm_slot; set_freezable(); set_user_nice(current, MAX_NICE); @@ -3101,10 +3120,10 @@ static int khugepaged(void *none) } spin_lock(&khugepaged_mm_lock); - slot = khugepaged_scan.mm_slot; + mm_slot = khugepaged_scan.mm_slot; khugepaged_scan.mm_slot = NULL; - if (slot) - collect_mm_slot(slot); + if (mm_slot) + collect_mm_slot(mm_slot); spin_unlock(&khugepaged_mm_lock); return 0; } -- 2.56.0