From: Xu Xin Add an additional member 'walk_lock' to folio_walk to indicate locking requirements for the walk. Similar to commit 49b0638502da0 ("mm: enable page walking API to lock vmas during the walk"). But no change is made on any existing locking behavior, all existing folio_walk_start() callers are currently still under mmap_read_lock() protection. This change is prepared for the later patch to enable VMA locking asserts. folio_walk_start now operate under write-locked mmap_lock. With introduction of vma locks at the next patch, the vmas have to be locked as well during such walks to prevent concurrent page faults in these areas. No functional change intended. Signed-off-by: Xu Xin --- arch/s390/mm/fault.c | 4 +++- include/linux/pagewalk.h | 1 + kernel/events/uprobes.c | 4 +++- mm/huge_memory.c | 4 +++- mm/ksm.c | 4 +++- mm/migrate.c | 8 ++++++-- mm/pagewalk.c | 10 +++++++++- mm/rmap.c | 4 +++- 8 files changed, 31 insertions(+), 8 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 46d828926009..968f128a5232 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -412,7 +412,9 @@ __context_unsafe(/* folio_walk_end() not instrumented */) unsigned long addr = get_fault_address(regs); struct mm_struct *mm = current->mm; struct vm_area_struct *vma; - struct folio_walk fw; + struct folio_walk fw = { + .walk_lock = PGWALK_RDLOCK, + }; struct folio *folio; int rc; diff --git a/include/linux/pagewalk.h b/include/linux/pagewalk.h index 1c397be0d092..455428eaf1fc 100644 --- a/include/linux/pagewalk.h +++ b/include/linux/pagewalk.h @@ -184,6 +184,7 @@ struct folio_walk { }; /* private */ spinlock_t *ptl; + enum page_walk_lock walk_lock; }; struct folio *folio_walk_start(struct folio_walk *fw, diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index caf444762a27..3e3927109ee9 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -507,7 +507,9 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, int ret, ref_ctr_updated = 0; unsigned int gup_flags = FOLL_FORCE; struct mmu_notifier_range range; - struct folio_walk fw; + struct folio_walk fw = { + .walk_lock = PGWALK_RDLOCK, + }; struct folio *folio; struct page *page; diff --git a/mm/huge_memory.c b/mm/huge_memory.c index bd44d58e8684..62ec81396c9d 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -4816,7 +4816,9 @@ static int split_huge_pages_pid(int pid, unsigned long vaddr_start, */ for (addr = vaddr_start; addr < vaddr_end; addr += PAGE_SIZE) { struct vm_area_struct *vma = vma_lookup(mm, addr); - struct folio_walk fw; + struct folio_walk fw = { + .walk_lock = PGWALK_RDLOCK, + }; struct folio *folio; struct address_space *mapping; unsigned int target_order = new_order; diff --git a/mm/ksm.c b/mm/ksm.c index d6f736a1aa2f..d4fc0b0965aa 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -819,8 +819,10 @@ static struct page *get_mergeable_page(struct ksm_rmap_item *rmap_item) unsigned long addr = rmap_item->address; struct vm_area_struct *vma; struct page *page = NULL; - struct folio_walk fw; struct folio *folio; + struct folio_walk fw = { + .walk_lock = PGWALK_RDLOCK, + }; mmap_read_lock(mm); vma = find_mergeable_vma(mm, addr); diff --git a/mm/migrate.c b/mm/migrate.c index 875079464930..89662d5bedf4 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -2301,7 +2301,9 @@ static int add_folio_for_migration(struct mm_struct *mm, const void __user *p, int node, struct list_head *pagelist, bool migrate_all) { struct vm_area_struct *vma; - struct folio_walk fw; + struct folio_walk fw = { + .walk_lock = PGWALK_RDLOCK, + }; struct folio *folio; unsigned long addr; int err = -EFAULT; @@ -2463,7 +2465,9 @@ static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages, for (i = 0; i < nr_pages; i++) { unsigned long addr = (unsigned long)untagged_addr_remote(mm, *pages); struct vm_area_struct *vma; - struct folio_walk fw; + struct folio_walk fw = { + .walk_lock = PGWALK_RDLOCK, + }; struct folio *folio; int err = -EFAULT; diff --git a/mm/pagewalk.c b/mm/pagewalk.c index e6493bbe6919..7d85289fcb87 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -910,7 +910,15 @@ struct folio *folio_walk_start(struct folio_walk *fw, pgd_t *pgdp; p4d_t *p4dp; - mmap_assert_locked(vma->vm_mm); + /* + * Other locking modes except for mmap or vma read locking are not + * expected. + */ + if (fw->walk_lock != PGWALK_RDLOCK && fw->walk_lock != PGWALK_VMA_RDLOCK_VERIFY) + WARN_ONCE(1, "walk_lock is not expected!\n"); + process_mm_walk_lock(vma->vm_mm, fw->walk_lock); + process_vma_walk_lock(vma, fw->walk_lock); + vma_pgtable_walk_begin(vma); if (WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end)) diff --git a/mm/rmap.c b/mm/rmap.c index 1131b76bbbc2..6e3bc2acd1d7 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -2862,7 +2862,9 @@ struct page *make_device_exclusive(struct mm_struct *mm, unsigned long addr, struct mmu_notifier_range range; struct folio *folio, *fw_folio; struct vm_area_struct *vma; - struct folio_walk fw; + struct folio_walk fw = { + .walk_lock = PGWALK_RDLOCK, + }; struct page *page; swp_entry_t entry; pte_t swp_pte; -- 2.34.1