In preparation for wp to support mthp, the single-page operation function is replaced with a multi-page operation function, without any functional changes. Signed-off-by: Vernon Yang --- include/linux/mm.h | 7 +++++++ mm/memory.c | 12 ++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index fa538feaa8d9..80c6673f419e 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2589,6 +2589,13 @@ static inline void inc_mm_counter(struct mm_struct *mm, int member) mm_trace_rss_stat(mm, member); } +static inline void sub_mm_counter(struct mm_struct *mm, int member, long value) +{ + percpu_counter_sub(&mm->rss_stat[member], value); + + mm_trace_rss_stat(mm, member); +} + static inline void dec_mm_counter(struct mm_struct *mm, int member) { percpu_counter_dec(&mm->rss_stat[member]); diff --git a/mm/memory.c b/mm/memory.c index b0cda5aab398..a6bc1db22387 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3606,14 +3606,14 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) if (likely(vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte))) { if (old_folio) { if (!folio_test_anon(old_folio)) { - dec_mm_counter(mm, mm_counter_file(old_folio)); - inc_mm_counter(mm, MM_ANONPAGES); + sub_mm_counter(mm, mm_counter_file(old_folio), 1); + add_mm_counter(mm, MM_ANONPAGES, 1); } } else { ksm_might_unmap_zero_page(mm, vmf->orig_pte); inc_mm_counter(mm, MM_ANONPAGES); } - flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte)); + flush_cache_range(vma, vmf->address, vmf->address + PAGE_SIZE); entry = folio_mk_pte(new_folio, vma->vm_page_prot); entry = pte_sw_mkyoung(entry); if (unlikely(unshare)) { @@ -3636,7 +3636,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) folio_add_new_anon_rmap(new_folio, vma, vmf->address, RMAP_EXCLUSIVE); folio_add_lru_vma(new_folio, vma); BUG_ON(unshare && pte_write(entry)); - set_pte_at(mm, vmf->address, vmf->pte, entry); + set_ptes(mm, vmf->address, vmf->pte, entry, 1); update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); if (old_folio) { /* @@ -3661,7 +3661,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) * mapcount is visible. So transitively, TLBs to * old page will be flushed before it can be reused. */ - folio_remove_rmap_pte(old_folio, vmf->page, vma); + folio_remove_rmap_ptes(old_folio, vmf->page, 1, vma); } /* Free the old page.. */ @@ -3676,7 +3676,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) mmu_notifier_invalidate_range_end(&range); if (new_folio) - folio_put(new_folio); + folio_put_refs(new_folio, 1); if (old_folio) { if (page_copied) free_swap_cache(old_folio); -- 2.50.1