On a VM_HUGEPAGE file mapping, a synchronous fault reads the PMD-sized folio that holds the faulting address and the one above it, and maps the faulting folio with a PMD. do_sync_mmap_readahead() counts each such fault in mmap_miss and stops reading ahead once it passes MMAP_LOTSAMISS, while filemap_map_pages() lowers the count for faults served from the page cache, but only when it maps them with PTEs. A reader that moves towards lower offsets, such as memmove() copying backwards out of a mapping, never finds its next folio cached: the folio above is the one it just read. Every folio is a miss and none is a hit, so after about a hundred folios mmap_miss passes the limit and every page after that is read on its own. Since the PTEs mapped around a fault no longer count as hits, nothing lowers the count for it. When the folio above the faulting one is cached, read the folio below instead, without a readahead mark, and count a PMD mapping of the faulting address as a hit the way a PTE mapping is. Such a reader then misses on every other folio and hits on the rest. Reading one byte from each page of a 2 GiB file, last page first, in a MADV_HUGEPAGE mapping with a cold page cache, read_ahead_kb 8192: before after time 36.76 s 0.41 s major faults 473191 516 Fixes: 0b9c0aeba938 ("mm/filemap: count only the faulting address as a mmap hit") Cc: stable@vger.kernel.org Signed-off-by: Qiliang Yuan --- mm/filemap.c | 101 ++++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 72 insertions(+), 29 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index 00fd89cf6f550..8a7d1238ae75d 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -3319,6 +3319,23 @@ static int lock_folio_maybe_drop_mmap(struct vm_fault *vmf, struct folio *folio, return 1; } +/* + * Whether a miss at @index, right below @nr cached pages, comes from a + * stream that runs towards lower offsets. + */ +static bool filemap_descending(struct address_space *mapping, pgoff_t index, + unsigned long nr) +{ + void *entry; + + if (index < nr) + return false; + rcu_read_lock(); + entry = xa_load(&mapping->i_pages, index + nr); + rcu_read_unlock(); + return entry && !xa_is_value(entry); +} + /* * Synchronous readahead happens when we don't even find a page in the page * cache at all. We don't want to perform IO under the mmap sem, so if we have @@ -3392,13 +3409,20 @@ static struct file *do_sync_mmap_readahead(struct vm_fault *vmf) fpin = maybe_unlock_mmap_for_io(vmf, fpin); ractl._index &= ~(folio_nr_pages - 1); ra->size = folio_nr_pages; + ra->async_size = folio_nr_pages; /* * Fetch two folios so we get the chance to actually - * readahead, unless we've been told not to. + * readahead, unless we've been told not to. A stream that + * runs towards lower offsets gets the folio below. */ - if (!(vm_flags & VM_RAND_READ)) + if (!(vm_flags & VM_RAND_READ)) { ra->size *= 2; - ra->async_size = folio_nr_pages; + if (filemap_descending(mapping, ractl._index, + folio_nr_pages)) { + ractl._index -= folio_nr_pages; + ra->async_size = 0; + } + } ra->order = thp_order; page_cache_ra_order(&ractl, ra); return fpin; @@ -3896,6 +3920,39 @@ static vm_fault_t filemap_map_order0_folio(struct vm_fault *vmf, return ret; } +/* + * Whether mapping @folio at the faulting address counts as an mmap hit, + * which lowers mmap_miss in do_sync_mmap_readahead(). + */ +static bool filemap_mmap_hit(struct vm_fault *vmf, struct folio *folio) +{ + /* Skip the retry of a fault, which maps the folio its miss read */ + if (vmf->flags & FAULT_FLAG_TRIED) + return false; + /* + * If there are too many folios that are recently evicted in a file, + * they will probably continue to be evicted. In such situation, + * read-ahead is only a waste of IO. Don't decrease mmap_miss in this + * scenario to make sure we can stop read-ahead. + */ + if (folio_test_workingset(folio)) + return false; + /* + * VM_SEQ_READ and VM_EXEC mappings skip the mmap_miss increment in + * do_sync_mmap_readahead(), so skip the decrement as well to keep + * the counter symmetric. + */ + return !(vmf->vma->vm_flags & (VM_SEQ_READ | VM_EXEC)); +} + +static void filemap_count_mmap_hit(struct file *file) +{ + unsigned short mmap_miss = READ_ONCE(file->f_ra.mmap_miss); + + if (mmap_miss) + WRITE_ONCE(file->f_ra.mmap_miss, mmap_miss - 1); +} + vm_fault_t filemap_map_pages(struct vm_fault *vmf, pgoff_t start_pgoff, pgoff_t end_pgoff) { @@ -3930,10 +3987,16 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, * Make an exception for shmem/tmpfs that for long time * intentionally mapped with PMDs across i_size. */ - if ((file_end >= folio_next_index(folio) || shmem_mapping(mapping)) && - filemap_map_pmd(vmf, folio, start_pgoff)) { - ret = VM_FAULT_NOPAGE; - goto out; + if (file_end >= folio_next_index(folio) || shmem_mapping(mapping)) { + /* Check before filemap_map_pmd(), which may drop the last ref */ + bool hit = filemap_mmap_hit(vmf, folio); + + if (filemap_map_pmd(vmf, folio, start_pgoff)) { + if (hit) + filemap_count_mmap_hit(file); + ret = VM_FAULT_NOPAGE; + goto out; + } } addr = vma->vm_start + @@ -3968,28 +4031,8 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, } ret |= map_ret; - /* - * If there are too many folios that are recently evicted - * in a file, they will probably continue to be evicted. - * In such situation, read-ahead is only a waste of IO. - * Don't decrease mmap_miss in this scenario to make sure - * we can stop read-ahead. - * - * VM_SEQ_READ and VM_EXEC mappings skip the mmap_miss - * increment in do_sync_mmap_readahead(), so skip the - * decrement here as well to keep the counter symmetric. - */ - if ((map_ret & VM_FAULT_NOPAGE) && - !(vmf->flags & FAULT_FLAG_TRIED) && - !folio_test_workingset(folio) && - !(vma->vm_flags & (VM_SEQ_READ | VM_EXEC))) { - unsigned short mmap_miss; - - mmap_miss = READ_ONCE(file->f_ra.mmap_miss); - if (mmap_miss) - WRITE_ONCE(file->f_ra.mmap_miss, - mmap_miss - 1); - } + if ((map_ret & VM_FAULT_NOPAGE) && filemap_mmap_hit(vmf, folio)) + filemap_count_mmap_hit(file); folio_unlock(folio); } while ((folio = next_uptodate_folio(&xas, mapping, end_pgoff)) != NULL); -- 2.43.0