Classical LRU protects mapped executable file folios through commit 8cab4754d24a0 ("vmscan: make mapped executable pages the first class citizen") and commit c909e99364c8 ("vmscan: activate executable pages after first usage"), giving executable code a better chance to stay in memory, avoiding IO thrashing and improving workload performance. However, MGLRU's protection of mapped executable file folios is less reliable. Although shrink_folio_list() checks references, the access flag of mapped executable file folios may have already been checked and cleared by lru_gen_look_around() or walk_mm(). Additionally, folio_update_gen() or lru_gen_set_refs() only sets the 'PG_referenced' flag for mapped executable file folios, which causes shrink_folio_list() to ignore the first usage of these mapped executable file folios and reclaim them easily. Follow the classical LRU's logic, promoting mapped executable file folios after their first usage in folio_update_gen() and lru_gen_set_refs(), giving executable code a better chance to stay in memory. On my 32-core Arm machine, with the memcg limit set to 2G, running 'make -j32' to build kernel showed some improvement in sys time. base patched 9248.543s 7861.579s Signed-off-by: Baolin Wang Acked-by: Johannes Weiner Reviewed-by: Axel Rasmussen --- mm/vmscan.c | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 8f0c31a4848e..b1ec65fb8947 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -841,10 +841,16 @@ enum folio_references { * with PG_active set. In contrast, the aging (page table walk) path uses * folio_update_gen(). */ -static bool lru_gen_set_refs(struct folio *folio) +static bool lru_gen_set_refs(struct folio *folio, const vma_flags_t *vma_flags) { /* see the comment on LRU_REFS_FLAGS */ if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) { + /* Activate file-backed executable folios after first usage. */ + if (is_exec_file_folio(folio, vma_flags)) { + set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_workingset)); + return true; + } + set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced)); return false; } @@ -857,7 +863,7 @@ static bool lru_gen_set_refs(struct folio *folio) return true; } #else -static bool lru_gen_set_refs(struct folio *folio) +static bool lru_gen_set_refs(struct folio *folio, const vma_flags_t *vma_flags) { return false; } @@ -892,7 +898,7 @@ static enum folio_references folio_check_references(struct folio *folio, if (!referenced_ptes) return FOLIOREF_RECLAIM; - return lru_gen_set_refs(folio) ? FOLIOREF_ACTIVATE : FOLIOREF_KEEP; + return lru_gen_set_refs(folio, &vma_flags) ? FOLIOREF_ACTIVATE : FOLIOREF_KEEP; } referenced_folio = folio_test_clear_referenced(folio); @@ -3192,14 +3198,19 @@ static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv) ******************************************************************************/ /* promote pages accessed through page tables */ -static int folio_update_gen(struct folio *folio, int gen) +static int folio_update_gen(struct folio *folio, int gen, const vma_flags_t *vma_flags) { unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f); VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); - /* see the comment on LRU_REFS_FLAGS */ - if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) { + /* + * See the comment on LRU_REFS_FLAGS, and activate file-backed + * executable folios after first usage to avoid typical IO + * thrashing from reclaiming. + */ + if (!folio_test_referenced(folio) && !folio_test_workingset(folio) && + !is_exec_file_folio(folio, vma_flags)) { set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced)); return -1; } @@ -3432,8 +3443,8 @@ static bool suitable_to_scan(int total, int young) return young * n >= total; } -static void walk_update_folio(struct lru_gen_mm_walk *walk, struct folio *folio, - int new_gen, bool dirty) +static void walk_update_folio(struct lru_gen_mm_walk *walk, struct vm_area_struct *vma, + struct folio *folio, int new_gen, bool dirty) { int old_gen; @@ -3446,10 +3457,10 @@ static void walk_update_folio(struct lru_gen_mm_walk *walk, struct folio *folio, folio_mark_dirty(folio); if (walk) { - old_gen = folio_update_gen(folio, new_gen); + old_gen = folio_update_gen(folio, new_gen, &vma->flags); if (old_gen >= 0 && old_gen != new_gen) update_batch_size(walk, folio, old_gen, new_gen); - } else if (lru_gen_set_refs(folio)) { + } else if (lru_gen_set_refs(folio, &vma->flags)) { old_gen = folio_lru_gen(folio); if (old_gen >= 0 && old_gen != new_gen) folio_activate(folio); @@ -3522,7 +3533,7 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, continue; if (last != folio) { - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, args->vma, last, gen, dirty); last = folio; dirty = false; @@ -3535,7 +3546,7 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, walk->mm_stats[MM_LEAF_YOUNG] += nr; } - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, args->vma, last, gen, dirty); last = NULL; if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) @@ -3613,7 +3624,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area goto next; if (last != folio) { - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, vma, last, gen, dirty); last = folio; dirty = false; @@ -3627,7 +3638,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1; } while (i <= MIN_LRU_BATCH); - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, vma, last, gen, dirty); lazy_mmu_mode_disable(); spin_unlock(ptl); @@ -4262,7 +4273,7 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) continue; if (last != folio) { - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, vma, last, gen, dirty); last = folio; dirty = false; @@ -4274,7 +4285,7 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) young += nr; } - walk_update_folio(walk, last, gen, dirty); + walk_update_folio(walk, vma, last, gen, dirty); lazy_mmu_mode_disable(); -- 2.47.3