From: Kairui Song folio_test_workingset() currently tests the PG_workingset bit, which is only the second lowest access bit of the folio LRU refs count now. Folios with a higher refs count carry it in the LRU_REFS_MASK field and no longer have PG_workingset set, so the bit test misses them. folio_set_workingset() currently sets the PG_workingset bit with a plain set_bit(), which also over-promotes folios at a higher refs count: e.g. a folio at refs 4 is bumped to refs 6, advancing it toward LRU_REFS_MAX and an unintended generation promotion on the next access. Move the test into mm_inline.h as an inline helper based on folio_lru_refs(), checking refs >= LRU_REFS_WORKINGSET. The set stays a plain PG_workingset bit operation reserved for the classical LRU: under MGLRU the refs count is maintained by folio_inc_lru_refs(), and now triggers a debug WARN when called while MGLRU is fully on (the switching window is exempt, as the classical paths legitimately run alongside MGLRU then). The one PageWorkingset() user in erofs is converted to the folio helper. Under the classical LRU the refs count is not maintained, so the test falls back to the PG_workingset bit test, which is the old behavior. Signed-off-by: Kairui Song --- fs/btrfs/compression.c | 1 + fs/erofs/zdata.c | 3 ++- include/linux/mm_inline.h | 35 +++++++++++++++++++++++++++++++++++ include/linux/page-flags.h | 2 -- mm/filemap.c | 1 + mm/page_io.c | 1 + 6 files changed, 40 insertions(+), 3 deletions(-) diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index c62b5148d5ac..57d24413265e 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c index e981e371d6c2..34ee41e1ccdc 100644 --- a/fs/erofs/zdata.c +++ b/fs/erofs/zdata.c @@ -6,6 +6,7 @@ */ #include "compress.h" #include +#include #include #include @@ -1713,7 +1714,7 @@ static void z_erofs_submit_queue(struct z_erofs_frontend *f, DBG_BUGON(bvec.bv_len < sb->s_blocksize); } - if (unlikely(PageWorkingset(bvec.bv_page)) && + if (unlikely(folio_test_workingset(page_folio(bvec.bv_page))) && !memstall) { psi_memstall_enter(&pflags); memstall = 1; diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index b68d68101248..e651f60cb914 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -258,6 +258,24 @@ static inline bool lru_gen_enabled(void) } #endif +/** + * folio_test_workingset - Test if a folio is in the workingset. + * @folio: the folio + * + * A folio is workingset when its LRU refs count reaches + * LRU_REFS_WORKINGSET. Under the classical LRU the refs count never + * goes above it, so this is just testing the PG_workingset bit. + * NOTE: folio_set_workingset() must not be used under MGLRU, as the + * folio refs are tracked by folio_inc_lru_refs(), it triggers a debug + * WARN instead for MGLRU. + * + * Return: true if the folio is workingset. + */ +static __always_inline bool folio_test_workingset(const struct folio *folio) +{ + return folio_lru_refs(folio) >= LRU_REFS_WORKINGSET; +} + static inline bool lru_gen_in_fault(void) { return current->in_lru_fault; @@ -445,6 +463,11 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, { return false; } + +static inline bool folio_test_workingset(const struct folio *folio) +{ + return test_bit(PG_workingset, const_folio_flags(folio, FOLIO_HEAD_PAGE)); +} #endif /* CONFIG_LRU_GEN */ /** @@ -473,6 +496,18 @@ static __always_inline void folio_inc_lru_refs_fast(struct folio *folio) } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); } +/* + * For the classical LRU only: under MGLRU the PG_workingset bit is + * part of the folio refs count maintained by folio_inc_lru_refs(), + * and a raw set would corrupt it. The switching window is exempt + * because the classical paths legitimately run alongside MGLRU then. + */ +static __always_inline void folio_set_workingset(struct folio *folio) +{ + VM_WARN_ON_ONCE(lru_gen_enabled() && !lru_gen_switching()); + set_bit(PG_workingset, folio_flags(folio, FOLIO_HEAD_PAGE)); +} + static __always_inline void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio) { diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index b0ddc652e76c..f6625dc042da 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -549,8 +549,6 @@ PAGEFLAG(LRU, lru, PF_HEAD) __CLEARPAGEFLAG(LRU, lru, PF_HEAD) FOLIO_FLAG(active, FOLIO_HEAD_PAGE) __FOLIO_CLEAR_FLAG(active, FOLIO_HEAD_PAGE) FOLIO_TEST_CLEAR_FLAG(active, FOLIO_HEAD_PAGE) -PAGEFLAG(Workingset, workingset, PF_HEAD) - TESTCLEARFLAG(Workingset, workingset, PF_HEAD) PAGEFLAG(Checked, checked, PF_NO_COMPOUND) /* Used by some filesystems */ /* Xen */ diff --git a/mm/filemap.c b/mm/filemap.c index b74bc1e5015c..49e827058e14 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include diff --git a/mm/page_io.c b/mm/page_io.c index c6824fcd483e..540ae8caa932 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include -- 2.55.0