Create ALLOC_UNMAPPED, which requests pages that are not present in the direct map. Since this feature has a cost (e.g. more freelists), it's behind a kconfig. Unlike other conditionally-defined alloc flags, it doesn't fall back to being 0. This prevents building code that uses ALLOC_UNMAPPED but doesn't depend on the necessary kconfig, since that would lead to invisible security issues. Create a freetype flag to record that pages on the freelists with this flag are unmapped. This is currently only needed for MIGRATE_UNMOVABLE pages, so the freetype encoding remains trivial. Also create the corresponding pageblock flag to record the same thing. To keep patches from being too overwhelming, the actual implementation is added separately, this is just types, Kconfig boilerplate, etc. Acked-by: Vlastimil Babka (SUSE) Signed-off-by: Brendan Jackman --- include/linux/freetype.h | 70 ++++++++++++++++++++++++++++++++++++++++-------- mm/Kconfig | 3 +++ mm/page_alloc.h | 18 +++++++++++++ 3 files changed, 80 insertions(+), 11 deletions(-) diff --git a/include/linux/freetype.h b/include/linux/freetype.h index 3b0d44023b6a1..37e88dcccdecc 100644 --- a/include/linux/freetype.h +++ b/include/linux/freetype.h @@ -2,6 +2,7 @@ #ifndef _LINUX_FREETYPE_H #define _LINUX_FREETYPE_H +#include #include #include @@ -64,20 +65,47 @@ static inline bool migratetype_is_mergeable(int mt) return mt < MIGRATE_PCPTYPES; } +enum { + /* Defined unconditionally as a hack to avoid a zero-width bitfield. */ + FREETYPE_UNMAPPED_BIT, + NUM_FREETYPE_FLAGS, +}; + /* * A freetype is the identifier for a page freelist. This consists of a * migratetype, and other bits which encode orthogonal properties of memory. */ typedef struct { - int migratetype; + unsigned int migratetype : order_base_2(MIGRATE_TYPES); + unsigned int flags : NUM_FREETYPE_FLAGS; } freetype_t; +#ifdef CONFIG_PAGE_ALLOC_UNMAPPED +#define FREETYPE_UNMAPPED BIT(FREETYPE_UNMAPPED_BIT) +#define NUM_UNMAPPED_FREETYPES 1 +#else +#define FREETYPE_UNMAPPED 0 +#define NUM_UNMAPPED_FREETYPES 0 +#endif + +#define FREETYPE_FLAGS_MASK FREETYPE_UNMAPPED + /* * Return a dense linear index for freetypes that have lists in the free area. * Return -1 for other freetypes. */ static inline int freetype_idx(freetype_t freetype) { + /* For FREETYPE_UNMAPPED, only MIGRATE_UNMOVABLE has an index. */ + if (freetype.flags & FREETYPE_UNMAPPED) { + VM_WARN_ON_ONCE(freetype.flags & ~FREETYPE_UNMAPPED); + if (freetype.migratetype != MIGRATE_UNMOVABLE) + return -1; + return MIGRATE_TYPES; + } + /* No other flags are supported. */ + VM_WARN_ON_ONCE(freetype.flags); + return freetype.migratetype; } @@ -85,33 +113,53 @@ static inline freetype_t freetype_from_idx(unsigned int idx) { freetype_t freetype; - freetype.migratetype = idx; + if (idx == MIGRATE_TYPES) { + freetype.flags = FREETYPE_UNMAPPED; + freetype.migratetype = MIGRATE_UNMOVABLE; + } else { + VM_WARN_ON_ONCE(idx < 0 || idx > MIGRATE_TYPES); + freetype.flags = 0; + freetype.migratetype = idx; + } return freetype; } -/* No freetype flags actually exist yet. */ -#define NR_FREETYPE_IDXS MIGRATE_TYPES +/* One for each migratetype, plus one for MIGRATE_UNMOVABLE-FREETYPE_UNMAPPED */ +#define NR_FREETYPE_IDXS (MIGRATE_TYPES + NUM_UNMAPPED_FREETYPES) static inline unsigned int freetype_flags(freetype_t freetype) { - /* No flags supported yet. */ - return 0; + return freetype.flags; } +#ifdef CONFIG_PAGE_ALLOC_UNMAPPED +static inline bool freetype_unmapped(freetype_t freetype) +{ + return !!(freetype.flags & FREETYPE_UNMAPPED); +} +#else +static inline bool freetype_unmapped(freetype_t freetype) +{ + return false; +} +#endif + static inline bool freetypes_equal(freetype_t a, freetype_t b) { - return a.migratetype == b.migratetype; + return a.migratetype == b.migratetype && a.flags == b.flags; } static inline freetype_t migrate_to_freetype(enum migratetype mt, unsigned int flags) { - freetype_t freetype; + freetype_t freetype = { + .migratetype = mt, + .flags = flags, + }; - /* No flags supported yet. */ - VM_WARN_ON_ONCE(flags); + VM_WARN_ON_ONCE(flags & ~FREETYPE_FLAGS_MASK); + VM_WARN_ON_ONCE(mt >= MIGRATE_TYPES); - freetype.migratetype = mt; return freetype; } diff --git a/mm/Kconfig b/mm/Kconfig index adf71788971a2..aa5f041ce1a67 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -1530,3 +1530,6 @@ config MERMAP_KUNIT_TEST source "mm/damon/Kconfig" endmenu + +config PAGE_ALLOC_UNMAPPED + bool diff --git a/mm/page_alloc.h b/mm/page_alloc.h index 9928aa9012588..fac8e5304bb03 100644 --- a/mm/page_alloc.h +++ b/mm/page_alloc.h @@ -56,6 +56,24 @@ * alloc_tag_sub_check(). */ #define ALLOC_NO_CODETAG 0x1000 +#ifdef CONFIG_PAGE_ALLOC_UNMAPPED +/* + * Allocate pages that aren't present in the direct map. If the caller changes + * direct map presence, it must be restored to the previous state before freeing + * the page. (This is true regardless of ALLOC_UNMAPPED). + * + * This uses the mermap (when __GFP_ZERO), so it's only valid to allocate with + * this flag where that's valid, namely from process context after the mermap + * has been initialised for that process. This also means that the allocator + * leaves behind stale TLB entries in the mermap region. The caller is + * responsible for ensuring they are flushed as needed. + * + * This is currently incompatible with __GFP_MOVABLE and __GFP_RECLAIMABLE, but + * only because of allocator implementation details, if a usecase arises this + * restriction could be dropped. + */ +#define ALLOC_UNMAPPED 0x2000 +#endif /* Flags that allow allocations below the min watermark. */ #define ALLOC_RESERVES (ALLOC_NON_BLOCK|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM) -- 2.54.0