The KMSAN runtime needs an unchecked page extension lookup that can be inlined into its uninstrumented compilation unit while page_ext.c remains instrumented. Provide an inline lookup for this use case while keeping the existing RCU checks for other callers. Signed-off-by: Muchun Song --- v2: - Add a preparatory patch that inlines the unchecked lookup (suggested by Chris Down) - Use a PFN interface to avoid architecture header-order dependencies --- include/linux/page_ext.h | 46 ++++++++++++++++++++++++++++++ mm/page_ext.c | 60 +++------------------------------------- 2 files changed, 50 insertions(+), 56 deletions(-) diff --git a/include/linux/page_ext.h b/include/linux/page_ext.h index d00320106a58..166f8be11564 100644 --- a/include/linux/page_ext.h +++ b/include/linux/page_ext.h @@ -64,7 +64,14 @@ static inline bool early_page_ext_enabled(void) void page_ext_alloc(void); void page_ext_init(void); +static __always_inline struct page_ext *page_ext_entry(void *base, unsigned long index) +{ + return base + page_ext_size * index; +} + #ifdef CONFIG_SPARSEMEM +#define PAGE_EXT_INVALID 0x1 + static inline bool page_ext_iter_next_fast_possible(unsigned long next_pfn) { /* @@ -73,11 +80,50 @@ static inline bool page_ext_iter_next_fast_possible(unsigned long next_pfn) */ return next_pfn % PAGES_PER_SECTION; } + +static __always_inline bool page_ext_invalid(struct page_ext *page_ext) +{ + return !page_ext || (((unsigned long)page_ext & PAGE_EXT_INVALID) == PAGE_EXT_INVALID); +} + +static __always_inline struct page_ext *__page_ext_lookup(unsigned long pfn) +{ + struct mem_section *section = __pfn_to_section(pfn); + struct page_ext *page_ext = READ_ONCE(section->page_ext); + + /* + * The sanity checks the page allocator does upon freeing a page can + * reach here before the page_ext arrays are allocated when feeding a + * range of pages to the allocator for the first time during bootup or + * memory hotplug. + */ + if (page_ext_invalid(page_ext)) + return NULL; + return page_ext_entry(page_ext, pfn); +} #else static inline bool page_ext_iter_next_fast_possible(unsigned long next_pfn) { return true; } + +static __always_inline struct page_ext *__page_ext_lookup(unsigned long pfn) +{ + int nid = pfn_to_nid(pfn); + struct page_ext *base = NODE_DATA(nid)->node_page_ext; + unsigned long index; + + /* + * The sanity checks the page allocator does upon freeing a page can + * reach here before the page_ext arrays are allocated when feeding a + * range of pages to the allocator for the first time during bootup or + * memory hotplug. + */ + if (unlikely(!base)) + return NULL; + index = pfn - round_down(node_start_pfn(nid), MAX_ORDER_NR_PAGES); + return page_ext_entry(base, index); +} #endif extern struct page_ext *page_ext_get(const struct page *page); diff --git a/mm/page_ext.c b/mm/page_ext.c index 2910727e99e2..0f31e2da2611 100644 --- a/mm/page_ext.c +++ b/mm/page_ext.c @@ -63,10 +63,6 @@ * can utilize this callback to initialize the state of it correctly. */ -#ifdef CONFIG_SPARSEMEM -#define PAGE_EXT_INVALID (0x1) -#endif - #if defined(CONFIG_PAGE_IDLE_FLAG) && !defined(CONFIG_64BIT) static bool need_page_idle(void) { @@ -175,33 +171,7 @@ static void __init invoke_init_callbacks(void) } } -static inline struct page_ext *get_entry(void *base, unsigned long index) -{ - return base + page_ext_size * index; -} - #ifndef CONFIG_SPARSEMEM -static struct page_ext *lookup_page_ext(const struct page *page) -{ - unsigned long pfn = page_to_pfn(page); - unsigned long index; - struct page_ext *base; - - WARN_ON_ONCE(!rcu_read_lock_held()); - base = NODE_DATA(page_to_nid(page))->node_page_ext; - /* - * The sanity checks the page allocator does upon freeing a - * page can reach here before the page_ext arrays are - * allocated when feeding a range of pages to the allocator - * for the first time during bootup or memory hotplug. - */ - if (unlikely(!base)) - return NULL; - index = pfn - round_down(node_start_pfn(page_to_nid(page)), - MAX_ORDER_NR_PAGES); - return get_entry(base, index); -} - static int __init alloc_node_page_ext(int nid) { struct page_ext *base; @@ -247,29 +217,6 @@ static int __init __page_ext_alloc(void) } #else /* CONFIG_SPARSEMEM */ -static bool page_ext_invalid(struct page_ext *page_ext) -{ - return !page_ext || (((unsigned long)page_ext & PAGE_EXT_INVALID) == PAGE_EXT_INVALID); -} - -static struct page_ext *lookup_page_ext(const struct page *page) -{ - unsigned long pfn = page_to_pfn(page); - struct mem_section *section = __pfn_to_section(pfn); - struct page_ext *page_ext = READ_ONCE(section->page_ext); - - WARN_ON_ONCE(!rcu_read_lock_held()); - /* - * The sanity checks the page allocator does upon freeing a - * page can reach here before the page_ext arrays are - * allocated when feeding a range of pages to the allocator - * for the first time during bootup or memory hotplug. - */ - if (page_ext_invalid(page_ext)) - return NULL; - return get_entry(page_ext, pfn); -} - static __ref void *alloc_page_ext(size_t size, int nid) { void *addr; @@ -360,7 +307,7 @@ static void __free_page_ext(unsigned long pfn) base = (void *)base - PAGE_EXT_INVALID; WRITE_ONCE(ms->page_ext, NULL); - base = get_entry(base, pfn); + base = page_ext_entry(base, pfn); free_page_ext(base); } @@ -503,7 +450,8 @@ void __init page_ext_init(void) */ struct page_ext *page_ext_lookup(unsigned long pfn) { - return lookup_page_ext(pfn_to_page(pfn)); + WARN_ON_ONCE(!rcu_read_lock_held()); + return __page_ext_lookup(pfn); } /** @@ -522,7 +470,7 @@ struct page_ext *page_ext_get(const struct page *page) struct page_ext *page_ext; rcu_read_lock(); - page_ext = lookup_page_ext(page); + page_ext = page_ext_lookup(page_to_pfn(page)); if (!page_ext) { rcu_read_unlock(); return NULL; -- 2.54.0