Encapsulate the mhp_init_memmap_on_memory() and online_pages() into online_memory_block_pages(). Thus we can further optimize the set_zone_contiguous() to check the whole memory block range, instead of check the zone contiguous in separate range. Correspondingly, encapsulate the mhp_deinit_memmap_on_memory() and offline_pages() into offline_memory_block_pages(). Furthermore, move most of memory_block_online() to the new function mhp_block_online(struct memory_block *block) and correspondingly memory_block_offline() to mhp_block_offline(struct memory_block *block). Tested-by: Yuan Liu Reviewed-by: Yuan Liu Signed-off-by: Tianyou Li --- drivers/base/memory.c | 115 +--------------------------- include/linux/memory_hotplug.h | 13 +--- include/linux/mm.h | 6 ++ mm/memory_hotplug.c | 132 ++++++++++++++++++++++++++++++++- 4 files changed, 141 insertions(+), 125 deletions(-) diff --git a/drivers/base/memory.c b/drivers/base/memory.c index 751f248ca4a8..40f014c5dbb1 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -209,115 +209,6 @@ int memory_notify(enum memory_block_state state, void *v) return blocking_notifier_call_chain(&memory_chain, state, v); } -#if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_MEMORY_HOTPLUG) -static unsigned long memblk_nr_poison(struct memory_block *mem); -#else -static inline unsigned long memblk_nr_poison(struct memory_block *mem) -{ - return 0; -} -#endif - -/* - * Must acquire mem_hotplug_lock in write mode. - */ -static int memory_block_online(struct memory_block *mem) -{ - unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); - unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; - unsigned long nr_vmemmap_pages = 0; - struct zone *zone; - int ret; - - if (memblk_nr_poison(mem)) - return -EHWPOISON; - - zone = zone_for_pfn_range(mem->online_type, mem->nid, mem->group, - start_pfn, nr_pages); - - /* - * Although vmemmap pages have a different lifecycle than the pages - * they describe (they remain until the memory is unplugged), doing - * their initialization and accounting at memory onlining/offlining - * stage helps to keep accounting easier to follow - e.g vmemmaps - * belong to the same zone as the memory they backed. - */ - if (mem->altmap) - nr_vmemmap_pages = mem->altmap->free; - - mem_hotplug_begin(); - if (nr_vmemmap_pages) { - ret = mhp_init_memmap_on_memory(start_pfn, nr_vmemmap_pages, zone); - if (ret) - goto out; - } - - ret = online_pages(start_pfn + nr_vmemmap_pages, - nr_pages - nr_vmemmap_pages, zone, mem->group); - if (ret) { - if (nr_vmemmap_pages) - mhp_deinit_memmap_on_memory(start_pfn, nr_vmemmap_pages); - goto out; - } - - /* - * Account once onlining succeeded. If the zone was unpopulated, it is - * now already properly populated. - */ - if (nr_vmemmap_pages) - adjust_present_page_count(pfn_to_page(start_pfn), mem->group, - nr_vmemmap_pages); - - mem->zone = zone; -out: - mem_hotplug_done(); - return ret; -} - -/* - * Must acquire mem_hotplug_lock in write mode. - */ -static int memory_block_offline(struct memory_block *mem) -{ - unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); - unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; - unsigned long nr_vmemmap_pages = 0; - int ret; - - if (!mem->zone) - return -EINVAL; - - /* - * Unaccount before offlining, such that unpopulated zone and kthreads - * can properly be torn down in offline_pages(). - */ - if (mem->altmap) - nr_vmemmap_pages = mem->altmap->free; - - mem_hotplug_begin(); - if (nr_vmemmap_pages) - adjust_present_page_count(pfn_to_page(start_pfn), mem->group, - -nr_vmemmap_pages); - - ret = offline_pages(start_pfn + nr_vmemmap_pages, - nr_pages - nr_vmemmap_pages, mem->zone, mem->group); - if (ret) { - /* offline_pages() failed. Account back. */ - if (nr_vmemmap_pages) - adjust_present_page_count(pfn_to_page(start_pfn), - mem->group, nr_vmemmap_pages); - goto out; - } - - if (nr_vmemmap_pages) - mhp_deinit_memmap_on_memory(start_pfn, nr_vmemmap_pages); - - mem->zone = NULL; -out: - mem_hotplug_done(); - return ret; -} - /* * MEMORY_HOTPLUG depends on SPARSEMEM in mm/Kconfig, so it is * OK to have direct references to sparsemem variables in here. @@ -329,10 +220,10 @@ memory_block_action(struct memory_block *mem, unsigned long action) switch (action) { case MEM_ONLINE: - ret = memory_block_online(mem); + ret = mhp_block_online(mem); break; case MEM_OFFLINE: - ret = memory_block_offline(mem); + ret = mhp_block_offline(mem); break; default: WARN(1, KERN_WARNING "%s(%ld, %ld) unknown action: " @@ -1243,7 +1134,7 @@ void memblk_nr_poison_sub(unsigned long pfn, long i) atomic_long_sub(i, &mem->nr_hwpoison); } -static unsigned long memblk_nr_poison(struct memory_block *mem) +unsigned long memblk_nr_poison(struct memory_block *mem) { return atomic_long_read(&mem->nr_hwpoison); } diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index f2f16cdd73ee..8783a11da464 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -12,6 +12,7 @@ struct zone; struct pglist_data; struct mem_section; struct memory_group; +struct memory_block; struct resource; struct vmem_altmap; struct dev_pagemap; @@ -106,11 +107,7 @@ extern void adjust_present_page_count(struct page *page, struct memory_group *group, long nr_pages); /* VM interface that may be used by firmware interface */ -extern int mhp_init_memmap_on_memory(unsigned long pfn, unsigned long nr_pages, - struct zone *zone); -extern void mhp_deinit_memmap_on_memory(unsigned long pfn, unsigned long nr_pages); -extern int online_pages(unsigned long pfn, unsigned long nr_pages, - struct zone *zone, struct memory_group *group); +extern int mhp_block_online(struct memory_block *block); extern unsigned long __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn); @@ -261,8 +258,7 @@ static inline void pgdat_resize_init(struct pglist_data *pgdat) {} #ifdef CONFIG_MEMORY_HOTREMOVE extern void try_offline_node(int nid); -extern int offline_pages(unsigned long start_pfn, unsigned long nr_pages, - struct zone *zone, struct memory_group *group); +extern int mhp_block_offline(struct memory_block *block); extern int remove_memory(u64 start, u64 size); extern void __remove_memory(u64 start, u64 size); extern int offline_and_remove_memory(u64 start, u64 size); @@ -270,8 +266,7 @@ extern int offline_and_remove_memory(u64 start, u64 size); #else static inline void try_offline_node(int nid) {} -static inline int offline_pages(unsigned long start_pfn, unsigned long nr_pages, - struct zone *zone, struct memory_group *group) +static inline int mhp_block_offline(struct memory_block *block) { return -EINVAL; } diff --git a/include/linux/mm.h b/include/linux/mm.h index 6f959d8ca4b4..967605d95131 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4377,6 +4377,7 @@ static inline void num_poisoned_pages_sub(unsigned long pfn, long i) #if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_MEMORY_HOTPLUG) extern void memblk_nr_poison_inc(unsigned long pfn); extern void memblk_nr_poison_sub(unsigned long pfn, long i); +extern unsigned long memblk_nr_poison(struct memory_block *mem); #else static inline void memblk_nr_poison_inc(unsigned long pfn) { @@ -4385,6 +4386,11 @@ static inline void memblk_nr_poison_inc(unsigned long pfn) static inline void memblk_nr_poison_sub(unsigned long pfn, long i) { } + +static inline unsigned long memblk_nr_poison(struct memory_block *mem) +{ + return 0; +} #endif #ifndef arch_memory_failure diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index c8f492b5daf0..62d6bc8ea2dd 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1085,7 +1085,7 @@ void adjust_present_page_count(struct page *page, struct memory_group *group, group->present_kernel_pages += nr_pages; } -int mhp_init_memmap_on_memory(unsigned long pfn, unsigned long nr_pages, +static int mhp_init_memmap_on_memory(unsigned long pfn, unsigned long nr_pages, struct zone *zone) { unsigned long end_pfn = pfn + nr_pages; @@ -1116,7 +1116,7 @@ int mhp_init_memmap_on_memory(unsigned long pfn, unsigned long nr_pages, return ret; } -void mhp_deinit_memmap_on_memory(unsigned long pfn, unsigned long nr_pages) +static void mhp_deinit_memmap_on_memory(unsigned long pfn, unsigned long nr_pages) { unsigned long end_pfn = pfn + nr_pages; @@ -1139,7 +1139,7 @@ void mhp_deinit_memmap_on_memory(unsigned long pfn, unsigned long nr_pages) /* * Must be called with mem_hotplug_lock in write mode. */ -int online_pages(unsigned long pfn, unsigned long nr_pages, +static int online_pages(unsigned long pfn, unsigned long nr_pages, struct zone *zone, struct memory_group *group) { struct memory_notify mem_arg = { @@ -1254,6 +1254,74 @@ int online_pages(unsigned long pfn, unsigned long nr_pages, return ret; } +static int online_memory_block_pages(unsigned long start_pfn, unsigned long nr_pages, + unsigned long nr_vmemmap_pages, struct zone *zone, + struct memory_group *group) +{ + int ret; + + if (nr_vmemmap_pages) { + ret = mhp_init_memmap_on_memory(start_pfn, nr_vmemmap_pages, zone); + if (ret) + return ret; + } + + ret = online_pages(start_pfn + nr_vmemmap_pages, + nr_pages - nr_vmemmap_pages, zone, group); + if (ret) { + if (nr_vmemmap_pages) + mhp_deinit_memmap_on_memory(start_pfn, nr_vmemmap_pages); + return ret; + } + + /* + * Account once onlining succeeded. If the zone was unpopulated, it is + * now already properly populated. + */ + if (nr_vmemmap_pages) + adjust_present_page_count(pfn_to_page(start_pfn), group, + nr_vmemmap_pages); + + return ret; +} + +/* + * Must acquire mem_hotplug_lock in write mode. + */ +int mhp_block_online(struct memory_block *mem) +{ + unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); + unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; + unsigned long nr_vmemmap_pages = 0; + struct zone *zone; + int ret; + + if (memblk_nr_poison(mem)) + return -EHWPOISON; + + zone = zone_for_pfn_range(mem->online_type, mem->nid, mem->group, + start_pfn, nr_pages); + + /* + * Although vmemmap pages have a different lifecycle than the pages + * they describe (they remain until the memory is unplugged), doing + * their initialization and accounting at memory onlining/offlining + * stage helps to keep accounting easier to follow - e.g vmemmaps + * belong to the same zone as the memory they backed. + */ + if (mem->altmap) + nr_vmemmap_pages = mem->altmap->free; + + mem_hotplug_begin(); + ret = online_memory_block_pages(start_pfn, nr_pages, nr_vmemmap_pages, + zone, mem->group); + if (!ret) + mem->zone = zone; + mem_hotplug_done(); + + return ret; +} + /* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */ static pg_data_t *hotadd_init_pgdat(int nid) { @@ -1896,7 +1964,7 @@ static int count_system_ram_pages_cb(unsigned long start_pfn, /* * Must be called with mem_hotplug_lock in write mode. */ -int offline_pages(unsigned long start_pfn, unsigned long nr_pages, +static int offline_pages(unsigned long start_pfn, unsigned long nr_pages, struct zone *zone, struct memory_group *group) { unsigned long pfn, managed_pages, system_ram_pages = 0; @@ -2101,6 +2169,62 @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages, return ret; } +static int offline_memory_block_pages(unsigned long start_pfn, + unsigned long nr_pages, unsigned long nr_vmemmap_pages, + struct zone *zone, struct memory_group *group) +{ + int ret; + + if (nr_vmemmap_pages) + adjust_present_page_count(pfn_to_page(start_pfn), group, + -nr_vmemmap_pages); + + ret = offline_pages(start_pfn + nr_vmemmap_pages, + nr_pages - nr_vmemmap_pages, zone, group); + if (ret) { + /* offline_pages() failed. Account back. */ + if (nr_vmemmap_pages) + adjust_present_page_count(pfn_to_page(start_pfn), + group, nr_vmemmap_pages); + return ret; + } + + if (nr_vmemmap_pages) + mhp_deinit_memmap_on_memory(start_pfn, nr_vmemmap_pages); + + return ret; +} + +/* + * Must acquire mem_hotplug_lock in write mode. + */ +int mhp_block_offline(struct memory_block *mem) +{ + unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); + unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; + unsigned long nr_vmemmap_pages = 0; + int ret; + + if (!mem->zone) + return -EINVAL; + + /* + * Unaccount before offlining, such that unpopulated zone and kthreads + * can properly be torn down in offline_pages(). + */ + if (mem->altmap) + nr_vmemmap_pages = mem->altmap->free; + + mem_hotplug_begin(); + ret = offline_memory_block_pages(start_pfn, nr_pages, nr_vmemmap_pages, + mem->zone, mem->group); + if (!ret) + mem->zone = NULL; + mem_hotplug_done(); + + return ret; +} + static int check_memblock_offlined_cb(struct memory_block *mem, void *arg) { int *nid = arg; -- 2.47.1