The bitmap an earlier kernel filled in rides the EFI system table into this one, but nothing reads it back until now. Add efi_offline_poisoned_memory(), which vets the table header, copies the geometry out of it, and hands every frame of every set unit to hwpoison_boot_pfn(). The bitmap runs to megabytes on a large machine, so it is mapped and walked a page at a time rather than in one go. Call it through hwpoison_init_boot() from mm_core_init() right after memblock_free_all(), the first point at which the recorded frames have struct pages. They end up in the state a frame poisoned by this kernel would be in, so the placement check that already understands PG_hwpoison covers them too. Suggested-by: Kiryl Shutsemau Signed-off-by: Breno Leitao --- drivers/firmware/efi/poison.c | 113 +++++++++++++++++++++++++++++++++++++++++- include/linux/efi.h | 2 + include/linux/mm.h | 5 ++ mm/memory-failure.c | 6 +++ mm/mm_init.c | 1 + 5 files changed, 126 insertions(+), 1 deletion(-) diff --git a/drivers/firmware/efi/poison.c b/drivers/firmware/efi/poison.c index d6855e712832c..179843277457f 100644 --- a/drivers/firmware/efi/poison.c +++ b/drivers/firmware/efi/poison.c @@ -9,14 +9,20 @@ #define pr_fmt(fmt) "efi: " fmt -#include #include #include #include #include +#include #include #include +struct efi_poison_geometry { + u64 table; /* phys address of the table */ + u64 unit_size; + u64 bitmap_size; +}; + static struct linux_efi_poisoned_memory *efi_poison __ro_after_init; static u64 efi_poison_nbits __ro_after_init; @@ -124,3 +130,108 @@ void efi_hwpoison_record_pfn(unsigned long pfn) set_bit(unit, efi_poison->bitmap); } + +static bool __init efi_poison_read_geometry(u64 ppm, + struct efi_poison_geometry *g) +{ + struct linux_efi_poisoned_memory *pm; + bool valid; + + pm = early_memremap(ppm, sizeof(*pm)); + if (!pm) { + pr_warn("Could not map poisoned-memory table\n"); + return false; + } + + valid = efi_poison_table_valid(pm); + if (valid) { + g->table = ppm; + g->unit_size = pm->unit_size; + g->bitmap_size = efi_poison_usable_size(pm); + } else { + /* Keep the runtime side off a table this pass rejected. */ + efi.poisoned_memory = EFI_INVALID_TABLE_ADDR; + } + + early_memunmap(pm, sizeof(*pm)); + + return valid; +} + +static unsigned long __init +efi_poison_offline_unit(const struct efi_poison_geometry *g, u64 unit) +{ + unsigned long pfn = PHYS_PFN(unit * g->unit_size); + unsigned long i, nr_pages = 0; + + for (i = 0; i < g->unit_size >> PAGE_SHIFT; i++) + nr_pages += hwpoison_boot_pfn(pfn + i); + + return nr_pages; +} + +static long __init efi_poison_walk_chunk(const struct efi_poison_geometry *g, + u64 off, unsigned long *nr_units) +{ + u64 chunk = min_t(u64, PAGE_SIZE, g->bitmap_size - off); + unsigned long bit, nbits = chunk * BITS_PER_BYTE; + unsigned long *map, nr_pages = 0; + + map = early_memremap(g->table + offsetof(struct linux_efi_poisoned_memory, + bitmap) + off, chunk); + if (!map) + return -1; + + for_each_set_bit(bit, map, nbits) { + nr_pages += efi_poison_offline_unit(g, off * BITS_PER_BYTE + bit); + (*nr_units)++; + } + + early_memunmap(map, chunk); + + return nr_pages; +} + +static long __init efi_poison_walk(const struct efi_poison_geometry *g, + unsigned long *nr_units) +{ + unsigned long nr_pages = 0; + u64 off; + + for (off = 0; off < g->bitmap_size; off += PAGE_SIZE) { + long nr = efi_poison_walk_chunk(g, off, nr_units); + + if (nr < 0) { + pr_warn("Could not map poisoned-memory bitmap\n"); + return -1; + } + nr_pages += nr; + } + + return nr_pages; +} + +void __init efi_offline_poisoned_memory(void) +{ + struct efi_poison_geometry g; + unsigned long nr_units = 0; + long nr_pages, expected; + + if (efi.poisoned_memory == EFI_INVALID_TABLE_ADDR) + return; + + if (!efi_poison_read_geometry(efi.poisoned_memory, &g)) + return; + + nr_pages = efi_poison_walk(&g, &nr_units); + if (nr_pages < 0) + return; + + if (nr_pages) + pr_info("poisoned %ld page(s) inherited across kexec\n", nr_pages); + + expected = nr_units * (g.unit_size >> PAGE_SHIFT); + if (nr_pages < expected) + pr_warn("%ld inherited poisoned page(s) could not be taken out of use\n", + expected - nr_pages); +} diff --git a/include/linux/efi.h b/include/linux/efi.h index d579d75372248..03ba40ff70e7e 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -1285,8 +1285,10 @@ struct linux_efi_poisoned_memory { #define EFI_POISON_UNIT_SIZE SZ_2M #ifdef CONFIG_EFI_POISONED_MEMORY +void efi_offline_poisoned_memory(void); void efi_hwpoison_record_pfn(unsigned long pfn); #else +static inline void efi_offline_poisoned_memory(void) { } static inline void efi_hwpoison_record_pfn(unsigned long pfn) { } #endif diff --git a/include/linux/mm.h b/include/linux/mm.h index 52a00a0e090c7..092220943531e 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -5104,12 +5104,17 @@ extern const struct attribute_group memory_failure_attr_group; extern void memory_failure_queue(unsigned long pfn, int flags); void num_poisoned_pages_inc(unsigned long pfn); void num_poisoned_pages_sub(unsigned long pfn, long i); +void __init hwpoison_init_boot(void); bool __init hwpoison_boot_pfn(unsigned long pfn); #else static inline void memory_failure_queue(unsigned long pfn, int flags) { } +static inline void hwpoison_init_boot(void) +{ +} + static inline void num_poisoned_pages_inc(unsigned long pfn) { } diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 713fb2f8e0332..1d6a472267bdb 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -121,6 +121,12 @@ bool __init hwpoison_boot_pfn(unsigned long pfn) return true; } +/* The EFI table is the only source of inherited poison today. */ +void __init hwpoison_init_boot(void) +{ + efi_offline_poisoned_memory(); +} + /** * MF_ATTR_RO - Create sysfs entry for each memory failure statistics. * @_name: name of the file in the per NUMA sysfs directory. diff --git a/mm/mm_init.c b/mm/mm_init.c index ddda9d6837f32..f94fa221da0b8 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -2667,6 +2667,7 @@ void __init mm_core_init(void) kho_memory_init(); memblock_free_all(); + hwpoison_init_boot(); mem_init(); kmem_cache_init(); /* -- 2.53.0-Meta