Acked-by: Artem Kuzin Acked-by: Alexander Grubnikov Acked-by: Ilya Hanov Acked-by: Denis Darvish Signed-off-by: Nikita Panov --- include/linux/moduleloader.h | 4 ++++ kernel/module/main.c | 17 ++++++++++++++++ kernel/module/strict_rwx.c | 12 +++++------ mm/execmem.c | 39 +++++++++++++++++++++++++++++++----- 4 files changed, 61 insertions(+), 11 deletions(-) diff --git a/include/linux/moduleloader.h b/include/linux/moduleloader.h index e395461d59e5..1d100dfc8251 100644 --- a/include/linux/moduleloader.h +++ b/include/linux/moduleloader.h @@ -25,6 +25,10 @@ int module_frob_arch_sections(Elf_Ehdr *hdr, /* Additional bytes needed by arch in front of individual sections */ unsigned int arch_mod_section_prepend(struct module *mod, unsigned int section); +#ifdef CONFIG_KERNEL_REPLICATION +void module_replicate(void *ptr); +#endif /* CONFIG_KERNEL_REPLICATION */ + /* Determines if the section name is an init section (that is only used during * module loading). */ diff --git a/kernel/module/main.c b/kernel/module/main.c index d0e1e0bd2ad0..d5c8f041be48 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -60,6 +60,7 @@ #include #include #include +#include #include #include "internal.h" @@ -1338,6 +1339,18 @@ void __weak module_arch_freeing_init(struct module *mod) { } +#ifdef CONFIG_KERNEL_REPLICATION +static int sections_to_replicate[] = {MOD_TEXT, MOD_RODATA}; + +static void module_replicate_sections(struct module *mod) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(sections_to_replicate); i++) + module_replicate(mod->mem[sections_to_replicate[i]].base); +} +#endif /* CONFIG_KERNEL_REPLICATION */ + static int module_memory_alloc(struct module *mod, enum mod_mem_type type) { unsigned int size = PAGE_ALIGN(mod->mem[type].size); @@ -3330,6 +3343,10 @@ static int complete_formation(struct module *mod, struct load_info *info) module_bug_finalize(info->hdr, info->sechdrs, mod); module_cfi_finalize(info->hdr, info->sechdrs, mod); +#ifdef CONFIG_KERNEL_REPLICATION + module_replicate_sections(mod); +#endif + err = module_enable_rodata_ro(mod); if (err) goto out_strict_rwx; diff --git a/kernel/module/strict_rwx.c b/kernel/module/strict_rwx.c index 8fd438529fbc..07ad1af69eaa 100644 --- a/kernel/module/strict_rwx.c +++ b/kernel/module/strict_rwx.c @@ -39,9 +39,9 @@ int module_enable_text_rox(const struct module *mod) if (mem->is_rox) ret = execmem_restore_rox(mem->base, mem->size); else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) - ret = module_set_memory(mod, type, set_memory_rox); + ret = module_set_memory(mod, type, numa_set_memory_rox); else - ret = module_set_memory(mod, type, set_memory_x); + ret = module_set_memory(mod, type, numa_set_memory_x); if (ret) return ret; } @@ -55,10 +55,10 @@ int module_enable_rodata_ro(const struct module *mod) if (!IS_ENABLED(CONFIG_STRICT_MODULE_RWX) || !rodata_enabled) return 0; - ret = module_set_memory(mod, MOD_RODATA, set_memory_ro); + ret = module_set_memory(mod, MOD_RODATA, numa_set_memory_ro); if (ret) return ret; - ret = module_set_memory(mod, MOD_INIT_RODATA, set_memory_ro); + ret = module_set_memory(mod, MOD_INIT_RODATA, numa_set_memory_ro); if (ret) return ret; @@ -70,7 +70,7 @@ int module_enable_rodata_ro_after_init(const struct module *mod) if (!IS_ENABLED(CONFIG_STRICT_MODULE_RWX) || !rodata_enabled) return 0; - return module_set_memory(mod, MOD_RO_AFTER_INIT, set_memory_ro); + return module_set_memory(mod, MOD_RO_AFTER_INIT, numa_set_memory_ro); } int module_enable_data_nx(const struct module *mod) @@ -79,7 +79,7 @@ int module_enable_data_nx(const struct module *mod) return 0; for_class_mod_mem_type(type, data) { - int ret = module_set_memory(mod, type, set_memory_nx); + int ret = module_set_memory(mod, type, numa_set_memory_nx); if (ret) return ret; diff --git a/mm/execmem.c b/mm/execmem.c index 74a178a87e75..ea4b15c8a788 100644 --- a/mm/execmem.c +++ b/mm/execmem.c @@ -16,6 +16,7 @@ #include #include #include +#include #include @@ -26,8 +27,9 @@ static struct execmem_info *execmem_info __ro_after_init; static struct execmem_info default_execmem_info __ro_after_init; #ifdef CONFIG_MMU -static void *execmem_vmalloc(struct execmem_range *range, size_t size, - pgprot_t pgprot, unsigned long vm_flags) + +static void *execmem_vmalloc_node(struct execmem_range *range, size_t size, + pgprot_t pgprot, unsigned long vm_flags, int node) { bool kasan = range->flags & EXECMEM_KASAN_SHADOW; gfp_t gfp_flags = GFP_KERNEL | __GFP_NOWARN; @@ -40,13 +42,13 @@ static void *execmem_vmalloc(struct execmem_range *range, size_t size, vm_flags |= VM_DEFER_KMEMLEAK; p = __vmalloc_node_range(size, align, start, end, gfp_flags, - pgprot, vm_flags, NUMA_NO_NODE, + pgprot, vm_flags, node, __builtin_return_address(0)); if (!p && range->fallback_start) { start = range->fallback_start; end = range->fallback_end; p = __vmalloc_node_range(size, align, start, end, gfp_flags, - pgprot, vm_flags, NUMA_NO_NODE, + pgprot, vm_flags, node, __builtin_return_address(0)); } @@ -63,6 +65,26 @@ static void *execmem_vmalloc(struct execmem_range *range, size_t size, return p; } +#ifdef CONFIG_KERNEL_REPLICATION +static void *execmem_vmalloc_type(struct execmem_range *range, size_t size, + pgprot_t pgprot, unsigned long vm_flags, enum execmem_type type) +{ + if (is_text_replicated() && (type == EXECMEM_MODULE_TEXT || type == EXECMEM_MODULE_DATA)) + /* Need to specify some numa node id for correct allocation and further replication */ + return execmem_vmalloc_node(range, size, pgprot, + vm_flags | VM_NUMA_SHARED, numa_node_id()); + else + return execmem_vmalloc_node(range, size, pgprot, + vm_flags, NUMA_NO_NODE); +} +#else +static void *execmem_vmalloc_type(struct execmem_range *range, size_t size, + pgprot_t pgprot, unsigned long vm_flags, enum execmem_type type) +{ + return execmem_vmalloc_node(range, size, pgprot, vm_flags, NUMA_NO_NODE); +} +#endif + struct vm_struct *execmem_vmap(size_t size) { struct execmem_range *range = &execmem_info->ranges[EXECMEM_MODULE_DATA]; @@ -87,6 +109,13 @@ static void *execmem_vmalloc(struct execmem_range *range, size_t size, #endif /* CONFIG_MMU */ #ifdef CONFIG_ARCH_HAS_EXECMEM_ROX + +static void *execmem_vmalloc(struct execmem_range *range, size_t size, + pgprot_t pgprot, unsigned long vm_flags) +{ + return execmem_vmalloc_node(range, size, pgprot, vm_flags, NUMA_NO_NODE); +} + struct execmem_cache { struct mutex mutex; struct maple_tree busy_areas; @@ -475,7 +504,7 @@ void *execmem_alloc(enum execmem_type type, size_t size) if (use_cache) p = execmem_cache_alloc(range, size); else - p = execmem_vmalloc(range, size, pgprot, vm_flags); + p = execmem_vmalloc_type(range, size, pgprot, vm_flags, type); return kasan_reset_tag(p); } -- 2.34.1