The current codes only check if kasan is disabled for hw_tags mode. Here add the conditional checks for functional functions of generic mode and sw_tags mode. This is prepared for later adding kernel parameter kasan=on|off for all three kasan modes. Signed-off-by: Baoquan He --- mm/kasan/generic.c | 17 +++++++++++++++-- mm/kasan/init.c | 6 ++++++ mm/kasan/quarantine.c | 3 +++ mm/kasan/report.c | 4 +++- mm/kasan/shadow.c | 11 ++++++++++- mm/kasan/sw_tags.c | 3 +++ 6 files changed, 40 insertions(+), 4 deletions(-) diff --git a/mm/kasan/generic.c b/mm/kasan/generic.c index 2b8e73f5f6a7..aff822aa2bd6 100644 --- a/mm/kasan/generic.c +++ b/mm/kasan/generic.c @@ -214,12 +214,13 @@ bool kasan_byte_accessible(const void *addr) void kasan_cache_shrink(struct kmem_cache *cache) { - kasan_quarantine_remove_cache(cache); + if (kasan_enabled()) + kasan_quarantine_remove_cache(cache); } void kasan_cache_shutdown(struct kmem_cache *cache) { - if (!__kmem_cache_empty(cache)) + if (kasan_enabled() && !__kmem_cache_empty(cache)) kasan_quarantine_remove_cache(cache); } @@ -239,6 +240,9 @@ void __asan_register_globals(void *ptr, ssize_t size) int i; struct kasan_global *globals = ptr; + if (!kasan_enabled()) + return; + for (i = 0; i < size; i++) register_global(&globals[i]); } @@ -369,6 +373,9 @@ void kasan_cache_create(struct kmem_cache *cache, unsigned int *size, unsigned int rem_free_meta_size; unsigned int orig_alloc_meta_offset; + if (!kasan_enabled()) + return; + if (!kasan_requires_meta()) return; @@ -518,6 +525,9 @@ size_t kasan_metadata_size(struct kmem_cache *cache, bool in_object) { struct kasan_cache *info = &cache->kasan_info; + if (!kasan_enabled()) + return 0; + if (!kasan_requires_meta()) return 0; @@ -543,6 +553,9 @@ void kasan_record_aux_stack(void *addr) struct kasan_alloc_meta *alloc_meta; void *object; + if (!kasan_enabled()) + return; + if (is_kfence_address(addr) || !slab) return; diff --git a/mm/kasan/init.c b/mm/kasan/init.c index f084e7a5df1e..c78d77ed47bc 100644 --- a/mm/kasan/init.c +++ b/mm/kasan/init.c @@ -447,6 +447,9 @@ void kasan_remove_zero_shadow(void *start, unsigned long size) unsigned long addr, end, next; pgd_t *pgd; + if (!kasan_enabled()) + return; + addr = (unsigned long)kasan_mem_to_shadow(start); end = addr + (size >> KASAN_SHADOW_SCALE_SHIFT); @@ -482,6 +485,9 @@ int kasan_add_zero_shadow(void *start, unsigned long size) int ret; void *shadow_start, *shadow_end; + if (!kasan_enabled()) + return 0; + shadow_start = kasan_mem_to_shadow(start); shadow_end = shadow_start + (size >> KASAN_SHADOW_SCALE_SHIFT); diff --git a/mm/kasan/quarantine.c b/mm/kasan/quarantine.c index 6958aa713c67..a6dc2c3d8a15 100644 --- a/mm/kasan/quarantine.c +++ b/mm/kasan/quarantine.c @@ -405,6 +405,9 @@ static int __init kasan_cpu_quarantine_init(void) { int ret = 0; + if (!kasan_enabled()) + return 0; + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "mm/kasan:online", kasan_cpu_online, kasan_cpu_offline); if (ret < 0) diff --git a/mm/kasan/report.c b/mm/kasan/report.c index 62c01b4527eb..884357fa74ed 100644 --- a/mm/kasan/report.c +++ b/mm/kasan/report.c @@ -576,7 +576,9 @@ bool kasan_report(const void *addr, size_t size, bool is_write, unsigned long irq_flags; struct kasan_report_info info; - if (unlikely(report_suppressed_sw()) || unlikely(!report_enabled())) { + if (unlikely(report_suppressed_sw()) || + unlikely(!report_enabled()) || + !kasan_enabled()) { ret = false; goto out; } diff --git a/mm/kasan/shadow.c b/mm/kasan/shadow.c index 29a751a8a08d..f73a691421de 100644 --- a/mm/kasan/shadow.c +++ b/mm/kasan/shadow.c @@ -164,6 +164,8 @@ void kasan_unpoison(const void *addr, size_t size, bool init) { u8 tag = get_tag(addr); + if (!kasan_enabled()) + return; /* * Perform shadow offset calculation based on untagged address, as * some of the callers (e.g. kasan_unpoison_new_object) pass tagged @@ -277,7 +279,8 @@ static int __meminit kasan_mem_notifier(struct notifier_block *nb, static int __init kasan_memhotplug_init(void) { - hotplug_memory_notifier(kasan_mem_notifier, DEFAULT_CALLBACK_PRI); + if (kasan_enabled()) + hotplug_memory_notifier(kasan_mem_notifier, DEFAULT_CALLBACK_PRI); return 0; } @@ -658,6 +661,9 @@ int kasan_alloc_module_shadow(void *addr, size_t size, gfp_t gfp_mask) size_t shadow_size; unsigned long shadow_start; + if (!kasan_enabled()) + return 0; + shadow_start = (unsigned long)kasan_mem_to_shadow(addr); scaled_size = (size + KASAN_GRANULE_SIZE - 1) >> KASAN_SHADOW_SCALE_SHIFT; @@ -694,6 +700,9 @@ int kasan_alloc_module_shadow(void *addr, size_t size, gfp_t gfp_mask) void kasan_free_module_shadow(const struct vm_struct *vm) { + if (!kasan_enabled()) + return; + if (IS_ENABLED(CONFIG_UML)) return; diff --git a/mm/kasan/sw_tags.c b/mm/kasan/sw_tags.c index c75741a74602..6c1caec4261a 100644 --- a/mm/kasan/sw_tags.c +++ b/mm/kasan/sw_tags.c @@ -79,6 +79,9 @@ bool kasan_check_range(const void *addr, size_t size, bool write, u8 *shadow_first, *shadow_last, *shadow; void *untagged_addr; + if (!kasan_enabled()) + return true; + if (unlikely(size == 0)) return true; -- 2.41.0 This allows generic and sw_tags to be set in kernel cmdline too. When at it, rename 'kasan_arg' to 'kasan_arg_disabled' as a bool variable. And expose 'kasan_flag_enabled' to kasan common place too. This is prepared for later adding kernel parameter kasan=on|off for all three kasan modes. Signed-off-by: Baoquan He --- include/linux/kasan-enabled.h | 4 +++- mm/kasan/common.c | 20 ++++++++++++++++++-- mm/kasan/hw_tags.c | 28 ++-------------------------- 3 files changed, 23 insertions(+), 29 deletions(-) diff --git a/include/linux/kasan-enabled.h b/include/linux/kasan-enabled.h index 9eca967d8526..b05ec6329fbe 100644 --- a/include/linux/kasan-enabled.h +++ b/include/linux/kasan-enabled.h @@ -4,13 +4,15 @@ #include -#if defined(CONFIG_ARCH_DEFER_KASAN) || defined(CONFIG_KASAN_HW_TAGS) +extern bool kasan_arg_disabled; + /* * Global runtime flag for KASAN modes that need runtime control. * Used by ARCH_DEFER_KASAN architectures and HW_TAGS mode. */ DECLARE_STATIC_KEY_FALSE(kasan_flag_enabled); +#if defined(CONFIG_ARCH_DEFER_KASAN) || defined(CONFIG_KASAN_HW_TAGS) /* * Runtime control for shadow memory initialization or HW_TAGS mode. * Uses static key for architectures that need deferred KASAN or HW_TAGS. diff --git a/mm/kasan/common.c b/mm/kasan/common.c index 1d27f1bd260b..ac14956986ee 100644 --- a/mm/kasan/common.c +++ b/mm/kasan/common.c @@ -32,14 +32,30 @@ #include "kasan.h" #include "../slab.h" -#if defined(CONFIG_ARCH_DEFER_KASAN) || defined(CONFIG_KASAN_HW_TAGS) /* * Definition of the unified static key declared in kasan-enabled.h. * This provides consistent runtime enable/disable across KASAN modes. */ DEFINE_STATIC_KEY_FALSE(kasan_flag_enabled); EXPORT_SYMBOL_GPL(kasan_flag_enabled); -#endif + +bool kasan_arg_disabled __ro_after_init; +/* kasan=off/on */ +static int __init early_kasan_flag(char *arg) +{ + if (!arg) + return -EINVAL; + + if (!strcmp(arg, "off")) + kasan_arg_disabled = true; + else if (!strcmp(arg, "on")) + kasan_arg_disabled = false; + else + return -EINVAL; + + return 0; +} +early_param("kasan", early_kasan_flag); struct slab *kasan_addr_to_slab(const void *addr) { diff --git a/mm/kasan/hw_tags.c b/mm/kasan/hw_tags.c index 1c373cc4b3fa..709c91abc1b1 100644 --- a/mm/kasan/hw_tags.c +++ b/mm/kasan/hw_tags.c @@ -22,12 +22,6 @@ #include "kasan.h" -enum kasan_arg { - KASAN_ARG_DEFAULT, - KASAN_ARG_OFF, - KASAN_ARG_ON, -}; - enum kasan_arg_mode { KASAN_ARG_MODE_DEFAULT, KASAN_ARG_MODE_SYNC, @@ -41,7 +35,6 @@ enum kasan_arg_vmalloc { KASAN_ARG_VMALLOC_ON, }; -static enum kasan_arg kasan_arg __ro_after_init; static enum kasan_arg_mode kasan_arg_mode __ro_after_init; static enum kasan_arg_vmalloc kasan_arg_vmalloc __initdata; @@ -81,23 +74,6 @@ unsigned int kasan_page_alloc_sample_order = PAGE_ALLOC_SAMPLE_ORDER_DEFAULT; DEFINE_PER_CPU(long, kasan_page_alloc_skip); -/* kasan=off/on */ -static int __init early_kasan_flag(char *arg) -{ - if (!arg) - return -EINVAL; - - if (!strcmp(arg, "off")) - kasan_arg = KASAN_ARG_OFF; - else if (!strcmp(arg, "on")) - kasan_arg = KASAN_ARG_ON; - else - return -EINVAL; - - return 0; -} -early_param("kasan", early_kasan_flag); - /* kasan.mode=sync/async/asymm */ static int __init early_kasan_mode(char *arg) { @@ -222,7 +198,7 @@ void kasan_init_hw_tags_cpu(void) * When this function is called, kasan_flag_enabled is not yet * set by kasan_init_hw_tags(). Thus, check kasan_arg instead. */ - if (kasan_arg == KASAN_ARG_OFF) + if (kasan_arg_disabled) return; /* @@ -240,7 +216,7 @@ void __init kasan_init_hw_tags(void) return; /* If KASAN is disabled via command line, don't initialize it. */ - if (kasan_arg == KASAN_ARG_OFF) + if (kasan_arg_disabled) return; switch (kasan_arg_mode) { -- 2.41.0 Signed-off-by: Baoquan He --- mm/kasan/sw_tags.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mm/kasan/sw_tags.c b/mm/kasan/sw_tags.c index 6c1caec4261a..58edb68efc09 100644 --- a/mm/kasan/sw_tags.c +++ b/mm/kasan/sw_tags.c @@ -40,6 +40,9 @@ void __init kasan_init_sw_tags(void) { int cpu; + if (kasan_arg_disabled) + return; + for_each_possible_cpu(cpu) per_cpu(prng_state, cpu) = (u32)get_cycles(); -- 2.41.0 Here call jump_label_init() early in setup_arch() so that later kasan_init() can enable static key kasan_flag_enabled. Put jump_label_init() beofre parse_early_param() as other architectures do. Signed-off-by: Baoquan He Cc: linux-arm-kernel@lists.infradead.org --- arch/arm/kernel/setup.c | 6 ++++++ arch/arm/mm/kasan_init.c | 2 ++ 2 files changed, 8 insertions(+) diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index 0bfd66c7ada0..453a47a4c715 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -1135,6 +1135,12 @@ void __init setup_arch(char **cmdline_p) early_fixmap_init(); early_ioremap_init(); + /* + * Initialise the static keys early as they may be enabled by the + * kasan_init() or early parameters. + */ + jump_label_init(); + parse_early_param(); #ifdef CONFIG_MMU diff --git a/arch/arm/mm/kasan_init.c b/arch/arm/mm/kasan_init.c index c6625e808bf8..488916c7d29e 100644 --- a/arch/arm/mm/kasan_init.c +++ b/arch/arm/mm/kasan_init.c @@ -212,6 +212,8 @@ void __init kasan_init(void) phys_addr_t pa_start, pa_end; u64 i; + if (kasan_arg_disabled) + return; /* * We are going to perform proper setup of shadow memory. * -- 2.41.0 And also need skip kasan_populate_early_vm_area_shadow() if kasan is disabled. Signed-off-by: Baoquan He Cc: linux-arm-kernel@lists.infradead.org --- arch/arm64/mm/kasan_init.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/arm64/mm/kasan_init.c b/arch/arm64/mm/kasan_init.c index abeb81bf6ebd..eb49fdad4ef1 100644 --- a/arch/arm64/mm/kasan_init.c +++ b/arch/arm64/mm/kasan_init.c @@ -384,6 +384,9 @@ void __init kasan_populate_early_vm_area_shadow(void *start, unsigned long size) { unsigned long shadow_start, shadow_end; + if (!kasan_enabled()) + return; + if (!is_vmalloc_or_module_addr(start)) return; @@ -397,6 +400,9 @@ void __init kasan_populate_early_vm_area_shadow(void *start, unsigned long size) void __init kasan_init(void) { + if (kasan_arg_disabled) + return; + kasan_init_shadow(); kasan_init_depth(); kasan_init_generic(); -- 2.41.0 Signed-off-by: Baoquan He Cc: loongarch@lists.linux.dev --- arch/loongarch/mm/kasan_init.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/loongarch/mm/kasan_init.c b/arch/loongarch/mm/kasan_init.c index 170da98ad4f5..61bce6a4b4bb 100644 --- a/arch/loongarch/mm/kasan_init.c +++ b/arch/loongarch/mm/kasan_init.c @@ -265,6 +265,8 @@ void __init kasan_init(void) u64 i; phys_addr_t pa_start, pa_end; + if (kasan_arg_disabled) + return; /* * If PGDIR_SIZE is too large for cpu_vabits, KASAN_SHADOW_END will * overflow UINTPTR_MAX and then looks like a user space address. -- 2.41.0 This includes 32bit, book3s/64 and book3e/64. Signed-off-by: Baoquan He Cc: linuxppc-dev@lists.ozlabs.org --- arch/powerpc/mm/kasan/init_32.c | 5 ++++- arch/powerpc/mm/kasan/init_book3e_64.c | 3 +++ arch/powerpc/mm/kasan/init_book3s_64.c | 3 +++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/arch/powerpc/mm/kasan/init_32.c b/arch/powerpc/mm/kasan/init_32.c index 1d083597464f..b0651ff9d44d 100644 --- a/arch/powerpc/mm/kasan/init_32.c +++ b/arch/powerpc/mm/kasan/init_32.c @@ -141,6 +141,9 @@ void __init kasan_init(void) u64 i; int ret; + if (kasan_arg_disabled) + return; + for_each_mem_range(i, &base, &end) { phys_addr_t top = min(end, total_lowmem); @@ -170,7 +173,7 @@ void __init kasan_init(void) void __init kasan_late_init(void) { - if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) + if (IS_ENABLED(CONFIG_KASAN_VMALLOC) && kasan_enabled()) kasan_unmap_early_shadow_vmalloc(); } diff --git a/arch/powerpc/mm/kasan/init_book3e_64.c b/arch/powerpc/mm/kasan/init_book3e_64.c index 0d3a73d6d4b0..f75c1e38a011 100644 --- a/arch/powerpc/mm/kasan/init_book3e_64.c +++ b/arch/powerpc/mm/kasan/init_book3e_64.c @@ -111,6 +111,9 @@ void __init kasan_init(void) u64 i; pte_t zero_pte = pfn_pte(virt_to_pfn(kasan_early_shadow_page), PAGE_KERNEL_RO); + if (kasan_arg_disabled) + return; + for_each_mem_range(i, &start, &end) kasan_init_phys_region(phys_to_virt(start), phys_to_virt(end)); diff --git a/arch/powerpc/mm/kasan/init_book3s_64.c b/arch/powerpc/mm/kasan/init_book3s_64.c index dcafa641804c..8c6940e835d4 100644 --- a/arch/powerpc/mm/kasan/init_book3s_64.c +++ b/arch/powerpc/mm/kasan/init_book3s_64.c @@ -54,6 +54,9 @@ void __init kasan_init(void) u64 i; pte_t zero_pte = pfn_pte(virt_to_pfn(kasan_early_shadow_page), PAGE_KERNEL); + if (kasan_arg_disabled) + return; + if (!early_radix_enabled()) { pr_warn("KASAN not enabled as it requires radix!"); return; -- 2.41.0 Signed-off-by: Baoquan He Cc: linux-riscv@lists.infradead.org --- arch/riscv/mm/kasan_init.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/riscv/mm/kasan_init.c b/arch/riscv/mm/kasan_init.c index c4a2a9e5586e..aa464466e482 100644 --- a/arch/riscv/mm/kasan_init.c +++ b/arch/riscv/mm/kasan_init.c @@ -485,6 +485,9 @@ void __init kasan_init(void) phys_addr_t p_start, p_end; u64 i; + if (kasan_arg_disabled) + return; + create_tmp_mapping(); csr_write(CSR_SATP, PFN_DOWN(__pa(tmp_pg_dir)) | satp_mode); -- 2.41.0 Signed-off-by: Baoquan He Cc: x86@kernel.org --- arch/x86/mm/kasan_init_64.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/x86/mm/kasan_init_64.c b/arch/x86/mm/kasan_init_64.c index 998b6010d6d3..d642ad364904 100644 --- a/arch/x86/mm/kasan_init_64.c +++ b/arch/x86/mm/kasan_init_64.c @@ -343,6 +343,9 @@ void __init kasan_init(void) unsigned long shadow_cea_begin, shadow_cea_per_cpu_begin, shadow_cea_end; int i; + if (kasan_arg_disabled) + return; + memcpy(early_top_pgt, init_top_pgt, sizeof(early_top_pgt)); /* -- 2.41.0 Here call jump_label_init() early in setup_arch() so that later kasan_init() can enable static key kasan_flag_enabled. Put jump_label_init() beofre parse_early_param() as other architectures do. Signed-off-by: Baoquan He Cc: Chris Zankel Cc: Max Filippov --- arch/xtensa/kernel/setup.c | 1 + arch/xtensa/mm/kasan_init.c | 3 +++ 2 files changed, 4 insertions(+) diff --git a/arch/xtensa/kernel/setup.c b/arch/xtensa/kernel/setup.c index f72e280363be..aabeb23f41fa 100644 --- a/arch/xtensa/kernel/setup.c +++ b/arch/xtensa/kernel/setup.c @@ -352,6 +352,7 @@ void __init setup_arch(char **cmdline_p) mem_reserve(__pa(_SecondaryResetVector_text_start), __pa(_SecondaryResetVector_text_end)); #endif + jump_label_init(); parse_early_param(); bootmem_init(); kasan_init(); diff --git a/arch/xtensa/mm/kasan_init.c b/arch/xtensa/mm/kasan_init.c index 0524b9ed5e63..a78a85da1f0d 100644 --- a/arch/xtensa/mm/kasan_init.c +++ b/arch/xtensa/mm/kasan_init.c @@ -70,6 +70,9 @@ void __init kasan_init(void) { int i; + if (kasan_arg_disabled) + return; + BUILD_BUG_ON(KASAN_SHADOW_OFFSET != KASAN_SHADOW_START - (KASAN_START_VADDR >> KASAN_SHADOW_SCALE_SHIFT)); BUILD_BUG_ON(VMALLOC_START < KASAN_START_VADDR); -- 2.41.0 And also do the kasan_arg_disabled chekcing before kasan_flag_enabled enabling to make sure kernel parameter kasan=on|off has been parsed. Signed-off-by: Baoquan He Cc: linux-um@lists.infradead.org --- arch/um/kernel/mem.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/um/kernel/mem.c b/arch/um/kernel/mem.c index 39c4a7e21c6f..08cd012a6bb8 100644 --- a/arch/um/kernel/mem.c +++ b/arch/um/kernel/mem.c @@ -62,8 +62,11 @@ static unsigned long brk_end; void __init arch_mm_preinit(void) { +#ifdef CONFIG_KASAN /* Safe to call after jump_label_init(). Enables KASAN. */ - kasan_init_generic(); + if (!kasan_arg_disabled) + kasan_init_generic(); +#endif /* clear the zero-page */ memset(empty_zero_page, 0, PAGE_SIZE); -- 2.41.0 Now everything is ready, set kasan=off can disable kasan for all three modes. Signed-off-by: Baoquan He --- include/linux/kasan-enabled.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/kasan-enabled.h b/include/linux/kasan-enabled.h index b05ec6329fbe..b33c92cc6bd8 100644 --- a/include/linux/kasan-enabled.h +++ b/include/linux/kasan-enabled.h @@ -4,6 +4,7 @@ #include +#ifdef CONFIG_KASAN extern bool kasan_arg_disabled; /* @@ -12,7 +13,6 @@ extern bool kasan_arg_disabled; */ DECLARE_STATIC_KEY_FALSE(kasan_flag_enabled); -#if defined(CONFIG_ARCH_DEFER_KASAN) || defined(CONFIG_KASAN_HW_TAGS) /* * Runtime control for shadow memory initialization or HW_TAGS mode. * Uses static key for architectures that need deferred KASAN or HW_TAGS. @@ -30,7 +30,7 @@ static inline void kasan_enable(void) /* For architectures that can enable KASAN early, use compile-time check. */ static __always_inline bool kasan_enabled(void) { - return IS_ENABLED(CONFIG_KASAN); + return false; } static inline void kasan_enable(void) {} -- 2.41.0