Acked-by: Artem Kuzin Acked-by: Alexander Grubnikov Acked-by: Ilya Hanov Acked-by: Denis Darvish Signed-off-by: Nikita Panov --- arch/arm64/kernel/alternative.c | 33 +++++++++++- arch/arm64/kernel/patching.c | 96 +++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kernel/alternative.c b/arch/arm64/kernel/alternative.c index f5ec7e7c1d3f..e85644d64bbd 100644 --- a/arch/arm64/kernel/alternative.c +++ b/arch/arm64/kernel/alternative.c @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -139,6 +140,36 @@ static noinstr void clean_dcache_range_nopatch(u64 start, u64 end) } while (cur += d_size, cur < end); } +#ifdef CONFIG_KERNEL_REPLICATION +static void __write_alternatives(struct alt_instr *alt, + alternative_cb_t alt_cb, + __le32 *origptr, __le32 *updptr, + int nr_inst) +{ + if (is_text_replicated() && is_kernel_text((unsigned long)origptr)) { + int nid; + + for_each_memory_node(nid) { + __le32 *ptr = numa_get_replica(origptr, nid); + + alt_cb(alt, origptr, ptr, nr_inst); + clean_dcache_range_nopatch((u64)ptr, + (u64)(ptr + nr_inst)); + } + } else { + alt_cb(alt, origptr, updptr, nr_inst); + } +} +#else +static void __write_alternatives(struct alt_instr *alt, + alternative_cb_t alt_cb, + __le32 *origptr, __le32 *updptr, + int nr_inst) +{ + alt_cb(alt, origptr, updptr, nr_inst); +} +#endif /* CONFIG_KERNEL_REPLICATION */ + static int __apply_alternatives(const struct alt_region *region, bool is_module, unsigned long *cpucap_mask) @@ -174,7 +205,7 @@ static int __apply_alternatives(const struct alt_region *region, alt_cb = patch_alternative; } - alt_cb(alt, origptr, updptr, nr_inst); + __write_alternatives(alt, alt_cb, origptr, updptr, nr_inst); if (!is_module) { clean_dcache_range_nopatch((u64)origptr, diff --git a/arch/arm64/kernel/patching.c b/arch/arm64/kernel/patching.c index 09f019c6547a..8bac14198459 100644 --- a/arch/arm64/kernel/patching.c +++ b/arch/arm64/kernel/patching.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -15,6 +16,7 @@ static DEFINE_RAW_SPINLOCK(patch_lock); +#ifndef CONFIG_KERNEL_REPLICATION static bool is_exit_text(unsigned long addr) { /* discarded with init text/data */ @@ -42,6 +44,17 @@ static void __kprobes *patch_map(void *addr, int fixmap) return (void *)set_fixmap_offset(fixmap, phys); } +#else +static void __kprobes *patch_map(void *addr, int fixmap, int nid) +{ + struct page *page; + + page = walk_to_page_node(nid, addr); + BUG_ON(!page); + + return (void *)set_fixmap_offset(fixmap, page_to_phys(page) + offset_in_page(addr)); +} +#endif /* CONFIG_KERNEL_REPLICATION */ static void __kprobes patch_unmap(int fixmap) { @@ -63,6 +76,28 @@ int __kprobes aarch64_insn_read(void *addr, u32 *insnp) return ret; } +#ifdef CONFIG_KERNEL_REPLICATION +static int __kprobes __aarch64_insn_write(void *addr, __le32 insn) +{ + int nid; + void *waddr = addr; + unsigned long flags = 0; + int ret; + + raw_spin_lock_irqsave(&patch_lock, flags); + for_each_memory_node(nid) { + waddr = patch_map(addr, FIX_TEXT_POKE0, nid); + ret = copy_to_kernel_nofault(waddr, &insn, AARCH64_INSN_SIZE); + patch_unmap(FIX_TEXT_POKE0); + + if (ret || !is_text_replicated()) + break; + } + raw_spin_unlock_irqrestore(&patch_lock, flags); + + return ret; +} +#else static int __kprobes __aarch64_insn_write(void *addr, __le32 insn) { void *waddr = addr; @@ -79,12 +114,37 @@ static int __kprobes __aarch64_insn_write(void *addr, __le32 insn) return ret; } +#endif /* CONFIG_KERNEL_REPLICATION */ int __kprobes aarch64_insn_write(void *addr, u32 insn) { return __aarch64_insn_write(addr, cpu_to_le32(insn)); } +#ifdef CONFIG_KERNEL_REPLICATION +noinstr int aarch64_insn_write_literal_u64(void *addr, u64 val) +{ + int nid; + u64 *waddr; + unsigned long flags; + int ret; + + raw_spin_lock_irqsave(&patch_lock, flags); + for_each_memory_node(nid) { + waddr = patch_map(addr, FIX_TEXT_POKE0, nid); + + ret = copy_to_kernel_nofault(waddr, &val, sizeof(val)); + + patch_unmap(FIX_TEXT_POKE0); + + if (ret || !is_text_replicated()) + break; + } + raw_spin_unlock_irqrestore(&patch_lock, flags); + + return ret; +} +#else noinstr int aarch64_insn_write_literal_u64(void *addr, u64 val) { u64 *waddr; @@ -101,9 +161,44 @@ noinstr int aarch64_insn_write_literal_u64(void *addr, u64 val) return ret; } +#endif /* CONFIG_KERNEL_REPLICATION */ typedef void text_poke_f(void *dst, void *src, size_t patched, size_t len); +#ifdef CONFIG_KERNEL_REPLICATION +static void *__text_poke(text_poke_f func, void *addr, void *src, size_t len) +{ + unsigned long flags; + size_t patched = 0; + size_t size; + void *waddr; + void *ptr; + + raw_spin_lock_irqsave(&patch_lock, flags); + + while (patched < len) { + int nid; + + ptr = addr + patched; + size = min(PAGE_SIZE - offset_in_page(ptr), len - patched); + + for_each_memory_node(nid) { + waddr = patch_map(ptr, FIX_TEXT_POKE0, nid); + func(waddr, src, patched, size); + patch_unmap(FIX_TEXT_POKE0); + + if (!is_text_replicated()) + break; + } + patched += size; + } + raw_spin_unlock_irqrestore(&patch_lock, flags); + + flush_icache_range((uintptr_t)addr, (uintptr_t)addr + len); + + return addr; +} +#else static void *__text_poke(text_poke_f func, void *addr, void *src, size_t len) { unsigned long flags; @@ -130,6 +225,7 @@ static void *__text_poke(text_poke_f func, void *addr, void *src, size_t len) return addr; } +#endif static void text_poke_memcpy(void *dst, void *src, size_t patched, size_t len) { -- 2.34.1