| Seq | Timestamp | Type | Name | Duration |
|---|---|---|---|---|
| 0/0 | 2026/07/15 22:40 | flow | patch-triage |
1mArgs:null Results: {
"EnableConfigs": [
"HAVE_LOCAL_PER_CPU_MAP",
"ARCH_HAS_PMD_POPULATE_SYNC",
"ARCH_HAS_PUD_POPULATE_SYNC",
"NEED_PER_CPU_PAGE_FIRST_CHUNK"
],
"FocusSymbols": [
"pcpu_get_local_vm_area",
"map_local_percpu_first_chunk",
"setup_percpu_pgd",
"arch_sync_kernel_mappings"
],
"Reasoning": "The patch series introduces local per-cpu mapping support for ARM64, modifying core kernel logic for percpu memory management and page tables. This is a functional change that should be fuzzed.",
"WorthFuzzing": true
} |
| 1/1 | 2026/07/15 22:40 | action | read-patch-diff |
0mArgs:null Results: {
"PatchDiff": "commit 2baccb3511fcadd0fa12b52c7793f8cf0180006d\nAuthor: syz-cluster \u003ctriage@syzkaller.com\u003e\nDate: Wed Jul 15 22:40:22 2026 +0000\n\n syz-cluster: applied patch under review\n\ndiff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig\nindex b3afe0688919bd..95ffd151187f24 100644\n--- a/arch/arm64/Kconfig\n+++ b/arch/arm64/Kconfig\n@@ -41,9 +41,11 @@ config ARM64\n \tselect ARCH_HAS_NMI_SAFE_THIS_CPU_OPS\n \tselect ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE\n \tselect ARCH_HAS_NONLEAF_PMD_YOUNG if ARM64_HAFT\n+\tselect ARCH_HAS_PMD_POPULATE_SYNC\n \tselect ARCH_HAS_PREEMPT_LAZY\n \tselect ARCH_HAS_PTDUMP\n \tselect ARCH_HAS_PTE_SPECIAL\n+\tselect ARCH_HAS_PUD_POPULATE_SYNC\n \tselect ARCH_HAS_HW_PTE_YOUNG\n \tselect ARCH_HAS_SETUP_DMA_OPS\n \tselect ARCH_HAS_SET_DIRECT_MAP\n@@ -224,6 +226,7 @@ config ARM64\n \tselect MMU_GATHER_RCU_TABLE_FREE\n \tselect HAVE_RSEQ\n \tselect HAVE_RUST if RUSTC_SUPPORTS_ARM64\n+\tselect HAVE_SETUP_PER_CPU_AREA\n \tselect HAVE_STACKPROTECTOR\n \tselect HAVE_STATIC_CALL if CFI\n \tselect HAVE_SYSCALL_TRACEPOINTS\n@@ -239,6 +242,7 @@ config ARM64\n \tselect LOCK_MM_AND_FIND_VMA\n \tselect MODULES_USE_ELF_RELA\n \tselect NEED_DMA_MAP_STATE\n+\tselect NEED_PER_CPU_PAGE_FIRST_CHUNK\n \tselect NEED_SG_DMA_LENGTH\n \tselect OF\n \tselect OF_EARLY_FLATTREE\n@@ -411,6 +415,11 @@ config KASAN_SHADOW_OFFSET\n config UNWIND_TABLES\n \tbool\n \n+config HAVE_LOCAL_PER_CPU_MAP\n+\tdef_bool y\n+\tdepends on (ARM64_VA_BITS_48 \u0026\u0026 (ARM64_4K_PAGES || ARM64_64K_PAGES)) || \\\n+\t\t (ARM64_VA_BITS_47 \u0026\u0026 ARM64_16K_PAGES)\n+\n source \"arch/arm64/Kconfig.platforms\"\n \n menu \"Kernel Features\"\n@@ -1581,9 +1590,6 @@ config NUMA\n \tbool \"NUMA Memory Allocation and Scheduler Support\"\n \tselect GENERIC_ARCH_NUMA\n \tselect OF_NUMA\n-\tselect HAVE_SETUP_PER_CPU_AREA\n-\tselect NEED_PER_CPU_EMBED_FIRST_CHUNK\n-\tselect NEED_PER_CPU_PAGE_FIRST_CHUNK\n \tselect USE_PERCPU_NUMA_NODE_ID\n \thelp\n \t Enable NUMA (Non-Uniform Memory Access) support.\ndiff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h\nindex 5e1211c540ab23..aa0818f7ec58c5 100644\n--- a/arch/arm64/include/asm/mmu.h\n+++ b/arch/arm64/include/asm/mmu.h\n@@ -63,6 +63,7 @@ static inline bool arm64_kernel_unmapped_at_el0(void)\n extern void arm64_memblock_init(void);\n extern void paging_init(void);\n extern void bootmem_init(void);\n+extern void setup_percpu_pgd(void);\n extern void create_mapping_noalloc(phys_addr_t phys, unsigned long virt,\n \t\t\t\t phys_addr_t size, pgprot_t prot);\n extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,\n@@ -72,6 +73,10 @@ extern void *fixmap_remap_fdt(phys_addr_t dt_phys, int *size, pgprot_t prot);\n extern void mark_linear_text_alias_ro(void);\n extern int split_kernel_leaf_mapping(unsigned long start, unsigned long end);\n extern void linear_map_maybe_split_to_ptes(void);\n+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP\n+extern void map_local_percpu_first_chunk(unsigned int cpu, unsigned long virt,\n+\t\t\t\tstruct page **pages, unsigned int nr);\n+#endif\n \n /*\n * This check is triggered during the early boot before the cpufeature\ndiff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h\nindex 803b6875815275..0e991b98ed9eaf 100644\n--- a/arch/arm64/include/asm/mmu_context.h\n+++ b/arch/arm64/include/asm/mmu_context.h\n@@ -27,6 +27,7 @@\n #include \u003casm/tlbflush.h\u003e\n \n extern bool rodata_full;\n+extern pgd_t *percpu_pgd[NR_CPUS];\n \n static inline void contextidr_thread_switch(struct task_struct *next)\n {\n@@ -138,7 +139,13 @@ void __cpu_replace_ttbr1(pgd_t *pgdp, bool cnp);\n \n static inline void cpu_enable_swapper_cnp(void)\n {\n-\t__cpu_replace_ttbr1(lm_alias(swapper_pg_dir), true);\n+\tunsigned int cpu = smp_processor_id();\n+\tpgd_t *ttbr1 = percpu_pgd[cpu];\n+\n+\tif (cpu == 0)\n+\t\tttbr1 = lm_alias(swapper_pg_dir);\n+\n+\t__cpu_replace_ttbr1(ttbr1, false);\n }\n \n static inline void cpu_replace_ttbr1(pgd_t *pgdp)\ndiff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h\nindex b57b2bb0096774..03d149f9177c0e 100644\n--- a/arch/arm64/include/asm/percpu.h\n+++ b/arch/arm64/include/asm/percpu.h\n@@ -142,6 +142,27 @@ PERCPU_RET_OP(add, add, ldadd)\n #undef PERCPU_OP\n #undef PERCPU_RET_OP\n \n+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP\n+extern unsigned long __per_cpu_local_off;\n+\n+#define local_cpu_ptr(ptr)\t\t\t\t\t\t\\\n+({\t\t\t\t\t\t\t\t\t\\\n+\t__verify_pcpu_ptr(ptr);\t\t\t\t\t\t\\\n+\tSHIFT_PERCPU_PTR(ptr, __per_cpu_local_off);\t\t\t\\\n+})\n+\n+#define _pcp_protect(op, pcp, ...)\t\t\t\t\t\\\n+({\t\t\t\t\t\t\t\t\t\\\n+\top(local_cpu_ptr(\u0026(pcp)), __VA_ARGS__);\t\t\t\t\\\n+})\n+\n+#define _pcp_protect_return(op, pcp, args...)\t\t\t\t\\\n+({\t\t\t\t\t\t\t\t\t\\\n+\ttypeof(pcp) __retval;\t\t\t\t\t\t\\\n+\t__retval = (typeof(pcp))op(local_cpu_ptr(\u0026(pcp)), ##args);\t\\\n+\t__retval;\t\t\t\t\t\t\t\\\n+})\n+#else\n /*\n * It would be nice to avoid the conditional call into the scheduler when\n * re-enabling preemption for preemptible kernels, but doing that in a way\n@@ -152,7 +173,6 @@ PERCPU_RET_OP(add, add, ldadd)\n * preemption point when TIF_NEED_RESCHED gets set while preemption is\n * disabled.\n */\n-\n #define _pcp_protect(op, pcp, ...)\t\t\t\t\t\\\n ({\t\t\t\t\t\t\t\t\t\\\n \tpreempt_disable_notrace();\t\t\t\t\t\\\n@@ -168,6 +188,7 @@ PERCPU_RET_OP(add, add, ldadd)\n \tpreempt_enable_notrace();\t\t\t\t\t\\\n \t__retval;\t\t\t\t\t\t\t\\\n })\n+#endif\n \n #define this_cpu_read_1(pcp)\t\t\\\n \t_pcp_protect_return(__percpu_read_8, pcp)\n@@ -243,6 +264,19 @@ PERCPU_RET_OP(add, add, ldadd)\n \n #define this_cpu_cmpxchg64(pcp, o, n)\tthis_cpu_cmpxchg_8(pcp, o, n)\n \n+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP\n+#define this_cpu_cmpxchg128(pcp, o, n)\t\t\t\t\t\\\n+({\t\t\t\t\t\t\t\t\t\\\n+\ttypedef typeof(pcp) pcp_op_T__;\t\t\t\t\t\\\n+\tu128 old__, new__, ret__;\t\t\t\t\t\\\n+\tpcp_op_T__ *ptr__;\t\t\t\t\t\t\\\n+\told__ = o;\t\t\t\t\t\t\t\\\n+\tnew__ = n;\t\t\t\t\t\t\t\\\n+\tptr__ = local_cpu_ptr(\u0026(pcp));\t\t\t\t\t\\\n+\tret__ = cmpxchg128_local((void *)ptr__, old__, new__);\t\t\\\n+\tret__;\t\t\t\t\t\t\t\t\\\n+})\n+#else\n #define this_cpu_cmpxchg128(pcp, o, n)\t\t\t\t\t\\\n ({\t\t\t\t\t\t\t\t\t\\\n \ttypedef typeof(pcp) pcp_op_T__;\t\t\t\t\t\\\n@@ -256,6 +290,7 @@ PERCPU_RET_OP(add, add, ldadd)\n \tpreempt_enable_notrace();\t\t\t\t\t\\\n \tret__;\t\t\t\t\t\t\t\t\\\n })\n+#endif\n \n #ifdef __KVM_NVHE_HYPERVISOR__\n extern unsigned long __hyp_per_cpu_offset(unsigned int cpu);\ndiff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h\nindex 1b4509d3382c63..dd1ac84a11605e 100644\n--- a/arch/arm64/include/asm/pgalloc.h\n+++ b/arch/arm64/include/asm/pgalloc.h\n@@ -33,11 +33,27 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp)\n \tpudval |= (mm == \u0026init_mm) ? PUD_TABLE_UXN : PUD_TABLE_PXN;\n \t__pud_populate(pudp, __pa(pmdp), pudval);\n }\n+\n+#ifdef CONFIG_ARCH_HAS_PUD_POPULATE_SYNC\n+static inline void pud_populate_sync(unsigned long addr, pud_t *pudp, pmd_t *pmdp)\n+{\n+\tpud_populate(\u0026init_mm, pudp, pmdp);\n+\tif (ARCH_PAGE_TABLE_SYNC_MASK \u0026 PGTBL_PUD_MODIFIED)\n+\t\tarch_sync_kernel_mappings(addr, addr);\n+}\n+#endif\n #else\n static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot)\n {\n \tBUILD_BUG();\n }\n+\n+#ifdef CONFIG_ARCH_HAS_PUD_POPULATE_SYNC\n+static inline void pud_populate_sync(unsigned long addr, pud_t *pudp, pmd_t *pmdp)\n+{\n+\tBUILD_BUG();\n+}\n+#endif\n #endif\t/* CONFIG_PGTABLE_LEVELS \u003e 2 */\n \n #if CONFIG_PGTABLE_LEVELS \u003e 3\n@@ -121,4 +137,12 @@ pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep)\n \t\t PMD_TYPE_TABLE | PMD_TABLE_AF | PMD_TABLE_PXN);\n }\n \n+#ifdef CONFIG_ARCH_HAS_PMD_POPULATE_SYNC\n+static inline void pmd_populate_sync(unsigned long addr, pmd_t *pmdp, pte_t *ptep)\n+{\n+\tpmd_populate_kernel(\u0026init_mm, pmdp, ptep);\n+\tif (ARCH_PAGE_TABLE_SYNC_MASK \u0026 PGTBL_PMD_MODIFIED)\n+\t\tarch_sync_kernel_mappings(addr, addr);\n+}\n+#endif\n #endif\ndiff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h\nindex 27689c62bd25dd..83cd65610d7f7d 100644\n--- a/arch/arm64/include/asm/pgtable.h\n+++ b/arch/arm64/include/asm/pgtable.h\n@@ -18,14 +18,35 @@\n * VMALLOC range.\n *\n * VMALLOC_START: beginning of the kernel vmalloc space\n- * VMALLOC_END: extends to the available space below vmemmap\n+ * VMALLOC_END: extends to the space below global percpu area\n */\n #define VMALLOC_START\t\t(MODULES_END)\n+#define VMALLOC_END\t\t(PERCPU_START - SZ_8M)\n+\n+/*\n+ * PERCPU range\n+ *\n+ * PERCPU_START: beginning of global percpu area\n+ * PERCPU_END: end of global percpu area\n+ * LOCAL_PERCPU_START: beginning of local percpu area\n+ * LOCAL_PERCPU_END: end of local percpu area, extend to the available\n+ * space below vmemmap\n+ */\n+#if defined(CONFIG_ARM64_4K_PAGES)\n+#define PERCPU_SIZE\t\t(2 * PGDIR_SIZE)\n+#elif defined(CONFIG_ARM64_16K_PAGES)\n+#define PERCPU_SIZE (16 * PUD_SIZE)\n+#elif defined(CONFIG_ARM64_64K_PAGES)\n+#define PERCPU_SIZE (PGDIR_SIZE)\n+#endif\n+#define PERCPU_START\t\t(PERCPU_END - PERCPU_SIZE)\n+#define PERCPU_END\t\t(LOCAL_PERCPU_START)\n+#define LOCAL_PERCPU_START\t(LOCAL_PERCPU_END - PERCPU_SIZE)\n #if VA_BITS == VA_BITS_MIN\n-#define VMALLOC_END\t\t(VMEMMAP_START - SZ_8M)\n+#define LOCAL_PERCPU_END\t(ALIGN_DOWN(VMEMMAP_START, PERCPU_SIZE))\n #else\n #define VMEMMAP_UNUSED_NPAGES\t((_PAGE_OFFSET(vabits_actual) - PAGE_OFFSET) \u003e\u003e PAGE_SHIFT)\n-#define VMALLOC_END\t\t(VMEMMAP_START + VMEMMAP_UNUSED_NPAGES * sizeof(struct page) - SZ_8M)\n+#define LOCAL_PERCPU_END\t(ALIGN_DOWN((VMEMMAP_START + VMEMMAP_UNUSED_NPAGES * sizeof(struct page)), PERCPU_SIZE))\n #endif\n \n #define vmemmap\t\t\t((struct page *)VMEMMAP_START - (memstart_addr \u003e\u003e PAGE_SHIFT))\n@@ -1216,6 +1237,16 @@ p4d_t *p4d_offset_lockless_folded(pgd_t *pgdp, pgd_t pgd, unsigned long addr)\n \n #endif /* CONFIG_PGTABLE_LEVELS \u003e 4 */\n \n+#if CONFIG_PGTABLE_LEVELS == 2\n+#define ARCH_PAGE_TABLE_SYNC_MASK PGTBL_PMD_MODIFIED\n+#elif CONFIG_PGTABLE_LEVELS == 3\n+#define ARCH_PAGE_TABLE_SYNC_MASK PGTBL_PUD_MODIFIED\n+#elif CONFIG_PGTABLE_LEVELS == 4\n+#define ARCH_PAGE_TABLE_SYNC_MASK PGTBL_P4D_MODIFIED\n+#elif CONFIG_PGTABLE_LEVELS == 5\n+#define ARCH_PAGE_TABLE_SYNC_MASK PGTBL_PGD_MODIFIED\n+#endif\n+\n #define pgd_ERROR(e)\t\\\n \tpr_err(\"%s:%d: bad pgd %016llx.\\n\", __FILE__, __LINE__, pgd_val(e))\n \ndiff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c\nindex 23c05dc7a8f2ac..6d420ad59af492 100644\n--- a/arch/arm64/kernel/setup.c\n+++ b/arch/arm64/kernel/setup.c\n@@ -360,6 +360,9 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p)\n \tsmp_init_cpus();\n \tsmp_build_mpidr_hash();\n \n+\t/* Must be called after smp_init_cpus */\n+\tsetup_percpu_pgd();\n+\n #ifdef CONFIG_ARM64_SW_TTBR0_PAN\n \t/*\n \t * Make sure init_thread_info.ttbr0 always generates translation\ndiff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c\nindex d46022f7207540..4a811379c19162 100644\n--- a/arch/arm64/kernel/smp.c\n+++ b/arch/arm64/kernel/smp.c\n@@ -55,6 +55,8 @@\n \n #include \u003ctrace/events/ipi.h\u003e\n \n+extern void idmap_cpu_replace_ttbr1(phys_addr_t pgdir);\n+\n /*\n * as from 2.5, kernels no longer have an init_tasks structure\n * so we need some other way of telling a new secondary core\n@@ -198,6 +200,12 @@ asmlinkage notrace void secondary_start_kernel(void)\n \tstruct mm_struct *mm = \u0026init_mm;\n \tconst struct cpu_operations *ops;\n \tunsigned int cpu = smp_processor_id();\n+\ttypedef void (ttbr_replace_func)(phys_addr_t);\n+\tttbr_replace_func *replace_ttbr;\n+\n+\tphys_addr_t ttbr1 = phys_to_ttbr(virt_to_phys(percpu_pgd[cpu]));\n+\treplace_ttbr = (void *)__pa_symbol(idmap_cpu_replace_ttbr1);\n+\treplace_ttbr(ttbr1);\n \n \t/*\n \t * All kernel threads share the same mm context; grab a\n@@ -819,6 +827,42 @@ void __init smp_prepare_cpus(unsigned int max_cpus)\n \t}\n }\n \n+#ifdef CONFIG_NUMA\n+extern int cpu_to_node_map[NR_CPUS];\n+\n+int early_cpu_to_node(int cpu)\n+{\n+\treturn cpu_to_node_map[cpu];\n+}\n+#endif\n+\n+#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA\n+unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;\n+EXPORT_SYMBOL(__per_cpu_offset);\n+/* Used to calculate pcpu local address, the offset is same for all CPUs */\n+unsigned long __per_cpu_local_off __ro_after_init;\n+EXPORT_SYMBOL(__per_cpu_local_off);\n+\n+void __init setup_per_cpu_areas(void)\n+{\n+\tunsigned long delta;\n+\tunsigned int cpu;\n+\tint rc = -EINVAL;\n+\n+#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK\n+\t/* PCPU page table just can support PCPU_FC_PAGE */\n+\trc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, early_cpu_to_node);\n+#endif\n+\tif (rc \u003c 0)\n+\t\tpanic(\"Failed to initialize percpu areas (err=%d).\", rc);\n+\n+\tdelta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;\n+\tfor_each_possible_cpu(cpu)\n+\t\t__per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];\n+\t__per_cpu_local_off = (unsigned long)pcpu_local_base - (unsigned long)__per_cpu_start;\n+}\n+#endif\n+\n static const char *ipi_types[MAX_IPI] __tracepoint_string = {\n \t[IPI_RESCHEDULE]\t= \"Rescheduling interrupts\",\n \t[IPI_CALL_FUNC]\t\t= \"Function call interrupts\",\ndiff --git a/arch/arm64/mm/kasan_init.c b/arch/arm64/mm/kasan_init.c\nindex 3fcad956fdf7cb..a8e272bfefe1a1 100644\n--- a/arch/arm64/mm/kasan_init.c\n+++ b/arch/arm64/mm/kasan_init.c\n@@ -60,13 +60,14 @@ static phys_addr_t __init kasan_alloc_raw_page(int node)\n }\n \n static pte_t *__init kasan_pte_offset(pmd_t *pmdp, unsigned long addr, int node,\n-\t\t\t\t bool early)\n+\t\t\t\t bool early, pgtbl_mod_mask *mask)\n {\n \tif (pmd_none(READ_ONCE(*pmdp))) {\n \t\tphys_addr_t pte_phys = early ?\n \t\t\t\t__pa_symbol(kasan_early_shadow_pte)\n \t\t\t\t\t: kasan_alloc_zeroed_page(node);\n \t\t__pmd_populate(pmdp, pte_phys, PMD_TYPE_TABLE);\n+\t\t*mask |= PGTBL_PMD_MODIFIED;\n \t}\n \n \treturn early ? pte_offset_kimg(pmdp, addr)\n@@ -74,49 +75,53 @@ static pte_t *__init kasan_pte_offset(pmd_t *pmdp, unsigned long addr, int node,\n }\n \n static pmd_t *__init kasan_pmd_offset(pud_t *pudp, unsigned long addr, int node,\n-\t\t\t\t bool early)\n+\t\t\t\t bool early, pgtbl_mod_mask *mask)\n {\n \tif (pud_none(READ_ONCE(*pudp))) {\n \t\tphys_addr_t pmd_phys = early ?\n \t\t\t\t__pa_symbol(kasan_early_shadow_pmd)\n \t\t\t\t\t: kasan_alloc_zeroed_page(node);\n \t\t__pud_populate(pudp, pmd_phys, PUD_TYPE_TABLE);\n+\t\t*mask |= PGTBL_PUD_MODIFIED;\n \t}\n \n \treturn early ? pmd_offset_kimg(pudp, addr) : pmd_offset(pudp, addr);\n }\n \n static pud_t *__init kasan_pud_offset(p4d_t *p4dp, unsigned long addr, int node,\n-\t\t\t\t bool early)\n+\t\t\t\t bool early, pgtbl_mod_mask *mask)\n {\n \tif (p4d_none(READ_ONCE(*p4dp))) {\n \t\tphys_addr_t pud_phys = early ?\n \t\t\t\t__pa_symbol(kasan_early_shadow_pud)\n \t\t\t\t\t: kasan_alloc_zeroed_page(node);\n \t\t__p4d_populate(p4dp, pud_phys, P4D_TYPE_TABLE);\n+\t\t*mask |= PGTBL_P4D_MODIFIED;\n \t}\n \n \treturn early ? pud_offset_kimg(p4dp, addr) : pud_offset(p4dp, addr);\n }\n \n static p4d_t *__init kasan_p4d_offset(pgd_t *pgdp, unsigned long addr, int node,\n-\t\t\t\t bool early)\n+\t\t\t\t bool early, pgtbl_mod_mask *mask)\n {\n \tif (pgd_none(READ_ONCE(*pgdp))) {\n \t\tphys_addr_t p4d_phys = early ?\n \t\t\t\t__pa_symbol(kasan_early_shadow_p4d)\n \t\t\t\t\t: kasan_alloc_zeroed_page(node);\n \t\t__pgd_populate(pgdp, p4d_phys, PGD_TYPE_TABLE);\n+\t\t*mask |= PGTBL_PGD_MODIFIED;\n \t}\n \n \treturn early ? p4d_offset_kimg(pgdp, addr) : p4d_offset(pgdp, addr);\n }\n \n static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr,\n-\t\t\t\t unsigned long end, int node, bool early)\n+\t\t\t\t unsigned long end, int node, bool early,\n+\t\t\t\t pgtbl_mod_mask *mask)\n {\n \tunsigned long next;\n-\tpte_t *ptep = kasan_pte_offset(pmdp, addr, node, early);\n+\tpte_t *ptep = kasan_pte_offset(pmdp, addr, node, early, mask);\n \n \tdo {\n \t\tphys_addr_t page_phys = early ?\n@@ -130,38 +135,41 @@ static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr,\n }\n \n static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr,\n-\t\t\t\t unsigned long end, int node, bool early)\n+\t\t\t\t unsigned long end, int node, bool early,\n+\t\t\t\t pgtbl_mod_mask *mask)\n {\n \tunsigned long next;\n-\tpmd_t *pmdp = kasan_pmd_offset(pudp, addr, node, early);\n+\tpmd_t *pmdp = kasan_pmd_offset(pudp, addr, node, early, mask);\n \n \tdo {\n \t\tnext = pmd_addr_end(addr, end);\n-\t\tkasan_pte_populate(pmdp, addr, next, node, early);\n+\t\tkasan_pte_populate(pmdp, addr, next, node, early, mask);\n \t} while (pmdp++, addr = next, addr != end \u0026\u0026 pmd_none(READ_ONCE(*pmdp)));\n }\n \n static void __init kasan_pud_populate(p4d_t *p4dp, unsigned long addr,\n-\t\t\t\t unsigned long end, int node, bool early)\n+\t\t\t\t unsigned long end, int node, bool early,\n+\t\t\t\t pgtbl_mod_mask *mask)\n {\n \tunsigned long next;\n-\tpud_t *pudp = kasan_pud_offset(p4dp, addr, node, early);\n+\tpud_t *pudp = kasan_pud_offset(p4dp, addr, node, early, mask);\n \n \tdo {\n \t\tnext = pud_addr_end(addr, end);\n-\t\tkasan_pmd_populate(pudp, addr, next, node, early);\n+\t\tkasan_pmd_populate(pudp, addr, next, node, early, mask);\n \t} while (pudp++, addr = next, addr != end \u0026\u0026 pud_none(READ_ONCE(*pudp)));\n }\n \n static void __init kasan_p4d_populate(pgd_t *pgdp, unsigned long addr,\n-\t\t\t\t unsigned long end, int node, bool early)\n+\t\t\t\t unsigned long end, int node, bool early,\n+\t\t\t\t pgtbl_mod_mask *mask)\n {\n \tunsigned long next;\n-\tp4d_t *p4dp = kasan_p4d_offset(pgdp, addr, node, early);\n+\tp4d_t *p4dp = kasan_p4d_offset(pgdp, addr, node, early, mask);\n \n \tdo {\n \t\tnext = p4d_addr_end(addr, end);\n-\t\tkasan_pud_populate(p4dp, addr, next, node, early);\n+\t\tkasan_pud_populate(p4dp, addr, next, node, early, mask);\n \t} while (p4dp++, addr = next, addr != end \u0026\u0026 p4d_none(READ_ONCE(*p4dp)));\n }\n \n@@ -170,12 +178,17 @@ static void __init kasan_pgd_populate(unsigned long addr, unsigned long end,\n {\n \tunsigned long next;\n \tpgd_t *pgdp;\n+\tpgtbl_mod_mask mask = 0;\n+\tunsigned long start = addr;\n \n \tpgdp = pgd_offset_k(addr);\n \tdo {\n \t\tnext = pgd_addr_end(addr, end);\n-\t\tkasan_p4d_populate(pgdp, addr, next, node, early);\n+\t\tkasan_p4d_populate(pgdp, addr, next, node, early, \u0026mask);\n \t} while (pgdp++, addr = next, addr != end);\n+\n+\tif (mask \u0026 ARCH_PAGE_TABLE_SYNC_MASK)\n+\t\tarch_sync_kernel_mappings(start, end);\n }\n \n #if defined(CONFIG_ARM64_64K_PAGES) || CONFIG_PGTABLE_LEVELS \u003e 4\n@@ -311,7 +324,7 @@ static void __init kasan_init_shadow(void)\n \n \tmod_shadow_start = (u64)kasan_mem_to_shadow((void *)MODULES_VADDR);\n \n-\tvmalloc_shadow_end = (u64)kasan_mem_to_shadow((void *)VMALLOC_END);\n+\tvmalloc_shadow_end = (u64)kasan_mem_to_shadow((void *)LOCAL_PERCPU_END);\n \n \t/*\n \t * We are going to perform proper setup of shadow memory.\ndiff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c\nindex f2be501468ce5a..ee4ab031ebf8d7 100644\n--- a/arch/arm64/mm/mmu.c\n+++ b/arch/arm64/mm/mmu.c\n@@ -67,6 +67,63 @@ long __section(\".mmuoff.data.write\") __early_cpu_boot_status;\n static DEFINE_SPINLOCK(swapper_pgdir_lock);\n static DEFINE_MUTEX(fixmap_lock);\n \n+pgd_t *percpu_pgd[NR_CPUS] __ro_after_init;\n+bool percpu_pgd_setup_done __ro_after_init = false;\n+\n+void __init setup_percpu_pgd(void)\n+{\n+\tint cpu;\n+\n+\tfor_each_possible_cpu(cpu) {\n+\t\tvoid *addr;\n+\n+\t\tif (cpu == 0) {\n+\t\t\tpercpu_pgd[cpu] = swapper_pg_dir;\n+\t\t\tcontinue;\n+\t\t}\n+\n+\t\taddr = memblock_alloc(PAGE_SIZE, PAGE_SIZE);\n+\t\tif (!addr)\n+\t\t\tpanic(\"Can't alloc percpu pgd\\n\");\n+\n+\t\tmemcpy(addr, (void *)swapper_pg_dir, PAGE_SIZE);\n+\t\tpercpu_pgd[cpu] = (pgd_t *)addr;\n+\t}\n+\n+\tdsb(ishst);\n+\n+\tpercpu_pgd_setup_done = true;\n+}\n+\n+void arch_sync_kernel_mappings(unsigned long start, unsigned long end)\n+{\n+\tunsigned long addr, next;\n+\tint cpu;\n+\tpgd_t *pgdp = pgd_offset_k(start);\n+\tpgd_t pgd;\n+\tunsigned int index = pgd_index(start);\n+\n+\tBUG_ON(start \u003e end);\n+\n+\tif (!percpu_pgd_setup_done)\n+\t\treturn;\n+\n+\t/* Don't sync local percpu area page table */\n+\tif (start \u003e= LOCAL_PERCPU_START \u0026\u0026 end \u003c LOCAL_PERCPU_END)\n+\t\treturn;\n+\n+\taddr = start;\n+\tdo {\n+\t\tpgd = READ_ONCE(*pgdp);\n+\t\tnext = pgd_addr_end(addr, end);\n+\t\tfor_each_possible_cpu(cpu) {\n+\t\t\tif (cpu == 0)\n+\t\t\t\tcontinue;\n+\t\t\tset_pgd(percpu_pgd[cpu] + index, pgd);\n+\t\t}\n+\t} while (pgdp++, index++, addr = next, addr != end);\n+}\n+\n void noinstr set_swapper_pgd(pgd_t *pgdp, pgd_t pgd)\n {\n \tpgd_t *fixmap_pgdp;\n@@ -161,7 +218,7 @@ static void init_clear_pgtable(void *table)\n }\n \n static void init_pte(pte_t *ptep, unsigned long addr, unsigned long end,\n-\t\t phys_addr_t phys, pgprot_t prot)\n+\t\t phys_addr_t phys, pgprot_t prot, pgtbl_mod_mask *mask)\n {\n \tdo {\n \t\tpte_t old_pte = __ptep_get(ptep);\n@@ -171,6 +228,7 @@ static void init_pte(pte_t *ptep, unsigned long addr, unsigned long end,\n \t\t * are deferred to the end of alloc_init_cont_pte().\n \t\t */\n \t\t__set_pte_nosync(ptep, pfn_pte(__phys_to_pfn(phys), prot));\n+\t\t*mask |= PGTBL_PTE_MODIFIED;\n \n \t\t/*\n \t\t * After the PTE entry has been populated once, we\n@@ -198,7 +256,7 @@ static int alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,\n \t\t\t unsigned long end, phys_addr_t phys,\n \t\t\t pgprot_t prot,\n \t\t\t phys_addr_t (*pgtable_alloc)(enum pgtable_level),\n-\t\t\t int flags)\n+\t\t\t int flags, pgtbl_mod_mask *mask)\n {\n \tunsigned long next;\n \tpmd_t pmd = READ_ONCE(*pmdp);\n@@ -219,6 +277,7 @@ static int alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,\n \t\tinit_clear_pgtable(ptep);\n \t\tptep += pte_index(addr);\n \t\t__pmd_populate(pmdp, pte_phys, pmdval);\n+\t\t*mask |= PGTBL_PMD_MODIFIED;\n \t} else {\n \t\tBUG_ON(pmd_bad(pmd));\n \t\tptep = pte_set_fixmap_offset(pmdp, addr);\n@@ -235,7 +294,7 @@ static int alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,\n \t\t !pte_range_has_valid_noncont(ptep))\n \t\t\t__prot = __pgprot(pgprot_val(prot) | PTE_CONT);\n \n-\t\tinit_pte(ptep, addr, next, phys, __prot);\n+\t\tinit_pte(ptep, addr, next, phys, __prot, mask);\n \n \t\tptep += pte_index(next) - pte_index(addr);\n \t\tphys += next - addr;\n@@ -253,7 +312,8 @@ static int alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,\n \n static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end,\n \t\t phys_addr_t phys, pgprot_t prot,\n-\t\t phys_addr_t (*pgtable_alloc)(enum pgtable_level), int flags)\n+\t\t phys_addr_t (*pgtable_alloc)(enum pgtable_level), int flags,\n+\t\t pgtbl_mod_mask *mask)\n {\n \tunsigned long next;\n \n@@ -266,7 +326,10 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end,\n \t\tif (((addr | next | phys) \u0026 ~PMD_MASK) == 0 \u0026\u0026\n \t\t (flags \u0026 NO_BLOCK_MAPPINGS) == 0 \u0026\u0026\n \t\t !pmd_table(old_pmd)) {\n-\t\t\tWARN_ON(!pmd_set_huge(pmdp, phys, prot));\n+\t\t\tif (pmd_set_huge(pmdp, phys, prot))\n+\t\t\t\t*mask |= PGTBL_PMD_MODIFIED;\n+\t\t\telse\n+\t\t\t\tWARN_ON(1);\n \n \t\t\t/*\n \t\t\t * After the PMD entry has been populated once, we\n@@ -278,7 +341,7 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end,\n \t\t\tint ret;\n \n \t\t\tret = alloc_init_cont_pte(pmdp, addr, next, phys, prot,\n-\t\t\t\t\t\t pgtable_alloc, flags);\n+\t\t\t\t\t\t pgtable_alloc, flags, mask);\n \t\t\tif (ret)\n \t\t\t\treturn ret;\n \n@@ -306,7 +369,7 @@ static int alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,\n \t\t\t unsigned long end, phys_addr_t phys,\n \t\t\t pgprot_t prot,\n \t\t\t phys_addr_t (*pgtable_alloc)(enum pgtable_level),\n-\t\t\t int flags)\n+\t\t\t int flags, pgtbl_mod_mask *mask)\n {\n \tint ret;\n \tunsigned long next;\n@@ -331,6 +394,7 @@ static int alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,\n \t\tinit_clear_pgtable(pmdp);\n \t\tpmdp += pmd_index(addr);\n \t\t__pud_populate(pudp, pmd_phys, pudval);\n+\t\t*mask |= PGTBL_PUD_MODIFIED;\n \t} else {\n \t\tBUG_ON(pud_bad(pud));\n \t\tpmdp = pmd_set_fixmap_offset(pudp, addr);\n@@ -347,7 +411,8 @@ static int alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,\n \t\t !pmd_range_has_valid_noncont(pmdp))\n \t\t\t__prot = __pgprot(pgprot_val(prot) | PTE_CONT);\n \n-\t\tret = init_pmd(pmdp, addr, next, phys, __prot, pgtable_alloc, flags);\n+\t\tret = init_pmd(pmdp, addr, next, phys, __prot, pgtable_alloc, flags,\n+\t\t\t mask);\n \t\tif (ret)\n \t\t\tgoto out;\n \n@@ -364,7 +429,7 @@ static int alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,\n static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,\n \t\t\t phys_addr_t phys, pgprot_t prot,\n \t\t\t phys_addr_t (*pgtable_alloc)(enum pgtable_level),\n-\t\t\t int flags)\n+\t\t\t int flags, pgtbl_mod_mask *mask)\n {\n \tint ret = 0;\n \tunsigned long next;\n@@ -385,6 +450,7 @@ static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,\n \t\tinit_clear_pgtable(pudp);\n \t\tpudp += pud_index(addr);\n \t\t__p4d_populate(p4dp, pud_phys, p4dval);\n+\t\t*mask |= PGTBL_P4D_MODIFIED;\n \t} else {\n \t\tBUG_ON(p4d_bad(p4d));\n \t\tpudp = pud_set_fixmap_offset(p4dp, addr);\n@@ -402,7 +468,10 @@ static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,\n \t\t ((addr | next | phys) \u0026 ~PUD_MASK) == 0 \u0026\u0026\n \t\t (flags \u0026 NO_BLOCK_MAPPINGS) == 0 \u0026\u0026\n \t\t !pud_table(old_pud)) {\n-\t\t\tWARN_ON(!pud_set_huge(pudp, phys, prot));\n+\t\t\tif (pud_set_huge(pudp, phys, prot))\n+\t\t\t\t*mask |= PGTBL_PUD_MODIFIED;\n+\t\t\telse\n+\t\t\t\tWARN_ON(1);\n \n \t\t\t/*\n \t\t\t * After the PUD entry has been populated once, we\n@@ -412,7 +481,7 @@ static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,\n \t\t\t\t\t\t READ_ONCE(pud_val(*pudp))));\n \t\t} else {\n \t\t\tret = alloc_init_cont_pmd(pudp, addr, next, phys, prot,\n-\t\t\t\t\t\t pgtable_alloc, flags);\n+\t\t\t\t\t\t pgtable_alloc, flags, mask);\n \t\t\tif (ret)\n \t\t\t\tgoto out;\n \n@@ -431,7 +500,7 @@ static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,\n static int alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end,\n \t\t\t phys_addr_t phys, pgprot_t prot,\n \t\t\t phys_addr_t (*pgtable_alloc)(enum pgtable_level),\n-\t\t\t int flags)\n+\t\t\t int flags, pgtbl_mod_mask *mask)\n {\n \tint ret;\n \tunsigned long next;\n@@ -452,6 +521,7 @@ static int alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end,\n \t\tinit_clear_pgtable(p4dp);\n \t\tp4dp += p4d_index(addr);\n \t\t__pgd_populate(pgdp, p4d_phys, pgdval);\n+\t\t*mask |= PGTBL_PGD_MODIFIED;\n \t} else {\n \t\tBUG_ON(pgd_bad(pgd));\n \t\tp4dp = p4d_set_fixmap_offset(pgdp, addr);\n@@ -463,7 +533,7 @@ static int alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end,\n \t\tnext = p4d_addr_end(addr, end);\n \n \t\tret = alloc_init_pud(p4dp, addr, next, phys, prot,\n-\t\t\t\t pgtable_alloc, flags);\n+\t\t\t\t pgtable_alloc, flags, mask);\n \t\tif (ret)\n \t\t\tgoto out;\n \n@@ -483,7 +553,7 @@ static int __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys,\n \t\t\t\t unsigned long virt, phys_addr_t size,\n \t\t\t\t pgprot_t prot,\n \t\t\t\t phys_addr_t (*pgtable_alloc)(enum pgtable_level),\n-\t\t\t\t int flags)\n+\t\t\t\t int flags, pgtbl_mod_mask *mask)\n {\n \tint ret;\n \tunsigned long addr, end, next;\n@@ -503,7 +573,7 @@ static int __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys,\n \tdo {\n \t\tnext = pgd_addr_end(addr, end);\n \t\tret = alloc_init_p4d(pgdp, addr, next, phys, prot, pgtable_alloc,\n-\t\t\t\t flags);\n+\t\t\t\t flags, mask);\n \t\tif (ret)\n \t\t\treturn ret;\n \t\tphys += next - addr;\n@@ -516,13 +586,13 @@ static int __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,\n \t\t\t\tunsigned long virt, phys_addr_t size,\n \t\t\t\tpgprot_t prot,\n \t\t\t\tphys_addr_t (*pgtable_alloc)(enum pgtable_level),\n-\t\t\t\tint flags)\n+\t\t\t\tint flags, pgtbl_mod_mask *mask)\n {\n \tint ret;\n \n \tmutex_lock(\u0026fixmap_lock);\n \tret = __create_pgd_mapping_locked(pgdir, phys, virt, size, prot,\n-\t\t\t\t\t pgtable_alloc, flags);\n+\t\t\t\t\t pgtable_alloc, flags, mask);\n \tmutex_unlock(\u0026fixmap_lock);\n \n \treturn ret;\n@@ -535,9 +605,10 @@ static void early_create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,\n \t\t\t\t int flags)\n {\n \tint ret;\n+\tpgtbl_mod_mask mask = 0;\n \n \tret = __create_pgd_mapping(pgdir, phys, virt, size, prot, pgtable_alloc,\n-\t\t\t\t flags);\n+\t\t\t\t flags, \u0026mask);\n \tif (ret)\n \t\tpanic(\"Failed to create page tables\\n\");\n }\n@@ -1009,6 +1080,29 @@ void __init linear_map_maybe_split_to_ptes(void)\n \t}\n }\n \n+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP\n+void __init map_local_percpu_first_chunk(unsigned int cpu, unsigned long virt,\n+\t\t\t struct page **pages, unsigned int nr)\n+{\n+\tint i, ret;\n+\tpgtbl_mod_mask mask = 0;\n+\n+\tarch_enter_lazy_mmu_mode();\n+\n+\tfor (i = 0; i \u003c nr; i++) {\n+\t\tphys_addr_t phys = page_to_phys(pages[i]);\n+\t\tret = __create_pgd_mapping(percpu_pgd[cpu], phys, virt, PAGE_SIZE, PAGE_KERNEL,\n+\t\t\t\t\t early_pgtable_alloc, NO_EXEC_MAPPINGS, \u0026mask);\n+\t\tif (ret)\n+\t\t\tpanic(\"Failed to create local percpu first chunk\\n\");\n+\n+\t\tvirt += PAGE_SIZE;\n+\t}\n+\n+\tarch_leave_lazy_mmu_mode();\n+}\n+#endif\n+\n /*\n * This function can only be used to modify existing table entries,\n * without allocating new levels of table. Note that this permits the\n@@ -1279,6 +1373,7 @@ static int __init __kpti_install_ng_mappings(void *__unused)\n \n \tif (!cpu) {\n \t\tint ret;\n+\t\tpgtbl_mod_mask mask = 0;\n \n \t\talloc = __get_free_pages(GFP_ATOMIC | __GFP_ZERO, order);\n \t\tkpti_ng_temp_pgd = (pgd_t *)(alloc + (levels - 1) * PAGE_SIZE);\n@@ -1302,7 +1397,7 @@ static int __init __kpti_install_ng_mappings(void *__unused)\n \t\t//\n \t\tret = __create_pgd_mapping_locked(kpti_ng_temp_pgd, __pa(alloc),\n \t\t\t\t\t\t KPTI_NG_TEMP_VA, PAGE_SIZE, PAGE_KERNEL,\n-\t\t\t\t\t\t kpti_ng_pgd_alloc, 0);\n+\t\t\t\t\t\t kpti_ng_pgd_alloc, 0, \u0026mask);\n \t\tif (ret)\n \t\t\tpanic(\"Failed to create page tables\\n\");\n \t}\n@@ -1608,7 +1703,7 @@ static void unmap_hotplug_range(unsigned long addr, unsigned long end,\n \n static void free_empty_pte_table(pmd_t *pmdp, unsigned long addr,\n \t\t\t\t unsigned long end, unsigned long floor,\n-\t\t\t\t unsigned long ceiling)\n+\t\t\t\t unsigned long ceiling, pgtbl_mod_mask *mask)\n {\n \tpte_t *ptep, pte;\n \tunsigned long i, start = addr;\n@@ -1624,6 +1719,7 @@ static void free_empty_pte_table(pmd_t *pmdp, unsigned long addr,\n \t\tWARN_ON(!pte_none(pte));\n \t} while (addr += PAGE_SIZE, addr \u003c end);\n \n+\t*mask |= PGTBL_PTE_MODIFIED;\n \tif (!pgtable_range_aligned(start, end, floor, ceiling, PMD_MASK))\n \t\treturn;\n \n@@ -1639,13 +1735,14 @@ static void free_empty_pte_table(pmd_t *pmdp, unsigned long addr,\n \t}\n \n \tpmd_clear(pmdp);\n+\t*mask |= PGTBL_PMD_MODIFIED;\n \t__flush_tlb_kernel_pgtable(start);\n \tfree_hotplug_pgtable_page(virt_to_page(ptep));\n }\n \n static void free_empty_pmd_table(pud_t *pudp, unsigned long addr,\n \t\t\t\t unsigned long end, unsigned long floor,\n-\t\t\t\t unsigned long ceiling)\n+\t\t\t\t unsigned long ceiling, pgtbl_mod_mask *mask)\n {\n \tpmd_t *pmdp, pmd;\n \tunsigned long i, next, start = addr;\n@@ -1658,7 +1755,7 @@ static void free_empty_pmd_table(pud_t *pudp, unsigned long addr,\n \t\t\tcontinue;\n \n \t\tWARN_ON(!pmd_present(pmd) || !pmd_table(pmd));\n-\t\tfree_empty_pte_table(pmdp, addr, next, floor, ceiling);\n+\t\tfree_empty_pte_table(pmdp, addr, next, floor, ceiling, mask);\n \t} while (addr = next, addr \u003c end);\n \n \tif (CONFIG_PGTABLE_LEVELS \u003c= 2)\n@@ -1679,13 +1776,14 @@ static void free_empty_pmd_table(pud_t *pudp, unsigned long addr,\n \t}\n \n \tpud_clear(pudp);\n+\t*mask |= PGTBL_PUD_MODIFIED;\n \t__flush_tlb_kernel_pgtable(start);\n \tfree_hotplug_pgtable_page(virt_to_page(pmdp));\n }\n \n static void free_empty_pud_table(p4d_t *p4dp, unsigned long addr,\n \t\t\t\t unsigned long end, unsigned long floor,\n-\t\t\t\t unsigned long ceiling)\n+\t\t\t\t unsigned long ceiling, pgtbl_mod_mask *mask)\n {\n \tpud_t *pudp, pud;\n \tunsigned long i, next, start = addr;\n@@ -1698,7 +1796,7 @@ static void free_empty_pud_table(p4d_t *p4dp, unsigned long addr,\n \t\t\tcontinue;\n \n \t\tWARN_ON(!pud_present(pud) || !pud_table(pud));\n-\t\tfree_empty_pmd_table(pudp, addr, next, floor, ceiling);\n+\t\tfree_empty_pmd_table(pudp, addr, next, floor, ceiling, mask);\n \t} while (addr = next, addr \u003c end);\n \n \tif (!pgtable_l4_enabled())\n@@ -1719,13 +1817,14 @@ static void free_empty_pud_table(p4d_t *p4dp, unsigned long addr,\n \t}\n \n \tp4d_clear(p4dp);\n+\t*mask |= PGTBL_P4D_MODIFIED;\n \t__flush_tlb_kernel_pgtable(start);\n \tfree_hotplug_pgtable_page(virt_to_page(pudp));\n }\n \n static void free_empty_p4d_table(pgd_t *pgdp, unsigned long addr,\n \t\t\t\t unsigned long end, unsigned long floor,\n-\t\t\t\t unsigned long ceiling)\n+\t\t\t\t unsigned long ceiling, pgtbl_mod_mask *mask)\n {\n \tp4d_t *p4dp, p4d;\n \tunsigned long i, next, start = addr;\n@@ -1738,7 +1837,7 @@ static void free_empty_p4d_table(pgd_t *pgdp, unsigned long addr,\n \t\t\tcontinue;\n \n \t\tWARN_ON(!p4d_present(p4d));\n-\t\tfree_empty_pud_table(p4dp, addr, next, floor, ceiling);\n+\t\tfree_empty_pud_table(p4dp, addr, next, floor, ceiling, mask);\n \t} while (addr = next, addr \u003c end);\n \n \tif (!pgtable_l5_enabled())\n@@ -1759,6 +1858,7 @@ static void free_empty_p4d_table(pgd_t *pgdp, unsigned long addr,\n \t}\n \n \tpgd_clear(pgdp);\n+\t*mask |= PGTBL_PGD_MODIFIED;\n \t__flush_tlb_kernel_pgtable(start);\n \tfree_hotplug_pgtable_page(virt_to_page(p4dp));\n }\n@@ -1768,6 +1868,8 @@ static void free_empty_tables(unsigned long addr, unsigned long end,\n {\n \tunsigned long next;\n \tpgd_t *pgdp, pgd;\n+\tpgtbl_mod_mask mask = 0;\n+\tunsigned long start = addr;\n \n \tdo {\n \t\tnext = pgd_addr_end(addr, end);\n@@ -1777,8 +1879,11 @@ static void free_empty_tables(unsigned long addr, unsigned long end,\n \t\t\tcontinue;\n \n \t\tWARN_ON(!pgd_present(pgd));\n-\t\tfree_empty_p4d_table(pgdp, addr, next, floor, ceiling);\n+\t\tfree_empty_p4d_table(pgdp, addr, next, floor, ceiling, \u0026mask);\n \t} while (addr = next, addr \u003c end);\n+\n+\tif (mask \u0026 ARCH_PAGE_TABLE_SYNC_MASK)\n+\t\tarch_sync_kernel_mappings(start, end);\n }\n #endif\n \n@@ -1986,6 +2091,7 @@ int arch_add_memory(int nid, u64 start, u64 size,\n \t\t struct mhp_params *params)\n {\n \tint ret, flags = NO_EXEC_MAPPINGS;\n+\tpgtbl_mod_mask mask = 0;\n \n \tVM_BUG_ON(!mhp_range_allowed(start, size, true));\n \n@@ -1994,10 +2100,13 @@ int arch_add_memory(int nid, u64 start, u64 size,\n \n \tret = __create_pgd_mapping(swapper_pg_dir, start, __phys_to_virt(start),\n \t\t\t\t size, params-\u003epgprot, pgd_pgtable_alloc_init_mm,\n-\t\t\t\t flags);\n+\t\t\t\t flags, \u0026mask);\n \tif (ret)\n \t\tgoto err;\n \n+\tif (mask \u0026 ARCH_PAGE_TABLE_SYNC_MASK)\n+\t\tarch_sync_kernel_mappings(start, start + size);\n+\n \tmemblock_clear_nomap(start, size);\n \n \tret = __add_pages(nid, start \u003e\u003e PAGE_SHIFT, size \u003e\u003e PAGE_SHIFT,\ndiff --git a/arch/arm64/mm/ptdump.c b/arch/arm64/mm/ptdump.c\nindex 1c20144700d7d7..dd293319823e3c 100644\n--- a/arch/arm64/mm/ptdump.c\n+++ b/arch/arm64/mm/ptdump.c\n@@ -389,6 +389,10 @@ static int __init ptdump_init(void)\n \t\t{ MODULES_END,\t\t\"Modules end\" },\n \t\t{ VMALLOC_START,\t\"vmalloc() area\" },\n \t\t{ VMALLOC_END,\t\t\"vmalloc() end\" },\n+\t\t{ PERCPU_START,\t\t\"Global percpu start\" },\n+\t\t{ PERCPU_END,\t\t\"Global percpu end\" },\n+\t\t{ LOCAL_PERCPU_START,\t\"Local percpu start\" },\n+\t\t{ LOCAL_PERCPU_END,\t\"Local percpu end\" },\n \t\t{ vmemmap_start,\t\"vmemmap start\" },\n \t\t{ VMEMMAP_END,\t\t\"vmemmap end\" },\n \t\t{ PCI_IO_START,\t\t\"PCI I/O start\" },\ndiff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c\nindex fa66f9c97d748d..d420ff9dcf4fc8 100644\n--- a/arch/riscv/kernel/smp.c\n+++ b/arch/riscv/kernel/smp.c\n@@ -364,3 +364,54 @@ void kgdb_roundup_cpus(void)\n \t}\n }\n #endif\n+\n+#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA\n+extern int cpu_to_node_map[NR_CPUS];\n+\n+unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;\n+EXPORT_SYMBOL(__per_cpu_offset);\n+\n+int early_cpu_to_node(int cpu)\n+{\n+\treturn cpu_to_node_map[cpu];\n+}\n+\n+static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)\n+{\n+\treturn node_distance(early_cpu_to_node(from), early_cpu_to_node(to));\n+}\n+\n+void __init setup_per_cpu_areas(void)\n+{\n+\tunsigned long delta;\n+\tunsigned int cpu;\n+\tint rc = -EINVAL;\n+\n+\tif (pcpu_chosen_fc != PCPU_FC_PAGE) {\n+\t\t/*\n+\t\t * Always reserve area for module percpu variables. That's\n+\t\t * what the legacy allocator did.\n+\t\t */\n+\t\trc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE,\n+\t\t\t\t\t PERCPU_DYNAMIC_RESERVE, PAGE_SIZE,\n+\t\t\t\t\t pcpu_cpu_distance,\n+\t\t\t\t\t early_cpu_to_node);\n+#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK\n+\t\tif (rc \u003c 0)\n+\t\t\tpr_warn(\"PERCPU: %s allocator failed (%d), falling back to page size\\n\",\n+\t\t\t\t pcpu_fc_names[pcpu_chosen_fc], rc);\n+#endif\n+\t}\n+\n+#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK\n+\tif (rc \u003c 0)\n+\t\trc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, early_cpu_to_node);\n+#endif\n+\tif (rc \u003c 0)\n+\t\tpanic(\"Failed to initialize percpu areas (err=%d).\", rc);\n+\n+\tdelta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;\n+\tfor_each_possible_cpu(cpu)\n+\t\t__per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];\n+}\n+#endif\ndiff --git a/drivers/base/arch_numa.c b/drivers/base/arch_numa.c\nindex c99f2ab105e5b7..b3b91ceed6a90d 100644\n--- a/drivers/base/arch_numa.c\n+++ b/drivers/base/arch_numa.c\n@@ -16,7 +16,7 @@\n \n #include \u003casm/sections.h\u003e\n \n-static int cpu_to_node_map[NR_CPUS] = { [0 ... NR_CPUS-1] = NUMA_NO_NODE };\n+int cpu_to_node_map[NR_CPUS] = { [0 ... NR_CPUS-1] = NUMA_NO_NODE };\n \n bool numa_off;\n \n@@ -140,55 +140,6 @@ void __init early_map_cpu_to_node(unsigned int cpu, int nid)\n \t\tset_cpu_numa_node(cpu, nid);\n }\n \n-#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA\n-unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;\n-EXPORT_SYMBOL(__per_cpu_offset);\n-\n-int early_cpu_to_node(int cpu)\n-{\n-\treturn cpu_to_node_map[cpu];\n-}\n-\n-static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)\n-{\n-\treturn node_distance(early_cpu_to_node(from), early_cpu_to_node(to));\n-}\n-\n-void __init setup_per_cpu_areas(void)\n-{\n-\tunsigned long delta;\n-\tunsigned int cpu;\n-\tint rc = -EINVAL;\n-\n-\tif (pcpu_chosen_fc != PCPU_FC_PAGE) {\n-\t\t/*\n-\t\t * Always reserve area for module percpu variables. That's\n-\t\t * what the legacy allocator did.\n-\t\t */\n-\t\trc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE,\n-\t\t\t\t\t PERCPU_DYNAMIC_RESERVE, PAGE_SIZE,\n-\t\t\t\t\t pcpu_cpu_distance,\n-\t\t\t\t\t early_cpu_to_node);\n-#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK\n-\t\tif (rc \u003c 0)\n-\t\t\tpr_warn(\"PERCPU: %s allocator failed (%d), falling back to page size\\n\",\n-\t\t\t\t pcpu_fc_names[pcpu_chosen_fc], rc);\n-#endif\n-\t}\n-\n-#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK\n-\tif (rc \u003c 0)\n-\t\trc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, early_cpu_to_node);\n-#endif\n-\tif (rc \u003c 0)\n-\t\tpanic(\"Failed to initialize percpu areas (err=%d).\", rc);\n-\n-\tdelta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;\n-\tfor_each_possible_cpu(cpu)\n-\t\t__per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];\n-}\n-#endif\n-\n /*\n * Initialize NODE_DATA for a node on the local memory\n */\ndiff --git a/include/linux/mm.h b/include/linux/mm.h\nindex 485df9c2dbddb3..c8c3800fbd9fa0 100644\n--- a/include/linux/mm.h\n+++ b/include/linux/mm.h\n@@ -1858,6 +1858,17 @@ unsigned long vmalloc_to_pfn(const void *addr);\n #ifdef CONFIG_MMU\n extern bool is_vmalloc_addr(const void *x);\n extern int is_vmalloc_or_module_addr(const void *x);\n+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP\n+static inline bool is_percpu_addr(unsigned long addr)\n+{\n+\treturn addr \u003e= PERCPU_START \u0026\u0026 addr \u003c LOCAL_PERCPU_END;\n+}\n+#else\n+static inline bool is_percpu_addr(unsigned long addr)\n+{\n+\treturn false;\n+}\n+#endif\n #else\n static inline bool is_vmalloc_addr(const void *x)\n {\ndiff --git a/include/linux/percpu.h b/include/linux/percpu.h\nindex 4f36b8585995c5..5a27031dff1a5c 100644\n--- a/include/linux/percpu.h\n+++ b/include/linux/percpu.h\n@@ -75,6 +75,8 @@\n \n extern void *pcpu_base_addr;\n extern const unsigned long *pcpu_unit_offsets;\n+/* percpu local mapping base */\n+extern void *pcpu_local_base;\n \n struct pcpu_group_info {\n \tint\t\t\tnr_units;\t/* aligned # of units */\n@@ -114,7 +116,7 @@ extern struct pcpu_alloc_info * __init pcpu_alloc_alloc_info(int nr_groups,\n extern void __init pcpu_free_alloc_info(struct pcpu_alloc_info *ai);\n \n extern void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,\n-\t\t\t\t\t void *base_addr);\n+\t\t\t\t\t void *base_addr, void *local_base);\n \n extern int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size,\n \t\t\t\tsize_t atom_size,\ndiff --git a/include/linux/pgalloc.h b/include/linux/pgalloc.h\nindex 9174fa59bbc54d..062c64a5d4bd0c 100644\n--- a/include/linux/pgalloc.h\n+++ b/include/linux/pgalloc.h\n@@ -26,4 +26,17 @@\n \t\t\tarch_sync_kernel_mappings(addr, addr);\t\t\\\n \t} while (0)\n \n+#ifndef CONFIG_ARCH_HAS_PUD_POPULATE_SYNC\n+#define pud_populate_sync(addr, pud, pmd)\t\t\t\t\\\n+\tdo {\t\t\t\t\t\t\t\t\\\n+\t\tpud_populate(\u0026init_mm, pud, pmd);\t\t\t\\\n+\t} while (0)\n+#endif\n+\n+#ifndef CONFIG_ARCH_HAS_PMD_POPULATE_SYNC\n+#define pmd_populate_sync(addr, pmd, pte)\t\t\t\t\\\n+\tdo {\t\t\t\t\t\t\t\t\\\n+\t\tpmd_populate_kernel(\u0026init_mm, pmd, pte);\t\t\\\n+\t} while (0)\n+#endif\n #endif /* _LINUX_PGALLOC_H */\ndiff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h\nindex d87dc7f77f4e8a..daf9965a111d1f 100644\n--- a/include/linux/vmalloc.h\n+++ b/include/linux/vmalloc.h\n@@ -313,6 +313,9 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,\n \t\t\t\t size_t align);\n \n void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms);\n+struct vm_struct *pcpu_get_local_vm_area(unsigned long hint,\n+\t\t\t\t int unit_size, size_t align);\n+\n # else\n static inline struct vm_struct **\n pcpu_get_vm_areas(const unsigned long *offsets,\ndiff --git a/mm/Kconfig b/mm/Kconfig\nindex 9e0ca48249054f..e1910c33263851 100644\n--- a/mm/Kconfig\n+++ b/mm/Kconfig\n@@ -1061,6 +1061,9 @@ config NEED_PER_CPU_PAGE_FIRST_CHUNK\n config USE_PERCPU_NUMA_NODE_ID\n \tbool\n \n+config HAVE_LOCAL_PER_CPU_MAP\n+\tbool\n+\n config HAVE_SETUP_PER_CPU_AREA\n \tbool\n \n@@ -1509,6 +1512,12 @@ config LAZY_MMU_MODE_KUNIT_TEST\n \n \t If unsure, say N.\n \n+config ARCH_HAS_PMD_POPULATE_SYNC\n+\tbool\n+\n+config ARCH_HAS_PUD_POPULATE_SYNC\n+\tbool\n+\n source \"mm/damon/Kconfig\"\n \n endmenu\ndiff --git a/mm/internal.h b/mm/internal.h\nindex 181e79f1d6a207..9d9f08cb26a114 100644\n--- a/mm/internal.h\n+++ b/mm/internal.h\n@@ -1552,10 +1552,13 @@ void clear_vm_uninitialized_flag(struct vm_struct *vm);\n int __must_check __vmap_pages_range_noflush(unsigned long addr,\n \t\t\t unsigned long end, pgprot_t prot,\n \t\t\t struct page **pages, unsigned int page_shift);\n+int __must_check vmap_range_noflush(pgd_t *pgdir, unsigned long addr,\n+\t\t\tunsigned long end, phys_addr_t phys_addr,\n+\t\t\tpgprot_t prot, unsigned int max_page_shift);\n \n void vunmap_range_noflush(unsigned long start, unsigned long end);\n \n-void __vunmap_range_noflush(unsigned long start, unsigned long end);\n+void __vunmap_range_noflush(pgd_t *pgdir, unsigned long start, unsigned long end);\n \n static inline bool vma_is_single_threaded_private(struct vm_area_struct *vma)\n {\ndiff --git a/mm/kmsan/hooks.c b/mm/kmsan/hooks.c\nindex 8f22d1f229813f..e2a0faf344b91b 100644\n--- a/mm/kmsan/hooks.c\n+++ b/mm/kmsan/hooks.c\n@@ -135,8 +135,8 @@ static unsigned long vmalloc_origin(unsigned long addr)\n \n void kmsan_vunmap_range_noflush(unsigned long start, unsigned long end)\n {\n-\t__vunmap_range_noflush(vmalloc_shadow(start), vmalloc_shadow(end));\n-\t__vunmap_range_noflush(vmalloc_origin(start), vmalloc_origin(end));\n+\t__vunmap_range_noflush(init_mm.pgd, vmalloc_shadow(start), vmalloc_shadow(end));\n+\t__vunmap_range_noflush(init_mm.pgd, vmalloc_origin(start), vmalloc_origin(end));\n \tflush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));\n \tflush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));\n }\n@@ -181,7 +181,7 @@ int kmsan_ioremap_page_range(unsigned long start, unsigned long end,\n \t\t\tvmalloc_origin(start + off + PAGE_SIZE), prot, \u0026origin,\n \t\t\tPAGE_SHIFT);\n \t\tif (mapped) {\n-\t\t\t__vunmap_range_noflush(\n+\t\t\t__vunmap_range_noflush(init_mm.pgd,\n \t\t\t\tvmalloc_shadow(start + off),\n \t\t\t\tvmalloc_shadow(start + off + PAGE_SIZE));\n \t\t\terr = mapped;\n@@ -203,10 +203,10 @@ int kmsan_ioremap_page_range(unsigned long start, unsigned long end,\n \t\t\t__free_pages(shadow, 1);\n \t\tif (origin)\n \t\t\t__free_pages(origin, 1);\n-\t\t__vunmap_range_noflush(\n+\t\t__vunmap_range_noflush(init_mm.pgd,\n \t\t\tvmalloc_shadow(start),\n \t\t\tvmalloc_shadow(start + clean * PAGE_SIZE));\n-\t\t__vunmap_range_noflush(\n+\t\t__vunmap_range_noflush(init_mm.pgd,\n \t\t\tvmalloc_origin(start),\n \t\t\tvmalloc_origin(start + clean * PAGE_SIZE));\n \t}\n@@ -233,8 +233,8 @@ void kmsan_iounmap_page_range(unsigned long start, unsigned long end)\n \t i++, v_shadow += PAGE_SIZE, v_origin += PAGE_SIZE) {\n \t\tshadow = kmsan_vmalloc_to_page_or_null((void *)v_shadow);\n \t\torigin = kmsan_vmalloc_to_page_or_null((void *)v_origin);\n-\t\t__vunmap_range_noflush(v_shadow, vmalloc_shadow(end));\n-\t\t__vunmap_range_noflush(v_origin, vmalloc_origin(end));\n+\t\t__vunmap_range_noflush(init_mm.pgd, v_shadow, vmalloc_shadow(end));\n+\t\t__vunmap_range_noflush(init_mm.pgd, v_origin, vmalloc_origin(end));\n \t\tif (shadow)\n \t\t\t__free_pages(shadow, 1);\n \t\tif (origin)\ndiff --git a/mm/percpu-internal.h b/mm/percpu-internal.h\nindex 8cbe039bf847ac..4d461305243bf0 100644\n--- a/mm/percpu-internal.h\n+++ b/mm/percpu-internal.h\n@@ -63,11 +63,14 @@ struct pcpu_chunk {\n \t * chunk_md.\n \t */\n \tvoid\t\t\t*base_addr ____cacheline_aligned_in_smp;\n+\t/* percpu local base address of the chunk */\n+\tvoid *local_base;\n \n \tunsigned long\t\t*alloc_map;\t/* allocation map */\n \tstruct pcpu_block_md\t*md_blocks;\t/* metadata blocks */\n \n \tvoid\t\t\t*data;\t\t/* chunk data */\n+\tvoid\t\t\t*local_data;\t/* chunk local vm */\n \tbool\t\t\timmutable;\t/* no [de]population allowed */\n \tbool\t\t\tisolated;\t/* isolated from active chunk\n \t\t\t\t\t\t slots */\n@@ -162,6 +165,17 @@ static inline size_t pcpu_obj_full_size(size_t size)\n \treturn size * num_possible_cpus() + extra_size;\n }\n \n+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP\n+extern void __init map_local_percpu_first_chunk(unsigned int cpu, unsigned long virt,\n+ struct page **pages, unsigned int nr);\n+#else\n+static inline void __init map_local_percpu_first_chunk(unsigned int cpu, unsigned long virt,\n+ struct page **pages, unsigned int nr)\n+{\n+\treturn;\n+}\n+#endif\n+\n #ifdef CONFIG_PERCPU_STATS\n \n #include \u003clinux/spinlock.h\u003e\ndiff --git a/mm/percpu-vm.c b/mm/percpu-vm.c\nindex 4f5937090590d8..5decaad7640fdc 100644\n--- a/mm/percpu-vm.c\n+++ b/mm/percpu-vm.c\n@@ -112,6 +112,10 @@ static int pcpu_alloc_pages(struct pcpu_chunk *chunk,\n \treturn -ENOMEM;\n }\n \n+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP\n+extern pgd_t *percpu_pgd[NR_CPUS];\n+#endif\n+\n /**\n * pcpu_pre_unmap_flush - flush cache prior to unmapping\n * @chunk: chunk the regions to be flushed belongs to\n@@ -130,6 +134,11 @@ static void pcpu_pre_unmap_flush(struct pcpu_chunk *chunk,\n \tflush_cache_vunmap(\n \t\tpcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),\n \t\tpcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));\n+\n+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP\n+\tflush_cache_vunmap((unsigned long)chunk-\u003elocal_base + (page_start \u003c\u003c PAGE_SHIFT),\n+\t\t\t(unsigned long)chunk-\u003elocal_base + (page_end \u003c\u003c PAGE_SHIFT));\n+#endif\n }\n \n static void __pcpu_unmap_pages(unsigned long addr, int nr_pages)\n@@ -137,6 +146,20 @@ static void __pcpu_unmap_pages(unsigned long addr, int nr_pages)\n \tvunmap_range_noflush(addr, addr + (nr_pages \u003c\u003c PAGE_SHIFT));\n }\n \n+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP\n+static void __pcpu_unmap_pages_local(unsigned int cpu, unsigned long virt,\n+\t\t\t int nr_pages)\n+{\n+\t__vunmap_range_noflush(percpu_pgd[cpu], virt, virt + (nr_pages \u003c\u003c PAGE_SHIFT));\n+}\n+#else\n+static void __pcpu_unmap_pages_local(unsigned int cpu, unsigned long virt,\n+\t\t\t int nr_pages)\n+{\n+\treturn;\n+}\n+#endif\n+\n /**\n * pcpu_unmap_pages - unmap pages out of a pcpu_chunk\n * @chunk: chunk of interest\n@@ -166,6 +189,10 @@ static void pcpu_unmap_pages(struct pcpu_chunk *chunk,\n \t\t}\n \t\t__pcpu_unmap_pages(pcpu_chunk_addr(chunk, cpu, page_start),\n \t\t\t\t page_end - page_start);\n+\n+\t\t__pcpu_unmap_pages_local(cpu,\n+\t\t\t\t\t(unsigned long)chunk-\u003elocal_base + (page_start \u003c\u003c PAGE_SHIFT),\n+\t\t\t\t\tpage_end - page_start);\n \t}\n }\n \n@@ -188,6 +215,12 @@ static void pcpu_post_unmap_tlb_flush(struct pcpu_chunk *chunk,\n \tflush_tlb_kernel_range(\n \t\tpcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),\n \t\tpcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));\n+\n+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP\n+\tflush_tlb_kernel_range(\n+\t\t(unsigned long)chunk-\u003elocal_base + (page_start \u003c\u003c PAGE_SHIFT),\n+\t\t(unsigned long)chunk-\u003elocal_base + (page_end \u003c\u003c PAGE_SHIFT));\n+#endif\n }\n \n static int __pcpu_map_pages(unsigned long addr, struct page **pages,\n@@ -197,6 +230,32 @@ static int __pcpu_map_pages(unsigned long addr, struct page **pages,\n \t\t\tPAGE_KERNEL, pages, PAGE_SHIFT, GFP_KERNEL);\n }\n \n+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP\n+static int __pcpu_map_pages_local(unsigned int cpu, unsigned long virt, struct page **pages,\n+\t\t\t int nr_pages)\n+{\n+\tunsigned int i;\n+\tint err = 0;\n+\n+\tfor (i = 0; i \u003c nr_pages; i++) {\n+\t\terr = vmap_range_noflush(percpu_pgd[cpu], virt, virt + PAGE_SIZE,\n+\t\t\t\tpage_to_phys(pages[i]), PAGE_KERNEL, PAGE_SHIFT);\n+\t\tif (err)\n+\t\t\treturn err;\n+\n+\t\tvirt += PAGE_SIZE;\n+\t}\n+\n+\treturn err;\n+}\n+#else\n+static int __pcpu_map_pages_local(unsigned int cpu, unsigned long virt, struct page **pages,\n+\t\t\t int nr_pages)\n+{\n+\treturn 0;\n+}\n+#endif\n+\n /**\n * pcpu_map_pages - map pages into a pcpu_chunk\n * @chunk: chunk of interest\n@@ -224,6 +283,13 @@ static int pcpu_map_pages(struct pcpu_chunk *chunk,\n \t\tif (err \u003c 0)\n \t\t\tgoto err;\n \n+\t\terr = __pcpu_map_pages_local(cpu,\n+\t\t\t\t\t(unsigned long)chunk-\u003elocal_base + (page_start \u003c\u003c PAGE_SHIFT),\n+\t\t\t\t\t\u0026pages[pcpu_page_idx(cpu, page_start)],\n+\t\t\t\t\tpage_end - page_start);\n+\t\tif (err \u003c 0)\n+\t\t\tgoto err;\n+\n \t\tfor (i = page_start; i \u003c page_end; i++)\n \t\t\tpcpu_set_page_chunk(pages[pcpu_page_idx(cpu, i)],\n \t\t\t\t\t chunk);\n@@ -233,6 +299,9 @@ static int pcpu_map_pages(struct pcpu_chunk *chunk,\n \tfor_each_possible_cpu(tcpu) {\n \t\t__pcpu_unmap_pages(pcpu_chunk_addr(chunk, tcpu, page_start),\n \t\t\t\t page_end - page_start);\n+\t\t__pcpu_unmap_pages_local(tcpu,\n+\t\t\t\t\t(unsigned long)chunk-\u003elocal_base + (page_start \u003c\u003c PAGE_SHIFT),\n+\t\t\t\t\tpage_end - page_start);\n \t\tif (tcpu == cpu)\n \t\t\tbreak;\n \t}\n@@ -258,6 +327,11 @@ static void pcpu_post_map_flush(struct pcpu_chunk *chunk,\n \tflush_cache_vmap(\n \t\tpcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),\n \t\tpcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));\n+\n+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP\n+\tflush_cache_vmap((unsigned long)chunk-\u003elocal_base + (page_start \u003c\u003c PAGE_SHIFT),\n+\t\t\t (unsigned long)chunk-\u003elocal_base + (page_end \u003c\u003c PAGE_SHIFT));\n+#endif\n }\n \n /**\n@@ -349,6 +423,24 @@ static struct pcpu_chunk *pcpu_create_chunk(gfp_t gfp)\n \tchunk-\u003edata = vms;\n \tchunk-\u003ebase_addr = vms[0]-\u003eaddr - pcpu_group_offsets[0];\n \n+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP\n+\tunsigned long delta = (unsigned long)chunk-\u003ebase_addr - (unsigned long)pcpu_base_addr;\n+\tunsigned long hint = delta + (unsigned long)pcpu_local_base;\n+\tstruct vm_struct *local_vm = pcpu_get_local_vm_area(hint,\n+\t\t\t\t\tpcpu_unit_size, pcpu_atom_size);\n+\tif (!local_vm) {\n+\t\tpcpu_free_vm_areas(vms, pcpu_nr_groups);\n+\t\tpcpu_free_chunk(chunk);\n+\t\treturn NULL;\n+\t}\n+\n+\tchunk-\u003elocal_base = local_vm-\u003eaddr;\n+\tchunk-\u003elocal_data = (void *)local_vm;\n+#else\n+\tchunk-\u003elocal_base = 0;\n+\tchunk-\u003elocal_data = NULL;\n+#endif\n+\n \tpcpu_stats_chunk_alloc();\n \ttrace_percpu_create_chunk(chunk-\u003ebase_addr);\n \n@@ -365,6 +457,8 @@ static void pcpu_destroy_chunk(struct pcpu_chunk *chunk)\n \n \tif (chunk-\u003edata)\n \t\tpcpu_free_vm_areas(chunk-\u003edata, pcpu_nr_groups);\n+\tif (chunk-\u003elocal_data)\n+\t\tfree_vm_area((struct vm_struct *)chunk-\u003elocal_data);\n \tpcpu_free_chunk(chunk);\n }\n \ndiff --git a/mm/percpu.c b/mm/percpu.c\nindex b0676b8054ed09..8b273ed8e5a8bb 100644\n--- a/mm/percpu.c\n+++ b/mm/percpu.c\n@@ -145,6 +145,8 @@ static unsigned int pcpu_high_unit_cpu __ro_after_init;\n \n /* the address of the first chunk which starts with the kernel static area */\n void *pcpu_base_addr __ro_after_init;\n+/* The address of the first chunk local mapping */\n+void *pcpu_local_base __ro_after_init;\n \n static const int *pcpu_unit_map __ro_after_init;\t\t/* cpu -\u003e unit */\n const unsigned long *pcpu_unit_offsets __ro_after_init;\t/* cpu -\u003e unit offset */\n@@ -1338,6 +1340,7 @@ static void pcpu_init_md_blocks(struct pcpu_chunk *chunk)\n /**\n * pcpu_alloc_first_chunk - creates chunks that serve the first chunk\n * @tmp_addr: the start of the region served\n+ * @local_tmp: the start of the local percpu region served\n * @map_size: size of the region served\n *\n * This is responsible for creating the chunks that serve the first chunk. The\n@@ -1349,15 +1352,16 @@ static void pcpu_init_md_blocks(struct pcpu_chunk *chunk)\n * Chunk serving the region at @tmp_addr of @map_size.\n */\n static struct pcpu_chunk * __init pcpu_alloc_first_chunk(unsigned long tmp_addr,\n-\t\t\t\t\t\t\t int map_size)\n+\t\t\t\t\t\t\t unsigned long local_tmp, int map_size)\n {\n \tstruct pcpu_chunk *chunk;\n-\tunsigned long aligned_addr;\n+\tunsigned long aligned_addr, aligned_local;\n \tint start_offset, offset_bits, region_size, region_bits;\n \tsize_t alloc_size;\n \n \t/* region calculations */\n \taligned_addr = tmp_addr \u0026 PAGE_MASK;\n+\taligned_local = local_tmp \u0026 PAGE_MASK;\n \n \tstart_offset = tmp_addr - aligned_addr;\n \tregion_size = ALIGN(start_offset + map_size, PAGE_SIZE);\n@@ -1370,6 +1374,7 @@ static struct pcpu_chunk * __init pcpu_alloc_first_chunk(unsigned long tmp_addr,\n \tINIT_LIST_HEAD(\u0026chunk-\u003elist);\n \n \tchunk-\u003ebase_addr = (void *)aligned_addr;\n+\tchunk-\u003elocal_base = (void *)aligned_local;\n \tchunk-\u003estart_offset = start_offset;\n \tchunk-\u003eend_offset = region_size - chunk-\u003estart_offset - map_size;\n \n@@ -2562,7 +2567,7 @@ static void pcpu_dump_alloc_info(const char *lvl,\n * and available for dynamic allocation like any other chunk.\n */\n void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,\n-\t\t\t\t void *base_addr)\n+\t\t\t\t void *base_addr, void *local_base)\n {\n \tsize_t size_sum = ai-\u003estatic_size + ai-\u003ereserved_size + ai-\u003edyn_size;\n \tsize_t static_size, dyn_size;\n@@ -2572,7 +2577,7 @@ void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,\n \tunsigned int cpu;\n \tint *unit_map;\n \tint group, unit, i;\n-\tunsigned long tmp_addr;\n+\tunsigned long tmp_addr, local_tmp = 0;\n \tsize_t alloc_size;\n \n #define PCPU_SETUP_BUG_ON(cond)\tdo {\t\t\t\t\t\\\n@@ -2713,11 +2718,15 @@ void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,\n \t * chunk.\n \t */\n \ttmp_addr = (unsigned long)base_addr + static_size;\n+\tif (local_base)\n+\t\tlocal_tmp = (unsigned long)local_base + static_size;\n \tif (ai-\u003ereserved_size)\n-\t\tpcpu_reserved_chunk = pcpu_alloc_first_chunk(tmp_addr,\n+\t\tpcpu_reserved_chunk = pcpu_alloc_first_chunk(tmp_addr, local_tmp,\n \t\t\t\t\t\tai-\u003ereserved_size);\n \ttmp_addr = (unsigned long)base_addr + static_size + ai-\u003ereserved_size;\n-\tpcpu_first_chunk = pcpu_alloc_first_chunk(tmp_addr, dyn_size);\n+\tif (local_base)\n+\t\tlocal_tmp = (unsigned long)local_base + static_size + ai-\u003ereserved_size;\n+\tpcpu_first_chunk = pcpu_alloc_first_chunk(tmp_addr, local_tmp, dyn_size);\n \n \tpcpu_nr_empty_pop_pages = pcpu_first_chunk-\u003enr_empty_pop_pages;\n \tpcpu_chunk_relocate(pcpu_first_chunk, -1);\n@@ -3108,7 +3117,7 @@ int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size,\n \t\tPFN_DOWN(size_sum), ai-\u003estatic_size, ai-\u003ereserved_size,\n \t\tai-\u003edyn_size, ai-\u003eunit_size);\n \n-\tpcpu_setup_first_chunk(ai, base);\n+\tpcpu_setup_first_chunk(ai, base, NULL);\n \tgoto out_free;\n \n out_free_areas:\n@@ -3163,7 +3172,7 @@ void __init __weak pcpu_populate_pte(unsigned long addr)\n \tpud = pud_offset(p4d, addr);\n \tif (pud_none(*pud)) {\n \t\tpmd = memblock_alloc_or_panic(PMD_TABLE_SIZE, PMD_TABLE_SIZE);\n-\t\tpud_populate(\u0026init_mm, pud, pmd);\n+\t\tpud_populate_sync(addr, pud, pmd);\n \t}\n \n \tpmd = pmd_offset(pud, addr);\n@@ -3171,7 +3180,7 @@ void __init __weak pcpu_populate_pte(unsigned long addr)\n \t\tpte_t *new;\n \n \t\tnew = memblock_alloc_or_panic(PTE_TABLE_SIZE, PTE_TABLE_SIZE);\n-\t\tpmd_populate_kernel(\u0026init_mm, pmd, new);\n+\t\tpmd_populate_sync(addr, pmd, new);\n \t}\n \n \treturn;\n@@ -3194,6 +3203,7 @@ void __init __weak pcpu_populate_pte(unsigned long addr)\n int __init pcpu_page_first_chunk(size_t reserved_size, pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn)\n {\n \tstatic struct vm_struct vm;\n+\tstatic struct vm_struct pcpu_vm;\n \tstruct pcpu_alloc_info *ai;\n \tchar psize_str[16];\n \tint unit_pages;\n@@ -3243,11 +3253,28 @@ int __init pcpu_page_first_chunk(size_t reserved_size, pcpu_fc_cpu_to_node_fn_t\n \t}\n \n \t/* allocate vm area, map the pages and copy static data */\n+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP\n+\tvm.flags = VM_ALLOC;\n+\tvm.addr = (void *)ALIGN(PERCPU_START, PAGE_SIZE);\n+\tvm.size = num_possible_cpus() * ai-\u003eunit_size;\n+\tvm_area_add_early(\u0026vm);\n+\tkasan_populate_early_vm_area_shadow(vm.addr, vm.size);\n+\n+\tpcpu_vm.flags = VM_ALLOC;\n+\tpcpu_vm.addr = (void *)ALIGN(LOCAL_PERCPU_START, PAGE_SIZE);\n+\tpcpu_vm.size = ai-\u003eunit_size;\n+\tvm_area_add_early(\u0026pcpu_vm);\n+\tkasan_populate_early_vm_area_shadow(pcpu_vm.addr, pcpu_vm.size);\n+#else\n \tvm.flags = VM_ALLOC;\n \tvm.size = num_possible_cpus() * ai-\u003eunit_size;\n \tvm_area_register_early(\u0026vm, PAGE_SIZE);\n \n+\tpcpu_vm.addr = NULL;\n+#endif\n+\n \tfor (unit = 0; unit \u003c num_possible_cpus(); unit++) {\n+\t\tunsigned int cpu = ai-\u003egroups[0].cpu_map[unit];\n \t\tunsigned long unit_addr =\n \t\t\t(unsigned long)vm.addr + unit * ai-\u003eunit_size;\n \n@@ -3264,6 +3291,14 @@ int __init pcpu_page_first_chunk(size_t reserved_size, pcpu_fc_cpu_to_node_fn_t\n \n \t\t/* copy static data */\n \t\tmemcpy((void *)unit_addr, __per_cpu_start, ai-\u003estatic_size);\n+\n+\t\t/*\n+\t\t * Map percpu data to PERCPU map.\n+\t\t *\n+\t\t * PCPU_FC_EMBED can't support it.\n+\t\t */\n+\t\tmap_local_percpu_first_chunk(cpu, (unsigned long)pcpu_vm.addr,\n+\t\t\t\t\u0026pages[unit * unit_pages], unit_pages);\n \t}\n \n \t/* we're ready, commit */\n@@ -3271,7 +3306,8 @@ int __init pcpu_page_first_chunk(size_t reserved_size, pcpu_fc_cpu_to_node_fn_t\n \t\tunit_pages, psize_str, ai-\u003estatic_size,\n \t\tai-\u003ereserved_size, ai-\u003edyn_size);\n \n-\tpcpu_setup_first_chunk(ai, vm.addr);\n+\tpcpu_setup_first_chunk(ai, vm.addr, pcpu_vm.addr);\n+\tpcpu_local_base = pcpu_vm.addr;\n \tgoto out_free_ar;\n \n enomem:\n@@ -3353,7 +3389,7 @@ void __init setup_per_cpu_areas(void)\n \tai-\u003egroups[0].nr_units = 1;\n \tai-\u003egroups[0].cpu_map[0] = 0;\n \n-\tpcpu_setup_first_chunk(ai, fc);\n+\tpcpu_setup_first_chunk(ai, fc, NULL);\n \tpcpu_free_alloc_info(ai);\n }\n \ndiff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c\nindex 99e2be39671b63..463c2f8b611c03 100644\n--- a/mm/sparse-vmemmap.c\n+++ b/mm/sparse-vmemmap.c\n@@ -198,7 +198,7 @@ static pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, in\n \t\tif (!p)\n \t\t\treturn NULL;\n \t\tkernel_pte_init(p);\n-\t\tpmd_populate_kernel(\u0026init_mm, pmd, p);\n+\t\tpmd_populate_sync(addr, pmd, p);\n \t}\n \treturn pmd;\n }\n@@ -211,7 +211,7 @@ static pud_t * __meminit vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, in\n \t\tif (!p)\n \t\t\treturn NULL;\n \t\tpmd_init(p);\n-\t\tpud_populate(\u0026init_mm, pud, p);\n+\t\tpud_populate_sync(addr, pud, p);\n \t}\n \treturn pud;\n }\ndiff --git a/mm/vmalloc.c b/mm/vmalloc.c\nindex 1afca3568b9b65..3d94ca4bbd17b1 100644\n--- a/mm/vmalloc.c\n+++ b/mm/vmalloc.c\n@@ -80,6 +80,9 @@ bool is_vmalloc_addr(const void *x)\n {\n \tunsigned long addr = (unsigned long)kasan_reset_tag(x);\n \n+\tif (is_percpu_addr(addr))\n+\t\treturn true;\n+\n \treturn addr \u003e= VMALLOC_START \u0026\u0026 addr \u003c VMALLOC_END;\n }\n EXPORT_SYMBOL(is_vmalloc_addr);\n@@ -295,9 +298,9 @@ static int vmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,\n \treturn err;\n }\n \n-static int vmap_range_noflush(unsigned long addr, unsigned long end,\n-\t\t\tphys_addr_t phys_addr, pgprot_t prot,\n-\t\t\tunsigned int max_page_shift)\n+int vmap_range_noflush(pgd_t *pgdir, unsigned long addr, unsigned long end,\n+\t\t phys_addr_t phys_addr, pgprot_t prot,\n+\t\t unsigned int max_page_shift)\n {\n \tpgd_t *pgd;\n \tunsigned long start;\n@@ -314,7 +317,7 @@ static int vmap_range_noflush(unsigned long addr, unsigned long end,\n \tBUG_ON(addr \u003e= end);\n \n \tstart = addr;\n-\tpgd = pgd_offset_k(addr);\n+\tpgd = pgd_offset_pgd(pgdir, addr);\n \tdo {\n \t\tnext = pgd_addr_end(addr, end);\n \t\terr = vmap_p4d_range(pgd, addr, next, phys_addr, prot,\n@@ -334,8 +337,8 @@ int vmap_page_range(unsigned long addr, unsigned long end,\n {\n \tint err;\n \n-\terr = vmap_range_noflush(addr, end, phys_addr, pgprot_nx(prot),\n-\t\t\t\t ioremap_max_page_shift);\n+\terr = vmap_range_noflush(init_mm.pgd, addr, end, phys_addr,\n+\t\t\t\t pgprot_nx(prot), ioremap_max_page_shift);\n \tflush_cache_vmap(addr, end);\n \tif (!err)\n \t\terr = kmsan_ioremap_page_range(addr, end, phys_addr, prot,\n@@ -478,7 +481,7 @@ static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,\n *\n * This is an internal function only. Do not use outside mm/.\n */\n-void __vunmap_range_noflush(unsigned long start, unsigned long end)\n+void __vunmap_range_noflush(pgd_t *pgdir, unsigned long start, unsigned long end)\n {\n \tunsigned long next;\n \tpgd_t *pgd;\n@@ -486,7 +489,7 @@ void __vunmap_range_noflush(unsigned long start, unsigned long end)\n \tpgtbl_mod_mask mask = 0;\n \n \tBUG_ON(addr \u003e= end);\n-\tpgd = pgd_offset_k(addr);\n+\tpgd = pgd_offset_pgd(pgdir, addr);\n \tdo {\n \t\tnext = pgd_addr_end(addr, end);\n \t\tif (pgd_bad(*pgd))\n@@ -503,7 +506,7 @@ void __vunmap_range_noflush(unsigned long start, unsigned long end)\n void vunmap_range_noflush(unsigned long start, unsigned long end)\n {\n \tkmsan_vunmap_range_noflush(start, end);\n-\t__vunmap_range_noflush(start, end);\n+\t__vunmap_range_noflush(init_mm.pgd, start, end);\n }\n \n /**\n@@ -670,9 +673,10 @@ int __vmap_pages_range_noflush(unsigned long addr, unsigned long end,\n \tfor (i = 0; i \u003c nr; i += 1U \u003c\u003c (page_shift - PAGE_SHIFT)) {\n \t\tint err;\n \n-\t\terr = vmap_range_noflush(addr, addr + (1UL \u003c\u003c page_shift),\n-\t\t\t\t\tpage_to_phys(pages[i]), prot,\n-\t\t\t\t\tpage_shift);\n+\t\terr = vmap_range_noflush(init_mm.pgd, addr,\n+\t\t\t\t\t addr + (1UL \u003c\u003c page_shift),\n+\t\t\t\t\t page_to_phys(pages[i]), prot,\n+\t\t\t\t\t page_shift);\n \t\tif (err)\n \t\t\treturn err;\n \n@@ -4919,15 +4923,25 @@ pvm_find_va_enclose_addr(unsigned long addr)\n * in - the VA we start the search(reverse order);\n * out - the VA with the highest aligned end address.\n * @align: alignment for required highest address\n+ * @pcpu: whether request allocation from local percpu area\n *\n * Returns: determined end address within vmap_area\n */\n static unsigned long\n-pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align)\n+pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align, bool pcpu)\n {\n-\tunsigned long vmalloc_end = VMALLOC_END \u0026 ~(align - 1);\n+\tunsigned long vmalloc_end;\n \tunsigned long addr;\n \n+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP\n+\tif (pcpu)\n+\t\tvmalloc_end = LOCAL_PERCPU_END \u0026 ~(align - 1);\n+\telse\n+\t\tvmalloc_end = PERCPU_END \u0026 ~(align - 1);\n+#else\n+\tvmalloc_end = VMALLOC_END \u0026 ~(align - 1);\n+#endif\n+\n \tif (likely(*va)) {\n \t\tlist_for_each_entry_from_reverse((*va),\n \t\t\t\t\u0026free_vmap_area_list, list) {\n@@ -4968,14 +4982,22 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,\n \t\t\t\t const size_t *sizes, int nr_vms,\n \t\t\t\t size_t align)\n {\n-\tconst unsigned long vmalloc_start = ALIGN(VMALLOC_START, align);\n-\tconst unsigned long vmalloc_end = VMALLOC_END \u0026 ~(align - 1);\n+\tunsigned long vmalloc_start;\n+\tunsigned long vmalloc_end;\n \tstruct vmap_area **vas, *va;\n \tstruct vm_struct **vms;\n \tint area, area2, last_area, term_area;\n \tunsigned long base, start, size, end, last_end, orig_start, orig_end;\n \tbool purged = false;\n \n+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP\n+\tvmalloc_start = ALIGN(PERCPU_START, align);\n+\tvmalloc_end = PERCPU_END \u0026 ~(align - 1);\n+#else\n+\tvmalloc_start = ALIGN(VMALLOC_START, align);\n+\tvmalloc_end = VMALLOC_END \u0026 ~(align - 1);\n+#endif\n+\n \t/* verify parameters and allocate data structures */\n \tBUG_ON(offset_in_page(align) || !is_power_of_2(align));\n \tfor (last_area = 0, area = 0; area \u003c nr_vms; area++) {\n@@ -5024,7 +5046,7 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,\n \tend = start + sizes[area];\n \n \tva = pvm_find_va_enclose_addr(vmalloc_end);\n-\tbase = pvm_determine_end_from_reverse(\u0026va, align) - end;\n+\tbase = pvm_determine_end_from_reverse(\u0026va, align, false) - end;\n \n \twhile (true) {\n \t\t/*\n@@ -5045,7 +5067,7 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,\n \t\t * base downwards and then recheck.\n \t\t */\n \t\tif (base + end \u003e va-\u003eva_end) {\n-\t\t\tbase = pvm_determine_end_from_reverse(\u0026va, align) - end;\n+\t\t\tbase = pvm_determine_end_from_reverse(\u0026va, align, false) - end;\n \t\t\tterm_area = area;\n \t\t\tcontinue;\n \t\t}\n@@ -5055,7 +5077,7 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,\n \t\t */\n \t\tif (base + start \u003c va-\u003eva_start) {\n \t\t\tva = node_to_va(rb_prev(\u0026va-\u003erb_node));\n-\t\t\tbase = pvm_determine_end_from_reverse(\u0026va, align) - end;\n+\t\t\tbase = pvm_determine_end_from_reverse(\u0026va, align, false) - end;\n \t\t\tterm_area = area;\n \t\t\tcontinue;\n \t\t}\n@@ -5218,6 +5240,84 @@ void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms)\n \t\tfree_vm_area(vms[i]);\n \tkfree(vms);\n }\n+\n+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP\n+/* Find free vm area starts from hint */\n+struct vm_struct *pcpu_get_local_vm_area(unsigned long hint,\n+\t\t\t\t int unit_size, size_t align)\n+{\n+\tstruct vmap_area *tmp_va, *va;\n+\tstruct vm_struct *vm;\n+\tstruct vmap_node *vn;\n+\tunsigned long end;\n+\tint ret;\n+\n+\tva = kmem_cache_zalloc(vmap_area_cachep, GFP_KERNEL);\n+\tvm = kzalloc(sizeof(struct vm_struct), GFP_KERNEL);\n+\tif (!va || !vm)\n+\t\tgoto err_free;\n+\n+\tspin_lock(\u0026free_vmap_area_lock);\n+\n+\ttmp_va = pvm_find_va_enclose_addr(hint);\n+\tif (!tmp_va) {\n+\t\tspin_unlock(\u0026free_vmap_area_lock);\n+\t\tgoto err_free;\n+\t}\n+\n+\tend = pvm_determine_end_from_reverse(\u0026tmp_va, align, true);\n+\n+\tif (hint + unit_size \u003e end) {\n+\t\tspin_unlock(\u0026free_vmap_area_lock);\n+\t\tgoto err_free;\n+\t}\n+\n+\tret = va_clip(\u0026free_vmap_area_root,\n+\t\t\t\u0026free_vmap_area_list, tmp_va, hint, unit_size);\n+\tif (ret) {\n+\t\tspin_unlock(\u0026free_vmap_area_lock);\n+\t\tgoto err_free;\n+\t}\n+\n+\tva-\u003eva_start = hint;\n+\tva-\u003eva_end = hint + unit_size;\n+\n+\tspin_unlock(\u0026free_vmap_area_lock);\n+\n+\tif (kasan_populate_vmalloc(va-\u003eva_start, unit_size, GFP_KERNEL))\n+\t\tgoto err_free_shadow;\n+\n+\tvn = addr_to_node(va-\u003eva_start);\n+\n+\tspin_lock(\u0026vn-\u003ebusy.lock);\n+\tinsert_vmap_area(va, \u0026vn-\u003ebusy.root, \u0026vn-\u003ebusy.head);\n+\tsetup_vmalloc_vm(vm, va, VM_ALLOC,\n+\t\t\t pcpu_get_local_vm_area);\n+\tspin_unlock(\u0026vn-\u003ebusy.lock);\n+\n+\tkasan_unpoison_vmap_areas(\u0026vm, 1, KASAN_VMALLOC_PROT_NORMAL);\n+\n+\treturn vm;\n+\n+err_free:\n+\tkmem_cache_free(vmap_area_cachep, va);\n+\tkfree(vm);\n+\n+\treturn NULL;\n+\n+err_free_shadow:\n+\tspin_lock(\u0026free_vmap_area_lock);\n+\tva = merge_or_add_vmap_area_augment(va, \u0026free_vmap_area_root,\n+\t\t\t\u0026free_vmap_area_list);\n+\tif (va)\n+\t\tkasan_release_vmalloc(va-\u003eva_start, va-\u003eva_end, va-\u003eva_start, va-\u003eva_end,\n+\t\t\t KASAN_VMALLOC_PAGE_RANGE | KASAN_VMALLOC_TLB_FLUSH);\n+\tkfree(vm);\n+\tspin_unlock(\u0026free_vmap_area_lock);\n+\treturn NULL;\n+\n+}\n+#endif\n #endif\t/* CONFIG_SMP */\n \n #ifdef CONFIG_PRINTK\n"
} |
| 2/1 | 2026/07/15 22:40 | agent | patch-evaluator |
1mModel:gemini-3.1-pro-preview Args: null Results: {
"EnableConfigs": [
"HAVE_LOCAL_PER_CPU_MAP",
"ARCH_HAS_PMD_POPULATE_SYNC",
"ARCH_HAS_PUD_POPULATE_SYNC",
"NEED_PER_CPU_PAGE_FIRST_CHUNK"
],
"FocusSymbols": [
"pcpu_get_local_vm_area",
"map_local_percpu_first_chunk",
"setup_percpu_pgd",
"arch_sync_kernel_mappings"
],
"Reasoning": "The patch series introduces local per-cpu mapping support for ARM64, modifying core kernel logic for percpu memory management and page tables. This is a functional change that should be fuzzed.",
"WorthFuzzing": true
}Instruction: You are an expert Linux kernel maintainer. Your job is to review a provided patch series and determine if it makes functional changes to the kernel that should be fuzzed. IMPORTANT: The changes have ALREADY been applied and committed as the HEAD commit in your workspace. Do NOT rely on your internal knowledge of the kernel. You must actively use your code access tools to examine the actual source code and confirm any assumptions. Return WorthFuzzing=false if the patch only contains: - Modifications to Documentation/, Kconfig files, or code comments. - Purely decorative changes, such as logging (e.g., pr_err, printk) or tracepoints. - Changes to numeric constants or macros that do not functionally alter execution flow. - Code paths that are impossible to reach in virtualized environments like GCE or QEMU, even when utilizing software-emulated hardware (e.g., usb gadget, mac80211_hwsim). If it modifies reachable core kernel logic, drivers, or architectures, use your code search tools to verify the code can be executed, then return WorthFuzzing=true. When returning WorthFuzzing=true, you MUST ALSO: 1. Extract any specific kernel functions that should be heavily fuzzed into FocusSymbols. Avoid listing generic hot-path functions to prevent skewed test distributions. 2. Identify any specific CONFIG_ options required to properly test this new/modified feature. Go and look into the Kconfig files and check for ifdefs around the code, do not make assumptions. Do not list too generic configs (we already have them enabled). Only list those that specifically cover the modified code. List them in the EnableConfigs output array, and DO NOT add a 'CONFIG_' prefix (e.g., return "NET_IPV4" instead of "CONFIG_NET_IPV4"). Prefer calling several tools at the same time to save round-trips. Use set-results tool to provide results of the analysis. It must be called exactly once before the final reply. Ignore results of this tool. Prompt: For your convenience, here is the diff of the changes:
commit 2baccb3511fcadd0fa12b52c7793f8cf0180006d
Author: syz-cluster <triage@syzkaller.com>
Date: Wed Jul 15 22:40:22 2026 +0000
syz-cluster: applied patch under review
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index b3afe0688919bd..95ffd151187f24 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -41,9 +41,11 @@ config ARM64
select ARCH_HAS_NMI_SAFE_THIS_CPU_OPS
select ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
select ARCH_HAS_NONLEAF_PMD_YOUNG if ARM64_HAFT
+ select ARCH_HAS_PMD_POPULATE_SYNC
select ARCH_HAS_PREEMPT_LAZY
select ARCH_HAS_PTDUMP
select ARCH_HAS_PTE_SPECIAL
+ select ARCH_HAS_PUD_POPULATE_SYNC
select ARCH_HAS_HW_PTE_YOUNG
select ARCH_HAS_SETUP_DMA_OPS
select ARCH_HAS_SET_DIRECT_MAP
@@ -224,6 +226,7 @@ config ARM64
select MMU_GATHER_RCU_TABLE_FREE
select HAVE_RSEQ
select HAVE_RUST if RUSTC_SUPPORTS_ARM64
+ select HAVE_SETUP_PER_CPU_AREA
select HAVE_STACKPROTECTOR
select HAVE_STATIC_CALL if CFI
select HAVE_SYSCALL_TRACEPOINTS
@@ -239,6 +242,7 @@ config ARM64
select LOCK_MM_AND_FIND_VMA
select MODULES_USE_ELF_RELA
select NEED_DMA_MAP_STATE
+ select NEED_PER_CPU_PAGE_FIRST_CHUNK
select NEED_SG_DMA_LENGTH
select OF
select OF_EARLY_FLATTREE
@@ -411,6 +415,11 @@ config KASAN_SHADOW_OFFSET
config UNWIND_TABLES
bool
+config HAVE_LOCAL_PER_CPU_MAP
+ def_bool y
+ depends on (ARM64_VA_BITS_48 && (ARM64_4K_PAGES || ARM64_64K_PAGES)) || \
+ (ARM64_VA_BITS_47 && ARM64_16K_PAGES)
+
source "arch/arm64/Kconfig.platforms"
menu "Kernel Features"
@@ -1581,9 +1590,6 @@ config NUMA
bool "NUMA Memory Allocation and Scheduler Support"
select GENERIC_ARCH_NUMA
select OF_NUMA
- select HAVE_SETUP_PER_CPU_AREA
- select NEED_PER_CPU_EMBED_FIRST_CHUNK
- select NEED_PER_CPU_PAGE_FIRST_CHUNK
select USE_PERCPU_NUMA_NODE_ID
help
Enable NUMA (Non-Uniform Memory Access) support.
diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h
index 5e1211c540ab23..aa0818f7ec58c5 100644
--- a/arch/arm64/include/asm/mmu.h
+++ b/arch/arm64/include/asm/mmu.h
@@ -63,6 +63,7 @@ static inline bool arm64_kernel_unmapped_at_el0(void)
extern void arm64_memblock_init(void);
extern void paging_init(void);
extern void bootmem_init(void);
+extern void setup_percpu_pgd(void);
extern void create_mapping_noalloc(phys_addr_t phys, unsigned long virt,
phys_addr_t size, pgprot_t prot);
extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
@@ -72,6 +73,10 @@ extern void *fixmap_remap_fdt(phys_addr_t dt_phys, int *size, pgprot_t prot);
extern void mark_linear_text_alias_ro(void);
extern int split_kernel_leaf_mapping(unsigned long start, unsigned long end);
extern void linear_map_maybe_split_to_ptes(void);
+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP
+extern void map_local_percpu_first_chunk(unsigned int cpu, unsigned long virt,
+ struct page **pages, unsigned int nr);
+#endif
/*
* This check is triggered during the early boot before the cpufeature
diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h
index 803b6875815275..0e991b98ed9eaf 100644
--- a/arch/arm64/include/asm/mmu_context.h
+++ b/arch/arm64/include/asm/mmu_context.h
@@ -27,6 +27,7 @@
#include <asm/tlbflush.h>
extern bool rodata_full;
+extern pgd_t *percpu_pgd[NR_CPUS];
static inline void contextidr_thread_switch(struct task_struct *next)
{
@@ -138,7 +139,13 @@ void __cpu_replace_ttbr1(pgd_t *pgdp, bool cnp);
static inline void cpu_enable_swapper_cnp(void)
{
- __cpu_replace_ttbr1(lm_alias(swapper_pg_dir), true);
+ unsigned int cpu = smp_processor_id();
+ pgd_t *ttbr1 = percpu_pgd[cpu];
+
+ if (cpu == 0)
+ ttbr1 = lm_alias(swapper_pg_dir);
+
+ __cpu_replace_ttbr1(ttbr1, false);
}
static inline void cpu_replace_ttbr1(pgd_t *pgdp)
diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h
index b57b2bb0096774..03d149f9177c0e 100644
--- a/arch/arm64/include/asm/percpu.h
+++ b/arch/arm64/include/asm/percpu.h
@@ -142,6 +142,27 @@ PERCPU_RET_OP(add, add, ldadd)
#undef PERCPU_OP
#undef PERCPU_RET_OP
+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP
+extern unsigned long __per_cpu_local_off;
+
+#define local_cpu_ptr(ptr) \
+({ \
+ __verify_pcpu_ptr(ptr); \
+ SHIFT_PERCPU_PTR(ptr, __per_cpu_local_off); \
+})
+
+#define _pcp_protect(op, pcp, ...) \
+({ \
+ op(local_cpu_ptr(&(pcp)), __VA_ARGS__); \
+})
+
+#define _pcp_protect_return(op, pcp, args...) \
+({ \
+ typeof(pcp) __retval; \
+ __retval = (typeof(pcp))op(local_cpu_ptr(&(pcp)), ##args); \
+ __retval; \
+})
+#else
/*
* It would be nice to avoid the conditional call into the scheduler when
* re-enabling preemption for preemptible kernels, but doing that in a way
@@ -152,7 +173,6 @@ PERCPU_RET_OP(add, add, ldadd)
* preemption point when TIF_NEED_RESCHED gets set while preemption is
* disabled.
*/
-
#define _pcp_protect(op, pcp, ...) \
({ \
preempt_disable_notrace(); \
@@ -168,6 +188,7 @@ PERCPU_RET_OP(add, add, ldadd)
preempt_enable_notrace(); \
__retval; \
})
+#endif
#define this_cpu_read_1(pcp) \
_pcp_protect_return(__percpu_read_8, pcp)
@@ -243,6 +264,19 @@ PERCPU_RET_OP(add, add, ldadd)
#define this_cpu_cmpxchg64(pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n)
+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP
+#define this_cpu_cmpxchg128(pcp, o, n) \
+({ \
+ typedef typeof(pcp) pcp_op_T__; \
+ u128 old__, new__, ret__; \
+ pcp_op_T__ *ptr__; \
+ old__ = o; \
+ new__ = n; \
+ ptr__ = local_cpu_ptr(&(pcp)); \
+ ret__ = cmpxchg128_local((void *)ptr__, old__, new__); \
+ ret__; \
+})
+#else
#define this_cpu_cmpxchg128(pcp, o, n) \
({ \
typedef typeof(pcp) pcp_op_T__; \
@@ -256,6 +290,7 @@ PERCPU_RET_OP(add, add, ldadd)
preempt_enable_notrace(); \
ret__; \
})
+#endif
#ifdef __KVM_NVHE_HYPERVISOR__
extern unsigned long __hyp_per_cpu_offset(unsigned int cpu);
diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h
index 1b4509d3382c63..dd1ac84a11605e 100644
--- a/arch/arm64/include/asm/pgalloc.h
+++ b/arch/arm64/include/asm/pgalloc.h
@@ -33,11 +33,27 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp)
pudval |= (mm == &init_mm) ? PUD_TABLE_UXN : PUD_TABLE_PXN;
__pud_populate(pudp, __pa(pmdp), pudval);
}
+
+#ifdef CONFIG_ARCH_HAS_PUD_POPULATE_SYNC
+static inline void pud_populate_sync(unsigned long addr, pud_t *pudp, pmd_t *pmdp)
+{
+ pud_populate(&init_mm, pudp, pmdp);
+ if (ARCH_PAGE_TABLE_SYNC_MASK & PGTBL_PUD_MODIFIED)
+ arch_sync_kernel_mappings(addr, addr);
+}
+#endif
#else
static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot)
{
BUILD_BUG();
}
+
+#ifdef CONFIG_ARCH_HAS_PUD_POPULATE_SYNC
+static inline void pud_populate_sync(unsigned long addr, pud_t *pudp, pmd_t *pmdp)
+{
+ BUILD_BUG();
+}
+#endif
#endif /* CONFIG_PGTABLE_LEVELS > 2 */
#if CONFIG_PGTABLE_LEVELS > 3
@@ -121,4 +137,12 @@ pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep)
PMD_TYPE_TABLE | PMD_TABLE_AF | PMD_TABLE_PXN);
}
+#ifdef CONFIG_ARCH_HAS_PMD_POPULATE_SYNC
+static inline void pmd_populate_sync(unsigned long addr, pmd_t *pmdp, pte_t *ptep)
+{
+ pmd_populate_kernel(&init_mm, pmdp, ptep);
+ if (ARCH_PAGE_TABLE_SYNC_MASK & PGTBL_PMD_MODIFIED)
+ arch_sync_kernel_mappings(addr, addr);
+}
+#endif
#endif
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 27689c62bd25dd..83cd65610d7f7d 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -18,14 +18,35 @@
* VMALLOC range.
*
* VMALLOC_START: beginning of the kernel vmalloc space
- * VMALLOC_END: extends to the available space below vmemmap
+ * VMALLOC_END: extends to the space below global percpu area
*/
#define VMALLOC_START (MODULES_END)
+#define VMALLOC_END (PERCPU_START - SZ_8M)
+
+/*
+ * PERCPU range
+ *
+ * PERCPU_START: beginning of global percpu area
+ * PERCPU_END: end of global percpu area
+ * LOCAL_PERCPU_START: beginning of local percpu area
+ * LOCAL_PERCPU_END: end of local percpu area, extend to the available
+ * space below vmemmap
+ */
+#if defined(CONFIG_ARM64_4K_PAGES)
+#define PERCPU_SIZE (2 * PGDIR_SIZE)
+#elif defined(CONFIG_ARM64_16K_PAGES)
+#define PERCPU_SIZE (16 * PUD_SIZE)
+#elif defined(CONFIG_ARM64_64K_PAGES)
+#define PERCPU_SIZE (PGDIR_SIZE)
+#endif
+#define PERCPU_START (PERCPU_END - PERCPU_SIZE)
+#define PERCPU_END (LOCAL_PERCPU_START)
+#define LOCAL_PERCPU_START (LOCAL_PERCPU_END - PERCPU_SIZE)
#if VA_BITS == VA_BITS_MIN
-#define VMALLOC_END (VMEMMAP_START - SZ_8M)
+#define LOCAL_PERCPU_END (ALIGN_DOWN(VMEMMAP_START, PERCPU_SIZE))
#else
#define VMEMMAP_UNUSED_NPAGES ((_PAGE_OFFSET(vabits_actual) - PAGE_OFFSET) >> PAGE_SHIFT)
-#define VMALLOC_END (VMEMMAP_START + VMEMMAP_UNUSED_NPAGES * sizeof(struct page) - SZ_8M)
+#define LOCAL_PERCPU_END (ALIGN_DOWN((VMEMMAP_START + VMEMMAP_UNUSED_NPAGES * sizeof(struct page)), PERCPU_SIZE))
#endif
#define vmemmap ((struct page *)VMEMMAP_START - (memstart_addr >> PAGE_SHIFT))
@@ -1216,6 +1237,16 @@ p4d_t *p4d_offset_lockless_folded(pgd_t *pgdp, pgd_t pgd, unsigned long addr)
#endif /* CONFIG_PGTABLE_LEVELS > 4 */
+#if CONFIG_PGTABLE_LEVELS == 2
+#define ARCH_PAGE_TABLE_SYNC_MASK PGTBL_PMD_MODIFIED
+#elif CONFIG_PGTABLE_LEVELS == 3
+#define ARCH_PAGE_TABLE_SYNC_MASK PGTBL_PUD_MODIFIED
+#elif CONFIG_PGTABLE_LEVELS == 4
+#define ARCH_PAGE_TABLE_SYNC_MASK PGTBL_P4D_MODIFIED
+#elif CONFIG_PGTABLE_LEVELS == 5
+#define ARCH_PAGE_TABLE_SYNC_MASK PGTBL_PGD_MODIFIED
+#endif
+
#define pgd_ERROR(e) \
pr_err("%s:%d: bad pgd %016llx.\n", __FILE__, __LINE__, pgd_val(e))
diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index 23c05dc7a8f2ac..6d420ad59af492 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -360,6 +360,9 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p)
smp_init_cpus();
smp_build_mpidr_hash();
+ /* Must be called after smp_init_cpus */
+ setup_percpu_pgd();
+
#ifdef CONFIG_ARM64_SW_TTBR0_PAN
/*
* Make sure init_thread_info.ttbr0 always generates translation
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index d46022f7207540..4a811379c19162 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -55,6 +55,8 @@
#include <trace/events/ipi.h>
+extern void idmap_cpu_replace_ttbr1(phys_addr_t pgdir);
+
/*
* as from 2.5, kernels no longer have an init_tasks structure
* so we need some other way of telling a new secondary core
@@ -198,6 +200,12 @@ asmlinkage notrace void secondary_start_kernel(void)
struct mm_struct *mm = &init_mm;
const struct cpu_operations *ops;
unsigned int cpu = smp_processor_id();
+ typedef void (ttbr_replace_func)(phys_addr_t);
+ ttbr_replace_func *replace_ttbr;
+
+ phys_addr_t ttbr1 = phys_to_ttbr(virt_to_phys(percpu_pgd[cpu]));
+ replace_ttbr = (void *)__pa_symbol(idmap_cpu_replace_ttbr1);
+ replace_ttbr(ttbr1);
/*
* All kernel threads share the same mm context; grab a
@@ -819,6 +827,42 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
}
}
+#ifdef CONFIG_NUMA
+extern int cpu_to_node_map[NR_CPUS];
+
+int early_cpu_to_node(int cpu)
+{
+ return cpu_to_node_map[cpu];
+}
+#endif
+
+#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
+unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
+EXPORT_SYMBOL(__per_cpu_offset);
+/* Used to calculate pcpu local address, the offset is same for all CPUs */
+unsigned long __per_cpu_local_off __ro_after_init;
+EXPORT_SYMBOL(__per_cpu_local_off);
+
+void __init setup_per_cpu_areas(void)
+{
+ unsigned long delta;
+ unsigned int cpu;
+ int rc = -EINVAL;
+
+#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
+ /* PCPU page table just can support PCPU_FC_PAGE */
+ rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, early_cpu_to_node);
+#endif
+ if (rc < 0)
+ panic("Failed to initialize percpu areas (err=%d).", rc);
+
+ delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
+ for_each_possible_cpu(cpu)
+ __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
+ __per_cpu_local_off = (unsigned long)pcpu_local_base - (unsigned long)__per_cpu_start;
+}
+#endif
+
static const char *ipi_types[MAX_IPI] __tracepoint_string = {
[IPI_RESCHEDULE] = "Rescheduling interrupts",
[IPI_CALL_FUNC] = "Function call interrupts",
diff --git a/arch/arm64/mm/kasan_init.c b/arch/arm64/mm/kasan_init.c
index 3fcad956fdf7cb..a8e272bfefe1a1 100644
--- a/arch/arm64/mm/kasan_init.c
+++ b/arch/arm64/mm/kasan_init.c
@@ -60,13 +60,14 @@ static phys_addr_t __init kasan_alloc_raw_page(int node)
}
static pte_t *__init kasan_pte_offset(pmd_t *pmdp, unsigned long addr, int node,
- bool early)
+ bool early, pgtbl_mod_mask *mask)
{
if (pmd_none(READ_ONCE(*pmdp))) {
phys_addr_t pte_phys = early ?
__pa_symbol(kasan_early_shadow_pte)
: kasan_alloc_zeroed_page(node);
__pmd_populate(pmdp, pte_phys, PMD_TYPE_TABLE);
+ *mask |= PGTBL_PMD_MODIFIED;
}
return early ? pte_offset_kimg(pmdp, addr)
@@ -74,49 +75,53 @@ static pte_t *__init kasan_pte_offset(pmd_t *pmdp, unsigned long addr, int node,
}
static pmd_t *__init kasan_pmd_offset(pud_t *pudp, unsigned long addr, int node,
- bool early)
+ bool early, pgtbl_mod_mask *mask)
{
if (pud_none(READ_ONCE(*pudp))) {
phys_addr_t pmd_phys = early ?
__pa_symbol(kasan_early_shadow_pmd)
: kasan_alloc_zeroed_page(node);
__pud_populate(pudp, pmd_phys, PUD_TYPE_TABLE);
+ *mask |= PGTBL_PUD_MODIFIED;
}
return early ? pmd_offset_kimg(pudp, addr) : pmd_offset(pudp, addr);
}
static pud_t *__init kasan_pud_offset(p4d_t *p4dp, unsigned long addr, int node,
- bool early)
+ bool early, pgtbl_mod_mask *mask)
{
if (p4d_none(READ_ONCE(*p4dp))) {
phys_addr_t pud_phys = early ?
__pa_symbol(kasan_early_shadow_pud)
: kasan_alloc_zeroed_page(node);
__p4d_populate(p4dp, pud_phys, P4D_TYPE_TABLE);
+ *mask |= PGTBL_P4D_MODIFIED;
}
return early ? pud_offset_kimg(p4dp, addr) : pud_offset(p4dp, addr);
}
static p4d_t *__init kasan_p4d_offset(pgd_t *pgdp, unsigned long addr, int node,
- bool early)
+ bool early, pgtbl_mod_mask *mask)
{
if (pgd_none(READ_ONCE(*pgdp))) {
phys_addr_t p4d_phys = early ?
__pa_symbol(kasan_early_shadow_p4d)
: kasan_alloc_zeroed_page(node);
__pgd_populate(pgdp, p4d_phys, PGD_TYPE_TABLE);
+ *mask |= PGTBL_PGD_MODIFIED;
}
return early ? p4d_offset_kimg(pgdp, addr) : p4d_offset(pgdp, addr);
}
static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr,
- unsigned long end, int node, bool early)
+ unsigned long end, int node, bool early,
+ pgtbl_mod_mask *mask)
{
unsigned long next;
- pte_t *ptep = kasan_pte_offset(pmdp, addr, node, early);
+ pte_t *ptep = kasan_pte_offset(pmdp, addr, node, early, mask);
do {
phys_addr_t page_phys = early ?
@@ -130,38 +135,41 @@ static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr,
}
static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr,
- unsigned long end, int node, bool early)
+ unsigned long end, int node, bool early,
+ pgtbl_mod_mask *mask)
{
unsigned long next;
- pmd_t *pmdp = kasan_pmd_offset(pudp, addr, node, early);
+ pmd_t *pmdp = kasan_pmd_offset(pudp, addr, node, early, mask);
do {
next = pmd_addr_end(addr, end);
- kasan_pte_populate(pmdp, addr, next, node, early);
+ kasan_pte_populate(pmdp, addr, next, node, early, mask);
} while (pmdp++, addr = next, addr != end && pmd_none(READ_ONCE(*pmdp)));
}
static void __init kasan_pud_populate(p4d_t *p4dp, unsigned long addr,
- unsigned long end, int node, bool early)
+ unsigned long end, int node, bool early,
+ pgtbl_mod_mask *mask)
{
unsigned long next;
- pud_t *pudp = kasan_pud_offset(p4dp, addr, node, early);
+ pud_t *pudp = kasan_pud_offset(p4dp, addr, node, early, mask);
do {
next = pud_addr_end(addr, end);
- kasan_pmd_populate(pudp, addr, next, node, early);
+ kasan_pmd_populate(pudp, addr, next, node, early, mask);
} while (pudp++, addr = next, addr != end && pud_none(READ_ONCE(*pudp)));
}
static void __init kasan_p4d_populate(pgd_t *pgdp, unsigned long addr,
- unsigned long end, int node, bool early)
+ unsigned long end, int node, bool early,
+ pgtbl_mod_mask *mask)
{
unsigned long next;
- p4d_t *p4dp = kasan_p4d_offset(pgdp, addr, node, early);
+ p4d_t *p4dp = kasan_p4d_offset(pgdp, addr, node, early, mask);
do {
next = p4d_addr_end(addr, end);
- kasan_pud_populate(p4dp, addr, next, node, early);
+ kasan_pud_populate(p4dp, addr, next, node, early, mask);
} while (p4dp++, addr = next, addr != end && p4d_none(READ_ONCE(*p4dp)));
}
@@ -170,12 +178,17 @@ static void __init kasan_pgd_populate(unsigned long addr, unsigned long end,
{
unsigned long next;
pgd_t *pgdp;
+ pgtbl_mod_mask mask = 0;
+ unsigned long start = addr;
pgdp = pgd_offset_k(addr);
do {
next = pgd_addr_end(addr, end);
- kasan_p4d_populate(pgdp, addr, next, node, early);
+ kasan_p4d_populate(pgdp, addr, next, node, early, &mask);
} while (pgdp++, addr = next, addr != end);
+
+ if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
+ arch_sync_kernel_mappings(start, end);
}
#if defined(CONFIG_ARM64_64K_PAGES) || CONFIG_PGTABLE_LEVELS > 4
@@ -311,7 +324,7 @@ static void __init kasan_init_shadow(void)
mod_shadow_start = (u64)kasan_mem_to_shadow((void *)MODULES_VADDR);
- vmalloc_shadow_end = (u64)kasan_mem_to_shadow((void *)VMALLOC_END);
+ vmalloc_shadow_end = (u64)kasan_mem_to_shadow((void *)LOCAL_PERCPU_END);
/*
* We are going to perform proper setup of shadow memory.
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index f2be501468ce5a..ee4ab031ebf8d7 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -67,6 +67,63 @@ long __section(".mmuoff.data.write") __early_cpu_boot_status;
static DEFINE_SPINLOCK(swapper_pgdir_lock);
static DEFINE_MUTEX(fixmap_lock);
+pgd_t *percpu_pgd[NR_CPUS] __ro_after_init;
+bool percpu_pgd_setup_done __ro_after_init = false;
+
+void __init setup_percpu_pgd(void)
+{
+ int cpu;
+
+ for_each_possible_cpu(cpu) {
+ void *addr;
+
+ if (cpu == 0) {
+ percpu_pgd[cpu] = swapper_pg_dir;
+ continue;
+ }
+
+ addr = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
+ if (!addr)
+ panic("Can't alloc percpu pgd\n");
+
+ memcpy(addr, (void *)swapper_pg_dir, PAGE_SIZE);
+ percpu_pgd[cpu] = (pgd_t *)addr;
+ }
+
+ dsb(ishst);
+
+ percpu_pgd_setup_done = true;
+}
+
+void arch_sync_kernel_mappings(unsigned long start, unsigned long end)
+{
+ unsigned long addr, next;
+ int cpu;
+ pgd_t *pgdp = pgd_offset_k(start);
+ pgd_t pgd;
+ unsigned int index = pgd_index(start);
+
+ BUG_ON(start > end);
+
+ if (!percpu_pgd_setup_done)
+ return;
+
+ /* Don't sync local percpu area page table */
+ if (start >= LOCAL_PERCPU_START && end < LOCAL_PERCPU_END)
+ return;
+
+ addr = start;
+ do {
+ pgd = READ_ONCE(*pgdp);
+ next = pgd_addr_end(addr, end);
+ for_each_possible_cpu(cpu) {
+ if (cpu == 0)
+ continue;
+ set_pgd(percpu_pgd[cpu] + index, pgd);
+ }
+ } while (pgdp++, index++, addr = next, addr != end);
+}
+
void noinstr set_swapper_pgd(pgd_t *pgdp, pgd_t pgd)
{
pgd_t *fixmap_pgdp;
@@ -161,7 +218,7 @@ static void init_clear_pgtable(void *table)
}
static void init_pte(pte_t *ptep, unsigned long addr, unsigned long end,
- phys_addr_t phys, pgprot_t prot)
+ phys_addr_t phys, pgprot_t prot, pgtbl_mod_mask *mask)
{
do {
pte_t old_pte = __ptep_get(ptep);
@@ -171,6 +228,7 @@ static void init_pte(pte_t *ptep, unsigned long addr, unsigned long end,
* are deferred to the end of alloc_init_cont_pte().
*/
__set_pte_nosync(ptep, pfn_pte(__phys_to_pfn(phys), prot));
+ *mask |= PGTBL_PTE_MODIFIED;
/*
* After the PTE entry has been populated once, we
@@ -198,7 +256,7 @@ static int alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
unsigned long end, phys_addr_t phys,
pgprot_t prot,
phys_addr_t (*pgtable_alloc)(enum pgtable_level),
- int flags)
+ int flags, pgtbl_mod_mask *mask)
{
unsigned long next;
pmd_t pmd = READ_ONCE(*pmdp);
@@ -219,6 +277,7 @@ static int alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
init_clear_pgtable(ptep);
ptep += pte_index(addr);
__pmd_populate(pmdp, pte_phys, pmdval);
+ *mask |= PGTBL_PMD_MODIFIED;
} else {
BUG_ON(pmd_bad(pmd));
ptep = pte_set_fixmap_offset(pmdp, addr);
@@ -235,7 +294,7 @@ static int alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
!pte_range_has_valid_noncont(ptep))
__prot = __pgprot(pgprot_val(prot) | PTE_CONT);
- init_pte(ptep, addr, next, phys, __prot);
+ init_pte(ptep, addr, next, phys, __prot, mask);
ptep += pte_index(next) - pte_index(addr);
phys += next - addr;
@@ -253,7 +312,8 @@ static int alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end,
phys_addr_t phys, pgprot_t prot,
- phys_addr_t (*pgtable_alloc)(enum pgtable_level), int flags)
+ phys_addr_t (*pgtable_alloc)(enum pgtable_level), int flags,
+ pgtbl_mod_mask *mask)
{
unsigned long next;
@@ -266,7 +326,10 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end,
if (((addr | next | phys) & ~PMD_MASK) == 0 &&
(flags & NO_BLOCK_MAPPINGS) == 0 &&
!pmd_table(old_pmd)) {
- WARN_ON(!pmd_set_huge(pmdp, phys, prot));
+ if (pmd_set_huge(pmdp, phys, prot))
+ *mask |= PGTBL_PMD_MODIFIED;
+ else
+ WARN_ON(1);
/*
* After the PMD entry has been populated once, we
@@ -278,7 +341,7 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end,
int ret;
ret = alloc_init_cont_pte(pmdp, addr, next, phys, prot,
- pgtable_alloc, flags);
+ pgtable_alloc, flags, mask);
if (ret)
return ret;
@@ -306,7 +369,7 @@ static int alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
unsigned long end, phys_addr_t phys,
pgprot_t prot,
phys_addr_t (*pgtable_alloc)(enum pgtable_level),
- int flags)
+ int flags, pgtbl_mod_mask *mask)
{
int ret;
unsigned long next;
@@ -331,6 +394,7 @@ static int alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
init_clear_pgtable(pmdp);
pmdp += pmd_index(addr);
__pud_populate(pudp, pmd_phys, pudval);
+ *mask |= PGTBL_PUD_MODIFIED;
} else {
BUG_ON(pud_bad(pud));
pmdp = pmd_set_fixmap_offset(pudp, addr);
@@ -347,7 +411,8 @@ static int alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
!pmd_range_has_valid_noncont(pmdp))
__prot = __pgprot(pgprot_val(prot) | PTE_CONT);
- ret = init_pmd(pmdp, addr, next, phys, __prot, pgtable_alloc, flags);
+ ret = init_pmd(pmdp, addr, next, phys, __prot, pgtable_alloc, flags,
+ mask);
if (ret)
goto out;
@@ -364,7 +429,7 @@ static int alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,
phys_addr_t phys, pgprot_t prot,
phys_addr_t (*pgtable_alloc)(enum pgtable_level),
- int flags)
+ int flags, pgtbl_mod_mask *mask)
{
int ret = 0;
unsigned long next;
@@ -385,6 +450,7 @@ static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,
init_clear_pgtable(pudp);
pudp += pud_index(addr);
__p4d_populate(p4dp, pud_phys, p4dval);
+ *mask |= PGTBL_P4D_MODIFIED;
} else {
BUG_ON(p4d_bad(p4d));
pudp = pud_set_fixmap_offset(p4dp, addr);
@@ -402,7 +468,10 @@ static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,
((addr | next | phys) & ~PUD_MASK) == 0 &&
(flags & NO_BLOCK_MAPPINGS) == 0 &&
!pud_table(old_pud)) {
- WARN_ON(!pud_set_huge(pudp, phys, prot));
+ if (pud_set_huge(pudp, phys, prot))
+ *mask |= PGTBL_PUD_MODIFIED;
+ else
+ WARN_ON(1);
/*
* After the PUD entry has been populated once, we
@@ -412,7 +481,7 @@ static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,
READ_ONCE(pud_val(*pudp))));
} else {
ret = alloc_init_cont_pmd(pudp, addr, next, phys, prot,
- pgtable_alloc, flags);
+ pgtable_alloc, flags, mask);
if (ret)
goto out;
@@ -431,7 +500,7 @@ static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,
static int alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end,
phys_addr_t phys, pgprot_t prot,
phys_addr_t (*pgtable_alloc)(enum pgtable_level),
- int flags)
+ int flags, pgtbl_mod_mask *mask)
{
int ret;
unsigned long next;
@@ -452,6 +521,7 @@ static int alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end,
init_clear_pgtable(p4dp);
p4dp += p4d_index(addr);
__pgd_populate(pgdp, p4d_phys, pgdval);
+ *mask |= PGTBL_PGD_MODIFIED;
} else {
BUG_ON(pgd_bad(pgd));
p4dp = p4d_set_fixmap_offset(pgdp, addr);
@@ -463,7 +533,7 @@ static int alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end,
next = p4d_addr_end(addr, end);
ret = alloc_init_pud(p4dp, addr, next, phys, prot,
- pgtable_alloc, flags);
+ pgtable_alloc, flags, mask);
if (ret)
goto out;
@@ -483,7 +553,7 @@ static int __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys,
unsigned long virt, phys_addr_t size,
pgprot_t prot,
phys_addr_t (*pgtable_alloc)(enum pgtable_level),
- int flags)
+ int flags, pgtbl_mod_mask *mask)
{
int ret;
unsigned long addr, end, next;
@@ -503,7 +573,7 @@ static int __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys,
do {
next = pgd_addr_end(addr, end);
ret = alloc_init_p4d(pgdp, addr, next, phys, prot, pgtable_alloc,
- flags);
+ flags, mask);
if (ret)
return ret;
phys += next - addr;
@@ -516,13 +586,13 @@ static int __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
unsigned long virt, phys_addr_t size,
pgprot_t prot,
phys_addr_t (*pgtable_alloc)(enum pgtable_level),
- int flags)
+ int flags, pgtbl_mod_mask *mask)
{
int ret;
mutex_lock(&fixmap_lock);
ret = __create_pgd_mapping_locked(pgdir, phys, virt, size, prot,
- pgtable_alloc, flags);
+ pgtable_alloc, flags, mask);
mutex_unlock(&fixmap_lock);
return ret;
@@ -535,9 +605,10 @@ static void early_create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
int flags)
{
int ret;
+ pgtbl_mod_mask mask = 0;
ret = __create_pgd_mapping(pgdir, phys, virt, size, prot, pgtable_alloc,
- flags);
+ flags, &mask);
if (ret)
panic("Failed to create page tables\n");
}
@@ -1009,6 +1080,29 @@ void __init linear_map_maybe_split_to_ptes(void)
}
}
+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP
+void __init map_local_percpu_first_chunk(unsigned int cpu, unsigned long virt,
+ struct page **pages, unsigned int nr)
+{
+ int i, ret;
+ pgtbl_mod_mask mask = 0;
+
+ arch_enter_lazy_mmu_mode();
+
+ for (i = 0; i < nr; i++) {
+ phys_addr_t phys = page_to_phys(pages[i]);
+ ret = __create_pgd_mapping(percpu_pgd[cpu], phys, virt, PAGE_SIZE, PAGE_KERNEL,
+ early_pgtable_alloc, NO_EXEC_MAPPINGS, &mask);
+ if (ret)
+ panic("Failed to create local percpu first chunk\n");
+
+ virt += PAGE_SIZE;
+ }
+
+ arch_leave_lazy_mmu_mode();
+}
+#endif
+
/*
* This function can only be used to modify existing table entries,
* without allocating new levels of table. Note that this permits the
@@ -1279,6 +1373,7 @@ static int __init __kpti_install_ng_mappings(void *__unused)
if (!cpu) {
int ret;
+ pgtbl_mod_mask mask = 0;
alloc = __get_free_pages(GFP_ATOMIC | __GFP_ZERO, order);
kpti_ng_temp_pgd = (pgd_t *)(alloc + (levels - 1) * PAGE_SIZE);
@@ -1302,7 +1397,7 @@ static int __init __kpti_install_ng_mappings(void *__unused)
//
ret = __create_pgd_mapping_locked(kpti_ng_temp_pgd, __pa(alloc),
KPTI_NG_TEMP_VA, PAGE_SIZE, PAGE_KERNEL,
- kpti_ng_pgd_alloc, 0);
+ kpti_ng_pgd_alloc, 0, &mask);
if (ret)
panic("Failed to create page tables\n");
}
@@ -1608,7 +1703,7 @@ static void unmap_hotplug_range(unsigned long addr, unsigned long end,
static void free_empty_pte_table(pmd_t *pmdp, unsigned long addr,
unsigned long end, unsigned long floor,
- unsigned long ceiling)
+ unsigned long ceiling, pgtbl_mod_mask *mask)
{
pte_t *ptep, pte;
unsigned long i, start = addr;
@@ -1624,6 +1719,7 @@ static void free_empty_pte_table(pmd_t *pmdp, unsigned long addr,
WARN_ON(!pte_none(pte));
} while (addr += PAGE_SIZE, addr < end);
+ *mask |= PGTBL_PTE_MODIFIED;
if (!pgtable_range_aligned(start, end, floor, ceiling, PMD_MASK))
return;
@@ -1639,13 +1735,14 @@ static void free_empty_pte_table(pmd_t *pmdp, unsigned long addr,
}
pmd_clear(pmdp);
+ *mask |= PGTBL_PMD_MODIFIED;
__flush_tlb_kernel_pgtable(start);
free_hotplug_pgtable_page(virt_to_page(ptep));
}
static void free_empty_pmd_table(pud_t *pudp, unsigned long addr,
unsigned long end, unsigned long floor,
- unsigned long ceiling)
+ unsigned long ceiling, pgtbl_mod_mask *mask)
{
pmd_t *pmdp, pmd;
unsigned long i, next, start = addr;
@@ -1658,7 +1755,7 @@ static void free_empty_pmd_table(pud_t *pudp, unsigned long addr,
continue;
WARN_ON(!pmd_present(pmd) || !pmd_table(pmd));
- free_empty_pte_table(pmdp, addr, next, floor, ceiling);
+ free_empty_pte_table(pmdp, addr, next, floor, ceiling, mask);
} while (addr = next, addr < end);
if (CONFIG_PGTABLE_LEVELS <= 2)
@@ -1679,13 +1776,14 @@ static void free_empty_pmd_table(pud_t *pudp, unsigned long addr,
}
pud_clear(pudp);
+ *mask |= PGTBL_PUD_MODIFIED;
__flush_tlb_kernel_pgtable(start);
free_hotplug_pgtable_page(virt_to_page(pmdp));
}
static void free_empty_pud_table(p4d_t *p4dp, unsigned long addr,
unsigned long end, unsigned long floor,
- unsigned long ceiling)
+ unsigned long ceiling, pgtbl_mod_mask *mask)
{
pud_t *pudp, pud;
unsigned long i, next, start = addr;
@@ -1698,7 +1796,7 @@ static void free_empty_pud_table(p4d_t *p4dp, unsigned long addr,
continue;
WARN_ON(!pud_present(pud) || !pud_table(pud));
- free_empty_pmd_table(pudp, addr, next, floor, ceiling);
+ free_empty_pmd_table(pudp, addr, next, floor, ceiling, mask);
} while (addr = next, addr < end);
if (!pgtable_l4_enabled())
@@ -1719,13 +1817,14 @@ static void free_empty_pud_table(p4d_t *p4dp, unsigned long addr,
}
p4d_clear(p4dp);
+ *mask |= PGTBL_P4D_MODIFIED;
__flush_tlb_kernel_pgtable(start);
free_hotplug_pgtable_page(virt_to_page(pudp));
}
static void free_empty_p4d_table(pgd_t *pgdp, unsigned long addr,
unsigned long end, unsigned long floor,
- unsigned long ceiling)
+ unsigned long ceiling, pgtbl_mod_mask *mask)
{
p4d_t *p4dp, p4d;
unsigned long i, next, start = addr;
@@ -1738,7 +1837,7 @@ static void free_empty_p4d_table(pgd_t *pgdp, unsigned long addr,
continue;
WARN_ON(!p4d_present(p4d));
- free_empty_pud_table(p4dp, addr, next, floor, ceiling);
+ free_empty_pud_table(p4dp, addr, next, floor, ceiling, mask);
} while (addr = next, addr < end);
if (!pgtable_l5_enabled())
@@ -1759,6 +1858,7 @@ static void free_empty_p4d_table(pgd_t *pgdp, unsigned long addr,
}
pgd_clear(pgdp);
+ *mask |= PGTBL_PGD_MODIFIED;
__flush_tlb_kernel_pgtable(start);
free_hotplug_pgtable_page(virt_to_page(p4dp));
}
@@ -1768,6 +1868,8 @@ static void free_empty_tables(unsigned long addr, unsigned long end,
{
unsigned long next;
pgd_t *pgdp, pgd;
+ pgtbl_mod_mask mask = 0;
+ unsigned long start = addr;
do {
next = pgd_addr_end(addr, end);
@@ -1777,8 +1879,11 @@ static void free_empty_tables(unsigned long addr, unsigned long end,
continue;
WARN_ON(!pgd_present(pgd));
- free_empty_p4d_table(pgdp, addr, next, floor, ceiling);
+ free_empty_p4d_table(pgdp, addr, next, floor, ceiling, &mask);
} while (addr = next, addr < end);
+
+ if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
+ arch_sync_kernel_mappings(start, end);
}
#endif
@@ -1986,6 +2091,7 @@ int arch_add_memory(int nid, u64 start, u64 size,
struct mhp_params *params)
{
int ret, flags = NO_EXEC_MAPPINGS;
+ pgtbl_mod_mask mask = 0;
VM_BUG_ON(!mhp_range_allowed(start, size, true));
@@ -1994,10 +2100,13 @@ int arch_add_memory(int nid, u64 start, u64 size,
ret = __create_pgd_mapping(swapper_pg_dir, start, __phys_to_virt(start),
size, params->pgprot, pgd_pgtable_alloc_init_mm,
- flags);
+ flags, &mask);
if (ret)
goto err;
+ if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
+ arch_sync_kernel_mappings(start, start + size);
+
memblock_clear_nomap(start, size);
ret = __add_pages(nid, start >> PAGE_SHIFT, size >> PAGE_SHIFT,
diff --git a/arch/arm64/mm/ptdump.c b/arch/arm64/mm/ptdump.c
index 1c20144700d7d7..dd293319823e3c 100644
--- a/arch/arm64/mm/ptdump.c
+++ b/arch/arm64/mm/ptdump.c
@@ -389,6 +389,10 @@ static int __init ptdump_init(void)
{ MODULES_END, "Modules end" },
{ VMALLOC_START, "vmalloc() area" },
{ VMALLOC_END, "vmalloc() end" },
+ { PERCPU_START, "Global percpu start" },
+ { PERCPU_END, "Global percpu end" },
+ { LOCAL_PERCPU_START, "Local percpu start" },
+ { LOCAL_PERCPU_END, "Local percpu end" },
{ vmemmap_start, "vmemmap start" },
{ VMEMMAP_END, "vmemmap end" },
{ PCI_IO_START, "PCI I/O start" },
diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c
index fa66f9c97d748d..d420ff9dcf4fc8 100644
--- a/arch/riscv/kernel/smp.c
+++ b/arch/riscv/kernel/smp.c
@@ -364,3 +364,54 @@ void kgdb_roundup_cpus(void)
}
}
#endif
+
+#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
+extern int cpu_to_node_map[NR_CPUS];
+
+unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
+EXPORT_SYMBOL(__per_cpu_offset);
+
+int early_cpu_to_node(int cpu)
+{
+ return cpu_to_node_map[cpu];
+}
+
+static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)
+{
+ return node_distance(early_cpu_to_node(from), early_cpu_to_node(to));
+}
+
+void __init setup_per_cpu_areas(void)
+{
+ unsigned long delta;
+ unsigned int cpu;
+ int rc = -EINVAL;
+
+ if (pcpu_chosen_fc != PCPU_FC_PAGE) {
+ /*
+ * Always reserve area for module percpu variables. That's
+ * what the legacy allocator did.
+ */
+ rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE,
+ PERCPU_DYNAMIC_RESERVE, PAGE_SIZE,
+ pcpu_cpu_distance,
+ early_cpu_to_node);
+#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
+ if (rc < 0)
+ pr_warn("PERCPU: %s allocator failed (%d), falling back to page size\n",
+ pcpu_fc_names[pcpu_chosen_fc], rc);
+#endif
+ }
+
+#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
+ if (rc < 0)
+ rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, early_cpu_to_node);
+#endif
+ if (rc < 0)
+ panic("Failed to initialize percpu areas (err=%d).", rc);
+
+ delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
+ for_each_possible_cpu(cpu)
+ __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
+}
+#endif
diff --git a/drivers/base/arch_numa.c b/drivers/base/arch_numa.c
index c99f2ab105e5b7..b3b91ceed6a90d 100644
--- a/drivers/base/arch_numa.c
+++ b/drivers/base/arch_numa.c
@@ -16,7 +16,7 @@
#include <asm/sections.h>
-static int cpu_to_node_map[NR_CPUS] = { [0 ... NR_CPUS-1] = NUMA_NO_NODE };
+int cpu_to_node_map[NR_CPUS] = { [0 ... NR_CPUS-1] = NUMA_NO_NODE };
bool numa_off;
@@ -140,55 +140,6 @@ void __init early_map_cpu_to_node(unsigned int cpu, int nid)
set_cpu_numa_node(cpu, nid);
}
-#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
-unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
-EXPORT_SYMBOL(__per_cpu_offset);
-
-int early_cpu_to_node(int cpu)
-{
- return cpu_to_node_map[cpu];
-}
-
-static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)
-{
- return node_distance(early_cpu_to_node(from), early_cpu_to_node(to));
-}
-
-void __init setup_per_cpu_areas(void)
-{
- unsigned long delta;
- unsigned int cpu;
- int rc = -EINVAL;
-
- if (pcpu_chosen_fc != PCPU_FC_PAGE) {
- /*
- * Always reserve area for module percpu variables. That's
- * what the legacy allocator did.
- */
- rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE,
- PERCPU_DYNAMIC_RESERVE, PAGE_SIZE,
- pcpu_cpu_distance,
- early_cpu_to_node);
-#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
- if (rc < 0)
- pr_warn("PERCPU: %s allocator failed (%d), falling back to page size\n",
- pcpu_fc_names[pcpu_chosen_fc], rc);
-#endif
- }
-
-#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
- if (rc < 0)
- rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, early_cpu_to_node);
-#endif
- if (rc < 0)
- panic("Failed to initialize percpu areas (err=%d).", rc);
-
- delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
- for_each_possible_cpu(cpu)
- __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
-}
-#endif
-
/*
* Initialize NODE_DATA for a node on the local memory
*/
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 485df9c2dbddb3..c8c3800fbd9fa0 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1858,6 +1858,17 @@ unsigned long vmalloc_to_pfn(const void *addr);
#ifdef CONFIG_MMU
extern bool is_vmalloc_addr(const void *x);
extern int is_vmalloc_or_module_addr(const void *x);
+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP
+static inline bool is_percpu_addr(unsigned long addr)
+{
+ return addr >= PERCPU_START && addr < LOCAL_PERCPU_END;
+}
+#else
+static inline bool is_percpu_addr(unsigned long addr)
+{
+ return false;
+}
+#endif
#else
static inline bool is_vmalloc_addr(const void *x)
{
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index 4f36b8585995c5..5a27031dff1a5c 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -75,6 +75,8 @@
extern void *pcpu_base_addr;
extern const unsigned long *pcpu_unit_offsets;
+/* percpu local mapping base */
+extern void *pcpu_local_base;
struct pcpu_group_info {
int nr_units; /* aligned # of units */
@@ -114,7 +116,7 @@ extern struct pcpu_alloc_info * __init pcpu_alloc_alloc_info(int nr_groups,
extern void __init pcpu_free_alloc_info(struct pcpu_alloc_info *ai);
extern void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
- void *base_addr);
+ void *base_addr, void *local_base);
extern int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size,
size_t atom_size,
diff --git a/include/linux/pgalloc.h b/include/linux/pgalloc.h
index 9174fa59bbc54d..062c64a5d4bd0c 100644
--- a/include/linux/pgalloc.h
+++ b/include/linux/pgalloc.h
@@ -26,4 +26,17 @@
arch_sync_kernel_mappings(addr, addr); \
} while (0)
+#ifndef CONFIG_ARCH_HAS_PUD_POPULATE_SYNC
+#define pud_populate_sync(addr, pud, pmd) \
+ do { \
+ pud_populate(&init_mm, pud, pmd); \
+ } while (0)
+#endif
+
+#ifndef CONFIG_ARCH_HAS_PMD_POPULATE_SYNC
+#define pmd_populate_sync(addr, pmd, pte) \
+ do { \
+ pmd_populate_kernel(&init_mm, pmd, pte); \
+ } while (0)
+#endif
#endif /* _LINUX_PGALLOC_H */
diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h
index d87dc7f77f4e8a..daf9965a111d1f 100644
--- a/include/linux/vmalloc.h
+++ b/include/linux/vmalloc.h
@@ -313,6 +313,9 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
size_t align);
void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms);
+struct vm_struct *pcpu_get_local_vm_area(unsigned long hint,
+ int unit_size, size_t align);
+
# else
static inline struct vm_struct **
pcpu_get_vm_areas(const unsigned long *offsets,
diff --git a/mm/Kconfig b/mm/Kconfig
index 9e0ca48249054f..e1910c33263851 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -1061,6 +1061,9 @@ config NEED_PER_CPU_PAGE_FIRST_CHUNK
config USE_PERCPU_NUMA_NODE_ID
bool
+config HAVE_LOCAL_PER_CPU_MAP
+ bool
+
config HAVE_SETUP_PER_CPU_AREA
bool
@@ -1509,6 +1512,12 @@ config LAZY_MMU_MODE_KUNIT_TEST
If unsure, say N.
+config ARCH_HAS_PMD_POPULATE_SYNC
+ bool
+
+config ARCH_HAS_PUD_POPULATE_SYNC
+ bool
+
source "mm/damon/Kconfig"
endmenu
diff --git a/mm/internal.h b/mm/internal.h
index 181e79f1d6a207..9d9f08cb26a114 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -1552,10 +1552,13 @@ void clear_vm_uninitialized_flag(struct vm_struct *vm);
int __must_check __vmap_pages_range_noflush(unsigned long addr,
unsigned long end, pgprot_t prot,
struct page **pages, unsigned int page_shift);
+int __must_check vmap_range_noflush(pgd_t *pgdir, unsigned long addr,
+ unsigned long end, phys_addr_t phys_addr,
+ pgprot_t prot, unsigned int max_page_shift);
void vunmap_range_noflush(unsigned long start, unsigned long end);
-void __vunmap_range_noflush(unsigned long start, unsigned long end);
+void __vunmap_range_noflush(pgd_t *pgdir, unsigned long start, unsigned long end);
static inline bool vma_is_single_threaded_private(struct vm_area_struct *vma)
{
diff --git a/mm/kmsan/hooks.c b/mm/kmsan/hooks.c
index 8f22d1f229813f..e2a0faf344b91b 100644
--- a/mm/kmsan/hooks.c
+++ b/mm/kmsan/hooks.c
@@ -135,8 +135,8 @@ static unsigned long vmalloc_origin(unsigned long addr)
void kmsan_vunmap_range_noflush(unsigned long start, unsigned long end)
{
- __vunmap_range_noflush(vmalloc_shadow(start), vmalloc_shadow(end));
- __vunmap_range_noflush(vmalloc_origin(start), vmalloc_origin(end));
+ __vunmap_range_noflush(init_mm.pgd, vmalloc_shadow(start), vmalloc_shadow(end));
+ __vunmap_range_noflush(init_mm.pgd, vmalloc_origin(start), vmalloc_origin(end));
flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));
flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));
}
@@ -181,7 +181,7 @@ int kmsan_ioremap_page_range(unsigned long start, unsigned long end,
vmalloc_origin(start + off + PAGE_SIZE), prot, &origin,
PAGE_SHIFT);
if (mapped) {
- __vunmap_range_noflush(
+ __vunmap_range_noflush(init_mm.pgd,
vmalloc_shadow(start + off),
vmalloc_shadow(start + off + PAGE_SIZE));
err = mapped;
@@ -203,10 +203,10 @@ int kmsan_ioremap_page_range(unsigned long start, unsigned long end,
__free_pages(shadow, 1);
if (origin)
__free_pages(origin, 1);
- __vunmap_range_noflush(
+ __vunmap_range_noflush(init_mm.pgd,
vmalloc_shadow(start),
vmalloc_shadow(start + clean * PAGE_SIZE));
- __vunmap_range_noflush(
+ __vunmap_range_noflush(init_mm.pgd,
vmalloc_origin(start),
vmalloc_origin(start + clean * PAGE_SIZE));
}
@@ -233,8 +233,8 @@ void kmsan_iounmap_page_range(unsigned long start, unsigned long end)
i++, v_shadow += PAGE_SIZE, v_origin += PAGE_SIZE) {
shadow = kmsan_vmalloc_to_page_or_null((void *)v_shadow);
origin = kmsan_vmalloc_to_page_or_null((void *)v_origin);
- __vunmap_range_noflush(v_shadow, vmalloc_shadow(end));
- __vunmap_range_noflush(v_origin, vmalloc_origin(end));
+ __vunmap_range_noflush(init_mm.pgd, v_shadow, vmalloc_shadow(end));
+ __vunmap_range_noflush(init_mm.pgd, v_origin, vmalloc_origin(end));
if (shadow)
__free_pages(shadow, 1);
if (origin)
diff --git a/mm/percpu-internal.h b/mm/percpu-internal.h
index 8cbe039bf847ac..4d461305243bf0 100644
--- a/mm/percpu-internal.h
+++ b/mm/percpu-internal.h
@@ -63,11 +63,14 @@ struct pcpu_chunk {
* chunk_md.
*/
void *base_addr ____cacheline_aligned_in_smp;
+ /* percpu local base address of the chunk */
+ void *local_base;
unsigned long *alloc_map; /* allocation map */
struct pcpu_block_md *md_blocks; /* metadata blocks */
void *data; /* chunk data */
+ void *local_data; /* chunk local vm */
bool immutable; /* no [de]population allowed */
bool isolated; /* isolated from active chunk
slots */
@@ -162,6 +165,17 @@ static inline size_t pcpu_obj_full_size(size_t size)
return size * num_possible_cpus() + extra_size;
}
+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP
+extern void __init map_local_percpu_first_chunk(unsigned int cpu, unsigned long virt,
+ struct page **pages, unsigned int nr);
+#else
+static inline void __init map_local_percpu_first_chunk(unsigned int cpu, unsigned long virt,
+ struct page **pages, unsigned int nr)
+{
+ return;
+}
+#endif
+
#ifdef CONFIG_PERCPU_STATS
#include <linux/spinlock.h>
diff --git a/mm/percpu-vm.c b/mm/percpu-vm.c
index 4f5937090590d8..5decaad7640fdc 100644
--- a/mm/percpu-vm.c
+++ b/mm/percpu-vm.c
@@ -112,6 +112,10 @@ static int pcpu_alloc_pages(struct pcpu_chunk *chunk,
return -ENOMEM;
}
+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP
+extern pgd_t *percpu_pgd[NR_CPUS];
+#endif
+
/**
* pcpu_pre_unmap_flush - flush cache prior to unmapping
* @chunk: chunk the regions to be flushed belongs to
@@ -130,6 +134,11 @@ static void pcpu_pre_unmap_flush(struct pcpu_chunk *chunk,
flush_cache_vunmap(
pcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),
pcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));
+
+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP
+ flush_cache_vunmap((unsigned long)chunk->local_base + (page_start << PAGE_SHIFT),
+ (unsigned long)chunk->local_base + (page_end << PAGE_SHIFT));
+#endif
}
static void __pcpu_unmap_pages(unsigned long addr, int nr_pages)
@@ -137,6 +146,20 @@ static void __pcpu_unmap_pages(unsigned long addr, int nr_pages)
vunmap_range_noflush(addr, addr + (nr_pages << PAGE_SHIFT));
}
+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP
+static void __pcpu_unmap_pages_local(unsigned int cpu, unsigned long virt,
+ int nr_pages)
+{
+ __vunmap_range_noflush(percpu_pgd[cpu], virt, virt + (nr_pages << PAGE_SHIFT));
+}
+#else
+static void __pcpu_unmap_pages_local(unsigned int cpu, unsigned long virt,
+ int nr_pages)
+{
+ return;
+}
+#endif
+
/**
* pcpu_unmap_pages - unmap pages out of a pcpu_chunk
* @chunk: chunk of interest
@@ -166,6 +189,10 @@ static void pcpu_unmap_pages(struct pcpu_chunk *chunk,
}
__pcpu_unmap_pages(pcpu_chunk_addr(chunk, cpu, page_start),
page_end - page_start);
+
+ __pcpu_unmap_pages_local(cpu,
+ (unsigned long)chunk->local_base + (page_start << PAGE_SHIFT),
+ page_end - page_start);
}
}
@@ -188,6 +215,12 @@ static void pcpu_post_unmap_tlb_flush(struct pcpu_chunk *chunk,
flush_tlb_kernel_range(
pcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),
pcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));
+
+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP
+ flush_tlb_kernel_range(
+ (unsigned long)chunk->local_base + (page_start << PAGE_SHIFT),
+ (unsigned long)chunk->local_base + (page_end << PAGE_SHIFT));
+#endif
}
static int __pcpu_map_pages(unsigned long addr, struct page **pages,
@@ -197,6 +230,32 @@ static int __pcpu_map_pages(unsigned long addr, struct page **pages,
PAGE_KERNEL, pages, PAGE_SHIFT, GFP_KERNEL);
}
+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP
+static int __pcpu_map_pages_local(unsigned int cpu, unsigned long virt, struct page **pages,
+ int nr_pages)
+{
+ unsigned int i;
+ int err = 0;
+
+ for (i = 0; i < nr_pages; i++) {
+ err = vmap_range_noflush(percpu_pgd[cpu], virt, virt + PAGE_SIZE,
+ page_to_phys(pages[i]), PAGE_KERNEL, PAGE_SHIFT);
+ if (err)
+ return err;
+
+ virt += PAGE_SIZE;
+ }
+
+ return err;
+}
+#else
+static int __pcpu_map_pages_local(unsigned int cpu, unsigned long virt, struct page **pages,
+ int nr_pages)
+{
+ return 0;
+}
+#endif
+
/**
* pcpu_map_pages - map pages into a pcpu_chunk
* @chunk: chunk of interest
@@ -224,6 +283,13 @@ static int pcpu_map_pages(struct pcpu_chunk *chunk,
if (err < 0)
goto err;
+ err = __pcpu_map_pages_local(cpu,
+ (unsigned long)chunk->local_base + (page_start << PAGE_SHIFT),
+ &pages[pcpu_page_idx(cpu, page_start)],
+ page_end - page_start);
+ if (err < 0)
+ goto err;
+
for (i = page_start; i < page_end; i++)
pcpu_set_page_chunk(pages[pcpu_page_idx(cpu, i)],
chunk);
@@ -233,6 +299,9 @@ static int pcpu_map_pages(struct pcpu_chunk *chunk,
for_each_possible_cpu(tcpu) {
__pcpu_unmap_pages(pcpu_chunk_addr(chunk, tcpu, page_start),
page_end - page_start);
+ __pcpu_unmap_pages_local(tcpu,
+ (unsigned long)chunk->local_base + (page_start << PAGE_SHIFT),
+ page_end - page_start);
if (tcpu == cpu)
break;
}
@@ -258,6 +327,11 @@ static void pcpu_post_map_flush(struct pcpu_chunk *chunk,
flush_cache_vmap(
pcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),
pcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));
+
+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP
+ flush_cache_vmap((unsigned long)chunk->local_base + (page_start << PAGE_SHIFT),
+ (unsigned long)chunk->local_base + (page_end << PAGE_SHIFT));
+#endif
}
/**
@@ -349,6 +423,24 @@ static struct pcpu_chunk *pcpu_create_chunk(gfp_t gfp)
chunk->data = vms;
chunk->base_addr = vms[0]->addr - pcpu_group_offsets[0];
+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP
+ unsigned long delta = (unsigned long)chunk->base_addr - (unsigned long)pcpu_base_addr;
+ unsigned long hint = delta + (unsigned long)pcpu_local_base;
+ struct vm_struct *local_vm = pcpu_get_local_vm_area(hint,
+ pcpu_unit_size, pcpu_atom_size);
+ if (!local_vm) {
+ pcpu_free_vm_areas(vms, pcpu_nr_groups);
+ pcpu_free_chunk(chunk);
+ return NULL;
+ }
+
+ chunk->local_base = local_vm->addr;
+ chunk->local_data = (void *)local_vm;
+#else
+ chunk->local_base = 0;
+ chunk->local_data = NULL;
+#endif
+
pcpu_stats_chunk_alloc();
trace_percpu_create_chunk(chunk->base_addr);
@@ -365,6 +457,8 @@ static void pcpu_destroy_chunk(struct pcpu_chunk *chunk)
if (chunk->data)
pcpu_free_vm_areas(chunk->data, pcpu_nr_groups);
+ if (chunk->local_data)
+ free_vm_area((struct vm_struct *)chunk->local_data);
pcpu_free_chunk(chunk);
}
diff --git a/mm/percpu.c b/mm/percpu.c
index b0676b8054ed09..8b273ed8e5a8bb 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -145,6 +145,8 @@ static unsigned int pcpu_high_unit_cpu __ro_after_init;
/* the address of the first chunk which starts with the kernel static area */
void *pcpu_base_addr __ro_after_init;
+/* The address of the first chunk local mapping */
+void *pcpu_local_base __ro_after_init;
static const int *pcpu_unit_map __ro_after_init; /* cpu -> unit */
const unsigned long *pcpu_unit_offsets __ro_after_init; /* cpu -> unit offset */
@@ -1338,6 +1340,7 @@ static void pcpu_init_md_blocks(struct pcpu_chunk *chunk)
/**
* pcpu_alloc_first_chunk - creates chunks that serve the first chunk
* @tmp_addr: the start of the region served
+ * @local_tmp: the start of the local percpu region served
* @map_size: size of the region served
*
* This is responsible for creating the chunks that serve the first chunk. The
@@ -1349,15 +1352,16 @@ static void pcpu_init_md_blocks(struct pcpu_chunk *chunk)
* Chunk serving the region at @tmp_addr of @map_size.
*/
static struct pcpu_chunk * __init pcpu_alloc_first_chunk(unsigned long tmp_addr,
- int map_size)
+ unsigned long local_tmp, int map_size)
{
struct pcpu_chunk *chunk;
- unsigned long aligned_addr;
+ unsigned long aligned_addr, aligned_local;
int start_offset, offset_bits, region_size, region_bits;
size_t alloc_size;
/* region calculations */
aligned_addr = tmp_addr & PAGE_MASK;
+ aligned_local = local_tmp & PAGE_MASK;
start_offset = tmp_addr - aligned_addr;
region_size = ALIGN(start_offset + map_size, PAGE_SIZE);
@@ -1370,6 +1374,7 @@ static struct pcpu_chunk * __init pcpu_alloc_first_chunk(unsigned long tmp_addr,
INIT_LIST_HEAD(&chunk->list);
chunk->base_addr = (void *)aligned_addr;
+ chunk->local_base = (void *)aligned_local;
chunk->start_offset = start_offset;
chunk->end_offset = region_size - chunk->start_offset - map_size;
@@ -2562,7 +2567,7 @@ static void pcpu_dump_alloc_info(const char *lvl,
* and available for dynamic allocation like any other chunk.
*/
void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
- void *base_addr)
+ void *base_addr, void *local_base)
{
size_t size_sum = ai->static_size + ai->reserved_size + ai->dyn_size;
size_t static_size, dyn_size;
@@ -2572,7 +2577,7 @@ void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
unsigned int cpu;
int *unit_map;
int group, unit, i;
- unsigned long tmp_addr;
+ unsigned long tmp_addr, local_tmp = 0;
size_t alloc_size;
#define PCPU_SETUP_BUG_ON(cond) do { \
@@ -2713,11 +2718,15 @@ void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
* chunk.
*/
tmp_addr = (unsigned long)base_addr + static_size;
+ if (local_base)
+ local_tmp = (unsigned long)local_base + static_size;
if (ai->reserved_size)
- pcpu_reserved_chunk = pcpu_alloc_first_chunk(tmp_addr,
+ pcpu_reserved_chunk = pcpu_alloc_first_chunk(tmp_addr, local_tmp,
ai->reserved_size);
tmp_addr = (unsigned long)base_addr + static_size + ai->reserved_size;
- pcpu_first_chunk = pcpu_alloc_first_chunk(tmp_addr, dyn_size);
+ if (local_base)
+ local_tmp = (unsigned long)local_base + static_size + ai->reserved_size;
+ pcpu_first_chunk = pcpu_alloc_first_chunk(tmp_addr, local_tmp, dyn_size);
pcpu_nr_empty_pop_pages = pcpu_first_chunk->nr_empty_pop_pages;
pcpu_chunk_relocate(pcpu_first_chunk, -1);
@@ -3108,7 +3117,7 @@ int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size,
PFN_DOWN(size_sum), ai->static_size, ai->reserved_size,
ai->dyn_size, ai->unit_size);
- pcpu_setup_first_chunk(ai, base);
+ pcpu_setup_first_chunk(ai, base, NULL);
goto out_free;
out_free_areas:
@@ -3163,7 +3172,7 @@ void __init __weak pcpu_populate_pte(unsigned long addr)
pud = pud_offset(p4d, addr);
if (pud_none(*pud)) {
pmd = memblock_alloc_or_panic(PMD_TABLE_SIZE, PMD_TABLE_SIZE);
- pud_populate(&init_mm, pud, pmd);
+ pud_populate_sync(addr, pud, pmd);
}
pmd = pmd_offset(pud, addr);
@@ -3171,7 +3180,7 @@ void __init __weak pcpu_populate_pte(unsigned long addr)
pte_t *new;
new = memblock_alloc_or_panic(PTE_TABLE_SIZE, PTE_TABLE_SIZE);
- pmd_populate_kernel(&init_mm, pmd, new);
+ pmd_populate_sync(addr, pmd, new);
}
return;
@@ -3194,6 +3203,7 @@ void __init __weak pcpu_populate_pte(unsigned long addr)
int __init pcpu_page_first_chunk(size_t reserved_size, pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn)
{
static struct vm_struct vm;
+ static struct vm_struct pcpu_vm;
struct pcpu_alloc_info *ai;
char psize_str[16];
int unit_pages;
@@ -3243,11 +3253,28 @@ int __init pcpu_page_first_chunk(size_t reserved_size, pcpu_fc_cpu_to_node_fn_t
}
/* allocate vm area, map the pages and copy static data */
+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP
+ vm.flags = VM_ALLOC;
+ vm.addr = (void *)ALIGN(PERCPU_START, PAGE_SIZE);
+ vm.size = num_possible_cpus() * ai->unit_size;
+ vm_area_add_early(&vm);
+ kasan_populate_early_vm_area_shadow(vm.addr, vm.size);
+
+ pcpu_vm.flags = VM_ALLOC;
+ pcpu_vm.addr = (void *)ALIGN(LOCAL_PERCPU_START, PAGE_SIZE);
+ pcpu_vm.size = ai->unit_size;
+ vm_area_add_early(&pcpu_vm);
+ kasan_populate_early_vm_area_shadow(pcpu_vm.addr, pcpu_vm.size);
+#else
vm.flags = VM_ALLOC;
vm.size = num_possible_cpus() * ai->unit_size;
vm_area_register_early(&vm, PAGE_SIZE);
+ pcpu_vm.addr = NULL;
+#endif
+
for (unit = 0; unit < num_possible_cpus(); unit++) {
+ unsigned int cpu = ai->groups[0].cpu_map[unit];
unsigned long unit_addr =
(unsigned long)vm.addr + unit * ai->unit_size;
@@ -3264,6 +3291,14 @@ int __init pcpu_page_first_chunk(size_t reserved_size, pcpu_fc_cpu_to_node_fn_t
/* copy static data */
memcpy((void *)unit_addr, __per_cpu_start, ai->static_size);
+
+ /*
+ * Map percpu data to PERCPU map.
+ *
+ * PCPU_FC_EMBED can't support it.
+ */
+ map_local_percpu_first_chunk(cpu, (unsigned long)pcpu_vm.addr,
+ &pages[unit * unit_pages], unit_pages);
}
/* we're ready, commit */
@@ -3271,7 +3306,8 @@ int __init pcpu_page_first_chunk(size_t reserved_size, pcpu_fc_cpu_to_node_fn_t
unit_pages, psize_str, ai->static_size,
ai->reserved_size, ai->dyn_size);
- pcpu_setup_first_chunk(ai, vm.addr);
+ pcpu_setup_first_chunk(ai, vm.addr, pcpu_vm.addr);
+ pcpu_local_base = pcpu_vm.addr;
goto out_free_ar;
enomem:
@@ -3353,7 +3389,7 @@ void __init setup_per_cpu_areas(void)
ai->groups[0].nr_units = 1;
ai->groups[0].cpu_map[0] = 0;
- pcpu_setup_first_chunk(ai, fc);
+ pcpu_setup_first_chunk(ai, fc, NULL);
pcpu_free_alloc_info(ai);
}
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index 99e2be39671b63..463c2f8b611c03 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -198,7 +198,7 @@ static pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, in
if (!p)
return NULL;
kernel_pte_init(p);
- pmd_populate_kernel(&init_mm, pmd, p);
+ pmd_populate_sync(addr, pmd, p);
}
return pmd;
}
@@ -211,7 +211,7 @@ static pud_t * __meminit vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, in
if (!p)
return NULL;
pmd_init(p);
- pud_populate(&init_mm, pud, p);
+ pud_populate_sync(addr, pud, p);
}
return pud;
}
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 1afca3568b9b65..3d94ca4bbd17b1 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -80,6 +80,9 @@ bool is_vmalloc_addr(const void *x)
{
unsigned long addr = (unsigned long)kasan_reset_tag(x);
+ if (is_percpu_addr(addr))
+ return true;
+
return addr >= VMALLOC_START && addr < VMALLOC_END;
}
EXPORT_SYMBOL(is_vmalloc_addr);
@@ -295,9 +298,9 @@ static int vmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
return err;
}
-static int vmap_range_noflush(unsigned long addr, unsigned long end,
- phys_addr_t phys_addr, pgprot_t prot,
- unsigned int max_page_shift)
+int vmap_range_noflush(pgd_t *pgdir, unsigned long addr, unsigned long end,
+ phys_addr_t phys_addr, pgprot_t prot,
+ unsigned int max_page_shift)
{
pgd_t *pgd;
unsigned long start;
@@ -314,7 +317,7 @@ static int vmap_range_noflush(unsigned long addr, unsigned long end,
BUG_ON(addr >= end);
start = addr;
- pgd = pgd_offset_k(addr);
+ pgd = pgd_offset_pgd(pgdir, addr);
do {
next = pgd_addr_end(addr, end);
err = vmap_p4d_range(pgd, addr, next, phys_addr, prot,
@@ -334,8 +337,8 @@ int vmap_page_range(unsigned long addr, unsigned long end,
{
int err;
- err = vmap_range_noflush(addr, end, phys_addr, pgprot_nx(prot),
- ioremap_max_page_shift);
+ err = vmap_range_noflush(init_mm.pgd, addr, end, phys_addr,
+ pgprot_nx(prot), ioremap_max_page_shift);
flush_cache_vmap(addr, end);
if (!err)
err = kmsan_ioremap_page_range(addr, end, phys_addr, prot,
@@ -478,7 +481,7 @@ static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
*
* This is an internal function only. Do not use outside mm/.
*/
-void __vunmap_range_noflush(unsigned long start, unsigned long end)
+void __vunmap_range_noflush(pgd_t *pgdir, unsigned long start, unsigned long end)
{
unsigned long next;
pgd_t *pgd;
@@ -486,7 +489,7 @@ void __vunmap_range_noflush(unsigned long start, unsigned long end)
pgtbl_mod_mask mask = 0;
BUG_ON(addr >= end);
- pgd = pgd_offset_k(addr);
+ pgd = pgd_offset_pgd(pgdir, addr);
do {
next = pgd_addr_end(addr, end);
if (pgd_bad(*pgd))
@@ -503,7 +506,7 @@ void __vunmap_range_noflush(unsigned long start, unsigned long end)
void vunmap_range_noflush(unsigned long start, unsigned long end)
{
kmsan_vunmap_range_noflush(start, end);
- __vunmap_range_noflush(start, end);
+ __vunmap_range_noflush(init_mm.pgd, start, end);
}
/**
@@ -670,9 +673,10 @@ int __vmap_pages_range_noflush(unsigned long addr, unsigned long end,
for (i = 0; i < nr; i += 1U << (page_shift - PAGE_SHIFT)) {
int err;
- err = vmap_range_noflush(addr, addr + (1UL << page_shift),
- page_to_phys(pages[i]), prot,
- page_shift);
+ err = vmap_range_noflush(init_mm.pgd, addr,
+ addr + (1UL << page_shift),
+ page_to_phys(pages[i]), prot,
+ page_shift);
if (err)
return err;
@@ -4919,15 +4923,25 @@ pvm_find_va_enclose_addr(unsigned long addr)
* in - the VA we start the search(reverse order);
* out - the VA with the highest aligned end address.
* @align: alignment for required highest address
+ * @pcpu: whether request allocation from local percpu area
*
* Returns: determined end address within vmap_area
*/
static unsigned long
-pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align)
+pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align, bool pcpu)
{
- unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
+ unsigned long vmalloc_end;
unsigned long addr;
+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP
+ if (pcpu)
+ vmalloc_end = LOCAL_PERCPU_END & ~(align - 1);
+ else
+ vmalloc_end = PERCPU_END & ~(align - 1);
+#else
+ vmalloc_end = VMALLOC_END & ~(align - 1);
+#endif
+
if (likely(*va)) {
list_for_each_entry_from_reverse((*va),
&free_vmap_area_list, list) {
@@ -4968,14 +4982,22 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
const size_t *sizes, int nr_vms,
size_t align)
{
- const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align);
- const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
+ unsigned long vmalloc_start;
+ unsigned long vmalloc_end;
struct vmap_area **vas, *va;
struct vm_struct **vms;
int area, area2, last_area, term_area;
unsigned long base, start, size, end, last_end, orig_start, orig_end;
bool purged = false;
+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP
+ vmalloc_start = ALIGN(PERCPU_START, align);
+ vmalloc_end = PERCPU_END & ~(align - 1);
+#else
+ vmalloc_start = ALIGN(VMALLOC_START, align);
+ vmalloc_end = VMALLOC_END & ~(align - 1);
+#endif
+
/* verify parameters and allocate data structures */
BUG_ON(offset_in_page(align) || !is_power_of_2(align));
for (last_area = 0, area = 0; area < nr_vms; area++) {
@@ -5024,7 +5046,7 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
end = start + sizes[area];
va = pvm_find_va_enclose_addr(vmalloc_end);
- base = pvm_determine_end_from_reverse(&va, align) - end;
+ base = pvm_determine_end_from_reverse(&va, align, false) - end;
while (true) {
/*
@@ -5045,7 +5067,7 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
* base downwards and then recheck.
*/
if (base + end > va->va_end) {
- base = pvm_determine_end_from_reverse(&va, align) - end;
+ base = pvm_determine_end_from_reverse(&va, align, false) - end;
term_area = area;
continue;
}
@@ -5055,7 +5077,7 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
*/
if (base + start < va->va_start) {
va = node_to_va(rb_prev(&va->rb_node));
- base = pvm_determine_end_from_reverse(&va, align) - end;
+ base = pvm_determine_end_from_reverse(&va, align, false) - end;
term_area = area;
continue;
}
@@ -5218,6 +5240,84 @@ void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms)
free_vm_area(vms[i]);
kfree(vms);
}
+
+#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP
+/* Find free vm area starts from hint */
+struct vm_struct *pcpu_get_local_vm_area(unsigned long hint,
+ int unit_size, size_t align)
+{
+ struct vmap_area *tmp_va, *va;
+ struct vm_struct *vm;
+ struct vmap_node *vn;
+ unsigned long end;
+ int ret;
+
+ va = kmem_cache_zalloc(vmap_area_cachep, GFP_KERNEL);
+ vm = kzalloc(sizeof(struct vm_struct), GFP_KERNEL);
+ if (!va || !vm)
+ goto err_free;
+
+ spin_lock(&free_vmap_area_lock);
+
+ tmp_va = pvm_find_va_enclose_addr(hint);
+ if (!tmp_va) {
+ spin_unlock(&free_vmap_area_lock);
+ goto err_free;
+ }
+
+ end = pvm_determine_end_from_reverse(&tmp_va, align, true);
+
+ if (hint + unit_size > end) {
+ spin_unlock(&free_vmap_area_lock);
+ goto err_free;
+ }
+
+ ret = va_clip(&free_vmap_area_root,
+ &free_vmap_area_list, tmp_va, hint, unit_size);
+ if (ret) {
+ spin_unlock(&free_vmap_area_lock);
+ goto err_free;
+ }
+
+ va->va_start = hint;
+ va->va_end = hint + unit_size;
+
+ spin_unlock(&free_vmap_area_lock);
+
+ if (kasan_populate_vmalloc(va->va_start, unit_size, GFP_KERNEL))
+ goto err_free_shadow;
+
+ vn = addr_to_node(va->va_start);
+
+ spin_lock(&vn->busy.lock);
+ insert_vmap_area(va, &vn->busy.root, &vn->busy.head);
+ setup_vmalloc_vm(vm, va, VM_ALLOC,
+ pcpu_get_local_vm_area);
+ spin_unlock(&vn->busy.lock);
+
+ kasan_unpoison_vmap_areas(&vm, 1, KASAN_VMALLOC_PROT_NORMAL);
+
+ return vm;
+
+err_free:
+ kmem_cache_free(vmap_area_cachep, va);
+ kfree(vm);
+
+ return NULL;
+
+err_free_shadow:
+ spin_lock(&free_vmap_area_lock);
+ va = merge_or_add_vmap_area_augment(va, &free_vmap_area_root,
+ &free_vmap_area_list);
+ if (va)
+ kasan_release_vmalloc(va->va_start, va->va_end, va->va_start, va->va_end,
+ KASAN_VMALLOC_PAGE_RANGE | KASAN_VMALLOC_TLB_FLUSH);
+ kfree(vm);
+ spin_unlock(&free_vmap_area_lock);
+ return NULL;
+
+}
+#endif
#endif /* CONFIG_SMP */
#ifdef CONFIG_PRINTK
|
| 3/2 | 2026/07/15 22:40 | llm | patch-evaluator |
0mModel:gemini-3.1-pro-preview Args: null Results: null Tokens: input: 29182 output: 36 thoughts: 890 Thoughts: **Fuzzing Target Identification: Local Per-CPU Mapping for ARM64** Okay, so I need to assess this patch series. My initial thought is, based on the file modifications, this looks like a functional change that introduces a new feature: `HAVE_LOCAL_PER_CPU_MAP`. It touches core kernel components related to memory management (percpu, page tables) on ARM64, and this *definitely* triggers my fuzzing instincts. Anything that messes with memory mapping is a potential goldmine for finding subtle bugs. I need to make a note of the files being touched: `arch/arm64/...`, `include/...`, and `mm/...`, specifically focusing on `percpu`, `mm` and page table related files. The core of this seems to be adding local per-cpu mapping support for ARM64. That screams "functional change" to me, and it modifies fundamental memory management logic. We have to fuzz this. Now, let's identify the key functions that are relevant here. I'm seeing: `setup_percpu_pgd`, `map_local_percpu_first_chunk`, `arch_sync_kernel_mappings`, `pcpu_get_local_vm_area`, `pcpu_alloc_first_chunk`, `pcpu_setup_first_chunk`, `pcpu_page_first_chunk`, `vmap_range_noflush`, and `__vunmap_range_noflush`. These are the functions I want to target with fuzzing. I'll check the config options now. `HAVE_LOCAL_PER_CPU_MAP` is a `def_bool y` depending on ARM64 page sizes which means it does not need to be explicitly enabled. That said, it would be useful to list this config in our fuzzing list. I see that `NEED_PER_CPU_PAGE_FIRST_CHUNK` is selected by ARM64 now. Also, `ARCH_HAS_PMD_POPULATE_SYNC` and `ARCH_HAS_PUD_POPULATE_SYNC` are selected by ARM64. Good to know. Therefore, my initial list of functions to focus on for fuzzing are: `pcpu_get_local_vm_area`, `pcpu_alloc_first_chunk`, `pcpu_setup_first_chunk`, `pcpu_page_first_chunk`, `map_local_percpu_first_chunk`, `setup_percpu_pgd` and `arch_sync_kernel_mappings`. Now, I need to verify that these functions exist and are part of the target code. |
| 4/2 | 2026/07/15 22:40 | tool | grepper |
0mError:no matches Args: {
"Expression": "^(void|int|struct vm_struct \\*) .*pcpu_get_local_vm_area"
}Results: {
"error": "no matches"
} |
| 5/2 | 2026/07/15 22:40 | llm | patch-evaluator |
0mModel:gemini-3.1-pro-preview Args: null Results: null Tokens: input: 30687 output: 24 thoughts: 10 Thoughts: Okay, here's my summary, tailored for an expert audience: **Refining My Approach** Right, hold on a second. I'm taking another look at this. The current regular expression isn't quite as robust as it needs to be. I can see a potential edge case or two that it's going to miss, and that's not acceptable, especially given the scale I'm working at. I need to tighten things up. I'm going to have to dive back in and refine the pattern. It's about being precise; leaving no room for ambiguity. This isn't just about getting *a* match; it's about getting *the right* match every single time. Time for a more elegant and comprehensive solution. I'll need to consider lookarounds, perhaps even a recursive pattern, depending on the complexity of the source data. Let's see... |
| 6/2 | 2026/07/15 22:40 | tool | grepper |
0mArgs:{
"Expression": "pcpu_get_local_vm_area"
}Results: {
"Output": "include/linux/vmalloc.h=315=void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms);\ninclude/linux/vmalloc.h:316:struct vm_struct *pcpu_get_local_vm_area(unsigned long hint,\ninclude/linux/vmalloc.h-317-\t\t\t\t int unit_size, size_t align);\n--\nmm/percpu-vm.c=407=static struct pcpu_chunk *pcpu_create_chunk(gfp_t gfp)\n--\nmm/percpu-vm.c-428-\tunsigned long hint = delta + (unsigned long)pcpu_local_base;\nmm/percpu-vm.c:429:\tstruct vm_struct *local_vm = pcpu_get_local_vm_area(hint,\nmm/percpu-vm.c-430-\t\t\t\t\tpcpu_unit_size, pcpu_atom_size);\n--\nmm/vmalloc.c=5235=void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms)\n--\nmm/vmalloc.c-5245-/* Find free vm area starts from hint */\nmm/vmalloc.c:5246:struct vm_struct *pcpu_get_local_vm_area(unsigned long hint,\nmm/vmalloc.c-5247-\t\t\t\t int unit_size, size_t align)\n--\nmm/vmalloc.c-5294-\tsetup_vmalloc_vm(vm, va, VM_ALLOC,\nmm/vmalloc.c:5295:\t\t\t pcpu_get_local_vm_area);\nmm/vmalloc.c-5296-\tspin_unlock(\u0026vn-\u003ebusy.lock);\n"
} |
| 7/2 | 2026/07/15 22:40 | llm | patch-evaluator |
0mModel:gemini-3.1-pro-preview Args: null Results: null Tokens: input: 31357 output: 24 thoughts: 34 Thoughts: Here's my take on the thought process, as requested: **Navigating the Per-CPU Virtual Memory Landscape** Alright, so `pcpu_get_local_vm_area` is definitely present and accounted for. That's good. Now, to understand how this system is actually managing and mapping these local per-CPU virtual memory areas, I need to dive into `map_local_percpu_first_chunk`. That's the logical next step to figure out the initialization and allocation strategy. Let's see how that function works and figure out what the kernel's doing under the hood. |
| 8/2 | 2026/07/15 22:40 | tool | grepper |
0mArgs:{
"Expression": "map_local_percpu_first_chunk"
}Results: {
"Output": "arch/arm64/include/asm/mmu.h=75=extern void linear_map_maybe_split_to_ptes(void);\narch/arm64/include/asm/mmu.h-76-#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP\narch/arm64/include/asm/mmu.h:77:extern void map_local_percpu_first_chunk(unsigned int cpu, unsigned long virt,\narch/arm64/include/asm/mmu.h-78-\t\t\t\tstruct page **pages, unsigned int nr);\n--\narch/arm64/mm/mmu.c=1075=void __init linear_map_maybe_split_to_ptes(void)\n--\narch/arm64/mm/mmu.c-1083-#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP\narch/arm64/mm/mmu.c:1084:void __init map_local_percpu_first_chunk(unsigned int cpu, unsigned long virt,\narch/arm64/mm/mmu.c-1085-\t\t\t struct page **pages, unsigned int nr)\n--\nmm/percpu-internal.h=156=static inline size_t pcpu_obj_full_size(size_t size)\n--\nmm/percpu-internal.h-168-#ifdef CONFIG_HAVE_LOCAL_PER_CPU_MAP\nmm/percpu-internal.h:169:extern void __init map_local_percpu_first_chunk(unsigned int cpu, unsigned long virt,\nmm/percpu-internal.h-170- struct page **pages, unsigned int nr);\nmm/percpu-internal.h-171-#else\nmm/percpu-internal.h:172:static inline void __init map_local_percpu_first_chunk(unsigned int cpu, unsigned long virt,\nmm/percpu-internal.h-173- struct page **pages, unsigned int nr)\n--\nmm/percpu.c=3203=int __init pcpu_page_first_chunk(size_t reserved_size, pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn)\n--\nmm/percpu.c-3299-\t\t */\nmm/percpu.c:3300:\t\tmap_local_percpu_first_chunk(cpu, (unsigned long)pcpu_vm.addr,\nmm/percpu.c-3301-\t\t\t\t\u0026pages[unit * unit_pages], unit_pages);\n"
} |
| 9/2 | 2026/07/15 22:41 | llm | patch-evaluator |
0mModel:gemini-3.1-pro-preview Args: null Results: null Tokens: input: 32177 output: 21 thoughts: 13 Thoughts: Okay, here's my attempt at summarizing my thought process, assuming I'm a seasoned kernel developer and the text refers to examining kernel code: **Checking `arch_sync_kernel_mappings`** Alright, let's dive into this `arch_sync_kernel_mappings` function. Immediately, my mind goes to the potential architecture-specific nuances this function likely handles. This isn't just a generic memory synchronization routine; the "arch" prefix screams that we're dealing with hardware details. I need to understand what specific architectural context this applies to. Is it x86, ARM, RISC-V, or something else entirely? The architecture will dictate the underlying memory models, cache coherency protocols, and memory barriers that are relevant here. My focus should be on how the function synchronizes the kernel's virtual memory mappings. We're talking about ensuring consistency between the virtual address space visible to the kernel and the underlying physical memory. What's the specific scenario? Is it related to handling TLB flushes? Page table updates? Or possibly cache invalidation for memory regions shared between different processes or devices? The comments within the function itself should be my first port of call, but I anticipate looking at the surrounding code to determine its specific use cases and the broader context of its role in the kernel's memory management. I'm also interested in its efficiency, I will have to look to see if this operation has any performance bottlenecks. |
| 10/2 | 2026/07/15 22:41 | tool | grepper |
0mArgs:{
"Expression": "arch_sync_kernel_mappings"
}Results: {
"Output": "arch/arm/kernel/traps.c=921=asmlinkage void handle_bad_stack(struct pt_regs *regs)\n--\narch/arm/kernel/traps.c-958- */\narch/arm/kernel/traps.c:959:void arch_sync_kernel_mappings(unsigned long start, unsigned long end)\narch/arm/kernel/traps.c-960-{\n--\narch/arm64/include/asm/pgalloc.h=38=static inline void pud_populate_sync(unsigned long addr, pud_t *pudp, pmd_t *pmdp)\n--\narch/arm64/include/asm/pgalloc.h-41-\tif (ARCH_PAGE_TABLE_SYNC_MASK \u0026 PGTBL_PUD_MODIFIED)\narch/arm64/include/asm/pgalloc.h:42:\t\tarch_sync_kernel_mappings(addr, addr);\narch/arm64/include/asm/pgalloc.h-43-}\n--\narch/arm64/include/asm/pgalloc.h=141=static inline void pmd_populate_sync(unsigned long addr, pmd_t *pmdp, pte_t *ptep)\n--\narch/arm64/include/asm/pgalloc.h-144-\tif (ARCH_PAGE_TABLE_SYNC_MASK \u0026 PGTBL_PMD_MODIFIED)\narch/arm64/include/asm/pgalloc.h:145:\t\tarch_sync_kernel_mappings(addr, addr);\narch/arm64/include/asm/pgalloc.h-146-}\n--\narch/arm64/mm/kasan_init.c=176=static void __init kasan_pgd_populate(unsigned long addr, unsigned long end,\n--\narch/arm64/mm/kasan_init.c-190-\tif (mask \u0026 ARCH_PAGE_TABLE_SYNC_MASK)\narch/arm64/mm/kasan_init.c:191:\t\tarch_sync_kernel_mappings(start, end);\narch/arm64/mm/kasan_init.c-192-}\n--\narch/arm64/mm/mmu.c=73=void __init setup_percpu_pgd(void)\n--\narch/arm64/mm/mmu.c-97-\narch/arm64/mm/mmu.c:98:void arch_sync_kernel_mappings(unsigned long start, unsigned long end)\narch/arm64/mm/mmu.c-99-{\n--\narch/arm64/mm/mmu.c=1866=static void free_empty_tables(unsigned long addr, unsigned long end,\n--\narch/arm64/mm/mmu.c-1885-\tif (mask \u0026 ARCH_PAGE_TABLE_SYNC_MASK)\narch/arm64/mm/mmu.c:1886:\t\tarch_sync_kernel_mappings(start, end);\narch/arm64/mm/mmu.c-1887-}\n--\narch/arm64/mm/mmu.c=2090=int arch_add_memory(int nid, u64 start, u64 size,\n--\narch/arm64/mm/mmu.c-2107-\tif (mask \u0026 ARCH_PAGE_TABLE_SYNC_MASK)\narch/arm64/mm/mmu.c:2108:\t\tarch_sync_kernel_mappings(start, start + size);\narch/arm64/mm/mmu.c-2109-\n--\narch/x86/kernel/setup_percpu.c=111=void __init setup_per_cpu_areas(void)\n--\narch/x86/kernel/setup_percpu.c-220-\t * setup_arch(), but percpu data also needs to be available in\narch/x86/kernel/setup_percpu.c:221:\t * the smpboot asm and arch_sync_kernel_mappings() doesn't sync to\narch/x86/kernel/setup_percpu.c-222-\t * swapper_pg_dir on 32-bit. The per-cpu mappings need to be available\n--\narch/x86/mm/fault.c=269=NOKPROBE_SYMBOL(vmalloc_fault);\narch/x86/mm/fault.c-270-\narch/x86/mm/fault.c:271:void arch_sync_kernel_mappings(unsigned long start, unsigned long end)\narch/x86/mm/fault.c-272-{\n--\narch/x86/mm/init_64.c=218=static void sync_global_pgds(unsigned long start, unsigned long end)\n--\narch/x86/mm/init_64.c-238- */\narch/x86/mm/init_64.c:239:void arch_sync_kernel_mappings(unsigned long start, unsigned long end)\narch/x86/mm/init_64.c-240-{\n--\ninclude/linux/pgalloc.h-18-\t\tif (ARCH_PAGE_TABLE_SYNC_MASK \u0026 PGTBL_PGD_MODIFIED)\t\\\ninclude/linux/pgalloc.h:19:\t\t\tarch_sync_kernel_mappings(addr, addr);\t\t\\\ninclude/linux/pgalloc.h-20-\t} while (0)\n--\ninclude/linux/pgalloc.h-25-\t\tif (ARCH_PAGE_TABLE_SYNC_MASK \u0026 PGTBL_P4D_MODIFIED)\t\\\ninclude/linux/pgalloc.h:26:\t\t\tarch_sync_kernel_mappings(addr, addr);\t\t\\\ninclude/linux/pgalloc.h-27-\t} while (0)\n--\ninclude/linux/pgtable.h=1728=static inline void modify_prot_commit_ptes(struct vm_area_struct *vma, unsigned long addr,\n--\ninclude/linux/pgtable.h-1745- * and let generic vmalloc, ioremap and page table update code know when\ninclude/linux/pgtable.h:1746: * arch_sync_kernel_mappings() needs to be called.\ninclude/linux/pgtable.h-1747- */\n--\ninclude/linux/pgtable.h-1752-/*\ninclude/linux/pgtable.h:1753: * There is no default implementation for arch_sync_kernel_mappings(). It is\ninclude/linux/pgtable.h-1754- * relied upon the compiler to optimize calls out if ARCH_PAGE_TABLE_SYNC_MASK\n--\ninclude/linux/pgtable.h-1756- */\ninclude/linux/pgtable.h:1757:void arch_sync_kernel_mappings(unsigned long start, unsigned long end);\ninclude/linux/pgtable.h-1758-\n--\nmm/memory.c=3463=static int __apply_to_page_range(struct mm_struct *mm, unsigned long addr,\n--\nmm/memory.c-3496-\tif (mask \u0026 ARCH_PAGE_TABLE_SYNC_MASK)\nmm/memory.c:3497:\t\tarch_sync_kernel_mappings(start, start + size);\nmm/memory.c-3498-\n--\nmm/vmalloc.c=301=int vmap_range_noflush(pgd_t *pgdir, unsigned long addr, unsigned long end,\n--\nmm/vmalloc.c-329-\tif (mask \u0026 ARCH_PAGE_TABLE_SYNC_MASK)\nmm/vmalloc.c:330:\t\tarch_sync_kernel_mappings(start, end);\nmm/vmalloc.c-331-\n--\nmm/vmalloc.c=484=void __vunmap_range_noflush(pgd_t *pgdir, unsigned long start, unsigned long end)\n--\nmm/vmalloc.c-502-\tif (mask \u0026 ARCH_PAGE_TABLE_SYNC_MASK)\nmm/vmalloc.c:503:\t\tarch_sync_kernel_mappings(start, end);\nmm/vmalloc.c-504-}\n--\nmm/vmalloc.c=626=static int vmap_small_pages_range_noflush(unsigned long addr, unsigned long end,\n--\nmm/vmalloc.c-647-\tif (mask \u0026 ARCH_PAGE_TABLE_SYNC_MASK)\nmm/vmalloc.c:648:\t\tarch_sync_kernel_mappings(start, end);\nmm/vmalloc.c-649-\n"
} |
| 11/2 | 2026/07/15 22:41 | llm | patch-evaluator |
0mModel:gemini-3.1-pro-preview Args: null Results: null Tokens: input: 34650 output: 172 thoughts: 1615 Thoughts: **Initial Assessment and Exploration of Per-CPU Mapping Implementation** Okay, let's dive into this per-CPU mapping stuff. My initial instinct is to zero in on a few key functions: `arch_sync_kernel_mappings`, `setup_percpu_pgd`, `map_local_percpu_first_chunk`, and `pcpu_get_local_vm_area`. These seem like the core building blocks for this feature, and likely contain the most interesting logic. First, I want to pin down the configuration options that govern this feature's enablement. Let's start with `HAVE_LOCAL_PER_CPU_MAP`. Quickly checking `arch/arm64/Kconfig`, I see it's a `def_bool y` which means it's automatically enabled if the dependencies (ARM64 with specific VA bit sizes and page sizes) are met. This means we don't have to manually enable it. However, it's still worthwhile to document that it's the critical configuration option here. Then, I'll move on to `NEED_PER_CPU_PAGE_FIRST_CHUNK`. A quick check in `arch/arm64/Kconfig` reveals that ARM64 *selects* this config, meaning it's always enabled when ARM64 is active. Next, I'll verify what I think: `ARCH_HAS_PMD_POPULATE_SYNC` and `ARCH_HAS_PUD_POPULATE_SYNC` are also selected by ARM64. Confirmed. Now, let's jump to the main configuration option, `HAVE_LOCAL_PER_CPU_MAP` and see it's a `bool` in `mm/Kconfig`. Now, I want to verify if the functions I initially listed are good targets for investigation. `pcpu_get_local_vm_area` from `mm/vmalloc.c` is enabled by the `CONFIG_HAVE_LOCAL_PER_CPU_MAP`, so it's a good place to start. `map_local_percpu_first_chunk` from `arch/arm64/mm/mmu.c` is also enabled by `CONFIG_HAVE_LOCAL_PER_CPU_MAP`, so it's a good place to start. `setup_percpu_pgd` from `arch/arm64/mm/mmu.c` is also a key function. `arch_sync_kernel_mappings` from `arch/arm64/mm/mmu.c` is a new function and is also a key function. Now, let's look at a few more functions. `vmap_range_noflush` in `mm/vmalloc.c` and `__vunmap_range_noflush` also in `mm/vmalloc.c` have been changed with a new parameter. Let's keep in mind that they are not new, but have been extended. However, let's keep them on the list just in case. `pcpu_alloc_first_chunk` in `mm/percpu.c` `pcpu_setup_first_chunk` in `mm/percpu.c` `pcpu_page_first_chunk` in `mm/percpu.c` Okay, so I have a more comprehensive list of functions to review, though I can prioritize based on novelty and specificity to this per-CPU feature: Based on the investigation, I have the following set of key symbols: - `pcpu_get_local_vm_area` - `map_local_percpu_first_chunk` - `setup_percpu_pgd` - `arch_sync_kernel_mappings` - `vmap_range_noflush` (extended) - `__vunmap_range_noflush` (extended) - `pcpu_alloc_first_chunk` - `pcpu_setup_first_chunk` - `pcpu_page_first_chunk` And the key configuration options are: - `HAVE_LOCAL_PER_CPU_MAP` - `ARCH_HAS_PMD_POPULATE_SYNC` - `ARCH_HAS_PUD_POPULATE_SYNC` - `NEED_PER_CPU_PAGE_FIRST_CHUNK` Since the other functions `vmap_range_noflush`, `__vunmap_range_noflush`, `pcpu_alloc_first_chunk`, `pcpu_setup_first_chunk` and `pcpu_page_first_chunk` are generic, it's clear |
| 12/2 | 2026/07/15 22:41 | tool | set-results |
0mArgs:{
"EnableConfigs": [
"HAVE_LOCAL_PER_CPU_MAP",
"ARCH_HAS_PMD_POPULATE_SYNC",
"ARCH_HAS_PUD_POPULATE_SYNC",
"NEED_PER_CPU_PAGE_FIRST_CHUNK"
],
"FocusSymbols": [
"pcpu_get_local_vm_area",
"map_local_percpu_first_chunk",
"setup_percpu_pgd",
"arch_sync_kernel_mappings"
],
"Reasoning": "The patch series introduces local per-cpu mapping support for ARM64, modifying core kernel logic for percpu memory management and page tables. This is a functional change that should be fuzzed.",
"WorthFuzzing": true
}Results: {
"EnableConfigs": [
"HAVE_LOCAL_PER_CPU_MAP",
"ARCH_HAS_PMD_POPULATE_SYNC",
"ARCH_HAS_PUD_POPULATE_SYNC",
"NEED_PER_CPU_PAGE_FIRST_CHUNK"
],
"FocusSymbols": [
"pcpu_get_local_vm_area",
"map_local_percpu_first_chunk",
"setup_percpu_pgd",
"arch_sync_kernel_mappings"
],
"Reasoning": "The patch series introduces local per-cpu mapping support for ARM64, modifying core kernel logic for percpu memory management and page tables. This is a functional change that should be fuzzed.",
"WorthFuzzing": true
} |
| Total Calls | Total Tokens | Avg Tokens | Total Duration (Seconds) | Avg Duration (Seconds) |
|---|
| Total Calls | Total Duration (Seconds) | Avg Duration (Seconds) |
|---|