AI Trajectory

Trajectory: (Expand/Collapse All)
Seq	Timestamp	Type	Name	Duration
0/0	2026/07/22 11:11	flow	patch-triage	0m Args: null Results: { "EnableConfigs": [ "CPA_DEBUG" ], "FocusSymbols": [ "change_page_attr_set_clr", "__change_page_attr_set_clr", "__change_page_attr", "cpa_handle_large_page", "should_split_large_page", "arch_should_split_large_page", "arch_split_large_page", "arch_change_pte", "arch_cpa_process_fault", "arch_cpa_process_alias", "arch_cpa_flush", "lookup_address_in_pgd_attr" ], "Reasoning": "The patch refactors the change_page_attr (CPA) logic, moving it from architecture-specific code (x86, riscv) to a generic implementation in mm/set_memory.c. This is a significant functional change to core kernel memory management. Fuzzing should focus on the new generic functions and their architecture-specific callbacks. Enabling CPA_DEBUG will also run the CPA self-tests to help uncover issues.", "WorthFuzzing": true }
1/1	2026/07/22 11:11	action	read-patch-diff	0m Args: null Results: { "PatchDiff": "commit d544b2bff4c34ff4e46cc081518a266ab316515c\nAuthor: syz-cluster \u003ctriage@syzkaller.com\u003e\nDate: Wed Jul 22 11:11:39 2026 +0000\n\n syz-cluster: applied patch under review\n\ndiff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig\nindex d8d25232501791..1bfa94797b1b4b 100644\n--- a/arch/loongarch/Kconfig\n+++ b/arch/loongarch/Kconfig\n@@ -109,6 +109,7 @@ config LOONGARCH\n \tselect GENERIC_LIB_DEVMEM_IS_ALLOWED\n \tselect GENERIC_PCI_IOMAP\n \tselect GENERIC_SCHED_CLOCK\n+\tselect GENERIC_SET_MEMORY\n \tselect GENERIC_SMP_IDLE_THREAD\n \tselect GPIOLIB\n \tselect HAS_IOPORT\ndiff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h\nindex 223528c04d7392..377bef0894d771 100644\n--- a/arch/loongarch/include/asm/pgtable.h\n+++ b/arch/loongarch/include/asm/pgtable.h\n@@ -638,6 +638,48 @@ static inline long pmd_protnone(pmd_t pmd)\n #define pmd_leaf(pmd)\t\t((pmd_val(pmd) \u0026 _PAGE_HUGE) != 0)\n #define pud_leaf(pud)\t\t((pud_val(pud) \u0026 _PAGE_HUGE) != 0)\n \n+#define pgd_write pgd_write\n+static inline int pgd_write(pgd_t pgd)\n+{\n+\treturn !!(pgd_val(pgd) \u0026 _PAGE_WRITE);\n+}\n+\n+#define p4d_write p4d_write\n+static inline int p4d_write(p4d_t p4d)\n+{\n+\treturn !!(p4d_val(p4d) \u0026 _PAGE_WRITE);\n+}\n+\n+#define pud_write pud_write\n+static inline int pud_write(pud_t pud)\n+{\n+\treturn !!(pud_val(pud) \u0026 _PAGE_WRITE);\n+}\n+\n+#define pgd_exec pgd_exec\n+static inline int pgd_exec(pgd_t pgd)\n+{\n+\treturn !(pgd_val(pgd) \u0026 _PAGE_NO_EXEC);\n+}\n+\n+#define p4d_exec p4d_exec\n+static inline int p4d_exec(p4d_t p4d)\n+{\n+\treturn !(p4d_val(p4d) \u0026 _PAGE_NO_EXEC);\n+}\n+\n+#define pud_exec pud_exec\n+static inline int pud_exec(pud_t pud)\n+{\n+\treturn !(pud_val(pud) \u0026 _PAGE_NO_EXEC);\n+}\n+\n+#define pmd_exec pmd_exec\n+static inline int pmd_exec(pmd_t pmd)\n+{\n+\treturn !(pmd_val(pmd) \u0026 _PAGE_NO_EXEC);\n+}\n+\n /\n We provide our own get_unmapped area to cope with the virtual aliasing\n * constraints placed on us by the cache architecture.\ndiff --git a/arch/loongarch/include/asm/set_memory.h b/arch/loongarch/include/asm/set_memory.h\nindex 55dfaefd02c8a6..6ffb4fe5a75b0e 100644\n--- a/arch/loongarch/include/asm/set_memory.h\n+++ b/arch/loongarch/include/asm/set_memory.h\n@@ -19,4 +19,13 @@ int set_direct_map_default_noflush(struct page page);\n int set_direct_map_invalid_noflush(struct page page);\n int set_direct_map_valid_noflush(struct page page, unsigned nr, bool valid);\n \n+/\n+ * The direct map is a fixed address window that is not backed by page tables,\n+ * so a page never has a second mapping whose protection must be synchronised.\n+ /\n+static inline bool cpa_should_update_alias(unsigned long vaddr, unsigned long pfn)\n+{\n+\treturn false;\n+}\n+\n #endif / _ASM_LOONGARCH_SET_MEMORY_H /\ndiff --git a/arch/loongarch/mm/pageattr.c b/arch/loongarch/mm/pageattr.c\nindex 614ccc7afccbea..dbbd408bd3d5e6 100644\n--- a/arch/loongarch/mm/pageattr.c\n+++ b/arch/loongarch/mm/pageattr.c\n@@ -4,126 +4,83 @@\n /\n \n #include \u003clinux/memblock.h\u003e\n-#include \u003clinux/pagewalk.h\u003e\n+#include \u003clinux/mm.h\u003e\n #include \u003clinux/pgtable.h\u003e\n+#include \u003clinux/set_memory.h\u003e\n #include \u003casm/set_memory.h\u003e\n #include \u003casm/tlbflush.h\u003e\n \n-struct pageattr_masks {\n-\tpgprot_t set_mask;\n-\tpgprot_t clear_mask;\n-};\n-\n-static unsigned long set_pageattr_masks(unsigned long val, struct mm_walk walk)\n+static unsigned long set_pageattr_masks(struct cpa_data cpa, unsigned long val)\n {\n-\tunsigned long new_val = val;\n-\tstruct pageattr_masks masks = walk-\u003eprivate;\n-\n-\tnew_val \u0026= ~(pgprot_val(masks-\u003eclear_mask));\n-\tnew_val \|= (pgprot_val(masks-\u003eset_mask));\n+\tval \u0026= ~pgprot_val(cpa-\u003emask_clr);\n+\tval \|= pgprot_val(cpa-\u003emask_set);\n \n-\treturn new_val;\n+\treturn val;\n }\n \n-static int pageattr_pgd_entry(pgd_t pgd, unsigned long addr,\n-\t\t\t unsigned long next, struct mm_walk walk)\n+int arch_should_split_large_page(struct cpa_data cpa, struct cpa_split_data sd)\n {\n-\tpgd_t val = pgdp_get(pgd);\n-\n-\tif (pgd_leaf(val)) {\n-\t\tval = __pgd(set_pageattr_masks(pgd_val(val), walk));\n-\t\tset_pgd(pgd, val);\n-\t}\n-\n-\treturn 0;\n-}\n+\tpte_t old = ptep_get(sd-\u003ekpte);\n+\tunsigned long val = pte_val(old);\n+\tunsigned long new_val = set_pageattr_masks(cpa, val);\n \n-static int pageattr_p4d_entry(p4d_t p4d, unsigned long addr,\n-\t\t\t unsigned long next, struct mm_walk walk)\n-{\n-\tp4d_t val = p4dp_get(p4d);\n+\tcpa-\u003epfn = pte_pfn(old) + ((sd-\u003eaddress \u0026 (sd-\u003epsize - 1)) \u003e\u003e PAGE_SHIFT);\n \n-\tif (p4d_leaf(val)) {\n-\t\tval = __p4d(set_pageattr_masks(p4d_val(val), walk));\n-\t\tset_p4d(p4d, val);\n+\tif (new_val != val) {\n+\t\tset_pte(sd-\u003ekpte, __pte(new_val));\n+\t\tcpa-\u003eflags \|= CPA_FLUSHTLB;\n \t}\n \n \treturn 0;\n }\n \n-static int pageattr_pud_entry(pud_t pud, unsigned long addr,\n-\t\t\t unsigned long next, struct mm_walk walk)\n+int arch_split_large_page(struct cpa_data cpa, struct cpa_split_data sd)\n {\n-\tpud_t val = pudp_get(pud);\n-\n-\tif (pud_leaf(val)) {\n-\t\tval = __pud(set_pageattr_masks(pud_val(val), walk));\n-\t\tset_pud(pud, val);\n-\t}\n+\tWARN_ON_ONCE(1);\n \n-\treturn 0;\n+\treturn -EINVAL;\n }\n \n-static int pageattr_pmd_entry(pmd_t pmd, unsigned long addr,\n-\t\t\t unsigned long next, struct mm_walk walk)\n+void arch_change_pte(struct cpa_data cpa, unsigned long address,\n+\t\t pte_t kpte, pte_t old_pte, bool nx, bool rw)\n {\n-\tpmd_t val = pmdp_get(pmd);\n+\tunsigned long val = pte_val(old_pte);\n+\tpte_t new_pte = __pte(set_pageattr_masks(cpa, val));\n \n-\tif (pmd_leaf(val)) {\n-\t\tval = __pmd(set_pageattr_masks(pmd_val(val), walk));\n-\t\tset_pmd(pmd, val);\n-\t}\n+\tcpa-\u003epfn = pte_pfn(old_pte);\n \n-\treturn 0;\n+\tif (pte_val(old_pte) != pte_val(new_pte)) {\n+\t\tset_pte(kpte, new_pte);\n+\t\tcpa-\u003eflags \|= CPA_FLUSHTLB;\n+\t}\n }\n \n-static int pageattr_pte_entry(pte_t pte, unsigned long addr,\n-\t\t\t unsigned long next, struct mm_walk walk)\n+int arch_cpa_process_fault(struct cpa_data cpa, unsigned long vaddr,\n+\t\t\t int primary)\n {\n-\tpte_t val = ptep_get(pte);\n-\n-\tval = __pte(set_pageattr_masks(pte_val(val), walk));\n-\tset_pte(pte, val);\n+\t/\n+\t The direct map has no page tables and vmalloc ranges may contain\n+\t * holes. Both are silently skipped.\n+\t /\n+\tcpa-\u003enumpages = 1;\n \n \treturn 0;\n }\n \n-static int pageattr_pte_hole(unsigned long addr, unsigned long next,\n-\t\t\t int depth, struct mm_walk walk)\n+int arch_cpa_process_alias(struct cpa_data cpa)\n {\n \treturn 0;\n }\n \n-static const struct mm_walk_ops pageattr_ops = {\n-\t.pgd_entry = pageattr_pgd_entry,\n-\t.p4d_entry = pageattr_p4d_entry,\n-\t.pud_entry = pageattr_pud_entry,\n-\t.pmd_entry = pageattr_pmd_entry,\n-\t.pte_entry = pageattr_pte_entry,\n-\t.pte_hole = pageattr_pte_hole,\n-\t.walk_lock = PGWALK_RDLOCK,\n-};\n-\n-static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask, pgprot_t clear_mask)\n+void arch_cpa_flush(struct cpa_data cpa, int err)\n {\n-\tint ret;\n-\tunsigned long start = addr;\n-\tunsigned long end = start + PAGE_SIZE * numpages;\n-\tstruct pageattr_masks masks = {\n-\t\t.set_mask = set_mask,\n-\t\t.clear_mask = clear_mask\n-\t};\n-\n-\tif (!numpages)\n-\t\treturn 0;\n-\n-\tmmap_write_lock(\u0026init_mm);\n-\tret = walk_kernel_page_table_range(start, end, \u0026pageattr_ops, NULL, \u0026masks);\n-\tmmap_write_unlock(\u0026init_mm);\n-\n-\tflush_tlb_kernel_range(start, end);\n+\tif (err \|\| cpa-\u003eforce_flush_all \|\|\n+\t (cpa-\u003eflags \u0026 (CPA_ARRAY \| CPA_PAGES_ARRAY))) {\n+\t\tflush_tlb_all();\n+\t\treturn;\n+\t}\n \n-\treturn ret;\n+\tflush_tlb_kernel_range(cpa-\u003evaddr, cpa-\u003evaddr + cpa-\u003enumpages * PAGE_SIZE);\n }\n \n int set_memory_x(unsigned long addr, int numpages)\n@@ -131,7 +88,7 @@ int set_memory_x(unsigned long addr, int numpages)\n \tif (addr \u003c vm_map_base)\n \t\treturn 0;\n \n-\treturn __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_NO_EXEC));\n+\treturn change_page_attr_clear(\u0026addr, numpages, __pgprot(_PAGE_NO_EXEC), 0);\n }\n \n int set_memory_nx(unsigned long addr, int numpages)\n@@ -139,7 +96,7 @@ int set_memory_nx(unsigned long addr, int numpages)\n \tif (addr \u003c vm_map_base)\n \t\treturn 0;\n \n-\treturn __set_memory(addr, numpages, __pgprot(_PAGE_NO_EXEC), __pgprot(0));\n+\treturn change_page_attr_set(\u0026addr, numpages, __pgprot(_PAGE_NO_EXEC), 0);\n }\n \n int set_memory_ro(unsigned long addr, int numpages)\n@@ -147,7 +104,7 @@ int set_memory_ro(unsigned long addr, int numpages)\n \tif (addr \u003c vm_map_base)\n \t\treturn 0;\n \n-\treturn __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_WRITE \| _PAGE_DIRTY));\n+\treturn change_page_attr_clear(\u0026addr, numpages, __pgprot(_PAGE_WRITE \| _PAGE_DIRTY), 0);\n }\n \n int set_memory_rw(unsigned long addr, int numpages)\n@@ -155,84 +112,75 @@ int set_memory_rw(unsigned long addr, int numpages)\n \tif (addr \u003c vm_map_base)\n \t\treturn 0;\n \n-\treturn __set_memory(addr, numpages, __pgprot(_PAGE_WRITE \| _PAGE_DIRTY), __pgprot(0));\n+\treturn change_page_attr_set(\u0026addr, numpages, __pgprot(_PAGE_WRITE \| _PAGE_DIRTY), 0);\n }\n \n bool kernel_page_present(struct page page)\n {\n-\tpgd_t pgd;\n-\tp4d_t p4d;\n-\tpud_t pud;\n-\tpmd_t pmd;\n-\tpte_t pte;\n+\tunsigned int level;\n \tunsigned long addr = (unsigned long)page_address(page);\n+\tpte_t pte;\n \n \tif (addr \u003c vm_map_base)\n \t\treturn memblock_is_memory(__pa(addr));\n \n-\tpgd = pgd_offset_k(addr);\n-\tif (pgd_none(pgdp_get(pgd)))\n-\t\treturn false;\n-\tif (pgd_leaf(pgdp_get(pgd)))\n-\t\treturn true;\n-\n-\tp4d = p4d_offset(pgd, addr);\n-\tif (p4d_none(p4dp_get(p4d)))\n-\t\treturn false;\n-\tif (p4d_leaf(p4dp_get(p4d)))\n-\t\treturn true;\n-\n-\tpud = pud_offset(p4d, addr);\n-\tif (pud_none(pudp_get(pud)))\n-\t\treturn false;\n-\tif (pud_leaf(pudp_get(pud)))\n-\t\treturn true;\n-\n-\tpmd = pmd_offset(pud, addr);\n-\tif (pmd_none(pmdp_get(pmd)))\n+\tpte = lookup_address(addr, \u0026level);\n+\tif (!pte)\n \t\treturn false;\n-\tif (pmd_leaf(pmdp_get(pmd)))\n-\t\treturn true;\n \n-\tpte = pte_offset_kernel(pmd, addr);\n \treturn pte_present(ptep_get(pte));\n }\n \n int set_direct_map_default_noflush(struct page page)\n {\n \tunsigned long addr = (unsigned long)page_address(page);\n+\tstruct cpa_data cpa = { .vaddr = \u0026addr,\n+\t\t\t\t.pgd = NULL,\n+\t\t\t\t.numpages = 1,\n+\t\t\t\t.mask_set = PAGE_KERNEL,\n+\t\t\t\t.mask_clr = __pgprot(0),\n+\t\t\t\t.flags = CPA_NO_CHECK_ALIAS };\n \n \tif (addr \u003c vm_map_base)\n \t\treturn 0;\n \n-\treturn __set_memory(addr, 1, PAGE_KERNEL, __pgprot(0));\n+\treturn __change_page_attr_set_clr(\u0026cpa, 1);\n }\n \n int set_direct_map_invalid_noflush(struct page page)\n {\n \tunsigned long addr = (unsigned long)page_address(page);\n+\tstruct cpa_data cpa = { .vaddr = \u0026addr,\n+\t\t\t\t.pgd = NULL,\n+\t\t\t\t.numpages = 1,\n+\t\t\t\t.mask_set = __pgprot(0),\n+\t\t\t\t.mask_clr = __pgprot(_PAGE_PRESENT \| _PAGE_VALID),\n+\t\t\t\t.flags = CPA_NO_CHECK_ALIAS };\n \n \tif (addr \u003c vm_map_base)\n \t\treturn 0;\n \n-\treturn __set_memory(addr, 1, __pgprot(0), __pgprot(_PAGE_PRESENT \| _PAGE_VALID));\n+\treturn __change_page_attr_set_clr(\u0026cpa, 1);\n }\n \n int set_direct_map_valid_noflush(struct page page, unsigned nr, bool valid)\n {\n \tunsigned long addr = (unsigned long)page_address(page);\n-\tpgprot_t set, clear;\n+\tstruct cpa_data cpa = { .vaddr = \u0026addr,\n+\t\t\t\t.pgd = NULL,\n+\t\t\t\t.numpages = nr,\n+\t\t\t\t.flags = CPA_NO_CHECK_ALIAS };\n \n \tif (addr \u003c vm_map_base)\n \t\treturn 0;\n \n \tif (valid) {\n-\t\tset = PAGE_KERNEL;\n-\t\tclear = __pgprot(0);\n+\t\tcpa.mask_set = PAGE_KERNEL;\n+\t\tcpa.mask_clr = __pgprot(0);\n \t} else {\n-\t\tset = __pgprot(0);\n-\t\tclear = __pgprot(_PAGE_PRESENT \| _PAGE_VALID);\n+\t\tcpa.mask_set = __pgprot(0);\n+\t\tcpa.mask_clr = __pgprot(_PAGE_PRESENT \| _PAGE_VALID);\n \t}\n \n-\treturn __set_memory(addr, nr, set, clear);\n+\treturn __change_page_attr_set_clr(\u0026cpa, 1);\n }\ndiff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig\nindex 3f0a647218e407..5d069bf290ce78 100644\n--- a/arch/riscv/Kconfig\n+++ b/arch/riscv/Kconfig\n@@ -48,6 +48,7 @@ config RISCV\n \tselect ARCH_HAS_PTE_SPECIAL\n \tselect ARCH_HAS_SET_DIRECT_MAP if MMU\n \tselect ARCH_HAS_SET_MEMORY if MMU\n+\tselect GENERIC_SET_MEMORY if MMU\n \tselect ARCH_HAS_STRICT_KERNEL_RWX if MMU\n \tselect ARCH_HAS_STRICT_MODULE_RWX if MMU\n \tselect ARCH_HAS_SYNC_CORE_BEFORE_USERMODE\ndiff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h\nindex 5d5756bda82e3d..027b3e7e4f177d 100644\n--- a/arch/riscv/include/asm/pgtable.h\n+++ b/arch/riscv/include/asm/pgtable.h\n@@ -819,6 +819,42 @@ static inline int pud_write(pud_t pud)\n \treturn pte_write(pud_pte(pud));\n }\n \n+#define pgd_write pgd_write\n+static inline int pgd_write(pgd_t pgd)\n+{\n+\treturn pgd_val(pgd) \u0026 _PAGE_WRITE;\n+}\n+\n+#define p4d_write p4d_write\n+static inline int p4d_write(p4d_t p4d)\n+{\n+\treturn p4d_val(p4d) \u0026 _PAGE_WRITE;\n+}\n+\n+#define pgd_exec pgd_exec\n+static inline int pgd_exec(pgd_t pgd)\n+{\n+\treturn !!(pgd_val(pgd) \u0026 _PAGE_EXEC);\n+}\n+\n+#define p4d_exec p4d_exec\n+static inline int p4d_exec(p4d_t p4d)\n+{\n+\treturn !!(p4d_val(p4d) \u0026 _PAGE_EXEC);\n+}\n+\n+#define pud_exec pud_exec\n+static inline int pud_exec(pud_t pud)\n+{\n+\treturn !!(pud_val(pud) \u0026 _PAGE_EXEC);\n+}\n+\n+#define pmd_exec pmd_exec\n+static inline int pmd_exec(pmd_t pmd)\n+{\n+\treturn !!(pmd_val(pmd) \u0026 _PAGE_EXEC);\n+}\n+\n #define pmd_dirty pmd_dirty\n static inline int pmd_dirty(pmd_t pmd)\n {\ndiff --git a/arch/riscv/include/asm/set_memory.h b/arch/riscv/include/asm/set_memory.h\nindex ef59e1716a2cfd..4ba46a33103433 100644\n--- a/arch/riscv/include/asm/set_memory.h\n+++ b/arch/riscv/include/asm/set_memory.h\n@@ -7,6 +7,8 @@\n #define _ASM_RISCV_SET_MEMORY_H\n \n #ifndef __ASSEMBLER__\n+#include \u003casm/page.h\u003e\n+#include \u003casm/pgtable.h\u003e\n /\n Functions to change memory attributes.\n /\n@@ -26,6 +28,11 @@ static __always_inline int set_kernel_memory(char startp, char endp,\n \n \treturn set_memory(start, num_pages);\n }\n+\n+static inline bool cpa_should_update_alias(unsigned long vaddr, unsigned long pfn)\n+{\n+\treturn !is_linear_mapping(vaddr);\n+}\n #else\n static inline int set_memory_ro(unsigned long addr, int numpages) { return 0; }\n static inline int set_memory_rw(unsigned long addr, int numpages) { return 0; }\ndiff --git a/arch/riscv/mm/pageattr.c b/arch/riscv/mm/pageattr.c\nindex 3f76db3d276992..66a20fc3d80574 100644\n--- a/arch/riscv/mm/pageattr.c\n+++ b/arch/riscv/mm/pageattr.c\n@@ -3,402 +3,242 @@\n Copyright (C) 2019 SiFive\n /\n \n-#include \u003clinux/pagewalk.h\u003e\n+#include \u003clinux/mm.h\u003e\n #include \u003clinux/pgtable.h\u003e\n-#include \u003clinux/vmalloc.h\u003e\n+#include \u003clinux/set_memory.h\u003e\n #include \u003casm/tlbflush.h\u003e\n-#include \u003casm/bitops.h\u003e\n #include \u003casm/set_memory.h\u003e\n \n-struct pageattr_masks {\n-\tpgprot_t set_mask;\n-\tpgprot_t clear_mask;\n-};\n-\n-static unsigned long set_pageattr_masks(unsigned long val, struct mm_walk walk)\n-{\n-\tstruct pageattr_masks masks = walk-\u003eprivate;\n-\tunsigned long new_val = val;\n-\n-\tnew_val \u0026= ~(pgprot_val(masks-\u003eclear_mask));\n-\tnew_val \|= (pgprot_val(masks-\u003eset_mask));\n-\n-\treturn new_val;\n-}\n-\n-static int pageattr_p4d_entry(p4d_t p4d, unsigned long addr,\n-\t\t\t unsigned long next, struct mm_walk walk)\n+int arch_should_split_large_page(struct cpa_data cpa, struct cpa_split_data sd)\n {\n-\tp4d_t val = p4dp_get(p4d);\n-\n-\tif (p4d_leaf(val)) {\n-\t\tval = __p4d(set_pageattr_masks(p4d_val(val), walk));\n-\t\tset_p4d(p4d, val);\n-\t}\n+\tpte_t old = ptep_get(sd-\u003ekpte);\n+\tpgprot_t old_prot = __pgprot(pte_val(old) \u0026 ~_PAGE_PFN_MASK);\n+\tpgprot_t new_prot = old_prot;\n+\tunsigned long old_pfn = pte_pfn(old);\n+\tunsigned long lpaddr, numpages;\n \n-\treturn 0;\n-}\n+\tpgprot_val(new_prot) \u0026= ~pgprot_val(cpa-\u003emask_clr);\n+\tpgprot_val(new_prot) \|= pgprot_val(cpa-\u003emask_set);\n \n-static int pageattr_pud_entry(pud_t pud, unsigned long addr,\n-\t\t\t unsigned long next, struct mm_walk walk)\n-{\n-\tpud_t val = pudp_get(pud);\n-\n-\tif (pud_leaf(val)) {\n-\t\tval = __pud(set_pageattr_masks(pud_val(val), walk));\n-\t\tset_pud(pud, val);\n-\t}\n+\t/\n+\t * Record the pfn mapped at @address so the alias check can locate the\n+\t * matching linear map entry.\n+\t /\n+\tcpa-\u003epfn = old_pfn + ((sd-\u003eaddress \u0026 (sd-\u003epsize - 1)) \u003e\u003e PAGE_SHIFT);\n \n-\treturn 0;\n-}\n+\t/ If the protections do not change, keep the large page intact. /\n+\tif (pgprot_val(new_prot) == pgprot_val(old_prot))\n+\t\treturn 0;\n \n-static int pageattr_pmd_entry(pmd_t pmd, unsigned long addr,\n-\t\t\t unsigned long next, struct mm_walk walk)\n-{\n-\tpmd_t val = pmdp_get(pmd);\n+\t/ If the request does not cover the whole large page, split it. /\n+\tlpaddr = sd-\u003eaddress \u0026 sd-\u003epmask;\n+\tnumpages = sd-\u003epsize \u003e\u003e PAGE_SHIFT;\n+\tif (sd-\u003eaddress != lpaddr \|\| cpa-\u003enumpages != numpages)\n+\t\treturn 1;\n \n-\tif (pmd_leaf(val)) {\n-\t\tval = __pmd(set_pageattr_masks(pmd_val(val), walk));\n-\t\tset_pmd(pmd, val);\n-\t}\n+\t/ The whole large page is covered: update it in place. /\n+\tset_pte(sd-\u003ekpte, pfn_pte(old_pfn, new_prot));\n+\tcpa-\u003eflags \|= CPA_FLUSHTLB;\n \n \treturn 0;\n }\n \n-static int pageattr_pte_entry(pte_t pte, unsigned long addr,\n-\t\t\t unsigned long next, struct mm_walk walk)\n+static void split_set_ptes(pte_t ptep, unsigned long pfn, unsigned long pfninc,\n+\t\t\t pgprot_t prot)\n {\n-\tpte_t val = ptep_get(pte);\n-\n-\tval = __pte(set_pageattr_masks(pte_val(val), walk));\n-\tset_pte(pte, val);\n+\tunsigned int i;\n \n-\treturn 0;\n+\tfor (i = 0; i \u003c PTRS_PER_PTE; ++i, ++ptep, pfn += pfninc)\n+\t\tset_pte(ptep, pfn_pte(pfn, prot));\n }\n \n-static int pageattr_pte_hole(unsigned long addr, unsigned long next,\n-\t\t\t int depth, struct mm_walk walk)\n+int arch_split_large_page(struct cpa_data cpa, struct cpa_split_data sd)\n {\n-\t/ Nothing to do here /\n-\treturn 0;\n-}\n-\n-static const struct mm_walk_ops pageattr_ops = {\n-\t.p4d_entry = pageattr_p4d_entry,\n-\t.pud_entry = pageattr_pud_entry,\n-\t.pmd_entry = pageattr_pmd_entry,\n-\t.pte_entry = pageattr_pte_entry,\n-\t.pte_hole = pageattr_pte_hole,\n-\t.walk_lock = PGWALK_RDLOCK,\n-};\n-\n #ifdef CONFIG_64BIT\n-static int __split_linear_mapping_pmd(pud_t pudp,\n-\t\t\t\t unsigned long vaddr, unsigned long end)\n-{\n-\tpmd_t pmdp;\n-\tunsigned long next;\n-\n-\tpmdp = pmd_offset(pudp, vaddr);\n-\n-\tdo {\n-\t\tnext = pmd_addr_end(vaddr, end);\n-\n-\t\tif (next - vaddr \u003e= PMD_SIZE \u0026\u0026\n-\t\t vaddr \u003c= (vaddr \u0026 PMD_MASK) \u0026\u0026 end \u003e= next)\n-\t\t\tcontinue;\n-\n-\t\tif (pmd_leaf(pmdp_get(pmdp))) {\n-\t\t\tstruct page pte_page;\n-\t\t\tunsigned long pfn = _pmd_pfn(pmdp_get(pmdp));\n-\t\t\tpgprot_t prot = __pgprot(pmd_val(pmdp_get(pmdp)) \u0026 ~_PAGE_PFN_MASK);\n-\t\t\tpte_t ptep_new;\n-\t\t\tint i;\n-\n-\t\t\tpte_page = alloc_page(GFP_KERNEL);\n-\t\t\tif (!pte_page)\n-\t\t\t\treturn -ENOMEM;\n+\tstruct page base = ptdesc_page(sd-\u003eptdesc);\n+\tpte_t old = ptep_get(sd-\u003ekpte);\n+\tpgprot_t prot = __pgprot(pte_val(old) \u0026 ~_PAGE_PFN_MASK);\n+\tunsigned long pfn = pte_pfn(old);\n+\tunsigned long pfninc;\n+\n+\tswitch (sd-\u003elevel) {\n+\tcase PGTABLE_LEVEL_PMD:\n+\t\tpfninc = 1;\n+\t\tbreak;\n+\tcase PGTABLE_LEVEL_PUD:\n+\t\tpfninc = PMD_SIZE \u003e\u003e PAGE_SHIFT;\n+\t\tbreak;\n+\tcase PGTABLE_LEVEL_P4D:\n+\t\tpfninc = PUD_SIZE \u003e\u003e PAGE_SHIFT;\n+\t\tbreak;\n+\tdefault:\n+\t\treturn -EINVAL;\n+\t}\n \n-\t\t\tptep_new = (pte_t )page_address(pte_page);\n-\t\t\tfor (i = 0; i \u003c PTRS_PER_PTE; ++i, ++ptep_new)\n-\t\t\t\tset_pte(ptep_new, pfn_pte(pfn + i, prot));\n+\tsplit_set_ptes((pte_t )page_address(base), pfn, pfninc, prot);\n \n-\t\t\tsmp_wmb();\n+\tsmp_wmb();\n+\tset_pte(sd-\u003ekpte, pfn_pte(page_to_pfn(base), PAGE_TABLE));\n \n-\t\t\tset_pmd(pmdp, pfn_pmd(page_to_pfn(pte_page), PAGE_TABLE));\n-\t\t}\n-\t} while (pmdp++, vaddr = next, vaddr != end);\n+\tcpa-\u003eflags \|= CPA_FLUSHTLB;\n+\tcpa-\u003eforce_flush_all = 1;\n \n \treturn 0;\n+#else\n+\tWARN_ON_ONCE(1);\n+\treturn -EINVAL;\n+#endif\n }\n \n-static int __split_linear_mapping_pud(p4d_t p4dp,\n-\t\t\t\t unsigned long vaddr, unsigned long end)\n+void arch_change_pte(struct cpa_data cpa, unsigned long address,\n+\t\t pte_t kpte, pte_t old_pte, bool nx, bool rw)\n {\n-\tpud_t pudp;\n-\tunsigned long next;\n-\tint ret;\n-\n-\tpudp = pud_offset(p4dp, vaddr);\n-\n-\tdo {\n-\t\tnext = pud_addr_end(vaddr, end);\n-\n-\t\tif (next - vaddr \u003e= PUD_SIZE \u0026\u0026\n-\t\t vaddr \u003c= (vaddr \u0026 PUD_MASK) \u0026\u0026 end \u003e= next)\n-\t\t\tcontinue;\n+\tpgprot_t new_prot = __pgprot(pte_val(old_pte) \u0026 ~_PAGE_PFN_MASK);\n+\tunsigned long pfn = pte_pfn(old_pte);\n+\tpte_t new_pte;\n \n-\t\tif (pud_leaf(pudp_get(pudp))) {\n-\t\t\tstruct page pmd_page;\n-\t\t\tunsigned long pfn = _pud_pfn(pudp_get(pudp));\n-\t\t\tpgprot_t prot = __pgprot(pud_val(pudp_get(pudp)) \u0026 ~_PAGE_PFN_MASK);\n-\t\t\tpmd_t pmdp_new;\n-\t\t\tint i;\n+\tpgprot_val(new_prot) \u0026= ~pgprot_val(cpa-\u003emask_clr);\n+\tpgprot_val(new_prot) \|= pgprot_val(cpa-\u003emask_set);\n \n-\t\t\tpmd_page = alloc_page(GFP_KERNEL);\n-\t\t\tif (!pmd_page)\n-\t\t\t\treturn -ENOMEM;\n+\tnew_pte = pfn_pte(pfn, new_prot);\n+\tcpa-\u003epfn = pfn;\n \n-\t\t\tpmdp_new = (pmd_t )page_address(pmd_page);\n-\t\t\tfor (i = 0; i \u003c PTRS_PER_PMD; ++i, ++pmdp_new)\n-\t\t\t\tset_pmd(pmdp_new,\n-\t\t\t\t\tpfn_pmd(pfn + ((i PMD_SIZE) \u003e\u003e PAGE_SHIFT), prot));\n-\n-\t\t\tsmp_wmb();\n-\n-\t\t\tset_pud(pudp, pfn_pud(page_to_pfn(pmd_page), PAGE_TABLE));\n-\t\t}\n-\n-\t\tret = __split_linear_mapping_pmd(pudp, vaddr, next);\n-\t\tif (ret)\n-\t\t\treturn ret;\n-\t} while (pudp++, vaddr = next, vaddr != end);\n-\n-\treturn 0;\n+\tif (pte_val(old_pte) != pte_val(new_pte)) {\n+\t\tset_pte(kpte, new_pte);\n+\t\tcpa-\u003eflags \|= CPA_FLUSHTLB;\n+\t}\n }\n \n-static int __split_linear_mapping_p4d(pgd_t pgdp,\n-\t\t\t\t unsigned long vaddr, unsigned long end)\n+int arch_cpa_process_fault(struct cpa_data cpa, unsigned long vaddr,\n+\t\t\t int primary)\n {\n-\tp4d_t p4dp;\n-\tunsigned long next;\n-\tint ret;\n-\n-\tp4dp = p4d_offset(pgdp, vaddr);\n-\n-\tdo {\n-\t\tnext = p4d_addr_end(vaddr, end);\n-\n-\t\t/\n-\t\t * If [vaddr; end] contains [vaddr \u0026 P4D_MASK; next], we don't\n-\t\t * need to split, we'll change the protections on the whole P4D.\n-\t\t /\n-\t\tif (next - vaddr \u003e= P4D_SIZE \u0026\u0026\n-\t\t vaddr \u003c= (vaddr \u0026 P4D_MASK) \u0026\u0026 end \u003e= next)\n-\t\t\tcontinue;\n-\n-\t\tif (p4d_leaf(p4dp_get(p4dp))) {\n-\t\t\tstruct page pud_page;\n-\t\t\tunsigned long pfn = _p4d_pfn(p4dp_get(p4dp));\n-\t\t\tpgprot_t prot = __pgprot(p4d_val(p4dp_get(p4dp)) \u0026 ~_PAGE_PFN_MASK);\n-\t\t\tpud_t pudp_new;\n-\t\t\tint i;\n-\n-\t\t\tpud_page = alloc_page(GFP_KERNEL);\n-\t\t\tif (!pud_page)\n-\t\t\t\treturn -ENOMEM;\n-\n-\t\t\t/\n-\t\t\t * Fill the pud level with leaf puds that have the same\n-\t\t\t * protections as the leaf p4d.\n-\t\t\t /\n-\t\t\tpudp_new = (pud_t )page_address(pud_page);\n-\t\t\tfor (i = 0; i \u003c PTRS_PER_PUD; ++i, ++pudp_new)\n-\t\t\t\tset_pud(pudp_new,\n-\t\t\t\t\tpfn_pud(pfn + ((i * PUD_SIZE) \u003e\u003e PAGE_SHIFT), prot));\n-\n-\t\t\t/\n-\t\t\t Make sure the pud filling is not reordered with the\n-\t\t\t * p4d store which could result in seeing a partially\n-\t\t\t * filled pud level.\n-\t\t\t /\n-\t\t\tsmp_wmb();\n-\n-\t\t\tset_p4d(p4dp, pfn_p4d(page_to_pfn(pud_page), PAGE_TABLE));\n-\t\t}\n-\n-\t\tret = __split_linear_mapping_pud(p4dp, vaddr, next);\n-\t\tif (ret)\n-\t\t\treturn ret;\n-\t} while (p4dp++, vaddr = next, vaddr != end);\n+\tcpa-\u003enumpages = 1;\n \n-\treturn 0;\n-}\n-\n-static int __split_linear_mapping_pgd(pgd_t pgdp,\n-\t\t\t\t unsigned long vaddr,\n-\t\t\t\t unsigned long end)\n-{\n-\tunsigned long next;\n-\tint ret;\n+\tif (!primary)\n+\t\treturn 0;\n \n-\tdo {\n-\t\tnext = pgd_addr_end(vaddr, end);\n-\t\t/* We never use PGD mappings for the linear mapping /\n-\t\tret = __split_linear_mapping_p4d(pgdp, vaddr, next);\n-\t\tif (ret)\n-\t\t\treturn ret;\n-\t} while (pgdp++, vaddr = next, vaddr != end);\n+\t/ The linear map is expected to have holes /\n+\tif (is_linear_mapping(vaddr)) {\n+\t\tcpa-\u003epfn = PFN_DOWN(__pa(vaddr));\n+\t\treturn 0;\n+\t}\n \n-\treturn 0;\n-}\n+\tWARN(1, \"CPA: called for zero pte. vaddr = %lx cpa-\u003evaddr = %lx\\n\",\n+\t vaddr, cpa-\u003evaddr);\n \n-static int split_linear_mapping(unsigned long start, unsigned long end)\n-{\n-\treturn __split_linear_mapping_pgd(pgd_offset_k(start), start, end);\n+\treturn -EFAULT;\n }\n-#endif\t/* CONFIG_64BIT /\n \n-static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask,\n-\t\t\tpgprot_t clear_mask)\n+int arch_cpa_process_alias(struct cpa_data cpa)\n {\n-\tint ret;\n-\tunsigned long start = addr;\n-\tunsigned long end = start + PAGE_SIZE * numpages;\n-\tunsigned long __maybe_unused lm_start;\n-\tunsigned long __maybe_unused lm_end;\n-\tstruct pageattr_masks masks = {\n-\t\t.set_mask = set_mask,\n-\t\t.clear_mask = clear_mask\n-\t};\n-\n-\tif (!numpages)\n-\t\treturn 0;\n+\tstruct cpa_data alias_cpa;\n+\tunsigned long laddr;\n \n-\tmmap_write_lock(\u0026init_mm);\n-\n-#ifdef CONFIG_64BIT\n \t/\n-\t We are about to change the permissions of a kernel mapping, we must\n-\t * apply the same changes to its linear mapping alias, which may imply\n-\t * splitting a huge mapping.\n+\t * cpa_should_update_alias() only lets non linear map primaries reach\n+\t * here, so @cpa-\u003epfn always has a linear map alias that must receive\n+\t * the same protection change.\n \t /\n+\tladdr = (unsigned long)__va(PFN_PHYS(cpa-\u003epfn));\n \n-\tif (is_vmalloc_or_module_addr((void )start)) {\n-\t\tstruct vm_struct area = NULL;\n-\t\tint i, page_start;\n-\n-\t\tarea = find_vm_area((void )start);\n-\t\tpage_start = (start - (unsigned long)area-\u003eaddr) \u003e\u003e PAGE_SHIFT;\n-\n-\t\tfor (i = page_start; i \u003c page_start + numpages; ++i) {\n-\t\t\tlm_start = (unsigned long)page_address(area-\u003epages[i]);\n-\t\t\tlm_end = lm_start + PAGE_SIZE;\n-\n-\t\t\tret = split_linear_mapping(lm_start, lm_end);\n-\t\t\tif (ret)\n-\t\t\t\tgoto unlock;\n-\n-\t\t\tret = walk_kernel_page_table_range(lm_start, lm_end,\n-\t\t\t\t\t\t \u0026pageattr_ops, NULL, \u0026masks);\n-\t\t\tif (ret)\n-\t\t\t\tgoto unlock;\n-\t\t}\n-\t} else if (is_kernel_mapping(start) \|\| is_linear_mapping(start)) {\n-\t\tif (is_kernel_mapping(start)) {\n-\t\t\tlm_start = (unsigned long)lm_alias(start);\n-\t\t\tlm_end = (unsigned long)lm_alias(end);\n-\t\t} else {\n-\t\t\tlm_start = start;\n-\t\t\tlm_end = end;\n-\t\t}\n-\n-\t\tret = split_linear_mapping(lm_start, lm_end);\n-\t\tif (ret)\n-\t\t\tgoto unlock;\n-\n-\t\tret = walk_kernel_page_table_range(lm_start, lm_end,\n-\t\t\t\t\t \u0026pageattr_ops, NULL, \u0026masks);\n-\t\tif (ret)\n-\t\t\tgoto unlock;\n-\t}\n-\n-\tret = walk_kernel_page_table_range(start, end, \u0026pageattr_ops, NULL,\n-\t\t\t\t \u0026masks);\n+\talias_cpa = cpa;\n+\talias_cpa.vaddr = \u0026laddr;\n+\talias_cpa.flags \u0026= ~(CPA_PAGES_ARRAY \| CPA_ARRAY);\n+\talias_cpa.curpage = 0;\n \n-unlock:\n-\tmmap_write_unlock(\u0026init_mm);\n+\t/ The linear map alias must never be made executable /\n+\talias_cpa.mask_set = __pgprot(pgprot_val(alias_cpa.mask_set) \u0026 ~_PAGE_EXEC);\n+\talias_cpa.mask_clr = __pgprot(pgprot_val(alias_cpa.mask_clr) \u0026 ~_PAGE_EXEC);\n \n-\t/\n-\t * We can't use flush_tlb_kernel_range() here as we may have split a\n-\t * hugepage that is larger than that, so let's flush everything.\n-\t /\n-\tflush_tlb_all();\n-#else\n-\tret = walk_kernel_page_table_range(start, end, \u0026pageattr_ops, NULL,\n-\t\t\t\t \u0026masks);\n+\tcpa-\u003eforce_flush_all = 1;\n \n-\tmmap_write_unlock(\u0026init_mm);\n+\treturn __change_page_attr_set_clr(\u0026alias_cpa, 0);\n+}\n \n-\tflush_tlb_kernel_range(start, end);\n-#endif\n+void arch_cpa_flush(struct cpa_data cpa, int err)\n+{\n+\tif (err \|\| cpa-\u003eforce_flush_all \|\|\n+\t (cpa-\u003eflags \u0026 (CPA_ARRAY \| CPA_PAGES_ARRAY))) {\n+\t\tflush_tlb_all();\n+\t\treturn;\n+\t}\n \n-\treturn ret;\n+\tflush_tlb_kernel_range(cpa-\u003evaddr,\n+\t\t\t cpa-\u003evaddr + cpa-\u003enumpages * PAGE_SIZE);\n }\n \n int set_memory_rw_nx(unsigned long addr, int numpages)\n {\n-\treturn __set_memory(addr, numpages, __pgprot(_PAGE_READ \| _PAGE_WRITE),\n-\t\t\t __pgprot(_PAGE_EXEC));\n+\treturn change_page_attr_set_clr(\u0026addr, numpages,\n+\t\t\t\t\t__pgprot(_PAGE_READ \| _PAGE_WRITE),\n+\t\t\t\t\t__pgprot(_PAGE_EXEC), 0, 0, NULL);\n }\n \n int set_memory_ro(unsigned long addr, int numpages)\n {\n-\treturn __set_memory(addr, numpages, __pgprot(_PAGE_READ),\n-\t\t\t __pgprot(_PAGE_WRITE));\n+\treturn change_page_attr_set_clr(\u0026addr, numpages, __pgprot(_PAGE_READ),\n+\t\t\t\t\t__pgprot(_PAGE_WRITE), 0, 0, NULL);\n }\n \n int set_memory_rw(unsigned long addr, int numpages)\n {\n-\treturn __set_memory(addr, numpages, __pgprot(_PAGE_READ \| _PAGE_WRITE),\n-\t\t\t __pgprot(0));\n+\treturn change_page_attr_set(\u0026addr, numpages,\n+\t\t\t\t __pgprot(_PAGE_READ \| _PAGE_WRITE), 0);\n }\n \n int set_memory_x(unsigned long addr, int numpages)\n {\n-\treturn __set_memory(addr, numpages, __pgprot(_PAGE_EXEC), __pgprot(0));\n+\treturn change_page_attr_set(\u0026addr, numpages, __pgprot(_PAGE_EXEC), 0);\n }\n \n int set_memory_nx(unsigned long addr, int numpages)\n {\n-\treturn __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_EXEC));\n+\treturn change_page_attr_clear(\u0026addr, numpages, __pgprot(_PAGE_EXEC), 0);\n }\n \n int set_direct_map_invalid_noflush(struct page page)\n {\n-\treturn __set_memory((unsigned long)page_address(page), 1,\n-\t\t\t __pgprot(0), __pgprot(_PAGE_PRESENT));\n+\tunsigned long start = (unsigned long)page_address(page);\n+\tstruct cpa_data cpa = { .vaddr = \u0026start,\n+\t\t\t\t.pgd = NULL,\n+\t\t\t\t.numpages = 1,\n+\t\t\t\t.mask_set = __pgprot(0),\n+\t\t\t\t.mask_clr = __pgprot(_PAGE_PRESENT),\n+\t\t\t\t.flags = CPA_NO_CHECK_ALIAS };\n+\n+\treturn __change_page_attr_set_clr(\u0026cpa, 1);\n }\n \n int set_direct_map_default_noflush(struct page page)\n {\n-\treturn __set_memory((unsigned long)page_address(page), 1,\n-\t\t\t PAGE_KERNEL, __pgprot(_PAGE_EXEC));\n+\tunsigned long start = (unsigned long)page_address(page);\n+\tstruct cpa_data cpa = { .vaddr = \u0026start,\n+\t\t\t\t.pgd = NULL,\n+\t\t\t\t.numpages = 1,\n+\t\t\t\t.mask_set = PAGE_KERNEL,\n+\t\t\t\t.mask_clr = __pgprot(_PAGE_EXEC),\n+\t\t\t\t.flags = CPA_NO_CHECK_ALIAS };\n+\n+\treturn __change_page_attr_set_clr(\u0026cpa, 1);\n }\n \n int set_direct_map_valid_noflush(struct page page, unsigned nr, bool valid)\n {\n-\tpgprot_t set, clear;\n+\tunsigned long start = (unsigned long)page_address(page);\n+\tstruct cpa_data cpa = { .vaddr = \u0026start,\n+\t\t\t\t.pgd = NULL,\n+\t\t\t\t.numpages = nr,\n+\t\t\t\t.flags = CPA_NO_CHECK_ALIAS };\n \n \tif (valid) {\n-\t\tset = PAGE_KERNEL;\n-\t\tclear = __pgprot(_PAGE_EXEC);\n+\t\tcpa.mask_set = PAGE_KERNEL;\n+\t\tcpa.mask_clr = __pgprot(_PAGE_EXEC);\n \t} else {\n-\t\tset = __pgprot(0);\n-\t\tclear = __pgprot(_PAGE_PRESENT);\n+\t\tcpa.mask_set = __pgprot(0);\n+\t\tcpa.mask_clr = __pgprot(_PAGE_PRESENT);\n \t}\n \n-\treturn __set_memory((unsigned long)page_address(page), nr, set, clear);\n+\treturn __change_page_attr_set_clr(\u0026cpa, 1);\n }\n \n #ifdef CONFIG_DEBUG_PAGEALLOC\n@@ -435,36 +275,11 @@ void __kernel_map_pages(struct page page, int numpages, int enable)\n bool kernel_page_present(struct page page)\n {\n \tunsigned long addr = (unsigned long)page_address(page);\n-\tpgd_t pgd;\n-\tpud_t pud;\n-\tp4d_t p4d;\n-\tpmd_t pmd;\n-\tpte_t pte;\n-\n-\tpgd = pgd_offset_k(addr);\n-\tif (!pgd_present(pgdp_get(pgd)))\n-\t\treturn false;\n-\tif (pgd_leaf(pgdp_get(pgd)))\n-\t\treturn true;\n-\n-\tp4d = p4d_offset(pgd, addr);\n-\tif (!p4d_present(p4dp_get(p4d)))\n-\t\treturn false;\n-\tif (p4d_leaf(p4dp_get(p4d)))\n-\t\treturn true;\n-\n-\tpud = pud_offset(p4d, addr);\n-\tif (!pud_present(pudp_get(pud)))\n-\t\treturn false;\n-\tif (pud_leaf(pudp_get(pud)))\n-\t\treturn true;\n+\tunsigned int level;\n+\tpte_t pte = lookup_address(addr, \u0026level);\n \n-\tpmd = pmd_offset(pud, addr);\n-\tif (!pmd_present(pmdp_get(pmd)))\n+\tif (!pte)\n \t\treturn false;\n-\tif (pmd_leaf(pmdp_get(pmd)))\n-\t\treturn true;\n \n-\tpte = pte_offset_kernel(pmd, addr);\n \treturn pte_present(ptep_get(pte));\n }\ndiff --git a/arch/x86/Kconfig b/arch/x86/Kconfig\nindex bdad90f210e4bf..e1f105e52d6a12 100644\n--- a/arch/x86/Kconfig\n+++ b/arch/x86/Kconfig\n@@ -336,6 +336,7 @@ config X86\n \tselect ARCH_SUPPORTS_SCHED_CLUSTER\tif SMP\n \tselect ARCH_SUPPORTS_SCHED_MC\t\tif SMP\n \tselect HAVE_SINGLE_FTRACE_DIRECT_OPS\tif X86_64 \u0026\u0026 DYNAMIC_FTRACE_WITH_DIRECT_CALLS\n+\tselect GENERIC_SET_MEMORY\n \n config INSTRUCTION_DECODER\n \tdef_bool y\ndiff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug\nindex c95c3aaadf97c3..314b72aad6fff4 100644\n--- a/arch/x86/Kconfig.debug\n+++ b/arch/x86/Kconfig.debug\n@@ -157,12 +157,6 @@ config DEBUG_BOOT_PARAMS\n \thelp\n \t This option will cause struct boot_params to be exported via debugfs.\n \n-config CPA_DEBUG\n-\tbool \"CPA self-test code\"\n-\tdepends on DEBUG_KERNEL\n-\thelp\n-\t Do change_page_attr() self-tests every 30 seconds.\n-\n config DEBUG_ENTRY\n \tbool \"Debug low-level entry code\"\n \tdepends on DEBUG_KERNEL\ndiff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h\nindex ac295ca6c92f99..497aaf08ad3616 100644\n--- a/arch/x86/include/asm/pgtable.h\n+++ b/arch/x86/include/asm/pgtable.h\n@@ -227,6 +227,42 @@ static inline int pud_write(pud_t pud)\n \treturn pud_flags(pud) \u0026 _PAGE_RW;\n }\n \n+#define p4d_write p4d_write\n+static inline int p4d_write(p4d_t p4d)\n+{\n+\treturn p4d_flags(p4d) \u0026 _PAGE_RW;\n+}\n+\n+#define pgd_write pgd_write\n+static inline int pgd_write(pgd_t pgd)\n+{\n+\treturn pgd_flags(pgd) \u0026 _PAGE_RW;\n+}\n+\n+#define pmd_exec pmd_exec\n+static inline int pmd_exec(pmd_t pmd)\n+{\n+\treturn !(pmd_flags(pmd) \u0026 _PAGE_NX);\n+}\n+\n+#define pud_exec pud_exec\n+static inline int pud_exec(pud_t pud)\n+{\n+\treturn !(pud_flags(pud) \u0026 _PAGE_NX);\n+}\n+\n+#define p4d_exec p4d_exec\n+static inline int p4d_exec(p4d_t p4d)\n+{\n+\treturn !(p4d_flags(p4d) \u0026 _PAGE_NX);\n+}\n+\n+#define pgd_exec pgd_exec\n+static inline int pgd_exec(pgd_t pgd)\n+{\n+\treturn !(pgd_flags(pgd) \u0026 _PAGE_NX);\n+}\n+\n static inline int pte_huge(pte_t pte)\n {\n \treturn pte_flags(pte) \u0026 _PAGE_PSE;\ndiff --git a/arch/x86/include/asm/set_memory.h b/arch/x86/include/asm/set_memory.h\nindex 4362c26aa992db..332ec300c2ed9c 100644\n--- a/arch/x86/include/asm/set_memory.h\n+++ b/arch/x86/include/asm/set_memory.h\n@@ -3,12 +3,62 @@\n #define _ASM_X86_SET_MEMORY_H\n \n #include \u003casm/page.h\u003e\n+#include \u003casm/sections.h\u003e\n #include \u003casm-generic/set_memory.h\u003e\n #include \u003casm/pgtable.h\u003e\n \n #define set_memory_rox set_memory_rox\n int set_memory_rox(unsigned long addr, int numpages);\n \n+#ifdef CONFIG_X86_64\n+/\n+ * The kernel image is mapped into two places in the virtual address space\n+ * (addresses without KASLR, of course):\n+ \n+ 1. The kernel direct map (0xffff880000000000)\n+ * 2. The \"high kernel map\" (0xffffffff81000000)\n+ \n+ We actually execute out of #2. If we get the address of a kernel symbol, it\n+ * points to #2, but almost all physical-to-virtual translations point to #1.\n+ \n+ This is so that we can have both a directmap of all physical memory and\n+ * take full advantage of the limited (s32) immediate addressing range (2G)\n+ * of x86_64.\n+ \n+ See Documentation/arch/x86/x86_64/mm.rst for more detail.\n+ /\n+static inline bool pfn_is_kernel(unsigned long pfn)\n+{\n+\tunsigned long spfn = __pa_symbol(_text) \u003e\u003e PAGE_SHIFT;\n+\t/ Do not reference a physical address outside the kernel. /\n+\tunsigned long epfn = __pa_symbol(roundup(_brk_end, PMD_SIZE) - 1) \u003e\u003e PAGE_SHIFT;\n+\n+\treturn pfn \u003e= spfn \u0026\u0026 pfn \u003c= epfn;\n+}\n+#else\n+static inline bool pfn_is_kernel(unsigned long pfn)\n+{\n+\t/ There is no highmap on 32-bit /\n+\treturn false;\n+}\n+#endif\n+\n+static inline bool cpa_should_update_alias(unsigned long vaddr,\n+\t\t\t\t\t unsigned long pfn)\n+{\n+\t/ Primary is not in the direct map, its direct map alias needs update /\n+\tif (vaddr \u003c PAGE_OFFSET \|\|\n+\t vaddr \u003e= PAGE_OFFSET + (max_pfn_mapped \u003c\u003c PAGE_SHIFT))\n+\t\treturn true;\n+\n+\t/ direct map page that is also part of the kernel highmap /\n+\tif ((vaddr \u003c (unsigned long)_text \|\| vaddr \u003e= _brk_end) \u0026\u0026\n+\t pfn_is_kernel(pfn))\n+\t\treturn true;\n+\n+\treturn false;\n+}\n+\n /\n * The set_memory_* API can be used to change various attributes of a virtual\n * address range. The attributes include:\ndiff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c\nindex d023a40a1e0345..c460121c96b949 100644\n--- a/arch/x86/mm/pat/set_memory.c\n+++ b/arch/x86/mm/pat/set_memory.c\n@@ -38,21 +38,6 @@\n /\n The current flushing context - we pass it instead of 5 arguments:\n /\n-struct cpa_data {\n-\tunsigned long\tvaddr;\n-\tpgd_t\t\tpgd;\n-\tpgprot_t\tmask_set;\n-\tpgprot_t\tmask_clr;\n-\tunsigned long\tnumpages;\n-\tunsigned long\tcurpage;\n-\tunsigned long\tpfn;\n-\tunsigned int\tflags;\n-\tunsigned int\tforce_split\t\t: 1,\n-\t\t\tforce_static_prot\t: 1,\n-\t\t\tforce_flush_all\t\t: 1;\n-\tstruct page\tpages;\n-};\n-\n enum cpa_warn {\n \tCPA_CONFLICT,\n \tCPA_PROTECT,\n@@ -61,27 +46,13 @@ enum cpa_warn {\n \n static const int cpa_warn_level = CPA_PROTECT;\n \n-/\n- * Serialize cpa() (for !DEBUG_PAGEALLOC which uses large identity mappings)\n- * using cpa_lock. So that we don't allow any other cpu, with stale large tlb\n- * entries change the page attribute in parallel to some other cpu\n- * splitting a large page entry along with changing the attribute.\n- /\n-static DEFINE_SPINLOCK(cpa_lock);\n-\n-#define CPA_FLUSHTLB 1\n-#define CPA_ARRAY 2\n-#define CPA_PAGES_ARRAY 4\n-#define CPA_NO_CHECK_ALIAS 8 / Do not search for aliases /\n-#define CPA_COLLAPSE 16 / try to collapse large pages /\n-\n static inline pgprot_t cachemode2pgprot(enum page_cache_mode pcm)\n {\n \treturn __pgprot(cachemode2protval(pcm));\n }\n \n #ifdef CONFIG_PROC_FS\n-static unsigned long direct_pages_count[PG_LEVEL_NUM];\n+static unsigned long direct_pages_count[PGTABLE_LEVEL_NUM];\n \n void update_page_count(int level, unsigned long pages)\n {\n@@ -98,9 +69,9 @@ static void split_page_count(int level)\n \n \tdirect_pages_count[level]--;\n \tif (system_state == SYSTEM_RUNNING) {\n-\t\tif (level == PG_LEVEL_2M)\n+\t\tif (level == PGTABLE_LEVEL_PMD)\n \t\t\tcount_vm_event(DIRECT_MAP_LEVEL2_SPLIT);\n-\t\telse if (level == PG_LEVEL_1G)\n+\t\telse if (level == PGTABLE_LEVEL_PUD)\n \t\t\tcount_vm_event(DIRECT_MAP_LEVEL3_SPLIT);\n \t}\n \tdirect_pages_count[level - 1] += PTRS_PER_PTE;\n@@ -110,9 +81,9 @@ static void collapse_page_count(int level)\n {\n \tdirect_pages_count[level]++;\n \tif (system_state == SYSTEM_RUNNING) {\n-\t\tif (level == PG_LEVEL_2M)\n+\t\tif (level == PGTABLE_LEVEL_PMD)\n \t\t\tcount_vm_event(DIRECT_MAP_LEVEL2_COLLAPSE);\n-\t\telse if (level == PG_LEVEL_1G)\n+\t\telse if (level == PGTABLE_LEVEL_PUD)\n \t\t\tcount_vm_event(DIRECT_MAP_LEVEL3_COLLAPSE);\n \t}\n \tdirect_pages_count[level - 1] -= PTRS_PER_PTE;\n@@ -121,17 +92,17 @@ static void collapse_page_count(int level)\n void arch_report_meminfo(struct seq_file m)\n {\n \tseq_printf(m, \"DirectMap4k: %8lu kB\\n\",\n-\t\t\tdirect_pages_count[PG_LEVEL_4K] \u003c\u003c 2);\n+\t\t\tdirect_pages_count[PGTABLE_LEVEL_PTE] \u003c\u003c 2);\n #if defined(CONFIG_X86_64) \|\| defined(CONFIG_X86_PAE)\n \tseq_printf(m, \"DirectMap2M: %8lu kB\\n\",\n-\t\t\tdirect_pages_count[PG_LEVEL_2M] \u003c\u003c 11);\n+\t\t\tdirect_pages_count[PGTABLE_LEVEL_PMD] \u003c\u003c 11);\n #else\n \tseq_printf(m, \"DirectMap4M: %8lu kB\\n\",\n-\t\t\tdirect_pages_count[PG_LEVEL_2M] \u003c\u003c 12);\n+\t\t\tdirect_pages_count[PGTABLE_LEVEL_PMD] \u003c\u003c 12);\n #endif\n \tif (direct_gbpages)\n \t\tseq_printf(m, \"DirectMap1G: %8lu kB\\n\",\n-\t\t\tdirect_pages_count[PG_LEVEL_1G] \u003c\u003c 20);\n+\t\t\tdirect_pages_count[PGTABLE_LEVEL_PUD] \u003c\u003c 20);\n }\n #else\n static inline void split_page_count(int level) { }\n@@ -165,7 +136,7 @@ static inline void cpa_inc_4k_install(void)\n \n static inline void cpa_inc_lp_sameprot(int level)\n {\n-\tif (level == PG_LEVEL_1G)\n+\tif (level == PGTABLE_LEVEL_PUD)\n \t\tcpa_1g_sameprot++;\n \telse\n \t\tcpa_2m_sameprot++;\n@@ -173,7 +144,7 @@ static inline void cpa_inc_lp_sameprot(int level)\n \n static inline void cpa_inc_lp_preserved(int level)\n {\n-\tif (level == PG_LEVEL_1G)\n+\tif (level == PGTABLE_LEVEL_PUD)\n \t\tcpa_1g_preserved++;\n \telse\n \t\tcpa_2m_preserved++;\n@@ -225,61 +196,6 @@ within(unsigned long addr, unsigned long start, unsigned long end)\n \treturn addr \u003e= start \u0026\u0026 addr \u003c end;\n }\n \n-#ifdef CONFIG_X86_64\n-\n-static inline int\n-within_inclusive(unsigned long addr, unsigned long start, unsigned long end)\n-{\n-\treturn addr \u003e= start \u0026\u0026 addr \u003c= end;\n-}\n-\n-/\n- The kernel image is mapped into two places in the virtual address space\n- * (addresses without KASLR, of course):\n- \n- 1. The kernel direct map (0xffff880000000000)\n- * 2. The \"high kernel map\" (0xffffffff81000000)\n- \n- We actually execute out of #2. If we get the address of a kernel symbol, it\n- * points to #2, but almost all physical-to-virtual translations point to #1.\n- \n- This is so that we can have both a directmap of all physical memory and\n- * take full advantage of the limited (s32) immediate addressing range (2G)\n- * of x86_64.\n- \n- See Documentation/arch/x86/x86_64/mm.rst for more detail.\n- /\n-\n-static inline unsigned long highmap_start_pfn(void)\n-{\n-\treturn __pa_symbol(_text) \u003e\u003e PAGE_SHIFT;\n-}\n-\n-static inline unsigned long highmap_end_pfn(void)\n-{\n-\t/ Do not reference physical address outside the kernel. /\n-\treturn __pa_symbol(roundup(_brk_end, PMD_SIZE) - 1) \u003e\u003e PAGE_SHIFT;\n-}\n-\n-static bool __cpa_pfn_in_highmap(unsigned long pfn)\n-{\n-\t/\n-\t * Kernel text has an alias mapping at a high address, known\n-\t * here as \"highmap\".\n-\t /\n-\treturn within_inclusive(pfn, highmap_start_pfn(), highmap_end_pfn());\n-}\n-\n-#else\n-\n-static bool __cpa_pfn_in_highmap(unsigned long pfn)\n-{\n-\t/ There is no highmap on 32-bit /\n-\treturn false;\n-}\n-\n-#endif\n-\n /\n * See set_mce_nospec().\n \n@@ -303,23 +219,6 @@ static inline unsigned long fix_addr(unsigned long addr)\n #endif\n }\n \n-static unsigned long __cpa_addr(struct cpa_data cpa, unsigned long idx)\n-{\n-\tif (cpa-\u003eflags \u0026 CPA_PAGES_ARRAY) {\n-\t\tstruct page page = cpa-\u003epages[idx];\n-\n-\t\tif (unlikely(PageHighMem(page)))\n-\t\t\treturn 0;\n-\n-\t\treturn (unsigned long)page_address(page);\n-\t}\n-\n-\tif (cpa-\u003eflags \u0026 CPA_ARRAY)\n-\t\treturn cpa-\u003evaddr[idx];\n-\n-\treturn cpa-\u003evaddr + idx * PAGE_SIZE;\n-}\n-\n /\n Flushing functions\n /\n@@ -405,7 +304,7 @@ static void __cpa_flush_tlb(void data)\n \tunsigned int i;\n \n \tfor (i = 0; i \u003c cpa-\u003enumpages; i++)\n-\t\tflush_tlb_one_kernel(fix_addr(__cpa_addr(cpa, i)));\n+\t\tflush_tlb_one_kernel(fix_addr(cpa_addr(cpa, i)));\n }\n \n static int collapse_large_pages(unsigned long addr, struct list_head pgtables);\n@@ -420,10 +319,10 @@ static void cpa_collapse_large_pages(struct cpa_data cpa)\n \n \tif (cpa-\u003eflags \u0026 (CPA_PAGES_ARRAY \| CPA_ARRAY)) {\n \t\tfor (i = 0; i \u003c cpa-\u003enumpages; i++)\n-\t\t\tcollapsed += collapse_large_pages(__cpa_addr(cpa, i),\n+\t\t\tcollapsed += collapse_large_pages(cpa_addr(cpa, i),\n \t\t\t\t\t\t\t \u0026pgtables);\n \t} else {\n-\t\taddr = __cpa_addr(cpa, 0);\n+\t\taddr = cpa_addr(cpa, 0);\n \t\tstart = addr \u0026 PMD_MASK;\n \t\tend = addr + PAGE_SIZE * cpa-\u003enumpages;\n \n@@ -463,7 +362,7 @@ static void cpa_flush(struct cpa_data cpa, int cache)\n \n \tmb();\n \tfor (i = 0; i \u003c cpa-\u003enumpages; i++) {\n-\t\tunsigned long addr = __cpa_addr(cpa, i);\n+\t\tunsigned long addr = cpa_addr(cpa, i);\n \t\tunsigned int level;\n \n \t\tpte_t pte = lookup_address(addr, \u0026level);\n@@ -578,7 +477,7 @@ static pgprotval_t protect_kernel_text_ro(unsigned long start,\n \t * so the protections for kernel text and identity mappings have to\n \t * be the same.\n \t /\n-\tif (lookup_address(start, \u0026level) \u0026\u0026 (level != PG_LEVEL_4K))\n+\tif (lookup_address(start, \u0026level) \u0026\u0026 (level != PGTABLE_LEVEL_PTE))\n \t\treturn _PAGE_RW;\n \treturn 0;\n }\n@@ -709,104 +608,6 @@ static inline pgprot_t verify_rwx(pgprot_t old, pgprot_t new, unsigned long star\n \treturn new;\n }\n \n-/\n- * Lookup the page table entry for a virtual address in a specific pgd.\n- * Return a pointer to the entry (or NULL if the entry does not exist),\n- * the level of the entry, and the effective NX and RW bits of all\n- * page table levels.\n- /\n-pte_t lookup_address_in_pgd_attr(pgd_t pgd, unsigned long address,\n-\t\t\t\t unsigned int level, bool nx, bool rw)\n-{\n-\tp4d_t p4d;\n-\tpud_t pud;\n-\tpmd_t pmd;\n-\n-\tlevel = PG_LEVEL_256T;\n-\tnx = false;\n-\trw = true;\n-\n-\tif (pgd_none(pgd))\n-\t\treturn NULL;\n-\n-\tlevel = PG_LEVEL_512G;\n-\tnx \|= pgd_flags(pgd) \u0026 _PAGE_NX;\n-\trw \u0026= pgd_flags(pgd) \u0026 _PAGE_RW;\n-\n-\tp4d = p4d_offset(pgd, address);\n-\tif (p4d_none(p4d))\n-\t\treturn NULL;\n-\n-\tif (p4d_leaf(p4d) \|\| !p4d_present(p4d))\n-\t\treturn (pte_t )p4d;\n-\n-\tlevel = PG_LEVEL_1G;\n-\tnx \|= p4d_flags(p4d) \u0026 _PAGE_NX;\n-\trw \u0026= p4d_flags(p4d) \u0026 _PAGE_RW;\n-\n-\tpud = pud_offset(p4d, address);\n-\tif (pud_none(pud))\n-\t\treturn NULL;\n-\n-\tif (pud_leaf(pud) \|\| !pud_present(pud))\n-\t\treturn (pte_t )pud;\n-\n-\tlevel = PG_LEVEL_2M;\n-\tnx \|= pud_flags(pud) \u0026 _PAGE_NX;\n-\trw \u0026= pud_flags(pud) \u0026 _PAGE_RW;\n-\n-\tpmd = pmd_offset(pud, address);\n-\tif (pmd_none(pmd))\n-\t\treturn NULL;\n-\n-\tif (pmd_leaf(pmd) \|\| !pmd_present(pmd))\n-\t\treturn (pte_t )pmd;\n-\n-\tlevel = PG_LEVEL_4K;\n-\tnx \|= pmd_flags(pmd) \u0026 _PAGE_NX;\n-\trw \u0026= pmd_flags(pmd) \u0026 _PAGE_RW;\n-\n-\treturn pte_offset_kernel(pmd, address);\n-}\n-\n-/\n- * Lookup the page table entry for a virtual address in a specific pgd.\n- * Return a pointer to the entry and the level of the mapping.\n- /\n-pte_t lookup_address_in_pgd(pgd_t pgd, unsigned long address,\n-\t\t\t unsigned int level)\n-{\n-\tbool nx, rw;\n-\n-\treturn lookup_address_in_pgd_attr(pgd, address, level, \u0026nx, \u0026rw);\n-}\n-\n-/\n- Lookup the page table entry for a virtual address. Return a pointer\n- * to the entry and the level of the mapping.\n- \n- Note: the function returns p4d, pud or pmd either when the entry is marked\n- * large or when the present bit is not set. Otherwise it returns NULL.\n- /\n-pte_t lookup_address(unsigned long address, unsigned int level)\n-{\n-\treturn lookup_address_in_pgd(pgd_offset_k(address), address, level);\n-}\n-EXPORT_SYMBOL_GPL(lookup_address);\n-\n-static pte_t _lookup_address_cpa(struct cpa_data cpa, unsigned long address,\n-\t\t\t\t unsigned int level, bool nx, bool rw)\n-{\n-\tpgd_t pgd;\n-\n-\tif (!cpa-\u003epgd)\n-\t\tpgd = pgd_offset_k(address);\n-\telse\n-\t\tpgd = cpa-\u003epgd + pgd_index(address);\n-\n-\treturn lookup_address_in_pgd_attr(pgd, address, level, nx, rw);\n-}\n-\n /\n * Lookup the PMD entry for a virtual address. Return a pointer to the entry\n * or NULL if not present.\n@@ -850,9 +651,9 @@ pmd_t lookup_pmd_address(unsigned long address)\n phys_addr_t slow_virt_to_phys(void __virt_addr)\n {\n \tunsigned long virt_addr = (unsigned long)__virt_addr;\n+\tenum pgtable_level level;\n \tphys_addr_t phys_addr;\n \tunsigned long offset;\n-\tenum pg_level level;\n \tpte_t pte;\n \n \tpte = lookup_address(virt_addr, \u0026level);\n@@ -864,11 +665,11 @@ phys_addr_t slow_virt_to_phys(void __virt_addr)\n \t * make 32-PAE kernel work correctly.\n \t /\n \tswitch (level) {\n-\tcase PG_LEVEL_1G:\n+\tcase PGTABLE_LEVEL_PUD:\n \t\tphys_addr = (phys_addr_t)pud_pfn((pud_t )pte) \u003c\u003c PAGE_SHIFT;\n \t\toffset = virt_addr \u0026 ~PUD_MASK;\n \t\tbreak;\n-\tcase PG_LEVEL_2M:\n+\tcase PGTABLE_LEVEL_PMD:\n \t\tphys_addr = (phys_addr_t)pmd_pfn((pmd_t )pte) \u003c\u003c PAGE_SHIFT;\n \t\toffset = virt_addr \u0026 ~PMD_MASK;\n \t\tbreak;\n@@ -925,30 +726,27 @@ static pgprot_t pgprot_clear_protnone_bits(pgprot_t prot)\n \treturn prot;\n }\n \n-static int __should_split_large_page(pte_t kpte, unsigned long address,\n-\t\t\t\t struct cpa_data cpa)\n+int arch_should_split_large_page(struct cpa_data cpa,\n+\t\t\t\t struct cpa_split_data sd)\n {\n-\tunsigned long numpages, pmask, psize, lpaddr, pfn, old_pfn;\n \tpgprot_t old_prot, new_prot, req_prot, chk_prot;\n-\tpte_t new_pte, tmp;\n-\tenum pg_level level;\n-\tbool nx, rw;\n-\n-\t/\n-\t Check for races, another CPU might have split this page\n-\t * up already:\n-\t /\n-\ttmp = _lookup_address_cpa(cpa, address, \u0026level, \u0026nx, \u0026rw);\n-\tif (tmp != kpte)\n-\t\treturn 1;\n+\tunsigned long numpages, lpaddr, pfn, old_pfn;\n+\tenum pgtable_level level = sd-\u003elevel;\n+\tunsigned long address = sd-\u003eaddress;\n+\tunsigned long psize = sd-\u003epsize;\n+\tunsigned long pmask = sd-\u003epmask;\n+\tpte_t kpte = sd-\u003ekpte;\n+\tbool nx = sd-\u003enx;\n+\tbool rw = sd-\u003erw;\n+\tpte_t new_pte;\n \n \tswitch (level) {\n-\tcase PG_LEVEL_2M:\n+\tcase PGTABLE_LEVEL_PMD:\n \t\told_prot = pmd_pgprot((pmd_t )kpte);\n \t\told_pfn = pmd_pfn((pmd_t )kpte);\n \t\tcpa_inc_2m_checked();\n \t\tbreak;\n-\tcase PG_LEVEL_1G:\n+\tcase PGTABLE_LEVEL_PUD:\n \t\told_prot = pud_pgprot((pud_t )kpte);\n \t\told_pfn = pud_pfn((pud_t )kpte);\n \t\tcpa_inc_1g_checked();\n@@ -957,18 +755,6 @@ static int __should_split_large_page(pte_t kpte, unsigned long address,\n \t\treturn -EINVAL;\n \t}\n \n-\tpsize = page_level_size(level);\n-\tpmask = page_level_mask(level);\n-\n-\t/\n-\t * Calculate the number of pages, which fit into this large\n-\t * page starting at address:\n-\t /\n-\tlpaddr = (address + psize) \u0026 pmask;\n-\tnumpages = (lpaddr - address) \u003e\u003e PAGE_SHIFT;\n-\tif (numpages \u003c cpa-\u003enumpages)\n-\t\tcpa-\u003enumpages = numpages;\n-\n \t/\n \t * We are safe now. Check whether the new pgprot is the same:\n \t * Convert protection attributes to 4k-format, as cpa-\u003emask* are set\n@@ -1072,21 +858,6 @@ static int __should_split_large_page(pte_t kpte, unsigned long address,\n \treturn 0;\n }\n \n-static int should_split_large_page(pte_t kpte, unsigned long address,\n-\t\t\t\t struct cpa_data cpa)\n-{\n-\tint do_split;\n-\n-\tif (cpa-\u003eforce_split)\n-\t\treturn 1;\n-\n-\tspin_lock(\u0026pgd_lock);\n-\tdo_split = __should_split_large_page(kpte, address, cpa);\n-\tspin_unlock(\u0026pgd_lock);\n-\n-\treturn do_split;\n-}\n-\n static void split_set_pte(struct cpa_data cpa, pte_t pte, unsigned long pfn,\n \t\t\t pgprot_t ref_prot, unsigned long address,\n \t\t\t unsigned long size)\n@@ -1123,33 +894,22 @@ static void split_set_pte(struct cpa_data cpa, pte_t pte, unsigned long pfn,\n \tset_pte(pte, pfn_pte(pfn, ref_prot));\n }\n \n-static int\n-__split_large_page(struct cpa_data cpa, pte_t kpte, unsigned long address,\n-\t\t struct ptdesc ptdesc)\n+int arch_split_large_page(struct cpa_data cpa, struct cpa_split_data sd)\n {\n \tunsigned long lpaddr, lpinc, ref_pfn, pfn, pfninc = 1;\n+\tstruct ptdesc ptdesc = sd-\u003eptdesc;\n \tstruct page base = ptdesc_page(ptdesc);\n \tpte_t pbase = (pte_t )page_address(base);\n-\tunsigned int i, level;\n+\tenum pgtable_level level = sd-\u003elevel;\n+\tunsigned long address = sd-\u003eaddress;\n+\tpte_t kpte = sd-\u003ekpte;\n+\tunsigned int i;\n \tpgprot_t ref_prot;\n-\tbool nx, rw;\n-\tpte_t tmp;\n-\n-\tspin_lock(\u0026pgd_lock);\n-\t/\n-\t Check for races, another CPU might have split this page\n-\t * up for us already:\n-\t /\n-\ttmp = _lookup_address_cpa(cpa, address, \u0026level, \u0026nx, \u0026rw);\n-\tif (tmp != kpte) {\n-\t\tspin_unlock(\u0026pgd_lock);\n-\t\treturn 1;\n-\t}\n \n \tparavirt_alloc_pte(\u0026init_mm, page_to_pfn(base));\n \n \tswitch (level) {\n-\tcase PG_LEVEL_2M:\n+\tcase PGTABLE_LEVEL_PMD:\n \t\tref_prot = pmd_pgprot((pmd_t )kpte);\n \t\t/\n \t\t * Clear PSE (aka _PAGE_PAT) and move\n@@ -1161,7 +921,7 @@ __split_large_page(struct cpa_data cpa, pte_t kpte, unsigned long address,\n \t\tlpinc = PAGE_SIZE;\n \t\tbreak;\n \n-\tcase PG_LEVEL_1G:\n+\tcase PGTABLE_LEVEL_PUD:\n \t\tref_prot = pud_pgprot((pud_t )kpte);\n \t\tref_pfn = pud_pfn((pud_t )kpte);\n \t\tpfninc = PMD_SIZE \u003e\u003e PAGE_SHIFT;\n@@ -1177,7 +937,6 @@ __split_large_page(struct cpa_data cpa, pte_t kpte, unsigned long address,\n \t\tbreak;\n \n \tdefault:\n-\t\tspin_unlock(\u0026pgd_lock);\n \t\treturn 1;\n \t}\n \n@@ -1225,26 +984,6 @@ __split_large_page(struct cpa_data cpa, pte_t kpte, unsigned long address,\n \t * just split large page entry.\n \t /\n \tflush_tlb_all();\n-\tspin_unlock(\u0026pgd_lock);\n-\n-\treturn 0;\n-}\n-\n-static int split_large_page(struct cpa_data cpa, pte_t kpte,\n-\t\t\t unsigned long address)\n-{\n-\tstruct ptdesc ptdesc;\n-\n-\tif (!debug_pagealloc_enabled())\n-\t\tspin_unlock(\u0026cpa_lock);\n-\tptdesc = pagetable_alloc(GFP_KERNEL, 0);\n-\tif (!debug_pagealloc_enabled())\n-\t\tspin_lock(\u0026cpa_lock);\n-\tif (!ptdesc)\n-\t\treturn -ENOMEM;\n-\n-\tif (__split_large_page(cpa, kpte, address, ptdesc))\n-\t\tpagetable_free(ptdesc);\n \n \treturn 0;\n }\n@@ -1314,7 +1053,7 @@ static int collapse_pmd_page(pmd_t pmd, unsigned long addr,\n \t}\n \n \tif (virt_addr_valid(addr) \u0026\u0026 pfn_range_is_mapped(pfn, pfn + 1))\n-\t\tcollapse_page_count(PG_LEVEL_2M);\n+\t\tcollapse_page_count(PGTABLE_LEVEL_PMD);\n \n \treturn 1;\n }\n@@ -1361,7 +1100,7 @@ static int collapse_pud_page(pud_t pud, unsigned long addr,\n \tset_pud(pud, pfn_pud(pfn, pmd_pgprot(first)));\n \n \tif (virt_addr_valid(addr) \u0026\u0026 pfn_range_is_mapped(pfn, pfn + 1))\n-\t\tcollapse_page_count(PG_LEVEL_1G);\n+\t\tcollapse_page_count(PGTABLE_LEVEL_PUD);\n \n \treturn 1;\n }\n@@ -1800,8 +1539,8 @@ static int populate_pgd(struct cpa_data cpa, unsigned long addr)\n \treturn 0;\n }\n \n-static int __cpa_process_fault(struct cpa_data cpa, unsigned long vaddr,\n-\t\t\t int primary)\n+int arch_cpa_process_fault(struct cpa_data cpa, unsigned long vaddr,\n+\t\t\t int primary)\n {\n \tif (cpa-\u003epgd) {\n \t\t/\n@@ -1833,7 +1572,7 @@ static int __cpa_process_fault(struct cpa_data cpa, unsigned long vaddr,\n \t\tcpa-\u003epfn = __pa(vaddr) \u003e\u003e PAGE_SHIFT;\n \t\treturn 0;\n \n-\t} else if (__cpa_pfn_in_highmap(cpa-\u003epfn)) {\n+\t} else if (pfn_is_kernel(cpa-\u003epfn)) {\n \t\t/ Faults in the highmap are OK, so do not warn: /\n \t\treturn -EFAULT;\n \t} else {\n@@ -1845,90 +1584,45 @@ static int __cpa_process_fault(struct cpa_data cpa, unsigned long vaddr,\n \t}\n }\n \n-static int __change_page_attr(struct cpa_data cpa, int primary)\n+void arch_change_pte(struct cpa_data cpa, unsigned long address,\n+\t\t pte_t kpte, pte_t old_pte, bool nx, bool rw)\n {\n-\tunsigned long address;\n-\tint do_split, err;\n-\tunsigned int level;\n-\tpte_t kpte, old_pte;\n-\tbool nx, rw;\n-\n-\taddress = __cpa_addr(cpa, cpa-\u003ecurpage);\n-repeat:\n-\tkpte = _lookup_address_cpa(cpa, address, \u0026level, \u0026nx, \u0026rw);\n-\tif (!kpte)\n-\t\treturn __cpa_process_fault(cpa, address, primary);\n+\tpte_t new_pte;\n+\tpgprot_t old_prot = pte_pgprot(old_pte);\n+\tpgprot_t new_prot = pte_pgprot(old_pte);\n+\tunsigned long pfn = pte_pfn(old_pte);\n \n-\told_pte = kpte;\n-\tif (pte_none(old_pte))\n-\t\treturn __cpa_process_fault(cpa, address, primary);\n+\tpgprot_val(new_prot) \u0026= ~pgprot_val(cpa-\u003emask_clr);\n+\tpgprot_val(new_prot) \|= pgprot_val(cpa-\u003emask_set);\n \n-\tif (level == PG_LEVEL_4K) {\n-\t\tpte_t new_pte;\n-\t\tpgprot_t old_prot = pte_pgprot(old_pte);\n-\t\tpgprot_t new_prot = pte_pgprot(old_pte);\n-\t\tunsigned long pfn = pte_pfn(old_pte);\n-\n-\t\tpgprot_val(new_prot) \u0026= ~pgprot_val(cpa-\u003emask_clr);\n-\t\tpgprot_val(new_prot) \|= pgprot_val(cpa-\u003emask_set);\n-\n-\t\tcpa_inc_4k_install();\n-\t\t/ Hand in lpsize = 0 to enforce the protection mechanism /\n-\t\tnew_prot = static_protections(new_prot, address, pfn, 1, 0,\n-\t\t\t\t\t CPA_PROTECT);\n-\n-\t\tnew_prot = verify_rwx(old_prot, new_prot, address, pfn, 1,\n-\t\t\t\t nx, rw);\n+\tcpa_inc_4k_install();\n+\t/ Hand in lpsize = 0 to enforce the protection mechanism /\n+\tnew_prot = static_protections(new_prot, address, pfn, 1, 0,\n+\t\t\t\t CPA_PROTECT);\n \n-\t\tnew_prot = pgprot_clear_protnone_bits(new_prot);\n+\tnew_prot = verify_rwx(old_prot, new_prot, address, pfn, 1, nx, rw);\n \n-\t\t/\n-\t\t * We need to keep the pfn from the existing PTE,\n-\t\t * after all we're only going to change its attributes\n-\t\t * not the memory it points to\n-\t\t /\n-\t\tnew_pte = pfn_pte(pfn, new_prot);\n-\t\tcpa-\u003epfn = pfn;\n-\t\t/\n-\t\t * Do we really change anything ?\n-\t\t /\n-\t\tif (pte_val(old_pte) != pte_val(new_pte)) {\n-\t\t\tset_pte_atomic(kpte, new_pte);\n-\t\t\tcpa-\u003eflags \|= CPA_FLUSHTLB;\n-\t\t}\n-\t\tcpa-\u003enumpages = 1;\n-\t\treturn 0;\n-\t}\n+\tnew_prot = pgprot_clear_protnone_bits(new_prot);\n \n \t/\n-\t * Check, whether we can keep the large page intact\n-\t * and just change the pte:\n-\t /\n-\tdo_split = should_split_large_page(kpte, address, cpa);\n-\t/\n-\t * When the range fits into the existing large page,\n-\t * return. cp-\u003enumpages and cpa-\u003etlbflush have been updated in\n-\t * try_large_page:\n+\t * We need to keep the pfn from the existing PTE, after all we're only\n+\t * going to change its attributes not the memory it points to\n \t /\n-\tif (do_split \u003c= 0)\n-\t\treturn do_split;\n-\n+\tnew_pte = pfn_pte(pfn, new_prot);\n+\tcpa-\u003epfn = pfn;\n \t/\n-\t * We have to split the large page:\n+\t * Do we really change anything ?\n \t /\n-\terr = split_large_page(cpa, kpte, address);\n-\tif (!err)\n-\t\tgoto repeat;\n-\n-\treturn err;\n+\tif (pte_val(old_pte) != pte_val(new_pte)) {\n+\t\tset_pte_atomic(kpte, new_pte);\n+\t\tcpa-\u003eflags \|= CPA_FLUSHTLB;\n+\t}\n }\n \n-static int __change_page_attr_set_clr(struct cpa_data cpa, int primary);\n-\n /\n Check the directmap and \"high kernel map\" 'aliases'.\n /\n-static int cpa_process_alias(struct cpa_data cpa)\n+int arch_cpa_process_alias(struct cpa_data cpa)\n {\n \tstruct cpa_data alias_cpa;\n \tunsigned long laddr = (unsigned long)__va(cpa-\u003epfn \u003c\u003c PAGE_SHIFT);\n@@ -1942,7 +1636,7 @@ static int cpa_process_alias(struct cpa_data cpa)\n \t * No need to redo, when the primary call touched the direct\n \t * mapping already:\n \t /\n-\tvaddr = __cpa_addr(cpa, cpa-\u003ecurpage);\n+\tvaddr = cpa_addr(cpa, cpa-\u003ecurpage);\n \tif (!(within(vaddr, PAGE_OFFSET,\n \t\t PAGE_OFFSET + (max_pfn_mapped \u003c\u003c PAGE_SHIFT)))) {\n \n@@ -1971,7 +1665,7 @@ static int cpa_process_alias(struct cpa_data cpa)\n \t * to touch the high mapped kernel as well:\n \t /\n \tif (!within(vaddr, (unsigned long)_text, _brk_end) \u0026\u0026\n-\t __cpa_pfn_in_highmap(cpa-\u003epfn)) {\n+\t pfn_is_kernel(cpa-\u003epfn)) {\n \t\tunsigned long temp_cpa_vaddr = (cpa-\u003epfn \u003c\u003c PAGE_SHIFT) +\n \t\t\t\t\t __START_KERNEL_map - phys_base;\n \t\talias_cpa = cpa;\n@@ -2000,168 +1694,25 @@ static int cpa_process_alias(struct cpa_data cpa)\n \treturn 0;\n }\n \n-static int __change_page_attr_set_clr(struct cpa_data cpa, int primary)\n-{\n-\tunsigned long numpages = cpa-\u003enumpages;\n-\tunsigned long rempages = numpages;\n-\tint ret = 0;\n-\n-\t/\n-\t No changes, easy!\n-\t /\n-\tif (!(pgprot_val(cpa-\u003emask_set) \| pgprot_val(cpa-\u003emask_clr)) \u0026\u0026\n-\t !cpa-\u003eforce_split)\n-\t\treturn ret;\n-\n-\twhile (rempages) {\n-\t\t/\n-\t\t * Store the remaining nr of pages for the large page\n-\t\t * preservation check.\n-\t\t /\n-\t\tcpa-\u003enumpages = rempages;\n-\t\t/ for array changes, we can't use large page /\n-\t\tif (cpa-\u003eflags \u0026 (CPA_ARRAY \| CPA_PAGES_ARRAY))\n-\t\t\tcpa-\u003enumpages = 1;\n-\n-\t\tif (!debug_pagealloc_enabled())\n-\t\t\tspin_lock(\u0026cpa_lock);\n-\t\tret = __change_page_attr(cpa, primary);\n-\t\tif (!debug_pagealloc_enabled())\n-\t\t\tspin_unlock(\u0026cpa_lock);\n-\t\tif (ret)\n-\t\t\tgoto out;\n-\n-\t\tif (primary \u0026\u0026 !(cpa-\u003eflags \u0026 CPA_NO_CHECK_ALIAS)) {\n-\t\t\tret = cpa_process_alias(cpa);\n-\t\t\tif (ret)\n-\t\t\t\tgoto out;\n-\t\t}\n-\n-\t\t/\n-\t\t * Adjust the number of pages with the result of the\n-\t\t * CPA operation. Either a large page has been\n-\t\t * preserved or a single page update happened.\n-\t\t /\n-\t\tBUG_ON(cpa-\u003enumpages \u003e rempages \|\| !cpa-\u003enumpages);\n-\t\trempages -= cpa-\u003enumpages;\n-\t\tcpa-\u003ecurpage += cpa-\u003enumpages;\n-\t}\n-\n-out:\n-\t/ Restore the original numpages /\n-\tcpa-\u003enumpages = numpages;\n-\treturn ret;\n-}\n-\n-static int change_page_attr_set_clr(unsigned long addr, int numpages,\n-\t\t\t\t pgprot_t mask_set, pgprot_t mask_clr,\n-\t\t\t\t int force_split, int in_flag,\n-\t\t\t\t struct page *pages)\n+void arch_cpa_flush(struct cpa_data cpa, int err)\n {\n-\tstruct cpa_data cpa;\n-\tint ret, cache;\n-\n-\tmemset(\u0026cpa, 0, sizeof(cpa));\n-\n-\t/\n-\t Check, if we are requested to set a not supported\n-\t * feature. Clearing non-supported features is OK.\n-\t /\n-\tmask_set = canon_pgprot(mask_set);\n-\n-\tif (!pgprot_val(mask_set) \u0026\u0026 !pgprot_val(mask_clr) \u0026\u0026 !force_split)\n-\t\treturn 0;\n-\n-\t/ Ensure we are PAGE_SIZE aligned /\n-\tif (in_flag \u0026 CPA_ARRAY) {\n-\t\tint i;\n-\t\tfor (i = 0; i \u003c numpages; i++) {\n-\t\t\tif (addr[i] \u0026 ~PAGE_MASK) {\n-\t\t\t\taddr[i] \u0026= PAGE_MASK;\n-\t\t\t\tWARN_ON_ONCE(1);\n-\t\t\t}\n-\t\t}\n-\t} else if (!(in_flag \u0026 CPA_PAGES_ARRAY)) {\n-\t\t/\n-\t\t * in_flag of CPA_PAGES_ARRAY implies it is aligned.\n-\t\t * No need to check in that case\n-\t\t /\n-\t\tif (addr \u0026 ~PAGE_MASK) {\n-\t\t\taddr \u0026= PAGE_MASK;\n-\t\t\t/\n-\t\t\t * People should not be passing in unaligned addresses:\n-\t\t\t /\n-\t\t\tWARN_ON_ONCE(1);\n-\t\t}\n-\t}\n-\n-\t/ Must avoid aliasing mappings in the highmem code /\n-\tkmap_flush_unused();\n-\n-\tvm_unmap_aliases();\n-\n-\tcpa.vaddr = addr;\n-\tcpa.pages = pages;\n-\tcpa.numpages = numpages;\n-\tcpa.mask_set = mask_set;\n-\tcpa.mask_clr = mask_clr;\n-\tcpa.flags = in_flag;\n-\tcpa.curpage = 0;\n-\tcpa.force_split = force_split;\n-\n-\tret = __change_page_attr_set_clr(\u0026cpa, 1);\n-\n-\t/\n-\t * Check whether we really changed something:\n-\t /\n-\tif (!(cpa.flags \u0026 CPA_FLUSHTLB))\n-\t\tgoto out;\n+\tint cache;\n \n \t/\n \t * No need to flush, when we did not set any of the caching\n \t * attributes:\n \t /\n-\tcache = !!pgprot2cachemode(mask_set);\n+\tcache = !!pgprot2cachemode(cpa-\u003emask_set);\n \n \t/\n \t * On error; flush everything to be sure.\n \t /\n-\tif (ret) {\n+\tif (err) {\n \t\tcpa_flush_all(cache);\n-\t\tgoto out;\n+\t\treturn;\n \t}\n \n-\tcpa_flush(\u0026cpa, cache);\n-out:\n-\treturn ret;\n-}\n-\n-static inline int change_page_attr_set(unsigned long addr, int numpages,\n-\t\t\t\t pgprot_t mask, int array)\n-{\n-\treturn change_page_attr_set_clr(addr, numpages, mask, __pgprot(0), 0,\n-\t\t(array ? CPA_ARRAY : 0), NULL);\n-}\n-\n-static inline int change_page_attr_clear(unsigned long addr, int numpages,\n-\t\t\t\t\t pgprot_t mask, int array)\n-{\n-\treturn change_page_attr_set_clr(addr, numpages, __pgprot(0), mask, 0,\n-\t\t(array ? CPA_ARRAY : 0), NULL);\n-}\n-\n-static inline int cpa_set_pages_array(struct page pages, int numpages,\n-\t\t\t\t pgprot_t mask)\n-{\n-\treturn change_page_attr_set_clr(NULL, numpages, mask, __pgprot(0), 0,\n-\t\tCPA_PAGES_ARRAY, pages);\n-}\n-\n-static inline int cpa_clear_pages_array(struct page pages, int numpages,\n-\t\t\t\t\t pgprot_t mask)\n-{\n-\treturn change_page_attr_set_clr(NULL, numpages, __pgprot(0), mask, 0,\n-\t\tCPA_PAGES_ARRAY, pages);\n+\tcpa_flush(cpa, cache);\n }\n \n int _set_memory_uc(unsigned long addr, int numpages)\n@@ -2773,11 +2324,3 @@ int __init kernel_unmap_pages_in_pgd(pgd_t pgd, unsigned long address,\n \n \treturn retval;\n }\n-\n-/\n- The testcases use internal knowledge of the implementation that shouldn't\n- * be exposed to the rest of the kernel. Include these directly here.\n- /\n-#ifdef CONFIG_CPA_DEBUG\n-#include \"cpa-test.c\"\n-#endif\ndiff --git a/include/linux/pgtable.h b/include/linux/pgtable.h\nindex 2981e386da7b3c..d1df8345852be8 100644\n--- a/include/linux/pgtable.h\n+++ b/include/linux/pgtable.h\n@@ -1782,6 +1782,10 @@ void arch_sync_kernel_mappings(unsigned long start, unsigned long end);\n because these macros can be used even if CONFIG_MMU is not defined.\n /\n \n+#ifndef canon_pgprot\n+#define canon_pgprot(prot)\t(prot)\n+#endif\n+\n #ifndef pgprot_nx\n #define pgprot_nx(prot)\t(prot)\n #endif\n@@ -2105,6 +2109,46 @@ static inline int pud_write(pud_t pud)\n }\n #endif / pud_write /\n \n+#ifndef p4d_write\n+static inline int p4d_write(p4d_t p4d)\n+{\n+\tBUG();\n+\treturn 0;\n+}\n+#endif / p4d_write /\n+\n+#ifndef pmd_exec\n+static inline int pmd_exec(pmd_t pmd)\n+{\n+\tBUG();\n+\treturn 0;\n+}\n+#endif / pmd_exec /\n+\n+#ifndef pud_exec\n+static inline int pud_exec(pud_t pud)\n+{\n+\tBUG();\n+\treturn 0;\n+}\n+#endif / pud_exec /\n+\n+#ifndef p4d_exec\n+static inline int p4d_exec(p4d_t p4d)\n+{\n+\tBUG();\n+\treturn 0;\n+}\n+#endif / p4d_exec /\n+\n+#ifndef pgd_exec\n+static inline int pgd_exec(pgd_t pgd)\n+{\n+\tBUG();\n+\treturn 0;\n+}\n+#endif / pgd_exec /\n+\n #if !defined(CONFIG_TRANSPARENT_HUGEPAGE) \|\| \\\n \t!defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)\n static inline int pud_trans_huge(pud_t pud)\n@@ -2295,16 +2339,20 @@ static inline bool arch_has_pfn_modify_check(void)\n typedef unsigned int pgtbl_mod_mask;\n \n enum pgtable_level {\n-\tPGTABLE_LEVEL_PTE = 0,\n+\tPGTABLE_LEVEL_NONE = 0,\n+\tPGTABLE_LEVEL_PTE,\n \tPGTABLE_LEVEL_PMD,\n \tPGTABLE_LEVEL_PUD,\n \tPGTABLE_LEVEL_P4D,\n \tPGTABLE_LEVEL_PGD,\n+\tPGTABLE_LEVEL_NUM,\n };\n \n static inline const char pgtable_level_to_str(enum pgtable_level level)\n {\n \tswitch (level) {\n+\tcase PGTABLE_LEVEL_NONE:\n+\t\treturn \"none\";\n \tcase PGTABLE_LEVEL_PTE:\n \t\treturn \"pte\";\n \tcase PGTABLE_LEVEL_PMD:\n@@ -2320,6 +2368,28 @@ static inline const char pgtable_level_to_str(enum pgtable_level level)\n \t}\n }\n \n+/ Size of the memory mapped by a single leaf entry at @level /\n+static inline unsigned long pgtable_level_size(enum pgtable_level level)\n+{\n+\tswitch (level) {\n+\tcase PGTABLE_LEVEL_PTE:\n+\t\treturn PAGE_SIZE;\n+\tcase PGTABLE_LEVEL_PMD:\n+\t\treturn PMD_SIZE;\n+\tcase PGTABLE_LEVEL_PUD:\n+\t\treturn PUD_SIZE;\n+\tcase PGTABLE_LEVEL_P4D:\n+\t\treturn P4D_SIZE;\n+\tdefault:\n+\t\treturn PGDIR_SIZE;\n+\t}\n+}\n+\n+static inline unsigned long pgtable_level_mask(enum pgtable_level level)\n+{\n+\treturn ~(pgtable_level_size(level) - 1);\n+}\n+\n #endif / !__ASSEMBLY__ /\n \n #if !defined(MAX_POSSIBLE_PHYSMEM_BITS) \u0026\u0026 !defined(CONFIG_64BIT)\ndiff --git a/include/linux/set_memory.h b/include/linux/set_memory.h\nindex 3030d9245f5ac8..17cbe6e26cb732 100644\n--- a/include/linux/set_memory.h\n+++ b/include/linux/set_memory.h\n@@ -84,4 +84,101 @@ static inline int set_memory_decrypted(unsigned long addr, int numpages)\n }\n #endif / CONFIG_ARCH_HAS_MEM_ENCRYPT /\n \n+#ifdef CONFIG_GENERIC_SET_MEMORY\n+\n+#include \u003clinux/pgtable.h\u003e\n+\n+#define CPA_FLUSHTLB 1\n+#define CPA_ARRAY 2\n+#define CPA_PAGES_ARRAY 4\n+#define CPA_NO_CHECK_ALIAS 8 / Do not search for aliases /\n+#define CPA_COLLAPSE 16 / try to collapse large pages /\n+\n+/\n+ * The current flushing context - we pass it instead of 5 arguments:\n+ /\n+struct cpa_data {\n+\tunsigned long\tvaddr;\n+\tpgd_t\t\tpgd;\n+\tpgprot_t\tmask_set;\n+\tpgprot_t\tmask_clr;\n+\tunsigned long\tnumpages;\n+\tunsigned long\tcurpage;\n+\tunsigned long\tpfn;\n+\tunsigned int\tflags;\n+\tunsigned int\tforce_split\t\t: 1,\n+\t\t\tforce_static_prot\t: 1,\n+\t\t\tforce_flush_all\t\t: 1;\n+\tstruct page\tpages;\n+};\n+\n+/\n+ * context for checking for a split a large page and splitting it if required.\n+ /\n+struct ptdesc;\n+struct cpa_split_data {\n+\tunsigned long\taddress;\n+\tunsigned long\tpsize;\n+\tunsigned long\tpmask;\n+\tpte_t\t\tkpte;\n+\tstruct ptdesc\tptdesc;\n+\tenum pgtable_level level;\n+\tbool\t\tnx;\n+\tbool\t\trw;\n+};\n+\n+unsigned long cpa_addr(struct cpa_data cpa, unsigned long idx);\n+\n+pte_t lookup_address(unsigned long address, unsigned int level);\n+pte_t lookup_address_in_pgd(pgd_t pgd, unsigned long address,\n+\t\t\t unsigned int level);\n+pte_t lookup_address_in_pgd_attr(pgd_t pgd, unsigned long address,\n+\t\t\t\t unsigned int level, bool nx, bool rw);\n+\n+int arch_cpa_process_fault(struct cpa_data cpa, unsigned long vaddr,\n+\t\t\t int primary);\n+int arch_cpa_process_alias(struct cpa_data cpa);\n+int arch_should_split_large_page(struct cpa_data cpa,\n+\t\t\t\t struct cpa_split_data sd);\n+int arch_split_large_page(struct cpa_data cpa, struct cpa_split_data sd);\n+void arch_change_pte(struct cpa_data cpa, unsigned long address,\n+\t\t pte_t kpte, pte_t old_pte, bool nx, bool rw);\n+void arch_cpa_flush(struct cpa_data cpa, int err);\n+\n+int __change_page_attr_set_clr(struct cpa_data cpa, int primary);\n+int change_page_attr_set_clr(unsigned long addr, int numpages,\n+\t\t\t pgprot_t mask_set, pgprot_t mask_clr,\n+\t\t\t int force_split, int in_flag,\n+\t\t\t struct page pages);\n+\n+static inline int change_page_attr_set(unsigned long addr, int numpages,\n+\t\t\t\t pgprot_t mask, int array)\n+{\n+\treturn change_page_attr_set_clr(addr, numpages, mask, __pgprot(0), 0,\n+\t\t(array ? CPA_ARRAY : 0), NULL);\n+}\n+\n+static inline int change_page_attr_clear(unsigned long addr, int numpages,\n+\t\t\t\t\t pgprot_t mask, int array)\n+{\n+\treturn change_page_attr_set_clr(addr, numpages, __pgprot(0), mask, 0,\n+\t\t(array ? CPA_ARRAY : 0), NULL);\n+}\n+\n+static inline int cpa_set_pages_array(struct page pages, int numpages,\n+\t\t\t\t pgprot_t mask)\n+{\n+\treturn change_page_attr_set_clr(NULL, numpages, mask, __pgprot(0), 0,\n+\t\tCPA_PAGES_ARRAY, pages);\n+}\n+\n+static inline int cpa_clear_pages_array(struct page pages, int numpages,\n+\t\t\t\t\tpgprot_t mask)\n+{\n+\treturn change_page_attr_set_clr(NULL, numpages, __pgprot(0), mask, 0,\n+\t\tCPA_PAGES_ARRAY, pages);\n+}\n+\n+#endif / CONFIG_GENERIC_SET_MEMORY /\n+\n #endif / _LINUX_SET_MEMORY_H_ /\ndiff --git a/mm/Kconfig b/mm/Kconfig\nindex 9e0ca48249054f..541edd1e11b610 100644\n--- a/mm/Kconfig\n+++ b/mm/Kconfig\n@@ -1509,6 +1509,9 @@ config LAZY_MMU_MODE_KUNIT_TEST\n \n \t If unsure, say N.\n \n+config GENERIC_SET_MEMORY\n+\tbool\n+\n source \"mm/damon/Kconfig\"\n \n endmenu\ndiff --git a/mm/Kconfig.debug b/mm/Kconfig.debug\nindex 91b3e027b75357..83cfef6b2d8cfb 100644\n--- a/mm/Kconfig.debug\n+++ b/mm/Kconfig.debug\n@@ -320,3 +320,10 @@ config PER_VMA_LOCK_STATS\n \t overhead in the page fault path.\n \n \t If in doubt, say N.\n+\n+config CPA_DEBUG\n+\tbool \"CPA self-test code\"\n+\tdepends on GENERIC_SET_MEMORY\n+\tdepends on DEBUG_KERNEL\n+\thelp\n+\t Do change_page_attr() self-tests every 30 seconds.\ndiff --git a/mm/Makefile b/mm/Makefile\nindex eff9f9e7e061c1..e3f6c39c2ba991 100644\n--- a/mm/Makefile\n+++ b/mm/Makefile\n@@ -147,3 +147,4 @@ obj-$(CONFIG_SHRINKER_DEBUG) += shrinker_debug.o\n obj-$(CONFIG_EXECMEM) += execmem.o\n obj-$(CONFIG_TMPFS_QUOTA) += shmem_quota.o\n obj-$(CONFIG_LAZY_MMU_MODE_KUNIT_TEST) += tests/lazy_mmu_mode_kunit.o\n+obj-$(CONFIG_GENERIC_SET_MEMORY) += set_memory.o\ndiff --git a/arch/x86/mm/pat/cpa-test.c b/mm/cpa-test.c\nsimilarity index 80%\nrename from arch/x86/mm/pat/cpa-test.c\nrename to mm/cpa-test.c\nindex ad3c1feec990db..63c3463a5a7d85 100644\n--- a/arch/x86/mm/pat/cpa-test.c\n+++ b/mm/cpa-test.c\n@@ -13,32 +13,24 @@\n #include \u003clinux/mm.h\u003e\n #include \u003clinux/vmalloc.h\u003e\n \n-#include \u003casm/cacheflush.h\u003e\n-#include \u003casm/kdebug.h\u003e\n-\n /\n * Only print the results of the first pass:\n /\n static __read_mostly int print = 1;\n+static unsigned long max_test_pfn;\n \n enum {\n \tNTEST\t\t\t= 3 100,\n \tNPAGES\t\t\t= 100,\n-#ifdef CONFIG_X86_64\n-\tLPS\t\t\t= (1 \u003c\u003c PMD_SHIFT),\n-#elif defined(CONFIG_X86_PAE)\n-\tLPS\t\t\t= (1 \u003c\u003c PMD_SHIFT),\n-#else\n-\tLPS\t\t\t= (1 \u003c\u003c 22),\n-#endif\n-\tGPS\t\t\t= (1\u003c\u003c30)\n+\tLPS\t\t\t= PMD_SIZE,\n+\tGPS\t\t\t= PUD_SIZE\n };\n \n-#define PAGE_CPA_TEST\t__pgprot(_PAGE_CPA_TEST)\n+#define PAGE_CPA_TEST\t__pgprot(_PAGE_SPECIAL)\n \n static int pte_testbit(pte_t pte)\n {\n-\treturn pte_flags(pte) \u0026 _PAGE_SOFTW1;\n+\treturn pte_special(pte);\n }\n \n struct split_state {\n@@ -54,10 +46,10 @@ static int print_split(struct split_state s)\n \ts-\u003elpg = s-\u003egpg = s-\u003espg = s-\u003eexec = 0;\n \ts-\u003emin_exec = ~0UL;\n \ts-\u003emax_exec = 0;\n-\tfor (i = 0; i \u003c max_pfn_mapped; ) {\n+\tfor (i = 0; i \u003c max_test_pfn; ) {\n \t\tunsigned long addr = (unsigned long)__va(i \u003c\u003c PAGE_SHIFT);\n \t\tunsigned int level;\n-\t\tpte_t pte;\n+\t\tpte_t pte, pteval;\n \n \t\tpte = lookup_address(addr, \u0026level);\n \t\tif (!pte) {\n@@ -65,15 +57,16 @@ static int print_split(struct split_state s)\n \t\t\ti++;\n \t\t\tcontinue;\n \t\t}\n+\t\tpteval = ptep_get(pte);\n \n-\t\tif (level == PG_LEVEL_1G \u0026\u0026 sizeof(long) == 8) {\n+\t\tif (level == PGTABLE_LEVEL_PUD \u0026\u0026 sizeof(long) == 8) {\n \t\t\ts-\u003egpg++;\n \t\t\ti += GPS/PAGE_SIZE;\n-\t\t} else if (level == PG_LEVEL_2M) {\n-\t\t\tif ((pte_val(pte) \u0026 _PAGE_PRESENT) \u0026\u0026 !(pte_val(pte) \u0026 _PAGE_PSE)) {\n+\t\t} else if (level == PGTABLE_LEVEL_PMD) {\n+\t\t\tif (pte_present(pteval) \u0026\u0026 !pte_huge(pteval)) {\n \t\t\t\tprintk(KERN_ERR\n-\t\t\t\t\t\"%lx level %d but not PSE %Lx\\n\",\n-\t\t\t\t\taddr, level, (u64)pte_val(pte));\n+\t\t\t\t\t\"%lx level %d but not leaf %Lx\\n\",\n+\t\t\t\t\taddr, level, (u64)pte_val(pteval));\n \t\t\t\terr = 1;\n \t\t\t}\n \t\t\ts-\u003elpg++;\n@@ -82,7 +75,7 @@ static int print_split(struct split_state s)\n \t\t\ts-\u003espg++;\n \t\t\ti++;\n \t\t}\n-\t\tif (!(pte_val(pte) \u0026 _PAGE_NX)) {\n+\t\tif (pte_exec(pteval)) {\n \t\t\ts-\u003eexec++;\n \t\t\tif (addr \u003c s-\u003emin_exec)\n \t\t\t\ts-\u003emin_exec = addr;\n@@ -100,8 +93,8 @@ static int print_split(struct split_state s)\n \n \texpected = (s-\u003egpgGPS + s-\u003elpgLPS)/PAGE_SIZE + s-\u003espg + missed;\n \tif (expected != i) {\n-\t\tprintk(KERN_ERR \"CPA max_pfn_mapped %lu but expected %lu\\n\",\n-\t\t\tmax_pfn_mapped, expected);\n+\t\tprintk(KERN_ERR \"CPA max_test_pfn %lu but expected %lu\\n\",\n+\t\t\tmax_test_pfn, expected);\n \t\treturn 1;\n \t}\n \treturn err;\n@@ -118,7 +111,7 @@ static int pageattr_test(void)\n {\n \tstruct split_state sa, sb, sc;\n \tunsigned long bm;\n-\tpte_t pte, pte0;\n+\tpte_t pte, pte0, pte_val;\n \tint failed = 0;\n \tunsigned int level;\n \tint i, k;\n@@ -127,7 +120,7 @@ static int pageattr_test(void)\n \tif (print)\n \t\tprintk(KERN_INFO \"CPA self-test:\\n\");\n \n-\tbm = vzalloc((max_pfn_mapped + 7) / 8);\n+\tbm = vzalloc((max_test_pfn + 7) / 8);\n \tif (!bm) {\n \t\tprintk(KERN_ERR \"CPA Cannot vmalloc bitmap\\n\");\n \t\treturn -ENOMEM;\n@@ -136,11 +129,11 @@ static int pageattr_test(void)\n \tfailed += print_split(\u0026sa);\n \n \tfor (i = 0; i \u003c NTEST; i++) {\n-\t\tunsigned long pfn = get_random_u32_below(max_pfn_mapped);\n+\t\tunsigned long pfn = get_random_u32_below(max_test_pfn);\n \n \t\taddr[i] = (unsigned long)__va(pfn \u003c\u003c PAGE_SHIFT);\n \t\tlen[i] = get_random_u32_below(NPAGES);\n-\t\tlen[i] = min_t(unsigned long, len[i], max_pfn_mapped - pfn - 1);\n+\t\tlen[i] = min_t(unsigned long, len[i], max_test_pfn - pfn - 1);\n \n \t\tif (len[i] == 0)\n \t\t\tlen[i] = 1;\n@@ -150,15 +143,20 @@ static int pageattr_test(void)\n \n \t\tfor (k = 0; k \u003c len[i]; k++) {\n \t\t\tpte = lookup_address(addr[i] + kPAGE_SIZE, \u0026level);\n-\t\t\tif (!pte \|\| pgprot_val(pte_pgprot(pte)) == 0 \|\|\n-\t\t\t !(pte_val(pte) \u0026 _PAGE_PRESENT)) {\n+\t\t\tif (!pte) {\n+\t\t\t\taddr[i] = 0;\n+\t\t\t\tbreak;\n+\t\t\t}\n+\t\t\tpte_val = ptep_get(pte);\n+\t\t\tif (pgprot_val(pte_pgprot(pte_val)) == 0 \|\|\n+\t\t\t !pte_present(pte_val)) {\n \t\t\t\taddr[i] = 0;\n \t\t\t\tbreak;\n \t\t\t}\n \t\t\tif (k == 0) {\n-\t\t\t\tpte0 = pte;\n+\t\t\t\tpte0 = pte_val;\n \t\t\t} else {\n-\t\t\t\tif (pgprot_val(pte_pgprot(pte)) !=\n+\t\t\t\tif (pgprot_val(pte_pgprot(pte_val)) !=\n \t\t\t\t\tpgprot_val(pte_pgprot(pte0))) {\n \t\t\t\t\tlen[i] = k;\n \t\t\t\t\tbreak;\n@@ -198,12 +196,12 @@ static int pageattr_test(void)\n \t\t}\n \n \t\tpte = lookup_address(addr[i], \u0026level);\n-\t\tif (!pte \|\| !pte_testbit(pte) \|\| pte_huge(pte)) {\n+\t\tif (!pte \|\| !pte_testbit(pte) \|\| level != PGTABLE_LEVEL_PTE) {\n \t\t\tprintk(KERN_ERR \"CPA %lx: bad pte %Lx\\n\", addr[i],\n \t\t\t\tpte ? (u64)pte_val(pte) : 0ULL);\n \t\t\tfailed++;\n \t\t}\n-\t\tif (level != PG_LEVEL_4K) {\n+\t\tif (level != PGTABLE_LEVEL_PTE) {\n \t\t\tprintk(KERN_ERR \"CPA %lx: unexpected level %d\\n\",\n \t\t\t\taddr[i], level);\n \t\t\tfailed++;\n@@ -266,6 +264,8 @@ static int start_pageattr_test(void)\n {\n \tstruct task_struct p;\n \n+\tmax_test_pfn = PFN_DOWN(__pa(high_memory - 1));\n+\n \tp = kthread_create(do_pageattr_test, NULL, \"pageattr-test\");\n \tif (!IS_ERR(p))\n \t\twake_up_process(p);\ndiff --git a/mm/set_memory.c b/mm/set_memory.c\nnew file mode 100644\nindex 00000000000000..744405efcc3401\n--- /dev/null\n+++ b/mm/set_memory.c\n@@ -0,0 +1,396 @@\n+// SPDX-License-Identifier: GPL-2.0-only\n+\n+#include \u003clinux/highmem.h\u003e\n+#include \u003clinux/vmalloc.h\u003e\n+#include \u003clinux/pgtable.h\u003e\n+#include \u003clinux/export.h\u003e\n+\n+#include \u003clinux/set_memory.h\u003e\n+\n+/\n+ * Serialize cpa() so that we don't allow any other cpu, with stale large tlb\n+ * entries, to change the page attribute in parallel to some other cpu\n+ * splitting a large page entry along with changing the attribute.\n+ /\n+static DEFINE_SPINLOCK(cpa_lock);\n+\n+static inline void arch_lock(void)\n+{\n+#ifdef CONFIG_X86\n+\tspin_lock(\u0026pgd_lock);\n+#endif\n+}\n+\n+static inline void arch_unlock(void)\n+{\n+#ifdef CONFIG_X86\n+\tspin_unlock(\u0026pgd_lock);\n+#endif\n+}\n+\n+unsigned long cpa_addr(struct cpa_data cpa, unsigned long idx)\n+{\n+\tif (cpa-\u003eflags \u0026 CPA_PAGES_ARRAY) {\n+\t\tstruct page page = cpa-\u003epages[idx];\n+\n+\t\tif (unlikely(PageHighMem(page)))\n+\t\t\treturn 0;\n+\n+\t\treturn (unsigned long)page_address(page);\n+\t}\n+\n+\tif (cpa-\u003eflags \u0026 CPA_ARRAY)\n+\t\treturn cpa-\u003evaddr[idx];\n+\n+\treturn cpa-\u003evaddr + idx * PAGE_SIZE;\n+}\n+\n+/\n+ Lookup the page table entry for a virtual address in a specific pgd.\n+ * Return a pointer to the entry (or NULL if the entry does not exist),\n+ * the level of the entry, and the effective NX and RW bits of all\n+ * page table levels.\n+ /\n+pte_t lookup_address_in_pgd_attr(pgd_t pgdp, unsigned long address,\n+\t\t\t\t unsigned int level, bool nx, bool rw)\n+{\n+\tpgd_t pgd = pgdp_get(pgdp);\n+\tp4d_t p4dp, p4d;\n+\tpud_t pudp, pud;\n+\tpmd_t pmdp, pmd;\n+\n+\tlevel = PGTABLE_LEVEL_PGD;\n+\tnx = false;\n+\trw = true;\n+\n+\tif (pgd_none(pgd))\n+\t\treturn NULL;\n+\n+\tlevel = PGTABLE_LEVEL_P4D;\n+\tnx \|= !pgd_exec(pgd);\n+\trw \u0026= !!(pgd_write(pgd));\n+\n+\tp4dp = p4d_offset(pgdp, address);\n+\tp4d = p4dp_get(p4dp);\n+\tif (p4d_none(p4d))\n+\t\treturn NULL;\n+\n+\tif (p4d_leaf(p4d) \|\| !p4d_present(p4d))\n+\t\treturn (pte_t )p4dp;\n+\n+\tlevel = PGTABLE_LEVEL_PUD;\n+\tnx \|= !p4d_exec(p4d);\n+\trw \u0026= !!(p4d_write(p4d));\n+\n+\tpudp = pud_offset(p4dp, address);\n+\tpud = pudp_get(pudp);\n+\tif (pud_none(pud))\n+\t\treturn NULL;\n+\n+\tif (pud_leaf(pud) \|\| !pud_present(pud))\n+\t\treturn (pte_t )pudp;\n+\n+\tlevel = PGTABLE_LEVEL_PMD;\n+\tnx \|= !pud_exec(pud);\n+\trw \u0026= !!(pud_write(pud));\n+\n+\tpmdp = pmd_offset(pudp, address);\n+\tpmd = pmdp_get(pmdp);\n+\tif (pmd_none(pmd))\n+\t\treturn NULL;\n+\n+\tif (pmd_leaf(pmd) \|\| !pmd_present(pmd))\n+\t\treturn (pte_t )pmdp;\n+\n+\tlevel = PGTABLE_LEVEL_PTE;\n+\tnx \|= !pmd_exec(pmd);\n+\trw \u0026= !!(pmd_write(pmd));\n+\n+\treturn pte_offset_kernel(pmdp, address);\n+}\n+\n+/\n+ * Lookup the page table entry for a virtual address in a specific pgd.\n+ * Return a pointer to the entry and the level of the mapping.\n+ /\n+pte_t lookup_address_in_pgd(pgd_t pgd, unsigned long address,\n+\t\t\t unsigned int level)\n+{\n+\tbool nx, rw;\n+\n+\treturn lookup_address_in_pgd_attr(pgd, address, level, \u0026nx, \u0026rw);\n+}\n+\n+/\n+ Lookup the page table entry for a virtual address. Return a pointer\n+ * to the entry and the level of the mapping.\n+ \n+ Note: the function returns p4d, pud or pmd either when the entry is marked\n+ * large or when the present bit is not set. Otherwise it returns NULL.\n+ /\n+pte_t lookup_address(unsigned long address, unsigned int level)\n+{\n+\treturn lookup_address_in_pgd(pgd_offset_k(address), address, level);\n+}\n+EXPORT_SYMBOL_GPL(lookup_address);\n+\n+static pte_t _lookup_address_cpa(struct cpa_data cpa, unsigned long address,\n+\t\t\t unsigned int level, bool nx, bool rw)\n+{\n+\tpgd_t pgd;\n+\n+\tif (!cpa-\u003epgd)\n+\t\tpgd = pgd_offset_k(address);\n+\telse\n+\t\tpgd = cpa-\u003epgd + pgd_index(address);\n+\n+\treturn lookup_address_in_pgd_attr(pgd, address, level, nx, rw);\n+}\n+\n+static int should_split_large_page(struct cpa_data cpa,\n+\t\t\t\t struct cpa_split_data sd)\n+{\n+\tenum pgtable_level level = sd-\u003elevel;\n+\tunsigned long address = sd-\u003eaddress;\n+\tunsigned long numpages, lpaddr;\n+\n+\tif (cpa-\u003eforce_split)\n+\t\treturn 1;\n+\n+\tsd-\u003epsize = pgtable_level_size(level);\n+\tsd-\u003epmask = pgtable_level_mask(level);\n+\n+\t/\n+\t * Calculate the number of pages, which fit into this large\n+\t * page starting at address:\n+\t /\n+\tlpaddr = (address + sd-\u003epsize) \u0026 sd-\u003epmask;\n+\tnumpages = (lpaddr - address) \u003e\u003e PAGE_SHIFT;\n+\tif (numpages \u003c cpa-\u003enumpages)\n+\t\tcpa-\u003enumpages = numpages;\n+\n+\treturn arch_should_split_large_page(cpa, sd);\n+}\n+\n+static int cpa_handle_large_page(struct cpa_data cpa, pte_t kpte,\n+\t\t\t\t unsigned long address)\n+{\n+\tstruct cpa_split_data sd = {\n+\t\t.address = address,\n+\t\t.kpte = kpte,\n+\t};\n+\tint do_split, ret = 1;\n+\tpte_t tmp;\n+\n+\tspin_unlock(\u0026cpa_lock);\n+\tsd.ptdesc = pagetable_alloc(GFP_KERNEL, 0);\n+\tspin_lock(\u0026cpa_lock);\n+\tif (!sd.ptdesc)\n+\t\treturn -ENOMEM;\n+\n+\tarch_lock();\n+\n+\t/\n+\t Check for races, another CPU might have split this page\n+\t * up already:\n+\t /\n+\ttmp = _lookup_address_cpa(cpa, address, \u0026sd.level, \u0026sd.nx, \u0026sd.rw);\n+\tif (tmp != kpte)\n+\t\tgoto out_free_ptdesc;\n+\n+\t/\n+\t * Check, whether we can keep the large page intact\n+\t * and just change the pte:\n+\t /\n+\tdo_split = should_split_large_page(cpa, \u0026sd);\n+\tret = do_split;\n+\t/\n+\t * When the range fits into the existing large page, no split is\n+\t * required.\n+\t * should_split_large_page() updated the large page attributes and\n+\t * cpa-\u003enumpages and cpa-\u003epfn.\n+\t /\n+\tif (do_split \u003c= 0)\n+\t\tgoto out_free_ptdesc;\n+\n+\t/\n+\t * We have to split the large page:\n+\t /\n+\tret = arch_split_large_page(cpa, \u0026sd);\n+\tif (ret)\n+\t\tgoto out_free_ptdesc;\n+\n+\tarch_unlock();\n+\treturn do_split;\n+\n+out_free_ptdesc:\n+\tarch_unlock();\n+\tpagetable_free(sd.ptdesc);\n+\treturn ret;\n+}\n+\n+static int __change_page_attr(struct cpa_data cpa, int primary)\n+{\n+\tunsigned long address;\n+\tint split_res;\n+\tunsigned int level;\n+\tpte_t kpte, old_pte;\n+\tbool nx, rw;\n+\n+\taddress = cpa_addr(cpa, cpa-\u003ecurpage);\n+repeat:\n+\tkpte = _lookup_address_cpa(cpa, address, \u0026level, \u0026nx, \u0026rw);\n+\tif (!kpte)\n+\t\treturn arch_cpa_process_fault(cpa, address, primary);\n+\n+\told_pte = kpte;\n+\tif (pte_none(old_pte))\n+\t\treturn arch_cpa_process_fault(cpa, address, primary);\n+\n+\tif (level == PGTABLE_LEVEL_PTE) {\n+\t\tarch_change_pte(cpa, address, kpte, old_pte, nx, rw);\n+\t\tcpa-\u003enumpages = 1;\n+\t\treturn 0;\n+\t}\n+\n+\t/\n+\t The address maps a large page. Try to keep it, otherwise split it\n+\t * up and retry on the freshly installed smaller mapping.\n+\t /\n+\tsplit_res = cpa_handle_large_page(cpa, kpte, address);\n+\tif (split_res \u003e 0)\n+\t\tgoto repeat;\n+\n+\treturn split_res;\n+}\n+\n+int __change_page_attr_set_clr(struct cpa_data cpa, int primary)\n+{\n+\tunsigned long numpages = cpa-\u003enumpages;\n+\tunsigned long rempages = numpages;\n+\tint ret = 0;\n+\n+\t/\n+\t No changes, easy!\n+\t /\n+\tif (!(pgprot_val(cpa-\u003emask_set) \| pgprot_val(cpa-\u003emask_clr)) \u0026\u0026\n+\t !cpa-\u003eforce_split)\n+\t\treturn ret;\n+\n+\twhile (rempages) {\n+\t\t/\n+\t\t * Store the remaining nr of pages for the large page\n+\t\t * preservation check.\n+\t\t /\n+\t\tcpa-\u003enumpages = rempages;\n+\t\t/ for array changes, we can't use large page /\n+\t\tif (cpa-\u003eflags \u0026 (CPA_ARRAY \| CPA_PAGES_ARRAY))\n+\t\t\tcpa-\u003enumpages = 1;\n+\n+\t\tspin_lock(\u0026cpa_lock);\n+\t\tret = __change_page_attr(cpa, primary);\n+\t\tspin_unlock(\u0026cpa_lock);\n+\t\tif (ret)\n+\t\t\tgoto out;\n+\n+\t\tif (primary \u0026\u0026 !(cpa-\u003eflags \u0026 CPA_NO_CHECK_ALIAS) \u0026\u0026\n+\t\t cpa_should_update_alias(cpa_addr(cpa, cpa-\u003ecurpage), cpa-\u003epfn)) {\n+\t\t\tret = arch_cpa_process_alias(cpa);\n+\t\t\tif (ret)\n+\t\t\t\tgoto out;\n+\t\t}\n+\n+\t\t/\n+\t\t * Adjust the number of pages with the result of the\n+\t\t * CPA operation. Either a large page has been\n+\t\t * preserved or a single page update happened.\n+\t\t /\n+\t\tBUG_ON(cpa-\u003enumpages \u003e rempages \|\| !cpa-\u003enumpages);\n+\t\trempages -= cpa-\u003enumpages;\n+\t\tcpa-\u003ecurpage += cpa-\u003enumpages;\n+\t}\n+\n+out:\n+\t/ Restore the original numpages /\n+\tcpa-\u003enumpages = numpages;\n+\treturn ret;\n+}\n+\n+int change_page_attr_set_clr(unsigned long addr, int numpages,\n+\t\t\t pgprot_t mask_set, pgprot_t mask_clr,\n+\t\t\t int force_split, int in_flag,\n+\t\t\t struct page *pages)\n+{\n+\tstruct cpa_data cpa;\n+\tint err;\n+\n+\tmemset(\u0026cpa, 0, sizeof(cpa));\n+\n+\t/\n+\t * Check, if we are requested to set a not supported\n+\t * feature. Clearing non-supported features is OK.\n+\t /\n+\tmask_set = canon_pgprot(mask_set);\n+\n+\tif (!pgprot_val(mask_set) \u0026\u0026 !pgprot_val(mask_clr) \u0026\u0026 !force_split)\n+\t\treturn 0;\n+\n+\t/ Ensure we are PAGE_SIZE aligned /\n+\tif (in_flag \u0026 CPA_ARRAY) {\n+\t\tint i;\n+\n+\t\tfor (i = 0; i \u003c numpages; i++) {\n+\t\t\tif (addr[i] \u0026 ~PAGE_MASK) {\n+\t\t\t\taddr[i] \u0026= PAGE_MASK;\n+\t\t\t\tWARN_ON_ONCE(1);\n+\t\t\t}\n+\t\t}\n+\t} else if (!(in_flag \u0026 CPA_PAGES_ARRAY)) {\n+\t\t/\n+\t\t * in_flag of CPA_PAGES_ARRAY implies it is aligned.\n+\t\t * No need to check in that case\n+\t\t /\n+\t\tif (addr \u0026 ~PAGE_MASK) {\n+\t\t\taddr \u0026= PAGE_MASK;\n+\t\t\t/\n+\t\t\t * People should not be passing in unaligned addresses:\n+\t\t\t /\n+\t\t\tWARN_ON_ONCE(1);\n+\t\t}\n+\t}\n+\n+\t/ Must avoid aliasing mappings in the highmem code /\n+\tkmap_flush_unused();\n+\n+\tvm_unmap_aliases();\n+\n+\tcpa.vaddr = addr;\n+\tcpa.pages = pages;\n+\tcpa.numpages = numpages;\n+\tcpa.mask_set = mask_set;\n+\tcpa.mask_clr = mask_clr;\n+\tcpa.flags = in_flag;\n+\tcpa.curpage = 0;\n+\tcpa.force_split = force_split;\n+\n+\terr = __change_page_attr_set_clr(\u0026cpa, 1);\n+\n+\t/\n+\t * Check whether we really changed something:\n+\t /\n+\tif (!(cpa.flags \u0026 CPA_FLUSHTLB))\n+\t\tgoto out;\n+\n+\tarch_cpa_flush(\u0026cpa, err);\n+\n+out:\n+\treturn err;\n+}\n+\n+/\n+ * The testcases use internal knowledge of the implementation that shouldn't\n+ * be exposed to the rest of the kernel. Include these directly here.\n+ */\n+#ifdef CONFIG_CPA_DEBUG\n+#include \"cpa-test.c\"\n+#endif\n" }
2/1	2026/07/22 11:11	agent	patch-evaluator	0m Model: best-expensive Args: null Results: { "EnableConfigs": [ "CPA_DEBUG" ], "FocusSymbols": [ "change_page_attr_set_clr", "__change_page_attr_set_clr", "__change_page_attr", "cpa_handle_large_page", "should_split_large_page", "arch_should_split_large_page", "arch_split_large_page", "arch_change_pte", "arch_cpa_process_fault", "arch_cpa_process_alias", "arch_cpa_flush", "lookup_address_in_pgd_attr" ], "Reasoning": "The patch refactors the change_page_attr (CPA) logic, moving it from architecture-specific code (x86, riscv) to a generic implementation in mm/set_memory.c. This is a significant functional change to core kernel memory management. Fuzzing should focus on the new generic functions and their architecture-specific callbacks. Enabling CPA_DEBUG will also run the CPA self-tests to help uncover issues.", "WorthFuzzing": true } Instruction: You are an expert Linux kernel maintainer. Your job is to review a provided patch series and determine if it makes functional changes to the kernel that should be fuzzed. IMPORTANT: The changes have ALREADY been applied and committed as the HEAD commit in your workspace. Do NOT rely on your internal knowledge of the kernel. You must actively use your code access tools to examine the actual source code and confirm any assumptions. Return WorthFuzzing=false if the patch only contains: - Modifications to Documentation/, Kconfig files, or code comments. - Purely decorative changes, such as logging (e.g., pr_err, printk) or tracepoints. - Changes to numeric constants or macros that do not functionally alter execution flow. - Code paths that are impossible to reach in virtualized environments like GCE or QEMU, even when utilizing software-emulated hardware (e.g., usb gadget, mac80211_hwsim). If it modifies reachable core kernel logic, drivers, or architectures, use your code search tools to verify the code can be executed, then return WorthFuzzing=true. When returning WorthFuzzing=true, you MUST ALSO: 1. Extract any specific kernel functions that should be heavily fuzzed into FocusSymbols. Avoid listing generic hot-path functions to prevent skewed test distributions. 2. Identify any specific CONFIG_ options required to properly test this new/modified feature. Go and look into the Kconfig files and check for ifdefs around the code, do not make assumptions. Do not list too generic configs (we already have them enabled). Only list those that specifically cover the modified code. List them in the EnableConfigs output array, and DO NOT add a 'CONFIG_' prefix (e.g., return "NET_IPV4" instead of "CONFIG_NET_IPV4"). Prefer calling several tools at the same time to save round-trips. Use set-results tool to provide results of the analysis. It must be called exactly once before the final reply. Ignore results of this tool. Prompt: For your convenience, here is the diff of the changes: commit d544b2bff4c34ff4e46cc081518a266ab316515c Author: syz-cluster <triage@syzkaller.com> Date: Wed Jul 22 11:11:39 2026 +0000 syz-cluster: applied patch under review diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index d8d25232501791..1bfa94797b1b4b 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -109,6 +109,7 @@ config LOONGARCH select GENERIC_LIB_DEVMEM_IS_ALLOWED select GENERIC_PCI_IOMAP select GENERIC_SCHED_CLOCK + select GENERIC_SET_MEMORY select GENERIC_SMP_IDLE_THREAD select GPIOLIB select HAS_IOPORT diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h index 223528c04d7392..377bef0894d771 100644 --- a/arch/loongarch/include/asm/pgtable.h +++ b/arch/loongarch/include/asm/pgtable.h @@ -638,6 +638,48 @@ static inline long pmd_protnone(pmd_t pmd) #define pmd_leaf(pmd) ((pmd_val(pmd) & _PAGE_HUGE) != 0) #define pud_leaf(pud) ((pud_val(pud) & _PAGE_HUGE) != 0) +#define pgd_write pgd_write +static inline int pgd_write(pgd_t pgd) +{ + return !!(pgd_val(pgd) & _PAGE_WRITE); +} + +#define p4d_write p4d_write +static inline int p4d_write(p4d_t p4d) +{ + return !!(p4d_val(p4d) & _PAGE_WRITE); +} + +#define pud_write pud_write +static inline int pud_write(pud_t pud) +{ + return !!(pud_val(pud) & _PAGE_WRITE); +} + +#define pgd_exec pgd_exec +static inline int pgd_exec(pgd_t pgd) +{ + return !(pgd_val(pgd) & _PAGE_NO_EXEC); +} + +#define p4d_exec p4d_exec +static inline int p4d_exec(p4d_t p4d) +{ + return !(p4d_val(p4d) & _PAGE_NO_EXEC); +} + +#define pud_exec pud_exec +static inline int pud_exec(pud_t pud) +{ + return !(pud_val(pud) & _PAGE_NO_EXEC); +} + +#define pmd_exec pmd_exec +static inline int pmd_exec(pmd_t pmd) +{ + return !(pmd_val(pmd) & _PAGE_NO_EXEC); +} + /* * We provide our own get_unmapped area to cope with the virtual aliasing * constraints placed on us by the cache architecture. diff --git a/arch/loongarch/include/asm/set_memory.h b/arch/loongarch/include/asm/set_memory.h index 55dfaefd02c8a6..6ffb4fe5a75b0e 100644 --- a/arch/loongarch/include/asm/set_memory.h +++ b/arch/loongarch/include/asm/set_memory.h @@ -19,4 +19,13 @@ int set_direct_map_default_noflush(struct page page); int set_direct_map_invalid_noflush(struct page page); int set_direct_map_valid_noflush(struct page page, unsigned nr, bool valid); +/ + * The direct map is a fixed address window that is not backed by page tables, + * so a page never has a second mapping whose protection must be synchronised. + / +static inline bool cpa_should_update_alias(unsigned long vaddr, unsigned long pfn) +{ + return false; +} + #endif / _ASM_LOONGARCH_SET_MEMORY_H / diff --git a/arch/loongarch/mm/pageattr.c b/arch/loongarch/mm/pageattr.c index 614ccc7afccbea..dbbd408bd3d5e6 100644 --- a/arch/loongarch/mm/pageattr.c +++ b/arch/loongarch/mm/pageattr.c @@ -4,126 +4,83 @@ / #include <linux/memblock.h> -#include <linux/pagewalk.h> +#include <linux/mm.h> #include <linux/pgtable.h> +#include <linux/set_memory.h> #include <asm/set_memory.h> #include <asm/tlbflush.h> -struct pageattr_masks { - pgprot_t set_mask; - pgprot_t clear_mask; -}; - -static unsigned long set_pageattr_masks(unsigned long val, struct mm_walk walk) +static unsigned long set_pageattr_masks(struct cpa_data cpa, unsigned long val) { - unsigned long new_val = val; - struct pageattr_masks masks = walk->private; - - new_val &= ~(pgprot_val(masks->clear_mask)); - new_val \|= (pgprot_val(masks->set_mask)); + val &= ~pgprot_val(cpa->mask_clr); + val \|= pgprot_val(cpa->mask_set); - return new_val; + return val; } -static int pageattr_pgd_entry(pgd_t pgd, unsigned long addr, - unsigned long next, struct mm_walk walk) +int arch_should_split_large_page(struct cpa_data cpa, struct cpa_split_data sd) { - pgd_t val = pgdp_get(pgd); - - if (pgd_leaf(val)) { - val = __pgd(set_pageattr_masks(pgd_val(val), walk)); - set_pgd(pgd, val); - } - - return 0; -} + pte_t old = ptep_get(sd->kpte); + unsigned long val = pte_val(old); + unsigned long new_val = set_pageattr_masks(cpa, val); -static int pageattr_p4d_entry(p4d_t p4d, unsigned long addr, - unsigned long next, struct mm_walk walk) -{ - p4d_t val = p4dp_get(p4d); + cpa->pfn = pte_pfn(old) + ((sd->address & (sd->psize - 1)) >> PAGE_SHIFT); - if (p4d_leaf(val)) { - val = __p4d(set_pageattr_masks(p4d_val(val), walk)); - set_p4d(p4d, val); + if (new_val != val) { + set_pte(sd->kpte, __pte(new_val)); + cpa->flags \|= CPA_FLUSHTLB; } return 0; } -static int pageattr_pud_entry(pud_t pud, unsigned long addr, - unsigned long next, struct mm_walk walk) +int arch_split_large_page(struct cpa_data cpa, struct cpa_split_data sd) { - pud_t val = pudp_get(pud); - - if (pud_leaf(val)) { - val = __pud(set_pageattr_masks(pud_val(val), walk)); - set_pud(pud, val); - } + WARN_ON_ONCE(1); - return 0; + return -EINVAL; } -static int pageattr_pmd_entry(pmd_t pmd, unsigned long addr, - unsigned long next, struct mm_walk walk) +void arch_change_pte(struct cpa_data cpa, unsigned long address, + pte_t kpte, pte_t old_pte, bool nx, bool rw) { - pmd_t val = pmdp_get(pmd); + unsigned long val = pte_val(old_pte); + pte_t new_pte = __pte(set_pageattr_masks(cpa, val)); - if (pmd_leaf(val)) { - val = __pmd(set_pageattr_masks(pmd_val(val), walk)); - set_pmd(pmd, val); - } + cpa->pfn = pte_pfn(old_pte); - return 0; + if (pte_val(old_pte) != pte_val(new_pte)) { + set_pte(kpte, new_pte); + cpa->flags \|= CPA_FLUSHTLB; + } } -static int pageattr_pte_entry(pte_t pte, unsigned long addr, - unsigned long next, struct mm_walk walk) +int arch_cpa_process_fault(struct cpa_data cpa, unsigned long vaddr, + int primary) { - pte_t val = ptep_get(pte); - - val = __pte(set_pageattr_masks(pte_val(val), walk)); - set_pte(pte, val); + /* + * The direct map has no page tables and vmalloc ranges may contain + * holes. Both are silently skipped. + / + cpa->numpages = 1; return 0; } -static int pageattr_pte_hole(unsigned long addr, unsigned long next, - int depth, struct mm_walk walk) +int arch_cpa_process_alias(struct cpa_data cpa) { return 0; } -static const struct mm_walk_ops pageattr_ops = { - .pgd_entry = pageattr_pgd_entry, - .p4d_entry = pageattr_p4d_entry, - .pud_entry = pageattr_pud_entry, - .pmd_entry = pageattr_pmd_entry, - .pte_entry = pageattr_pte_entry, - .pte_hole = pageattr_pte_hole, - .walk_lock = PGWALK_RDLOCK, -}; - -static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask, pgprot_t clear_mask) +void arch_cpa_flush(struct cpa_data cpa, int err) { - int ret; - unsigned long start = addr; - unsigned long end = start + PAGE_SIZE * numpages; - struct pageattr_masks masks = { - .set_mask = set_mask, - .clear_mask = clear_mask - }; - - if (!numpages) - return 0; - - mmap_write_lock(&init_mm); - ret = walk_kernel_page_table_range(start, end, &pageattr_ops, NULL, &masks); - mmap_write_unlock(&init_mm); - - flush_tlb_kernel_range(start, end); + if (err \|\| cpa->force_flush_all \|\| + (cpa->flags & (CPA_ARRAY \| CPA_PAGES_ARRAY))) { + flush_tlb_all(); + return; + } - return ret; + flush_tlb_kernel_range(cpa->vaddr, cpa->vaddr + cpa->numpages * PAGE_SIZE); } int set_memory_x(unsigned long addr, int numpages) @@ -131,7 +88,7 @@ int set_memory_x(unsigned long addr, int numpages) if (addr < vm_map_base) return 0; - return __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_NO_EXEC)); + return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_NO_EXEC), 0); } int set_memory_nx(unsigned long addr, int numpages) @@ -139,7 +96,7 @@ int set_memory_nx(unsigned long addr, int numpages) if (addr < vm_map_base) return 0; - return __set_memory(addr, numpages, __pgprot(_PAGE_NO_EXEC), __pgprot(0)); + return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_NO_EXEC), 0); } int set_memory_ro(unsigned long addr, int numpages) @@ -147,7 +104,7 @@ int set_memory_ro(unsigned long addr, int numpages) if (addr < vm_map_base) return 0; - return __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_WRITE \| _PAGE_DIRTY)); + return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_WRITE \| _PAGE_DIRTY), 0); } int set_memory_rw(unsigned long addr, int numpages) @@ -155,84 +112,75 @@ int set_memory_rw(unsigned long addr, int numpages) if (addr < vm_map_base) return 0; - return __set_memory(addr, numpages, __pgprot(_PAGE_WRITE \| _PAGE_DIRTY), __pgprot(0)); + return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_WRITE \| _PAGE_DIRTY), 0); } bool kernel_page_present(struct page page) { - pgd_t pgd; - p4d_t p4d; - pud_t pud; - pmd_t pmd; - pte_t pte; + unsigned int level; unsigned long addr = (unsigned long)page_address(page); + pte_t pte; if (addr < vm_map_base) return memblock_is_memory(__pa(addr)); - pgd = pgd_offset_k(addr); - if (pgd_none(pgdp_get(pgd))) - return false; - if (pgd_leaf(pgdp_get(pgd))) - return true; - - p4d = p4d_offset(pgd, addr); - if (p4d_none(p4dp_get(p4d))) - return false; - if (p4d_leaf(p4dp_get(p4d))) - return true; - - pud = pud_offset(p4d, addr); - if (pud_none(pudp_get(pud))) - return false; - if (pud_leaf(pudp_get(pud))) - return true; - - pmd = pmd_offset(pud, addr); - if (pmd_none(pmdp_get(pmd))) + pte = lookup_address(addr, &level); + if (!pte) return false; - if (pmd_leaf(pmdp_get(pmd))) - return true; - pte = pte_offset_kernel(pmd, addr); return pte_present(ptep_get(pte)); } int set_direct_map_default_noflush(struct page page) { unsigned long addr = (unsigned long)page_address(page); + struct cpa_data cpa = { .vaddr = &addr, + .pgd = NULL, + .numpages = 1, + .mask_set = PAGE_KERNEL, + .mask_clr = __pgprot(0), + .flags = CPA_NO_CHECK_ALIAS }; if (addr < vm_map_base) return 0; - return __set_memory(addr, 1, PAGE_KERNEL, __pgprot(0)); + return __change_page_attr_set_clr(&cpa, 1); } int set_direct_map_invalid_noflush(struct page page) { unsigned long addr = (unsigned long)page_address(page); + struct cpa_data cpa = { .vaddr = &addr, + .pgd = NULL, + .numpages = 1, + .mask_set = __pgprot(0), + .mask_clr = __pgprot(_PAGE_PRESENT \| _PAGE_VALID), + .flags = CPA_NO_CHECK_ALIAS }; if (addr < vm_map_base) return 0; - return __set_memory(addr, 1, __pgprot(0), __pgprot(_PAGE_PRESENT \| _PAGE_VALID)); + return __change_page_attr_set_clr(&cpa, 1); } int set_direct_map_valid_noflush(struct page page, unsigned nr, bool valid) { unsigned long addr = (unsigned long)page_address(page); - pgprot_t set, clear; + struct cpa_data cpa = { .vaddr = &addr, + .pgd = NULL, + .numpages = nr, + .flags = CPA_NO_CHECK_ALIAS }; if (addr < vm_map_base) return 0; if (valid) { - set = PAGE_KERNEL; - clear = __pgprot(0); + cpa.mask_set = PAGE_KERNEL; + cpa.mask_clr = __pgprot(0); } else { - set = __pgprot(0); - clear = __pgprot(_PAGE_PRESENT \| _PAGE_VALID); + cpa.mask_set = __pgprot(0); + cpa.mask_clr = __pgprot(_PAGE_PRESENT \| _PAGE_VALID); } - return __set_memory(addr, nr, set, clear); + return __change_page_attr_set_clr(&cpa, 1); } diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 3f0a647218e407..5d069bf290ce78 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -48,6 +48,7 @@ config RISCV select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_SET_DIRECT_MAP if MMU select ARCH_HAS_SET_MEMORY if MMU + select GENERIC_SET_MEMORY if MMU select ARCH_HAS_STRICT_KERNEL_RWX if MMU select ARCH_HAS_STRICT_MODULE_RWX if MMU select ARCH_HAS_SYNC_CORE_BEFORE_USERMODE diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 5d5756bda82e3d..027b3e7e4f177d 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -819,6 +819,42 @@ static inline int pud_write(pud_t pud) return pte_write(pud_pte(pud)); } +#define pgd_write pgd_write +static inline int pgd_write(pgd_t pgd) +{ + return pgd_val(pgd) & _PAGE_WRITE; +} + +#define p4d_write p4d_write +static inline int p4d_write(p4d_t p4d) +{ + return p4d_val(p4d) & _PAGE_WRITE; +} + +#define pgd_exec pgd_exec +static inline int pgd_exec(pgd_t pgd) +{ + return !!(pgd_val(pgd) & _PAGE_EXEC); +} + +#define p4d_exec p4d_exec +static inline int p4d_exec(p4d_t p4d) +{ + return !!(p4d_val(p4d) & _PAGE_EXEC); +} + +#define pud_exec pud_exec +static inline int pud_exec(pud_t pud) +{ + return !!(pud_val(pud) & _PAGE_EXEC); +} + +#define pmd_exec pmd_exec +static inline int pmd_exec(pmd_t pmd) +{ + return !!(pmd_val(pmd) & _PAGE_EXEC); +} + #define pmd_dirty pmd_dirty static inline int pmd_dirty(pmd_t pmd) { diff --git a/arch/riscv/include/asm/set_memory.h b/arch/riscv/include/asm/set_memory.h index ef59e1716a2cfd..4ba46a33103433 100644 --- a/arch/riscv/include/asm/set_memory.h +++ b/arch/riscv/include/asm/set_memory.h @@ -7,6 +7,8 @@ #define _ASM_RISCV_SET_MEMORY_H #ifndef __ASSEMBLER__ +#include <asm/page.h> +#include <asm/pgtable.h> /* * Functions to change memory attributes. / @@ -26,6 +28,11 @@ static __always_inline int set_kernel_memory(char startp, char endp, return set_memory(start, num_pages); } + +static inline bool cpa_should_update_alias(unsigned long vaddr, unsigned long pfn) +{ + return !is_linear_mapping(vaddr); +} #else static inline int set_memory_ro(unsigned long addr, int numpages) { return 0; } static inline int set_memory_rw(unsigned long addr, int numpages) { return 0; } diff --git a/arch/riscv/mm/pageattr.c b/arch/riscv/mm/pageattr.c index 3f76db3d276992..66a20fc3d80574 100644 --- a/arch/riscv/mm/pageattr.c +++ b/arch/riscv/mm/pageattr.c @@ -3,402 +3,242 @@ Copyright (C) 2019 SiFive / -#include <linux/pagewalk.h> +#include <linux/mm.h> #include <linux/pgtable.h> -#include <linux/vmalloc.h> +#include <linux/set_memory.h> #include <asm/tlbflush.h> -#include <asm/bitops.h> #include <asm/set_memory.h> -struct pageattr_masks { - pgprot_t set_mask; - pgprot_t clear_mask; -}; - -static unsigned long set_pageattr_masks(unsigned long val, struct mm_walk walk) -{ - struct pageattr_masks masks = walk->private; - unsigned long new_val = val; - - new_val &= ~(pgprot_val(masks->clear_mask)); - new_val \|= (pgprot_val(masks->set_mask)); - - return new_val; -} - -static int pageattr_p4d_entry(p4d_t p4d, unsigned long addr, - unsigned long next, struct mm_walk walk) +int arch_should_split_large_page(struct cpa_data cpa, struct cpa_split_data sd) { - p4d_t val = p4dp_get(p4d); - - if (p4d_leaf(val)) { - val = __p4d(set_pageattr_masks(p4d_val(val), walk)); - set_p4d(p4d, val); - } + pte_t old = ptep_get(sd->kpte); + pgprot_t old_prot = __pgprot(pte_val(old) & ~_PAGE_PFN_MASK); + pgprot_t new_prot = old_prot; + unsigned long old_pfn = pte_pfn(old); + unsigned long lpaddr, numpages; - return 0; -} + pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr); + pgprot_val(new_prot) \|= pgprot_val(cpa->mask_set); -static int pageattr_pud_entry(pud_t pud, unsigned long addr, - unsigned long next, struct mm_walk walk) -{ - pud_t val = pudp_get(pud); - - if (pud_leaf(val)) { - val = __pud(set_pageattr_masks(pud_val(val), walk)); - set_pud(pud, val); - } + / + * Record the pfn mapped at @address so the alias check can locate the + * matching linear map entry. + / + cpa->pfn = old_pfn + ((sd->address & (sd->psize - 1)) >> PAGE_SHIFT); - return 0; -} + / If the protections do not change, keep the large page intact. / + if (pgprot_val(new_prot) == pgprot_val(old_prot)) + return 0; -static int pageattr_pmd_entry(pmd_t pmd, unsigned long addr, - unsigned long next, struct mm_walk walk) -{ - pmd_t val = pmdp_get(pmd); + / If the request does not cover the whole large page, split it. / + lpaddr = sd->address & sd->pmask; + numpages = sd->psize >> PAGE_SHIFT; + if (sd->address != lpaddr \|\| cpa->numpages != numpages) + return 1; - if (pmd_leaf(val)) { - val = __pmd(set_pageattr_masks(pmd_val(val), walk)); - set_pmd(pmd, val); - } + / The whole large page is covered: update it in place. / + set_pte(sd->kpte, pfn_pte(old_pfn, new_prot)); + cpa->flags \|= CPA_FLUSHTLB; return 0; } -static int pageattr_pte_entry(pte_t pte, unsigned long addr, - unsigned long next, struct mm_walk walk) +static void split_set_ptes(pte_t ptep, unsigned long pfn, unsigned long pfninc, + pgprot_t prot) { - pte_t val = ptep_get(pte); - - val = __pte(set_pageattr_masks(pte_val(val), walk)); - set_pte(pte, val); + unsigned int i; - return 0; + for (i = 0; i < PTRS_PER_PTE; ++i, ++ptep, pfn += pfninc) + set_pte(ptep, pfn_pte(pfn, prot)); } -static int pageattr_pte_hole(unsigned long addr, unsigned long next, - int depth, struct mm_walk walk) +int arch_split_large_page(struct cpa_data cpa, struct cpa_split_data sd) { - / Nothing to do here / - return 0; -} - -static const struct mm_walk_ops pageattr_ops = { - .p4d_entry = pageattr_p4d_entry, - .pud_entry = pageattr_pud_entry, - .pmd_entry = pageattr_pmd_entry, - .pte_entry = pageattr_pte_entry, - .pte_hole = pageattr_pte_hole, - .walk_lock = PGWALK_RDLOCK, -}; - #ifdef CONFIG_64BIT -static int __split_linear_mapping_pmd(pud_t pudp, - unsigned long vaddr, unsigned long end) -{ - pmd_t pmdp; - unsigned long next; - - pmdp = pmd_offset(pudp, vaddr); - - do { - next = pmd_addr_end(vaddr, end); - - if (next - vaddr >= PMD_SIZE && - vaddr <= (vaddr & PMD_MASK) && end >= next) - continue; - - if (pmd_leaf(pmdp_get(pmdp))) { - struct page pte_page; - unsigned long pfn = _pmd_pfn(pmdp_get(pmdp)); - pgprot_t prot = __pgprot(pmd_val(pmdp_get(pmdp)) & ~_PAGE_PFN_MASK); - pte_t ptep_new; - int i; - - pte_page = alloc_page(GFP_KERNEL); - if (!pte_page) - return -ENOMEM; + struct page base = ptdesc_page(sd->ptdesc); + pte_t old = ptep_get(sd->kpte); + pgprot_t prot = __pgprot(pte_val(old) & ~_PAGE_PFN_MASK); + unsigned long pfn = pte_pfn(old); + unsigned long pfninc; + + switch (sd->level) { + case PGTABLE_LEVEL_PMD: + pfninc = 1; + break; + case PGTABLE_LEVEL_PUD: + pfninc = PMD_SIZE >> PAGE_SHIFT; + break; + case PGTABLE_LEVEL_P4D: + pfninc = PUD_SIZE >> PAGE_SHIFT; + break; + default: + return -EINVAL; + } - ptep_new = (pte_t )page_address(pte_page); - for (i = 0; i < PTRS_PER_PTE; ++i, ++ptep_new) - set_pte(ptep_new, pfn_pte(pfn + i, prot)); + split_set_ptes((pte_t )page_address(base), pfn, pfninc, prot); - smp_wmb(); + smp_wmb(); + set_pte(sd->kpte, pfn_pte(page_to_pfn(base), PAGE_TABLE)); - set_pmd(pmdp, pfn_pmd(page_to_pfn(pte_page), PAGE_TABLE)); - } - } while (pmdp++, vaddr = next, vaddr != end); + cpa->flags \|= CPA_FLUSHTLB; + cpa->force_flush_all = 1; return 0; +#else + WARN_ON_ONCE(1); + return -EINVAL; +#endif } -static int __split_linear_mapping_pud(p4d_t p4dp, - unsigned long vaddr, unsigned long end) +void arch_change_pte(struct cpa_data cpa, unsigned long address, + pte_t kpte, pte_t old_pte, bool nx, bool rw) { - pud_t pudp; - unsigned long next; - int ret; - - pudp = pud_offset(p4dp, vaddr); - - do { - next = pud_addr_end(vaddr, end); - - if (next - vaddr >= PUD_SIZE && - vaddr <= (vaddr & PUD_MASK) && end >= next) - continue; + pgprot_t new_prot = __pgprot(pte_val(old_pte) & ~_PAGE_PFN_MASK); + unsigned long pfn = pte_pfn(old_pte); + pte_t new_pte; - if (pud_leaf(pudp_get(pudp))) { - struct page pmd_page; - unsigned long pfn = _pud_pfn(pudp_get(pudp)); - pgprot_t prot = __pgprot(pud_val(pudp_get(pudp)) & ~_PAGE_PFN_MASK); - pmd_t pmdp_new; - int i; + pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr); + pgprot_val(new_prot) \|= pgprot_val(cpa->mask_set); - pmd_page = alloc_page(GFP_KERNEL); - if (!pmd_page) - return -ENOMEM; + new_pte = pfn_pte(pfn, new_prot); + cpa->pfn = pfn; - pmdp_new = (pmd_t )page_address(pmd_page); - for (i = 0; i < PTRS_PER_PMD; ++i, ++pmdp_new) - set_pmd(pmdp_new, - pfn_pmd(pfn + ((i PMD_SIZE) >> PAGE_SHIFT), prot)); - - smp_wmb(); - - set_pud(pudp, pfn_pud(page_to_pfn(pmd_page), PAGE_TABLE)); - } - - ret = __split_linear_mapping_pmd(pudp, vaddr, next); - if (ret) - return ret; - } while (pudp++, vaddr = next, vaddr != end); - - return 0; + if (pte_val(old_pte) != pte_val(new_pte)) { + set_pte(kpte, new_pte); + cpa->flags \|= CPA_FLUSHTLB; + } } -static int __split_linear_mapping_p4d(pgd_t pgdp, - unsigned long vaddr, unsigned long end) +int arch_cpa_process_fault(struct cpa_data cpa, unsigned long vaddr, + int primary) { - p4d_t p4dp; - unsigned long next; - int ret; - - p4dp = p4d_offset(pgdp, vaddr); - - do { - next = p4d_addr_end(vaddr, end); - - / - * If [vaddr; end] contains [vaddr & P4D_MASK; next], we don't - * need to split, we'll change the protections on the whole P4D. - / - if (next - vaddr >= P4D_SIZE && - vaddr <= (vaddr & P4D_MASK) && end >= next) - continue; - - if (p4d_leaf(p4dp_get(p4dp))) { - struct page pud_page; - unsigned long pfn = _p4d_pfn(p4dp_get(p4dp)); - pgprot_t prot = __pgprot(p4d_val(p4dp_get(p4dp)) & ~_PAGE_PFN_MASK); - pud_t pudp_new; - int i; - - pud_page = alloc_page(GFP_KERNEL); - if (!pud_page) - return -ENOMEM; - - / - * Fill the pud level with leaf puds that have the same - * protections as the leaf p4d. - / - pudp_new = (pud_t )page_address(pud_page); - for (i = 0; i < PTRS_PER_PUD; ++i, ++pudp_new) - set_pud(pudp_new, - pfn_pud(pfn + ((i * PUD_SIZE) >> PAGE_SHIFT), prot)); - - /* - * Make sure the pud filling is not reordered with the - * p4d store which could result in seeing a partially - * filled pud level. - / - smp_wmb(); - - set_p4d(p4dp, pfn_p4d(page_to_pfn(pud_page), PAGE_TABLE)); - } - - ret = __split_linear_mapping_pud(p4dp, vaddr, next); - if (ret) - return ret; - } while (p4dp++, vaddr = next, vaddr != end); + cpa->numpages = 1; - return 0; -} - -static int __split_linear_mapping_pgd(pgd_t pgdp, - unsigned long vaddr, - unsigned long end) -{ - unsigned long next; - int ret; + if (!primary) + return 0; - do { - next = pgd_addr_end(vaddr, end); - /* We never use PGD mappings for the linear mapping / - ret = __split_linear_mapping_p4d(pgdp, vaddr, next); - if (ret) - return ret; - } while (pgdp++, vaddr = next, vaddr != end); + / The linear map is expected to have holes / + if (is_linear_mapping(vaddr)) { + cpa->pfn = PFN_DOWN(__pa(vaddr)); + return 0; + } - return 0; -} + WARN(1, "CPA: called for zero pte. vaddr = %lx cpa->vaddr = %lx\n", + vaddr, cpa->vaddr); -static int split_linear_mapping(unsigned long start, unsigned long end) -{ - return __split_linear_mapping_pgd(pgd_offset_k(start), start, end); + return -EFAULT; } -#endif /* CONFIG_64BIT / -static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask, - pgprot_t clear_mask) +int arch_cpa_process_alias(struct cpa_data cpa) { - int ret; - unsigned long start = addr; - unsigned long end = start + PAGE_SIZE * numpages; - unsigned long __maybe_unused lm_start; - unsigned long __maybe_unused lm_end; - struct pageattr_masks masks = { - .set_mask = set_mask, - .clear_mask = clear_mask - }; - - if (!numpages) - return 0; + struct cpa_data alias_cpa; + unsigned long laddr; - mmap_write_lock(&init_mm); - -#ifdef CONFIG_64BIT /* - * We are about to change the permissions of a kernel mapping, we must - * apply the same changes to its linear mapping alias, which may imply - * splitting a huge mapping. + * cpa_should_update_alias() only lets non linear map primaries reach + * here, so @cpa->pfn always has a linear map alias that must receive + * the same protection change. / + laddr = (unsigned long)__va(PFN_PHYS(cpa->pfn)); - if (is_vmalloc_or_module_addr((void )start)) { - struct vm_struct area = NULL; - int i, page_start; - - area = find_vm_area((void )start); - page_start = (start - (unsigned long)area->addr) >> PAGE_SHIFT; - - for (i = page_start; i < page_start + numpages; ++i) { - lm_start = (unsigned long)page_address(area->pages[i]); - lm_end = lm_start + PAGE_SIZE; - - ret = split_linear_mapping(lm_start, lm_end); - if (ret) - goto unlock; - - ret = walk_kernel_page_table_range(lm_start, lm_end, - &pageattr_ops, NULL, &masks); - if (ret) - goto unlock; - } - } else if (is_kernel_mapping(start) \|\| is_linear_mapping(start)) { - if (is_kernel_mapping(start)) { - lm_start = (unsigned long)lm_alias(start); - lm_end = (unsigned long)lm_alias(end); - } else { - lm_start = start; - lm_end = end; - } - - ret = split_linear_mapping(lm_start, lm_end); - if (ret) - goto unlock; - - ret = walk_kernel_page_table_range(lm_start, lm_end, - &pageattr_ops, NULL, &masks); - if (ret) - goto unlock; - } - - ret = walk_kernel_page_table_range(start, end, &pageattr_ops, NULL, - &masks); + alias_cpa = cpa; + alias_cpa.vaddr = &laddr; + alias_cpa.flags &= ~(CPA_PAGES_ARRAY \| CPA_ARRAY); + alias_cpa.curpage = 0; -unlock: - mmap_write_unlock(&init_mm); + / The linear map alias must never be made executable / + alias_cpa.mask_set = __pgprot(pgprot_val(alias_cpa.mask_set) & ~_PAGE_EXEC); + alias_cpa.mask_clr = __pgprot(pgprot_val(alias_cpa.mask_clr) & ~_PAGE_EXEC); - / - * We can't use flush_tlb_kernel_range() here as we may have split a - * hugepage that is larger than that, so let's flush everything. - / - flush_tlb_all(); -#else - ret = walk_kernel_page_table_range(start, end, &pageattr_ops, NULL, - &masks); + cpa->force_flush_all = 1; - mmap_write_unlock(&init_mm); + return __change_page_attr_set_clr(&alias_cpa, 0); +} - flush_tlb_kernel_range(start, end); -#endif +void arch_cpa_flush(struct cpa_data cpa, int err) +{ + if (err \|\| cpa->force_flush_all \|\| + (cpa->flags & (CPA_ARRAY \| CPA_PAGES_ARRAY))) { + flush_tlb_all(); + return; + } - return ret; + flush_tlb_kernel_range(cpa->vaddr, + cpa->vaddr + cpa->numpages * PAGE_SIZE); } int set_memory_rw_nx(unsigned long addr, int numpages) { - return __set_memory(addr, numpages, __pgprot(_PAGE_READ \| _PAGE_WRITE), - __pgprot(_PAGE_EXEC)); + return change_page_attr_set_clr(&addr, numpages, + __pgprot(_PAGE_READ \| _PAGE_WRITE), + __pgprot(_PAGE_EXEC), 0, 0, NULL); } int set_memory_ro(unsigned long addr, int numpages) { - return __set_memory(addr, numpages, __pgprot(_PAGE_READ), - __pgprot(_PAGE_WRITE)); + return change_page_attr_set_clr(&addr, numpages, __pgprot(_PAGE_READ), + __pgprot(_PAGE_WRITE), 0, 0, NULL); } int set_memory_rw(unsigned long addr, int numpages) { - return __set_memory(addr, numpages, __pgprot(_PAGE_READ \| _PAGE_WRITE), - __pgprot(0)); + return change_page_attr_set(&addr, numpages, + __pgprot(_PAGE_READ \| _PAGE_WRITE), 0); } int set_memory_x(unsigned long addr, int numpages) { - return __set_memory(addr, numpages, __pgprot(_PAGE_EXEC), __pgprot(0)); + return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_EXEC), 0); } int set_memory_nx(unsigned long addr, int numpages) { - return __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_EXEC)); + return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_EXEC), 0); } int set_direct_map_invalid_noflush(struct page page) { - return __set_memory((unsigned long)page_address(page), 1, - __pgprot(0), __pgprot(_PAGE_PRESENT)); + unsigned long start = (unsigned long)page_address(page); + struct cpa_data cpa = { .vaddr = &start, + .pgd = NULL, + .numpages = 1, + .mask_set = __pgprot(0), + .mask_clr = __pgprot(_PAGE_PRESENT), + .flags = CPA_NO_CHECK_ALIAS }; + + return __change_page_attr_set_clr(&cpa, 1); } int set_direct_map_default_noflush(struct page page) { - return __set_memory((unsigned long)page_address(page), 1, - PAGE_KERNEL, __pgprot(_PAGE_EXEC)); + unsigned long start = (unsigned long)page_address(page); + struct cpa_data cpa = { .vaddr = &start, + .pgd = NULL, + .numpages = 1, + .mask_set = PAGE_KERNEL, + .mask_clr = __pgprot(_PAGE_EXEC), + .flags = CPA_NO_CHECK_ALIAS }; + + return __change_page_attr_set_clr(&cpa, 1); } int set_direct_map_valid_noflush(struct page page, unsigned nr, bool valid) { - pgprot_t set, clear; + unsigned long start = (unsigned long)page_address(page); + struct cpa_data cpa = { .vaddr = &start, + .pgd = NULL, + .numpages = nr, + .flags = CPA_NO_CHECK_ALIAS }; if (valid) { - set = PAGE_KERNEL; - clear = __pgprot(_PAGE_EXEC); + cpa.mask_set = PAGE_KERNEL; + cpa.mask_clr = __pgprot(_PAGE_EXEC); } else { - set = __pgprot(0); - clear = __pgprot(_PAGE_PRESENT); + cpa.mask_set = __pgprot(0); + cpa.mask_clr = __pgprot(_PAGE_PRESENT); } - return __set_memory((unsigned long)page_address(page), nr, set, clear); + return __change_page_attr_set_clr(&cpa, 1); } #ifdef CONFIG_DEBUG_PAGEALLOC @@ -435,36 +275,11 @@ void __kernel_map_pages(struct page page, int numpages, int enable) bool kernel_page_present(struct page page) { unsigned long addr = (unsigned long)page_address(page); - pgd_t pgd; - pud_t pud; - p4d_t p4d; - pmd_t pmd; - pte_t pte; - - pgd = pgd_offset_k(addr); - if (!pgd_present(pgdp_get(pgd))) - return false; - if (pgd_leaf(pgdp_get(pgd))) - return true; - - p4d = p4d_offset(pgd, addr); - if (!p4d_present(p4dp_get(p4d))) - return false; - if (p4d_leaf(p4dp_get(p4d))) - return true; - - pud = pud_offset(p4d, addr); - if (!pud_present(pudp_get(pud))) - return false; - if (pud_leaf(pudp_get(pud))) - return true; + unsigned int level; + pte_t pte = lookup_address(addr, &level); - pmd = pmd_offset(pud, addr); - if (!pmd_present(pmdp_get(pmd))) + if (!pte) return false; - if (pmd_leaf(pmdp_get(pmd))) - return true; - pte = pte_offset_kernel(pmd, addr); return pte_present(ptep_get(pte)); } diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index bdad90f210e4bf..e1f105e52d6a12 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -336,6 +336,7 @@ config X86 select ARCH_SUPPORTS_SCHED_CLUSTER if SMP select ARCH_SUPPORTS_SCHED_MC if SMP select HAVE_SINGLE_FTRACE_DIRECT_OPS if X86_64 && DYNAMIC_FTRACE_WITH_DIRECT_CALLS + select GENERIC_SET_MEMORY config INSTRUCTION_DECODER def_bool y diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug index c95c3aaadf97c3..314b72aad6fff4 100644 --- a/arch/x86/Kconfig.debug +++ b/arch/x86/Kconfig.debug @@ -157,12 +157,6 @@ config DEBUG_BOOT_PARAMS help This option will cause struct boot_params to be exported via debugfs. -config CPA_DEBUG - bool "CPA self-test code" - depends on DEBUG_KERNEL - help - Do change_page_attr() self-tests every 30 seconds. - config DEBUG_ENTRY bool "Debug low-level entry code" depends on DEBUG_KERNEL diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index ac295ca6c92f99..497aaf08ad3616 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -227,6 +227,42 @@ static inline int pud_write(pud_t pud) return pud_flags(pud) & _PAGE_RW; } +#define p4d_write p4d_write +static inline int p4d_write(p4d_t p4d) +{ + return p4d_flags(p4d) & _PAGE_RW; +} + +#define pgd_write pgd_write +static inline int pgd_write(pgd_t pgd) +{ + return pgd_flags(pgd) & _PAGE_RW; +} + +#define pmd_exec pmd_exec +static inline int pmd_exec(pmd_t pmd) +{ + return !(pmd_flags(pmd) & _PAGE_NX); +} + +#define pud_exec pud_exec +static inline int pud_exec(pud_t pud) +{ + return !(pud_flags(pud) & _PAGE_NX); +} + +#define p4d_exec p4d_exec +static inline int p4d_exec(p4d_t p4d) +{ + return !(p4d_flags(p4d) & _PAGE_NX); +} + +#define pgd_exec pgd_exec +static inline int pgd_exec(pgd_t pgd) +{ + return !(pgd_flags(pgd) & _PAGE_NX); +} + static inline int pte_huge(pte_t pte) { return pte_flags(pte) & _PAGE_PSE; diff --git a/arch/x86/include/asm/set_memory.h b/arch/x86/include/asm/set_memory.h index 4362c26aa992db..332ec300c2ed9c 100644 --- a/arch/x86/include/asm/set_memory.h +++ b/arch/x86/include/asm/set_memory.h @@ -3,12 +3,62 @@ #define _ASM_X86_SET_MEMORY_H #include <asm/page.h> +#include <asm/sections.h> #include <asm-generic/set_memory.h> #include <asm/pgtable.h> #define set_memory_rox set_memory_rox int set_memory_rox(unsigned long addr, int numpages); +#ifdef CONFIG_X86_64 +/ + * The kernel image is mapped into two places in the virtual address space + * (addresses without KASLR, of course): + * + * 1. The kernel direct map (0xffff880000000000) + * 2. The "high kernel map" (0xffffffff81000000) + * + * We actually execute out of #2. If we get the address of a kernel symbol, it + * points to #2, but almost all physical-to-virtual translations point to #1. + * + * This is so that we can have both a directmap of all physical memory and + * take full advantage of the limited (s32) immediate addressing range (2G) + * of x86_64. + * + * See Documentation/arch/x86/x86_64/mm.rst for more detail. + / +static inline bool pfn_is_kernel(unsigned long pfn) +{ + unsigned long spfn = __pa_symbol(_text) >> PAGE_SHIFT; + / Do not reference a physical address outside the kernel. / + unsigned long epfn = __pa_symbol(roundup(_brk_end, PMD_SIZE) - 1) >> PAGE_SHIFT; + + return pfn >= spfn && pfn <= epfn; +} +#else +static inline bool pfn_is_kernel(unsigned long pfn) +{ + / There is no highmap on 32-bit / + return false; +} +#endif + +static inline bool cpa_should_update_alias(unsigned long vaddr, + unsigned long pfn) +{ + / Primary is not in the direct map, its direct map alias needs update / + if (vaddr < PAGE_OFFSET \|\| + vaddr >= PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT)) + return true; + + / direct map page that is also part of the kernel highmap / + if ((vaddr < (unsigned long)_text \|\| vaddr >= _brk_end) && + pfn_is_kernel(pfn)) + return true; + + return false; +} + / * The set_memory_* API can be used to change various attributes of a virtual * address range. The attributes include: diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index d023a40a1e0345..c460121c96b949 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -38,21 +38,6 @@ /* * The current flushing context - we pass it instead of 5 arguments: / -struct cpa_data { - unsigned long vaddr; - pgd_t pgd; - pgprot_t mask_set; - pgprot_t mask_clr; - unsigned long numpages; - unsigned long curpage; - unsigned long pfn; - unsigned int flags; - unsigned int force_split : 1, - force_static_prot : 1, - force_flush_all : 1; - struct page pages; -}; - enum cpa_warn { CPA_CONFLICT, CPA_PROTECT, @@ -61,27 +46,13 @@ enum cpa_warn { static const int cpa_warn_level = CPA_PROTECT; -/ - * Serialize cpa() (for !DEBUG_PAGEALLOC which uses large identity mappings) - * using cpa_lock. So that we don't allow any other cpu, with stale large tlb - * entries change the page attribute in parallel to some other cpu - * splitting a large page entry along with changing the attribute. - / -static DEFINE_SPINLOCK(cpa_lock); - -#define CPA_FLUSHTLB 1 -#define CPA_ARRAY 2 -#define CPA_PAGES_ARRAY 4 -#define CPA_NO_CHECK_ALIAS 8 / Do not search for aliases / -#define CPA_COLLAPSE 16 / try to collapse large pages / - static inline pgprot_t cachemode2pgprot(enum page_cache_mode pcm) { return __pgprot(cachemode2protval(pcm)); } #ifdef CONFIG_PROC_FS -static unsigned long direct_pages_count[PG_LEVEL_NUM]; +static unsigned long direct_pages_count[PGTABLE_LEVEL_NUM]; void update_page_count(int level, unsigned long pages) { @@ -98,9 +69,9 @@ static void split_page_count(int level) direct_pages_count[level]--; if (system_state == SYSTEM_RUNNING) { - if (level == PG_LEVEL_2M) + if (level == PGTABLE_LEVEL_PMD) count_vm_event(DIRECT_MAP_LEVEL2_SPLIT); - else if (level == PG_LEVEL_1G) + else if (level == PGTABLE_LEVEL_PUD) count_vm_event(DIRECT_MAP_LEVEL3_SPLIT); } direct_pages_count[level - 1] += PTRS_PER_PTE; @@ -110,9 +81,9 @@ static void collapse_page_count(int level) { direct_pages_count[level]++; if (system_state == SYSTEM_RUNNING) { - if (level == PG_LEVEL_2M) + if (level == PGTABLE_LEVEL_PMD) count_vm_event(DIRECT_MAP_LEVEL2_COLLAPSE); - else if (level == PG_LEVEL_1G) + else if (level == PGTABLE_LEVEL_PUD) count_vm_event(DIRECT_MAP_LEVEL3_COLLAPSE); } direct_pages_count[level - 1] -= PTRS_PER_PTE; @@ -121,17 +92,17 @@ static void collapse_page_count(int level) void arch_report_meminfo(struct seq_file m) { seq_printf(m, "DirectMap4k: %8lu kB\n", - direct_pages_count[PG_LEVEL_4K] << 2); + direct_pages_count[PGTABLE_LEVEL_PTE] << 2); #if defined(CONFIG_X86_64) \|\| defined(CONFIG_X86_PAE) seq_printf(m, "DirectMap2M: %8lu kB\n", - direct_pages_count[PG_LEVEL_2M] << 11); + direct_pages_count[PGTABLE_LEVEL_PMD] << 11); #else seq_printf(m, "DirectMap4M: %8lu kB\n", - direct_pages_count[PG_LEVEL_2M] << 12); + direct_pages_count[PGTABLE_LEVEL_PMD] << 12); #endif if (direct_gbpages) seq_printf(m, "DirectMap1G: %8lu kB\n", - direct_pages_count[PG_LEVEL_1G] << 20); + direct_pages_count[PGTABLE_LEVEL_PUD] << 20); } #else static inline void split_page_count(int level) { } @@ -165,7 +136,7 @@ static inline void cpa_inc_4k_install(void) static inline void cpa_inc_lp_sameprot(int level) { - if (level == PG_LEVEL_1G) + if (level == PGTABLE_LEVEL_PUD) cpa_1g_sameprot++; else cpa_2m_sameprot++; @@ -173,7 +144,7 @@ static inline void cpa_inc_lp_sameprot(int level) static inline void cpa_inc_lp_preserved(int level) { - if (level == PG_LEVEL_1G) + if (level == PGTABLE_LEVEL_PUD) cpa_1g_preserved++; else cpa_2m_preserved++; @@ -225,61 +196,6 @@ within(unsigned long addr, unsigned long start, unsigned long end) return addr >= start && addr < end; } -#ifdef CONFIG_X86_64 - -static inline int -within_inclusive(unsigned long addr, unsigned long start, unsigned long end) -{ - return addr >= start && addr <= end; -} - -/* - * The kernel image is mapped into two places in the virtual address space - * (addresses without KASLR, of course): - * - * 1. The kernel direct map (0xffff880000000000) - * 2. The "high kernel map" (0xffffffff81000000) - * - * We actually execute out of #2. If we get the address of a kernel symbol, it - * points to #2, but almost all physical-to-virtual translations point to #1. - * - * This is so that we can have both a directmap of all physical memory and - * take full advantage of the limited (s32) immediate addressing range (2G) - * of x86_64. - * - * See Documentation/arch/x86/x86_64/mm.rst for more detail. - / - -static inline unsigned long highmap_start_pfn(void) -{ - return __pa_symbol(_text) >> PAGE_SHIFT; -} - -static inline unsigned long highmap_end_pfn(void) -{ - / Do not reference physical address outside the kernel. / - return __pa_symbol(roundup(_brk_end, PMD_SIZE) - 1) >> PAGE_SHIFT; -} - -static bool __cpa_pfn_in_highmap(unsigned long pfn) -{ - / - * Kernel text has an alias mapping at a high address, known - * here as "highmap". - / - return within_inclusive(pfn, highmap_start_pfn(), highmap_end_pfn()); -} - -#else - -static bool __cpa_pfn_in_highmap(unsigned long pfn) -{ - / There is no highmap on 32-bit / - return false; -} - -#endif - / * See set_mce_nospec(). * @@ -303,23 +219,6 @@ static inline unsigned long fix_addr(unsigned long addr) #endif } -static unsigned long __cpa_addr(struct cpa_data cpa, unsigned long idx) -{ - if (cpa->flags & CPA_PAGES_ARRAY) { - struct page page = cpa->pages[idx]; - - if (unlikely(PageHighMem(page))) - return 0; - - return (unsigned long)page_address(page); - } - - if (cpa->flags & CPA_ARRAY) - return cpa->vaddr[idx]; - - return cpa->vaddr + idx PAGE_SIZE; -} - /* * Flushing functions / @@ -405,7 +304,7 @@ static void __cpa_flush_tlb(void data) unsigned int i; for (i = 0; i < cpa->numpages; i++) - flush_tlb_one_kernel(fix_addr(__cpa_addr(cpa, i))); + flush_tlb_one_kernel(fix_addr(cpa_addr(cpa, i))); } static int collapse_large_pages(unsigned long addr, struct list_head pgtables); @@ -420,10 +319,10 @@ static void cpa_collapse_large_pages(struct cpa_data cpa) if (cpa->flags & (CPA_PAGES_ARRAY \| CPA_ARRAY)) { for (i = 0; i < cpa->numpages; i++) - collapsed += collapse_large_pages(__cpa_addr(cpa, i), + collapsed += collapse_large_pages(cpa_addr(cpa, i), &pgtables); } else { - addr = __cpa_addr(cpa, 0); + addr = cpa_addr(cpa, 0); start = addr & PMD_MASK; end = addr + PAGE_SIZE * cpa->numpages; @@ -463,7 +362,7 @@ static void cpa_flush(struct cpa_data cpa, int cache) mb(); for (i = 0; i < cpa->numpages; i++) { - unsigned long addr = __cpa_addr(cpa, i); + unsigned long addr = cpa_addr(cpa, i); unsigned int level; pte_t pte = lookup_address(addr, &level); @@ -578,7 +477,7 @@ static pgprotval_t protect_kernel_text_ro(unsigned long start, * so the protections for kernel text and identity mappings have to * be the same. / - if (lookup_address(start, &level) && (level != PG_LEVEL_4K)) + if (lookup_address(start, &level) && (level != PGTABLE_LEVEL_PTE)) return _PAGE_RW; return 0; } @@ -709,104 +608,6 @@ static inline pgprot_t verify_rwx(pgprot_t old, pgprot_t new, unsigned long star return new; } -/ - * Lookup the page table entry for a virtual address in a specific pgd. - * Return a pointer to the entry (or NULL if the entry does not exist), - * the level of the entry, and the effective NX and RW bits of all - * page table levels. - / -pte_t lookup_address_in_pgd_attr(pgd_t pgd, unsigned long address, - unsigned int level, bool nx, bool rw) -{ - p4d_t p4d; - pud_t pud; - pmd_t pmd; - - level = PG_LEVEL_256T; - nx = false; - rw = true; - - if (pgd_none(pgd)) - return NULL; - - level = PG_LEVEL_512G; - nx \|= pgd_flags(pgd) & _PAGE_NX; - rw &= pgd_flags(pgd) & _PAGE_RW; - - p4d = p4d_offset(pgd, address); - if (p4d_none(p4d)) - return NULL; - - if (p4d_leaf(p4d) \|\| !p4d_present(p4d)) - return (pte_t )p4d; - - level = PG_LEVEL_1G; - nx \|= p4d_flags(p4d) & _PAGE_NX; - rw &= p4d_flags(p4d) & _PAGE_RW; - - pud = pud_offset(p4d, address); - if (pud_none(pud)) - return NULL; - - if (pud_leaf(pud) \|\| !pud_present(pud)) - return (pte_t )pud; - - level = PG_LEVEL_2M; - nx \|= pud_flags(pud) & _PAGE_NX; - rw &= pud_flags(pud) & _PAGE_RW; - - pmd = pmd_offset(pud, address); - if (pmd_none(pmd)) - return NULL; - - if (pmd_leaf(pmd) \|\| !pmd_present(pmd)) - return (pte_t )pmd; - - level = PG_LEVEL_4K; - nx \|= pmd_flags(pmd) & _PAGE_NX; - rw &= pmd_flags(pmd) & _PAGE_RW; - - return pte_offset_kernel(pmd, address); -} - -/ - * Lookup the page table entry for a virtual address in a specific pgd. - * Return a pointer to the entry and the level of the mapping. - / -pte_t lookup_address_in_pgd(pgd_t pgd, unsigned long address, - unsigned int level) -{ - bool nx, rw; - - return lookup_address_in_pgd_attr(pgd, address, level, &nx, &rw); -} - -/* - * Lookup the page table entry for a virtual address. Return a pointer - * to the entry and the level of the mapping. - * - * Note: the function returns p4d, pud or pmd either when the entry is marked - * large or when the present bit is not set. Otherwise it returns NULL. - / -pte_t lookup_address(unsigned long address, unsigned int level) -{ - return lookup_address_in_pgd(pgd_offset_k(address), address, level); -} -EXPORT_SYMBOL_GPL(lookup_address); - -static pte_t _lookup_address_cpa(struct cpa_data cpa, unsigned long address, - unsigned int level, bool nx, bool rw) -{ - pgd_t pgd; - - if (!cpa->pgd) - pgd = pgd_offset_k(address); - else - pgd = cpa->pgd + pgd_index(address); - - return lookup_address_in_pgd_attr(pgd, address, level, nx, rw); -} - / * Lookup the PMD entry for a virtual address. Return a pointer to the entry * or NULL if not present. @@ -850,9 +651,9 @@ pmd_t lookup_pmd_address(unsigned long address) phys_addr_t slow_virt_to_phys(void __virt_addr) { unsigned long virt_addr = (unsigned long)__virt_addr; + enum pgtable_level level; phys_addr_t phys_addr; unsigned long offset; - enum pg_level level; pte_t pte; pte = lookup_address(virt_addr, &level); @@ -864,11 +665,11 @@ phys_addr_t slow_virt_to_phys(void __virt_addr) * make 32-PAE kernel work correctly. / switch (level) { - case PG_LEVEL_1G: + case PGTABLE_LEVEL_PUD: phys_addr = (phys_addr_t)pud_pfn((pud_t )pte) << PAGE_SHIFT; offset = virt_addr & ~PUD_MASK; break; - case PG_LEVEL_2M: + case PGTABLE_LEVEL_PMD: phys_addr = (phys_addr_t)pmd_pfn((pmd_t )pte) << PAGE_SHIFT; offset = virt_addr & ~PMD_MASK; break; @@ -925,30 +726,27 @@ static pgprot_t pgprot_clear_protnone_bits(pgprot_t prot) return prot; } -static int __should_split_large_page(pte_t kpte, unsigned long address, - struct cpa_data cpa) +int arch_should_split_large_page(struct cpa_data cpa, + struct cpa_split_data sd) { - unsigned long numpages, pmask, psize, lpaddr, pfn, old_pfn; pgprot_t old_prot, new_prot, req_prot, chk_prot; - pte_t new_pte, tmp; - enum pg_level level; - bool nx, rw; - - /* - * Check for races, another CPU might have split this page - * up already: - / - tmp = _lookup_address_cpa(cpa, address, &level, &nx, &rw); - if (tmp != kpte) - return 1; + unsigned long numpages, lpaddr, pfn, old_pfn; + enum pgtable_level level = sd->level; + unsigned long address = sd->address; + unsigned long psize = sd->psize; + unsigned long pmask = sd->pmask; + pte_t kpte = sd->kpte; + bool nx = sd->nx; + bool rw = sd->rw; + pte_t new_pte; switch (level) { - case PG_LEVEL_2M: + case PGTABLE_LEVEL_PMD: old_prot = pmd_pgprot((pmd_t )kpte); old_pfn = pmd_pfn((pmd_t )kpte); cpa_inc_2m_checked(); break; - case PG_LEVEL_1G: + case PGTABLE_LEVEL_PUD: old_prot = pud_pgprot((pud_t )kpte); old_pfn = pud_pfn((pud_t )kpte); cpa_inc_1g_checked(); @@ -957,18 +755,6 @@ static int __should_split_large_page(pte_t kpte, unsigned long address, return -EINVAL; } - psize = page_level_size(level); - pmask = page_level_mask(level); - - / - * Calculate the number of pages, which fit into this large - * page starting at address: - / - lpaddr = (address + psize) & pmask; - numpages = (lpaddr - address) >> PAGE_SHIFT; - if (numpages < cpa->numpages) - cpa->numpages = numpages; - / * We are safe now. Check whether the new pgprot is the same: * Convert protection attributes to 4k-format, as cpa->mask* are set @@ -1072,21 +858,6 @@ static int __should_split_large_page(pte_t kpte, unsigned long address, return 0; } -static int should_split_large_page(pte_t kpte, unsigned long address, - struct cpa_data cpa) -{ - int do_split; - - if (cpa->force_split) - return 1; - - spin_lock(&pgd_lock); - do_split = __should_split_large_page(kpte, address, cpa); - spin_unlock(&pgd_lock); - - return do_split; -} - static void split_set_pte(struct cpa_data cpa, pte_t pte, unsigned long pfn, pgprot_t ref_prot, unsigned long address, unsigned long size) @@ -1123,33 +894,22 @@ static void split_set_pte(struct cpa_data cpa, pte_t pte, unsigned long pfn, set_pte(pte, pfn_pte(pfn, ref_prot)); } -static int -__split_large_page(struct cpa_data cpa, pte_t kpte, unsigned long address, - struct ptdesc ptdesc) +int arch_split_large_page(struct cpa_data cpa, struct cpa_split_data sd) { unsigned long lpaddr, lpinc, ref_pfn, pfn, pfninc = 1; + struct ptdesc ptdesc = sd->ptdesc; struct page base = ptdesc_page(ptdesc); pte_t pbase = (pte_t )page_address(base); - unsigned int i, level; + enum pgtable_level level = sd->level; + unsigned long address = sd->address; + pte_t kpte = sd->kpte; + unsigned int i; pgprot_t ref_prot; - bool nx, rw; - pte_t tmp; - - spin_lock(&pgd_lock); - /* - * Check for races, another CPU might have split this page - * up for us already: - / - tmp = _lookup_address_cpa(cpa, address, &level, &nx, &rw); - if (tmp != kpte) { - spin_unlock(&pgd_lock); - return 1; - } paravirt_alloc_pte(&init_mm, page_to_pfn(base)); switch (level) { - case PG_LEVEL_2M: + case PGTABLE_LEVEL_PMD: ref_prot = pmd_pgprot((pmd_t )kpte); / * Clear PSE (aka _PAGE_PAT) and move @@ -1161,7 +921,7 @@ __split_large_page(struct cpa_data cpa, pte_t kpte, unsigned long address, lpinc = PAGE_SIZE; break; - case PG_LEVEL_1G: + case PGTABLE_LEVEL_PUD: ref_prot = pud_pgprot((pud_t )kpte); ref_pfn = pud_pfn((pud_t )kpte); pfninc = PMD_SIZE >> PAGE_SHIFT; @@ -1177,7 +937,6 @@ __split_large_page(struct cpa_data cpa, pte_t kpte, unsigned long address, break; default: - spin_unlock(&pgd_lock); return 1; } @@ -1225,26 +984,6 @@ __split_large_page(struct cpa_data cpa, pte_t kpte, unsigned long address, * just split large page entry. / flush_tlb_all(); - spin_unlock(&pgd_lock); - - return 0; -} - -static int split_large_page(struct cpa_data cpa, pte_t kpte, - unsigned long address) -{ - struct ptdesc ptdesc; - - if (!debug_pagealloc_enabled()) - spin_unlock(&cpa_lock); - ptdesc = pagetable_alloc(GFP_KERNEL, 0); - if (!debug_pagealloc_enabled()) - spin_lock(&cpa_lock); - if (!ptdesc) - return -ENOMEM; - - if (__split_large_page(cpa, kpte, address, ptdesc)) - pagetable_free(ptdesc); return 0; } @@ -1314,7 +1053,7 @@ static int collapse_pmd_page(pmd_t pmd, unsigned long addr, } if (virt_addr_valid(addr) && pfn_range_is_mapped(pfn, pfn + 1)) - collapse_page_count(PG_LEVEL_2M); + collapse_page_count(PGTABLE_LEVEL_PMD); return 1; } @@ -1361,7 +1100,7 @@ static int collapse_pud_page(pud_t pud, unsigned long addr, set_pud(pud, pfn_pud(pfn, pmd_pgprot(first))); if (virt_addr_valid(addr) && pfn_range_is_mapped(pfn, pfn + 1)) - collapse_page_count(PG_LEVEL_1G); + collapse_page_count(PGTABLE_LEVEL_PUD); return 1; } @@ -1800,8 +1539,8 @@ static int populate_pgd(struct cpa_data cpa, unsigned long addr) return 0; } -static int __cpa_process_fault(struct cpa_data cpa, unsigned long vaddr, - int primary) +int arch_cpa_process_fault(struct cpa_data cpa, unsigned long vaddr, + int primary) { if (cpa->pgd) { / @@ -1833,7 +1572,7 @@ static int __cpa_process_fault(struct cpa_data cpa, unsigned long vaddr, cpa->pfn = __pa(vaddr) >> PAGE_SHIFT; return 0; - } else if (__cpa_pfn_in_highmap(cpa->pfn)) { + } else if (pfn_is_kernel(cpa->pfn)) { / Faults in the highmap are OK, so do not warn: / return -EFAULT; } else { @@ -1845,90 +1584,45 @@ static int __cpa_process_fault(struct cpa_data cpa, unsigned long vaddr, } } -static int __change_page_attr(struct cpa_data cpa, int primary) +void arch_change_pte(struct cpa_data cpa, unsigned long address, + pte_t kpte, pte_t old_pte, bool nx, bool rw) { - unsigned long address; - int do_split, err; - unsigned int level; - pte_t kpte, old_pte; - bool nx, rw; - - address = __cpa_addr(cpa, cpa->curpage); -repeat: - kpte = _lookup_address_cpa(cpa, address, &level, &nx, &rw); - if (!kpte) - return __cpa_process_fault(cpa, address, primary); + pte_t new_pte; + pgprot_t old_prot = pte_pgprot(old_pte); + pgprot_t new_prot = pte_pgprot(old_pte); + unsigned long pfn = pte_pfn(old_pte); - old_pte = kpte; - if (pte_none(old_pte)) - return __cpa_process_fault(cpa, address, primary); + pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr); + pgprot_val(new_prot) \|= pgprot_val(cpa->mask_set); - if (level == PG_LEVEL_4K) { - pte_t new_pte; - pgprot_t old_prot = pte_pgprot(old_pte); - pgprot_t new_prot = pte_pgprot(old_pte); - unsigned long pfn = pte_pfn(old_pte); - - pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr); - pgprot_val(new_prot) \|= pgprot_val(cpa->mask_set); - - cpa_inc_4k_install(); - / Hand in lpsize = 0 to enforce the protection mechanism / - new_prot = static_protections(new_prot, address, pfn, 1, 0, - CPA_PROTECT); - - new_prot = verify_rwx(old_prot, new_prot, address, pfn, 1, - nx, rw); + cpa_inc_4k_install(); + / Hand in lpsize = 0 to enforce the protection mechanism / + new_prot = static_protections(new_prot, address, pfn, 1, 0, + CPA_PROTECT); - new_prot = pgprot_clear_protnone_bits(new_prot); + new_prot = verify_rwx(old_prot, new_prot, address, pfn, 1, nx, rw); - / - * We need to keep the pfn from the existing PTE, - * after all we're only going to change its attributes - * not the memory it points to - / - new_pte = pfn_pte(pfn, new_prot); - cpa->pfn = pfn; - / - * Do we really change anything ? - / - if (pte_val(old_pte) != pte_val(new_pte)) { - set_pte_atomic(kpte, new_pte); - cpa->flags \|= CPA_FLUSHTLB; - } - cpa->numpages = 1; - return 0; - } + new_prot = pgprot_clear_protnone_bits(new_prot); / - * Check, whether we can keep the large page intact - * and just change the pte: - / - do_split = should_split_large_page(kpte, address, cpa); - / - * When the range fits into the existing large page, - * return. cp->numpages and cpa->tlbflush have been updated in - * try_large_page: + * We need to keep the pfn from the existing PTE, after all we're only + * going to change its attributes not the memory it points to / - if (do_split <= 0) - return do_split; - + new_pte = pfn_pte(pfn, new_prot); + cpa->pfn = pfn; / - * We have to split the large page: + * Do we really change anything ? / - err = split_large_page(cpa, kpte, address); - if (!err) - goto repeat; - - return err; + if (pte_val(old_pte) != pte_val(new_pte)) { + set_pte_atomic(kpte, new_pte); + cpa->flags \|= CPA_FLUSHTLB; + } } -static int __change_page_attr_set_clr(struct cpa_data cpa, int primary); - /* * Check the directmap and "high kernel map" 'aliases'. / -static int cpa_process_alias(struct cpa_data cpa) +int arch_cpa_process_alias(struct cpa_data cpa) { struct cpa_data alias_cpa; unsigned long laddr = (unsigned long)__va(cpa->pfn << PAGE_SHIFT); @@ -1942,7 +1636,7 @@ static int cpa_process_alias(struct cpa_data cpa) * No need to redo, when the primary call touched the direct * mapping already: / - vaddr = __cpa_addr(cpa, cpa->curpage); + vaddr = cpa_addr(cpa, cpa->curpage); if (!(within(vaddr, PAGE_OFFSET, PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT)))) { @@ -1971,7 +1665,7 @@ static int cpa_process_alias(struct cpa_data cpa) * to touch the high mapped kernel as well: / if (!within(vaddr, (unsigned long)_text, _brk_end) && - __cpa_pfn_in_highmap(cpa->pfn)) { + pfn_is_kernel(cpa->pfn)) { unsigned long temp_cpa_vaddr = (cpa->pfn << PAGE_SHIFT) + __START_KERNEL_map - phys_base; alias_cpa = cpa; @@ -2000,168 +1694,25 @@ static int cpa_process_alias(struct cpa_data cpa) return 0; } -static int __change_page_attr_set_clr(struct cpa_data cpa, int primary) -{ - unsigned long numpages = cpa->numpages; - unsigned long rempages = numpages; - int ret = 0; - - /* - * No changes, easy! - / - if (!(pgprot_val(cpa->mask_set) \| pgprot_val(cpa->mask_clr)) && - !cpa->force_split) - return ret; - - while (rempages) { - / - * Store the remaining nr of pages for the large page - * preservation check. - / - cpa->numpages = rempages; - / for array changes, we can't use large page / - if (cpa->flags & (CPA_ARRAY \| CPA_PAGES_ARRAY)) - cpa->numpages = 1; - - if (!debug_pagealloc_enabled()) - spin_lock(&cpa_lock); - ret = __change_page_attr(cpa, primary); - if (!debug_pagealloc_enabled()) - spin_unlock(&cpa_lock); - if (ret) - goto out; - - if (primary && !(cpa->flags & CPA_NO_CHECK_ALIAS)) { - ret = cpa_process_alias(cpa); - if (ret) - goto out; - } - - / - * Adjust the number of pages with the result of the - * CPA operation. Either a large page has been - * preserved or a single page update happened. - / - BUG_ON(cpa->numpages > rempages \|\| !cpa->numpages); - rempages -= cpa->numpages; - cpa->curpage += cpa->numpages; - } - -out: - / Restore the original numpages / - cpa->numpages = numpages; - return ret; -} - -static int change_page_attr_set_clr(unsigned long addr, int numpages, - pgprot_t mask_set, pgprot_t mask_clr, - int force_split, int in_flag, - struct page *pages) +void arch_cpa_flush(struct cpa_data cpa, int err) { - struct cpa_data cpa; - int ret, cache; - - memset(&cpa, 0, sizeof(cpa)); - - /* - * Check, if we are requested to set a not supported - * feature. Clearing non-supported features is OK. - / - mask_set = canon_pgprot(mask_set); - - if (!pgprot_val(mask_set) && !pgprot_val(mask_clr) && !force_split) - return 0; - - / Ensure we are PAGE_SIZE aligned / - if (in_flag & CPA_ARRAY) { - int i; - for (i = 0; i < numpages; i++) { - if (addr[i] & ~PAGE_MASK) { - addr[i] &= PAGE_MASK; - WARN_ON_ONCE(1); - } - } - } else if (!(in_flag & CPA_PAGES_ARRAY)) { - / - * in_flag of CPA_PAGES_ARRAY implies it is aligned. - * No need to check in that case - / - if (addr & ~PAGE_MASK) { - addr &= PAGE_MASK; - / - * People should not be passing in unaligned addresses: - / - WARN_ON_ONCE(1); - } - } - - / Must avoid aliasing mappings in the highmem code / - kmap_flush_unused(); - - vm_unmap_aliases(); - - cpa.vaddr = addr; - cpa.pages = pages; - cpa.numpages = numpages; - cpa.mask_set = mask_set; - cpa.mask_clr = mask_clr; - cpa.flags = in_flag; - cpa.curpage = 0; - cpa.force_split = force_split; - - ret = __change_page_attr_set_clr(&cpa, 1); - - / - * Check whether we really changed something: - / - if (!(cpa.flags & CPA_FLUSHTLB)) - goto out; + int cache; / * No need to flush, when we did not set any of the caching * attributes: / - cache = !!pgprot2cachemode(mask_set); + cache = !!pgprot2cachemode(cpa->mask_set); / * On error; flush everything to be sure. / - if (ret) { + if (err) { cpa_flush_all(cache); - goto out; + return; } - cpa_flush(&cpa, cache); -out: - return ret; -} - -static inline int change_page_attr_set(unsigned long addr, int numpages, - pgprot_t mask, int array) -{ - return change_page_attr_set_clr(addr, numpages, mask, __pgprot(0), 0, - (array ? CPA_ARRAY : 0), NULL); -} - -static inline int change_page_attr_clear(unsigned long addr, int numpages, - pgprot_t mask, int array) -{ - return change_page_attr_set_clr(addr, numpages, __pgprot(0), mask, 0, - (array ? CPA_ARRAY : 0), NULL); -} - -static inline int cpa_set_pages_array(struct page pages, int numpages, - pgprot_t mask) -{ - return change_page_attr_set_clr(NULL, numpages, mask, __pgprot(0), 0, - CPA_PAGES_ARRAY, pages); -} - -static inline int cpa_clear_pages_array(struct page pages, int numpages, - pgprot_t mask) -{ - return change_page_attr_set_clr(NULL, numpages, __pgprot(0), mask, 0, - CPA_PAGES_ARRAY, pages); + cpa_flush(cpa, cache); } int _set_memory_uc(unsigned long addr, int numpages) @@ -2773,11 +2324,3 @@ int __init kernel_unmap_pages_in_pgd(pgd_t pgd, unsigned long address, return retval; } - -/* - * The testcases use internal knowledge of the implementation that shouldn't - * be exposed to the rest of the kernel. Include these directly here. - / -#ifdef CONFIG_CPA_DEBUG -#include "cpa-test.c" -#endif diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 2981e386da7b3c..d1df8345852be8 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -1782,6 +1782,10 @@ void arch_sync_kernel_mappings(unsigned long start, unsigned long end); because these macros can be used even if CONFIG_MMU is not defined. / +#ifndef canon_pgprot +#define canon_pgprot(prot) (prot) +#endif + #ifndef pgprot_nx #define pgprot_nx(prot) (prot) #endif @@ -2105,6 +2109,46 @@ static inline int pud_write(pud_t pud) } #endif / pud_write / +#ifndef p4d_write +static inline int p4d_write(p4d_t p4d) +{ + BUG(); + return 0; +} +#endif / p4d_write / + +#ifndef pmd_exec +static inline int pmd_exec(pmd_t pmd) +{ + BUG(); + return 0; +} +#endif / pmd_exec / + +#ifndef pud_exec +static inline int pud_exec(pud_t pud) +{ + BUG(); + return 0; +} +#endif / pud_exec / + +#ifndef p4d_exec +static inline int p4d_exec(p4d_t p4d) +{ + BUG(); + return 0; +} +#endif / p4d_exec / + +#ifndef pgd_exec +static inline int pgd_exec(pgd_t pgd) +{ + BUG(); + return 0; +} +#endif / pgd_exec / + #if !defined(CONFIG_TRANSPARENT_HUGEPAGE) \|\| \ !defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) static inline int pud_trans_huge(pud_t pud) @@ -2295,16 +2339,20 @@ static inline bool arch_has_pfn_modify_check(void) typedef unsigned int pgtbl_mod_mask; enum pgtable_level { - PGTABLE_LEVEL_PTE = 0, + PGTABLE_LEVEL_NONE = 0, + PGTABLE_LEVEL_PTE, PGTABLE_LEVEL_PMD, PGTABLE_LEVEL_PUD, PGTABLE_LEVEL_P4D, PGTABLE_LEVEL_PGD, + PGTABLE_LEVEL_NUM, }; static inline const char pgtable_level_to_str(enum pgtable_level level) { switch (level) { + case PGTABLE_LEVEL_NONE: + return "none"; case PGTABLE_LEVEL_PTE: return "pte"; case PGTABLE_LEVEL_PMD: @@ -2320,6 +2368,28 @@ static inline const char pgtable_level_to_str(enum pgtable_level level) } } +/ Size of the memory mapped by a single leaf entry at @level / +static inline unsigned long pgtable_level_size(enum pgtable_level level) +{ + switch (level) { + case PGTABLE_LEVEL_PTE: + return PAGE_SIZE; + case PGTABLE_LEVEL_PMD: + return PMD_SIZE; + case PGTABLE_LEVEL_PUD: + return PUD_SIZE; + case PGTABLE_LEVEL_P4D: + return P4D_SIZE; + default: + return PGDIR_SIZE; + } +} + +static inline unsigned long pgtable_level_mask(enum pgtable_level level) +{ + return ~(pgtable_level_size(level) - 1); +} + #endif / !__ASSEMBLY__ / #if !defined(MAX_POSSIBLE_PHYSMEM_BITS) && !defined(CONFIG_64BIT) diff --git a/include/linux/set_memory.h b/include/linux/set_memory.h index 3030d9245f5ac8..17cbe6e26cb732 100644 --- a/include/linux/set_memory.h +++ b/include/linux/set_memory.h @@ -84,4 +84,101 @@ static inline int set_memory_decrypted(unsigned long addr, int numpages) } #endif / CONFIG_ARCH_HAS_MEM_ENCRYPT / +#ifdef CONFIG_GENERIC_SET_MEMORY + +#include <linux/pgtable.h> + +#define CPA_FLUSHTLB 1 +#define CPA_ARRAY 2 +#define CPA_PAGES_ARRAY 4 +#define CPA_NO_CHECK_ALIAS 8 / Do not search for aliases / +#define CPA_COLLAPSE 16 / try to collapse large pages / + +/ + * The current flushing context - we pass it instead of 5 arguments: + / +struct cpa_data { + unsigned long vaddr; + pgd_t pgd; + pgprot_t mask_set; + pgprot_t mask_clr; + unsigned long numpages; + unsigned long curpage; + unsigned long pfn; + unsigned int flags; + unsigned int force_split : 1, + force_static_prot : 1, + force_flush_all : 1; + struct page pages; +}; + +/ + * context for checking for a split a large page and splitting it if required. + / +struct ptdesc; +struct cpa_split_data { + unsigned long address; + unsigned long psize; + unsigned long pmask; + pte_t kpte; + struct ptdesc ptdesc; + enum pgtable_level level; + bool nx; + bool rw; +}; + +unsigned long cpa_addr(struct cpa_data cpa, unsigned long idx); + +pte_t lookup_address(unsigned long address, unsigned int level); +pte_t lookup_address_in_pgd(pgd_t pgd, unsigned long address, + unsigned int level); +pte_t lookup_address_in_pgd_attr(pgd_t pgd, unsigned long address, + unsigned int level, bool nx, bool rw); + +int arch_cpa_process_fault(struct cpa_data cpa, unsigned long vaddr, + int primary); +int arch_cpa_process_alias(struct cpa_data cpa); +int arch_should_split_large_page(struct cpa_data cpa, + struct cpa_split_data sd); +int arch_split_large_page(struct cpa_data cpa, struct cpa_split_data sd); +void arch_change_pte(struct cpa_data cpa, unsigned long address, + pte_t kpte, pte_t old_pte, bool nx, bool rw); +void arch_cpa_flush(struct cpa_data cpa, int err); + +int __change_page_attr_set_clr(struct cpa_data cpa, int primary); +int change_page_attr_set_clr(unsigned long addr, int numpages, + pgprot_t mask_set, pgprot_t mask_clr, + int force_split, int in_flag, + struct page pages); + +static inline int change_page_attr_set(unsigned long addr, int numpages, + pgprot_t mask, int array) +{ + return change_page_attr_set_clr(addr, numpages, mask, __pgprot(0), 0, + (array ? CPA_ARRAY : 0), NULL); +} + +static inline int change_page_attr_clear(unsigned long addr, int numpages, + pgprot_t mask, int array) +{ + return change_page_attr_set_clr(addr, numpages, __pgprot(0), mask, 0, + (array ? CPA_ARRAY : 0), NULL); +} + +static inline int cpa_set_pages_array(struct page pages, int numpages, + pgprot_t mask) +{ + return change_page_attr_set_clr(NULL, numpages, mask, __pgprot(0), 0, + CPA_PAGES_ARRAY, pages); +} + +static inline int cpa_clear_pages_array(struct page pages, int numpages, + pgprot_t mask) +{ + return change_page_attr_set_clr(NULL, numpages, __pgprot(0), mask, 0, + CPA_PAGES_ARRAY, pages); +} + +#endif / CONFIG_GENERIC_SET_MEMORY / + #endif / _LINUX_SET_MEMORY_H_ / diff --git a/mm/Kconfig b/mm/Kconfig index 9e0ca48249054f..541edd1e11b610 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -1509,6 +1509,9 @@ config LAZY_MMU_MODE_KUNIT_TEST If unsure, say N. +config GENERIC_SET_MEMORY + bool + source "mm/damon/Kconfig" endmenu diff --git a/mm/Kconfig.debug b/mm/Kconfig.debug index 91b3e027b75357..83cfef6b2d8cfb 100644 --- a/mm/Kconfig.debug +++ b/mm/Kconfig.debug @@ -320,3 +320,10 @@ config PER_VMA_LOCK_STATS overhead in the page fault path. If in doubt, say N. + +config CPA_DEBUG + bool "CPA self-test code" + depends on GENERIC_SET_MEMORY + depends on DEBUG_KERNEL + help + Do change_page_attr() self-tests every 30 seconds. diff --git a/mm/Makefile b/mm/Makefile index eff9f9e7e061c1..e3f6c39c2ba991 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -147,3 +147,4 @@ obj-$(CONFIG_SHRINKER_DEBUG) += shrinker_debug.o obj-$(CONFIG_EXECMEM) += execmem.o obj-$(CONFIG_TMPFS_QUOTA) += shmem_quota.o obj-$(CONFIG_LAZY_MMU_MODE_KUNIT_TEST) += tests/lazy_mmu_mode_kunit.o +obj-$(CONFIG_GENERIC_SET_MEMORY) += set_memory.o diff --git a/arch/x86/mm/pat/cpa-test.c b/mm/cpa-test.c similarity index 80% rename from arch/x86/mm/pat/cpa-test.c rename to mm/cpa-test.c index ad3c1feec990db..63c3463a5a7d85 100644 --- a/arch/x86/mm/pat/cpa-test.c +++ b/mm/cpa-test.c @@ -13,32 +13,24 @@ #include <linux/mm.h> #include <linux/vmalloc.h> -#include <asm/cacheflush.h> -#include <asm/kdebug.h> - / * Only print the results of the first pass: / static __read_mostly int print = 1; +static unsigned long max_test_pfn; enum { NTEST = 3 100, NPAGES = 100, -#ifdef CONFIG_X86_64 - LPS = (1 << PMD_SHIFT), -#elif defined(CONFIG_X86_PAE) - LPS = (1 << PMD_SHIFT), -#else - LPS = (1 << 22), -#endif - GPS = (1<<30) + LPS = PMD_SIZE, + GPS = PUD_SIZE }; -#define PAGE_CPA_TEST __pgprot(_PAGE_CPA_TEST) +#define PAGE_CPA_TEST __pgprot(_PAGE_SPECIAL) static int pte_testbit(pte_t pte) { - return pte_flags(pte) & _PAGE_SOFTW1; + return pte_special(pte); } struct split_state { @@ -54,10 +46,10 @@ static int print_split(struct split_state s) s->lpg = s->gpg = s->spg = s->exec = 0; s->min_exec = ~0UL; s->max_exec = 0; - for (i = 0; i < max_pfn_mapped; ) { + for (i = 0; i < max_test_pfn; ) { unsigned long addr = (unsigned long)__va(i << PAGE_SHIFT); unsigned int level; - pte_t pte; + pte_t pte, pteval; pte = lookup_address(addr, &level); if (!pte) { @@ -65,15 +57,16 @@ static int print_split(struct split_state s) i++; continue; } + pteval = ptep_get(pte); - if (level == PG_LEVEL_1G && sizeof(long) == 8) { + if (level == PGTABLE_LEVEL_PUD && sizeof(long) == 8) { s->gpg++; i += GPS/PAGE_SIZE; - } else if (level == PG_LEVEL_2M) { - if ((pte_val(pte) & _PAGE_PRESENT) && !(pte_val(pte) & _PAGE_PSE)) { + } else if (level == PGTABLE_LEVEL_PMD) { + if (pte_present(pteval) && !pte_huge(pteval)) { printk(KERN_ERR - "%lx level %d but not PSE %Lx\n", - addr, level, (u64)pte_val(pte)); + "%lx level %d but not leaf %Lx\n", + addr, level, (u64)pte_val(pteval)); err = 1; } s->lpg++; @@ -82,7 +75,7 @@ static int print_split(struct split_state s) s->spg++; i++; } - if (!(pte_val(pte) & _PAGE_NX)) { + if (pte_exec(pteval)) { s->exec++; if (addr < s->min_exec) s->min_exec = addr; @@ -100,8 +93,8 @@ static int print_split(struct split_state s) expected = (s->gpgGPS + s->lpgLPS)/PAGE_SIZE + s->spg + missed; if (expected != i) { - printk(KERN_ERR "CPA max_pfn_mapped %lu but expected %lu\n", - max_pfn_mapped, expected); + printk(KERN_ERR "CPA max_test_pfn %lu but expected %lu\n", + max_test_pfn, expected); return 1; } return err; @@ -118,7 +111,7 @@ static int pageattr_test(void) { struct split_state sa, sb, sc; unsigned long bm; - pte_t pte, pte0; + pte_t pte, pte0, pte_val; int failed = 0; unsigned int level; int i, k; @@ -127,7 +120,7 @@ static int pageattr_test(void) if (print) printk(KERN_INFO "CPA self-test:\n"); - bm = vzalloc((max_pfn_mapped + 7) / 8); + bm = vzalloc((max_test_pfn + 7) / 8); if (!bm) { printk(KERN_ERR "CPA Cannot vmalloc bitmap\n"); return -ENOMEM; @@ -136,11 +129,11 @@ static int pageattr_test(void) failed += print_split(&sa); for (i = 0; i < NTEST; i++) { - unsigned long pfn = get_random_u32_below(max_pfn_mapped); + unsigned long pfn = get_random_u32_below(max_test_pfn); addr[i] = (unsigned long)__va(pfn << PAGE_SHIFT); len[i] = get_random_u32_below(NPAGES); - len[i] = min_t(unsigned long, len[i], max_pfn_mapped - pfn - 1); + len[i] = min_t(unsigned long, len[i], max_test_pfn - pfn - 1); if (len[i] == 0) len[i] = 1; @@ -150,15 +143,20 @@ static int pageattr_test(void) for (k = 0; k < len[i]; k++) { pte = lookup_address(addr[i] + kPAGE_SIZE, &level); - if (!pte \|\| pgprot_val(pte_pgprot(pte)) == 0 \|\| - !(pte_val(pte) & _PAGE_PRESENT)) { + if (!pte) { + addr[i] = 0; + break; + } + pte_val = ptep_get(pte); + if (pgprot_val(pte_pgprot(pte_val)) == 0 \|\| + !pte_present(pte_val)) { addr[i] = 0; break; } if (k == 0) { - pte0 = pte; + pte0 = pte_val; } else { - if (pgprot_val(pte_pgprot(pte)) != + if (pgprot_val(pte_pgprot(pte_val)) != pgprot_val(pte_pgprot(pte0))) { len[i] = k; break; @@ -198,12 +196,12 @@ static int pageattr_test(void) } pte = lookup_address(addr[i], &level); - if (!pte \|\| !pte_testbit(pte) \|\| pte_huge(pte)) { + if (!pte \|\| !pte_testbit(pte) \|\| level != PGTABLE_LEVEL_PTE) { printk(KERN_ERR "CPA %lx: bad pte %Lx\n", addr[i], pte ? (u64)pte_val(pte) : 0ULL); failed++; } - if (level != PG_LEVEL_4K) { + if (level != PGTABLE_LEVEL_PTE) { printk(KERN_ERR "CPA %lx: unexpected level %d\n", addr[i], level); failed++; @@ -266,6 +264,8 @@ static int start_pageattr_test(void) { struct task_struct p; + max_test_pfn = PFN_DOWN(__pa(high_memory - 1)); + p = kthread_create(do_pageattr_test, NULL, "pageattr-test"); if (!IS_ERR(p)) wake_up_process(p); diff --git a/mm/set_memory.c b/mm/set_memory.c new file mode 100644 index 00000000000000..744405efcc3401 --- /dev/null +++ b/mm/set_memory.c @@ -0,0 +1,396 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <linux/highmem.h> +#include <linux/vmalloc.h> +#include <linux/pgtable.h> +#include <linux/export.h> + +#include <linux/set_memory.h> + +/ + * Serialize cpa() so that we don't allow any other cpu, with stale large tlb + * entries, to change the page attribute in parallel to some other cpu + * splitting a large page entry along with changing the attribute. + / +static DEFINE_SPINLOCK(cpa_lock); + +static inline void arch_lock(void) +{ +#ifdef CONFIG_X86 + spin_lock(&pgd_lock); +#endif +} + +static inline void arch_unlock(void) +{ +#ifdef CONFIG_X86 + spin_unlock(&pgd_lock); +#endif +} + +unsigned long cpa_addr(struct cpa_data cpa, unsigned long idx) +{ + if (cpa->flags & CPA_PAGES_ARRAY) { + struct page page = cpa->pages[idx]; + + if (unlikely(PageHighMem(page))) + return 0; + + return (unsigned long)page_address(page); + } + + if (cpa->flags & CPA_ARRAY) + return cpa->vaddr[idx]; + + return cpa->vaddr + idx * PAGE_SIZE; +} + +/* + * Lookup the page table entry for a virtual address in a specific pgd. + * Return a pointer to the entry (or NULL if the entry does not exist), + * the level of the entry, and the effective NX and RW bits of all + * page table levels. + / +pte_t lookup_address_in_pgd_attr(pgd_t pgdp, unsigned long address, + unsigned int level, bool nx, bool rw) +{ + pgd_t pgd = pgdp_get(pgdp); + p4d_t p4dp, p4d; + pud_t pudp, pud; + pmd_t pmdp, pmd; + + level = PGTABLE_LEVEL_PGD; + nx = false; + rw = true; + + if (pgd_none(pgd)) + return NULL; + + level = PGTABLE_LEVEL_P4D; + nx \|= !pgd_exec(pgd); + rw &= !!(pgd_write(pgd)); + + p4dp = p4d_offset(pgdp, address); + p4d = p4dp_get(p4dp); + if (p4d_none(p4d)) + return NULL; + + if (p4d_leaf(p4d) \|\| !p4d_present(p4d)) + return (pte_t )p4dp; + + level = PGTABLE_LEVEL_PUD; + nx \|= !p4d_exec(p4d); + rw &= !!(p4d_write(p4d)); + + pudp = pud_offset(p4dp, address); + pud = pudp_get(pudp); + if (pud_none(pud)) + return NULL; + + if (pud_leaf(pud) \|\| !pud_present(pud)) + return (pte_t )pudp; + + level = PGTABLE_LEVEL_PMD; + nx \|= !pud_exec(pud); + rw &= !!(pud_write(pud)); + + pmdp = pmd_offset(pudp, address); + pmd = pmdp_get(pmdp); + if (pmd_none(pmd)) + return NULL; + + if (pmd_leaf(pmd) \|\| !pmd_present(pmd)) + return (pte_t )pmdp; + + level = PGTABLE_LEVEL_PTE; + nx \|= !pmd_exec(pmd); + rw &= !!(pmd_write(pmd)); + + return pte_offset_kernel(pmdp, address); +} + +/ + * Lookup the page table entry for a virtual address in a specific pgd. + * Return a pointer to the entry and the level of the mapping. + / +pte_t lookup_address_in_pgd(pgd_t pgd, unsigned long address, + unsigned int level) +{ + bool nx, rw; + + return lookup_address_in_pgd_attr(pgd, address, level, &nx, &rw); +} + +/* + * Lookup the page table entry for a virtual address. Return a pointer + * to the entry and the level of the mapping. + * + * Note: the function returns p4d, pud or pmd either when the entry is marked + * large or when the present bit is not set. Otherwise it returns NULL. + / +pte_t lookup_address(unsigned long address, unsigned int level) +{ + return lookup_address_in_pgd(pgd_offset_k(address), address, level); +} +EXPORT_SYMBOL_GPL(lookup_address); + +static pte_t _lookup_address_cpa(struct cpa_data cpa, unsigned long address, + unsigned int level, bool nx, bool rw) +{ + pgd_t pgd; + + if (!cpa->pgd) + pgd = pgd_offset_k(address); + else + pgd = cpa->pgd + pgd_index(address); + + return lookup_address_in_pgd_attr(pgd, address, level, nx, rw); +} + +static int should_split_large_page(struct cpa_data cpa, + struct cpa_split_data sd) +{ + enum pgtable_level level = sd->level; + unsigned long address = sd->address; + unsigned long numpages, lpaddr; + + if (cpa->force_split) + return 1; + + sd->psize = pgtable_level_size(level); + sd->pmask = pgtable_level_mask(level); + + / + * Calculate the number of pages, which fit into this large + * page starting at address: + / + lpaddr = (address + sd->psize) & sd->pmask; + numpages = (lpaddr - address) >> PAGE_SHIFT; + if (numpages < cpa->numpages) + cpa->numpages = numpages; + + return arch_should_split_large_page(cpa, sd); +} + +static int cpa_handle_large_page(struct cpa_data cpa, pte_t kpte, + unsigned long address) +{ + struct cpa_split_data sd = { + .address = address, + .kpte = kpte, + }; + int do_split, ret = 1; + pte_t tmp; + + spin_unlock(&cpa_lock); + sd.ptdesc = pagetable_alloc(GFP_KERNEL, 0); + spin_lock(&cpa_lock); + if (!sd.ptdesc) + return -ENOMEM; + + arch_lock(); + + /* + * Check for races, another CPU might have split this page + * up already: + / + tmp = _lookup_address_cpa(cpa, address, &sd.level, &sd.nx, &sd.rw); + if (tmp != kpte) + goto out_free_ptdesc; + + / + * Check, whether we can keep the large page intact + * and just change the pte: + / + do_split = should_split_large_page(cpa, &sd); + ret = do_split; + / + * When the range fits into the existing large page, no split is + * required. + * should_split_large_page() updated the large page attributes and + * cpa->numpages and cpa->pfn. + / + if (do_split <= 0) + goto out_free_ptdesc; + + / + * We have to split the large page: + / + ret = arch_split_large_page(cpa, &sd); + if (ret) + goto out_free_ptdesc; + + arch_unlock(); + return do_split; + +out_free_ptdesc: + arch_unlock(); + pagetable_free(sd.ptdesc); + return ret; +} + +static int __change_page_attr(struct cpa_data cpa, int primary) +{ + unsigned long address; + int split_res; + unsigned int level; + pte_t kpte, old_pte; + bool nx, rw; + + address = cpa_addr(cpa, cpa->curpage); +repeat: + kpte = _lookup_address_cpa(cpa, address, &level, &nx, &rw); + if (!kpte) + return arch_cpa_process_fault(cpa, address, primary); + + old_pte = kpte; + if (pte_none(old_pte)) + return arch_cpa_process_fault(cpa, address, primary); + + if (level == PGTABLE_LEVEL_PTE) { + arch_change_pte(cpa, address, kpte, old_pte, nx, rw); + cpa->numpages = 1; + return 0; + } + + /* + * The address maps a large page. Try to keep it, otherwise split it + * up and retry on the freshly installed smaller mapping. + / + split_res = cpa_handle_large_page(cpa, kpte, address); + if (split_res > 0) + goto repeat; + + return split_res; +} + +int __change_page_attr_set_clr(struct cpa_data cpa, int primary) +{ + unsigned long numpages = cpa->numpages; + unsigned long rempages = numpages; + int ret = 0; + + /* + * No changes, easy! + / + if (!(pgprot_val(cpa->mask_set) \| pgprot_val(cpa->mask_clr)) && + !cpa->force_split) + return ret; + + while (rempages) { + / + * Store the remaining nr of pages for the large page + * preservation check. + / + cpa->numpages = rempages; + / for array changes, we can't use large page / + if (cpa->flags & (CPA_ARRAY \| CPA_PAGES_ARRAY)) + cpa->numpages = 1; + + spin_lock(&cpa_lock); + ret = __change_page_attr(cpa, primary); + spin_unlock(&cpa_lock); + if (ret) + goto out; + + if (primary && !(cpa->flags & CPA_NO_CHECK_ALIAS) && + cpa_should_update_alias(cpa_addr(cpa, cpa->curpage), cpa->pfn)) { + ret = arch_cpa_process_alias(cpa); + if (ret) + goto out; + } + + / + * Adjust the number of pages with the result of the + * CPA operation. Either a large page has been + * preserved or a single page update happened. + / + BUG_ON(cpa->numpages > rempages \|\| !cpa->numpages); + rempages -= cpa->numpages; + cpa->curpage += cpa->numpages; + } + +out: + / Restore the original numpages / + cpa->numpages = numpages; + return ret; +} + +int change_page_attr_set_clr(unsigned long addr, int numpages, + pgprot_t mask_set, pgprot_t mask_clr, + int force_split, int in_flag, + struct page *pages) +{ + struct cpa_data cpa; + int err; + + memset(&cpa, 0, sizeof(cpa)); + + / + * Check, if we are requested to set a not supported + * feature. Clearing non-supported features is OK. + / + mask_set = canon_pgprot(mask_set); + + if (!pgprot_val(mask_set) && !pgprot_val(mask_clr) && !force_split) + return 0; + + / Ensure we are PAGE_SIZE aligned / + if (in_flag & CPA_ARRAY) { + int i; + + for (i = 0; i < numpages; i++) { + if (addr[i] & ~PAGE_MASK) { + addr[i] &= PAGE_MASK; + WARN_ON_ONCE(1); + } + } + } else if (!(in_flag & CPA_PAGES_ARRAY)) { + / + * in_flag of CPA_PAGES_ARRAY implies it is aligned. + * No need to check in that case + / + if (addr & ~PAGE_MASK) { + addr &= PAGE_MASK; + / + * People should not be passing in unaligned addresses: + / + WARN_ON_ONCE(1); + } + } + + / Must avoid aliasing mappings in the highmem code / + kmap_flush_unused(); + + vm_unmap_aliases(); + + cpa.vaddr = addr; + cpa.pages = pages; + cpa.numpages = numpages; + cpa.mask_set = mask_set; + cpa.mask_clr = mask_clr; + cpa.flags = in_flag; + cpa.curpage = 0; + cpa.force_split = force_split; + + err = __change_page_attr_set_clr(&cpa, 1); + + / + * Check whether we really changed something: + / + if (!(cpa.flags & CPA_FLUSHTLB)) + goto out; + + arch_cpa_flush(&cpa, err); + +out: + return err; +} + +/ + * The testcases use internal knowledge of the implementation that shouldn't + * be exposed to the rest of the kernel. Include these directly here. + */ +#ifdef CONFIG_CPA_DEBUG +#include "cpa-test.c" +#endif
3/2	2026/07/22 11:11	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 36394 output: 255 thoughts: 984 Thoughts: Refactoring Core Memory Management for Enhanced Testing Okay, so I'm looking at this patch. It's refactoring the `change_page_attr` (CPA) logic, moving it into a more generic `mm/set_memory.c` from the architecture-specific locations. This introduces `CONFIG_GENERIC_SET_MEMORY` and updates a few architecture Kconfig files (`arch/loongarch/Kconfig`, `arch/riscv/Kconfig`, and `arch/x86/Kconfig`) to select it. They're also moving the test file, `cpa-test.c`, into the `mm` directory. It looks like the architectures are expected to provide some new functions now too: `arch_should_split_large_page`, `arch_split_large_page`, `arch_change_pte`, `arch_cpa_process_fault`, `arch_cpa_process_alias`, and `arch_cpa_flush`. This is a pretty significant change to how we handle page attributes, so fuzzing is definitely a priority here. I need to figure out the right approach. Let's start by identifying the critical functions that are being refactored. Functions that I need to keep a close eye on are: `change_page_attr_set_clr`, `__change_page_attr_set_clr`, `__change_page_attr`, `cpa_handle_large_page`, `should_split_large_page`, `lookup_address_in_pgd_attr`, `lookup_address_in_pgd`, `lookup_address`, and `cpa_addr`. Also the new architecture-specific implementations of: `arch_should_split_large_page`, `arch_split_large_page`, `arch_change_pte`, `arch_cpa_process_fault`, `arch_cpa_process_alias`, and `arch_cpa_flush`. Now, is `CONFIG_GENERIC_SET_MEMORY` something I can enable directly? No, it's selected by the architectures, so it should already be enabled on x86, RISC-V, and Loongarch. However, hang on… it looks like we also have a `CONFIG_CPA_DEBUG` now, available for all architectures that select `GENERIC_SET_MEMORY`. That sounds like a gift! I'll quickly check the Kconfig.debug file in `mm`: ``` config CPA_DEBUG bool "CPA self-test code" depends on GENERIC_SET_MEMORY depends on DEBUG_KERNEL ``` Yes, enabling `CPA_DEBUG` would be a great way to exercise this new code. Let's see if it's already used anywhere, or if I need to add it. I'll search for `CPA_DEBUG` in the kernel and use `set-results` with `WorthFuzzing=true`. For the focus symbols, I'll use those core function names I identified before: `change_page_attr_set_clr`, `__change_page_attr_set_clr`, `__change_page_attr`, `cpa_handle_large_page`, `should_split_large_page`, `arch_should_split_large_page`, `arch_split_large_page`, `arch_change_pte`, `arch_cpa_process_fault`, `arch_cpa_process_alias`, `arch_cpa_flush`, and `lookup_address_in_pgd_attr`. Let me just double-check those names… yep, they all look good. I'll add `lookup_address_in_pgd`, and `lookup_address` while I'm at it. Configs: `CPA_DEBUG` Let me make sure that `CPA_DEBUG` is a valid config. Yes, it's defined in `mm/Kconfig.debug`. Alright, I don't see any other relevant configs, `GENERIC_SET_MEMORY` is selected by the architectures. I'll run `set-results` with those parameters, and hopefully, this will highlight the best areas for focused fuzzing.
4/2	2026/07/22 11:11	tool	set-results	0m Args: { "EnableConfigs": [ "CPA_DEBUG" ], "FocusSymbols": [ "change_page_attr_set_clr", "__change_page_attr_set_clr", "__change_page_attr", "cpa_handle_large_page", "should_split_large_page", "arch_should_split_large_page", "arch_split_large_page", "arch_change_pte", "arch_cpa_process_fault", "arch_cpa_process_alias", "arch_cpa_flush", "lookup_address_in_pgd_attr" ], "Reasoning": "The patch refactors the change_page_attr (CPA) logic, moving it from architecture-specific code (x86, riscv) to a generic implementation in mm/set_memory.c. This is a significant functional change to core kernel memory management. Fuzzing should focus on the new generic functions and their architecture-specific callbacks. Enabling CPA_DEBUG will also run the CPA self-tests to help uncover issues.", "WorthFuzzing": true } Results: { "EnableConfigs": [ "CPA_DEBUG" ], "FocusSymbols": [ "change_page_attr_set_clr", "__change_page_attr_set_clr", "__change_page_attr", "cpa_handle_large_page", "should_split_large_page", "arch_should_split_large_page", "arch_split_large_page", "arch_change_pte", "arch_cpa_process_fault", "arch_cpa_process_alias", "arch_cpa_flush", "lookup_address_in_pgd_attr" ], "Reasoning": "The patch refactors the change_page_attr (CPA) logic, moving it from architecture-specific code (x86, riscv) to a generic implementation in mm/set_memory.c. This is a significant functional change to core kernel memory management. Fuzzing should focus on the new generic functions and their architecture-specific callbacks. Enabling CPA_DEBUG will also run the CPA self-tests to help uncover issues.", "WorthFuzzing": true }

{
  "PatchDiff": "commit d544b2bff4c34ff4e46cc081518a266ab316515c\nAuthor: syz-cluster \u003ctriage@syzkaller.com\u003e\nDate:   Wed Jul 22 11:11:39 2026 +0000\n\n    syz-cluster: applied patch under review\n\ndiff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig\nindex d8d25232501791..1bfa94797b1b4b 100644\n--- a/arch/loongarch/Kconfig\n+++ b/arch/loongarch/Kconfig\n@@ -109,6 +109,7 @@ config LOONGARCH\n \tselect GENERIC_LIB_DEVMEM_IS_ALLOWED\n \tselect GENERIC_PCI_IOMAP\n \tselect GENERIC_SCHED_CLOCK\n+\tselect GENERIC_SET_MEMORY\n \tselect GENERIC_SMP_IDLE_THREAD\n \tselect GPIOLIB\n \tselect HAS_IOPORT\ndiff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h\nindex 223528c04d7392..377bef0894d771 100644\n--- a/arch/loongarch/include/asm/pgtable.h\n+++ b/arch/loongarch/include/asm/pgtable.h\n@@ -638,6 +638,48 @@ static inline long pmd_protnone(pmd_t pmd)\n #define pmd_leaf(pmd)\t\t((pmd_val(pmd) \u0026 _PAGE_HUGE) != 0)\n #define pud_leaf(pud)\t\t((pud_val(pud) \u0026 _PAGE_HUGE) != 0)\n \n+#define pgd_write pgd_write\n+static inline int pgd_write(pgd_t pgd)\n+{\n+\treturn !!(pgd_val(pgd) \u0026 _PAGE_WRITE);\n+}\n+\n+#define p4d_write p4d_write\n+static inline int p4d_write(p4d_t p4d)\n+{\n+\treturn !!(p4d_val(p4d) \u0026 _PAGE_WRITE);\n+}\n+\n+#define pud_write pud_write\n+static inline int pud_write(pud_t pud)\n+{\n+\treturn !!(pud_val(pud) \u0026 _PAGE_WRITE);\n+}\n+\n+#define pgd_exec pgd_exec\n+static inline int pgd_exec(pgd_t pgd)\n+{\n+\treturn !(pgd_val(pgd) \u0026 _PAGE_NO_EXEC);\n+}\n+\n+#define p4d_exec p4d_exec\n+static inline int p4d_exec(p4d_t p4d)\n+{\n+\treturn !(p4d_val(p4d) \u0026 _PAGE_NO_EXEC);\n+}\n+\n+#define pud_exec pud_exec\n+static inline int pud_exec(pud_t pud)\n+{\n+\treturn !(pud_val(pud) \u0026 _PAGE_NO_EXEC);\n+}\n+\n+#define pmd_exec pmd_exec\n+static inline int pmd_exec(pmd_t pmd)\n+{\n+\treturn !(pmd_val(pmd) \u0026 _PAGE_NO_EXEC);\n+}\n+\n /*\n  * We provide our own get_unmapped area to cope with the virtual aliasing\n  * constraints placed on us by the cache architecture.\ndiff --git a/arch/loongarch/include/asm/set_memory.h b/arch/loongarch/include/asm/set_memory.h\nindex 55dfaefd02c8a6..6ffb4fe5a75b0e 100644\n--- a/arch/loongarch/include/asm/set_memory.h\n+++ b/arch/loongarch/include/asm/set_memory.h\n@@ -19,4 +19,13 @@ int set_direct_map_default_noflush(struct page *page);\n int set_direct_map_invalid_noflush(struct page *page);\n int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid);\n \n+/*\n+ * The direct map is a fixed address window that is not backed by page tables,\n+ * so a page never has a second mapping whose protection must be synchronised.\n+ */\n+static inline bool cpa_should_update_alias(unsigned long vaddr, unsigned long pfn)\n+{\n+\treturn false;\n+}\n+\n #endif /* _ASM_LOONGARCH_SET_MEMORY_H */\ndiff --git a/arch/loongarch/mm/pageattr.c b/arch/loongarch/mm/pageattr.c\nindex 614ccc7afccbea..dbbd408bd3d5e6 100644\n--- a/arch/loongarch/mm/pageattr.c\n+++ b/arch/loongarch/mm/pageattr.c\n@@ -4,126 +4,83 @@\n  */\n \n #include \u003clinux/memblock.h\u003e\n-#include \u003clinux/pagewalk.h\u003e\n+#include \u003clinux/mm.h\u003e\n #include \u003clinux/pgtable.h\u003e\n+#include \u003clinux/set_memory.h\u003e\n #include \u003casm/set_memory.h\u003e\n #include \u003casm/tlbflush.h\u003e\n \n-struct pageattr_masks {\n-\tpgprot_t set_mask;\n-\tpgprot_t clear_mask;\n-};\n-\n-static unsigned long set_pageattr_masks(unsigned long val, struct mm_walk *walk)\n+static unsigned long set_pageattr_masks(struct cpa_data *cpa, unsigned long val)\n {\n-\tunsigned long new_val = val;\n-\tstruct pageattr_masks *masks = walk-\u003eprivate;\n-\n-\tnew_val \u0026= ~(pgprot_val(masks-\u003eclear_mask));\n-\tnew_val |= (pgprot_val(masks-\u003eset_mask));\n+\tval \u0026= ~pgprot_val(cpa-\u003emask_clr);\n+\tval |= pgprot_val(cpa-\u003emask_set);\n \n-\treturn new_val;\n+\treturn val;\n }\n \n-static int pageattr_pgd_entry(pgd_t *pgd, unsigned long addr,\n-\t\t\t      unsigned long next, struct mm_walk *walk)\n+int arch_should_split_large_page(struct cpa_data *cpa, struct cpa_split_data *sd)\n {\n-\tpgd_t val = pgdp_get(pgd);\n-\n-\tif (pgd_leaf(val)) {\n-\t\tval = __pgd(set_pageattr_masks(pgd_val(val), walk));\n-\t\tset_pgd(pgd, val);\n-\t}\n-\n-\treturn 0;\n-}\n+\tpte_t old = ptep_get(sd-\u003ekpte);\n+\tunsigned long val = pte_val(old);\n+\tunsigned long new_val = set_pageattr_masks(cpa, val);\n \n-static int pageattr_p4d_entry(p4d_t *p4d, unsigned long addr,\n-\t\t\t      unsigned long next, struct mm_walk *walk)\n-{\n-\tp4d_t val = p4dp_get(p4d);\n+\tcpa-\u003epfn = pte_pfn(old) + ((sd-\u003eaddress \u0026 (sd-\u003epsize - 1)) \u003e\u003e PAGE_SHIFT);\n \n-\tif (p4d_leaf(val)) {\n-\t\tval = __p4d(set_pageattr_masks(p4d_val(val), walk));\n-\t\tset_p4d(p4d, val);\n+\tif (new_val != val) {\n+\t\tset_pte(sd-\u003ekpte, __pte(new_val));\n+\t\tcpa-\u003eflags |= CPA_FLUSHTLB;\n \t}\n \n \treturn 0;\n }\n \n-static int pageattr_pud_entry(pud_t *pud, unsigned long addr,\n-\t\t\t      unsigned long next, struct mm_walk *walk)\n+int arch_split_large_page(struct cpa_data *cpa, struct cpa_split_data *sd)\n {\n-\tpud_t val = pudp_get(pud);\n-\n-\tif (pud_leaf(val)) {\n-\t\tval = __pud(set_pageattr_masks(pud_val(val), walk));\n-\t\tset_pud(pud, val);\n-\t}\n+\tWARN_ON_ONCE(1);\n \n-\treturn 0;\n+\treturn -EINVAL;\n }\n \n-static int pageattr_pmd_entry(pmd_t *pmd, unsigned long addr,\n-\t\t\t      unsigned long next, struct mm_walk *walk)\n+void arch_change_pte(struct cpa_data *cpa, unsigned long address,\n+\t\t     pte_t *kpte, pte_t old_pte, bool nx, bool rw)\n {\n-\tpmd_t val = pmdp_get(pmd);\n+\tunsigned long val = pte_val(old_pte);\n+\tpte_t new_pte = __pte(set_pageattr_masks(cpa, val));\n \n-\tif (pmd_leaf(val)) {\n-\t\tval = __pmd(set_pageattr_masks(pmd_val(val), walk));\n-\t\tset_pmd(pmd, val);\n-\t}\n+\tcpa-\u003epfn = pte_pfn(old_pte);\n \n-\treturn 0;\n+\tif (pte_val(old_pte) != pte_val(new_pte)) {\n+\t\tset_pte(kpte, new_pte);\n+\t\tcpa-\u003eflags |= CPA_FLUSHTLB;\n+\t}\n }\n \n-static int pageattr_pte_entry(pte_t *pte, unsigned long addr,\n-\t\t\t      unsigned long next, struct mm_walk *walk)\n+int arch_cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,\n+\t\t\t   int primary)\n {\n-\tpte_t val = ptep_get(pte);\n-\n-\tval = __pte(set_pageattr_masks(pte_val(val), walk));\n-\tset_pte(pte, val);\n+\t/*\n+\t * The direct map has no page tables and vmalloc ranges may contain\n+\t * holes. Both are silently skipped.\n+\t */\n+\tcpa-\u003enumpages = 1;\n \n \treturn 0;\n }\n \n-static int pageattr_pte_hole(unsigned long addr, unsigned long next,\n-\t\t\t     int depth, struct mm_walk *walk)\n+int arch_cpa_process_alias(struct cpa_data *cpa)\n {\n \treturn 0;\n }\n \n-static const struct mm_walk_ops pageattr_ops = {\n-\t.pgd_entry = pageattr_pgd_entry,\n-\t.p4d_entry = pageattr_p4d_entry,\n-\t.pud_entry = pageattr_pud_entry,\n-\t.pmd_entry = pageattr_pmd_entry,\n-\t.pte_entry = pageattr_pte_entry,\n-\t.pte_hole = pageattr_pte_hole,\n-\t.walk_lock = PGWALK_RDLOCK,\n-};\n-\n-static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask, pgprot_t clear_mask)\n+void arch_cpa_flush(struct cpa_data *cpa, int err)\n {\n-\tint ret;\n-\tunsigned long start = addr;\n-\tunsigned long end = start + PAGE_SIZE * numpages;\n-\tstruct pageattr_masks masks = {\n-\t\t.set_mask = set_mask,\n-\t\t.clear_mask = clear_mask\n-\t};\n-\n-\tif (!numpages)\n-\t\treturn 0;\n-\n-\tmmap_write_lock(\u0026init_mm);\n-\tret = walk_kernel_page_table_range(start, end, \u0026pageattr_ops, NULL, \u0026masks);\n-\tmmap_write_unlock(\u0026init_mm);\n-\n-\tflush_tlb_kernel_range(start, end);\n+\tif (err || cpa-\u003eforce_flush_all ||\n+\t    (cpa-\u003eflags \u0026 (CPA_ARRAY | CPA_PAGES_ARRAY))) {\n+\t\tflush_tlb_all();\n+\t\treturn;\n+\t}\n \n-\treturn ret;\n+\tflush_tlb_kernel_range(*cpa-\u003evaddr, *cpa-\u003evaddr + cpa-\u003enumpages * PAGE_SIZE);\n }\n \n int set_memory_x(unsigned long addr, int numpages)\n@@ -131,7 +88,7 @@ int set_memory_x(unsigned long addr, int numpages)\n \tif (addr \u003c vm_map_base)\n \t\treturn 0;\n \n-\treturn __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_NO_EXEC));\n+\treturn change_page_attr_clear(\u0026addr, numpages, __pgprot(_PAGE_NO_EXEC), 0);\n }\n \n int set_memory_nx(unsigned long addr, int numpages)\n@@ -139,7 +96,7 @@ int set_memory_nx(unsigned long addr, int numpages)\n \tif (addr \u003c vm_map_base)\n \t\treturn 0;\n \n-\treturn __set_memory(addr, numpages, __pgprot(_PAGE_NO_EXEC), __pgprot(0));\n+\treturn change_page_attr_set(\u0026addr, numpages, __pgprot(_PAGE_NO_EXEC), 0);\n }\n \n int set_memory_ro(unsigned long addr, int numpages)\n@@ -147,7 +104,7 @@ int set_memory_ro(unsigned long addr, int numpages)\n \tif (addr \u003c vm_map_base)\n \t\treturn 0;\n \n-\treturn __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_WRITE | _PAGE_DIRTY));\n+\treturn change_page_attr_clear(\u0026addr, numpages, __pgprot(_PAGE_WRITE | _PAGE_DIRTY), 0);\n }\n \n int set_memory_rw(unsigned long addr, int numpages)\n@@ -155,84 +112,75 @@ int set_memory_rw(unsigned long addr, int numpages)\n \tif (addr \u003c vm_map_base)\n \t\treturn 0;\n \n-\treturn __set_memory(addr, numpages, __pgprot(_PAGE_WRITE | _PAGE_DIRTY), __pgprot(0));\n+\treturn change_page_attr_set(\u0026addr, numpages, __pgprot(_PAGE_WRITE | _PAGE_DIRTY), 0);\n }\n \n bool kernel_page_present(struct page *page)\n {\n-\tpgd_t *pgd;\n-\tp4d_t *p4d;\n-\tpud_t *pud;\n-\tpmd_t *pmd;\n-\tpte_t *pte;\n+\tunsigned int level;\n \tunsigned long addr = (unsigned long)page_address(page);\n+\tpte_t *pte;\n \n \tif (addr \u003c vm_map_base)\n \t\treturn memblock_is_memory(__pa(addr));\n \n-\tpgd = pgd_offset_k(addr);\n-\tif (pgd_none(pgdp_get(pgd)))\n-\t\treturn false;\n-\tif (pgd_leaf(pgdp_get(pgd)))\n-\t\treturn true;\n-\n-\tp4d = p4d_offset(pgd, addr);\n-\tif (p4d_none(p4dp_get(p4d)))\n-\t\treturn false;\n-\tif (p4d_leaf(p4dp_get(p4d)))\n-\t\treturn true;\n-\n-\tpud = pud_offset(p4d, addr);\n-\tif (pud_none(pudp_get(pud)))\n-\t\treturn false;\n-\tif (pud_leaf(pudp_get(pud)))\n-\t\treturn true;\n-\n-\tpmd = pmd_offset(pud, addr);\n-\tif (pmd_none(pmdp_get(pmd)))\n+\tpte = lookup_address(addr, \u0026level);\n+\tif (!pte)\n \t\treturn false;\n-\tif (pmd_leaf(pmdp_get(pmd)))\n-\t\treturn true;\n \n-\tpte = pte_offset_kernel(pmd, addr);\n \treturn pte_present(ptep_get(pte));\n }\n \n int set_direct_map_default_noflush(struct page *page)\n {\n \tunsigned long addr = (unsigned long)page_address(page);\n+\tstruct cpa_data cpa = { .vaddr = \u0026addr,\n+\t\t\t\t.pgd = NULL,\n+\t\t\t\t.numpages = 1,\n+\t\t\t\t.mask_set = PAGE_KERNEL,\n+\t\t\t\t.mask_clr = __pgprot(0),\n+\t\t\t\t.flags = CPA_NO_CHECK_ALIAS };\n \n \tif (addr \u003c vm_map_base)\n \t\treturn 0;\n \n-\treturn __set_memory(addr, 1, PAGE_KERNEL, __pgprot(0));\n+\treturn __change_page_attr_set_clr(\u0026cpa, 1);\n }\n \n int set_direct_map_invalid_noflush(struct page *page)\n {\n \tunsigned long addr = (unsigned long)page_address(page);\n+\tstruct cpa_data cpa = { .vaddr = \u0026addr,\n+\t\t\t\t.pgd = NULL,\n+\t\t\t\t.numpages = 1,\n+\t\t\t\t.mask_set = __pgprot(0),\n+\t\t\t\t.mask_clr = __pgprot(_PAGE_PRESENT | _PAGE_VALID),\n+\t\t\t\t.flags = CPA_NO_CHECK_ALIAS };\n \n \tif (addr \u003c vm_map_base)\n \t\treturn 0;\n \n-\treturn __set_memory(addr, 1, __pgprot(0), __pgprot(_PAGE_PRESENT | _PAGE_VALID));\n+\treturn __change_page_attr_set_clr(\u0026cpa, 1);\n }\n \n int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid)\n {\n \tunsigned long addr = (unsigned long)page_address(page);\n-\tpgprot_t set, clear;\n+\tstruct cpa_data cpa = { .vaddr = \u0026addr,\n+\t\t\t\t.pgd = NULL,\n+\t\t\t\t.numpages = nr,\n+\t\t\t\t.flags = CPA_NO_CHECK_ALIAS };\n \n \tif (addr \u003c vm_map_base)\n \t\treturn 0;\n \n \tif (valid) {\n-\t\tset = PAGE_KERNEL;\n-\t\tclear = __pgprot(0);\n+\t\tcpa.mask_set = PAGE_KERNEL;\n+\t\tcpa.mask_clr = __pgprot(0);\n \t} else {\n-\t\tset = __pgprot(0);\n-\t\tclear = __pgprot(_PAGE_PRESENT | _PAGE_VALID);\n+\t\tcpa.mask_set = __pgprot(0);\n+\t\tcpa.mask_clr = __pgprot(_PAGE_PRESENT | _PAGE_VALID);\n \t}\n \n-\treturn __set_memory(addr, nr, set, clear);\n+\treturn __change_page_attr_set_clr(\u0026cpa, 1);\n }\ndiff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig\nindex 3f0a647218e407..5d069bf290ce78 100644\n--- a/arch/riscv/Kconfig\n+++ b/arch/riscv/Kconfig\n@@ -48,6 +48,7 @@ config RISCV\n \tselect ARCH_HAS_PTE_SPECIAL\n \tselect ARCH_HAS_SET_DIRECT_MAP if MMU\n \tselect ARCH_HAS_SET_MEMORY if MMU\n+\tselect GENERIC_SET_MEMORY if MMU\n \tselect ARCH_HAS_STRICT_KERNEL_RWX if MMU\n \tselect ARCH_HAS_STRICT_MODULE_RWX if MMU\n \tselect ARCH_HAS_SYNC_CORE_BEFORE_USERMODE\ndiff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h\nindex 5d5756bda82e3d..027b3e7e4f177d 100644\n--- a/arch/riscv/include/asm/pgtable.h\n+++ b/arch/riscv/include/asm/pgtable.h\n@@ -819,6 +819,42 @@ static inline int pud_write(pud_t pud)\n \treturn pte_write(pud_pte(pud));\n }\n \n+#define pgd_write pgd_write\n+static inline int pgd_write(pgd_t pgd)\n+{\n+\treturn pgd_val(pgd) \u0026 _PAGE_WRITE;\n+}\n+\n+#define p4d_write p4d_write\n+static inline int p4d_write(p4d_t p4d)\n+{\n+\treturn p4d_val(p4d) \u0026 _PAGE_WRITE;\n+}\n+\n+#define pgd_exec pgd_exec\n+static inline int pgd_exec(pgd_t pgd)\n+{\n+\treturn !!(pgd_val(pgd) \u0026 _PAGE_EXEC);\n+}\n+\n+#define p4d_exec p4d_exec\n+static inline int p4d_exec(p4d_t p4d)\n+{\n+\treturn !!(p4d_val(p4d) \u0026 _PAGE_EXEC);\n+}\n+\n+#define pud_exec pud_exec\n+static inline int pud_exec(pud_t pud)\n+{\n+\treturn !!(pud_val(pud) \u0026 _PAGE_EXEC);\n+}\n+\n+#define pmd_exec pmd_exec\n+static inline int pmd_exec(pmd_t pmd)\n+{\n+\treturn !!(pmd_val(pmd) \u0026 _PAGE_EXEC);\n+}\n+\n #define pmd_dirty pmd_dirty\n static inline int pmd_dirty(pmd_t pmd)\n {\ndiff --git a/arch/riscv/include/asm/set_memory.h b/arch/riscv/include/asm/set_memory.h\nindex ef59e1716a2cfd..4ba46a33103433 100644\n--- a/arch/riscv/include/asm/set_memory.h\n+++ b/arch/riscv/include/asm/set_memory.h\n@@ -7,6 +7,8 @@\n #define _ASM_RISCV_SET_MEMORY_H\n \n #ifndef __ASSEMBLER__\n+#include \u003casm/page.h\u003e\n+#include \u003casm/pgtable.h\u003e\n /*\n  * Functions to change memory attributes.\n  */\n@@ -26,6 +28,11 @@ static __always_inline int set_kernel_memory(char *startp, char *endp,\n \n \treturn set_memory(start, num_pages);\n }\n+\n+static inline bool cpa_should_update_alias(unsigned long vaddr, unsigned long pfn)\n+{\n+\treturn !is_linear_mapping(vaddr);\n+}\n #else\n static inline int set_memory_ro(unsigned long addr, int numpages) { return 0; }\n static inline int set_memory_rw(unsigned long addr, int numpages) { return 0; }\ndiff --git a/arch/riscv/mm/pageattr.c b/arch/riscv/mm/pageattr.c\nindex 3f76db3d276992..66a20fc3d80574 100644\n--- a/arch/riscv/mm/pageattr.c\n+++ b/arch/riscv/mm/pageattr.c\n@@ -3,402 +3,242 @@\n  * Copyright (C) 2019 SiFive\n  */\n \n-#include \u003clinux/pagewalk.h\u003e\n+#include \u003clinux/mm.h\u003e\n #include \u003clinux/pgtable.h\u003e\n-#include \u003clinux/vmalloc.h\u003e\n+#include \u003clinux/set_memory.h\u003e\n #include \u003casm/tlbflush.h\u003e\n-#include \u003casm/bitops.h\u003e\n #include \u003casm/set_memory.h\u003e\n \n-struct pageattr_masks {\n-\tpgprot_t set_mask;\n-\tpgprot_t clear_mask;\n-};\n-\n-static unsigned long set_pageattr_masks(unsigned long val, struct mm_walk *walk)\n-{\n-\tstruct pageattr_masks *masks = walk-\u003eprivate;\n-\tunsigned long new_val = val;\n-\n-\tnew_val \u0026= ~(pgprot_val(masks-\u003eclear_mask));\n-\tnew_val |= (pgprot_val(masks-\u003eset_mask));\n-\n-\treturn new_val;\n-}\n-\n-static int pageattr_p4d_entry(p4d_t *p4d, unsigned long addr,\n-\t\t\t      unsigned long next, struct mm_walk *walk)\n+int arch_should_split_large_page(struct cpa_data *cpa, struct cpa_split_data *sd)\n {\n-\tp4d_t val = p4dp_get(p4d);\n-\n-\tif (p4d_leaf(val)) {\n-\t\tval = __p4d(set_pageattr_masks(p4d_val(val), walk));\n-\t\tset_p4d(p4d, val);\n-\t}\n+\tpte_t old = ptep_get(sd-\u003ekpte);\n+\tpgprot_t old_prot = __pgprot(pte_val(old) \u0026 ~_PAGE_PFN_MASK);\n+\tpgprot_t new_prot = old_prot;\n+\tunsigned long old_pfn = pte_pfn(old);\n+\tunsigned long lpaddr, numpages;\n \n-\treturn 0;\n-}\n+\tpgprot_val(new_prot) \u0026= ~pgprot_val(cpa-\u003emask_clr);\n+\tpgprot_val(new_prot) |= pgprot_val(cpa-\u003emask_set);\n \n-static int pageattr_pud_entry(pud_t *pud, unsigned long addr,\n-\t\t\t      unsigned long next, struct mm_walk *walk)\n-{\n-\tpud_t val = pudp_get(pud);\n-\n-\tif (pud_leaf(val)) {\n-\t\tval = __pud(set_pageattr_masks(pud_val(val), walk));\n-\t\tset_pud(pud, val);\n-\t}\n+\t/*\n+\t * Record the pfn mapped at @address so the alias check can locate the\n+\t * matching linear map entry.\n+\t */\n+\tcpa-\u003epfn = old_pfn + ((sd-\u003eaddress \u0026 (sd-\u003epsize - 1)) \u003e\u003e PAGE_SHIFT);\n \n-\treturn 0;\n-}\n+\t/* If the protections do not change, keep the large page intact. */\n+\tif (pgprot_val(new_prot) == pgprot_val(old_prot))\n+\t\treturn 0;\n \n-static int pageattr_pmd_entry(pmd_t *pmd, unsigned long addr,\n-\t\t\t      unsigned long next, struct mm_walk *walk)\n-{\n-\tpmd_t val = pmdp_get(pmd);\n+\t/* If the request does not cover the whole large page, split it. */\n+\tlpaddr = sd-\u003eaddress \u0026 sd-\u003epmask;\n+\tnumpages = sd-\u003epsize \u003e\u003e PAGE_SHIFT;\n+\tif (sd-\u003eaddress != lpaddr || cpa-\u003enumpages != numpages)\n+\t\treturn 1;\n \n-\tif (pmd_leaf(val)) {\n-\t\tval = __pmd(set_pageattr_masks(pmd_val(val), walk));\n-\t\tset_pmd(pmd, val);\n-\t}\n+\t/* The whole large page is covered: update it in place. */\n+\tset_pte(sd-\u003ekpte, pfn_pte(old_pfn, new_prot));\n+\tcpa-\u003eflags |= CPA_FLUSHTLB;\n \n \treturn 0;\n }\n \n-static int pageattr_pte_entry(pte_t *pte, unsigned long addr,\n-\t\t\t      unsigned long next, struct mm_walk *walk)\n+static void split_set_ptes(pte_t *ptep, unsigned long pfn, unsigned long pfninc,\n+\t\t\t   pgprot_t prot)\n {\n-\tpte_t val = ptep_get(pte);\n-\n-\tval = __pte(set_pageattr_masks(pte_val(val), walk));\n-\tset_pte(pte, val);\n+\tunsigned int i;\n \n-\treturn 0;\n+\tfor (i = 0; i \u003c PTRS_PER_PTE; ++i, ++ptep, pfn += pfninc)\n+\t\tset_pte(ptep, pfn_pte(pfn, prot));\n }\n \n-static int pageattr_pte_hole(unsigned long addr, unsigned long next,\n-\t\t\t     int depth, struct mm_walk *walk)\n+int arch_split_large_page(struct cpa_data *cpa, struct cpa_split_data *sd)\n {\n-\t/* Nothing to do here */\n-\treturn 0;\n-}\n-\n-static const struct mm_walk_ops pageattr_ops = {\n-\t.p4d_entry = pageattr_p4d_entry,\n-\t.pud_entry = pageattr_pud_entry,\n-\t.pmd_entry = pageattr_pmd_entry,\n-\t.pte_entry = pageattr_pte_entry,\n-\t.pte_hole = pageattr_pte_hole,\n-\t.walk_lock = PGWALK_RDLOCK,\n-};\n-\n #ifdef CONFIG_64BIT\n-static int __split_linear_mapping_pmd(pud_t *pudp,\n-\t\t\t\t      unsigned long vaddr, unsigned long end)\n-{\n-\tpmd_t *pmdp;\n-\tunsigned long next;\n-\n-\tpmdp = pmd_offset(pudp, vaddr);\n-\n-\tdo {\n-\t\tnext = pmd_addr_end(vaddr, end);\n-\n-\t\tif (next - vaddr \u003e= PMD_SIZE \u0026\u0026\n-\t\t    vaddr \u003c= (vaddr \u0026 PMD_MASK) \u0026\u0026 end \u003e= next)\n-\t\t\tcontinue;\n-\n-\t\tif (pmd_leaf(pmdp_get(pmdp))) {\n-\t\t\tstruct page *pte_page;\n-\t\t\tunsigned long pfn = _pmd_pfn(pmdp_get(pmdp));\n-\t\t\tpgprot_t prot = __pgprot(pmd_val(pmdp_get(pmdp)) \u0026 ~_PAGE_PFN_MASK);\n-\t\t\tpte_t *ptep_new;\n-\t\t\tint i;\n-\n-\t\t\tpte_page = alloc_page(GFP_KERNEL);\n-\t\t\tif (!pte_page)\n-\t\t\t\treturn -ENOMEM;\n+\tstruct page *base = ptdesc_page(sd-\u003eptdesc);\n+\tpte_t old = ptep_get(sd-\u003ekpte);\n+\tpgprot_t prot = __pgprot(pte_val(old) \u0026 ~_PAGE_PFN_MASK);\n+\tunsigned long pfn = pte_pfn(old);\n+\tunsigned long pfninc;\n+\n+\tswitch (sd-\u003elevel) {\n+\tcase PGTABLE_LEVEL_PMD:\n+\t\tpfninc = 1;\n+\t\tbreak;\n+\tcase PGTABLE_LEVEL_PUD:\n+\t\tpfninc = PMD_SIZE \u003e\u003e PAGE_SHIFT;\n+\t\tbreak;\n+\tcase PGTABLE_LEVEL_P4D:\n+\t\tpfninc = PUD_SIZE \u003e\u003e PAGE_SHIFT;\n+\t\tbreak;\n+\tdefault:\n+\t\treturn -EINVAL;\n+\t}\n \n-\t\t\tptep_new = (pte_t *)page_address(pte_page);\n-\t\t\tfor (i = 0; i \u003c PTRS_PER_PTE; ++i, ++ptep_new)\n-\t\t\t\tset_pte(ptep_new, pfn_pte(pfn + i, prot));\n+\tsplit_set_ptes((pte_t *)page_address(base), pfn, pfninc, prot);\n \n-\t\t\tsmp_wmb();\n+\tsmp_wmb();\n+\tset_pte(sd-\u003ekpte, pfn_pte(page_to_pfn(base), PAGE_TABLE));\n \n-\t\t\tset_pmd(pmdp, pfn_pmd(page_to_pfn(pte_page), PAGE_TABLE));\n-\t\t}\n-\t} while (pmdp++, vaddr = next, vaddr != end);\n+\tcpa-\u003eflags |= CPA_FLUSHTLB;\n+\tcpa-\u003eforce_flush_all = 1;\n \n \treturn 0;\n+#else\n+\tWARN_ON_ONCE(1);\n+\treturn -EINVAL;\n+#endif\n }\n \n-static int __split_linear_mapping_pud(p4d_t *p4dp,\n-\t\t\t\t      unsigned long vaddr, unsigned long end)\n+void arch_change_pte(struct cpa_data *cpa, unsigned long address,\n+\t\t     pte_t *kpte, pte_t old_pte, bool nx, bool rw)\n {\n-\tpud_t *pudp;\n-\tunsigned long next;\n-\tint ret;\n-\n-\tpudp = pud_offset(p4dp, vaddr);\n-\n-\tdo {\n-\t\tnext = pud_addr_end(vaddr, end);\n-\n-\t\tif (next - vaddr \u003e= PUD_SIZE \u0026\u0026\n-\t\t    vaddr \u003c= (vaddr \u0026 PUD_MASK) \u0026\u0026 end \u003e= next)\n-\t\t\tcontinue;\n+\tpgprot_t new_prot = __pgprot(pte_val(old_pte) \u0026 ~_PAGE_PFN_MASK);\n+\tunsigned long pfn = pte_pfn(old_pte);\n+\tpte_t new_pte;\n \n-\t\tif (pud_leaf(pudp_get(pudp))) {\n-\t\t\tstruct page *pmd_page;\n-\t\t\tunsigned long pfn = _pud_pfn(pudp_get(pudp));\n-\t\t\tpgprot_t prot = __pgprot(pud_val(pudp_get(pudp)) \u0026 ~_PAGE_PFN_MASK);\n-\t\t\tpmd_t *pmdp_new;\n-\t\t\tint i;\n+\tpgprot_val(new_prot) \u0026= ~pgprot_val(cpa-\u003emask_clr);\n+\tpgprot_val(new_prot) |= pgprot_val(cpa-\u003emask_set);\n \n-\t\t\tpmd_page = alloc_page(GFP_KERNEL);\n-\t\t\tif (!pmd_page)\n-\t\t\t\treturn -ENOMEM;\n+\tnew_pte = pfn_pte(pfn, new_prot);\n+\tcpa-\u003epfn = pfn;\n \n-\t\t\tpmdp_new = (pmd_t *)page_address(pmd_page);\n-\t\t\tfor (i = 0; i \u003c PTRS_PER_PMD; ++i, ++pmdp_new)\n-\t\t\t\tset_pmd(pmdp_new,\n-\t\t\t\t\tpfn_pmd(pfn + ((i * PMD_SIZE) \u003e\u003e PAGE_SHIFT), prot));\n-\n-\t\t\tsmp_wmb();\n-\n-\t\t\tset_pud(pudp, pfn_pud(page_to_pfn(pmd_page), PAGE_TABLE));\n-\t\t}\n-\n-\t\tret = __split_linear_mapping_pmd(pudp, vaddr, next);\n-\t\tif (ret)\n-\t\t\treturn ret;\n-\t} while (pudp++, vaddr = next, vaddr != end);\n-\n-\treturn 0;\n+\tif (pte_val(old_pte) != pte_val(new_pte)) {\n+\t\tset_pte(kpte, new_pte);\n+\t\tcpa-\u003eflags |= CPA_FLUSHTLB;\n+\t}\n }\n \n-static int __split_linear_mapping_p4d(pgd_t *pgdp,\n-\t\t\t\t      unsigned long vaddr, unsigned long end)\n+int arch_cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,\n+\t\t\t   int primary)\n {\n-\tp4d_t *p4dp;\n-\tunsigned long next;\n-\tint ret;\n-\n-\tp4dp = p4d_offset(pgdp, vaddr);\n-\n-\tdo {\n-\t\tnext = p4d_addr_end(vaddr, end);\n-\n-\t\t/*\n-\t\t * If [vaddr; end] contains [vaddr \u0026 P4D_MASK; next], we don't\n-\t\t * need to split, we'll change the protections on the whole P4D.\n-\t\t */\n-\t\tif (next - vaddr \u003e= P4D_SIZE \u0026\u0026\n-\t\t    vaddr \u003c= (vaddr \u0026 P4D_MASK) \u0026\u0026 end \u003e= next)\n-\t\t\tcontinue;\n-\n-\t\tif (p4d_leaf(p4dp_get(p4dp))) {\n-\t\t\tstruct page *pud_page;\n-\t\t\tunsigned long pfn = _p4d_pfn(p4dp_get(p4dp));\n-\t\t\tpgprot_t prot = __pgprot(p4d_val(p4dp_get(p4dp)) \u0026 ~_PAGE_PFN_MASK);\n-\t\t\tpud_t *pudp_new;\n-\t\t\tint i;\n-\n-\t\t\tpud_page = alloc_page(GFP_KERNEL);\n-\t\t\tif (!pud_page)\n-\t\t\t\treturn -ENOMEM;\n-\n-\t\t\t/*\n-\t\t\t * Fill the pud level with leaf puds that have the same\n-\t\t\t * protections as the leaf p4d.\n-\t\t\t */\n-\t\t\tpudp_new = (pud_t *)page_address(pud_page);\n-\t\t\tfor (i = 0; i \u003c PTRS_PER_PUD; ++i, ++pudp_new)\n-\t\t\t\tset_pud(pudp_new,\n-\t\t\t\t\tpfn_pud(pfn + ((i * PUD_SIZE) \u003e\u003e PAGE_SHIFT), prot));\n-\n-\t\t\t/*\n-\t\t\t * Make sure the pud filling is not reordered with the\n-\t\t\t * p4d store which could result in seeing a partially\n-\t\t\t * filled pud level.\n-\t\t\t */\n-\t\t\tsmp_wmb();\n-\n-\t\t\tset_p4d(p4dp, pfn_p4d(page_to_pfn(pud_page), PAGE_TABLE));\n-\t\t}\n-\n-\t\tret = __split_linear_mapping_pud(p4dp, vaddr, next);\n-\t\tif (ret)\n-\t\t\treturn ret;\n-\t} while (p4dp++, vaddr = next, vaddr != end);\n+\tcpa-\u003enumpages = 1;\n \n-\treturn 0;\n-}\n-\n-static int __split_linear_mapping_pgd(pgd_t *pgdp,\n-\t\t\t\t      unsigned long vaddr,\n-\t\t\t\t      unsigned long end)\n-{\n-\tunsigned long next;\n-\tint ret;\n+\tif (!primary)\n+\t\treturn 0;\n \n-\tdo {\n-\t\tnext = pgd_addr_end(vaddr, end);\n-\t\t/* We never use PGD mappings for the linear mapping */\n-\t\tret = __split_linear_mapping_p4d(pgdp, vaddr, next);\n-\t\tif (ret)\n-\t\t\treturn ret;\n-\t} while (pgdp++, vaddr = next, vaddr != end);\n+\t/* The linear map is expected to have holes */\n+\tif (is_linear_mapping(vaddr)) {\n+\t\tcpa-\u003epfn = PFN_DOWN(__pa(vaddr));\n+\t\treturn 0;\n+\t}\n \n-\treturn 0;\n-}\n+\tWARN(1, \"CPA: called for zero pte. vaddr = %lx cpa-\u003evaddr = %lx\\n\",\n+\t     vaddr, *cpa-\u003evaddr);\n \n-static int split_linear_mapping(unsigned long start, unsigned long end)\n-{\n-\treturn __split_linear_mapping_pgd(pgd_offset_k(start), start, end);\n+\treturn -EFAULT;\n }\n-#endif\t/* CONFIG_64BIT */\n \n-static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask,\n-\t\t\tpgprot_t clear_mask)\n+int arch_cpa_process_alias(struct cpa_data *cpa)\n {\n-\tint ret;\n-\tunsigned long start = addr;\n-\tunsigned long end = start + PAGE_SIZE * numpages;\n-\tunsigned long __maybe_unused lm_start;\n-\tunsigned long __maybe_unused lm_end;\n-\tstruct pageattr_masks masks = {\n-\t\t.set_mask = set_mask,\n-\t\t.clear_mask = clear_mask\n-\t};\n-\n-\tif (!numpages)\n-\t\treturn 0;\n+\tstruct cpa_data alias_cpa;\n+\tunsigned long laddr;\n \n-\tmmap_write_lock(\u0026init_mm);\n-\n-#ifdef CONFIG_64BIT\n \t/*\n-\t * We are about to change the permissions of a kernel mapping, we must\n-\t * apply the same changes to its linear mapping alias, which may imply\n-\t * splitting a huge mapping.\n+\t * cpa_should_update_alias() only lets non linear map primaries reach\n+\t * here, so @cpa-\u003epfn always has a linear map alias that must receive\n+\t * the same protection change.\n \t */\n+\tladdr = (unsigned long)__va(PFN_PHYS(cpa-\u003epfn));\n \n-\tif (is_vmalloc_or_module_addr((void *)start)) {\n-\t\tstruct vm_struct *area = NULL;\n-\t\tint i, page_start;\n-\n-\t\tarea = find_vm_area((void *)start);\n-\t\tpage_start = (start - (unsigned long)area-\u003eaddr) \u003e\u003e PAGE_SHIFT;\n-\n-\t\tfor (i = page_start; i \u003c page_start + numpages; ++i) {\n-\t\t\tlm_start = (unsigned long)page_address(area-\u003epages[i]);\n-\t\t\tlm_end = lm_start + PAGE_SIZE;\n-\n-\t\t\tret = split_linear_mapping(lm_start, lm_end);\n-\t\t\tif (ret)\n-\t\t\t\tgoto unlock;\n-\n-\t\t\tret = walk_kernel_page_table_range(lm_start, lm_end,\n-\t\t\t\t\t\t    \u0026pageattr_ops, NULL, \u0026masks);\n-\t\t\tif (ret)\n-\t\t\t\tgoto unlock;\n-\t\t}\n-\t} else if (is_kernel_mapping(start) || is_linear_mapping(start)) {\n-\t\tif (is_kernel_mapping(start)) {\n-\t\t\tlm_start = (unsigned long)lm_alias(start);\n-\t\t\tlm_end = (unsigned long)lm_alias(end);\n-\t\t} else {\n-\t\t\tlm_start = start;\n-\t\t\tlm_end = end;\n-\t\t}\n-\n-\t\tret = split_linear_mapping(lm_start, lm_end);\n-\t\tif (ret)\n-\t\t\tgoto unlock;\n-\n-\t\tret = walk_kernel_page_table_range(lm_start, lm_end,\n-\t\t\t\t\t    \u0026pageattr_ops, NULL, \u0026masks);\n-\t\tif (ret)\n-\t\t\tgoto unlock;\n-\t}\n-\n-\tret =  walk_kernel_page_table_range(start, end, \u0026pageattr_ops, NULL,\n-\t\t\t\t     \u0026masks);\n+\talias_cpa = *cpa;\n+\talias_cpa.vaddr = \u0026laddr;\n+\talias_cpa.flags \u0026= ~(CPA_PAGES_ARRAY | CPA_ARRAY);\n+\talias_cpa.curpage = 0;\n \n-unlock:\n-\tmmap_write_unlock(\u0026init_mm);\n+\t/* The linear map alias must never be made executable */\n+\talias_cpa.mask_set = __pgprot(pgprot_val(alias_cpa.mask_set) \u0026 ~_PAGE_EXEC);\n+\talias_cpa.mask_clr = __pgprot(pgprot_val(alias_cpa.mask_clr) \u0026 ~_PAGE_EXEC);\n \n-\t/*\n-\t * We can't use flush_tlb_kernel_range() here as we may have split a\n-\t * hugepage that is larger than that, so let's flush everything.\n-\t */\n-\tflush_tlb_all();\n-#else\n-\tret =  walk_kernel_page_table_range(start, end, \u0026pageattr_ops, NULL,\n-\t\t\t\t     \u0026masks);\n+\tcpa-\u003eforce_flush_all = 1;\n \n-\tmmap_write_unlock(\u0026init_mm);\n+\treturn __change_page_attr_set_clr(\u0026alias_cpa, 0);\n+}\n \n-\tflush_tlb_kernel_range(start, end);\n-#endif\n+void arch_cpa_flush(struct cpa_data *cpa, int err)\n+{\n+\tif (err || cpa-\u003eforce_flush_all ||\n+\t    (cpa-\u003eflags \u0026 (CPA_ARRAY | CPA_PAGES_ARRAY))) {\n+\t\tflush_tlb_all();\n+\t\treturn;\n+\t}\n \n-\treturn ret;\n+\tflush_tlb_kernel_range(*cpa-\u003evaddr,\n+\t\t\t       *cpa-\u003evaddr + cpa-\u003enumpages * PAGE_SIZE);\n }\n \n int set_memory_rw_nx(unsigned long addr, int numpages)\n {\n-\treturn __set_memory(addr, numpages, __pgprot(_PAGE_READ | _PAGE_WRITE),\n-\t\t\t    __pgprot(_PAGE_EXEC));\n+\treturn change_page_attr_set_clr(\u0026addr, numpages,\n+\t\t\t\t\t__pgprot(_PAGE_READ | _PAGE_WRITE),\n+\t\t\t\t\t__pgprot(_PAGE_EXEC), 0, 0, NULL);\n }\n \n int set_memory_ro(unsigned long addr, int numpages)\n {\n-\treturn __set_memory(addr, numpages, __pgprot(_PAGE_READ),\n-\t\t\t    __pgprot(_PAGE_WRITE));\n+\treturn change_page_attr_set_clr(\u0026addr, numpages, __pgprot(_PAGE_READ),\n+\t\t\t\t\t__pgprot(_PAGE_WRITE), 0, 0, NULL);\n }\n \n int set_memory_rw(unsigned long addr, int numpages)\n {\n-\treturn __set_memory(addr, numpages, __pgprot(_PAGE_READ | _PAGE_WRITE),\n-\t\t\t    __pgprot(0));\n+\treturn change_page_attr_set(\u0026addr, numpages,\n+\t\t\t\t    __pgprot(_PAGE_READ | _PAGE_WRITE), 0);\n }\n \n int set_memory_x(unsigned long addr, int numpages)\n {\n-\treturn __set_memory(addr, numpages, __pgprot(_PAGE_EXEC), __pgprot(0));\n+\treturn change_page_attr_set(\u0026addr, numpages, __pgprot(_PAGE_EXEC), 0);\n }\n \n int set_memory_nx(unsigned long addr, int numpages)\n {\n-\treturn __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_EXEC));\n+\treturn change_page_attr_clear(\u0026addr, numpages, __pgprot(_PAGE_EXEC), 0);\n }\n \n int set_direct_map_invalid_noflush(struct page *page)\n {\n-\treturn __set_memory((unsigned long)page_address(page), 1,\n-\t\t\t    __pgprot(0), __pgprot(_PAGE_PRESENT));\n+\tunsigned long start = (unsigned long)page_address(page);\n+\tstruct cpa_data cpa = { .vaddr = \u0026start,\n+\t\t\t\t.pgd = NULL,\n+\t\t\t\t.numpages = 1,\n+\t\t\t\t.mask_set = __pgprot(0),\n+\t\t\t\t.mask_clr = __pgprot(_PAGE_PRESENT),\n+\t\t\t\t.flags = CPA_NO_CHECK_ALIAS };\n+\n+\treturn __change_page_attr_set_clr(\u0026cpa, 1);\n }\n \n int set_direct_map_default_noflush(struct page *page)\n {\n-\treturn __set_memory((unsigned long)page_address(page), 1,\n-\t\t\t    PAGE_KERNEL, __pgprot(_PAGE_EXEC));\n+\tunsigned long start = (unsigned long)page_address(page);\n+\tstruct cpa_data cpa = { .vaddr = \u0026start,\n+\t\t\t\t.pgd = NULL,\n+\t\t\t\t.numpages = 1,\n+\t\t\t\t.mask_set = PAGE_KERNEL,\n+\t\t\t\t.mask_clr = __pgprot(_PAGE_EXEC),\n+\t\t\t\t.flags = CPA_NO_CHECK_ALIAS };\n+\n+\treturn __change_page_attr_set_clr(\u0026cpa, 1);\n }\n \n int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid)\n {\n-\tpgprot_t set, clear;\n+\tunsigned long start = (unsigned long)page_address(page);\n+\tstruct cpa_data cpa = { .vaddr = \u0026start,\n+\t\t\t\t.pgd = NULL,\n+\t\t\t\t.numpages = nr,\n+\t\t\t\t.flags = CPA_NO_CHECK_ALIAS };\n \n \tif (valid) {\n-\t\tset = PAGE_KERNEL;\n-\t\tclear = __pgprot(_PAGE_EXEC);\n+\t\tcpa.mask_set = PAGE_KERNEL;\n+\t\tcpa.mask_clr = __pgprot(_PAGE_EXEC);\n \t} else {\n-\t\tset = __pgprot(0);\n-\t\tclear = __pgprot(_PAGE_PRESENT);\n+\t\tcpa.mask_set = __pgprot(0);\n+\t\tcpa.mask_clr = __pgprot(_PAGE_PRESENT);\n \t}\n \n-\treturn __set_memory((unsigned long)page_address(page), nr, set, clear);\n+\treturn __change_page_attr_set_clr(\u0026cpa, 1);\n }\n \n #ifdef CONFIG_DEBUG_PAGEALLOC\n@@ -435,36 +275,11 @@ void __kernel_map_pages(struct page *page, int numpages, int enable)\n bool kernel_page_present(struct page *page)\n {\n \tunsigned long addr = (unsigned long)page_address(page);\n-\tpgd_t *pgd;\n-\tpud_t *pud;\n-\tp4d_t *p4d;\n-\tpmd_t *pmd;\n-\tpte_t *pte;\n-\n-\tpgd = pgd_offset_k(addr);\n-\tif (!pgd_present(pgdp_get(pgd)))\n-\t\treturn false;\n-\tif (pgd_leaf(pgdp_get(pgd)))\n-\t\treturn true;\n-\n-\tp4d = p4d_offset(pgd, addr);\n-\tif (!p4d_present(p4dp_get(p4d)))\n-\t\treturn false;\n-\tif (p4d_leaf(p4dp_get(p4d)))\n-\t\treturn true;\n-\n-\tpud = pud_offset(p4d, addr);\n-\tif (!pud_present(pudp_get(pud)))\n-\t\treturn false;\n-\tif (pud_leaf(pudp_get(pud)))\n-\t\treturn true;\n+\tunsigned int level;\n+\tpte_t *pte = lookup_address(addr, \u0026level);\n \n-\tpmd = pmd_offset(pud, addr);\n-\tif (!pmd_present(pmdp_get(pmd)))\n+\tif (!pte)\n \t\treturn false;\n-\tif (pmd_leaf(pmdp_get(pmd)))\n-\t\treturn true;\n \n-\tpte = pte_offset_kernel(pmd, addr);\n \treturn pte_present(ptep_get(pte));\n }\ndiff --git a/arch/x86/Kconfig b/arch/x86/Kconfig\nindex bdad90f210e4bf..e1f105e52d6a12 100644\n--- a/arch/x86/Kconfig\n+++ b/arch/x86/Kconfig\n@@ -336,6 +336,7 @@ config X86\n \tselect ARCH_SUPPORTS_SCHED_CLUSTER\tif SMP\n \tselect ARCH_SUPPORTS_SCHED_MC\t\tif SMP\n \tselect HAVE_SINGLE_FTRACE_DIRECT_OPS\tif X86_64 \u0026\u0026 DYNAMIC_FTRACE_WITH_DIRECT_CALLS\n+\tselect GENERIC_SET_MEMORY\n \n config INSTRUCTION_DECODER\n \tdef_bool y\ndiff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug\nindex c95c3aaadf97c3..314b72aad6fff4 100644\n--- a/arch/x86/Kconfig.debug\n+++ b/arch/x86/Kconfig.debug\n@@ -157,12 +157,6 @@ config DEBUG_BOOT_PARAMS\n \thelp\n \t  This option will cause struct boot_params to be exported via debugfs.\n \n-config CPA_DEBUG\n-\tbool \"CPA self-test code\"\n-\tdepends on DEBUG_KERNEL\n-\thelp\n-\t  Do change_page_attr() self-tests every 30 seconds.\n-\n config DEBUG_ENTRY\n \tbool \"Debug low-level entry code\"\n \tdepends on DEBUG_KERNEL\ndiff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h\nindex ac295ca6c92f99..497aaf08ad3616 100644\n--- a/arch/x86/include/asm/pgtable.h\n+++ b/arch/x86/include/asm/pgtable.h\n@@ -227,6 +227,42 @@ static inline int pud_write(pud_t pud)\n \treturn pud_flags(pud) \u0026 _PAGE_RW;\n }\n \n+#define p4d_write p4d_write\n+static inline int p4d_write(p4d_t p4d)\n+{\n+\treturn p4d_flags(p4d) \u0026 _PAGE_RW;\n+}\n+\n+#define pgd_write pgd_write\n+static inline int pgd_write(pgd_t pgd)\n+{\n+\treturn pgd_flags(pgd) \u0026 _PAGE_RW;\n+}\n+\n+#define pmd_exec pmd_exec\n+static inline int pmd_exec(pmd_t pmd)\n+{\n+\treturn !(pmd_flags(pmd) \u0026 _PAGE_NX);\n+}\n+\n+#define pud_exec pud_exec\n+static inline int pud_exec(pud_t pud)\n+{\n+\treturn !(pud_flags(pud) \u0026 _PAGE_NX);\n+}\n+\n+#define p4d_exec p4d_exec\n+static inline int p4d_exec(p4d_t p4d)\n+{\n+\treturn !(p4d_flags(p4d) \u0026 _PAGE_NX);\n+}\n+\n+#define pgd_exec pgd_exec\n+static inline int pgd_exec(pgd_t pgd)\n+{\n+\treturn !(pgd_flags(pgd) \u0026 _PAGE_NX);\n+}\n+\n static inline int pte_huge(pte_t pte)\n {\n \treturn pte_flags(pte) \u0026 _PAGE_PSE;\ndiff --git a/arch/x86/include/asm/set_memory.h b/arch/x86/include/asm/set_memory.h\nindex 4362c26aa992db..332ec300c2ed9c 100644\n--- a/arch/x86/include/asm/set_memory.h\n+++ b/arch/x86/include/asm/set_memory.h\n@@ -3,12 +3,62 @@\n #define _ASM_X86_SET_MEMORY_H\n \n #include \u003casm/page.h\u003e\n+#include \u003casm/sections.h\u003e\n #include \u003casm-generic/set_memory.h\u003e\n #include \u003casm/pgtable.h\u003e\n \n #define set_memory_rox set_memory_rox\n int set_memory_rox(unsigned long addr, int numpages);\n \n+#ifdef CONFIG_X86_64\n+/*\n+ * The kernel image is mapped into two places in the virtual address space\n+ * (addresses without KASLR, of course):\n+ *\n+ * 1. The kernel direct map (0xffff880000000000)\n+ * 2. The \"high kernel map\" (0xffffffff81000000)\n+ *\n+ * We actually execute out of #2. If we get the address of a kernel symbol, it\n+ * points to #2, but almost all physical-to-virtual translations point to #1.\n+ *\n+ * This is so that we can have both a directmap of all physical memory *and*\n+ * take full advantage of the limited (s32) immediate addressing range (2G)\n+ * of x86_64.\n+ *\n+ * See Documentation/arch/x86/x86_64/mm.rst for more detail.\n+ */\n+static inline bool pfn_is_kernel(unsigned long pfn)\n+{\n+\tunsigned long spfn = __pa_symbol(_text) \u003e\u003e PAGE_SHIFT;\n+\t/* Do not reference a physical address outside the kernel. */\n+\tunsigned long epfn = __pa_symbol(roundup(_brk_end, PMD_SIZE) - 1) \u003e\u003e PAGE_SHIFT;\n+\n+\treturn pfn \u003e= spfn \u0026\u0026 pfn \u003c= epfn;\n+}\n+#else\n+static inline bool pfn_is_kernel(unsigned long pfn)\n+{\n+\t/* There is no highmap on 32-bit */\n+\treturn false;\n+}\n+#endif\n+\n+static inline bool cpa_should_update_alias(unsigned long vaddr,\n+\t\t\t\t\t   unsigned long pfn)\n+{\n+\t/* Primary is not in the direct map, its direct map alias needs update */\n+\tif (vaddr \u003c PAGE_OFFSET ||\n+\t    vaddr \u003e= PAGE_OFFSET + (max_pfn_mapped \u003c\u003c PAGE_SHIFT))\n+\t\treturn true;\n+\n+\t/* direct map page that is also part of the kernel highmap */\n+\tif ((vaddr \u003c (unsigned long)_text || vaddr \u003e= _brk_end) \u0026\u0026\n+\t    pfn_is_kernel(pfn))\n+\t\treturn true;\n+\n+\treturn false;\n+}\n+\n /*\n  * The set_memory_* API can be used to change various attributes of a virtual\n  * address range. The attributes include:\ndiff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c\nindex d023a40a1e0345..c460121c96b949 100644\n--- a/arch/x86/mm/pat/set_memory.c\n+++ b/arch/x86/mm/pat/set_memory.c\n@@ -38,21 +38,6 @@\n /*\n  * The current flushing context - we pass it instead of 5 arguments:\n  */\n-struct cpa_data {\n-\tunsigned long\t*vaddr;\n-\tpgd_t\t\t*pgd;\n-\tpgprot_t\tmask_set;\n-\tpgprot_t\tmask_clr;\n-\tunsigned long\tnumpages;\n-\tunsigned long\tcurpage;\n-\tunsigned long\tpfn;\n-\tunsigned int\tflags;\n-\tunsigned int\tforce_split\t\t: 1,\n-\t\t\tforce_static_prot\t: 1,\n-\t\t\tforce_flush_all\t\t: 1;\n-\tstruct page\t**pages;\n-};\n-\n enum cpa_warn {\n \tCPA_CONFLICT,\n \tCPA_PROTECT,\n@@ -61,27 +46,13 @@ enum cpa_warn {\n \n static const int cpa_warn_level = CPA_PROTECT;\n \n-/*\n- * Serialize cpa() (for !DEBUG_PAGEALLOC which uses large identity mappings)\n- * using cpa_lock. So that we don't allow any other cpu, with stale large tlb\n- * entries change the page attribute in parallel to some other cpu\n- * splitting a large page entry along with changing the attribute.\n- */\n-static DEFINE_SPINLOCK(cpa_lock);\n-\n-#define CPA_FLUSHTLB 1\n-#define CPA_ARRAY 2\n-#define CPA_PAGES_ARRAY 4\n-#define CPA_NO_CHECK_ALIAS 8 /* Do not search for aliases */\n-#define CPA_COLLAPSE 16 /* try to collapse large pages */\n-\n static inline pgprot_t cachemode2pgprot(enum page_cache_mode pcm)\n {\n \treturn __pgprot(cachemode2protval(pcm));\n }\n \n #ifdef CONFIG_PROC_FS\n-static unsigned long direct_pages_count[PG_LEVEL_NUM];\n+static unsigned long direct_pages_count[PGTABLE_LEVEL_NUM];\n \n void update_page_count(int level, unsigned long pages)\n {\n@@ -98,9 +69,9 @@ static void split_page_count(int level)\n \n \tdirect_pages_count[level]--;\n \tif (system_state == SYSTEM_RUNNING) {\n-\t\tif (level == PG_LEVEL_2M)\n+\t\tif (level == PGTABLE_LEVEL_PMD)\n \t\t\tcount_vm_event(DIRECT_MAP_LEVEL2_SPLIT);\n-\t\telse if (level == PG_LEVEL_1G)\n+\t\telse if (level == PGTABLE_LEVEL_PUD)\n \t\t\tcount_vm_event(DIRECT_MAP_LEVEL3_SPLIT);\n \t}\n \tdirect_pages_count[level - 1] += PTRS_PER_PTE;\n@@ -110,9 +81,9 @@ static void collapse_page_count(int level)\n {\n \tdirect_pages_count[level]++;\n \tif (system_state == SYSTEM_RUNNING) {\n-\t\tif (level == PG_LEVEL_2M)\n+\t\tif (level == PGTABLE_LEVEL_PMD)\n \t\t\tcount_vm_event(DIRECT_MAP_LEVEL2_COLLAPSE);\n-\t\telse if (level == PG_LEVEL_1G)\n+\t\telse if (level == PGTABLE_LEVEL_PUD)\n \t\t\tcount_vm_event(DIRECT_MAP_LEVEL3_COLLAPSE);\n \t}\n \tdirect_pages_count[level - 1] -= PTRS_PER_PTE;\n@@ -121,17 +92,17 @@ static void collapse_page_count(int level)\n void arch_report_meminfo(struct seq_file *m)\n {\n \tseq_printf(m, \"DirectMap4k:    %8lu kB\\n\",\n-\t\t\tdirect_pages_count[PG_LEVEL_4K] \u003c\u003c 2);\n+\t\t\tdirect_pages_count[PGTABLE_LEVEL_PTE] \u003c\u003c 2);\n #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)\n \tseq_printf(m, \"DirectMap2M:    %8lu kB\\n\",\n-\t\t\tdirect_pages_count[PG_LEVEL_2M] \u003c\u003c 11);\n+\t\t\tdirect_pages_count[PGTABLE_LEVEL_PMD] \u003c\u003c 11);\n #else\n \tseq_printf(m, \"DirectMap4M:    %8lu kB\\n\",\n-\t\t\tdirect_pages_count[PG_LEVEL_2M] \u003c\u003c 12);\n+\t\t\tdirect_pages_count[PGTABLE_LEVEL_PMD] \u003c\u003c 12);\n #endif\n \tif (direct_gbpages)\n \t\tseq_printf(m, \"DirectMap1G:    %8lu kB\\n\",\n-\t\t\tdirect_pages_count[PG_LEVEL_1G] \u003c\u003c 20);\n+\t\t\tdirect_pages_count[PGTABLE_LEVEL_PUD] \u003c\u003c 20);\n }\n #else\n static inline void split_page_count(int level) { }\n@@ -165,7 +136,7 @@ static inline void cpa_inc_4k_install(void)\n \n static inline void cpa_inc_lp_sameprot(int level)\n {\n-\tif (level == PG_LEVEL_1G)\n+\tif (level == PGTABLE_LEVEL_PUD)\n \t\tcpa_1g_sameprot++;\n \telse\n \t\tcpa_2m_sameprot++;\n@@ -173,7 +144,7 @@ static inline void cpa_inc_lp_sameprot(int level)\n \n static inline void cpa_inc_lp_preserved(int level)\n {\n-\tif (level == PG_LEVEL_1G)\n+\tif (level == PGTABLE_LEVEL_PUD)\n \t\tcpa_1g_preserved++;\n \telse\n \t\tcpa_2m_preserved++;\n@@ -225,61 +196,6 @@ within(unsigned long addr, unsigned long start, unsigned long end)\n \treturn addr \u003e= start \u0026\u0026 addr \u003c end;\n }\n \n-#ifdef CONFIG_X86_64\n-\n-static inline int\n-within_inclusive(unsigned long addr, unsigned long start, unsigned long end)\n-{\n-\treturn addr \u003e= start \u0026\u0026 addr \u003c= end;\n-}\n-\n-/*\n- * The kernel image is mapped into two places in the virtual address space\n- * (addresses without KASLR, of course):\n- *\n- * 1. The kernel direct map (0xffff880000000000)\n- * 2. The \"high kernel map\" (0xffffffff81000000)\n- *\n- * We actually execute out of #2. If we get the address of a kernel symbol, it\n- * points to #2, but almost all physical-to-virtual translations point to #1.\n- *\n- * This is so that we can have both a directmap of all physical memory *and*\n- * take full advantage of the limited (s32) immediate addressing range (2G)\n- * of x86_64.\n- *\n- * See Documentation/arch/x86/x86_64/mm.rst for more detail.\n- */\n-\n-static inline unsigned long highmap_start_pfn(void)\n-{\n-\treturn __pa_symbol(_text) \u003e\u003e PAGE_SHIFT;\n-}\n-\n-static inline unsigned long highmap_end_pfn(void)\n-{\n-\t/* Do not reference physical address outside the kernel. */\n-\treturn __pa_symbol(roundup(_brk_end, PMD_SIZE) - 1) \u003e\u003e PAGE_SHIFT;\n-}\n-\n-static bool __cpa_pfn_in_highmap(unsigned long pfn)\n-{\n-\t/*\n-\t * Kernel text has an alias mapping at a high address, known\n-\t * here as \"highmap\".\n-\t */\n-\treturn within_inclusive(pfn, highmap_start_pfn(), highmap_end_pfn());\n-}\n-\n-#else\n-\n-static bool __cpa_pfn_in_highmap(unsigned long pfn)\n-{\n-\t/* There is no highmap on 32-bit */\n-\treturn false;\n-}\n-\n-#endif\n-\n /*\n  * See set_mce_nospec().\n  *\n@@ -303,23 +219,6 @@ static inline unsigned long fix_addr(unsigned long addr)\n #endif\n }\n \n-static unsigned long __cpa_addr(struct cpa_data *cpa, unsigned long idx)\n-{\n-\tif (cpa-\u003eflags \u0026 CPA_PAGES_ARRAY) {\n-\t\tstruct page *page = cpa-\u003epages[idx];\n-\n-\t\tif (unlikely(PageHighMem(page)))\n-\t\t\treturn 0;\n-\n-\t\treturn (unsigned long)page_address(page);\n-\t}\n-\n-\tif (cpa-\u003eflags \u0026 CPA_ARRAY)\n-\t\treturn cpa-\u003evaddr[idx];\n-\n-\treturn *cpa-\u003evaddr + idx * PAGE_SIZE;\n-}\n-\n /*\n  * Flushing functions\n  */\n@@ -405,7 +304,7 @@ static void __cpa_flush_tlb(void *data)\n \tunsigned int i;\n \n \tfor (i = 0; i \u003c cpa-\u003enumpages; i++)\n-\t\tflush_tlb_one_kernel(fix_addr(__cpa_addr(cpa, i)));\n+\t\tflush_tlb_one_kernel(fix_addr(cpa_addr(cpa, i)));\n }\n \n static int collapse_large_pages(unsigned long addr, struct list_head *pgtables);\n@@ -420,10 +319,10 @@ static void cpa_collapse_large_pages(struct cpa_data *cpa)\n \n \tif (cpa-\u003eflags \u0026 (CPA_PAGES_ARRAY | CPA_ARRAY)) {\n \t\tfor (i = 0; i \u003c cpa-\u003enumpages; i++)\n-\t\t\tcollapsed += collapse_large_pages(__cpa_addr(cpa, i),\n+\t\t\tcollapsed += collapse_large_pages(cpa_addr(cpa, i),\n \t\t\t\t\t\t\t  \u0026pgtables);\n \t} else {\n-\t\taddr = __cpa_addr(cpa, 0);\n+\t\taddr = cpa_addr(cpa, 0);\n \t\tstart = addr \u0026 PMD_MASK;\n \t\tend = addr + PAGE_SIZE * cpa-\u003enumpages;\n \n@@ -463,7 +362,7 @@ static void cpa_flush(struct cpa_data *cpa, int cache)\n \n \tmb();\n \tfor (i = 0; i \u003c cpa-\u003enumpages; i++) {\n-\t\tunsigned long addr = __cpa_addr(cpa, i);\n+\t\tunsigned long addr = cpa_addr(cpa, i);\n \t\tunsigned int level;\n \n \t\tpte_t *pte = lookup_address(addr, \u0026level);\n@@ -578,7 +477,7 @@ static pgprotval_t protect_kernel_text_ro(unsigned long start,\n \t * so the protections for kernel text and identity mappings have to\n \t * be the same.\n \t */\n-\tif (lookup_address(start, \u0026level) \u0026\u0026 (level != PG_LEVEL_4K))\n+\tif (lookup_address(start, \u0026level) \u0026\u0026 (level != PGTABLE_LEVEL_PTE))\n \t\treturn _PAGE_RW;\n \treturn 0;\n }\n@@ -709,104 +608,6 @@ static inline pgprot_t verify_rwx(pgprot_t old, pgprot_t new, unsigned long star\n \treturn new;\n }\n \n-/*\n- * Lookup the page table entry for a virtual address in a specific pgd.\n- * Return a pointer to the entry (or NULL if the entry does not exist),\n- * the level of the entry, and the effective NX and RW bits of all\n- * page table levels.\n- */\n-pte_t *lookup_address_in_pgd_attr(pgd_t *pgd, unsigned long address,\n-\t\t\t\t  unsigned int *level, bool *nx, bool *rw)\n-{\n-\tp4d_t *p4d;\n-\tpud_t *pud;\n-\tpmd_t *pmd;\n-\n-\t*level = PG_LEVEL_256T;\n-\t*nx = false;\n-\t*rw = true;\n-\n-\tif (pgd_none(*pgd))\n-\t\treturn NULL;\n-\n-\t*level = PG_LEVEL_512G;\n-\t*nx |= pgd_flags(*pgd) \u0026 _PAGE_NX;\n-\t*rw \u0026= pgd_flags(*pgd) \u0026 _PAGE_RW;\n-\n-\tp4d = p4d_offset(pgd, address);\n-\tif (p4d_none(*p4d))\n-\t\treturn NULL;\n-\n-\tif (p4d_leaf(*p4d) || !p4d_present(*p4d))\n-\t\treturn (pte_t *)p4d;\n-\n-\t*level = PG_LEVEL_1G;\n-\t*nx |= p4d_flags(*p4d) \u0026 _PAGE_NX;\n-\t*rw \u0026= p4d_flags(*p4d) \u0026 _PAGE_RW;\n-\n-\tpud = pud_offset(p4d, address);\n-\tif (pud_none(*pud))\n-\t\treturn NULL;\n-\n-\tif (pud_leaf(*pud) || !pud_present(*pud))\n-\t\treturn (pte_t *)pud;\n-\n-\t*level = PG_LEVEL_2M;\n-\t*nx |= pud_flags(*pud) \u0026 _PAGE_NX;\n-\t*rw \u0026= pud_flags(*pud) \u0026 _PAGE_RW;\n-\n-\tpmd = pmd_offset(pud, address);\n-\tif (pmd_none(*pmd))\n-\t\treturn NULL;\n-\n-\tif (pmd_leaf(*pmd) || !pmd_present(*pmd))\n-\t\treturn (pte_t *)pmd;\n-\n-\t*level = PG_LEVEL_4K;\n-\t*nx |= pmd_flags(*pmd) \u0026 _PAGE_NX;\n-\t*rw \u0026= pmd_flags(*pmd) \u0026 _PAGE_RW;\n-\n-\treturn pte_offset_kernel(pmd, address);\n-}\n-\n-/*\n- * Lookup the page table entry for a virtual address in a specific pgd.\n- * Return a pointer to the entry and the level of the mapping.\n- */\n-pte_t *lookup_address_in_pgd(pgd_t *pgd, unsigned long address,\n-\t\t\t     unsigned int *level)\n-{\n-\tbool nx, rw;\n-\n-\treturn lookup_address_in_pgd_attr(pgd, address, level, \u0026nx, \u0026rw);\n-}\n-\n-/*\n- * Lookup the page table entry for a virtual address. Return a pointer\n- * to the entry and the level of the mapping.\n- *\n- * Note: the function returns p4d, pud or pmd either when the entry is marked\n- * large or when the present bit is not set. Otherwise it returns NULL.\n- */\n-pte_t *lookup_address(unsigned long address, unsigned int *level)\n-{\n-\treturn lookup_address_in_pgd(pgd_offset_k(address), address, level);\n-}\n-EXPORT_SYMBOL_GPL(lookup_address);\n-\n-static pte_t *_lookup_address_cpa(struct cpa_data *cpa, unsigned long address,\n-\t\t\t\t  unsigned int *level, bool *nx, bool *rw)\n-{\n-\tpgd_t *pgd;\n-\n-\tif (!cpa-\u003epgd)\n-\t\tpgd = pgd_offset_k(address);\n-\telse\n-\t\tpgd = cpa-\u003epgd + pgd_index(address);\n-\n-\treturn lookup_address_in_pgd_attr(pgd, address, level, nx, rw);\n-}\n-\n /*\n  * Lookup the PMD entry for a virtual address. Return a pointer to the entry\n  * or NULL if not present.\n@@ -850,9 +651,9 @@ pmd_t *lookup_pmd_address(unsigned long address)\n phys_addr_t slow_virt_to_phys(void *__virt_addr)\n {\n \tunsigned long virt_addr = (unsigned long)__virt_addr;\n+\tenum pgtable_level level;\n \tphys_addr_t phys_addr;\n \tunsigned long offset;\n-\tenum pg_level level;\n \tpte_t *pte;\n \n \tpte = lookup_address(virt_addr, \u0026level);\n@@ -864,11 +665,11 @@ phys_addr_t slow_virt_to_phys(void *__virt_addr)\n \t * make 32-PAE kernel work correctly.\n \t */\n \tswitch (level) {\n-\tcase PG_LEVEL_1G:\n+\tcase PGTABLE_LEVEL_PUD:\n \t\tphys_addr = (phys_addr_t)pud_pfn(*(pud_t *)pte) \u003c\u003c PAGE_SHIFT;\n \t\toffset = virt_addr \u0026 ~PUD_MASK;\n \t\tbreak;\n-\tcase PG_LEVEL_2M:\n+\tcase PGTABLE_LEVEL_PMD:\n \t\tphys_addr = (phys_addr_t)pmd_pfn(*(pmd_t *)pte) \u003c\u003c PAGE_SHIFT;\n \t\toffset = virt_addr \u0026 ~PMD_MASK;\n \t\tbreak;\n@@ -925,30 +726,27 @@ static pgprot_t pgprot_clear_protnone_bits(pgprot_t prot)\n \treturn prot;\n }\n \n-static int __should_split_large_page(pte_t *kpte, unsigned long address,\n-\t\t\t\t     struct cpa_data *cpa)\n+int arch_should_split_large_page(struct cpa_data *cpa,\n+\t\t\t\t struct cpa_split_data *sd)\n {\n-\tunsigned long numpages, pmask, psize, lpaddr, pfn, old_pfn;\n \tpgprot_t old_prot, new_prot, req_prot, chk_prot;\n-\tpte_t new_pte, *tmp;\n-\tenum pg_level level;\n-\tbool nx, rw;\n-\n-\t/*\n-\t * Check for races, another CPU might have split this page\n-\t * up already:\n-\t */\n-\ttmp = _lookup_address_cpa(cpa, address, \u0026level, \u0026nx, \u0026rw);\n-\tif (tmp != kpte)\n-\t\treturn 1;\n+\tunsigned long numpages, lpaddr, pfn, old_pfn;\n+\tenum pgtable_level level = sd-\u003elevel;\n+\tunsigned long address = sd-\u003eaddress;\n+\tunsigned long psize = sd-\u003epsize;\n+\tunsigned long pmask = sd-\u003epmask;\n+\tpte_t *kpte = sd-\u003ekpte;\n+\tbool nx = sd-\u003enx;\n+\tbool rw = sd-\u003erw;\n+\tpte_t new_pte;\n \n \tswitch (level) {\n-\tcase PG_LEVEL_2M:\n+\tcase PGTABLE_LEVEL_PMD:\n \t\told_prot = pmd_pgprot(*(pmd_t *)kpte);\n \t\told_pfn = pmd_pfn(*(pmd_t *)kpte);\n \t\tcpa_inc_2m_checked();\n \t\tbreak;\n-\tcase PG_LEVEL_1G:\n+\tcase PGTABLE_LEVEL_PUD:\n \t\told_prot = pud_pgprot(*(pud_t *)kpte);\n \t\told_pfn = pud_pfn(*(pud_t *)kpte);\n \t\tcpa_inc_1g_checked();\n@@ -957,18 +755,6 @@ static int __should_split_large_page(pte_t *kpte, unsigned long address,\n \t\treturn -EINVAL;\n \t}\n \n-\tpsize = page_level_size(level);\n-\tpmask = page_level_mask(level);\n-\n-\t/*\n-\t * Calculate the number of pages, which fit into this large\n-\t * page starting at address:\n-\t */\n-\tlpaddr = (address + psize) \u0026 pmask;\n-\tnumpages = (lpaddr - address) \u003e\u003e PAGE_SHIFT;\n-\tif (numpages \u003c cpa-\u003enumpages)\n-\t\tcpa-\u003enumpages = numpages;\n-\n \t/*\n \t * We are safe now. Check whether the new pgprot is the same:\n \t * Convert protection attributes to 4k-format, as cpa-\u003emask* are set\n@@ -1072,21 +858,6 @@ static int __should_split_large_page(pte_t *kpte, unsigned long address,\n \treturn 0;\n }\n \n-static int should_split_large_page(pte_t *kpte, unsigned long address,\n-\t\t\t\t   struct cpa_data *cpa)\n-{\n-\tint do_split;\n-\n-\tif (cpa-\u003eforce_split)\n-\t\treturn 1;\n-\n-\tspin_lock(\u0026pgd_lock);\n-\tdo_split = __should_split_large_page(kpte, address, cpa);\n-\tspin_unlock(\u0026pgd_lock);\n-\n-\treturn do_split;\n-}\n-\n static void split_set_pte(struct cpa_data *cpa, pte_t *pte, unsigned long pfn,\n \t\t\t  pgprot_t ref_prot, unsigned long address,\n \t\t\t  unsigned long size)\n@@ -1123,33 +894,22 @@ static void split_set_pte(struct cpa_data *cpa, pte_t *pte, unsigned long pfn,\n \tset_pte(pte, pfn_pte(pfn, ref_prot));\n }\n \n-static int\n-__split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,\n-\t\t   struct ptdesc *ptdesc)\n+int arch_split_large_page(struct cpa_data *cpa, struct cpa_split_data *sd)\n {\n \tunsigned long lpaddr, lpinc, ref_pfn, pfn, pfninc = 1;\n+\tstruct ptdesc *ptdesc = sd-\u003eptdesc;\n \tstruct page *base = ptdesc_page(ptdesc);\n \tpte_t *pbase = (pte_t *)page_address(base);\n-\tunsigned int i, level;\n+\tenum pgtable_level level = sd-\u003elevel;\n+\tunsigned long address = sd-\u003eaddress;\n+\tpte_t *kpte = sd-\u003ekpte;\n+\tunsigned int i;\n \tpgprot_t ref_prot;\n-\tbool nx, rw;\n-\tpte_t *tmp;\n-\n-\tspin_lock(\u0026pgd_lock);\n-\t/*\n-\t * Check for races, another CPU might have split this page\n-\t * up for us already:\n-\t */\n-\ttmp = _lookup_address_cpa(cpa, address, \u0026level, \u0026nx, \u0026rw);\n-\tif (tmp != kpte) {\n-\t\tspin_unlock(\u0026pgd_lock);\n-\t\treturn 1;\n-\t}\n \n \tparavirt_alloc_pte(\u0026init_mm, page_to_pfn(base));\n \n \tswitch (level) {\n-\tcase PG_LEVEL_2M:\n+\tcase PGTABLE_LEVEL_PMD:\n \t\tref_prot = pmd_pgprot(*(pmd_t *)kpte);\n \t\t/*\n \t\t * Clear PSE (aka _PAGE_PAT) and move\n@@ -1161,7 +921,7 @@ __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,\n \t\tlpinc = PAGE_SIZE;\n \t\tbreak;\n \n-\tcase PG_LEVEL_1G:\n+\tcase PGTABLE_LEVEL_PUD:\n \t\tref_prot = pud_pgprot(*(pud_t *)kpte);\n \t\tref_pfn = pud_pfn(*(pud_t *)kpte);\n \t\tpfninc = PMD_SIZE \u003e\u003e PAGE_SHIFT;\n@@ -1177,7 +937,6 @@ __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,\n \t\tbreak;\n \n \tdefault:\n-\t\tspin_unlock(\u0026pgd_lock);\n \t\treturn 1;\n \t}\n \n@@ -1225,26 +984,6 @@ __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,\n \t * just split large page entry.\n \t */\n \tflush_tlb_all();\n-\tspin_unlock(\u0026pgd_lock);\n-\n-\treturn 0;\n-}\n-\n-static int split_large_page(struct cpa_data *cpa, pte_t *kpte,\n-\t\t\t    unsigned long address)\n-{\n-\tstruct ptdesc *ptdesc;\n-\n-\tif (!debug_pagealloc_enabled())\n-\t\tspin_unlock(\u0026cpa_lock);\n-\tptdesc = pagetable_alloc(GFP_KERNEL, 0);\n-\tif (!debug_pagealloc_enabled())\n-\t\tspin_lock(\u0026cpa_lock);\n-\tif (!ptdesc)\n-\t\treturn -ENOMEM;\n-\n-\tif (__split_large_page(cpa, kpte, address, ptdesc))\n-\t\tpagetable_free(ptdesc);\n \n \treturn 0;\n }\n@@ -1314,7 +1053,7 @@ static int collapse_pmd_page(pmd_t *pmd, unsigned long addr,\n \t}\n \n \tif (virt_addr_valid(addr) \u0026\u0026 pfn_range_is_mapped(pfn, pfn + 1))\n-\t\tcollapse_page_count(PG_LEVEL_2M);\n+\t\tcollapse_page_count(PGTABLE_LEVEL_PMD);\n \n \treturn 1;\n }\n@@ -1361,7 +1100,7 @@ static int collapse_pud_page(pud_t *pud, unsigned long addr,\n \tset_pud(pud, pfn_pud(pfn, pmd_pgprot(first)));\n \n \tif (virt_addr_valid(addr) \u0026\u0026 pfn_range_is_mapped(pfn, pfn + 1))\n-\t\tcollapse_page_count(PG_LEVEL_1G);\n+\t\tcollapse_page_count(PGTABLE_LEVEL_PUD);\n \n \treturn 1;\n }\n@@ -1800,8 +1539,8 @@ static int populate_pgd(struct cpa_data *cpa, unsigned long addr)\n \treturn 0;\n }\n \n-static int __cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,\n-\t\t\t       int primary)\n+int arch_cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,\n+\t\t\t   int primary)\n {\n \tif (cpa-\u003epgd) {\n \t\t/*\n@@ -1833,7 +1572,7 @@ static int __cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,\n \t\tcpa-\u003epfn = __pa(vaddr) \u003e\u003e PAGE_SHIFT;\n \t\treturn 0;\n \n-\t} else if (__cpa_pfn_in_highmap(cpa-\u003epfn)) {\n+\t} else if (pfn_is_kernel(cpa-\u003epfn)) {\n \t\t/* Faults in the highmap are OK, so do not warn: */\n \t\treturn -EFAULT;\n \t} else {\n@@ -1845,90 +1584,45 @@ static int __cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,\n \t}\n }\n \n-static int __change_page_attr(struct cpa_data *cpa, int primary)\n+void arch_change_pte(struct cpa_data *cpa, unsigned long address,\n+\t\t     pte_t *kpte, pte_t old_pte, bool nx, bool rw)\n {\n-\tunsigned long address;\n-\tint do_split, err;\n-\tunsigned int level;\n-\tpte_t *kpte, old_pte;\n-\tbool nx, rw;\n-\n-\taddress = __cpa_addr(cpa, cpa-\u003ecurpage);\n-repeat:\n-\tkpte = _lookup_address_cpa(cpa, address, \u0026level, \u0026nx, \u0026rw);\n-\tif (!kpte)\n-\t\treturn __cpa_process_fault(cpa, address, primary);\n+\tpte_t new_pte;\n+\tpgprot_t old_prot = pte_pgprot(old_pte);\n+\tpgprot_t new_prot = pte_pgprot(old_pte);\n+\tunsigned long pfn = pte_pfn(old_pte);\n \n-\told_pte = *kpte;\n-\tif (pte_none(old_pte))\n-\t\treturn __cpa_process_fault(cpa, address, primary);\n+\tpgprot_val(new_prot) \u0026= ~pgprot_val(cpa-\u003emask_clr);\n+\tpgprot_val(new_prot) |= pgprot_val(cpa-\u003emask_set);\n \n-\tif (level == PG_LEVEL_4K) {\n-\t\tpte_t new_pte;\n-\t\tpgprot_t old_prot = pte_pgprot(old_pte);\n-\t\tpgprot_t new_prot = pte_pgprot(old_pte);\n-\t\tunsigned long pfn = pte_pfn(old_pte);\n-\n-\t\tpgprot_val(new_prot) \u0026= ~pgprot_val(cpa-\u003emask_clr);\n-\t\tpgprot_val(new_prot) |= pgprot_val(cpa-\u003emask_set);\n-\n-\t\tcpa_inc_4k_install();\n-\t\t/* Hand in lpsize = 0 to enforce the protection mechanism */\n-\t\tnew_prot = static_protections(new_prot, address, pfn, 1, 0,\n-\t\t\t\t\t      CPA_PROTECT);\n-\n-\t\tnew_prot = verify_rwx(old_prot, new_prot, address, pfn, 1,\n-\t\t\t\t      nx, rw);\n+\tcpa_inc_4k_install();\n+\t/* Hand in lpsize = 0 to enforce the protection mechanism */\n+\tnew_prot = static_protections(new_prot, address, pfn, 1, 0,\n+\t\t\t\t      CPA_PROTECT);\n \n-\t\tnew_prot = pgprot_clear_protnone_bits(new_prot);\n+\tnew_prot = verify_rwx(old_prot, new_prot, address, pfn, 1, nx, rw);\n \n-\t\t/*\n-\t\t * We need to keep the pfn from the existing PTE,\n-\t\t * after all we're only going to change its attributes\n-\t\t * not the memory it points to\n-\t\t */\n-\t\tnew_pte = pfn_pte(pfn, new_prot);\n-\t\tcpa-\u003epfn = pfn;\n-\t\t/*\n-\t\t * Do we really change anything ?\n-\t\t */\n-\t\tif (pte_val(old_pte) != pte_val(new_pte)) {\n-\t\t\tset_pte_atomic(kpte, new_pte);\n-\t\t\tcpa-\u003eflags |= CPA_FLUSHTLB;\n-\t\t}\n-\t\tcpa-\u003enumpages = 1;\n-\t\treturn 0;\n-\t}\n+\tnew_prot = pgprot_clear_protnone_bits(new_prot);\n \n \t/*\n-\t * Check, whether we can keep the large page intact\n-\t * and just change the pte:\n-\t */\n-\tdo_split = should_split_large_page(kpte, address, cpa);\n-\t/*\n-\t * When the range fits into the existing large page,\n-\t * return. cp-\u003enumpages and cpa-\u003etlbflush have been updated in\n-\t * try_large_page:\n+\t * We need to keep the pfn from the existing PTE, after all we're only\n+\t * going to change its attributes not the memory it points to\n \t */\n-\tif (do_split \u003c= 0)\n-\t\treturn do_split;\n-\n+\tnew_pte = pfn_pte(pfn, new_prot);\n+\tcpa-\u003epfn = pfn;\n \t/*\n-\t * We have to split the large page:\n+\t * Do we really change anything ?\n \t */\n-\terr = split_large_page(cpa, kpte, address);\n-\tif (!err)\n-\t\tgoto repeat;\n-\n-\treturn err;\n+\tif (pte_val(old_pte) != pte_val(new_pte)) {\n+\t\tset_pte_atomic(kpte, new_pte);\n+\t\tcpa-\u003eflags |= CPA_FLUSHTLB;\n+\t}\n }\n \n-static int __change_page_attr_set_clr(struct cpa_data *cpa, int primary);\n-\n /*\n  * Check the directmap and \"high kernel map\" 'aliases'.\n  */\n-static int cpa_process_alias(struct cpa_data *cpa)\n+int arch_cpa_process_alias(struct cpa_data *cpa)\n {\n \tstruct cpa_data alias_cpa;\n \tunsigned long laddr = (unsigned long)__va(cpa-\u003epfn \u003c\u003c PAGE_SHIFT);\n@@ -1942,7 +1636,7 @@ static int cpa_process_alias(struct cpa_data *cpa)\n \t * No need to redo, when the primary call touched the direct\n \t * mapping already:\n \t */\n-\tvaddr = __cpa_addr(cpa, cpa-\u003ecurpage);\n+\tvaddr = cpa_addr(cpa, cpa-\u003ecurpage);\n \tif (!(within(vaddr, PAGE_OFFSET,\n \t\t    PAGE_OFFSET + (max_pfn_mapped \u003c\u003c PAGE_SHIFT)))) {\n \n@@ -1971,7 +1665,7 @@ static int cpa_process_alias(struct cpa_data *cpa)\n \t * to touch the high mapped kernel as well:\n \t */\n \tif (!within(vaddr, (unsigned long)_text, _brk_end) \u0026\u0026\n-\t    __cpa_pfn_in_highmap(cpa-\u003epfn)) {\n+\t    pfn_is_kernel(cpa-\u003epfn)) {\n \t\tunsigned long temp_cpa_vaddr = (cpa-\u003epfn \u003c\u003c PAGE_SHIFT) +\n \t\t\t\t\t       __START_KERNEL_map - phys_base;\n \t\talias_cpa = *cpa;\n@@ -2000,168 +1694,25 @@ static int cpa_process_alias(struct cpa_data *cpa)\n \treturn 0;\n }\n \n-static int __change_page_attr_set_clr(struct cpa_data *cpa, int primary)\n-{\n-\tunsigned long numpages = cpa-\u003enumpages;\n-\tunsigned long rempages = numpages;\n-\tint ret = 0;\n-\n-\t/*\n-\t * No changes, easy!\n-\t */\n-\tif (!(pgprot_val(cpa-\u003emask_set) | pgprot_val(cpa-\u003emask_clr)) \u0026\u0026\n-\t    !cpa-\u003eforce_split)\n-\t\treturn ret;\n-\n-\twhile (rempages) {\n-\t\t/*\n-\t\t * Store the remaining nr of pages for the large page\n-\t\t * preservation check.\n-\t\t */\n-\t\tcpa-\u003enumpages = rempages;\n-\t\t/* for array changes, we can't use large page */\n-\t\tif (cpa-\u003eflags \u0026 (CPA_ARRAY | CPA_PAGES_ARRAY))\n-\t\t\tcpa-\u003enumpages = 1;\n-\n-\t\tif (!debug_pagealloc_enabled())\n-\t\t\tspin_lock(\u0026cpa_lock);\n-\t\tret = __change_page_attr(cpa, primary);\n-\t\tif (!debug_pagealloc_enabled())\n-\t\t\tspin_unlock(\u0026cpa_lock);\n-\t\tif (ret)\n-\t\t\tgoto out;\n-\n-\t\tif (primary \u0026\u0026 !(cpa-\u003eflags \u0026 CPA_NO_CHECK_ALIAS)) {\n-\t\t\tret = cpa_process_alias(cpa);\n-\t\t\tif (ret)\n-\t\t\t\tgoto out;\n-\t\t}\n-\n-\t\t/*\n-\t\t * Adjust the number of pages with the result of the\n-\t\t * CPA operation. Either a large page has been\n-\t\t * preserved or a single page update happened.\n-\t\t */\n-\t\tBUG_ON(cpa-\u003enumpages \u003e rempages || !cpa-\u003enumpages);\n-\t\trempages -= cpa-\u003enumpages;\n-\t\tcpa-\u003ecurpage += cpa-\u003enumpages;\n-\t}\n-\n-out:\n-\t/* Restore the original numpages */\n-\tcpa-\u003enumpages = numpages;\n-\treturn ret;\n-}\n-\n-static int change_page_attr_set_clr(unsigned long *addr, int numpages,\n-\t\t\t\t    pgprot_t mask_set, pgprot_t mask_clr,\n-\t\t\t\t    int force_split, int in_flag,\n-\t\t\t\t    struct page **pages)\n+void arch_cpa_flush(struct cpa_data *cpa, int err)\n {\n-\tstruct cpa_data cpa;\n-\tint ret, cache;\n-\n-\tmemset(\u0026cpa, 0, sizeof(cpa));\n-\n-\t/*\n-\t * Check, if we are requested to set a not supported\n-\t * feature.  Clearing non-supported features is OK.\n-\t */\n-\tmask_set = canon_pgprot(mask_set);\n-\n-\tif (!pgprot_val(mask_set) \u0026\u0026 !pgprot_val(mask_clr) \u0026\u0026 !force_split)\n-\t\treturn 0;\n-\n-\t/* Ensure we are PAGE_SIZE aligned */\n-\tif (in_flag \u0026 CPA_ARRAY) {\n-\t\tint i;\n-\t\tfor (i = 0; i \u003c numpages; i++) {\n-\t\t\tif (addr[i] \u0026 ~PAGE_MASK) {\n-\t\t\t\taddr[i] \u0026= PAGE_MASK;\n-\t\t\t\tWARN_ON_ONCE(1);\n-\t\t\t}\n-\t\t}\n-\t} else if (!(in_flag \u0026 CPA_PAGES_ARRAY)) {\n-\t\t/*\n-\t\t * in_flag of CPA_PAGES_ARRAY implies it is aligned.\n-\t\t * No need to check in that case\n-\t\t */\n-\t\tif (*addr \u0026 ~PAGE_MASK) {\n-\t\t\t*addr \u0026= PAGE_MASK;\n-\t\t\t/*\n-\t\t\t * People should not be passing in unaligned addresses:\n-\t\t\t */\n-\t\t\tWARN_ON_ONCE(1);\n-\t\t}\n-\t}\n-\n-\t/* Must avoid aliasing mappings in the highmem code */\n-\tkmap_flush_unused();\n-\n-\tvm_unmap_aliases();\n-\n-\tcpa.vaddr = addr;\n-\tcpa.pages = pages;\n-\tcpa.numpages = numpages;\n-\tcpa.mask_set = mask_set;\n-\tcpa.mask_clr = mask_clr;\n-\tcpa.flags = in_flag;\n-\tcpa.curpage = 0;\n-\tcpa.force_split = force_split;\n-\n-\tret = __change_page_attr_set_clr(\u0026cpa, 1);\n-\n-\t/*\n-\t * Check whether we really changed something:\n-\t */\n-\tif (!(cpa.flags \u0026 CPA_FLUSHTLB))\n-\t\tgoto out;\n+\tint cache;\n \n \t/*\n \t * No need to flush, when we did not set any of the caching\n \t * attributes:\n \t */\n-\tcache = !!pgprot2cachemode(mask_set);\n+\tcache = !!pgprot2cachemode(cpa-\u003emask_set);\n \n \t/*\n \t * On error; flush everything to be sure.\n \t */\n-\tif (ret) {\n+\tif (err) {\n \t\tcpa_flush_all(cache);\n-\t\tgoto out;\n+\t\treturn;\n \t}\n \n-\tcpa_flush(\u0026cpa, cache);\n-out:\n-\treturn ret;\n-}\n-\n-static inline int change_page_attr_set(unsigned long *addr, int numpages,\n-\t\t\t\t       pgprot_t mask, int array)\n-{\n-\treturn change_page_attr_set_clr(addr, numpages, mask, __pgprot(0), 0,\n-\t\t(array ? CPA_ARRAY : 0), NULL);\n-}\n-\n-static inline int change_page_attr_clear(unsigned long *addr, int numpages,\n-\t\t\t\t\t pgprot_t mask, int array)\n-{\n-\treturn change_page_attr_set_clr(addr, numpages, __pgprot(0), mask, 0,\n-\t\t(array ? CPA_ARRAY : 0), NULL);\n-}\n-\n-static inline int cpa_set_pages_array(struct page **pages, int numpages,\n-\t\t\t\t       pgprot_t mask)\n-{\n-\treturn change_page_attr_set_clr(NULL, numpages, mask, __pgprot(0), 0,\n-\t\tCPA_PAGES_ARRAY, pages);\n-}\n-\n-static inline int cpa_clear_pages_array(struct page **pages, int numpages,\n-\t\t\t\t\t pgprot_t mask)\n-{\n-\treturn change_page_attr_set_clr(NULL, numpages, __pgprot(0), mask, 0,\n-\t\tCPA_PAGES_ARRAY, pages);\n+\tcpa_flush(cpa, cache);\n }\n \n int _set_memory_uc(unsigned long addr, int numpages)\n@@ -2773,11 +2324,3 @@ int __init kernel_unmap_pages_in_pgd(pgd_t *pgd, unsigned long address,\n \n \treturn retval;\n }\n-\n-/*\n- * The testcases use internal knowledge of the implementation that shouldn't\n- * be exposed to the rest of the kernel. Include these directly here.\n- */\n-#ifdef CONFIG_CPA_DEBUG\n-#include \"cpa-test.c\"\n-#endif\ndiff --git a/include/linux/pgtable.h b/include/linux/pgtable.h\nindex 2981e386da7b3c..d1df8345852be8 100644\n--- a/include/linux/pgtable.h\n+++ b/include/linux/pgtable.h\n@@ -1782,6 +1782,10 @@ void arch_sync_kernel_mappings(unsigned long start, unsigned long end);\n  * because these macros can be used even if CONFIG_MMU is not defined.\n  */\n \n+#ifndef canon_pgprot\n+#define canon_pgprot(prot)\t(prot)\n+#endif\n+\n #ifndef pgprot_nx\n #define pgprot_nx(prot)\t(prot)\n #endif\n@@ -2105,6 +2109,46 @@ static inline int pud_write(pud_t pud)\n }\n #endif /* pud_write */\n \n+#ifndef p4d_write\n+static inline int p4d_write(p4d_t p4d)\n+{\n+\tBUG();\n+\treturn 0;\n+}\n+#endif /* p4d_write */\n+\n+#ifndef pmd_exec\n+static inline int pmd_exec(pmd_t pmd)\n+{\n+\tBUG();\n+\treturn 0;\n+}\n+#endif /* pmd_exec */\n+\n+#ifndef pud_exec\n+static inline int pud_exec(pud_t pud)\n+{\n+\tBUG();\n+\treturn 0;\n+}\n+#endif /* pud_exec */\n+\n+#ifndef p4d_exec\n+static inline int p4d_exec(p4d_t p4d)\n+{\n+\tBUG();\n+\treturn 0;\n+}\n+#endif /* p4d_exec */\n+\n+#ifndef pgd_exec\n+static inline int pgd_exec(pgd_t pgd)\n+{\n+\tBUG();\n+\treturn 0;\n+}\n+#endif /* pgd_exec */\n+\n #if !defined(CONFIG_TRANSPARENT_HUGEPAGE) || \\\n \t!defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)\n static inline int pud_trans_huge(pud_t pud)\n@@ -2295,16 +2339,20 @@ static inline bool arch_has_pfn_modify_check(void)\n typedef unsigned int pgtbl_mod_mask;\n \n enum pgtable_level {\n-\tPGTABLE_LEVEL_PTE = 0,\n+\tPGTABLE_LEVEL_NONE = 0,\n+\tPGTABLE_LEVEL_PTE,\n \tPGTABLE_LEVEL_PMD,\n \tPGTABLE_LEVEL_PUD,\n \tPGTABLE_LEVEL_P4D,\n \tPGTABLE_LEVEL_PGD,\n+\tPGTABLE_LEVEL_NUM,\n };\n \n static inline const char *pgtable_level_to_str(enum pgtable_level level)\n {\n \tswitch (level) {\n+\tcase PGTABLE_LEVEL_NONE:\n+\t\treturn \"none\";\n \tcase PGTABLE_LEVEL_PTE:\n \t\treturn \"pte\";\n \tcase PGTABLE_LEVEL_PMD:\n@@ -2320,6 +2368,28 @@ static inline const char *pgtable_level_to_str(enum pgtable_level level)\n \t}\n }\n \n+/* Size of the memory mapped by a single leaf entry at @level */\n+static inline unsigned long pgtable_level_size(enum pgtable_level level)\n+{\n+\tswitch (level) {\n+\tcase PGTABLE_LEVEL_PTE:\n+\t\treturn PAGE_SIZE;\n+\tcase PGTABLE_LEVEL_PMD:\n+\t\treturn PMD_SIZE;\n+\tcase PGTABLE_LEVEL_PUD:\n+\t\treturn PUD_SIZE;\n+\tcase PGTABLE_LEVEL_P4D:\n+\t\treturn P4D_SIZE;\n+\tdefault:\n+\t\treturn PGDIR_SIZE;\n+\t}\n+}\n+\n+static inline unsigned long pgtable_level_mask(enum pgtable_level level)\n+{\n+\treturn ~(pgtable_level_size(level) - 1);\n+}\n+\n #endif /* !__ASSEMBLY__ */\n \n #if !defined(MAX_POSSIBLE_PHYSMEM_BITS) \u0026\u0026 !defined(CONFIG_64BIT)\ndiff --git a/include/linux/set_memory.h b/include/linux/set_memory.h\nindex 3030d9245f5ac8..17cbe6e26cb732 100644\n--- a/include/linux/set_memory.h\n+++ b/include/linux/set_memory.h\n@@ -84,4 +84,101 @@ static inline int set_memory_decrypted(unsigned long addr, int numpages)\n }\n #endif /* CONFIG_ARCH_HAS_MEM_ENCRYPT */\n \n+#ifdef CONFIG_GENERIC_SET_MEMORY\n+\n+#include \u003clinux/pgtable.h\u003e\n+\n+#define CPA_FLUSHTLB 1\n+#define CPA_ARRAY 2\n+#define CPA_PAGES_ARRAY 4\n+#define CPA_NO_CHECK_ALIAS 8 /* Do not search for aliases */\n+#define CPA_COLLAPSE 16 /* try to collapse large pages */\n+\n+/*\n+ * The current flushing context - we pass it instead of 5 arguments:\n+ */\n+struct cpa_data {\n+\tunsigned long\t*vaddr;\n+\tpgd_t\t\t*pgd;\n+\tpgprot_t\tmask_set;\n+\tpgprot_t\tmask_clr;\n+\tunsigned long\tnumpages;\n+\tunsigned long\tcurpage;\n+\tunsigned long\tpfn;\n+\tunsigned int\tflags;\n+\tunsigned int\tforce_split\t\t: 1,\n+\t\t\tforce_static_prot\t: 1,\n+\t\t\tforce_flush_all\t\t: 1;\n+\tstruct page\t**pages;\n+};\n+\n+/*\n+ * context for checking for a split a large page and splitting it if required.\n+ */\n+struct ptdesc;\n+struct cpa_split_data {\n+\tunsigned long\taddress;\n+\tunsigned long\tpsize;\n+\tunsigned long\tpmask;\n+\tpte_t\t\t*kpte;\n+\tstruct ptdesc\t*ptdesc;\n+\tenum pgtable_level level;\n+\tbool\t\tnx;\n+\tbool\t\trw;\n+};\n+\n+unsigned long cpa_addr(struct cpa_data *cpa, unsigned long idx);\n+\n+pte_t *lookup_address(unsigned long address, unsigned int *level);\n+pte_t *lookup_address_in_pgd(pgd_t *pgd, unsigned long address,\n+\t\t\t     unsigned int *level);\n+pte_t *lookup_address_in_pgd_attr(pgd_t *pgd, unsigned long address,\n+\t\t\t\t  unsigned int *level, bool *nx, bool *rw);\n+\n+int arch_cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,\n+\t\t\t   int primary);\n+int arch_cpa_process_alias(struct cpa_data *cpa);\n+int arch_should_split_large_page(struct cpa_data *cpa,\n+\t\t\t\t struct cpa_split_data *sd);\n+int arch_split_large_page(struct cpa_data *cpa, struct cpa_split_data *sd);\n+void arch_change_pte(struct cpa_data *cpa, unsigned long address,\n+\t\t     pte_t *kpte, pte_t old_pte, bool nx, bool rw);\n+void arch_cpa_flush(struct cpa_data *cpa, int err);\n+\n+int __change_page_attr_set_clr(struct cpa_data *cpa, int primary);\n+int change_page_attr_set_clr(unsigned long *addr, int numpages,\n+\t\t\t     pgprot_t mask_set, pgprot_t mask_clr,\n+\t\t\t     int force_split, int in_flag,\n+\t\t\t     struct page **pages);\n+\n+static inline int change_page_attr_set(unsigned long *addr, int numpages,\n+\t\t\t\t       pgprot_t mask, int array)\n+{\n+\treturn change_page_attr_set_clr(addr, numpages, mask, __pgprot(0), 0,\n+\t\t(array ? CPA_ARRAY : 0), NULL);\n+}\n+\n+static inline int change_page_attr_clear(unsigned long *addr, int numpages,\n+\t\t\t\t\t pgprot_t mask, int array)\n+{\n+\treturn change_page_attr_set_clr(addr, numpages, __pgprot(0), mask, 0,\n+\t\t(array ? CPA_ARRAY : 0), NULL);\n+}\n+\n+static inline int cpa_set_pages_array(struct page **pages, int numpages,\n+\t\t\t\t      pgprot_t mask)\n+{\n+\treturn change_page_attr_set_clr(NULL, numpages, mask, __pgprot(0), 0,\n+\t\tCPA_PAGES_ARRAY, pages);\n+}\n+\n+static inline int cpa_clear_pages_array(struct page **pages, int numpages,\n+\t\t\t\t\tpgprot_t mask)\n+{\n+\treturn change_page_attr_set_clr(NULL, numpages, __pgprot(0), mask, 0,\n+\t\tCPA_PAGES_ARRAY, pages);\n+}\n+\n+#endif /* CONFIG_GENERIC_SET_MEMORY */\n+\n #endif /* _LINUX_SET_MEMORY_H_ */\ndiff --git a/mm/Kconfig b/mm/Kconfig\nindex 9e0ca48249054f..541edd1e11b610 100644\n--- a/mm/Kconfig\n+++ b/mm/Kconfig\n@@ -1509,6 +1509,9 @@ config LAZY_MMU_MODE_KUNIT_TEST\n \n \t  If unsure, say N.\n \n+config GENERIC_SET_MEMORY\n+\tbool\n+\n source \"mm/damon/Kconfig\"\n \n endmenu\ndiff --git a/mm/Kconfig.debug b/mm/Kconfig.debug\nindex 91b3e027b75357..83cfef6b2d8cfb 100644\n--- a/mm/Kconfig.debug\n+++ b/mm/Kconfig.debug\n@@ -320,3 +320,10 @@ config PER_VMA_LOCK_STATS\n \t  overhead in the page fault path.\n \n \t  If in doubt, say N.\n+\n+config CPA_DEBUG\n+\tbool \"CPA self-test code\"\n+\tdepends on GENERIC_SET_MEMORY\n+\tdepends on DEBUG_KERNEL\n+\thelp\n+\t  Do change_page_attr() self-tests every 30 seconds.\ndiff --git a/mm/Makefile b/mm/Makefile\nindex eff9f9e7e061c1..e3f6c39c2ba991 100644\n--- a/mm/Makefile\n+++ b/mm/Makefile\n@@ -147,3 +147,4 @@ obj-$(CONFIG_SHRINKER_DEBUG) += shrinker_debug.o\n obj-$(CONFIG_EXECMEM) += execmem.o\n obj-$(CONFIG_TMPFS_QUOTA) += shmem_quota.o\n obj-$(CONFIG_LAZY_MMU_MODE_KUNIT_TEST) += tests/lazy_mmu_mode_kunit.o\n+obj-$(CONFIG_GENERIC_SET_MEMORY) += set_memory.o\ndiff --git a/arch/x86/mm/pat/cpa-test.c b/mm/cpa-test.c\nsimilarity index 80%\nrename from arch/x86/mm/pat/cpa-test.c\nrename to mm/cpa-test.c\nindex ad3c1feec990db..63c3463a5a7d85 100644\n--- a/arch/x86/mm/pat/cpa-test.c\n+++ b/mm/cpa-test.c\n@@ -13,32 +13,24 @@\n #include \u003clinux/mm.h\u003e\n #include \u003clinux/vmalloc.h\u003e\n \n-#include \u003casm/cacheflush.h\u003e\n-#include \u003casm/kdebug.h\u003e\n-\n /*\n  * Only print the results of the first pass:\n  */\n static __read_mostly int print = 1;\n+static unsigned long max_test_pfn;\n \n enum {\n \tNTEST\t\t\t= 3 * 100,\n \tNPAGES\t\t\t= 100,\n-#ifdef CONFIG_X86_64\n-\tLPS\t\t\t= (1 \u003c\u003c PMD_SHIFT),\n-#elif defined(CONFIG_X86_PAE)\n-\tLPS\t\t\t= (1 \u003c\u003c PMD_SHIFT),\n-#else\n-\tLPS\t\t\t= (1 \u003c\u003c 22),\n-#endif\n-\tGPS\t\t\t= (1\u003c\u003c30)\n+\tLPS\t\t\t= PMD_SIZE,\n+\tGPS\t\t\t= PUD_SIZE\n };\n \n-#define PAGE_CPA_TEST\t__pgprot(_PAGE_CPA_TEST)\n+#define PAGE_CPA_TEST\t__pgprot(_PAGE_SPECIAL)\n \n static int pte_testbit(pte_t pte)\n {\n-\treturn pte_flags(pte) \u0026 _PAGE_SOFTW1;\n+\treturn pte_special(pte);\n }\n \n struct split_state {\n@@ -54,10 +46,10 @@ static int print_split(struct split_state *s)\n \ts-\u003elpg = s-\u003egpg = s-\u003espg = s-\u003eexec = 0;\n \ts-\u003emin_exec = ~0UL;\n \ts-\u003emax_exec = 0;\n-\tfor (i = 0; i \u003c max_pfn_mapped; ) {\n+\tfor (i = 0; i \u003c max_test_pfn; ) {\n \t\tunsigned long addr = (unsigned long)__va(i \u003c\u003c PAGE_SHIFT);\n \t\tunsigned int level;\n-\t\tpte_t *pte;\n+\t\tpte_t *pte, pteval;\n \n \t\tpte = lookup_address(addr, \u0026level);\n \t\tif (!pte) {\n@@ -65,15 +57,16 @@ static int print_split(struct split_state *s)\n \t\t\ti++;\n \t\t\tcontinue;\n \t\t}\n+\t\tpteval = ptep_get(pte);\n \n-\t\tif (level == PG_LEVEL_1G \u0026\u0026 sizeof(long) == 8) {\n+\t\tif (level == PGTABLE_LEVEL_PUD \u0026\u0026 sizeof(long) == 8) {\n \t\t\ts-\u003egpg++;\n \t\t\ti += GPS/PAGE_SIZE;\n-\t\t} else if (level == PG_LEVEL_2M) {\n-\t\t\tif ((pte_val(*pte) \u0026 _PAGE_PRESENT) \u0026\u0026 !(pte_val(*pte) \u0026 _PAGE_PSE)) {\n+\t\t} else if (level == PGTABLE_LEVEL_PMD) {\n+\t\t\tif (pte_present(pteval) \u0026\u0026 !pte_huge(pteval)) {\n \t\t\t\tprintk(KERN_ERR\n-\t\t\t\t\t\"%lx level %d but not PSE %Lx\\n\",\n-\t\t\t\t\taddr, level, (u64)pte_val(*pte));\n+\t\t\t\t\t\"%lx level %d but not leaf %Lx\\n\",\n+\t\t\t\t\taddr, level, (u64)pte_val(pteval));\n \t\t\t\terr = 1;\n \t\t\t}\n \t\t\ts-\u003elpg++;\n@@ -82,7 +75,7 @@ static int print_split(struct split_state *s)\n \t\t\ts-\u003espg++;\n \t\t\ti++;\n \t\t}\n-\t\tif (!(pte_val(*pte) \u0026 _PAGE_NX)) {\n+\t\tif (pte_exec(pteval)) {\n \t\t\ts-\u003eexec++;\n \t\t\tif (addr \u003c s-\u003emin_exec)\n \t\t\t\ts-\u003emin_exec = addr;\n@@ -100,8 +93,8 @@ static int print_split(struct split_state *s)\n \n \texpected = (s-\u003egpg*GPS + s-\u003elpg*LPS)/PAGE_SIZE + s-\u003espg + missed;\n \tif (expected != i) {\n-\t\tprintk(KERN_ERR \"CPA max_pfn_mapped %lu but expected %lu\\n\",\n-\t\t\tmax_pfn_mapped, expected);\n+\t\tprintk(KERN_ERR \"CPA max_test_pfn %lu but expected %lu\\n\",\n+\t\t\tmax_test_pfn, expected);\n \t\treturn 1;\n \t}\n \treturn err;\n@@ -118,7 +111,7 @@ static int pageattr_test(void)\n {\n \tstruct split_state sa, sb, sc;\n \tunsigned long *bm;\n-\tpte_t *pte, pte0;\n+\tpte_t *pte, pte0, pte_val;\n \tint failed = 0;\n \tunsigned int level;\n \tint i, k;\n@@ -127,7 +120,7 @@ static int pageattr_test(void)\n \tif (print)\n \t\tprintk(KERN_INFO \"CPA self-test:\\n\");\n \n-\tbm = vzalloc((max_pfn_mapped + 7) / 8);\n+\tbm = vzalloc((max_test_pfn + 7) / 8);\n \tif (!bm) {\n \t\tprintk(KERN_ERR \"CPA Cannot vmalloc bitmap\\n\");\n \t\treturn -ENOMEM;\n@@ -136,11 +129,11 @@ static int pageattr_test(void)\n \tfailed += print_split(\u0026sa);\n \n \tfor (i = 0; i \u003c NTEST; i++) {\n-\t\tunsigned long pfn = get_random_u32_below(max_pfn_mapped);\n+\t\tunsigned long pfn = get_random_u32_below(max_test_pfn);\n \n \t\taddr[i] = (unsigned long)__va(pfn \u003c\u003c PAGE_SHIFT);\n \t\tlen[i] = get_random_u32_below(NPAGES);\n-\t\tlen[i] = min_t(unsigned long, len[i], max_pfn_mapped - pfn - 1);\n+\t\tlen[i] = min_t(unsigned long, len[i], max_test_pfn - pfn - 1);\n \n \t\tif (len[i] == 0)\n \t\t\tlen[i] = 1;\n@@ -150,15 +143,20 @@ static int pageattr_test(void)\n \n \t\tfor (k = 0; k \u003c len[i]; k++) {\n \t\t\tpte = lookup_address(addr[i] + k*PAGE_SIZE, \u0026level);\n-\t\t\tif (!pte || pgprot_val(pte_pgprot(*pte)) == 0 ||\n-\t\t\t    !(pte_val(*pte) \u0026 _PAGE_PRESENT)) {\n+\t\t\tif (!pte) {\n+\t\t\t\taddr[i] = 0;\n+\t\t\t\tbreak;\n+\t\t\t}\n+\t\t\tpte_val = ptep_get(pte);\n+\t\t\tif (pgprot_val(pte_pgprot(pte_val)) == 0 ||\n+\t\t\t    !pte_present(pte_val)) {\n \t\t\t\taddr[i] = 0;\n \t\t\t\tbreak;\n \t\t\t}\n \t\t\tif (k == 0) {\n-\t\t\t\tpte0 = *pte;\n+\t\t\t\tpte0 = pte_val;\n \t\t\t} else {\n-\t\t\t\tif (pgprot_val(pte_pgprot(*pte)) !=\n+\t\t\t\tif (pgprot_val(pte_pgprot(pte_val)) !=\n \t\t\t\t\tpgprot_val(pte_pgprot(pte0))) {\n \t\t\t\t\tlen[i] = k;\n \t\t\t\t\tbreak;\n@@ -198,12 +196,12 @@ static int pageattr_test(void)\n \t\t}\n \n \t\tpte = lookup_address(addr[i], \u0026level);\n-\t\tif (!pte || !pte_testbit(*pte) || pte_huge(*pte)) {\n+\t\tif (!pte || !pte_testbit(*pte) || level != PGTABLE_LEVEL_PTE) {\n \t\t\tprintk(KERN_ERR \"CPA %lx: bad pte %Lx\\n\", addr[i],\n \t\t\t\tpte ? (u64)pte_val(*pte) : 0ULL);\n \t\t\tfailed++;\n \t\t}\n-\t\tif (level != PG_LEVEL_4K) {\n+\t\tif (level != PGTABLE_LEVEL_PTE) {\n \t\t\tprintk(KERN_ERR \"CPA %lx: unexpected level %d\\n\",\n \t\t\t\taddr[i], level);\n \t\t\tfailed++;\n@@ -266,6 +264,8 @@ static int start_pageattr_test(void)\n {\n \tstruct task_struct *p;\n \n+\tmax_test_pfn = PFN_DOWN(__pa(high_memory - 1));\n+\n \tp = kthread_create(do_pageattr_test, NULL, \"pageattr-test\");\n \tif (!IS_ERR(p))\n \t\twake_up_process(p);\ndiff --git a/mm/set_memory.c b/mm/set_memory.c\nnew file mode 100644\nindex 00000000000000..744405efcc3401\n--- /dev/null\n+++ b/mm/set_memory.c\n@@ -0,0 +1,396 @@\n+// SPDX-License-Identifier: GPL-2.0-only\n+\n+#include \u003clinux/highmem.h\u003e\n+#include \u003clinux/vmalloc.h\u003e\n+#include \u003clinux/pgtable.h\u003e\n+#include \u003clinux/export.h\u003e\n+\n+#include \u003clinux/set_memory.h\u003e\n+\n+/*\n+ * Serialize cpa() so that we don't allow any other cpu, with stale large tlb\n+ * entries, to change the page attribute in parallel to some other cpu\n+ * splitting a large page entry along with changing the attribute.\n+ */\n+static DEFINE_SPINLOCK(cpa_lock);\n+\n+static inline void arch_lock(void)\n+{\n+#ifdef CONFIG_X86\n+\tspin_lock(\u0026pgd_lock);\n+#endif\n+}\n+\n+static inline void arch_unlock(void)\n+{\n+#ifdef CONFIG_X86\n+\tspin_unlock(\u0026pgd_lock);\n+#endif\n+}\n+\n+unsigned long cpa_addr(struct cpa_data *cpa, unsigned long idx)\n+{\n+\tif (cpa-\u003eflags \u0026 CPA_PAGES_ARRAY) {\n+\t\tstruct page *page = cpa-\u003epages[idx];\n+\n+\t\tif (unlikely(PageHighMem(page)))\n+\t\t\treturn 0;\n+\n+\t\treturn (unsigned long)page_address(page);\n+\t}\n+\n+\tif (cpa-\u003eflags \u0026 CPA_ARRAY)\n+\t\treturn cpa-\u003evaddr[idx];\n+\n+\treturn *cpa-\u003evaddr + idx * PAGE_SIZE;\n+}\n+\n+/*\n+ * Lookup the page table entry for a virtual address in a specific pgd.\n+ * Return a pointer to the entry (or NULL if the entry does not exist),\n+ * the level of the entry, and the effective NX and RW bits of all\n+ * page table levels.\n+ */\n+pte_t *lookup_address_in_pgd_attr(pgd_t *pgdp, unsigned long address,\n+\t\t\t\t  unsigned int *level, bool *nx, bool *rw)\n+{\n+\tpgd_t pgd = pgdp_get(pgdp);\n+\tp4d_t *p4dp, p4d;\n+\tpud_t *pudp, pud;\n+\tpmd_t *pmdp, pmd;\n+\n+\t*level = PGTABLE_LEVEL_PGD;\n+\t*nx = false;\n+\t*rw = true;\n+\n+\tif (pgd_none(pgd))\n+\t\treturn NULL;\n+\n+\t*level = PGTABLE_LEVEL_P4D;\n+\t*nx |= !pgd_exec(pgd);\n+\t*rw \u0026= !!(pgd_write(pgd));\n+\n+\tp4dp = p4d_offset(pgdp, address);\n+\tp4d = p4dp_get(p4dp);\n+\tif (p4d_none(p4d))\n+\t\treturn NULL;\n+\n+\tif (p4d_leaf(p4d) || !p4d_present(p4d))\n+\t\treturn (pte_t *)p4dp;\n+\n+\t*level = PGTABLE_LEVEL_PUD;\n+\t*nx |= !p4d_exec(p4d);\n+\t*rw \u0026= !!(p4d_write(p4d));\n+\n+\tpudp = pud_offset(p4dp, address);\n+\tpud = pudp_get(pudp);\n+\tif (pud_none(pud))\n+\t\treturn NULL;\n+\n+\tif (pud_leaf(pud) || !pud_present(pud))\n+\t\treturn (pte_t *)pudp;\n+\n+\t*level = PGTABLE_LEVEL_PMD;\n+\t*nx |= !pud_exec(pud);\n+\t*rw \u0026= !!(pud_write(pud));\n+\n+\tpmdp = pmd_offset(pudp, address);\n+\tpmd = pmdp_get(pmdp);\n+\tif (pmd_none(pmd))\n+\t\treturn NULL;\n+\n+\tif (pmd_leaf(pmd) || !pmd_present(pmd))\n+\t\treturn (pte_t *)pmdp;\n+\n+\t*level = PGTABLE_LEVEL_PTE;\n+\t*nx |= !pmd_exec(pmd);\n+\t*rw \u0026= !!(pmd_write(pmd));\n+\n+\treturn pte_offset_kernel(pmdp, address);\n+}\n+\n+/*\n+ * Lookup the page table entry for a virtual address in a specific pgd.\n+ * Return a pointer to the entry and the level of the mapping.\n+ */\n+pte_t *lookup_address_in_pgd(pgd_t *pgd, unsigned long address,\n+\t\t\t     unsigned int *level)\n+{\n+\tbool nx, rw;\n+\n+\treturn lookup_address_in_pgd_attr(pgd, address, level, \u0026nx, \u0026rw);\n+}\n+\n+/*\n+ * Lookup the page table entry for a virtual address. Return a pointer\n+ * to the entry and the level of the mapping.\n+ *\n+ * Note: the function returns p4d, pud or pmd either when the entry is marked\n+ * large or when the present bit is not set. Otherwise it returns NULL.\n+ */\n+pte_t *lookup_address(unsigned long address, unsigned int *level)\n+{\n+\treturn lookup_address_in_pgd(pgd_offset_k(address), address, level);\n+}\n+EXPORT_SYMBOL_GPL(lookup_address);\n+\n+static pte_t *_lookup_address_cpa(struct cpa_data *cpa, unsigned long address,\n+\t\t\t   unsigned int *level, bool *nx, bool *rw)\n+{\n+\tpgd_t *pgd;\n+\n+\tif (!cpa-\u003epgd)\n+\t\tpgd = pgd_offset_k(address);\n+\telse\n+\t\tpgd = cpa-\u003epgd + pgd_index(address);\n+\n+\treturn lookup_address_in_pgd_attr(pgd, address, level, nx, rw);\n+}\n+\n+static int should_split_large_page(struct cpa_data *cpa,\n+\t\t\t\t   struct cpa_split_data *sd)\n+{\n+\tenum pgtable_level level = sd-\u003elevel;\n+\tunsigned long address = sd-\u003eaddress;\n+\tunsigned long numpages, lpaddr;\n+\n+\tif (cpa-\u003eforce_split)\n+\t\treturn 1;\n+\n+\tsd-\u003epsize = pgtable_level_size(level);\n+\tsd-\u003epmask = pgtable_level_mask(level);\n+\n+\t/*\n+\t * Calculate the number of pages, which fit into this large\n+\t * page starting at address:\n+\t */\n+\tlpaddr = (address + sd-\u003epsize) \u0026 sd-\u003epmask;\n+\tnumpages = (lpaddr - address) \u003e\u003e PAGE_SHIFT;\n+\tif (numpages \u003c cpa-\u003enumpages)\n+\t\tcpa-\u003enumpages = numpages;\n+\n+\treturn arch_should_split_large_page(cpa, sd);\n+}\n+\n+static int cpa_handle_large_page(struct cpa_data *cpa, pte_t *kpte,\n+\t\t\t\t  unsigned long address)\n+{\n+\tstruct cpa_split_data sd = {\n+\t\t.address = address,\n+\t\t.kpte = kpte,\n+\t};\n+\tint do_split, ret = 1;\n+\tpte_t *tmp;\n+\n+\tspin_unlock(\u0026cpa_lock);\n+\tsd.ptdesc = pagetable_alloc(GFP_KERNEL, 0);\n+\tspin_lock(\u0026cpa_lock);\n+\tif (!sd.ptdesc)\n+\t\treturn -ENOMEM;\n+\n+\tarch_lock();\n+\n+\t/*\n+\t * Check for races, another CPU might have split this page\n+\t * up already:\n+\t */\n+\ttmp = _lookup_address_cpa(cpa, address, \u0026sd.level, \u0026sd.nx, \u0026sd.rw);\n+\tif (tmp != kpte)\n+\t\tgoto out_free_ptdesc;\n+\n+\t/*\n+\t * Check, whether we can keep the large page intact\n+\t * and just change the pte:\n+\t */\n+\tdo_split = should_split_large_page(cpa, \u0026sd);\n+\tret = do_split;\n+\t/*\n+\t * When the range fits into the existing large page, no split is\n+\t * required.\n+\t * should_split_large_page() updated the large page attributes and\n+\t * cpa-\u003enumpages and cpa-\u003epfn.\n+\t */\n+\tif (do_split \u003c= 0)\n+\t\tgoto out_free_ptdesc;\n+\n+\t/*\n+\t * We have to split the large page:\n+\t */\n+\tret = arch_split_large_page(cpa, \u0026sd);\n+\tif (ret)\n+\t\tgoto out_free_ptdesc;\n+\n+\tarch_unlock();\n+\treturn do_split;\n+\n+out_free_ptdesc:\n+\tarch_unlock();\n+\tpagetable_free(sd.ptdesc);\n+\treturn ret;\n+}\n+\n+static int __change_page_attr(struct cpa_data *cpa, int primary)\n+{\n+\tunsigned long address;\n+\tint split_res;\n+\tunsigned int level;\n+\tpte_t *kpte, old_pte;\n+\tbool nx, rw;\n+\n+\taddress = cpa_addr(cpa, cpa-\u003ecurpage);\n+repeat:\n+\tkpte = _lookup_address_cpa(cpa, address, \u0026level, \u0026nx, \u0026rw);\n+\tif (!kpte)\n+\t\treturn arch_cpa_process_fault(cpa, address, primary);\n+\n+\told_pte = *kpte;\n+\tif (pte_none(old_pte))\n+\t\treturn arch_cpa_process_fault(cpa, address, primary);\n+\n+\tif (level == PGTABLE_LEVEL_PTE) {\n+\t\tarch_change_pte(cpa, address, kpte, old_pte, nx, rw);\n+\t\tcpa-\u003enumpages = 1;\n+\t\treturn 0;\n+\t}\n+\n+\t/*\n+\t * The address maps a large page. Try to keep it, otherwise split it\n+\t * up and retry on the freshly installed smaller mapping.\n+\t */\n+\tsplit_res = cpa_handle_large_page(cpa, kpte, address);\n+\tif (split_res \u003e 0)\n+\t\tgoto repeat;\n+\n+\treturn split_res;\n+}\n+\n+int __change_page_attr_set_clr(struct cpa_data *cpa, int primary)\n+{\n+\tunsigned long numpages = cpa-\u003enumpages;\n+\tunsigned long rempages = numpages;\n+\tint ret = 0;\n+\n+\t/*\n+\t * No changes, easy!\n+\t */\n+\tif (!(pgprot_val(cpa-\u003emask_set) | pgprot_val(cpa-\u003emask_clr)) \u0026\u0026\n+\t    !cpa-\u003eforce_split)\n+\t\treturn ret;\n+\n+\twhile (rempages) {\n+\t\t/*\n+\t\t * Store the remaining nr of pages for the large page\n+\t\t * preservation check.\n+\t\t */\n+\t\tcpa-\u003enumpages = rempages;\n+\t\t/* for array changes, we can't use large page */\n+\t\tif (cpa-\u003eflags \u0026 (CPA_ARRAY | CPA_PAGES_ARRAY))\n+\t\t\tcpa-\u003enumpages = 1;\n+\n+\t\tspin_lock(\u0026cpa_lock);\n+\t\tret = __change_page_attr(cpa, primary);\n+\t\tspin_unlock(\u0026cpa_lock);\n+\t\tif (ret)\n+\t\t\tgoto out;\n+\n+\t\tif (primary \u0026\u0026 !(cpa-\u003eflags \u0026 CPA_NO_CHECK_ALIAS) \u0026\u0026\n+\t\t    cpa_should_update_alias(cpa_addr(cpa, cpa-\u003ecurpage), cpa-\u003epfn)) {\n+\t\t\tret = arch_cpa_process_alias(cpa);\n+\t\t\tif (ret)\n+\t\t\t\tgoto out;\n+\t\t}\n+\n+\t\t/*\n+\t\t * Adjust the number of pages with the result of the\n+\t\t * CPA operation. Either a large page has been\n+\t\t * preserved or a single page update happened.\n+\t\t */\n+\t\tBUG_ON(cpa-\u003enumpages \u003e rempages || !cpa-\u003enumpages);\n+\t\trempages -= cpa-\u003enumpages;\n+\t\tcpa-\u003ecurpage += cpa-\u003enumpages;\n+\t}\n+\n+out:\n+\t/* Restore the original numpages */\n+\tcpa-\u003enumpages = numpages;\n+\treturn ret;\n+}\n+\n+int change_page_attr_set_clr(unsigned long *addr, int numpages,\n+\t\t\t     pgprot_t mask_set, pgprot_t mask_clr,\n+\t\t\t     int force_split, int in_flag,\n+\t\t\t     struct page **pages)\n+{\n+\tstruct cpa_data cpa;\n+\tint err;\n+\n+\tmemset(\u0026cpa, 0, sizeof(cpa));\n+\n+\t/*\n+\t * Check, if we are requested to set a not supported\n+\t * feature.  Clearing non-supported features is OK.\n+\t */\n+\tmask_set = canon_pgprot(mask_set);\n+\n+\tif (!pgprot_val(mask_set) \u0026\u0026 !pgprot_val(mask_clr) \u0026\u0026 !force_split)\n+\t\treturn 0;\n+\n+\t/* Ensure we are PAGE_SIZE aligned */\n+\tif (in_flag \u0026 CPA_ARRAY) {\n+\t\tint i;\n+\n+\t\tfor (i = 0; i \u003c numpages; i++) {\n+\t\t\tif (addr[i] \u0026 ~PAGE_MASK) {\n+\t\t\t\taddr[i] \u0026= PAGE_MASK;\n+\t\t\t\tWARN_ON_ONCE(1);\n+\t\t\t}\n+\t\t}\n+\t} else if (!(in_flag \u0026 CPA_PAGES_ARRAY)) {\n+\t\t/*\n+\t\t * in_flag of CPA_PAGES_ARRAY implies it is aligned.\n+\t\t * No need to check in that case\n+\t\t */\n+\t\tif (*addr \u0026 ~PAGE_MASK) {\n+\t\t\t*addr \u0026= PAGE_MASK;\n+\t\t\t/*\n+\t\t\t * People should not be passing in unaligned addresses:\n+\t\t\t */\n+\t\t\tWARN_ON_ONCE(1);\n+\t\t}\n+\t}\n+\n+\t/* Must avoid aliasing mappings in the highmem code */\n+\tkmap_flush_unused();\n+\n+\tvm_unmap_aliases();\n+\n+\tcpa.vaddr = addr;\n+\tcpa.pages = pages;\n+\tcpa.numpages = numpages;\n+\tcpa.mask_set = mask_set;\n+\tcpa.mask_clr = mask_clr;\n+\tcpa.flags = in_flag;\n+\tcpa.curpage = 0;\n+\tcpa.force_split = force_split;\n+\n+\terr = __change_page_attr_set_clr(\u0026cpa, 1);\n+\n+\t/*\n+\t * Check whether we really changed something:\n+\t */\n+\tif (!(cpa.flags \u0026 CPA_FLUSHTLB))\n+\t\tgoto out;\n+\n+\tarch_cpa_flush(\u0026cpa, err);\n+\n+out:\n+\treturn err;\n+}\n+\n+/*\n+ * The testcases use internal knowledge of the implementation that shouldn't\n+ * be exposed to the rest of the kernel. Include these directly here.\n+ */\n+#ifdef CONFIG_CPA_DEBUG\n+#include \"cpa-test.c\"\n+#endif\n"
}

For your convenience, here is the diff of the changes:
commit d544b2bff4c34ff4e46cc081518a266ab316515c
Author: syz-cluster <triage@syzkaller.com>
Date:   Wed Jul 22 11:11:39 2026 +0000

    syz-cluster: applied patch under review

diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig
index d8d25232501791..1bfa94797b1b4b 100644
--- a/arch/loongarch/Kconfig
+++ b/arch/loongarch/Kconfig
@@ -109,6 +109,7 @@ config LOONGARCH
 	select GENERIC_LIB_DEVMEM_IS_ALLOWED
 	select GENERIC_PCI_IOMAP
 	select GENERIC_SCHED_CLOCK
+	select GENERIC_SET_MEMORY
 	select GENERIC_SMP_IDLE_THREAD
 	select GPIOLIB
 	select HAS_IOPORT
diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h
index 223528c04d7392..377bef0894d771 100644
--- a/arch/loongarch/include/asm/pgtable.h
+++ b/arch/loongarch/include/asm/pgtable.h
@@ -638,6 +638,48 @@ static inline long pmd_protnone(pmd_t pmd)
 #define pmd_leaf(pmd)		((pmd_val(pmd) & _PAGE_HUGE) != 0)
 #define pud_leaf(pud)		((pud_val(pud) & _PAGE_HUGE) != 0)
 
+#define pgd_write pgd_write
+static inline int pgd_write(pgd_t pgd)
+{
+	return !!(pgd_val(pgd) & _PAGE_WRITE);
+}
+
+#define p4d_write p4d_write
+static inline int p4d_write(p4d_t p4d)
+{
+	return !!(p4d_val(p4d) & _PAGE_WRITE);
+}
+
+#define pud_write pud_write
+static inline int pud_write(pud_t pud)
+{
+	return !!(pud_val(pud) & _PAGE_WRITE);
+}
+
+#define pgd_exec pgd_exec
+static inline int pgd_exec(pgd_t pgd)
+{
+	return !(pgd_val(pgd) & _PAGE_NO_EXEC);
+}
+
+#define p4d_exec p4d_exec
+static inline int p4d_exec(p4d_t p4d)
+{
+	return !(p4d_val(p4d) & _PAGE_NO_EXEC);
+}
+
+#define pud_exec pud_exec
+static inline int pud_exec(pud_t pud)
+{
+	return !(pud_val(pud) & _PAGE_NO_EXEC);
+}
+
+#define pmd_exec pmd_exec
+static inline int pmd_exec(pmd_t pmd)
+{
+	return !(pmd_val(pmd) & _PAGE_NO_EXEC);
+}
+
 /*
  * We provide our own get_unmapped area to cope with the virtual aliasing
  * constraints placed on us by the cache architecture.
diff --git a/arch/loongarch/include/asm/set_memory.h b/arch/loongarch/include/asm/set_memory.h
index 55dfaefd02c8a6..6ffb4fe5a75b0e 100644
--- a/arch/loongarch/include/asm/set_memory.h
+++ b/arch/loongarch/include/asm/set_memory.h
@@ -19,4 +19,13 @@ int set_direct_map_default_noflush(struct page *page);
 int set_direct_map_invalid_noflush(struct page *page);
 int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid);
 
+/*
+ * The direct map is a fixed address window that is not backed by page tables,
+ * so a page never has a second mapping whose protection must be synchronised.
+ */
+static inline bool cpa_should_update_alias(unsigned long vaddr, unsigned long pfn)
+{
+	return false;
+}
+
 #endif /* _ASM_LOONGARCH_SET_MEMORY_H */
diff --git a/arch/loongarch/mm/pageattr.c b/arch/loongarch/mm/pageattr.c
index 614ccc7afccbea..dbbd408bd3d5e6 100644
--- a/arch/loongarch/mm/pageattr.c
+++ b/arch/loongarch/mm/pageattr.c
@@ -4,126 +4,83 @@
  */
 
 #include <linux/memblock.h>
-#include <linux/pagewalk.h>
+#include <linux/mm.h>
 #include <linux/pgtable.h>
+#include <linux/set_memory.h>
 #include <asm/set_memory.h>
 #include <asm/tlbflush.h>
 
-struct pageattr_masks {
-	pgprot_t set_mask;
-	pgprot_t clear_mask;
-};
-
-static unsigned long set_pageattr_masks(unsigned long val, struct mm_walk *walk)
+static unsigned long set_pageattr_masks(struct cpa_data *cpa, unsigned long val)
 {
-	unsigned long new_val = val;
-	struct pageattr_masks *masks = walk->private;
-
-	new_val &= ~(pgprot_val(masks->clear_mask));
-	new_val |= (pgprot_val(masks->set_mask));
+	val &= ~pgprot_val(cpa->mask_clr);
+	val |= pgprot_val(cpa->mask_set);
 
-	return new_val;
+	return val;
 }
 
-static int pageattr_pgd_entry(pgd_t *pgd, unsigned long addr,
-			      unsigned long next, struct mm_walk *walk)
+int arch_should_split_large_page(struct cpa_data *cpa, struct cpa_split_data *sd)
 {
-	pgd_t val = pgdp_get(pgd);
-
-	if (pgd_leaf(val)) {
-		val = __pgd(set_pageattr_masks(pgd_val(val), walk));
-		set_pgd(pgd, val);
-	}
-
-	return 0;
-}
+	pte_t old = ptep_get(sd->kpte);
+	unsigned long val = pte_val(old);
+	unsigned long new_val = set_pageattr_masks(cpa, val);
 
-static int pageattr_p4d_entry(p4d_t *p4d, unsigned long addr,
-			      unsigned long next, struct mm_walk *walk)
-{
-	p4d_t val = p4dp_get(p4d);
+	cpa->pfn = pte_pfn(old) + ((sd->address & (sd->psize - 1)) >> PAGE_SHIFT);
 
-	if (p4d_leaf(val)) {
-		val = __p4d(set_pageattr_masks(p4d_val(val), walk));
-		set_p4d(p4d, val);
+	if (new_val != val) {
+		set_pte(sd->kpte, __pte(new_val));
+		cpa->flags |= CPA_FLUSHTLB;
 	}
 
 	return 0;
 }
 
-static int pageattr_pud_entry(pud_t *pud, unsigned long addr,
-			      unsigned long next, struct mm_walk *walk)
+int arch_split_large_page(struct cpa_data *cpa, struct cpa_split_data *sd)
 {
-	pud_t val = pudp_get(pud);
-
-	if (pud_leaf(val)) {
-		val = __pud(set_pageattr_masks(pud_val(val), walk));
-		set_pud(pud, val);
-	}
+	WARN_ON_ONCE(1);
 
-	return 0;
+	return -EINVAL;
 }
 
-static int pageattr_pmd_entry(pmd_t *pmd, unsigned long addr,
-			      unsigned long next, struct mm_walk *walk)
+void arch_change_pte(struct cpa_data *cpa, unsigned long address,
+		     pte_t *kpte, pte_t old_pte, bool nx, bool rw)
 {
-	pmd_t val = pmdp_get(pmd);
+	unsigned long val = pte_val(old_pte);
+	pte_t new_pte = __pte(set_pageattr_masks(cpa, val));
 
-	if (pmd_leaf(val)) {
-		val = __pmd(set_pageattr_masks(pmd_val(val), walk));
-		set_pmd(pmd, val);
-	}
+	cpa->pfn = pte_pfn(old_pte);
 
-	return 0;
+	if (pte_val(old_pte) != pte_val(new_pte)) {
+		set_pte(kpte, new_pte);
+		cpa->flags |= CPA_FLUSHTLB;
+	}
 }
 
-static int pageattr_pte_entry(pte_t *pte, unsigned long addr,
-			      unsigned long next, struct mm_walk *walk)
+int arch_cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,
+			   int primary)
 {
-	pte_t val = ptep_get(pte);
-
-	val = __pte(set_pageattr_masks(pte_val(val), walk));
-	set_pte(pte, val);
+	/*
+	 * The direct map has no page tables and vmalloc ranges may contain
+	 * holes. Both are silently skipped.
+	 */
+	cpa->numpages = 1;
 
 	return 0;
 }
 
-static int pageattr_pte_hole(unsigned long addr, unsigned long next,
-			     int depth, struct mm_walk *walk)
+int arch_cpa_process_alias(struct cpa_data *cpa)
 {
 	return 0;
 }
 
-static const struct mm_walk_ops pageattr_ops = {
-	.pgd_entry = pageattr_pgd_entry,
-	.p4d_entry = pageattr_p4d_entry,
-	.pud_entry = pageattr_pud_entry,
-	.pmd_entry = pageattr_pmd_entry,
-	.pte_entry = pageattr_pte_entry,
-	.pte_hole = pageattr_pte_hole,
-	.walk_lock = PGWALK_RDLOCK,
-};
-
-static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask, pgprot_t clear_mask)
+void arch_cpa_flush(struct cpa_data *cpa, int err)
 {
-	int ret;
-	unsigned long start = addr;
-	unsigned long end = start + PAGE_SIZE * numpages;
-	struct pageattr_masks masks = {
-		.set_mask = set_mask,
-		.clear_mask = clear_mask
-	};
-
-	if (!numpages)
-		return 0;
-
-	mmap_write_lock(&init_mm);
-	ret = walk_kernel_page_table_range(start, end, &pageattr_ops, NULL, &masks);
-	mmap_write_unlock(&init_mm);
-
-	flush_tlb_kernel_range(start, end);
+	if (err || cpa->force_flush_all ||
+	    (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY))) {
+		flush_tlb_all();
+		return;
+	}
 
-	return ret;
+	flush_tlb_kernel_range(*cpa->vaddr, *cpa->vaddr + cpa->numpages * PAGE_SIZE);
 }
 
 int set_memory_x(unsigned long addr, int numpages)
@@ -131,7 +88,7 @@ int set_memory_x(unsigned long addr, int numpages)
 	if (addr < vm_map_base)
 		return 0;
 
-	return __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_NO_EXEC));
+	return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_NO_EXEC), 0);
 }
 
 int set_memory_nx(unsigned long addr, int numpages)
@@ -139,7 +96,7 @@ int set_memory_nx(unsigned long addr, int numpages)
 	if (addr < vm_map_base)
 		return 0;
 
-	return __set_memory(addr, numpages, __pgprot(_PAGE_NO_EXEC), __pgprot(0));
+	return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_NO_EXEC), 0);
 }
 
 int set_memory_ro(unsigned long addr, int numpages)
@@ -147,7 +104,7 @@ int set_memory_ro(unsigned long addr, int numpages)
 	if (addr < vm_map_base)
 		return 0;
 
-	return __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_WRITE | _PAGE_DIRTY));
+	return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_WRITE | _PAGE_DIRTY), 0);
 }
 
 int set_memory_rw(unsigned long addr, int numpages)
@@ -155,84 +112,75 @@ int set_memory_rw(unsigned long addr, int numpages)
 	if (addr < vm_map_base)
 		return 0;
 
-	return __set_memory(addr, numpages, __pgprot(_PAGE_WRITE | _PAGE_DIRTY), __pgprot(0));
+	return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_WRITE | _PAGE_DIRTY), 0);
 }
 
 bool kernel_page_present(struct page *page)
 {
-	pgd_t *pgd;
-	p4d_t *p4d;
-	pud_t *pud;
-	pmd_t *pmd;
-	pte_t *pte;
+	unsigned int level;
 	unsigned long addr = (unsigned long)page_address(page);
+	pte_t *pte;
 
 	if (addr < vm_map_base)
 		return memblock_is_memory(__pa(addr));
 
-	pgd = pgd_offset_k(addr);
-	if (pgd_none(pgdp_get(pgd)))
-		return false;
-	if (pgd_leaf(pgdp_get(pgd)))
-		return true;
-
-	p4d = p4d_offset(pgd, addr);
-	if (p4d_none(p4dp_get(p4d)))
-		return false;
-	if (p4d_leaf(p4dp_get(p4d)))
-		return true;
-
-	pud = pud_offset(p4d, addr);
-	if (pud_none(pudp_get(pud)))
-		return false;
-	if (pud_leaf(pudp_get(pud)))
-		return true;
-
-	pmd = pmd_offset(pud, addr);
-	if (pmd_none(pmdp_get(pmd)))
+	pte = lookup_address(addr, &level);
+	if (!pte)
 		return false;
-	if (pmd_leaf(pmdp_get(pmd)))
-		return true;
 
-	pte = pte_offset_kernel(pmd, addr);
 	return pte_present(ptep_get(pte));
 }
 
 int set_direct_map_default_noflush(struct page *page)
 {
 	unsigned long addr = (unsigned long)page_address(page);
+	struct cpa_data cpa = { .vaddr = &addr,
+				.pgd = NULL,
+				.numpages = 1,
+				.mask_set = PAGE_KERNEL,
+				.mask_clr = __pgprot(0),
+				.flags = CPA_NO_CHECK_ALIAS };
 
 	if (addr < vm_map_base)
 		return 0;
 
-	return __set_memory(addr, 1, PAGE_KERNEL, __pgprot(0));
+	return __change_page_attr_set_clr(&cpa, 1);
 }
 
 int set_direct_map_invalid_noflush(struct page *page)
 {
 	unsigned long addr = (unsigned long)page_address(page);
+	struct cpa_data cpa = { .vaddr = &addr,
+				.pgd = NULL,
+				.numpages = 1,
+				.mask_set = __pgprot(0),
+				.mask_clr = __pgprot(_PAGE_PRESENT | _PAGE_VALID),
+				.flags = CPA_NO_CHECK_ALIAS };
 
 	if (addr < vm_map_base)
 		return 0;
 
-	return __set_memory(addr, 1, __pgprot(0), __pgprot(_PAGE_PRESENT | _PAGE_VALID));
+	return __change_page_attr_set_clr(&cpa, 1);
 }
 
 int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid)
 {
 	unsigned long addr = (unsigned long)page_address(page);
-	pgprot_t set, clear;
+	struct cpa_data cpa = { .vaddr = &addr,
+				.pgd = NULL,
+				.numpages = nr,
+				.flags = CPA_NO_CHECK_ALIAS };
 
 	if (addr < vm_map_base)
 		return 0;
 
 	if (valid) {
-		set = PAGE_KERNEL;
-		clear = __pgprot(0);
+		cpa.mask_set = PAGE_KERNEL;
+		cpa.mask_clr = __pgprot(0);
 	} else {
-		set = __pgprot(0);
-		clear = __pgprot(_PAGE_PRESENT | _PAGE_VALID);
+		cpa.mask_set = __pgprot(0);
+		cpa.mask_clr = __pgprot(_PAGE_PRESENT | _PAGE_VALID);
 	}
 
-	return __set_memory(addr, nr, set, clear);
+	return __change_page_attr_set_clr(&cpa, 1);
 }
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 3f0a647218e407..5d069bf290ce78 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -48,6 +48,7 @@ config RISCV
 	select ARCH_HAS_PTE_SPECIAL
 	select ARCH_HAS_SET_DIRECT_MAP if MMU
 	select ARCH_HAS_SET_MEMORY if MMU
+	select GENERIC_SET_MEMORY if MMU
 	select ARCH_HAS_STRICT_KERNEL_RWX if MMU
 	select ARCH_HAS_STRICT_MODULE_RWX if MMU
 	select ARCH_HAS_SYNC_CORE_BEFORE_USERMODE
diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index 5d5756bda82e3d..027b3e7e4f177d 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -819,6 +819,42 @@ static inline int pud_write(pud_t pud)
 	return pte_write(pud_pte(pud));
 }
 
+#define pgd_write pgd_write
+static inline int pgd_write(pgd_t pgd)
+{
+	return pgd_val(pgd) & _PAGE_WRITE;
+}
+
+#define p4d_write p4d_write
+static inline int p4d_write(p4d_t p4d)
+{
+	return p4d_val(p4d) & _PAGE_WRITE;
+}
+
+#define pgd_exec pgd_exec
+static inline int pgd_exec(pgd_t pgd)
+{
+	return !!(pgd_val(pgd) & _PAGE_EXEC);
+}
+
+#define p4d_exec p4d_exec
+static inline int p4d_exec(p4d_t p4d)
+{
+	return !!(p4d_val(p4d) & _PAGE_EXEC);
+}
+
+#define pud_exec pud_exec
+static inline int pud_exec(pud_t pud)
+{
+	return !!(pud_val(pud) & _PAGE_EXEC);
+}
+
+#define pmd_exec pmd_exec
+static inline int pmd_exec(pmd_t pmd)
+{
+	return !!(pmd_val(pmd) & _PAGE_EXEC);
+}
+
 #define pmd_dirty pmd_dirty
 static inline int pmd_dirty(pmd_t pmd)
 {
diff --git a/arch/riscv/include/asm/set_memory.h b/arch/riscv/include/asm/set_memory.h
index ef59e1716a2cfd..4ba46a33103433 100644
--- a/arch/riscv/include/asm/set_memory.h
+++ b/arch/riscv/include/asm/set_memory.h
@@ -7,6 +7,8 @@
 #define _ASM_RISCV_SET_MEMORY_H
 
 #ifndef __ASSEMBLER__
+#include <asm/page.h>
+#include <asm/pgtable.h>
 /*
  * Functions to change memory attributes.
  */
@@ -26,6 +28,11 @@ static __always_inline int set_kernel_memory(char *startp, char *endp,
 
 	return set_memory(start, num_pages);
 }
+
+static inline bool cpa_should_update_alias(unsigned long vaddr, unsigned long pfn)
+{
+	return !is_linear_mapping(vaddr);
+}
 #else
 static inline int set_memory_ro(unsigned long addr, int numpages) { return 0; }
 static inline int set_memory_rw(unsigned long addr, int numpages) { return 0; }
diff --git a/arch/riscv/mm/pageattr.c b/arch/riscv/mm/pageattr.c
index 3f76db3d276992..66a20fc3d80574 100644
--- a/arch/riscv/mm/pageattr.c
+++ b/arch/riscv/mm/pageattr.c
@@ -3,402 +3,242 @@
  * Copyright (C) 2019 SiFive
  */
 
-#include <linux/pagewalk.h>
+#include <linux/mm.h>
 #include <linux/pgtable.h>
-#include <linux/vmalloc.h>
+#include <linux/set_memory.h>
 #include <asm/tlbflush.h>
-#include <asm/bitops.h>
 #include <asm/set_memory.h>
 
-struct pageattr_masks {
-	pgprot_t set_mask;
-	pgprot_t clear_mask;
-};
-
-static unsigned long set_pageattr_masks(unsigned long val, struct mm_walk *walk)
-{
-	struct pageattr_masks *masks = walk->private;
-	unsigned long new_val = val;
-
-	new_val &= ~(pgprot_val(masks->clear_mask));
-	new_val |= (pgprot_val(masks->set_mask));
-
-	return new_val;
-}
-
-static int pageattr_p4d_entry(p4d_t *p4d, unsigned long addr,
-			      unsigned long next, struct mm_walk *walk)
+int arch_should_split_large_page(struct cpa_data *cpa, struct cpa_split_data *sd)
 {
-	p4d_t val = p4dp_get(p4d);
-
-	if (p4d_leaf(val)) {
-		val = __p4d(set_pageattr_masks(p4d_val(val), walk));
-		set_p4d(p4d, val);
-	}
+	pte_t old = ptep_get(sd->kpte);
+	pgprot_t old_prot = __pgprot(pte_val(old) & ~_PAGE_PFN_MASK);
+	pgprot_t new_prot = old_prot;
+	unsigned long old_pfn = pte_pfn(old);
+	unsigned long lpaddr, numpages;
 
-	return 0;
-}
+	pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr);
+	pgprot_val(new_prot) |= pgprot_val(cpa->mask_set);
 
-static int pageattr_pud_entry(pud_t *pud, unsigned long addr,
-			      unsigned long next, struct mm_walk *walk)
-{
-	pud_t val = pudp_get(pud);
-
-	if (pud_leaf(val)) {
-		val = __pud(set_pageattr_masks(pud_val(val), walk));
-		set_pud(pud, val);
-	}
+	/*
+	 * Record the pfn mapped at @address so the alias check can locate the
+	 * matching linear map entry.
+	 */
+	cpa->pfn = old_pfn + ((sd->address & (sd->psize - 1)) >> PAGE_SHIFT);
 
-	return 0;
-}
+	/* If the protections do not change, keep the large page intact. */
+	if (pgprot_val(new_prot) == pgprot_val(old_prot))
+		return 0;
 
-static int pageattr_pmd_entry(pmd_t *pmd, unsigned long addr,
-			      unsigned long next, struct mm_walk *walk)
-{
-	pmd_t val = pmdp_get(pmd);
+	/* If the request does not cover the whole large page, split it. */
+	lpaddr = sd->address & sd->pmask;
+	numpages = sd->psize >> PAGE_SHIFT;
+	if (sd->address != lpaddr || cpa->numpages != numpages)
+		return 1;
 
-	if (pmd_leaf(val)) {
-		val = __pmd(set_pageattr_masks(pmd_val(val), walk));
-		set_pmd(pmd, val);
-	}
+	/* The whole large page is covered: update it in place. */
+	set_pte(sd->kpte, pfn_pte(old_pfn, new_prot));
+	cpa->flags |= CPA_FLUSHTLB;
 
 	return 0;
 }
 
-static int pageattr_pte_entry(pte_t *pte, unsigned long addr,
-			      unsigned long next, struct mm_walk *walk)
+static void split_set_ptes(pte_t *ptep, unsigned long pfn, unsigned long pfninc,
+			   pgprot_t prot)
 {
-	pte_t val = ptep_get(pte);
-
-	val = __pte(set_pageattr_masks(pte_val(val), walk));
-	set_pte(pte, val);
+	unsigned int i;
 
-	return 0;
+	for (i = 0; i < PTRS_PER_PTE; ++i, ++ptep, pfn += pfninc)
+		set_pte(ptep, pfn_pte(pfn, prot));
 }
 
-static int pageattr_pte_hole(unsigned long addr, unsigned long next,
-			     int depth, struct mm_walk *walk)
+int arch_split_large_page(struct cpa_data *cpa, struct cpa_split_data *sd)
 {
-	/* Nothing to do here */
-	return 0;
-}
-
-static const struct mm_walk_ops pageattr_ops = {
-	.p4d_entry = pageattr_p4d_entry,
-	.pud_entry = pageattr_pud_entry,
-	.pmd_entry = pageattr_pmd_entry,
-	.pte_entry = pageattr_pte_entry,
-	.pte_hole = pageattr_pte_hole,
-	.walk_lock = PGWALK_RDLOCK,
-};
-
 #ifdef CONFIG_64BIT
-static int __split_linear_mapping_pmd(pud_t *pudp,
-				      unsigned long vaddr, unsigned long end)
-{
-	pmd_t *pmdp;
-	unsigned long next;
-
-	pmdp = pmd_offset(pudp, vaddr);
-
-	do {
-		next = pmd_addr_end(vaddr, end);
-
-		if (next - vaddr >= PMD_SIZE &&
-		    vaddr <= (vaddr & PMD_MASK) && end >= next)
-			continue;
-
-		if (pmd_leaf(pmdp_get(pmdp))) {
-			struct page *pte_page;
-			unsigned long pfn = _pmd_pfn(pmdp_get(pmdp));
-			pgprot_t prot = __pgprot(pmd_val(pmdp_get(pmdp)) & ~_PAGE_PFN_MASK);
-			pte_t *ptep_new;
-			int i;
-
-			pte_page = alloc_page(GFP_KERNEL);
-			if (!pte_page)
-				return -ENOMEM;
+	struct page *base = ptdesc_page(sd->ptdesc);
+	pte_t old = ptep_get(sd->kpte);
+	pgprot_t prot = __pgprot(pte_val(old) & ~_PAGE_PFN_MASK);
+	unsigned long pfn = pte_pfn(old);
+	unsigned long pfninc;
+
+	switch (sd->level) {
+	case PGTABLE_LEVEL_PMD:
+		pfninc = 1;
+		break;
+	case PGTABLE_LEVEL_PUD:
+		pfninc = PMD_SIZE >> PAGE_SHIFT;
+		break;
+	case PGTABLE_LEVEL_P4D:
+		pfninc = PUD_SIZE >> PAGE_SHIFT;
+		break;
+	default:
+		return -EINVAL;
+	}
 
-			ptep_new = (pte_t *)page_address(pte_page);
-			for (i = 0; i < PTRS_PER_PTE; ++i, ++ptep_new)
-				set_pte(ptep_new, pfn_pte(pfn + i, prot));
+	split_set_ptes((pte_t *)page_address(base), pfn, pfninc, prot);
 
-			smp_wmb();
+	smp_wmb();
+	set_pte(sd->kpte, pfn_pte(page_to_pfn(base), PAGE_TABLE));
 
-			set_pmd(pmdp, pfn_pmd(page_to_pfn(pte_page), PAGE_TABLE));
-		}
-	} while (pmdp++, vaddr = next, vaddr != end);
+	cpa->flags |= CPA_FLUSHTLB;
+	cpa->force_flush_all = 1;
 
 	return 0;
+#else
+	WARN_ON_ONCE(1);
+	return -EINVAL;
+#endif
 }
 
-static int __split_linear_mapping_pud(p4d_t *p4dp,
-				      unsigned long vaddr, unsigned long end)
+void arch_change_pte(struct cpa_data *cpa, unsigned long address,
+		     pte_t *kpte, pte_t old_pte, bool nx, bool rw)
 {
-	pud_t *pudp;
-	unsigned long next;
-	int ret;
-
-	pudp = pud_offset(p4dp, vaddr);
-
-	do {
-		next = pud_addr_end(vaddr, end);
-
-		if (next - vaddr >= PUD_SIZE &&
-		    vaddr <= (vaddr & PUD_MASK) && end >= next)
-			continue;
+	pgprot_t new_prot = __pgprot(pte_val(old_pte) & ~_PAGE_PFN_MASK);
+	unsigned long pfn = pte_pfn(old_pte);
+	pte_t new_pte;
 
-		if (pud_leaf(pudp_get(pudp))) {
-			struct page *pmd_page;
-			unsigned long pfn = _pud_pfn(pudp_get(pudp));
-			pgprot_t prot = __pgprot(pud_val(pudp_get(pudp)) & ~_PAGE_PFN_MASK);
-			pmd_t *pmdp_new;
-			int i;
+	pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr);
+	pgprot_val(new_prot) |= pgprot_val(cpa->mask_set);
 
-			pmd_page = alloc_page(GFP_KERNEL);
-			if (!pmd_page)
-				return -ENOMEM;
+	new_pte = pfn_pte(pfn, new_prot);
+	cpa->pfn = pfn;
 
-			pmdp_new = (pmd_t *)page_address(pmd_page);
-			for (i = 0; i < PTRS_PER_PMD; ++i, ++pmdp_new)
-				set_pmd(pmdp_new,
-					pfn_pmd(pfn + ((i * PMD_SIZE) >> PAGE_SHIFT), prot));
-
-			smp_wmb();
-
-			set_pud(pudp, pfn_pud(page_to_pfn(pmd_page), PAGE_TABLE));
-		}
-
-		ret = __split_linear_mapping_pmd(pudp, vaddr, next);
-		if (ret)
-			return ret;
-	} while (pudp++, vaddr = next, vaddr != end);
-
-	return 0;
+	if (pte_val(old_pte) != pte_val(new_pte)) {
+		set_pte(kpte, new_pte);
+		cpa->flags |= CPA_FLUSHTLB;
+	}
 }
 
-static int __split_linear_mapping_p4d(pgd_t *pgdp,
-				      unsigned long vaddr, unsigned long end)
+int arch_cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,
+			   int primary)
 {
-	p4d_t *p4dp;
-	unsigned long next;
-	int ret;
-
-	p4dp = p4d_offset(pgdp, vaddr);
-
-	do {
-		next = p4d_addr_end(vaddr, end);
-
-		/*
-		 * If [vaddr; end] contains [vaddr & P4D_MASK; next], we don't
-		 * need to split, we'll change the protections on the whole P4D.
-		 */
-		if (next - vaddr >= P4D_SIZE &&
-		    vaddr <= (vaddr & P4D_MASK) && end >= next)
-			continue;
-
-		if (p4d_leaf(p4dp_get(p4dp))) {
-			struct page *pud_page;
-			unsigned long pfn = _p4d_pfn(p4dp_get(p4dp));
-			pgprot_t prot = __pgprot(p4d_val(p4dp_get(p4dp)) & ~_PAGE_PFN_MASK);
-			pud_t *pudp_new;
-			int i;
-
-			pud_page = alloc_page(GFP_KERNEL);
-			if (!pud_page)
-				return -ENOMEM;
-
-			/*
-			 * Fill the pud level with leaf puds that have the same
-			 * protections as the leaf p4d.
-			 */
-			pudp_new = (pud_t *)page_address(pud_page);
-			for (i = 0; i < PTRS_PER_PUD; ++i, ++pudp_new)
-				set_pud(pudp_new,
-					pfn_pud(pfn + ((i * PUD_SIZE) >> PAGE_SHIFT), prot));
-
-			/*
-			 * Make sure the pud filling is not reordered with the
-			 * p4d store which could result in seeing a partially
-			 * filled pud level.
-			 */
-			smp_wmb();
-
-			set_p4d(p4dp, pfn_p4d(page_to_pfn(pud_page), PAGE_TABLE));
-		}
-
-		ret = __split_linear_mapping_pud(p4dp, vaddr, next);
-		if (ret)
-			return ret;
-	} while (p4dp++, vaddr = next, vaddr != end);
+	cpa->numpages = 1;
 
-	return 0;
-}
-
-static int __split_linear_mapping_pgd(pgd_t *pgdp,
-				      unsigned long vaddr,
-				      unsigned long end)
-{
-	unsigned long next;
-	int ret;
+	if (!primary)
+		return 0;
 
-	do {
-		next = pgd_addr_end(vaddr, end);
-		/* We never use PGD mappings for the linear mapping */
-		ret = __split_linear_mapping_p4d(pgdp, vaddr, next);
-		if (ret)
-			return ret;
-	} while (pgdp++, vaddr = next, vaddr != end);
+	/* The linear map is expected to have holes */
+	if (is_linear_mapping(vaddr)) {
+		cpa->pfn = PFN_DOWN(__pa(vaddr));
+		return 0;
+	}
 
-	return 0;
-}
+	WARN(1, "CPA: called for zero pte. vaddr = %lx cpa->vaddr = %lx\n",
+	     vaddr, *cpa->vaddr);
 
-static int split_linear_mapping(unsigned long start, unsigned long end)
-{
-	return __split_linear_mapping_pgd(pgd_offset_k(start), start, end);
+	return -EFAULT;
 }
-#endif	/* CONFIG_64BIT */
 
-static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask,
-			pgprot_t clear_mask)
+int arch_cpa_process_alias(struct cpa_data *cpa)
 {
-	int ret;
-	unsigned long start = addr;
-	unsigned long end = start + PAGE_SIZE * numpages;
-	unsigned long __maybe_unused lm_start;
-	unsigned long __maybe_unused lm_end;
-	struct pageattr_masks masks = {
-		.set_mask = set_mask,
-		.clear_mask = clear_mask
-	};
-
-	if (!numpages)
-		return 0;
+	struct cpa_data alias_cpa;
+	unsigned long laddr;
 
-	mmap_write_lock(&init_mm);
-
-#ifdef CONFIG_64BIT
 	/*
-	 * We are about to change the permissions of a kernel mapping, we must
-	 * apply the same changes to its linear mapping alias, which may imply
-	 * splitting a huge mapping.
+	 * cpa_should_update_alias() only lets non linear map primaries reach
+	 * here, so @cpa->pfn always has a linear map alias that must receive
+	 * the same protection change.
 	 */
+	laddr = (unsigned long)__va(PFN_PHYS(cpa->pfn));
 
-	if (is_vmalloc_or_module_addr((void *)start)) {
-		struct vm_struct *area = NULL;
-		int i, page_start;
-
-		area = find_vm_area((void *)start);
-		page_start = (start - (unsigned long)area->addr) >> PAGE_SHIFT;
-
-		for (i = page_start; i < page_start + numpages; ++i) {
-			lm_start = (unsigned long)page_address(area->pages[i]);
-			lm_end = lm_start + PAGE_SIZE;
-
-			ret = split_linear_mapping(lm_start, lm_end);
-			if (ret)
-				goto unlock;
-
-			ret = walk_kernel_page_table_range(lm_start, lm_end,
-						    &pageattr_ops, NULL, &masks);
-			if (ret)
-				goto unlock;
-		}
-	} else if (is_kernel_mapping(start) || is_linear_mapping(start)) {
-		if (is_kernel_mapping(start)) {
-			lm_start = (unsigned long)lm_alias(start);
-			lm_end = (unsigned long)lm_alias(end);
-		} else {
-			lm_start = start;
-			lm_end = end;
-		}
-
-		ret = split_linear_mapping(lm_start, lm_end);
-		if (ret)
-			goto unlock;
-
-		ret = walk_kernel_page_table_range(lm_start, lm_end,
-					    &pageattr_ops, NULL, &masks);
-		if (ret)
-			goto unlock;
-	}
-
-	ret =  walk_kernel_page_table_range(start, end, &pageattr_ops, NULL,
-				     &masks);
+	alias_cpa = *cpa;
+	alias_cpa.vaddr = &laddr;
+	alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY);
+	alias_cpa.curpage = 0;
 
-unlock:
-	mmap_write_unlock(&init_mm);
+	/* The linear map alias must never be made executable */
+	alias_cpa.mask_set = __pgprot(pgprot_val(alias_cpa.mask_set) & ~_PAGE_EXEC);
+	alias_cpa.mask_clr = __pgprot(pgprot_val(alias_cpa.mask_clr) & ~_PAGE_EXEC);
 
-	/*
-	 * We can't use flush_tlb_kernel_range() here as we may have split a
-	 * hugepage that is larger than that, so let's flush everything.
-	 */
-	flush_tlb_all();
-#else
-	ret =  walk_kernel_page_table_range(start, end, &pageattr_ops, NULL,
-				     &masks);
+	cpa->force_flush_all = 1;
 
-	mmap_write_unlock(&init_mm);
+	return __change_page_attr_set_clr(&alias_cpa, 0);
+}
 
-	flush_tlb_kernel_range(start, end);
-#endif
+void arch_cpa_flush(struct cpa_data *cpa, int err)
+{
+	if (err || cpa->force_flush_all ||
+	    (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY))) {
+		flush_tlb_all();
+		return;
+	}
 
-	return ret;
+	flush_tlb_kernel_range(*cpa->vaddr,
+			       *cpa->vaddr + cpa->numpages * PAGE_SIZE);
 }
 
 int set_memory_rw_nx(unsigned long addr, int numpages)
 {
-	return __set_memory(addr, numpages, __pgprot(_PAGE_READ | _PAGE_WRITE),
-			    __pgprot(_PAGE_EXEC));
+	return change_page_attr_set_clr(&addr, numpages,
+					__pgprot(_PAGE_READ | _PAGE_WRITE),
+					__pgprot(_PAGE_EXEC), 0, 0, NULL);
 }
 
 int set_memory_ro(unsigned long addr, int numpages)
 {
-	return __set_memory(addr, numpages, __pgprot(_PAGE_READ),
-			    __pgprot(_PAGE_WRITE));
+	return change_page_attr_set_clr(&addr, numpages, __pgprot(_PAGE_READ),
+					__pgprot(_PAGE_WRITE), 0, 0, NULL);
 }
 
 int set_memory_rw(unsigned long addr, int numpages)
 {
-	return __set_memory(addr, numpages, __pgprot(_PAGE_READ | _PAGE_WRITE),
-			    __pgprot(0));
+	return change_page_attr_set(&addr, numpages,
+				    __pgprot(_PAGE_READ | _PAGE_WRITE), 0);
 }
 
 int set_memory_x(unsigned long addr, int numpages)
 {
-	return __set_memory(addr, numpages, __pgprot(_PAGE_EXEC), __pgprot(0));
+	return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_EXEC), 0);
 }
 
 int set_memory_nx(unsigned long addr, int numpages)
 {
-	return __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_EXEC));
+	return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_EXEC), 0);
 }
 
 int set_direct_map_invalid_noflush(struct page *page)
 {
-	return __set_memory((unsigned long)page_address(page), 1,
-			    __pgprot(0), __pgprot(_PAGE_PRESENT));
+	unsigned long start = (unsigned long)page_address(page);
+	struct cpa_data cpa = { .vaddr = &start,
+				.pgd = NULL,
+				.numpages = 1,
+				.mask_set = __pgprot(0),
+				.mask_clr = __pgprot(_PAGE_PRESENT),
+				.flags = CPA_NO_CHECK_ALIAS };
+
+	return __change_page_attr_set_clr(&cpa, 1);
 }
 
 int set_direct_map_default_noflush(struct page *page)
 {
-	return __set_memory((unsigned long)page_address(page), 1,
-			    PAGE_KERNEL, __pgprot(_PAGE_EXEC));
+	unsigned long start = (unsigned long)page_address(page);
+	struct cpa_data cpa = { .vaddr = &start,
+				.pgd = NULL,
+				.numpages = 1,
+				.mask_set = PAGE_KERNEL,
+				.mask_clr = __pgprot(_PAGE_EXEC),
+				.flags = CPA_NO_CHECK_ALIAS };
+
+	return __change_page_attr_set_clr(&cpa, 1);
 }
 
 int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid)
 {
-	pgprot_t set, clear;
+	unsigned long start = (unsigned long)page_address(page);
+	struct cpa_data cpa = { .vaddr = &start,
+				.pgd = NULL,
+				.numpages = nr,
+				.flags = CPA_NO_CHECK_ALIAS };
 
 	if (valid) {
-		set = PAGE_KERNEL;
-		clear = __pgprot(_PAGE_EXEC);
+		cpa.mask_set = PAGE_KERNEL;
+		cpa.mask_clr = __pgprot(_PAGE_EXEC);
 	} else {
-		set = __pgprot(0);
-		clear = __pgprot(_PAGE_PRESENT);
+		cpa.mask_set = __pgprot(0);
+		cpa.mask_clr = __pgprot(_PAGE_PRESENT);
 	}
 
-	return __set_memory((unsigned long)page_address(page), nr, set, clear);
+	return __change_page_attr_set_clr(&cpa, 1);
 }
 
 #ifdef CONFIG_DEBUG_PAGEALLOC
@@ -435,36 +275,11 @@ void __kernel_map_pages(struct page *page, int numpages, int enable)
 bool kernel_page_present(struct page *page)
 {
 	unsigned long addr = (unsigned long)page_address(page);
-	pgd_t *pgd;
-	pud_t *pud;
-	p4d_t *p4d;
-	pmd_t *pmd;
-	pte_t *pte;
-
-	pgd = pgd_offset_k(addr);
-	if (!pgd_present(pgdp_get(pgd)))
-		return false;
-	if (pgd_leaf(pgdp_get(pgd)))
-		return true;
-
-	p4d = p4d_offset(pgd, addr);
-	if (!p4d_present(p4dp_get(p4d)))
-		return false;
-	if (p4d_leaf(p4dp_get(p4d)))
-		return true;
-
-	pud = pud_offset(p4d, addr);
-	if (!pud_present(pudp_get(pud)))
-		return false;
-	if (pud_leaf(pudp_get(pud)))
-		return true;
+	unsigned int level;
+	pte_t *pte = lookup_address(addr, &level);
 
-	pmd = pmd_offset(pud, addr);
-	if (!pmd_present(pmdp_get(pmd)))
+	if (!pte)
 		return false;
-	if (pmd_leaf(pmdp_get(pmd)))
-		return true;
 
-	pte = pte_offset_kernel(pmd, addr);
 	return pte_present(ptep_get(pte));
 }
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index bdad90f210e4bf..e1f105e52d6a12 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -336,6 +336,7 @@ config X86
 	select ARCH_SUPPORTS_SCHED_CLUSTER	if SMP
 	select ARCH_SUPPORTS_SCHED_MC		if SMP
 	select HAVE_SINGLE_FTRACE_DIRECT_OPS	if X86_64 && DYNAMIC_FTRACE_WITH_DIRECT_CALLS
+	select GENERIC_SET_MEMORY
 
 config INSTRUCTION_DECODER
 	def_bool y
diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug
index c95c3aaadf97c3..314b72aad6fff4 100644
--- a/arch/x86/Kconfig.debug
+++ b/arch/x86/Kconfig.debug
@@ -157,12 +157,6 @@ config DEBUG_BOOT_PARAMS
 	help
 	  This option will cause struct boot_params to be exported via debugfs.
 
-config CPA_DEBUG
-	bool "CPA self-test code"
-	depends on DEBUG_KERNEL
-	help
-	  Do change_page_attr() self-tests every 30 seconds.
-
 config DEBUG_ENTRY
 	bool "Debug low-level entry code"
 	depends on DEBUG_KERNEL
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index ac295ca6c92f99..497aaf08ad3616 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -227,6 +227,42 @@ static inline int pud_write(pud_t pud)
 	return pud_flags(pud) & _PAGE_RW;
 }
 
+#define p4d_write p4d_write
+static inline int p4d_write(p4d_t p4d)
+{
+	return p4d_flags(p4d) & _PAGE_RW;
+}
+
+#define pgd_write pgd_write
+static inline int pgd_write(pgd_t pgd)
+{
+	return pgd_flags(pgd) & _PAGE_RW;
+}
+
+#define pmd_exec pmd_exec
+static inline int pmd_exec(pmd_t pmd)
+{
+	return !(pmd_flags(pmd) & _PAGE_NX);
+}
+
+#define pud_exec pud_exec
+static inline int pud_exec(pud_t pud)
+{
+	return !(pud_flags(pud) & _PAGE_NX);
+}
+
+#define p4d_exec p4d_exec
+static inline int p4d_exec(p4d_t p4d)
+{
+	return !(p4d_flags(p4d) & _PAGE_NX);
+}
+
+#define pgd_exec pgd_exec
+static inline int pgd_exec(pgd_t pgd)
+{
+	return !(pgd_flags(pgd) & _PAGE_NX);
+}
+
 static inline int pte_huge(pte_t pte)
 {
 	return pte_flags(pte) & _PAGE_PSE;
diff --git a/arch/x86/include/asm/set_memory.h b/arch/x86/include/asm/set_memory.h
index 4362c26aa992db..332ec300c2ed9c 100644
--- a/arch/x86/include/asm/set_memory.h
+++ b/arch/x86/include/asm/set_memory.h
@@ -3,12 +3,62 @@
 #define _ASM_X86_SET_MEMORY_H
 
 #include <asm/page.h>
+#include <asm/sections.h>
 #include <asm-generic/set_memory.h>
 #include <asm/pgtable.h>
 
 #define set_memory_rox set_memory_rox
 int set_memory_rox(unsigned long addr, int numpages);
 
+#ifdef CONFIG_X86_64
+/*
+ * The kernel image is mapped into two places in the virtual address space
+ * (addresses without KASLR, of course):
+ *
+ * 1. The kernel direct map (0xffff880000000000)
+ * 2. The "high kernel map" (0xffffffff81000000)
+ *
+ * We actually execute out of #2. If we get the address of a kernel symbol, it
+ * points to #2, but almost all physical-to-virtual translations point to #1.
+ *
+ * This is so that we can have both a directmap of all physical memory *and*
+ * take full advantage of the limited (s32) immediate addressing range (2G)
+ * of x86_64.
+ *
+ * See Documentation/arch/x86/x86_64/mm.rst for more detail.
+ */
+static inline bool pfn_is_kernel(unsigned long pfn)
+{
+	unsigned long spfn = __pa_symbol(_text) >> PAGE_SHIFT;
+	/* Do not reference a physical address outside the kernel. */
+	unsigned long epfn = __pa_symbol(roundup(_brk_end, PMD_SIZE) - 1) >> PAGE_SHIFT;
+
+	return pfn >= spfn && pfn <= epfn;
+}
+#else
+static inline bool pfn_is_kernel(unsigned long pfn)
+{
+	/* There is no highmap on 32-bit */
+	return false;
+}
+#endif
+
+static inline bool cpa_should_update_alias(unsigned long vaddr,
+					   unsigned long pfn)
+{
+	/* Primary is not in the direct map, its direct map alias needs update */
+	if (vaddr < PAGE_OFFSET ||
+	    vaddr >= PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT))
+		return true;
+
+	/* direct map page that is also part of the kernel highmap */
+	if ((vaddr < (unsigned long)_text || vaddr >= _brk_end) &&
+	    pfn_is_kernel(pfn))
+		return true;
+
+	return false;
+}
+
 /*
  * The set_memory_* API can be used to change various attributes of a virtual
  * address range. The attributes include:
diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c
index d023a40a1e0345..c460121c96b949 100644
--- a/arch/x86/mm/pat/set_memory.c
+++ b/arch/x86/mm/pat/set_memory.c
@@ -38,21 +38,6 @@
 /*
  * The current flushing context - we pass it instead of 5 arguments:
  */
-struct cpa_data {
-	unsigned long	*vaddr;
-	pgd_t		*pgd;
-	pgprot_t	mask_set;
-	pgprot_t	mask_clr;
-	unsigned long	numpages;
-	unsigned long	curpage;
-	unsigned long	pfn;
-	unsigned int	flags;
-	unsigned int	force_split		: 1,
-			force_static_prot	: 1,
-			force_flush_all		: 1;
-	struct page	**pages;
-};
-
 enum cpa_warn {
 	CPA_CONFLICT,
 	CPA_PROTECT,
@@ -61,27 +46,13 @@ enum cpa_warn {
 
 static const int cpa_warn_level = CPA_PROTECT;
 
-/*
- * Serialize cpa() (for !DEBUG_PAGEALLOC which uses large identity mappings)
- * using cpa_lock. So that we don't allow any other cpu, with stale large tlb
- * entries change the page attribute in parallel to some other cpu
- * splitting a large page entry along with changing the attribute.
- */
-static DEFINE_SPINLOCK(cpa_lock);
-
-#define CPA_FLUSHTLB 1
-#define CPA_ARRAY 2
-#define CPA_PAGES_ARRAY 4
-#define CPA_NO_CHECK_ALIAS 8 /* Do not search for aliases */
-#define CPA_COLLAPSE 16 /* try to collapse large pages */
-
 static inline pgprot_t cachemode2pgprot(enum page_cache_mode pcm)
 {
 	return __pgprot(cachemode2protval(pcm));
 }
 
 #ifdef CONFIG_PROC_FS
-static unsigned long direct_pages_count[PG_LEVEL_NUM];
+static unsigned long direct_pages_count[PGTABLE_LEVEL_NUM];
 
 void update_page_count(int level, unsigned long pages)
 {
@@ -98,9 +69,9 @@ static void split_page_count(int level)
 
 	direct_pages_count[level]--;
 	if (system_state == SYSTEM_RUNNING) {
-		if (level == PG_LEVEL_2M)
+		if (level == PGTABLE_LEVEL_PMD)
 			count_vm_event(DIRECT_MAP_LEVEL2_SPLIT);
-		else if (level == PG_LEVEL_1G)
+		else if (level == PGTABLE_LEVEL_PUD)
 			count_vm_event(DIRECT_MAP_LEVEL3_SPLIT);
 	}
 	direct_pages_count[level - 1] += PTRS_PER_PTE;
@@ -110,9 +81,9 @@ static void collapse_page_count(int level)
 {
 	direct_pages_count[level]++;
 	if (system_state == SYSTEM_RUNNING) {
-		if (level == PG_LEVEL_2M)
+		if (level == PGTABLE_LEVEL_PMD)
 			count_vm_event(DIRECT_MAP_LEVEL2_COLLAPSE);
-		else if (level == PG_LEVEL_1G)
+		else if (level == PGTABLE_LEVEL_PUD)
 			count_vm_event(DIRECT_MAP_LEVEL3_COLLAPSE);
 	}
 	direct_pages_count[level - 1] -= PTRS_PER_PTE;
@@ -121,17 +92,17 @@ static void collapse_page_count(int level)
 void arch_report_meminfo(struct seq_file *m)
 {
 	seq_printf(m, "DirectMap4k:    %8lu kB\n",
-			direct_pages_count[PG_LEVEL_4K] << 2);
+			direct_pages_count[PGTABLE_LEVEL_PTE] << 2);
 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
 	seq_printf(m, "DirectMap2M:    %8lu kB\n",
-			direct_pages_count[PG_LEVEL_2M] << 11);
+			direct_pages_count[PGTABLE_LEVEL_PMD] << 11);
 #else
 	seq_printf(m, "DirectMap4M:    %8lu kB\n",
-			direct_pages_count[PG_LEVEL_2M] << 12);
+			direct_pages_count[PGTABLE_LEVEL_PMD] << 12);
 #endif
 	if (direct_gbpages)
 		seq_printf(m, "DirectMap1G:    %8lu kB\n",
-			direct_pages_count[PG_LEVEL_1G] << 20);
+			direct_pages_count[PGTABLE_LEVEL_PUD] << 20);
 }
 #else
 static inline void split_page_count(int level) { }
@@ -165,7 +136,7 @@ static inline void cpa_inc_4k_install(void)
 
 static inline void cpa_inc_lp_sameprot(int level)
 {
-	if (level == PG_LEVEL_1G)
+	if (level == PGTABLE_LEVEL_PUD)
 		cpa_1g_sameprot++;
 	else
 		cpa_2m_sameprot++;
@@ -173,7 +144,7 @@ static inline void cpa_inc_lp_sameprot(int level)
 
 static inline void cpa_inc_lp_preserved(int level)
 {
-	if (level == PG_LEVEL_1G)
+	if (level == PGTABLE_LEVEL_PUD)
 		cpa_1g_preserved++;
 	else
 		cpa_2m_preserved++;
@@ -225,61 +196,6 @@ within(unsigned long addr, unsigned long start, unsigned long end)
 	return addr >= start && addr < end;
 }
 
-#ifdef CONFIG_X86_64
-
-static inline int
-within_inclusive(unsigned long addr, unsigned long start, unsigned long end)
-{
-	return addr >= start && addr <= end;
-}
-
-/*
- * The kernel image is mapped into two places in the virtual address space
- * (addresses without KASLR, of course):
- *
- * 1. The kernel direct map (0xffff880000000000)
- * 2. The "high kernel map" (0xffffffff81000000)
- *
- * We actually execute out of #2. If we get the address of a kernel symbol, it
- * points to #2, but almost all physical-to-virtual translations point to #1.
- *
- * This is so that we can have both a directmap of all physical memory *and*
- * take full advantage of the limited (s32) immediate addressing range (2G)
- * of x86_64.
- *
- * See Documentation/arch/x86/x86_64/mm.rst for more detail.
- */
-
-static inline unsigned long highmap_start_pfn(void)
-{
-	return __pa_symbol(_text) >> PAGE_SHIFT;
-}
-
-static inline unsigned long highmap_end_pfn(void)
-{
-	/* Do not reference physical address outside the kernel. */
-	return __pa_symbol(roundup(_brk_end, PMD_SIZE) - 1) >> PAGE_SHIFT;
-}
-
-static bool __cpa_pfn_in_highmap(unsigned long pfn)
-{
-	/*
-	 * Kernel text has an alias mapping at a high address, known
-	 * here as "highmap".
-	 */
-	return within_inclusive(pfn, highmap_start_pfn(), highmap_end_pfn());
-}
-
-#else
-
-static bool __cpa_pfn_in_highmap(unsigned long pfn)
-{
-	/* There is no highmap on 32-bit */
-	return false;
-}
-
-#endif
-
 /*
  * See set_mce_nospec().
  *
@@ -303,23 +219,6 @@ static inline unsigned long fix_addr(unsigned long addr)
 #endif
 }
 
-static unsigned long __cpa_addr(struct cpa_data *cpa, unsigned long idx)
-{
-	if (cpa->flags & CPA_PAGES_ARRAY) {
-		struct page *page = cpa->pages[idx];
-
-		if (unlikely(PageHighMem(page)))
-			return 0;
-
-		return (unsigned long)page_address(page);
-	}
-
-	if (cpa->flags & CPA_ARRAY)
-		return cpa->vaddr[idx];
-
-	return *cpa->vaddr + idx * PAGE_SIZE;
-}
-
 /*
  * Flushing functions
  */
@@ -405,7 +304,7 @@ static void __cpa_flush_tlb(void *data)
 	unsigned int i;
 
 	for (i = 0; i < cpa->numpages; i++)
-		flush_tlb_one_kernel(fix_addr(__cpa_addr(cpa, i)));
+		flush_tlb_one_kernel(fix_addr(cpa_addr(cpa, i)));
 }
 
 static int collapse_large_pages(unsigned long addr, struct list_head *pgtables);
@@ -420,10 +319,10 @@ static void cpa_collapse_large_pages(struct cpa_data *cpa)
 
 	if (cpa->flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) {
 		for (i = 0; i < cpa->numpages; i++)
-			collapsed += collapse_large_pages(__cpa_addr(cpa, i),
+			collapsed += collapse_large_pages(cpa_addr(cpa, i),
 							  &pgtables);
 	} else {
-		addr = __cpa_addr(cpa, 0);
+		addr = cpa_addr(cpa, 0);
 		start = addr & PMD_MASK;
 		end = addr + PAGE_SIZE * cpa->numpages;
 
@@ -463,7 +362,7 @@ static void cpa_flush(struct cpa_data *cpa, int cache)
 
 	mb();
 	for (i = 0; i < cpa->numpages; i++) {
-		unsigned long addr = __cpa_addr(cpa, i);
+		unsigned long addr = cpa_addr(cpa, i);
 		unsigned int level;
 
 		pte_t *pte = lookup_address(addr, &level);
@@ -578,7 +477,7 @@ static pgprotval_t protect_kernel_text_ro(unsigned long start,
 	 * so the protections for kernel text and identity mappings have to
 	 * be the same.
 	 */
-	if (lookup_address(start, &level) && (level != PG_LEVEL_4K))
+	if (lookup_address(start, &level) && (level != PGTABLE_LEVEL_PTE))
 		return _PAGE_RW;
 	return 0;
 }
@@ -709,104 +608,6 @@ static inline pgprot_t verify_rwx(pgprot_t old, pgprot_t new, unsigned long star
 	return new;
 }
 
-/*
- * Lookup the page table entry for a virtual address in a specific pgd.
- * Return a pointer to the entry (or NULL if the entry does not exist),
- * the level of the entry, and the effective NX and RW bits of all
- * page table levels.
- */
-pte_t *lookup_address_in_pgd_attr(pgd_t *pgd, unsigned long address,
-				  unsigned int *level, bool *nx, bool *rw)
-{
-	p4d_t *p4d;
-	pud_t *pud;
-	pmd_t *pmd;
-
-	*level = PG_LEVEL_256T;
-	*nx = false;
-	*rw = true;
-
-	if (pgd_none(*pgd))
-		return NULL;
-
-	*level = PG_LEVEL_512G;
-	*nx |= pgd_flags(*pgd) & _PAGE_NX;
-	*rw &= pgd_flags(*pgd) & _PAGE_RW;
-
-	p4d = p4d_offset(pgd, address);
-	if (p4d_none(*p4d))
-		return NULL;
-
-	if (p4d_leaf(*p4d) || !p4d_present(*p4d))
-		return (pte_t *)p4d;
-
-	*level = PG_LEVEL_1G;
-	*nx |= p4d_flags(*p4d) & _PAGE_NX;
-	*rw &= p4d_flags(*p4d) & _PAGE_RW;
-
-	pud = pud_offset(p4d, address);
-	if (pud_none(*pud))
-		return NULL;
-
-	if (pud_leaf(*pud) || !pud_present(*pud))
-		return (pte_t *)pud;
-
-	*level = PG_LEVEL_2M;
-	*nx |= pud_flags(*pud) & _PAGE_NX;
-	*rw &= pud_flags(*pud) & _PAGE_RW;
-
-	pmd = pmd_offset(pud, address);
-	if (pmd_none(*pmd))
-		return NULL;
-
-	if (pmd_leaf(*pmd) || !pmd_present(*pmd))
-		return (pte_t *)pmd;
-
-	*level = PG_LEVEL_4K;
-	*nx |= pmd_flags(*pmd) & _PAGE_NX;
-	*rw &= pmd_flags(*pmd) & _PAGE_RW;
-
-	return pte_offset_kernel(pmd, address);
-}
-
-/*
- * Lookup the page table entry for a virtual address in a specific pgd.
- * Return a pointer to the entry and the level of the mapping.
- */
-pte_t *lookup_address_in_pgd(pgd_t *pgd, unsigned long address,
-			     unsigned int *level)
-{
-	bool nx, rw;
-
-	return lookup_address_in_pgd_attr(pgd, address, level, &nx, &rw);
-}
-
-/*
- * Lookup the page table entry for a virtual address. Return a pointer
- * to the entry and the level of the mapping.
- *
- * Note: the function returns p4d, pud or pmd either when the entry is marked
- * large or when the present bit is not set. Otherwise it returns NULL.
- */
-pte_t *lookup_address(unsigned long address, unsigned int *level)
-{
-	return lookup_address_in_pgd(pgd_offset_k(address), address, level);
-}
-EXPORT_SYMBOL_GPL(lookup_address);
-
-static pte_t *_lookup_address_cpa(struct cpa_data *cpa, unsigned long address,
-				  unsigned int *level, bool *nx, bool *rw)
-{
-	pgd_t *pgd;
-
-	if (!cpa->pgd)
-		pgd = pgd_offset_k(address);
-	else
-		pgd = cpa->pgd + pgd_index(address);
-
-	return lookup_address_in_pgd_attr(pgd, address, level, nx, rw);
-}
-
 /*
  * Lookup the PMD entry for a virtual address. Return a pointer to the entry
  * or NULL if not present.
@@ -850,9 +651,9 @@ pmd_t *lookup_pmd_address(unsigned long address)
 phys_addr_t slow_virt_to_phys(void *__virt_addr)
 {
 	unsigned long virt_addr = (unsigned long)__virt_addr;
+	enum pgtable_level level;
 	phys_addr_t phys_addr;
 	unsigned long offset;
-	enum pg_level level;
 	pte_t *pte;
 
 	pte = lookup_address(virt_addr, &level);
@@ -864,11 +665,11 @@ phys_addr_t slow_virt_to_phys(void *__virt_addr)
 	 * make 32-PAE kernel work correctly.
 	 */
 	switch (level) {
-	case PG_LEVEL_1G:
+	case PGTABLE_LEVEL_PUD:
 		phys_addr = (phys_addr_t)pud_pfn(*(pud_t *)pte) << PAGE_SHIFT;
 		offset = virt_addr & ~PUD_MASK;
 		break;
-	case PG_LEVEL_2M:
+	case PGTABLE_LEVEL_PMD:
 		phys_addr = (phys_addr_t)pmd_pfn(*(pmd_t *)pte) << PAGE_SHIFT;
 		offset = virt_addr & ~PMD_MASK;
 		break;
@@ -925,30 +726,27 @@ static pgprot_t pgprot_clear_protnone_bits(pgprot_t prot)
 	return prot;
 }
 
-static int __should_split_large_page(pte_t *kpte, unsigned long address,
-				     struct cpa_data *cpa)
+int arch_should_split_large_page(struct cpa_data *cpa,
+				 struct cpa_split_data *sd)
 {
-	unsigned long numpages, pmask, psize, lpaddr, pfn, old_pfn;
 	pgprot_t old_prot, new_prot, req_prot, chk_prot;
-	pte_t new_pte, *tmp;
-	enum pg_level level;
-	bool nx, rw;
-
-	/*
-	 * Check for races, another CPU might have split this page
-	 * up already:
-	 */
-	tmp = _lookup_address_cpa(cpa, address, &level, &nx, &rw);
-	if (tmp != kpte)
-		return 1;
+	unsigned long numpages, lpaddr, pfn, old_pfn;
+	enum pgtable_level level = sd->level;
+	unsigned long address = sd->address;
+	unsigned long psize = sd->psize;
+	unsigned long pmask = sd->pmask;
+	pte_t *kpte = sd->kpte;
+	bool nx = sd->nx;
+	bool rw = sd->rw;
+	pte_t new_pte;
 
 	switch (level) {
-	case PG_LEVEL_2M:
+	case PGTABLE_LEVEL_PMD:
 		old_prot = pmd_pgprot(*(pmd_t *)kpte);
 		old_pfn = pmd_pfn(*(pmd_t *)kpte);
 		cpa_inc_2m_checked();
 		break;
-	case PG_LEVEL_1G:
+	case PGTABLE_LEVEL_PUD:
 		old_prot = pud_pgprot(*(pud_t *)kpte);
 		old_pfn = pud_pfn(*(pud_t *)kpte);
 		cpa_inc_1g_checked();
@@ -957,18 +755,6 @@ static int __should_split_large_page(pte_t *kpte, unsigned long address,
 		return -EINVAL;
 	}
 
-	psize = page_level_size(level);
-	pmask = page_level_mask(level);
-
-	/*
-	 * Calculate the number of pages, which fit into this large
-	 * page starting at address:
-	 */
-	lpaddr = (address + psize) & pmask;
-	numpages = (lpaddr - address) >> PAGE_SHIFT;
-	if (numpages < cpa->numpages)
-		cpa->numpages = numpages;
-
 	/*
 	 * We are safe now. Check whether the new pgprot is the same:
 	 * Convert protection attributes to 4k-format, as cpa->mask* are set
@@ -1072,21 +858,6 @@ static int __should_split_large_page(pte_t *kpte, unsigned long address,
 	return 0;
 }
 
-static int should_split_large_page(pte_t *kpte, unsigned long address,
-				   struct cpa_data *cpa)
-{
-	int do_split;
-
-	if (cpa->force_split)
-		return 1;
-
-	spin_lock(&pgd_lock);
-	do_split = __should_split_large_page(kpte, address, cpa);
-	spin_unlock(&pgd_lock);
-
-	return do_split;
-}
-
 static void split_set_pte(struct cpa_data *cpa, pte_t *pte, unsigned long pfn,
 			  pgprot_t ref_prot, unsigned long address,
 			  unsigned long size)
@@ -1123,33 +894,22 @@ static void split_set_pte(struct cpa_data *cpa, pte_t *pte, unsigned long pfn,
 	set_pte(pte, pfn_pte(pfn, ref_prot));
 }
 
-static int
-__split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,
-		   struct ptdesc *ptdesc)
+int arch_split_large_page(struct cpa_data *cpa, struct cpa_split_data *sd)
 {
 	unsigned long lpaddr, lpinc, ref_pfn, pfn, pfninc = 1;
+	struct ptdesc *ptdesc = sd->ptdesc;
 	struct page *base = ptdesc_page(ptdesc);
 	pte_t *pbase = (pte_t *)page_address(base);
-	unsigned int i, level;
+	enum pgtable_level level = sd->level;
+	unsigned long address = sd->address;
+	pte_t *kpte = sd->kpte;
+	unsigned int i;
 	pgprot_t ref_prot;
-	bool nx, rw;
-	pte_t *tmp;
-
-	spin_lock(&pgd_lock);
-	/*
-	 * Check for races, another CPU might have split this page
-	 * up for us already:
-	 */
-	tmp = _lookup_address_cpa(cpa, address, &level, &nx, &rw);
-	if (tmp != kpte) {
-		spin_unlock(&pgd_lock);
-		return 1;
-	}
 
 	paravirt_alloc_pte(&init_mm, page_to_pfn(base));
 
 	switch (level) {
-	case PG_LEVEL_2M:
+	case PGTABLE_LEVEL_PMD:
 		ref_prot = pmd_pgprot(*(pmd_t *)kpte);
 		/*
 		 * Clear PSE (aka _PAGE_PAT) and move
@@ -1161,7 +921,7 @@ __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,
 		lpinc = PAGE_SIZE;
 		break;
 
-	case PG_LEVEL_1G:
+	case PGTABLE_LEVEL_PUD:
 		ref_prot = pud_pgprot(*(pud_t *)kpte);
 		ref_pfn = pud_pfn(*(pud_t *)kpte);
 		pfninc = PMD_SIZE >> PAGE_SHIFT;
@@ -1177,7 +937,6 @@ __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,
 		break;
 
 	default:
-		spin_unlock(&pgd_lock);
 		return 1;
 	}
 
@@ -1225,26 +984,6 @@ __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,
 	 * just split large page entry.
 	 */
 	flush_tlb_all();
-	spin_unlock(&pgd_lock);
-
-	return 0;
-}
-
-static int split_large_page(struct cpa_data *cpa, pte_t *kpte,
-			    unsigned long address)
-{
-	struct ptdesc *ptdesc;
-
-	if (!debug_pagealloc_enabled())
-		spin_unlock(&cpa_lock);
-	ptdesc = pagetable_alloc(GFP_KERNEL, 0);
-	if (!debug_pagealloc_enabled())
-		spin_lock(&cpa_lock);
-	if (!ptdesc)
-		return -ENOMEM;
-
-	if (__split_large_page(cpa, kpte, address, ptdesc))
-		pagetable_free(ptdesc);
 
 	return 0;
 }
@@ -1314,7 +1053,7 @@ static int collapse_pmd_page(pmd_t *pmd, unsigned long addr,
 	}
 
 	if (virt_addr_valid(addr) && pfn_range_is_mapped(pfn, pfn + 1))
-		collapse_page_count(PG_LEVEL_2M);
+		collapse_page_count(PGTABLE_LEVEL_PMD);
 
 	return 1;
 }
@@ -1361,7 +1100,7 @@ static int collapse_pud_page(pud_t *pud, unsigned long addr,
 	set_pud(pud, pfn_pud(pfn, pmd_pgprot(first)));
 
 	if (virt_addr_valid(addr) && pfn_range_is_mapped(pfn, pfn + 1))
-		collapse_page_count(PG_LEVEL_1G);
+		collapse_page_count(PGTABLE_LEVEL_PUD);
 
 	return 1;
 }
@@ -1800,8 +1539,8 @@ static int populate_pgd(struct cpa_data *cpa, unsigned long addr)
 	return 0;
 }
 
-static int __cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,
-			       int primary)
+int arch_cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,
+			   int primary)
 {
 	if (cpa->pgd) {
 		/*
@@ -1833,7 +1572,7 @@ static int __cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,
 		cpa->pfn = __pa(vaddr) >> PAGE_SHIFT;
 		return 0;
 
-	} else if (__cpa_pfn_in_highmap(cpa->pfn)) {
+	} else if (pfn_is_kernel(cpa->pfn)) {
 		/* Faults in the highmap are OK, so do not warn: */
 		return -EFAULT;
 	} else {
@@ -1845,90 +1584,45 @@ static int __cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,
 	}
 }
 
-static int __change_page_attr(struct cpa_data *cpa, int primary)
+void arch_change_pte(struct cpa_data *cpa, unsigned long address,
+		     pte_t *kpte, pte_t old_pte, bool nx, bool rw)
 {
-	unsigned long address;
-	int do_split, err;
-	unsigned int level;
-	pte_t *kpte, old_pte;
-	bool nx, rw;
-
-	address = __cpa_addr(cpa, cpa->curpage);
-repeat:
-	kpte = _lookup_address_cpa(cpa, address, &level, &nx, &rw);
-	if (!kpte)
-		return __cpa_process_fault(cpa, address, primary);
+	pte_t new_pte;
+	pgprot_t old_prot = pte_pgprot(old_pte);
+	pgprot_t new_prot = pte_pgprot(old_pte);
+	unsigned long pfn = pte_pfn(old_pte);
 
-	old_pte = *kpte;
-	if (pte_none(old_pte))
-		return __cpa_process_fault(cpa, address, primary);
+	pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr);
+	pgprot_val(new_prot) |= pgprot_val(cpa->mask_set);
 
-	if (level == PG_LEVEL_4K) {
-		pte_t new_pte;
-		pgprot_t old_prot = pte_pgprot(old_pte);
-		pgprot_t new_prot = pte_pgprot(old_pte);
-		unsigned long pfn = pte_pfn(old_pte);
-
-		pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr);
-		pgprot_val(new_prot) |= pgprot_val(cpa->mask_set);
-
-		cpa_inc_4k_install();
-		/* Hand in lpsize = 0 to enforce the protection mechanism */
-		new_prot = static_protections(new_prot, address, pfn, 1, 0,
-					      CPA_PROTECT);
-
-		new_prot = verify_rwx(old_prot, new_prot, address, pfn, 1,
-				      nx, rw);
+	cpa_inc_4k_install();
+	/* Hand in lpsize = 0 to enforce the protection mechanism */
+	new_prot = static_protections(new_prot, address, pfn, 1, 0,
+				      CPA_PROTECT);
 
-		new_prot = pgprot_clear_protnone_bits(new_prot);
+	new_prot = verify_rwx(old_prot, new_prot, address, pfn, 1, nx, rw);
 
-		/*
-		 * We need to keep the pfn from the existing PTE,
-		 * after all we're only going to change its attributes
-		 * not the memory it points to
-		 */
-		new_pte = pfn_pte(pfn, new_prot);
-		cpa->pfn = pfn;
-		/*
-		 * Do we really change anything ?
-		 */
-		if (pte_val(old_pte) != pte_val(new_pte)) {
-			set_pte_atomic(kpte, new_pte);
-			cpa->flags |= CPA_FLUSHTLB;
-		}
-		cpa->numpages = 1;
-		return 0;
-	}
+	new_prot = pgprot_clear_protnone_bits(new_prot);
 
 	/*
-	 * Check, whether we can keep the large page intact
-	 * and just change the pte:
-	 */
-	do_split = should_split_large_page(kpte, address, cpa);
-	/*
-	 * When the range fits into the existing large page,
-	 * return. cp->numpages and cpa->tlbflush have been updated in
-	 * try_large_page:
+	 * We need to keep the pfn from the existing PTE, after all we're only
+	 * going to change its attributes not the memory it points to
 	 */
-	if (do_split <= 0)
-		return do_split;
-
+	new_pte = pfn_pte(pfn, new_prot);
+	cpa->pfn = pfn;
 	/*
-	 * We have to split the large page:
+	 * Do we really change anything ?
 	 */
-	err = split_large_page(cpa, kpte, address);
-	if (!err)
-		goto repeat;
-
-	return err;
+	if (pte_val(old_pte) != pte_val(new_pte)) {
+		set_pte_atomic(kpte, new_pte);
+		cpa->flags |= CPA_FLUSHTLB;
+	}
 }
 
-static int __change_page_attr_set_clr(struct cpa_data *cpa, int primary);
-
 /*
  * Check the directmap and "high kernel map" 'aliases'.
  */
-static int cpa_process_alias(struct cpa_data *cpa)
+int arch_cpa_process_alias(struct cpa_data *cpa)
 {
 	struct cpa_data alias_cpa;
 	unsigned long laddr = (unsigned long)__va(cpa->pfn << PAGE_SHIFT);
@@ -1942,7 +1636,7 @@ static int cpa_process_alias(struct cpa_data *cpa)
 	 * No need to redo, when the primary call touched the direct
 	 * mapping already:
 	 */
-	vaddr = __cpa_addr(cpa, cpa->curpage);
+	vaddr = cpa_addr(cpa, cpa->curpage);
 	if (!(within(vaddr, PAGE_OFFSET,
 		    PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT)))) {
 
@@ -1971,7 +1665,7 @@ static int cpa_process_alias(struct cpa_data *cpa)
 	 * to touch the high mapped kernel as well:
 	 */
 	if (!within(vaddr, (unsigned long)_text, _brk_end) &&
-	    __cpa_pfn_in_highmap(cpa->pfn)) {
+	    pfn_is_kernel(cpa->pfn)) {
 		unsigned long temp_cpa_vaddr = (cpa->pfn << PAGE_SHIFT) +
 					       __START_KERNEL_map - phys_base;
 		alias_cpa = *cpa;
@@ -2000,168 +1694,25 @@ static int cpa_process_alias(struct cpa_data *cpa)
 	return 0;
 }
 
-static int __change_page_attr_set_clr(struct cpa_data *cpa, int primary)
-{
-	unsigned long numpages = cpa->numpages;
-	unsigned long rempages = numpages;
-	int ret = 0;
-
-	/*
-	 * No changes, easy!
-	 */
-	if (!(pgprot_val(cpa->mask_set) | pgprot_val(cpa->mask_clr)) &&
-	    !cpa->force_split)
-		return ret;
-
-	while (rempages) {
-		/*
-		 * Store the remaining nr of pages for the large page
-		 * preservation check.
-		 */
-		cpa->numpages = rempages;
-		/* for array changes, we can't use large page */
-		if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY))
-			cpa->numpages = 1;
-
-		if (!debug_pagealloc_enabled())
-			spin_lock(&cpa_lock);
-		ret = __change_page_attr(cpa, primary);
-		if (!debug_pagealloc_enabled())
-			spin_unlock(&cpa_lock);
-		if (ret)
-			goto out;
-
-		if (primary && !(cpa->flags & CPA_NO_CHECK_ALIAS)) {
-			ret = cpa_process_alias(cpa);
-			if (ret)
-				goto out;
-		}
-
-		/*
-		 * Adjust the number of pages with the result of the
-		 * CPA operation. Either a large page has been
-		 * preserved or a single page update happened.
-		 */
-		BUG_ON(cpa->numpages > rempages || !cpa->numpages);
-		rempages -= cpa->numpages;
-		cpa->curpage += cpa->numpages;
-	}
-
-out:
-	/* Restore the original numpages */
-	cpa->numpages = numpages;
-	return ret;
-}
-
-static int change_page_attr_set_clr(unsigned long *addr, int numpages,
-				    pgprot_t mask_set, pgprot_t mask_clr,
-				    int force_split, int in_flag,
-				    struct page **pages)
+void arch_cpa_flush(struct cpa_data *cpa, int err)
 {
-	struct cpa_data cpa;
-	int ret, cache;
-
-	memset(&cpa, 0, sizeof(cpa));
-
-	/*
-	 * Check, if we are requested to set a not supported
-	 * feature.  Clearing non-supported features is OK.
-	 */
-	mask_set = canon_pgprot(mask_set);
-
-	if (!pgprot_val(mask_set) && !pgprot_val(mask_clr) && !force_split)
-		return 0;
-
-	/* Ensure we are PAGE_SIZE aligned */
-	if (in_flag & CPA_ARRAY) {
-		int i;
-		for (i = 0; i < numpages; i++) {
-			if (addr[i] & ~PAGE_MASK) {
-				addr[i] &= PAGE_MASK;
-				WARN_ON_ONCE(1);
-			}
-		}
-	} else if (!(in_flag & CPA_PAGES_ARRAY)) {
-		/*
-		 * in_flag of CPA_PAGES_ARRAY implies it is aligned.
-		 * No need to check in that case
-		 */
-		if (*addr & ~PAGE_MASK) {
-			*addr &= PAGE_MASK;
-			/*
-			 * People should not be passing in unaligned addresses:
-			 */
-			WARN_ON_ONCE(1);
-		}
-	}
-
-	/* Must avoid aliasing mappings in the highmem code */
-	kmap_flush_unused();
-
-	vm_unmap_aliases();
-
-	cpa.vaddr = addr;
-	cpa.pages = pages;
-	cpa.numpages = numpages;
-	cpa.mask_set = mask_set;
-	cpa.mask_clr = mask_clr;
-	cpa.flags = in_flag;
-	cpa.curpage = 0;
-	cpa.force_split = force_split;
-
-	ret = __change_page_attr_set_clr(&cpa, 1);
-
-	/*
-	 * Check whether we really changed something:
-	 */
-	if (!(cpa.flags & CPA_FLUSHTLB))
-		goto out;
+	int cache;
 
 	/*
 	 * No need to flush, when we did not set any of the caching
 	 * attributes:
 	 */
-	cache = !!pgprot2cachemode(mask_set);
+	cache = !!pgprot2cachemode(cpa->mask_set);
 
 	/*
 	 * On error; flush everything to be sure.
 	 */
-	if (ret) {
+	if (err) {
 		cpa_flush_all(cache);
-		goto out;
+		return;
 	}
 
-	cpa_flush(&cpa, cache);
-out:
-	return ret;
-}
-
-static inline int change_page_attr_set(unsigned long *addr, int numpages,
-				       pgprot_t mask, int array)
-{
-	return change_page_attr_set_clr(addr, numpages, mask, __pgprot(0), 0,
-		(array ? CPA_ARRAY : 0), NULL);
-}
-
-static inline int change_page_attr_clear(unsigned long *addr, int numpages,
-					 pgprot_t mask, int array)
-{
-	return change_page_attr_set_clr(addr, numpages, __pgprot(0), mask, 0,
-		(array ? CPA_ARRAY : 0), NULL);
-}
-
-static inline int cpa_set_pages_array(struct page **pages, int numpages,
-				       pgprot_t mask)
-{
-	return change_page_attr_set_clr(NULL, numpages, mask, __pgprot(0), 0,
-		CPA_PAGES_ARRAY, pages);
-}
-
-static inline int cpa_clear_pages_array(struct page **pages, int numpages,
-					 pgprot_t mask)
-{
-	return change_page_attr_set_clr(NULL, numpages, __pgprot(0), mask, 0,
-		CPA_PAGES_ARRAY, pages);
+	cpa_flush(cpa, cache);
 }
 
 int _set_memory_uc(unsigned long addr, int numpages)
@@ -2773,11 +2324,3 @@ int __init kernel_unmap_pages_in_pgd(pgd_t *pgd, unsigned long address,
 
 	return retval;
 }
-
-/*
- * The testcases use internal knowledge of the implementation that shouldn't
- * be exposed to the rest of the kernel. Include these directly here.
- */
-#ifdef CONFIG_CPA_DEBUG
-#include "cpa-test.c"
-#endif
diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h
index 2981e386da7b3c..d1df8345852be8 100644
--- a/include/linux/pgtable.h
+++ b/include/linux/pgtable.h
@@ -1782,6 +1782,10 @@ void arch_sync_kernel_mappings(unsigned long start, unsigned long end);
  * because these macros can be used even if CONFIG_MMU is not defined.
  */
 
+#ifndef canon_pgprot
+#define canon_pgprot(prot)	(prot)
+#endif
+
 #ifndef pgprot_nx
 #define pgprot_nx(prot)	(prot)
 #endif
@@ -2105,6 +2109,46 @@ static inline int pud_write(pud_t pud)
 }
 #endif /* pud_write */
 
+#ifndef p4d_write
+static inline int p4d_write(p4d_t p4d)
+{
+	BUG();
+	return 0;
+}
+#endif /* p4d_write */
+
+#ifndef pmd_exec
+static inline int pmd_exec(pmd_t pmd)
+{
+	BUG();
+	return 0;
+}
+#endif /* pmd_exec */
+
+#ifndef pud_exec
+static inline int pud_exec(pud_t pud)
+{
+	BUG();
+	return 0;
+}
+#endif /* pud_exec */
+
+#ifndef p4d_exec
+static inline int p4d_exec(p4d_t p4d)
+{
+	BUG();
+	return 0;
+}
+#endif /* p4d_exec */
+
+#ifndef pgd_exec
+static inline int pgd_exec(pgd_t pgd)
+{
+	BUG();
+	return 0;
+}
+#endif /* pgd_exec */
+
 #if !defined(CONFIG_TRANSPARENT_HUGEPAGE) || \
 	!defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
 static inline int pud_trans_huge(pud_t pud)
@@ -2295,16 +2339,20 @@ static inline bool arch_has_pfn_modify_check(void)
 typedef unsigned int pgtbl_mod_mask;
 
 enum pgtable_level {
-	PGTABLE_LEVEL_PTE = 0,
+	PGTABLE_LEVEL_NONE = 0,
+	PGTABLE_LEVEL_PTE,
 	PGTABLE_LEVEL_PMD,
 	PGTABLE_LEVEL_PUD,
 	PGTABLE_LEVEL_P4D,
 	PGTABLE_LEVEL_PGD,
+	PGTABLE_LEVEL_NUM,
 };
 
 static inline const char *pgtable_level_to_str(enum pgtable_level level)
 {
 	switch (level) {
+	case PGTABLE_LEVEL_NONE:
+		return "none";
 	case PGTABLE_LEVEL_PTE:
 		return "pte";
 	case PGTABLE_LEVEL_PMD:
@@ -2320,6 +2368,28 @@ static inline const char *pgtable_level_to_str(enum pgtable_level level)
 	}
 }
 
+/* Size of the memory mapped by a single leaf entry at @level */
+static inline unsigned long pgtable_level_size(enum pgtable_level level)
+{
+	switch (level) {
+	case PGTABLE_LEVEL_PTE:
+		return PAGE_SIZE;
+	case PGTABLE_LEVEL_PMD:
+		return PMD_SIZE;
+	case PGTABLE_LEVEL_PUD:
+		return PUD_SIZE;
+	case PGTABLE_LEVEL_P4D:
+		return P4D_SIZE;
+	default:
+		return PGDIR_SIZE;
+	}
+}
+
+static inline unsigned long pgtable_level_mask(enum pgtable_level level)
+{
+	return ~(pgtable_level_size(level) - 1);
+}
+
 #endif /* !__ASSEMBLY__ */
 
 #if !defined(MAX_POSSIBLE_PHYSMEM_BITS) && !defined(CONFIG_64BIT)
diff --git a/include/linux/set_memory.h b/include/linux/set_memory.h
index 3030d9245f5ac8..17cbe6e26cb732 100644
--- a/include/linux/set_memory.h
+++ b/include/linux/set_memory.h
@@ -84,4 +84,101 @@ static inline int set_memory_decrypted(unsigned long addr, int numpages)
 }
 #endif /* CONFIG_ARCH_HAS_MEM_ENCRYPT */
 
+#ifdef CONFIG_GENERIC_SET_MEMORY
+
+#include <linux/pgtable.h>
+
+#define CPA_FLUSHTLB 1
+#define CPA_ARRAY 2
+#define CPA_PAGES_ARRAY 4
+#define CPA_NO_CHECK_ALIAS 8 /* Do not search for aliases */
+#define CPA_COLLAPSE 16 /* try to collapse large pages */
+
+/*
+ * The current flushing context - we pass it instead of 5 arguments:
+ */
+struct cpa_data {
+	unsigned long	*vaddr;
+	pgd_t		*pgd;
+	pgprot_t	mask_set;
+	pgprot_t	mask_clr;
+	unsigned long	numpages;
+	unsigned long	curpage;
+	unsigned long	pfn;
+	unsigned int	flags;
+	unsigned int	force_split		: 1,
+			force_static_prot	: 1,
+			force_flush_all		: 1;
+	struct page	**pages;
+};
+
+/*
+ * context for checking for a split a large page and splitting it if required.
+ */
+struct ptdesc;
+struct cpa_split_data {
+	unsigned long	address;
+	unsigned long	psize;
+	unsigned long	pmask;
+	pte_t		*kpte;
+	struct ptdesc	*ptdesc;
+	enum pgtable_level level;
+	bool		nx;
+	bool		rw;
+};
+
+unsigned long cpa_addr(struct cpa_data *cpa, unsigned long idx);
+
+pte_t *lookup_address(unsigned long address, unsigned int *level);
+pte_t *lookup_address_in_pgd(pgd_t *pgd, unsigned long address,
+			     unsigned int *level);
+pte_t *lookup_address_in_pgd_attr(pgd_t *pgd, unsigned long address,
+				  unsigned int *level, bool *nx, bool *rw);
+
+int arch_cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,
+			   int primary);
+int arch_cpa_process_alias(struct cpa_data *cpa);
+int arch_should_split_large_page(struct cpa_data *cpa,
+				 struct cpa_split_data *sd);
+int arch_split_large_page(struct cpa_data *cpa, struct cpa_split_data *sd);
+void arch_change_pte(struct cpa_data *cpa, unsigned long address,
+		     pte_t *kpte, pte_t old_pte, bool nx, bool rw);
+void arch_cpa_flush(struct cpa_data *cpa, int err);
+
+int __change_page_attr_set_clr(struct cpa_data *cpa, int primary);
+int change_page_attr_set_clr(unsigned long *addr, int numpages,
+			     pgprot_t mask_set, pgprot_t mask_clr,
+			     int force_split, int in_flag,
+			     struct page **pages);
+
+static inline int change_page_attr_set(unsigned long *addr, int numpages,
+				       pgprot_t mask, int array)
+{
+	return change_page_attr_set_clr(addr, numpages, mask, __pgprot(0), 0,
+		(array ? CPA_ARRAY : 0), NULL);
+}
+
+static inline int change_page_attr_clear(unsigned long *addr, int numpages,
+					 pgprot_t mask, int array)
+{
+	return change_page_attr_set_clr(addr, numpages, __pgprot(0), mask, 0,
+		(array ? CPA_ARRAY : 0), NULL);
+}
+
+static inline int cpa_set_pages_array(struct page **pages, int numpages,
+				      pgprot_t mask)
+{
+	return change_page_attr_set_clr(NULL, numpages, mask, __pgprot(0), 0,
+		CPA_PAGES_ARRAY, pages);
+}
+
+static inline int cpa_clear_pages_array(struct page **pages, int numpages,
+					pgprot_t mask)
+{
+	return change_page_attr_set_clr(NULL, numpages, __pgprot(0), mask, 0,
+		CPA_PAGES_ARRAY, pages);
+}
+
+#endif /* CONFIG_GENERIC_SET_MEMORY */
+
 #endif /* _LINUX_SET_MEMORY_H_ */
diff --git a/mm/Kconfig b/mm/Kconfig
index 9e0ca48249054f..541edd1e11b610 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -1509,6 +1509,9 @@ config LAZY_MMU_MODE_KUNIT_TEST
 
 	  If unsure, say N.
 
+config GENERIC_SET_MEMORY
+	bool
+
 source "mm/damon/Kconfig"
 
 endmenu
diff --git a/mm/Kconfig.debug b/mm/Kconfig.debug
index 91b3e027b75357..83cfef6b2d8cfb 100644
--- a/mm/Kconfig.debug
+++ b/mm/Kconfig.debug
@@ -320,3 +320,10 @@ config PER_VMA_LOCK_STATS
 	  overhead in the page fault path.
 
 	  If in doubt, say N.
+
+config CPA_DEBUG
+	bool "CPA self-test code"
+	depends on GENERIC_SET_MEMORY
+	depends on DEBUG_KERNEL
+	help
+	  Do change_page_attr() self-tests every 30 seconds.
diff --git a/mm/Makefile b/mm/Makefile
index eff9f9e7e061c1..e3f6c39c2ba991 100644
--- a/mm/Makefile
+++ b/mm/Makefile
@@ -147,3 +147,4 @@ obj-$(CONFIG_SHRINKER_DEBUG) += shrinker_debug.o
 obj-$(CONFIG_EXECMEM) += execmem.o
 obj-$(CONFIG_TMPFS_QUOTA) += shmem_quota.o
 obj-$(CONFIG_LAZY_MMU_MODE_KUNIT_TEST) += tests/lazy_mmu_mode_kunit.o
+obj-$(CONFIG_GENERIC_SET_MEMORY) += set_memory.o
diff --git a/arch/x86/mm/pat/cpa-test.c b/mm/cpa-test.c
similarity index 80%
rename from arch/x86/mm/pat/cpa-test.c
rename to mm/cpa-test.c
index ad3c1feec990db..63c3463a5a7d85 100644
--- a/arch/x86/mm/pat/cpa-test.c
+++ b/mm/cpa-test.c
@@ -13,32 +13,24 @@
 #include <linux/mm.h>
 #include <linux/vmalloc.h>
 
-#include <asm/cacheflush.h>
-#include <asm/kdebug.h>
-
 /*
  * Only print the results of the first pass:
  */
 static __read_mostly int print = 1;
+static unsigned long max_test_pfn;
 
 enum {
 	NTEST			= 3 * 100,
 	NPAGES			= 100,
-#ifdef CONFIG_X86_64
-	LPS			= (1 << PMD_SHIFT),
-#elif defined(CONFIG_X86_PAE)
-	LPS			= (1 << PMD_SHIFT),
-#else
-	LPS			= (1 << 22),
-#endif
-	GPS			= (1<<30)
+	LPS			= PMD_SIZE,
+	GPS			= PUD_SIZE
 };
 
-#define PAGE_CPA_TEST	__pgprot(_PAGE_CPA_TEST)
+#define PAGE_CPA_TEST	__pgprot(_PAGE_SPECIAL)
 
 static int pte_testbit(pte_t pte)
 {
-	return pte_flags(pte) & _PAGE_SOFTW1;
+	return pte_special(pte);
 }
 
 struct split_state {
@@ -54,10 +46,10 @@ static int print_split(struct split_state *s)
 	s->lpg = s->gpg = s->spg = s->exec = 0;
 	s->min_exec = ~0UL;
 	s->max_exec = 0;
-	for (i = 0; i < max_pfn_mapped; ) {
+	for (i = 0; i < max_test_pfn; ) {
 		unsigned long addr = (unsigned long)__va(i << PAGE_SHIFT);
 		unsigned int level;
-		pte_t *pte;
+		pte_t *pte, pteval;
 
 		pte = lookup_address(addr, &level);
 		if (!pte) {
@@ -65,15 +57,16 @@ static int print_split(struct split_state *s)
 			i++;
 			continue;
 		}
+		pteval = ptep_get(pte);
 
-		if (level == PG_LEVEL_1G && sizeof(long) == 8) {
+		if (level == PGTABLE_LEVEL_PUD && sizeof(long) == 8) {
 			s->gpg++;
 			i += GPS/PAGE_SIZE;
-		} else if (level == PG_LEVEL_2M) {
-			if ((pte_val(*pte) & _PAGE_PRESENT) && !(pte_val(*pte) & _PAGE_PSE)) {
+		} else if (level == PGTABLE_LEVEL_PMD) {
+			if (pte_present(pteval) && !pte_huge(pteval)) {
 				printk(KERN_ERR
-					"%lx level %d but not PSE %Lx\n",
-					addr, level, (u64)pte_val(*pte));
+					"%lx level %d but not leaf %Lx\n",
+					addr, level, (u64)pte_val(pteval));
 				err = 1;
 			}
 			s->lpg++;
@@ -82,7 +75,7 @@ static int print_split(struct split_state *s)
 			s->spg++;
 			i++;
 		}
-		if (!(pte_val(*pte) & _PAGE_NX)) {
+		if (pte_exec(pteval)) {
 			s->exec++;
 			if (addr < s->min_exec)
 				s->min_exec = addr;
@@ -100,8 +93,8 @@ static int print_split(struct split_state *s)
 
 	expected = (s->gpg*GPS + s->lpg*LPS)/PAGE_SIZE + s->spg + missed;
 	if (expected != i) {
-		printk(KERN_ERR "CPA max_pfn_mapped %lu but expected %lu\n",
-			max_pfn_mapped, expected);
+		printk(KERN_ERR "CPA max_test_pfn %lu but expected %lu\n",
+			max_test_pfn, expected);
 		return 1;
 	}
 	return err;
@@ -118,7 +111,7 @@ static int pageattr_test(void)
 {
 	struct split_state sa, sb, sc;
 	unsigned long *bm;
-	pte_t *pte, pte0;
+	pte_t *pte, pte0, pte_val;
 	int failed = 0;
 	unsigned int level;
 	int i, k;
@@ -127,7 +120,7 @@ static int pageattr_test(void)
 	if (print)
 		printk(KERN_INFO "CPA self-test:\n");
 
-	bm = vzalloc((max_pfn_mapped + 7) / 8);
+	bm = vzalloc((max_test_pfn + 7) / 8);
 	if (!bm) {
 		printk(KERN_ERR "CPA Cannot vmalloc bitmap\n");
 		return -ENOMEM;
@@ -136,11 +129,11 @@ static int pageattr_test(void)
 	failed += print_split(&sa);
 
 	for (i = 0; i < NTEST; i++) {
-		unsigned long pfn = get_random_u32_below(max_pfn_mapped);
+		unsigned long pfn = get_random_u32_below(max_test_pfn);
 
 		addr[i] = (unsigned long)__va(pfn << PAGE_SHIFT);
 		len[i] = get_random_u32_below(NPAGES);
-		len[i] = min_t(unsigned long, len[i], max_pfn_mapped - pfn - 1);
+		len[i] = min_t(unsigned long, len[i], max_test_pfn - pfn - 1);
 
 		if (len[i] == 0)
 			len[i] = 1;
@@ -150,15 +143,20 @@ static int pageattr_test(void)
 
 		for (k = 0; k < len[i]; k++) {
 			pte = lookup_address(addr[i] + k*PAGE_SIZE, &level);
-			if (!pte || pgprot_val(pte_pgprot(*pte)) == 0 ||
-			    !(pte_val(*pte) & _PAGE_PRESENT)) {
+			if (!pte) {
+				addr[i] = 0;
+				break;
+			}
+			pte_val = ptep_get(pte);
+			if (pgprot_val(pte_pgprot(pte_val)) == 0 ||
+			    !pte_present(pte_val)) {
 				addr[i] = 0;
 				break;
 			}
 			if (k == 0) {
-				pte0 = *pte;
+				pte0 = pte_val;
 			} else {
-				if (pgprot_val(pte_pgprot(*pte)) !=
+				if (pgprot_val(pte_pgprot(pte_val)) !=
 					pgprot_val(pte_pgprot(pte0))) {
 					len[i] = k;
 					break;
@@ -198,12 +196,12 @@ static int pageattr_test(void)
 		}
 
 		pte = lookup_address(addr[i], &level);
-		if (!pte || !pte_testbit(*pte) || pte_huge(*pte)) {
+		if (!pte || !pte_testbit(*pte) || level != PGTABLE_LEVEL_PTE) {
 			printk(KERN_ERR "CPA %lx: bad pte %Lx\n", addr[i],
 				pte ? (u64)pte_val(*pte) : 0ULL);
 			failed++;
 		}
-		if (level != PG_LEVEL_4K) {
+		if (level != PGTABLE_LEVEL_PTE) {
 			printk(KERN_ERR "CPA %lx: unexpected level %d\n",
 				addr[i], level);
 			failed++;
@@ -266,6 +264,8 @@ static int start_pageattr_test(void)
 {
 	struct task_struct *p;
 
+	max_test_pfn = PFN_DOWN(__pa(high_memory - 1));
+
 	p = kthread_create(do_pageattr_test, NULL, "pageattr-test");
 	if (!IS_ERR(p))
 		wake_up_process(p);
diff --git a/mm/set_memory.c b/mm/set_memory.c
new file mode 100644
index 00000000000000..744405efcc3401
--- /dev/null
+++ b/mm/set_memory.c
@@ -0,0 +1,396 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include <linux/highmem.h>
+#include <linux/vmalloc.h>
+#include <linux/pgtable.h>
+#include <linux/export.h>
+
+#include <linux/set_memory.h>
+
+/*
+ * Serialize cpa() so that we don't allow any other cpu, with stale large tlb
+ * entries, to change the page attribute in parallel to some other cpu
+ * splitting a large page entry along with changing the attribute.
+ */
+static DEFINE_SPINLOCK(cpa_lock);
+
+static inline void arch_lock(void)
+{
+#ifdef CONFIG_X86
+	spin_lock(&pgd_lock);
+#endif
+}
+
+static inline void arch_unlock(void)
+{
+#ifdef CONFIG_X86
+	spin_unlock(&pgd_lock);
+#endif
+}
+
+unsigned long cpa_addr(struct cpa_data *cpa, unsigned long idx)
+{
+	if (cpa->flags & CPA_PAGES_ARRAY) {
+		struct page *page = cpa->pages[idx];
+
+		if (unlikely(PageHighMem(page)))
+			return 0;
+
+		return (unsigned long)page_address(page);
+	}
+
+	if (cpa->flags & CPA_ARRAY)
+		return cpa->vaddr[idx];
+
+	return *cpa->vaddr + idx * PAGE_SIZE;
+}
+
+/*
+ * Lookup the page table entry for a virtual address in a specific pgd.
+ * Return a pointer to the entry (or NULL if the entry does not exist),
+ * the level of the entry, and the effective NX and RW bits of all
+ * page table levels.
+ */
+pte_t *lookup_address_in_pgd_attr(pgd_t *pgdp, unsigned long address,
+				  unsigned int *level, bool *nx, bool *rw)
+{
+	pgd_t pgd = pgdp_get(pgdp);
+	p4d_t *p4dp, p4d;
+	pud_t *pudp, pud;
+	pmd_t *pmdp, pmd;
+
+	*level = PGTABLE_LEVEL_PGD;
+	*nx = false;
+	*rw = true;
+
+	if (pgd_none(pgd))
+		return NULL;
+
+	*level = PGTABLE_LEVEL_P4D;
+	*nx |= !pgd_exec(pgd);
+	*rw &= !!(pgd_write(pgd));
+
+	p4dp = p4d_offset(pgdp, address);
+	p4d = p4dp_get(p4dp);
+	if (p4d_none(p4d))
+		return NULL;
+
+	if (p4d_leaf(p4d) || !p4d_present(p4d))
+		return (pte_t *)p4dp;
+
+	*level = PGTABLE_LEVEL_PUD;
+	*nx |= !p4d_exec(p4d);
+	*rw &= !!(p4d_write(p4d));
+
+	pudp = pud_offset(p4dp, address);
+	pud = pudp_get(pudp);
+	if (pud_none(pud))
+		return NULL;
+
+	if (pud_leaf(pud) || !pud_present(pud))
+		return (pte_t *)pudp;
+
+	*level = PGTABLE_LEVEL_PMD;
+	*nx |= !pud_exec(pud);
+	*rw &= !!(pud_write(pud));
+
+	pmdp = pmd_offset(pudp, address);
+	pmd = pmdp_get(pmdp);
+	if (pmd_none(pmd))
+		return NULL;
+
+	if (pmd_leaf(pmd) || !pmd_present(pmd))
+		return (pte_t *)pmdp;
+
+	*level = PGTABLE_LEVEL_PTE;
+	*nx |= !pmd_exec(pmd);
+	*rw &= !!(pmd_write(pmd));
+
+	return pte_offset_kernel(pmdp, address);
+}
+
+/*
+ * Lookup the page table entry for a virtual address in a specific pgd.
+ * Return a pointer to the entry and the level of the mapping.
+ */
+pte_t *lookup_address_in_pgd(pgd_t *pgd, unsigned long address,
+			     unsigned int *level)
+{
+	bool nx, rw;
+
+	return lookup_address_in_pgd_attr(pgd, address, level, &nx, &rw);
+}
+
+/*
+ * Lookup the page table entry for a virtual address. Return a pointer
+ * to the entry and the level of the mapping.
+ *
+ * Note: the function returns p4d, pud or pmd either when the entry is marked
+ * large or when the present bit is not set. Otherwise it returns NULL.
+ */
+pte_t *lookup_address(unsigned long address, unsigned int *level)
+{
+	return lookup_address_in_pgd(pgd_offset_k(address), address, level);
+}
+EXPORT_SYMBOL_GPL(lookup_address);
+
+static pte_t *_lookup_address_cpa(struct cpa_data *cpa, unsigned long address,
+			   unsigned int *level, bool *nx, bool *rw)
+{
+	pgd_t *pgd;
+
+	if (!cpa->pgd)
+		pgd = pgd_offset_k(address);
+	else
+		pgd = cpa->pgd + pgd_index(address);
+
+	return lookup_address_in_pgd_attr(pgd, address, level, nx, rw);
+}
+
+static int should_split_large_page(struct cpa_data *cpa,
+				   struct cpa_split_data *sd)
+{
+	enum pgtable_level level = sd->level;
+	unsigned long address = sd->address;
+	unsigned long numpages, lpaddr;
+
+	if (cpa->force_split)
+		return 1;
+
+	sd->psize = pgtable_level_size(level);
+	sd->pmask = pgtable_level_mask(level);
+
+	/*
+	 * Calculate the number of pages, which fit into this large
+	 * page starting at address:
+	 */
+	lpaddr = (address + sd->psize) & sd->pmask;
+	numpages = (lpaddr - address) >> PAGE_SHIFT;
+	if (numpages < cpa->numpages)
+		cpa->numpages = numpages;
+
+	return arch_should_split_large_page(cpa, sd);
+}
+
+static int cpa_handle_large_page(struct cpa_data *cpa, pte_t *kpte,
+				  unsigned long address)
+{
+	struct cpa_split_data sd = {
+		.address = address,
+		.kpte = kpte,
+	};
+	int do_split, ret = 1;
+	pte_t *tmp;
+
+	spin_unlock(&cpa_lock);
+	sd.ptdesc = pagetable_alloc(GFP_KERNEL, 0);
+	spin_lock(&cpa_lock);
+	if (!sd.ptdesc)
+		return -ENOMEM;
+
+	arch_lock();
+
+	/*
+	 * Check for races, another CPU might have split this page
+	 * up already:
+	 */
+	tmp = _lookup_address_cpa(cpa, address, &sd.level, &sd.nx, &sd.rw);
+	if (tmp != kpte)
+		goto out_free_ptdesc;
+
+	/*
+	 * Check, whether we can keep the large page intact
+	 * and just change the pte:
+	 */
+	do_split = should_split_large_page(cpa, &sd);
+	ret = do_split;
+	/*
+	 * When the range fits into the existing large page, no split is
+	 * required.
+	 * should_split_large_page() updated the large page attributes and
+	 * cpa->numpages and cpa->pfn.
+	 */
+	if (do_split <= 0)
+		goto out_free_ptdesc;
+
+	/*
+	 * We have to split the large page:
+	 */
+	ret = arch_split_large_page(cpa, &sd);
+	if (ret)
+		goto out_free_ptdesc;
+
+	arch_unlock();
+	return do_split;
+
+out_free_ptdesc:
+	arch_unlock();
+	pagetable_free(sd.ptdesc);
+	return ret;
+}
+
+static int __change_page_attr(struct cpa_data *cpa, int primary)
+{
+	unsigned long address;
+	int split_res;
+	unsigned int level;
+	pte_t *kpte, old_pte;
+	bool nx, rw;
+
+	address = cpa_addr(cpa, cpa->curpage);
+repeat:
+	kpte = _lookup_address_cpa(cpa, address, &level, &nx, &rw);
+	if (!kpte)
+		return arch_cpa_process_fault(cpa, address, primary);
+
+	old_pte = *kpte;
+	if (pte_none(old_pte))
+		return arch_cpa_process_fault(cpa, address, primary);
+
+	if (level == PGTABLE_LEVEL_PTE) {
+		arch_change_pte(cpa, address, kpte, old_pte, nx, rw);
+		cpa->numpages = 1;
+		return 0;
+	}
+
+	/*
+	 * The address maps a large page. Try to keep it, otherwise split it
+	 * up and retry on the freshly installed smaller mapping.
+	 */
+	split_res = cpa_handle_large_page(cpa, kpte, address);
+	if (split_res > 0)
+		goto repeat;
+
+	return split_res;
+}
+
+int __change_page_attr_set_clr(struct cpa_data *cpa, int primary)
+{
+	unsigned long numpages = cpa->numpages;
+	unsigned long rempages = numpages;
+	int ret = 0;
+
+	/*
+	 * No changes, easy!
+	 */
+	if (!(pgprot_val(cpa->mask_set) | pgprot_val(cpa->mask_clr)) &&
+	    !cpa->force_split)
+		return ret;
+
+	while (rempages) {
+		/*
+		 * Store the remaining nr of pages for the large page
+		 * preservation check.
+		 */
+		cpa->numpages = rempages;
+		/* for array changes, we can't use large page */
+		if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY))
+			cpa->numpages = 1;
+
+		spin_lock(&cpa_lock);
+		ret = __change_page_attr(cpa, primary);
+		spin_unlock(&cpa_lock);
+		if (ret)
+			goto out;
+
+		if (primary && !(cpa->flags & CPA_NO_CHECK_ALIAS) &&
+		    cpa_should_update_alias(cpa_addr(cpa, cpa->curpage), cpa->pfn)) {
+			ret = arch_cpa_process_alias(cpa);
+			if (ret)
+				goto out;
+		}
+
+		/*
+		 * Adjust the number of pages with the result of the
+		 * CPA operation. Either a large page has been
+		 * preserved or a single page update happened.
+		 */
+		BUG_ON(cpa->numpages > rempages || !cpa->numpages);
+		rempages -= cpa->numpages;
+		cpa->curpage += cpa->numpages;
+	}
+
+out:
+	/* Restore the original numpages */
+	cpa->numpages = numpages;
+	return ret;
+}
+
+int change_page_attr_set_clr(unsigned long *addr, int numpages,
+			     pgprot_t mask_set, pgprot_t mask_clr,
+			     int force_split, int in_flag,
+			     struct page **pages)
+{
+	struct cpa_data cpa;
+	int err;
+
+	memset(&cpa, 0, sizeof(cpa));
+
+	/*
+	 * Check, if we are requested to set a not supported
+	 * feature.  Clearing non-supported features is OK.
+	 */
+	mask_set = canon_pgprot(mask_set);
+
+	if (!pgprot_val(mask_set) && !pgprot_val(mask_clr) && !force_split)
+		return 0;
+
+	/* Ensure we are PAGE_SIZE aligned */
+	if (in_flag & CPA_ARRAY) {
+		int i;
+
+		for (i = 0; i < numpages; i++) {
+			if (addr[i] & ~PAGE_MASK) {
+				addr[i] &= PAGE_MASK;
+				WARN_ON_ONCE(1);
+			}
+		}
+	} else if (!(in_flag & CPA_PAGES_ARRAY)) {
+		/*
+		 * in_flag of CPA_PAGES_ARRAY implies it is aligned.
+		 * No need to check in that case
+		 */
+		if (*addr & ~PAGE_MASK) {
+			*addr &= PAGE_MASK;
+			/*
+			 * People should not be passing in unaligned addresses:
+			 */
+			WARN_ON_ONCE(1);
+		}
+	}
+
+	/* Must avoid aliasing mappings in the highmem code */
+	kmap_flush_unused();
+
+	vm_unmap_aliases();
+
+	cpa.vaddr = addr;
+	cpa.pages = pages;
+	cpa.numpages = numpages;
+	cpa.mask_set = mask_set;
+	cpa.mask_clr = mask_clr;
+	cpa.flags = in_flag;
+	cpa.curpage = 0;
+	cpa.force_split = force_split;
+
+	err = __change_page_attr_set_clr(&cpa, 1);
+
+	/*
+	 * Check whether we really changed something:
+	 */
+	if (!(cpa.flags & CPA_FLUSHTLB))
+		goto out;
+
+	arch_cpa_flush(&cpa, err);
+
+out:
+	return err;
+}
+
+/*
+ * The testcases use internal knowledge of the implementation that shouldn't
+ * be exposed to the rest of the kernel. Include these directly here.
+ */
+#ifdef CONFIG_CPA_DEBUG
+#include "cpa-test.c"
+#endif