Initial kernel page tables (kernel image, direct map, vmemmap, etc.) are not allocated using pagetable_alloc() - the buddy allocator isn't ready at that point. We need to use a different approach to protect them. This patch takes the "brute force" route: when kpkeys_hardened_pgtables is enabled, walk all installed kernel page table, setting the pkey of their backing pages in the direct map along the way. The standard pagewalk API is used for that purpose, albeit in a slightly unusual way: * Visiting an entry is used to set the pkey of the table it points to, hence: .pmd_entry -> protect_pte_table * PTE entries are always leaves, so no need to visit them (no .pte_entry callback) * The range is meant to capture all installed kernel page tables. There doesn't seem to be a generic way to express this; [PAGE_OFFSET, ULONG_MAX) seems to be good enough on arm64/x86. Locking probably isn't necessary so early during boot, but taking the mmap_write_lock shouldn't hurt either. We also assume that all page tables installed after that point are allocated using pagetable_alloc(), which seems to be the case, at least on arm64. There are some more assumptions - see comment above protect_early_kernel_pgtables(). This simple approach should reliably protect all early kernel page tables without invasive changes. However, adding support for large blocks (PMD/PUD) is likely to be difficult and/or inefficient as splitting will ocur when protecting individual page table pages. Modifying the early page table allocators directly may be preferable. Signed-off-by: Kevin Brodsky --- mm/kpkeys_hardened_pgtables.c | 122 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/mm/kpkeys_hardened_pgtables.c b/mm/kpkeys_hardened_pgtables.c index 2311fd8da6bb..f6825a0e0689 100644 --- a/mm/kpkeys_hardened_pgtables.c +++ b/mm/kpkeys_hardened_pgtables.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-only #include #include +#include +#include #include #include @@ -54,10 +56,130 @@ void kpkeys_pgtable_free(struct page *page, unsigned int order) __free_pages(page, order); } +static void __init protect_pgtable_page(void *page) +{ + int ret; + + ret = set_direct_map_pkey((unsigned long)page, 1, KPKEYS_PKEY_PGTABLES); + WARN_ON_ONCE(ret); +} + +static int __init protect_pte_table(pmd_t *pmd, unsigned long addr, + unsigned long next, struct mm_walk *walk) +{ + pmd_t entry = pmdp_get(pmd); + + if (!pmd_present(entry) || pmd_leaf(entry)) + return 0; + + protect_pgtable_page(pte_offset_kernel(pmd, 0)); + + return 0; +} + +static int __init protect_pmd_table(pud_t *pud, unsigned long addr, + unsigned long next, struct mm_walk *walk) +{ + pud_t entry; + + if (mm_pmd_folded(walk->mm)) + return 0; + + entry = pudp_get(pud); + if (!pud_present(entry) || pud_leaf(entry)) + return 0; + + protect_pgtable_page(pmd_offset(pud, 0)); + + return 0; +} + +static int __init protect_pud_table(p4d_t *p4d, unsigned long addr, + unsigned long next, struct mm_walk *walk) +{ + p4d_t entry; + + if (mm_pud_folded(walk->mm)) + return 0; + + entry = p4dp_get(p4d); + if (!p4d_present(entry) || p4d_leaf(entry)) + return 0; + + protect_pgtable_page(pud_offset(p4d, 0)); + + return 0; +} + +static int __init protect_p4d_table(pgd_t *pgd, unsigned long addr, + unsigned long next, struct mm_walk *walk) +{ + pgd_t entry; + + if (mm_p4d_folded(walk->mm)) + return 0; + + entry = pgdp_get(pgd); + if (!pgd_present(entry) || pgd_leaf(entry)) + return 0; + + protect_pgtable_page(p4d_offset(pgd, 0)); + + return 0; +} + +static const struct mm_walk_ops protect_pgtable_ops __initconst = { + .pmd_entry = protect_pte_table, + .pud_entry = protect_pmd_table, + .p4d_entry = protect_pud_table, + .pgd_entry = protect_p4d_table, +}; + +/* + * Protect all installed kernel page tables by walking them and setting the + * direct mapping of all backing pages to pkey KPKEYS_PKEY_PGTABLES. We assume + * all non-root levels (PTE to P4D) are page-sized (asserts below). + * + * The root PGD page is left untouched; it is normally accessed via the kimg + * mapping only. On arm64 it cannot be written directly via any mapping. + * + * This approach is unsafe if set_direct_map_pkey() may split blocks in the + * direct map; we rely on the direct map being fully PTE-mapped. + */ +static void __init protect_early_kernel_pgtables(void) +{ + int ret; + + mmap_write_lock(&init_mm); + /* + * [PAGE_OFFSET, ULONG_MAX) covers all early kernel mappings on arm64 + * and x86 (at least). + */ + ret = walk_kernel_page_table_range(PAGE_OFFSET, ULONG_MAX, + &protect_pgtable_ops, NULL, NULL); + mmap_write_unlock(&init_mm); + WARN_ON_ONCE(ret); +} + +static_assert(MAX_PTRS_PER_PTE * sizeof(pte_t) == PAGE_SIZE); + +#ifndef __PAGETABLE_PMD_FOLDED +static_assert(MAX_PTRS_PER_PMD * sizeof(pmd_t) == PAGE_SIZE); +#endif + +#ifndef __PAGETABLE_PUD_FOLDED +static_assert(MAX_PTRS_PER_PUD * sizeof(pud_t) == PAGE_SIZE); +#endif + +#ifndef __PAGETABLE_P4D_FOLDED +static_assert(MAX_PTRS_PER_P4D * sizeof(p4d_t) == PAGE_SIZE); +#endif + void __init kpkeys_hardened_pgtables_init(void) { if (!kpkeys_supported()) return; static_branch_enable(&kpkeys_hardened_pgtables_key); + protect_early_kernel_pgtables(); } -- 2.56.0