From: "Barry Song (Xiaomi)" Try to align the vmap virtual address to PMD_SHIFT or a larger PTE mapping size hinted by the architecture, so contiguous pages can be batch-mapped when setting PMD or PTE entries. Signed-off-by: Barry Song (Xiaomi) Signed-off-by: Wen Jiang Tested-by: Xueyuan Chen --- mm/vmalloc.c | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 50642246f4d40..040d400928aab 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -3620,6 +3620,37 @@ static int vmap_batched(unsigned long addr, unsigned long end, return err; } +static struct vm_struct *get_aligned_vm_area(unsigned long size, + unsigned long flags, const void *caller) +{ + struct vm_struct *vm_area; + unsigned int shift; + + /* Try PMD alignment for large sizes */ + if (size >= PMD_SIZE) { + vm_area = __get_vm_area_node(size, PMD_SIZE, PAGE_SHIFT, flags, + VMALLOC_START, VMALLOC_END, + NUMA_NO_NODE, GFP_KERNEL, caller); + if (vm_area) + return vm_area; + } + + /* Try CONT_PTE alignment */ + shift = arch_vmap_pte_supported_shift(size); + if (shift > PAGE_SHIFT) { + vm_area = __get_vm_area_node(size, 1UL << shift, PAGE_SHIFT, flags, + VMALLOC_START, VMALLOC_END, + NUMA_NO_NODE, GFP_KERNEL, caller); + if (vm_area) + return vm_area; + } + + /* Fall back to page alignment */ + return __get_vm_area_node(size, PAGE_SIZE, PAGE_SHIFT, flags, + VMALLOC_START, VMALLOC_END, + NUMA_NO_NODE, GFP_KERNEL, caller); +} + /** * vmap - map an array of pages into virtually contiguous space * @pages: array of page pointers @@ -3658,7 +3689,7 @@ void *vmap(struct page **pages, unsigned int count, return NULL; size = (unsigned long)count << PAGE_SHIFT; - area = get_vm_area_caller(size, flags, __builtin_return_address(0)); + area = get_aligned_vm_area(size, flags, __builtin_return_address(0)); if (!area) return NULL; -- 2.34.1