From: "Barry Song (Xiaomi)" Try to align the vmap virtual address to PMD_SHIFT or a larger PTE mapping size hinted by the architecture, so contiguous pages can be batch-mapped when setting PMD or PTE entries. Add __get_vm_area_node_aligned_caller() as a wrapper over __get_vm_area_node() to simplify repeated calls with fixed arguments. Signed-off-by: Barry Song (Xiaomi) Signed-off-by: Wen Jiang Tested-by: Xueyuan Chen --- mm/vmalloc.c | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/mm/vmalloc.c b/mm/vmalloc.c index fffb885cb2158..bc9fa93e2bdc6 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -3628,6 +3628,41 @@ static int vmap_batched(unsigned long addr, unsigned long end, return err; } +static struct vm_struct *__get_vm_area_node_aligned_caller(unsigned long size, + unsigned long align, unsigned long flags, const void *caller) +{ + return __get_vm_area_node(size, align, PAGE_SHIFT, flags, + VMALLOC_START, VMALLOC_END, + NUMA_NO_NODE, GFP_KERNEL, caller); +} + +static struct vm_struct *vmap_get_aligned_vm_area(unsigned long size, + unsigned long flags, const void *caller) +{ + struct vm_struct *vm_area; + unsigned int shift; + + /* Try PMD alignment for large sizes */ + if (size >= PMD_SIZE) { + vm_area = __get_vm_area_node_aligned_caller(size, PMD_SIZE, + flags, caller); + if (vm_area) + return vm_area; + } + + /* Try CONT_PTE alignment */ + shift = arch_vmap_pte_supported_shift(size); + if (shift > PAGE_SHIFT) { + vm_area = __get_vm_area_node_aligned_caller(size, 1UL << shift, + flags, caller); + if (vm_area) + return vm_area; + } + + /* Fall back to page alignment */ + return __get_vm_area_node_aligned_caller(size, PAGE_SIZE, flags, caller); +} + /** * vmap - map an array of pages into virtually contiguous space * @pages: array of page pointers @@ -3666,7 +3701,7 @@ void *vmap(struct page **pages, unsigned int count, return NULL; size = (unsigned long)count << PAGE_SHIFT; - area = get_vm_area_caller(size, flags, __builtin_return_address(0)); + area = vmap_get_aligned_vm_area(size, flags, __builtin_return_address(0)); if (!area) return NULL; -- 2.34.1