Refactor vm_area_alloc_pages() by moving the high-order allocation code into a separate function, vm_area_alloc_pages_large_order(). No functional changes. Signed-off-by: Uladzislau Rezki (Sony) --- mm/vmalloc.c | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/mm/vmalloc.c b/mm/vmalloc.c index ecbac900c35f..d3a4725e15ca 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -3624,18 +3624,20 @@ static inline gfp_t vmalloc_gfp_adjust(gfp_t flags, const bool large) return flags; } -static inline unsigned int -vm_area_alloc_pages(gfp_t gfp, int nid, - unsigned int order, unsigned int nr_pages, struct page **pages) +static unsigned int +vm_area_alloc_pages_large_order(gfp_t gfp, int nid, unsigned int order, + unsigned int nr_pages, struct page **pages) { unsigned int nr_allocated = 0; unsigned int nr_remaining = nr_pages; unsigned int max_attempt_order = MAX_PAGE_ORDER; struct page *page; + unsigned int large_order; + gfp_t large_gfp; int i; - unsigned int large_order = ilog2(nr_remaining); - gfp_t large_gfp = vmalloc_gfp_adjust(gfp, large_order) & ~__GFP_DIRECT_RECLAIM; + large_order = ilog2(nr_remaining); + large_gfp = vmalloc_gfp_adjust(gfp, large_order) & ~__GFP_DIRECT_RECLAIM; large_order = min(max_attempt_order, large_order); /* @@ -3666,6 +3668,20 @@ vm_area_alloc_pages(gfp_t gfp, int nid, large_order = min(max_attempt_order, large_order); } + return nr_allocated; +} + +static inline unsigned int +vm_area_alloc_pages(gfp_t gfp, int nid, + unsigned int order, unsigned int nr_pages, struct page **pages) +{ + unsigned int nr_allocated = 0; + struct page *page; + int i; + + nr_allocated = vm_area_alloc_pages_large_order(gfp, nid, + order, nr_pages, pages); + /* * For order-0 pages we make use of bulk allocator, if * the page array is partly or not at all populated due -- 2.47.3 Introduce a module parameter to enable or disable the large-order allocation path in vmalloc. High-order allocations are disabled by default so far, but users may explicitly enable them at runtime if desired. High-order pages allocated for vmalloc are immediately split into order-0 pages and later freed as order-0, which means they do not feed the per-CPU page caches. As a result, high-order attempts tend to bypass the PCP fastpath and fall back to the buddy allocator that can affect performance. However, when the PCP caches are empty, high-order allocations may show better performance characteristics especially for larger allocation requests. Since the best strategy is workload-dependent, this patch adds a parameter letting users to choose whether vmalloc should try high-order allocations or stay strictly on the order-0 fastpath. Signed-off-by: Uladzislau Rezki (Sony) --- mm/vmalloc.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mm/vmalloc.c b/mm/vmalloc.c index d3a4725e15ca..f66543896b16 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -43,6 +43,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include @@ -3671,6 +3672,9 @@ vm_area_alloc_pages_large_order(gfp_t gfp, int nid, unsigned int order, return nr_allocated; } +static int attempt_larger_order_alloc; +module_param(attempt_larger_order_alloc, int, 0644); + static inline unsigned int vm_area_alloc_pages(gfp_t gfp, int nid, unsigned int order, unsigned int nr_pages, struct page **pages) @@ -3679,8 +3683,9 @@ vm_area_alloc_pages(gfp_t gfp, int nid, struct page *page; int i; - nr_allocated = vm_area_alloc_pages_large_order(gfp, nid, - order, nr_pages, pages); + if (attempt_larger_order_alloc) + nr_allocated = vm_area_alloc_pages_large_order(gfp, nid, + order, nr_pages, pages); /* * For order-0 pages we make use of bulk allocator, if -- 2.47.3