From: Pasha Tatashin In preparation for the dynamic stacks, separate out the __vmalloc_node_range and vfree calls from the vmap based stack allocations. The dynamic stacks will use their own variants of these functions. Signed-off-by: Pasha Tatashin [Fix a bug in original patch: free_vmap_stack(vm_area->addr)] Signed-off-by: Linus Walleij [Add missing free_vmap_stack conversion, fix typos, rebase] Signed-off-by: David Stevens --- kernel/fork.c | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/kernel/fork.c b/kernel/fork.c index 72c081db492c..8bf32815f422 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -269,6 +269,21 @@ static bool try_release_thread_stack_to_cache(struct vm_struct *vm_area) return false; } +static inline struct vm_struct *alloc_vmap_stack(int node) +{ + void *stack; + + stack = __vmalloc_node(THREAD_SIZE, THREAD_ALIGN, GFP_VMAP_STACK, + node, __builtin_return_address(0)); + + return stack ? find_vm_area(stack) : NULL; +} + +static inline void free_vmap_stack(struct vm_struct *vm_area) +{ + vfree(vm_area->addr); +} + static void thread_stack_free_rcu(struct rcu_head *rh) { struct vm_stack *vm_stack = container_of(rh, struct vm_stack, rcu); @@ -277,7 +292,7 @@ static void thread_stack_free_rcu(struct rcu_head *rh) if (try_release_thread_stack_to_cache(vm_stack->stack_vm_area)) return; - vfree(vm_area->addr); + free_vmap_stack(vm_area); } static void thread_stack_delayed_free(struct task_struct *tsk) @@ -304,7 +319,7 @@ static int free_vm_stack_cache(unsigned int cpu) if (!vm_area) continue; - vfree(vm_area->addr); + free_vmap_stack(vm_area); cached_vm_stack_areas[i] = NULL; } @@ -333,41 +348,35 @@ static int memcg_charge_kernel_stack(struct vm_struct *vm_area) static int alloc_thread_stack_node(struct task_struct *tsk, int node) { struct vm_struct *vm_area; - void *stack; vm_area = alloc_thread_stack_node_from_cache(tsk, node); if (vm_area) { unsigned long memset_offset = 0; if (memcg_charge_kernel_stack(vm_area)) { - vfree(vm_area->addr); + free_vmap_stack(vm_area); return -ENOMEM; } /* Reset stack metadata. */ kasan_unpoison_range(vm_area->addr, THREAD_SIZE); - - stack = kasan_reset_tag(vm_area->addr); + tsk->stack = kasan_reset_tag(vm_area->addr); /* Clear stale pointers from reused stack. */ if (!IS_ENABLED(CONFIG_STACK_GROWSUP)) memset_offset = THREAD_SIZE - vm_area->nr_pages * PAGE_SIZE; - memset(stack + memset_offset, 0, vm_area->nr_pages * PAGE_SIZE); + memset(tsk->stack + memset_offset, 0, vm_area->nr_pages * PAGE_SIZE); tsk->stack_vm_area = vm_area; - tsk->stack = stack; return 0; } - stack = __vmalloc_node(THREAD_SIZE, THREAD_ALIGN, - GFP_VMAP_STACK, - node, __builtin_return_address(0)); - if (!stack) + vm_area = alloc_vmap_stack(node); + if (!vm_area) return -ENOMEM; - vm_area = find_vm_area(stack); if (memcg_charge_kernel_stack(vm_area)) { - vfree(stack); + free_vmap_stack(vm_area); return -ENOMEM; } /* @@ -376,8 +385,7 @@ static int alloc_thread_stack_node(struct task_struct *tsk, int node) * so cache the vm_struct. */ tsk->stack_vm_area = vm_area; - stack = kasan_reset_tag(stack); - tsk->stack = stack; + tsk->stack = kasan_reset_tag(vm_area->addr); return 0; } -- 2.54.0.rc2.544.gc7ae2d5bb8-goog